Redis/redis
1
0
Fork 0
Code Issues Pull Requests Actions 1 Packages Projects Releases Wiki Activity

Compare commits

base: Redis/redis:7.4.0
Branches Tags
Redis/redis:6.2 Redis/redis:unstable Redis/redis:7.2 Redis/redis:7.4 Redis/redis:8.0
Redis/redis:7.2.10 Redis/redis:6.2.19 Redis/redis:7.4.5 Redis/redis:8.0.3 Redis/redis:8.2-rc1 Redis/redis:8.2-rc1-int Redis/redis:8.2-m01 Redis/redis:8.2-m01-int2 Redis/redis:8.2-m01-int Redis/redis:8.0.2 Redis/redis:7.2.9 Redis/redis:7.4.4 Redis/redis:8.0.1 Redis/redis:8.0.1-int Redis/redis:8.0.0 Redis/redis:8.0-rc2-int Redis/redis:7.4.3 Redis/redis:7.2.8 Redis/redis:6.2.18 Redis/redis:8.0-rc1 Redis/redis:8.0-rc1-int2 Redis/redis:8.0-rc1-int Redis/redis:8.0-m04 Redis/redis:8.0-m04-int Redis/redis:8.0-m03 Redis/redis:7.2.7 Redis/redis:7.4.2 Redis/redis:6.2.17 Redis/redis:8.0-m02 Redis/redis:6.2.16 Redis/redis:7.2.6 Redis/redis:7.4.1 Redis/redis:8.0-m01 Redis/redis:7.4.0 Redis/redis:7.4-rc2 Redis/redis:7.4-rc1 Redis/redis:7.2.5 Redis/redis:7.2.4 Redis/redis:7.0.15 Redis/redis:7.2.3 Redis/redis:7.2.2 Redis/redis:7.0.14 Redis/redis:6.2.15 Redis/redis:6.2.14 Redis/redis:7.2.1 Redis/redis:7.0.13 Redis/redis:7.2.0 Redis/redis:7.2-rc3 Redis/redis:7.0.12 Redis/redis:6.2.13 Redis/redis:6.0.20 Redis/redis:7.2-rc2 Redis/redis:6.0.19 Redis/redis:6.2.12 Redis/redis:7.0.11 Redis/redis:7.2-rc1 Redis/redis:7.0.10 Redis/redis:7.0.9 Redis/redis:6.2.11 Redis/redis:6.0.18 Redis/redis:6.2.10 Redis/redis:6.0.17 Redis/redis:6.2.9 Redis/redis:7.0.8 Redis/redis:7.0.7 Redis/redis:7.0.6 Redis/redis:6.2.8 Redis/redis:7.0.5 Redis/redis:7.0.4 Redis/redis:7.0.3 Redis/redis:7.0.2 Redis/redis:7.0.1 Redis/redis:7.0.0 Redis/redis:6.2.7 Redis/redis:7.0-rc3 Redis/redis:7.0-rc2 Redis/redis:7.0-rc1 Redis/redis:6.2.6 Redis/redis:6.0.16 Redis/redis:5.0.14 Redis/redis:6.2.5 Redis/redis:5.0.13 Redis/redis:6.0.15 Redis/redis:6.0.14 Redis/redis:6.2.4 Redis/redis:6.2.3 Redis/redis:6.0.13 Redis/redis:6.2.2 Redis/redis:6.2.1 Redis/redis:6.0.12 Redis/redis:5.0.12 Redis/redis:6.0.11 Redis/redis:6.2.0 Redis/redis:5.0.11 Redis/redis:6.2-rc3 Redis/redis:6.0.10 Redis/redis:6.2-rc2 Redis/redis:6.2-rc1 Redis/redis:6.0.9 Redis/redis:5.0.10 Redis/redis:6.0.8 Redis/redis:6.0.7 Redis/redis:6.0.6 Redis/redis:6.0.5 Redis/redis:6.0.4 Redis/redis:6.0.3 Redis/redis:6.0.2 Redis/redis:6.0.1 Redis/redis:6.0.0 Redis/redis:5.0.9 Redis/redis:6.0-rc4 Redis/redis:6.0-rc3 Redis/redis:5.0.8 Redis/redis:6.0-rc2 Redis/redis:6.0-rc1 Redis/redis:5.0.7 Redis/redis:5.0.6 Redis/redis:5.0.5 Redis/redis:3.2.13 Redis/redis:4.0.14 Redis/redis:5.0.4 Redis/redis:4.0.13 Redis/redis:5.0.3 Redis/redis:4.0.12 Redis/redis:5.0.2 Redis/redis:5.0.1 Redis/redis:5.0.0 Redis/redis:5.0-rc6 Redis/redis:5.0-rc5 Redis/redis:4.0.11 Redis/redis:5.0-rc4 Redis/redis:5.0-rc3 Redis/redis:5.0-rc2 Redis/redis:4.0.10 Redis/redis:3.2.12 Redis/redis:5.0-rc1 Redis/redis:4.0.9 Redis/redis:4.0.8 Redis/redis:4.0.7 Redis/redis:4.0.6 Redis/redis:4.0.5 Redis/redis:4.0.4 Redis/redis:4.0.3 Redis/redis:3.2.11 Redis/redis:4.0.2 Redis/redis:3.2.10 Redis/redis:4.0.1 Redis/redis:4.0.0 Redis/redis:3.2.9 Redis/redis:4.0-rc3 Redis/redis:3.2.8 Redis/redis:3.2.7 Redis/redis:3.2.6 Redis/redis:4.0-rc2 Redis/redis:4.0-rc1 Redis/redis:3.2.5 Redis/redis:3.2.4 Redis/redis:3.2.3 Redis/redis:3.2.2 Redis/redis:3.2.1 Redis/redis:3.2.0 Redis/redis:3.2.0-rc3 Redis/redis:3.0.7 Redis/redis:3.2.0-rc2 Redis/redis:3.2-rc1 Redis/redis:3.0.6 Redis/redis:2.8.24 Redis/redis:3.0.5 Redis/redis:2.8.23 Redis/redis:2.8.22 Redis/redis:3.0.4 Redis/redis:3.0.3 Redis/redis:3.0.2 Redis/redis:2.8.21 Redis/redis:2.8.20 Redis/redis:3.0.1 Redis/redis:3.0.0 Redis/redis:3.0.0-rc6 Redis/redis:3.0.0-rc5 Redis/redis:3.0.0-rc4 Redis/redis:3.0.0-rc3 Redis/redis:3.0.0-rc2 Redis/redis:2.8.19 Redis/redis:2.8.18 Redis/redis:3.0.0-rc1 Redis/redis:2.8.17 Redis/redis:2.8.16 Redis/redis:2.8.15 Redis/redis:2.8.14 Redis/redis:3.0.0-beta8 Redis/redis:2.8.13 Redis/redis:3.0.0-beta7 Redis/redis:2.8.12 Redis/redis:2.8.11 Redis/redis:3.0.0-beta6 Redis/redis:2.8.10 Redis/redis:3.0.0-beta5 Redis/redis:3.0.0-beta4 Redis/redis:3.0.0-beta3 Redis/redis:2.8.9 Redis/redis:2.8.8 Redis/redis:3.0.0-beta2 Redis/redis:2.8.7 Redis/redis:2.8.6 Redis/redis:3.0.0-beta1 Redis/redis:2.8.5 Redis/redis:2.8.4 Redis/redis:2.6.17 Redis/redis:2.8.3 Redis/redis:2.8.2 Redis/redis:2.8.1 Redis/redis:2.8.0 Redis/redis:2.8.0-rc6 Redis/redis:2.8.0-rc5 Redis/redis:2.8.0-rc4 Redis/redis:2.6.16 Redis/redis:2.6.15 Redis/redis:2.8.0-rc3 Redis/redis:2.8.0-rc2 Redis/redis:2.8.0-rc1 Redis/redis:2.6.14-2 Redis/redis:2.6.14-1 Redis/redis:2.6.14 Redis/redis:2.6.13 Redis/redis:2.6.12 Redis/redis:2.6.10-3 Redis/redis:2.6.10-2 Redis/redis:2.6.11 Redis/redis:2.6.10-1 Redis/redis:2.6.10 Redis/redis:2.6.9-1 Redis/redis:2.6.9 Redis/redis:2.6.8-1 Redis/redis:2.6.8 Redis/redis:2.6.7-1 Redis/redis:2.6.7 Redis/redis:2.6.6 Redis/redis:2.4.18 Redis/redis:2.6.5 Redis/redis:2.6.4 Redis/redis:2.6.3 Redis/redis:2.6.2 Redis/redis:2.6.1 Redis/redis:2.6.0 Redis/redis:2.6.0-rc8 Redis/redis:2.6.0-rc7 Redis/redis:2.4.17 Redis/redis:2.6.0-rc6 Redis/redis:2.4.16 Redis/redis:2.4.15 Redis/redis:2.6.0-rc5 Redis/redis:2.6.0-rc4 Redis/redis:2.4.14 Redis/redis:2.6.0-rc3 Redis/redis:2.6.0-rc2 Redis/redis:2.4.13 Redis/redis:2.4.12 Redis/redis:2.6.0-rc1 Redis/redis:2.4.11 Redis/redis:2.4.10 Redis/redis:2.4.9 Redis/redis:2.4.8 Redis/redis:2.4.7 Redis/redis:2.4.6 Redis/redis:2.4.5 Redis/redis:2.4.4 Redis/redis:2.4.3 Redis/redis:2.4.2 Redis/redis:2.2.15 Redis/redis:2.4.1 Redis/redis:2.4.0 Redis/redis:2.2.14 Redis/redis:2.4.0-rc8 Redis/redis:2.2.13 Redis/redis:2.4.0-rc7 Redis/redis:2.4.0-rc6 Redis/redis:2.4.0-rc5 Redis/redis:2.4.0-rc4 Redis/redis:2.2.12 Redis/redis:2.4.0-rc3 Redis/redis:with-deprecated-diskstore Redis/redis:2.2.111-scripting Redis/redis:2.2.11 Redis/redis:2.2.110-scripting Redis/redis:2.2.10 Redis/redis:2.2.9 Redis/redis:2.4.0-rc2 Redis/redis:2.4.0-rc1 Redis/redis:2.2.8 Redis/redis:2.2.107-scripting Redis/redis:2.2.7 Redis/redis:2.2.106-scripting Redis/redis:2.2.6 Redis/redis:2.2.105-scripting Redis/redis:2.2.5 Redis/redis:2.2.4 Redis/redis:2.2.3 Redis/redis:3.0-alpha0 Redis/redis:2.0.5 Redis/redis:2.2.2 Redis/redis:2.2.1 Redis/redis:2.2.0 Redis/redis:2.2.0-rc4 Redis/redis:2.3-alpha0 Redis/redis:2.2.0-rc3 Redis/redis:2.2.0-rc2 Redis/redis:2.2.0-rc1 Redis/redis:2.2-alpha6 Redis/redis:2.2-alpha5 Redis/redis:v2.0.4-stable Redis/redis:2.0.4 Redis/redis:2.2-alpha4 Redis/redis:2.2-alpha3 Redis/redis:v2.0.3-stable Redis/redis:2.0.3 Redis/redis:v2.0.2-stable Redis/redis:2.0.2 Redis/redis:2.2-alpha2 Redis/redis:v2.0.1-stable Redis/redis:2.0.1 Redis/redis:v2.0.0-stable Redis/redis:2.0.0 Redis/redis:2.2-alpha1 Redis/redis:twitter-20100825 Redis/redis:twitter-20100804 Redis/redis:2.2-alpha0 Redis/redis:v2.0.0-rc4 Redis/redis:v2.0.0-rc3 Redis/redis:v2.0.0-rc2 Redis/redis:v2.0.0-rc1b Redis/redis:v2.1.1-watch Redis/redis:v2.0.0-rc1 Redis/redis:v1.3.12 Redis/redis:v1.3.11 Redis/redis:v1.3.10 Redis/redis:v1.3.9 Redis/redis:v1.3.8 Redis/redis:v1.3.7 Redis/redis:1.3.6 Redis/redis:vm-playpen
...
compare: Redis/redis:8.0-rc1
Branches Tags
Redis/redis:unstable Redis/redis:7.2 Redis/redis:6.2 Redis/redis:7.4 Redis/redis:8.0
Redis/redis:7.2.10 Redis/redis:6.2.19 Redis/redis:7.4.5 Redis/redis:8.0.3 Redis/redis:8.2-rc1 Redis/redis:8.2-rc1-int Redis/redis:8.2-m01 Redis/redis:8.2-m01-int2 Redis/redis:8.2-m01-int Redis/redis:8.0.2 Redis/redis:7.2.9 Redis/redis:7.4.4 Redis/redis:8.0.1 Redis/redis:8.0.1-int Redis/redis:8.0.0 Redis/redis:8.0-rc2-int Redis/redis:7.4.3 Redis/redis:7.2.8 Redis/redis:6.2.18 Redis/redis:8.0-rc1 Redis/redis:8.0-rc1-int2 Redis/redis:8.0-rc1-int Redis/redis:8.0-m04 Redis/redis:8.0-m04-int Redis/redis:8.0-m03 Redis/redis:7.2.7 Redis/redis:7.4.2 Redis/redis:6.2.17 Redis/redis:8.0-m02 Redis/redis:6.2.16 Redis/redis:7.2.6 Redis/redis:7.4.1 Redis/redis:8.0-m01 Redis/redis:7.4.0 Redis/redis:7.4-rc2 Redis/redis:7.4-rc1 Redis/redis:7.2.5 Redis/redis:7.2.4 Redis/redis:7.0.15 Redis/redis:7.2.3 Redis/redis:7.2.2 Redis/redis:7.0.14 Redis/redis:6.2.15 Redis/redis:6.2.14 Redis/redis:7.2.1 Redis/redis:7.0.13 Redis/redis:7.2.0 Redis/redis:7.2-rc3 Redis/redis:7.0.12 Redis/redis:6.2.13 Redis/redis:6.0.20 Redis/redis:7.2-rc2 Redis/redis:6.0.19 Redis/redis:6.2.12 Redis/redis:7.0.11 Redis/redis:7.2-rc1 Redis/redis:7.0.10 Redis/redis:7.0.9 Redis/redis:6.2.11 Redis/redis:6.0.18 Redis/redis:6.2.10 Redis/redis:6.0.17 Redis/redis:6.2.9 Redis/redis:7.0.8 Redis/redis:7.0.7 Redis/redis:7.0.6 Redis/redis:6.2.8 Redis/redis:7.0.5 Redis/redis:7.0.4 Redis/redis:7.0.3 Redis/redis:7.0.2 Redis/redis:7.0.1 Redis/redis:7.0.0 Redis/redis:6.2.7 Redis/redis:7.0-rc3 Redis/redis:7.0-rc2 Redis/redis:7.0-rc1 Redis/redis:6.2.6 Redis/redis:6.0.16 Redis/redis:5.0.14 Redis/redis:6.2.5 Redis/redis:5.0.13 Redis/redis:6.0.15 Redis/redis:6.0.14 Redis/redis:6.2.4 Redis/redis:6.2.3 Redis/redis:6.0.13 Redis/redis:6.2.2 Redis/redis:6.2.1 Redis/redis:6.0.12 Redis/redis:5.0.12 Redis/redis:6.0.11 Redis/redis:6.2.0 Redis/redis:5.0.11 Redis/redis:6.2-rc3 Redis/redis:6.0.10 Redis/redis:6.2-rc2 Redis/redis:6.2-rc1 Redis/redis:6.0.9 Redis/redis:5.0.10 Redis/redis:6.0.8 Redis/redis:6.0.7 Redis/redis:6.0.6 Redis/redis:6.0.5 Redis/redis:6.0.4 Redis/redis:6.0.3 Redis/redis:6.0.2 Redis/redis:6.0.1 Redis/redis:6.0.0 Redis/redis:5.0.9 Redis/redis:6.0-rc4 Redis/redis:6.0-rc3 Redis/redis:5.0.8 Redis/redis:6.0-rc2 Redis/redis:6.0-rc1 Redis/redis:5.0.7 Redis/redis:5.0.6 Redis/redis:5.0.5 Redis/redis:3.2.13 Redis/redis:4.0.14 Redis/redis:5.0.4 Redis/redis:4.0.13 Redis/redis:5.0.3 Redis/redis:4.0.12 Redis/redis:5.0.2 Redis/redis:5.0.1 Redis/redis:5.0.0 Redis/redis:5.0-rc6 Redis/redis:5.0-rc5 Redis/redis:4.0.11 Redis/redis:5.0-rc4 Redis/redis:5.0-rc3 Redis/redis:5.0-rc2 Redis/redis:4.0.10 Redis/redis:3.2.12 Redis/redis:5.0-rc1 Redis/redis:4.0.9 Redis/redis:4.0.8 Redis/redis:4.0.7 Redis/redis:4.0.6 Redis/redis:4.0.5 Redis/redis:4.0.4 Redis/redis:4.0.3 Redis/redis:3.2.11 Redis/redis:4.0.2 Redis/redis:3.2.10 Redis/redis:4.0.1 Redis/redis:4.0.0 Redis/redis:3.2.9 Redis/redis:4.0-rc3 Redis/redis:3.2.8 Redis/redis:3.2.7 Redis/redis:3.2.6 Redis/redis:4.0-rc2 Redis/redis:4.0-rc1 Redis/redis:3.2.5 Redis/redis:3.2.4 Redis/redis:3.2.3 Redis/redis:3.2.2 Redis/redis:3.2.1 Redis/redis:3.2.0 Redis/redis:3.2.0-rc3 Redis/redis:3.0.7 Redis/redis:3.2.0-rc2 Redis/redis:3.2-rc1 Redis/redis:3.0.6 Redis/redis:2.8.24 Redis/redis:3.0.5 Redis/redis:2.8.23 Redis/redis:2.8.22 Redis/redis:3.0.4 Redis/redis:3.0.3 Redis/redis:3.0.2 Redis/redis:2.8.21 Redis/redis:2.8.20 Redis/redis:3.0.1 Redis/redis:3.0.0 Redis/redis:3.0.0-rc6 Redis/redis:3.0.0-rc5 Redis/redis:3.0.0-rc4 Redis/redis:3.0.0-rc3 Redis/redis:3.0.0-rc2 Redis/redis:2.8.19 Redis/redis:2.8.18 Redis/redis:3.0.0-rc1 Redis/redis:2.8.17 Redis/redis:2.8.16 Redis/redis:2.8.15 Redis/redis:2.8.14 Redis/redis:3.0.0-beta8 Redis/redis:2.8.13 Redis/redis:3.0.0-beta7 Redis/redis:2.8.12 Redis/redis:2.8.11 Redis/redis:3.0.0-beta6 Redis/redis:2.8.10 Redis/redis:3.0.0-beta5 Redis/redis:3.0.0-beta4 Redis/redis:3.0.0-beta3 Redis/redis:2.8.9 Redis/redis:2.8.8 Redis/redis:3.0.0-beta2 Redis/redis:2.8.7 Redis/redis:2.8.6 Redis/redis:3.0.0-beta1 Redis/redis:2.8.5 Redis/redis:2.8.4 Redis/redis:2.6.17 Redis/redis:2.8.3 Redis/redis:2.8.2 Redis/redis:2.8.1 Redis/redis:2.8.0 Redis/redis:2.8.0-rc6 Redis/redis:2.8.0-rc5 Redis/redis:2.8.0-rc4 Redis/redis:2.6.16 Redis/redis:2.6.15 Redis/redis:2.8.0-rc3 Redis/redis:2.8.0-rc2 Redis/redis:2.8.0-rc1 Redis/redis:2.6.14-2 Redis/redis:2.6.14-1 Redis/redis:2.6.14 Redis/redis:2.6.13 Redis/redis:2.6.12 Redis/redis:2.6.10-3 Redis/redis:2.6.10-2 Redis/redis:2.6.11 Redis/redis:2.6.10-1 Redis/redis:2.6.10 Redis/redis:2.6.9-1 Redis/redis:2.6.9 Redis/redis:2.6.8-1 Redis/redis:2.6.8 Redis/redis:2.6.7-1 Redis/redis:2.6.7 Redis/redis:2.6.6 Redis/redis:2.4.18 Redis/redis:2.6.5 Redis/redis:2.6.4 Redis/redis:2.6.3 Redis/redis:2.6.2 Redis/redis:2.6.1 Redis/redis:2.6.0 Redis/redis:2.6.0-rc8 Redis/redis:2.6.0-rc7 Redis/redis:2.4.17 Redis/redis:2.6.0-rc6 Redis/redis:2.4.16 Redis/redis:2.4.15 Redis/redis:2.6.0-rc5 Redis/redis:2.6.0-rc4 Redis/redis:2.4.14 Redis/redis:2.6.0-rc3 Redis/redis:2.6.0-rc2 Redis/redis:2.4.13 Redis/redis:2.4.12 Redis/redis:2.6.0-rc1 Redis/redis:2.4.11 Redis/redis:2.4.10 Redis/redis:2.4.9 Redis/redis:2.4.8 Redis/redis:2.4.7 Redis/redis:2.4.6 Redis/redis:2.4.5 Redis/redis:2.4.4 Redis/redis:2.4.3 Redis/redis:2.4.2 Redis/redis:2.2.15 Redis/redis:2.4.1 Redis/redis:2.4.0 Redis/redis:2.2.14 Redis/redis:2.4.0-rc8 Redis/redis:2.2.13 Redis/redis:2.4.0-rc7 Redis/redis:2.4.0-rc6 Redis/redis:2.4.0-rc5 Redis/redis:2.4.0-rc4 Redis/redis:2.2.12 Redis/redis:2.4.0-rc3 Redis/redis:with-deprecated-diskstore Redis/redis:2.2.111-scripting Redis/redis:2.2.11 Redis/redis:2.2.110-scripting Redis/redis:2.2.10 Redis/redis:2.2.9 Redis/redis:2.4.0-rc2 Redis/redis:2.4.0-rc1 Redis/redis:2.2.8 Redis/redis:2.2.107-scripting Redis/redis:2.2.7 Redis/redis:2.2.106-scripting Redis/redis:2.2.6 Redis/redis:2.2.105-scripting Redis/redis:2.2.5 Redis/redis:2.2.4 Redis/redis:2.2.3 Redis/redis:3.0-alpha0 Redis/redis:2.0.5 Redis/redis:2.2.2 Redis/redis:2.2.1 Redis/redis:2.2.0 Redis/redis:2.2.0-rc4 Redis/redis:2.3-alpha0 Redis/redis:2.2.0-rc3 Redis/redis:2.2.0-rc2 Redis/redis:2.2.0-rc1 Redis/redis:2.2-alpha6 Redis/redis:2.2-alpha5 Redis/redis:v2.0.4-stable Redis/redis:2.0.4 Redis/redis:2.2-alpha4 Redis/redis:2.2-alpha3 Redis/redis:v2.0.3-stable Redis/redis:2.0.3 Redis/redis:v2.0.2-stable Redis/redis:2.0.2 Redis/redis:2.2-alpha2 Redis/redis:v2.0.1-stable Redis/redis:2.0.1 Redis/redis:v2.0.0-stable Redis/redis:2.0.0 Redis/redis:2.2-alpha1 Redis/redis:twitter-20100825 Redis/redis:twitter-20100804 Redis/redis:2.2-alpha0 Redis/redis:v2.0.0-rc4 Redis/redis:v2.0.0-rc3 Redis/redis:v2.0.0-rc2 Redis/redis:v2.0.0-rc1b Redis/redis:v2.1.1-watch Redis/redis:v2.0.0-rc1 Redis/redis:v1.3.12 Redis/redis:v1.3.11 Redis/redis:v1.3.10 Redis/redis:v1.3.9 Redis/redis:v1.3.8 Redis/redis:v1.3.7 Redis/redis:1.3.6 Redis/redis:vm-playpen
381 Commits
7.4.0 ... 8.0-rc1
Author SHA1 Message Date
YaacovHazan b0403f1fa2 Redis CE 8.0 RC1 (#13924)
CI / test-ubuntu-latest (push) Has been cancelled
Details
CI / test-sanitizer-address (push) Has been cancelled
Details
CI / build-debian-old (push) Has been cancelled
Details
CI / build-macos-latest (push) Has been cancelled
Details
CI / build-32bit (push) Has been cancelled
Details
CI / build-libc-malloc (push) Has been cancelled
Details
CI / build-centos-jemalloc (push) Has been cancelled
Details
CI / build-old-chain-jemalloc (push) Has been cancelled
Details
Codecov / code-coverage (push) Has been cancelled
Details
External Server Tests / test-external-standalone (push) Has been cancelled
Details
External Server Tests / test-external-cluster (push) Has been cancelled
Details
External Server Tests / test-external-nodebug (push) Has been cancelled
Details
Spellcheck / Spellcheck (push) Has been cancelled
Details
2025-04-07 11:38:16 +03:00
YaacovHazan ce8c3993c5 Merge remote-tracking branch 'upstream/unstable' into HEAD 2025-04-06 21:52:28 +03:00
YaacovHazan 5582a41bb6 Few fixes around make for modules (#13922) 
- Suppress errors when removing .so files that may not exist
- Fix -DINCLUDE_VEC_SETS duplication
 2025-04-06 11:09:07 +03:00
Slava Koyfman fd4b5cb3fa Improve refcount check in 'decrRefCount' (#13888) 
The code of 'decrRefCount' included a validity check that would panic
the server if the refcount ever became invalid. However, due to the way
it was written, this could only happen if a corrupted value was written
to the field, or we attempted to decrement a newly-allocated and
never-incremented object. Incorrectly-tracked refcounts would not be
caught, as the code would never actually reduce the refcount from 1 to
0. This left potential use-after-free errors unhandled.

Improved the code so that incorrect tracking of refcounts causes a
panic, even if the freed memory happens to still be owned by the
application and not re-allocated.
 2025-04-03 21:29:06 +08:00
Ozan TezcanandOran Agra 3cdb8c6046 Improve replication buffering on replica and fix a related bug (#13904) 
With RDB channel replication, we introduced parallel replication stream
and RDB delivery to the replica during a full sync. Currently, after the
replica loads the RDB and begins streaming the accumulated buffer to the
database, it does not read from the master connection during this
period. Although streaming the local buffer is generally a fast
operation, it can take some time if the buffer is large. This PR
introduces buffering during the streaming of the local buffer. One
important consideration is ensuring that we consume more than we read
during this operation; otherwise, it could take indefinitely. To
guarantee that it will eventually complete, we limit the read to at most
half of what we consume, e.g. read at most 1 mb once we consume at least
2 mb.

**Additional changes**

**Bug fix**
- Currently, when replica starts draining accumulated buffer, we call
protectClient() for the master client as we occasionally yield back to
event loop via processEventsWhileBlocked(). So, it prevents freeing the
master client. While we are in this loop, if replica receives "replicaof
newmaster" command, we call replicaSetMaster() which expects to free the
master client and trigger a new connection attempt. As the client object
is protected, its destruction will happen asynchronously. Though, a new
connection attempt to new master will be made immediately. Later, when
the replication buffer is drained, we realize master client was marked
as CLOSE_ASAP, and freeing master client triggers another connection
attempt to the new master. In most cases, we realize something is wrong
in the replication state machine and abort the second attempt later. So,
the bug may go undetected. Fix is not calling protectClient() for the
master client. Instead, trying to detect if master client is
disconnected during processEventsWhileBlocked() and if so, breaking the
loop immediately.

**Related improvement:** 
- Currently, the replication buffer is a linked list of buffers, each of
which is 1 MB in size. While consuming the buffer, we process one buffer
at a time and check if we need to yield back to
`processEventsWhileBlocked()`. However, if
`loading-process-events-interval-bytes` is set to less than 1 MB, this
approach doesn't handle it well. To improve this, I've modified the code
to process 16KB at a time and check
`loading-process-events-interval-bytes` more frequently. This way,
depending on the configuration, we may yield back to networking more
often.

- In replication.c, `disklessLoadingRio` will be set before a call to
`emptyData()`. This change should not introduce any behavioral change
but it is logically more correct as emptyData() may yield to networking
and we may need to call rioAbort() on disklessLoadingRio. Otherwise,
failure of main channel may go undetected until a failure on rdb channel
on a corner case.

**Config changes**
- The default value for the `loading-process-events-interval-bytes`
configuration is being lowered from 2MB to 512KB. This configuration
primarily used for testing and controls the frequency of networking
during the loading phase, specifically when loading the RDB or applying
accumulated buffers during a full sync on the replica side.

Before the introduction of RDB channel replication, the 2MB value was
sufficient for occasionally yielding to networking, mainly to reply
-loading to the clients. However, with RDB channel replication, during a
full sync on the replica side (either while loading the RDB or applying
the accumulated buffer), we need to yield back to networking more
frequently to continue accumulating the replication stream. If this
doesn’t happen often enough, the replication stream can accumulate on
the master side, which is undesirable.
  
To address this, we’ve decided to lower the default value to 512KB. One
concern with frequent yielding to networking is the potential
performance impact, as each call to processEventsWhileBlocked() involves
4 syscalls, which could slow down the RDB loading phase. However,
benchmarking with various configuration values has shown that using
512KB or higher does not negatively impact RDB loading performance.
Based on these results, 512KB is now selected as the default value.

**Test changes**
- Added improved version of a replication test which checks memory usage
on master during full sync.

---------

Co-authored-by: Oran Agra <oran@redislabs.com>
 2025-04-03 10:04:29 +03:00
YaacovHazan 385823ea49 Merge remote-tracking branch 'upstream/unstable' into HEAD
CI / test-ubuntu-latest (push) Has been cancelled
Details
CI / test-sanitizer-address (push) Has been cancelled
Details
CI / build-debian-old (push) Has been cancelled
Details
CI / build-macos-latest (push) Has been cancelled
Details
CI / build-32bit (push) Has been cancelled
Details
CI / build-libc-malloc (push) Has been cancelled
Details
CI / build-centos-jemalloc (push) Has been cancelled
Details
CI / build-old-chain-jemalloc (push) Has been cancelled
Details
Codecov / code-coverage (push) Has been cancelled
Details
External Server Tests / test-external-standalone (push) Has been cancelled
Details
External Server Tests / test-external-cluster (push) Has been cancelled
Details
External Server Tests / test-external-nodebug (push) Has been cancelled
Details
Spellcheck / Spellcheck (push) Has been cancelled
Details
2025-04-02 22:05:08 +03:00
YaacovHazan 5e7333d2dd Add vector-sets module (#13915) 
The vector-sets module is a part of Redis Core and is available by
default,
just like any other data type in Redis.

As a result, when building Redis from the source, the vector-sets module
is also compiled as part of the Redis binary and loaded at server
start-up (internal module).

This new data type added as a preview feature and currently doesn't
support all the capabilities in Redis like:
* 32-bit build
* C99 (requires C11 stdatomic)
* Short-read from RDB isn't handled and might lead to a memory leak
* AOF rewirte (when aof-use-rdb-preamble is off)
* active defrag
* others?
 2025-04-02 21:54:15 +03:00
YaacovHazan 41b1b5df18 Add vector-sets module 
The vector-sets module is a part of Redis Core and is available by default,
just like any other data type in Redis.

As a result, when building Redis from the source, the vector-sets module
is also compiled as part of the Redis binary and loaded at server start-up.

This new data type added as a preview currently doesn't support
all the capabilities in Redis like:
32-bit OS
C99
Short-read that might end with memory leak
AOF rewirte
defrag
 2025-04-02 15:06:24 +00:00
YaacovHazan 78e0d87177 Add 'modules/vector-sets/' from commit 'c6db0a7c20ff5638f3a0c9ce9c106303daeb2f67' 
git-subtree-dir: modules/vector-sets
git-subtree-mainline: 8ea8f4220c
git-subtree-split: c6db0a7c20
 2025-04-02 16:34:28 +03:00
antirez c6db0a7c20 Don't use cross-thread unlocking. 2025-04-02 10:12:03 +02:00
YaacovHazan c6e5d1d5fe Merge remote-tracking branch 'upstream/unstable' into HEAD
CI / test-ubuntu-latest (push) Has been cancelled
Details
CI / test-sanitizer-address (push) Has been cancelled
Details
CI / build-debian-old (push) Has been cancelled
Details
CI / build-macos-latest (push) Has been cancelled
Details
CI / build-32bit (push) Has been cancelled
Details
CI / build-libc-malloc (push) Has been cancelled
Details
CI / build-centos-jemalloc (push) Has been cancelled
Details
CI / build-old-chain-jemalloc (push) Has been cancelled
Details
Codecov / code-coverage (push) Has been cancelled
Details
External Server Tests / test-external-standalone (push) Has been cancelled
Details
External Server Tests / test-external-cluster (push) Has been cancelled
Details
External Server Tests / test-external-nodebug (push) Has been cancelled
Details
Spellcheck / Spellcheck (push) Has been cancelled
Details
2025-03-31 21:26:40 +03:00
DvirDukhan 8ea8f4220c Update RediSearch Makefile - 7.99.90 (#13905) 2025-03-31 21:26:07 +03:00
Eran Hadad 1c646662e9 Bump module version to v7.99.90 for RedisBloom, JSON and Timeseries (#13908) 2025-03-31 21:24:22 +03:00
Ozan Tezcan 366c6aff81 Put replica online when bgsave is done (#13895) 
Before https://github.com/redis/redis/pull/13732, replicas were brought
online immediately after master wrote the last bytes of the RDB file to
the socket. This behavior remains unchanged if rdbchannel replication is
not used. However, with rdbchannel replication, the replica is brought
online after receiving the first ack which is sent by replica after rdb
is loaded.

To align the behavior, reverting this change to put replica online once
bgsave is done.

Additonal changes:
- INFO field `mem_total_replication_buffers` will also contain
`server.repl_full_sync_buffer.mem_used` which shows accumulated
replication stream during rdbchannel replication on replica side.
- Deleted debug level logging from some replication tests. These tests
generate thousands of keys and it may cause per key logging on some
cases.
 2025-03-31 13:48:49 +03:00
YaacovHazan 5d887c58ae Merge unstable into 8.0 (#13901) 
Merging unstable towards GA
 2025-03-30 15:11:57 +03:00
Jason aa8e2d1712 Ignore shardId updates from replica nodes (#13877) 
Close https://github.com/redis/redis/issues/13868

This bug was introduced by https://github.com/redis/redis/pull/13468

## Issue
To maintain compatibility with older versions that do not support
shardid, when a replica passes a shardid, we also update the master’s
shardid accordingly.

However, when both the master and replica support shardid, an issue
arises: in one moment, the master may pass a shardid, causing us to
update both the master and all its replicas to match the master’s
shardid. But if the replica later passes a different shardid, we would
then update the master’s shardid again, leading to continuous changes in
shardid.

## Solution
Regardless of the situation, we always ensure that the replica’s shardid
remains consistent with the master’s shardid.
 2025-03-30 15:15:04 +08:00
YaacovHazan 452b5b8a3b Merge remote-tracking branch 'upstream/unstable' into HEAD 2025-03-30 09:54:48 +03:00
antirez 3dd48b5b45 README: Random Projection section. 2025-03-28 18:29:56 +01:00
Vitah Linanddebing.sun 057f039c4b Fix 'RESTORE can set LFU' test (#13896) 
When the `restore foo 0 $encoded freq 100` command and `set freq [r
object freq foo]` run in different minute timestamps (i.e., when
server.unixtime/60 changes between these operations), the assertion may
fail due to the LFU decay.

This PR updates the “RESTORE can set LFU” test to verify the actual freq
value based on minute timestamps.

---------

Co-authored-by: debing.sun <debing.sun@redis.com>
 2025-03-28 13:33:58 +08:00
antirez 4dca45ad24 Remove sprintf() from cJSON. 2025-03-27 12:19:56 +01:00
antirez b17499f907 Fix projection output len. 2025-03-27 12:19:40 +01:00
antirez 29c27bc13e Make HNSW CAS commit atomic. 
This way we don't need to mess with node->value at a latter time
where an explicit lock would be required. Now we have:

1. Prepare context (neighbors).
2. Commit, and set the associated value.
 2025-03-27 12:18:58 +01:00
antirez c61c535c32 Make Redis module merging simpler. 
This way there is no need to change any file: the only
needed change is the initialization function name, that
is now controlled by the define.
 2025-03-27 10:45:05 +01:00
antirez 2f17e4fb04 Prettify parseVector(). 2025-03-27 08:35:47 +01:00
antirez 63057253d8 Document threading model in a top comment. 2025-03-27 08:31:15 +01:00
antirez 3d31fc3bee VSIM thread: manipulate results while still locked. 2025-03-27 08:11:13 +01:00
debing.sunandRain Valentine 87d8e71708 Fix defrag when type/encoding changes during scan (#13883) 
This PR is based on: https://github.com/valkey-io/valkey/pull/1801

[SoftlyRaining](https://github.com/SoftlyRaining) was hunting for defrag
bugs with Jim and found a couple of improvements to make. Jim pointed
out that in several of the callbacks, if the encoding were to change it
simply returns without doing anything to `cursor` to make it reach 0,
meaning that it would continue no-op working on that item without making
any progress. Type and encoding can change while the defrag scan is in
progress if the value is mutated or replaced by something else with the
same key.

---------

Signed-off-by: Rain Valentine <rsg000@gmail.com>
Co-authored-by: Rain Valentine <rsg000@gmail.com>
 2025-03-27 08:58:57 +08:00
antirez f70dc8acb2 Clarify VRANDMEMBER tradeoff. 2025-03-26 23:47:47 +01:00
antirez 9180659f8b Clarify failure behaior of VectorSetRdbLoad(). 2025-03-26 23:44:39 +01:00
antirez c2d80e8ced Clarify that if CAS fails we insert blocking. 2025-03-26 23:41:55 +01:00
antirez e3243819ef Don't mess with node attributes without protection. 
The background VSIMs use the node attributes (via the callback)
so we can't modify them without waiting for the background
operations to terminate.
 2025-03-26 23:36:14 +01:00
antirez a6c8a15cad VADD: fix leak on thread creation failure. 2025-03-26 22:50:47 +01:00
antirez 3e2649f1f1 hnsw_insert() should never fail in practice. 
We pass our aborting allocation function to the HNSW lib, the
only other reason for it to fail is pthread mutex locking failing
but this is also practically impossible AFAIK in modern systems,
and if it happens (for kernel reosurces shortage) anyway to
abort is the best thing to do: otherwise we would have to return
that we could not complete the operation for some reason, which
is not uniform with everything Redis does. In Redis under
normal conditions writes must succeed if they are semantically
correct, or the server crash for OOM.
 2025-03-26 22:46:00 +01:00
Ozan TezcanandOran Agra a0da8390a2 Fix use-after-free when diskless load config is not swapdb (#13887) 
When the diskless load configuration is set to on-empty-db, we retain a
pointer to the function library context. When emptyData() is called, it
frees this function library context pointer, leading to a use-after-free
situation.

I refactored code to ensure that emptyData() is called first, followed
by retrieving the valid pointer to the function library context.

Refactored code should not introduce any runtime implications.

Bug introduced by https://github.com/redis/redis/pull/13495 (Redis 8.0)

Co-authored-by: Oran Agra <oran@redislabs.com>
 2025-03-26 21:50:10 +03:00
antirez 8dfc501fb8 VSIM: fix double free if thread creation fails. 2025-03-26 19:43:59 +01:00
antirez 9d4325ee25 VSIM NOTHREAD, mainly for testing goals. 2025-03-26 16:52:28 +01:00
antirez 707c132392 Count threaded exec time in stats. 2025-03-26 16:48:02 +01:00
antirez 08e3f958fa README: remove no longer valid RP issue. 
now the projection matrix is deterministic.
 2025-03-26 11:33:32 +01:00
antirez 23b3e21817 README: suggest using FP32 vs VALUES. 2025-03-26 11:28:05 +01:00
Cong Chen 981aa5c12f Fix timing issue in HEXPIREAT test (#13873) 
This fixes an error that occurs in the job
[test-valgrind-no-malloc-usable-size-test](https://github.com/redis/redis/actions/runs/13912357739/job/38929051397)
of the Daily workflow:

```
*** [err]: HEXPIREAT - Set time and then get TTL (listpackex) in tests/unit/type/hash-field-expire.tcl
Expected '999' to be between to '1000' and '2000' (context: type eval line 6 cmd {assert_range [r hpttl myhash FIELDS 1 field1] 1000 2000} proc ::test)
```
 2025-03-26 10:00:38 +08:00
antirez 16e3c5a8f9 Locks error checking improved. 2025-03-24 19:10:28 +01:00
antirez adfd2dc7c0 Remove useless OOM checks, but handle mutex creation failure. 2025-03-24 12:54:41 +01:00
antirez 8bf9b8abc1 Use Hadamard-based projection. 
Works better and being deterministic (only relative to the projection
size) the replicas will have the same matrix automatically.
 2025-03-24 12:48:04 +01:00
Oran Agra 2a189709e0 avoid possible use-after-free with module KSN changes (#13875) 
in #13505, we changed the code to use the string value of the key rather
than the integer value on the stack, but we have a test in
unit/moduleapi/keyspace_events that uses keyspace notification hook to
modify the value with RM_StringDMA, which can cause this value to be
released before used. the reason it didn't happen so far is because we
were using shared integers, so releasing the object doesn't free it.
 2025-03-24 12:24:52 +02:00
antirez 958ebee091 README: specify how to add REDUCE in VADD. 2025-03-24 09:55:45 +01:00
Yuan Wang 319bbcc1a7 Fix sdscatprintf error of the in output of info stats (#13871) 
CI failed: https://github.com/redis/redis/actions/runs/13981749993/job/39148249096,
since i don't reassign `info` after `sdscatprintf(info, xxx)`
Thanks to @sundb for spotting this
introduced in https://github.com/redis/redis/pull/13846
 2025-03-24 09:17:58 +08:00
debing.sunandoranagra 87b7c3ac1a Fix rax node defragmentaion being skipped (#13847) 
First, when we do `raxSeek()` and then call raxNext, we will get the
`RAX_ITER_JUST_SEEKED` flag and return success directly.
We always set the node defrag callback after `raxSeek()`, which means
that when we break from defragmentation, the first node that comes in
again will never be defragged.

In this PR, we save the last as the next node to be processed, not the
last node to be completed.
This way we defrag the next node when we exit to avoid it being skipped
on the next resume.

---------

Co-authored-by: oranagra <oran@redislabs.com>
 2025-03-24 08:57:08 +08:00
antirez 8007ccd51b Use RESP3-friendly bool replies. 2025-03-23 20:14:40 +01:00
antirez 9cc750fd66 Test: projection regression test fixed. 2025-03-23 15:04:58 +01:00
antirez aa92b37589 VINFO: use a single field for random projection info. 2025-03-23 14:49:52 +01:00
antirez 8f479b22b9 Tests: replication test. 2025-03-23 14:45:34 +01:00
Salvatore Sanfilippo 854c7fdddb Merge pull request #6 from rowantrollope/main 
Fix possible crash with random projection
 2025-03-23 14:44:53 +01:00
Rowan Trollope 31bc07955c Fix possible crash with random projection 2025-03-22 09:11:20 -07:00
antirez f330d6175a Clarify HNSW_MAX_THREADS vs one thread per request. 2025-03-20 15:42:11 +01:00
Benson-liandyoungmore1024 427c36888e Fix potential infinite loop of RANDOMKEY during client pause (#13863) 
The bug mentioned in this
[#13862](https://github.com/redis/redis/issues/13862) has been fixed.

---------

Signed-off-by: li-benson <1260437731@qq.com>
Signed-off-by: youngmore1024 <youngmore1024@outlook.com>
Co-authored-by: youngmore1024 <youngmore1024@outlook.com>
 2025-03-20 21:32:12 +08:00
debing.sun cb02bd190b Fix timing issue in module defrag test (#13870) 
After #13840, the data we populate becomes more complex and slower, we
always wait for a defragmentation cycle to end before verifying that the
test is okay.
However, in some slow environments, an entire defragmentation cycle can
exceed 5 seconds, and in my local test using 'taskset -c 0' it can reach
6 seconds, so increase the threshold to avoid test failures.
 2025-03-20 21:22:47 +08:00
Yuan Wang 951ec79654 Cluster compatibility check (#13846) 
### Background
The program runs normally in standalone mode, but migrating to cluster
mode may cause errors, this is because some cross slot commands can not
run in cluster mode. We should provide an approach to detect this issue
when running in standalone mode, and need to expose a metric which
indicates the usage of no incompatible commands.

### Solution
To avoid perf impact, we introduce a new config
`cluster-compatibility-sample-ratio` which define the sampling ratio
(0-100) for checking command compatibility in cluster mode. When a
command is executed, it is sampled at the specified ratio to determine
if it complies with Redis cluster constraints, such as cross-slot
restrictions.

A new metric is exposed: `cluster_incompatible_ops` in `info stats`
output.

The following operations will be considered incompatible operations.

- cross-slot command
   If a command has multiple cross slot keys, it is incompatible
- `swap, copy, move, select` command
These commands involve multi databases in some cases, we don't allow
multiple DB in cluster mode, so there are not compatible
- Module command with `no-cluster` flag
If a module command has `no-cluster` flag, we will encounter an error
when loading module, leading to fail to load module if cluster is
enabled, so this is incompatible.
- Script/function with `no-cluster` flag
Similar with module command, if we declare `no-cluster` in shebang of
script/function, we also can not run it in cluster mode
- `sort` command by/get pattern
When `sort` command has `by/get` pattern option, we must ask that the
pattern slot is equal with the slot of keys, otherwise it is
incompatible in cluster mode.

- The script/function command accesses the keys and declared keys have
different slots
For the script/function command, we not only check the slot of declared
keys, but only check the slot the accessing keys, if they are different,
we think it is incompatible.

**Besides**, commands like `keys, scan, flushall, script/function
flush`, that in standalone mode iterate over all data to perform the
operation, are only valid for the server that executes the command in
cluster mode and are not broadcasted. However, this does not lead to
errors, so we do not consider them as incompatible commands.

### Performance impact test
**cross slot test**
Below are the test commands and results. When using MSET with 8 keys,
performance drops by approximately 3%.

**single key test**
It may be due to the overhead of the sampling function, and single-key
commands could cause a 1-2% performance drop.
 2025-03-20 10:35:53 +08:00
Filipe Oliveira (Redis)anddebing.sun 3e012c9260 Fix string2d usage in case of hexadecimal strings parsing and overflow (#13845) 
Since https://github.com/redis/redis/pull/11884, what was previously
accepted as a valid input (hexadecimal string) before 8.0 returned an
error. This PR addresses it. To avoid performance penalties if hints the
compiler that the fallbacks are not likely to happen.
Furthermore, we were ignoring std::result_out_of_range outputs from
fast_float. This PR addresses it as well and includes tests for both
identified scenarios.

---------

Co-authored-by: debing.sun <debing.sun@redis.com>
 2025-03-19 20:08:45 +08:00
antirez 758e963a4e VRANDMEMBER documentation. 2025-03-19 09:02:15 +01:00
debing.sun 26dcec4812 Fix messed-up unblocked clients in flush command (#13865) 
Fix https://github.com/redis/redis/pull/13853#pullrequestreview-2675227138

This PR ensures that the client's current command is not reset by
unblockClient(), while still needing to be handled after `unblockclient()`.
The FLUSH command still requires reprocessing (update the replication
offset) after unblockClient(). Therefore, we mark such blocked clients
with the CLIENT_PENDING_COMMAND flag to prevent the command from being
reset during unblockClient().
 2025-03-19 10:22:47 +08:00
antirez 3424757f4d Test: added another threading stress test. 
This access pattern triggered the bug fixed
about VADD and CAS in 70ffa8c.
 2025-03-18 23:18:26 +01:00
antirez 70ffa8ce5c Fix VADD_CASReply() NULL reference on ID mismatch. 
This bug was fixed thanks to the kind help of Dvir Dukhan
(@DvirDukhan) that found it and provided useful context.
 2025-03-18 21:37:06 +01:00
antirez 99176b3e04 Test: VRANDMEMBER test added. 2025-03-18 16:49:27 +01:00
antirez 22ce9f3fad VRANDMEMBER command implemented. 2025-03-17 23:52:15 +01:00
debing.sunandoranagra a5a3afd923 Fix crash during SLAVEOF when clients are blocked on lazyfree (#13853) 
After https://github.com/redis/redis/pull/13167, when a client calls
`FLUSHDB` command, we still async empty database, and the client was
blocked until the lazyfree completes.

1) If another client calls `SLAVEOF` command during this time, the
server will unblock all blocked clients, including those blocked by the
lazyfree. However, when unblocking a lazyfree blocked client, we forgot
to call `updateStatsOnUnblock()`, which ultimately triggered the
following assertion.

2) If a client blocked by Lazyfree is unblocked midway, and at this
point the `bio_comp_list` has already received the completion
notification for the bio, we might end up processing a client that has
already been unblocked in `flushallSyncBgDone()`. Therefore, we need to
filter it out.

---------

Co-authored-by: oranagra <oran@redislabs.com>
 2025-03-17 20:27:05 +08:00
YaacovHazan 095c131fbb Redis 8.0 M04
CI / test-ubuntu-latest (push) Has been cancelled
Details
CI / test-sanitizer-address (push) Has been cancelled
Details
CI / build-debian-old (push) Has been cancelled
Details
CI / build-macos-latest (push) Has been cancelled
Details
CI / build-32bit (push) Has been cancelled
Details
CI / build-libc-malloc (push) Has been cancelled
Details
CI / build-centos-jemalloc (push) Has been cancelled
Details
CI / build-old-chain-jemalloc (push) Has been cancelled
Details
Codecov / code-coverage (push) Has been cancelled
Details
External Server Tests / test-external-standalone (push) Has been cancelled
Details
External Server Tests / test-external-cluster (push) Has been cancelled
Details
External Server Tests / test-external-nodebug (push) Has been cancelled
Details
Spellcheck / Spellcheck (push) Has been cancelled
Details
2025-03-16 10:12:40 +02:00
YaacovHazan 89eef40ca2 Merge remote-tracking branch 'upstream/unstable' into HEAD 2025-03-16 10:01:10 +02:00
kei-nan 752576ce47 Use Search v7.99.5 (#13859) 2025-03-16 10:00:51 +02:00
antirez 706721f8c8 HSNW: random node. 2025-03-16 00:08:43 +01:00
antirez 8a5cf17cb2 HNSW: cursor fixes and thread safety. 2025-03-15 23:31:24 +01:00
antirez a363e5fe6d README: memory usage section. 2025-03-15 23:16:28 +01:00
antirez 6e434bcaaf HNSW: use node max link property. 
This is both more correct in formal terms, and in practical
terms as well, as we could over-allocate nodes sometimes.
 2025-03-15 10:30:14 +01:00
antirez 68d3067125 w2v test: fix recall EF usage. 2025-03-15 10:24:20 +01:00
antirez d94058fad9 w2v test: recall histograms + configurable M. 2025-03-15 09:46:42 +01:00
antirez c1c7eeaa69 Document VADD M parameter. 2025-03-15 09:28:55 +01:00
antirez 542736ce25 w2v test: proper recall test added. 2025-03-15 00:24:10 +01:00
antirez 13a0a63bef Copyright Sanfilipo -> Redis Ltd. 2025-03-14 23:06:22 +01:00
antirez d996eb82ef VADD: make M configurable at creation time. 2025-03-13 16:58:55 +01:00
antirez 4e57d3f76f README: grammar. 2025-03-13 15:56:05 +01:00
antirez 2fcf389f2a README: troubleshooting and understandability. 2025-03-13 13:25:48 +01:00
antirez 9500539c55 HNSW: implement last resort node reallocation. 2025-03-13 11:30:07 +01:00
antirez 095842a748 README: scaling information. 2025-03-12 22:58:33 +01:00
antirez 63ae981599 README: show main scalability property of vsets. 2025-03-12 18:41:49 +01:00
antirez cc3874ab87 VADD CAS: fallback when thread creation fails. 2025-03-12 16:57:03 +01:00
antirez f05912dea2 cJSON updated to latest version. 2025-03-12 09:55:23 +01:00
YaacovHazan 84471e238e Redis 8.0 RC1 (#13851) 
- Merge latest unstable
- Update version and release notes
 2025-03-11 20:16:27 +02:00
YaacovHazan d1df881ec5 Redis 8.0 RC1 2025-03-11 10:03:17 +02:00
YaacovHazan 53949521de Merge remote-tracking branch 'upstream/unstable' into HEAD 2025-03-11 09:39:04 +02:00
Eran Hadad b704179f15 Update release of RedisJSON, RedisTS and RedisBloom 7.99.4 (#13850) 2025-03-11 09:36:28 +02:00
DvirDukhan 557e0b1c07 Update Makefile with search 7.99.4 (#13848) 2025-03-09 13:55:27 +02:00
YaacovHazan a39ffc1fe9 Merge remote-tracking branch 'upstream/unstable' into HEAD 2025-03-09 10:08:03 +02:00
antirez f829d46535 HNSW: creation time M parameter VS hardcoded. 2025-03-08 16:15:15 +01:00
antirez 0258e85186 VSIM TRUTH option for ground truth results. 2025-03-07 09:58:16 +01:00
debing.sunandephraimfeldblum f364dcca2d Make RM_DefragRedisModuleDict API support incremental defragmentation for dict leaf (#13840) 
After https://github.com/redis/redis/pull/13816, we make a new API to
defrag RedisModuleDict.
Currently, we only support incremental defragmentation of the dictionary
itself, but the defragmentation of values is still not incremental. If
the values are very large, it could lead to significant blocking.
Therefore, in this PR, we have added incremental defragmentation for the
values.

The main change is to the `RedisModuleDefragDictValueCallback`, we
modified the return value of this callback.
When the callback returns 1, we will save the `seekTo` as the key of the
current unfinished node, and the next time we enter, we will continue
defragmenting this node.
When the return value is 0, we will proceed to the next node.

## Test
Since each dictionary in the global dict originally contained only 10
strings, but now it has been changed to a nested dictionary, each
dictionary now has 10 sub-dictionaries, with each sub-dictionary
containing 10 strings, this has led to a corresponding reduction in the
defragmentation time obtained from other tests.
Therefore, the other tests have been modified to always wait for
defragmentation to be turned off before the test begins, then start it
after creating fragmentation, ensuring that they can always run for a
full defragmentation cycle.

---------

Co-authored-by: ephraimfeldblum <ephraim.feldblum@redis.com>
 2025-03-04 17:19:41 +08:00
antirez 2114c65012 VINFO: add attributes count. 2025-03-04 09:35:41 +01:00
antirez ed7c539303 Improve Vector Sets MEMORY USAGE implementation. 
Now attributes are added as well. Moreover the code no longer uses
the first node to guess the size of the items, but does an average
of the few first items/attributes found. Still O(1) but more precise.
 2025-03-04 09:35:41 +01:00
antirez 1d09d67909 Tests: regressios for MEMORY USAGE / DEBUG DIGEST. 2025-03-04 09:35:41 +01:00
antirez 1f92040fcf Fix MEMORY USAGE and DEBUG DIGEST crash. 
After adding attributes, the code was still accessing
node->value as a string, but now this is mediated by the value
object.

Close #5
 2025-03-04 09:35:41 +01:00
Rowan Trollope 0f2c356b07 Merge pull request #4 from rowantrollope/main 
Update readme to fix VGET
 2025-03-03 12:31:10 -08:00
Rowan Trollope 0e3ee9afb4 Update readme to fix VGET 
VGET -> VGETATTR :)
 2025-03-03 11:14:36 -08:00
antirez ab5e01d6bc README: Remove the term "hybrid" search. 2025-03-03 17:29:41 +01:00
antirez b49bc14f96 Fix README conflict. 2025-03-03 13:12:25 +01:00
antirez 883d9e3a75 README: example data set. 2025-03-03 13:10:10 +01:00
antirez 07fd2fa8a6 README: extensive hybrid search documentation. 2025-03-03 10:07:05 +01:00
antirez afcc2ff6e8 LICENSE: change copyright to Redis Ltd. 2025-03-03 09:51:26 +01:00
antirez 4b0bd5b0bd README: VSETATTR / VGETATTR first documentation. 2025-03-03 09:44:40 +01:00
antirez 1ad503001f README: initial documentation of hybrid search. 2025-03-02 22:41:34 +01:00
antirez 6c95ec1d6c Test: fix integration test for FILTER. 2025-03-02 13:38:32 +01:00
antirez abe33257d9 Expr: improve selectors / operators parsing. 2025-03-02 12:58:06 +01:00
antirez c8b6cbc6e1 Test: FILTER integration tests, work in progress. 2025-03-02 12:03:49 +01:00
antirez 1cb927aef6 Test: text FILTER expressions basics. 2025-02-28 17:52:46 +01:00
antirez b417685430 Expr: Allow _ in selectors. 2025-02-28 17:52:27 +01:00
antirez 89ef4c0702 Expr: convert Json bool to 1/0 numerical type. 2025-02-28 17:49:09 +01:00
antirez 2d311dbb01 Fix VLINKS after adding attributes. 2025-02-28 16:39:33 +01:00
antirez 68dccc55ad Fix CAS insertion broken when adding attributes. 2025-02-28 16:35:45 +01:00
antirez 68683e181c Add FILTER-EF option. 2025-02-28 13:05:19 +01:00
antirez ef74527d92 HNSW: binary distance: fix type for the xor var. 2025-02-27 10:13:27 +01:00
antirez f20684e7b5 HNSW: don't free layers, now is part of the node itself. 2025-02-27 10:13:27 +01:00
antirez 1a2da02db6 HNSW: calloc() -> hmalloc(). 2025-02-27 10:13:27 +01:00
Salvatore Sanfilippo 6e09e05af5 Merge pull request #3 from rowantrollope/main 
Adding more files to .gitignore and removing stray .rdb file
 2025-02-27 09:25:56 +01:00
YaacovHazan cb261828bd Merge unstable into 8.0 (#13835) 
preparing for 8.0 RC1
 2025-02-27 08:29:01 +02:00
YaacovHazan 9265234299 Merge remote-tracking branch 'upstream/unstable' into HEAD 2025-02-26 21:23:36 +02:00
debing.sunandoranagra 7939ba031d Enable the callback to be NULL for RM_DefragRedisModuleDict() and reduce the system calls of RM_DefragShouldStop() (#13830) 
1) Enable the callback to be NULL for RM_DefragRedisModuleDict()
    Because the dictionary may store only the key without the value.

2) Reduce the system calls of RM_DefragShouldStop()
The API checks the following thresholds before performing a time check:
over 512 defrag hits, or over 1024 defrag misses, and performs the time
judgment if any of these thresholds are reached.

3) Added defragmentation statistics for dictionary items to cover the
associated code for RM_DefragRedisModuleDict().

4) Removed `module_ctx` from `defragModuleCtx` struct, which can be
replaced by a temporary variable.

---------

Co-authored-by: oranagra <oran@redislabs.com>
 2025-02-26 20:04:29 +08:00
Rowan Trollope 2a8af82f50 Updated docs to show mutually exclusive quantization flags 2025-02-26 00:55:07 -08:00
antirez 3abc801d7a Fix leak on VSIM FILTER syntax error. 2025-02-25 16:15:33 +01:00
antirez 774c05ab55 Add cJSON library. 2025-02-25 16:06:40 +01:00
antirez 44c064c0b4 Expr: handle JSON arrays as expr tuples. 2025-02-25 16:05:28 +01:00
antirez 764fb8cc74 Expr: use attribute in test expression. 2025-02-25 12:39:36 +01:00
antirez ab06a5a058 Expr: conver to refcounting. 2025-02-25 12:38:20 +01:00
antirez 3627bbe12c Fixed a few memory handling/corruption bugs. 2025-02-25 09:43:36 +01:00
Yuan Wang f1d6542b1a Stabilize tcl test cases (#13829) 
Recently encountered some errors as bellow,

HGETEX/HSETEX with PXAT/EXAT options, after getting ttl, we calculate
current time by `[clock seconds]` that may have a delay that causes
results greater than expected.

Dismiss memory test error, now we introduced rdb-channel replication,
the full synchronization might finish before the child process exits. So
we may fail if calling `bgsave` immediately after full sync.
 2025-02-25 16:31:53 +08:00
Denis Nevmerzhitskiidebing.sunguybe7
 		33f03f6fc8 Fix wrong behavior of XREAD + after last entry of stream have been removed (#13632) 
Close #13628

This PR changes behavior of special `+` id of XREAD command. Now it uses
`streamLastValidID` to find last entry instead of `last_id` field of
stream object.
This PR adds test for the issue.

**Notes**

Initial idea to update `last_id` while executing XDEL seems to be wrong.
`last_id` is used to strore last generated id and not id of last entry.

---------

Co-authored-by: debing.sun <debing.sun@redis.com>
Co-authored-by: guybe7 <guy.benoish@redislabs.com>
 2025-02-25 13:40:24 +08:00
Filipe Oliveira (Redis)debing.sunShooterIT
 		985bf68f34 Reduce redundant key slot calculations on expiration checks (#13796) 
On high-pipeline/fast commands use-cases, expireIfNeeded can take up to
3% cpu cycles. 

This PR introduces an optimization where key expiration checks leverage
key slots to improve efficiency.

---------

Co-authored-by: debing.sun <debing.sun@redis.com>
Co-authored-by: ShooterIT <wangyuancode@163.com>
 2025-02-25 11:55:30 +08:00
Moti Cohen 0200e8ada6 Fix multiple issues with "INFO KEYSIZES" (#13825) 
This commit addresses several issues related to the `INFO KEYSIZES` feature:
- HyperLogLog commands: `KEYSIZES` hooks were not properly set or tested.
- HFE lazy expiration: `KEYSIZES` hooks were not properly set or tested.
- Empty DB & SYNC flow: On `blocking_async=0` flow, global `keysizes`
  histogram were not reset (can reproduced using `DEBUG RELOAD`).
- Empty string handling: Fix histogram for strings of size 0. Not 
  relevant to other data-types.
 2025-02-25 00:38:44 +02:00
antirez de32f40b98 Expr: fix pow associativeness and alloc size. 2025-02-24 23:37:49 +01:00
antirez 3b60921c53 Expr: implement tuple, fix many stuff. 2025-02-24 23:37:49 +01:00
antirez a9e02dfd29 Expr: add strdup alias for stand-alone build. 2025-02-24 23:37:49 +01:00
antirez e66a50ec3c Expr: remove useless allocation checks + fix leak. 2025-02-24 23:37:49 +01:00
antirez ef24ab7821 Allow decimals, exponent numbers in expressions. 2025-02-24 23:37:49 +01:00
antirez d3ada8090f Fix expression op && entry trailing space. 2025-02-24 23:37:49 +01:00
antirez 2f1d917cf1 Fix exprParseNumber() overflow check. 2025-02-24 23:37:49 +01:00
antirez 7ad3cea7fa Expr filtering: fix selector name copying. 2025-02-24 23:37:49 +01:00
antirez 5304318335 Expr filtering: VSIM FILTER first draft. 2025-02-24 23:37:49 +01:00
antirez 025790fc50 Expr filtering: implement HNSW filter in search_layer(). 2025-02-24 23:37:49 +01:00
antirez 438adc917b Expr filtering: fix initialization of attrib. 2025-02-24 23:37:49 +01:00
antirez e3a8921ab5 VSETATTR / VGETATTR. 2025-02-24 23:37:49 +01:00
antirez 2d1642504d Expr filtering: fixed a few bugs. 2025-02-24 23:37:49 +01:00
antirez 097f310797 Expr filtering: save/load attribs. 2025-02-24 23:37:49 +01:00
antirez 832090d821 Expr filtering: draft of the dual ported object. 2025-02-24 23:37:49 +01:00
antirez af6fa6f732 Expr filtering: conversions and other fixes. 2025-02-24 23:37:49 +01:00
antirez a90d2ea290 Expr filtering: first draft of exprRun(). 2025-02-24 23:37:49 +01:00
antirez 9e4413d1e3 Expr filtering: function to free tokens. 2025-02-24 23:37:49 +01:00
antirez c6bd5d3542 Expr filtering: int error offset + selector fix. 2025-02-24 23:37:49 +01:00
antirez f9d4a5c435 Expr filtering: first compilation draft. 2025-02-24 23:37:49 +01:00
antirez 56bc353717 Expr filtering: separate tokenization from compiling. 2025-02-24 23:37:49 +01:00
antirez 6ad37b6550 Expression filtering: parsing WIP. 2025-02-24 23:37:49 +01:00
antirez 710f70f963 Make allocators func pointers private to HNSW. 2025-02-24 23:37:49 +01:00
antirez 90c2349b35 HNSW: binary distance: fix type for the xor var. 2025-02-24 23:34:18 +01:00
antirez 92dcfeae8b HNSW: don't free layers, now is part of the node itself. 2025-02-24 23:32:51 +01:00
antirez a5cf561288 HNSW: calloc() -> hmalloc(). 2025-02-24 23:26:23 +01:00
Filipe Oliveira (Redis)debing.sunYuan Wang
 		1848809f66 Optimize dictFind by leveraging key length functions to avoid redundant computations. (#13792) 
This PR enhances dictFind by introducing support for key length
functions, allowing the use of keyCompareWithLen when available. This
avoids redundant key length computations, improving efficiency,
especially when the dictionary is rehashing or there are a significant
number of hash collisions.

Additionally, it maintains backward compatibility and optimizes key
lookups without altering existing behavior.

Performance improvement on 100% GETs use-case

benchmark command used
```
taskset -c 1-11 memtier_benchmark --ratio 0:1 --key-maximum 1000000 --key-minimum 1 -c 1 -t 5 --pipeline 100 --key-pattern P:P --test-time 30 --hide-histogram -d 1024 -S /tmp/1.socket  -x 3
```

In unstable dictFindByHash takes 29% (and sdslen within it takes 8.9%)
of CPU cycles for a high-pipeline 100% gets use-case.
After this change dictFindByHash takes 27.8% (and sdslen within it takes
7.7%)

---------

Co-authored-by: debing.sun <debing.sun@redis.com>
Co-authored-by: Yuan Wang <wangyuancode@163.com>
 2025-02-24 22:27:06 +08:00
Filipe Oliveira (Redis) d7a448f9ae Avoid redundant calls to sdslen(c->querybuf) in processMultibulkBuffer (#13787) 
Optimize processMultibulkBuffer by avoiding redundant calls to
sdslen(c->querybuf).
The cached length is updated only when querybuf is modified.
 2025-02-24 20:06:14 +08:00
debing.sun 658424fc83 Revert "Update history for ban-list propagation (#13749)" (#13827) 
As discussed in
https://github.com/redis/redis/pull/13749#issuecomment-2673612941.
After #10398 we should record only the arguments and output changes in
the command history, while placing all others in the redis-doc, so
revert #13749.
 2025-02-24 17:40:25 +08:00
Filipe Oliveira (Redis)anddebing.sun 3f06ddfb7b Reuse lookupCommand data on consecutive same command calls on main thread (#13764) 
We can see that on fast commands and fast pipeline use-cases,
lookupCommand() takes 1.9% to 3.4% of total cpu cyles (depending on
pipeline). In cases in which consecutives commands are the same we can
avoid the call to lookupCommand() completely without changing or adding
new fields to the client struct (we simply reuse the info already
avaiable in lastcmd). This change can represent an improvement of around
4.4% in QPS on the high pipeline use-cases.

---------

Co-authored-by: debing.sun <debing.sun@redis.com>
 2025-02-24 12:33:14 +08:00
YaacovHazan 1cddd0d3ba Merge unstable into 8.0 (#13824) 2025-02-23 08:27:33 +02:00
debing.sunandShooterIT ee933d9e2b Fixed passing incorrect endtime value for module context (#13822) 
1) Fix a bug that passing an incorrect endtime to module.
   This bug was found by @ShooterIT.
After #13814, all endtime will be monotonic time, and we should no
longer convert it to ustime relative.
Add assertions to prevent endtime from being much larger thatn the
current time.

2) Fix a race in test `Reduce defrag CPU usage when module data can't be
defragged`

---------

Co-authored-by: ShooterIT <wangyuancode@163.com>
 2025-02-23 12:58:48 +08:00
YaacovHazan 1d5e13e121 Merge remote-tracking branch 'upstream/unstable' into HEAD 2025-02-22 21:48:15 +02:00
debing.sunShooterIToranagra
 		032357ec0f Add RM_DefragRedisModuleDict module API (#13816) 
After #13815, we introduced incremental defragmentation for global data
for module.
Now we added a new module API `RM_DefragRedisModuleDict` to incremental
defrag `RedisModuleDict`.

This PR adds a new APIs and a new defrag callback:
```c
RedisModuleDict *RM_DefragRedisModuleDict(RedisModuleDefragCtx *ctx, RedisModuleDict *dict, RedisModuleDefragDictValueCallback valueCB, RedisModuleString **seekTo);

typedef void *(*RedisModuleDefragDictValueCallback)(RedisModuleDefragCtx *ctx, void *data, unsigned char *key, size_t keylen);
```

Usage:
```c
RedisModuleString *seekTo = NULL;
RedisModuleDict *dict = = RedisModule_CreateDict(ctx);
... populate the dict code ...
/* Defragment a dictionary completely */
do {
    RedisModuleDict *new = RedisModule_DefragRedisModuleDict(ctx, dict, defragGlobalDictValueCB, &seekTo);
    if (new != NULL) {
        dict = new;
    }
} while (seekTo);
```

---------

Co-authored-by: ShooterIT <wangyuancode@163.com>
Co-authored-by: oranagra <oran@redislabs.com>
 2025-02-20 21:09:29 +08:00
debing.sunShooterIToranagra
 		695126ccce Add support for incremental defragmentation of global module data (#13815) 
## Description

Currently, when performing defragmentation on non-key data within the
module, we cannot process the defragmentation incrementally. This
limitation affects the efficiency and flexibility of defragmentation in
certain scenarios.
The primary goal of this PR is to introduce support for incremental
defragmentation of global module data.

## Interface Change
New module API `RegisterDefragFunc2`

This is a more advanced version of `RM_RegisterDefragFunc`, in that it
takes a new callbacks(`RegisterDefragFunc2`) that has a return value,
and can use RM_DefragShouldStop in and indicate that it should be called
again later, or is it done (returned 0).

## Note
The `RegisterDefragFunc` API remains available.

---------

Co-authored-by: ShooterIT <wangyuancode@163.com>
Co-authored-by: oranagra <oran@redislabs.com>
 2025-02-20 00:28:16 +08:00
debing.sunMadelyn Olson madelyneolson@gmail.comShooterIT
 		725cd268e6 Refactor of ActiveDefrag to reduce latencies (#13814) 
This PR is based on: https://github.com/valkey-io/valkey/pull/1462

## Issue/Problems

Duty Cycle: Active Defrag has configuration values which determine the
intended percentage of CPU to be used based on a gradient of the
fragmentation percentage. However, Active Defrag performs its work on
the 100ms serverCron timer. It then computes a duty cycle and performs a
single long cycle. For example, if the intended CPU is computed to be
10%, Active Defrag will perform 10ms of work on this 100ms timer cron.

* This type of cycle introduces large latencies on the client (up to
25ms with default configurations)
* This mechanism is subject to starvation when slow commands delay the
serverCron

Maintainability: The current Active Defrag code is difficult to read &
maintain. Refactoring of the high level control mechanisms and functions
will allow us to more seamlessly adapt to new defragmentation needs.
Specific examples include:

* A single function (activeDefragCycle) includes the logic to
start/stop/modify the defragmentation as well as performing one "step"
of the defragmentation. This should be separated out, so that the actual
defrag activity can be performed on an independent timer (see duty cycle
above).
* The code is focused on kvstores, with other actions just thrown in at
the end (defragOtherGlobals). There's no mechanism to break this up to
reduce latencies.
* For the main dictionary (only), there is a mechanism to set aside
large keys to be processed in a later step. However this code creates a
separate list in each kvstore (main dict or not), bleeding/exposing
internal defrag logic. We only need 1 list - inside defrag. This logic
should be more contained for the main key store.
* The structure is not well suited towards other non-main-dictionary
items. For example, pub-sub and pub-sub-shard was added, but it's added
in such a way that in CMD mode, with multiple DBs, we will defrag
pub-sub repeatedly after each DB.

## Description of the feature

Primarily, this feature will split activeDefragCycle into 2 functions.

1. One function will be called from serverCron to determine if a defrag
cycle (a complete scan) needs to be started. It will also determine if
the CPU expenditure needs to be adjusted.
2. The 2nd function will be a timer proc dedicated to performing defrag.
This will be invoked independently from serverCron.

Once the functions are split, there is more control over the latency
created by the defrag process. A new configuration will be used to
determine the running time for the defrag timer proc. The default for
this will be 500us (one-half of the current minimum time). Then the
timer will be adjusted to achieve the desired CPU. As an example, 5% of
CPU will run the defrag process for 500us every 10ms. This is much
better than running for 5ms every 100ms.

The timer function will also adjust to compensate for starvation. If a
slow command delays the timer, the process will run proportionately
longer to ensure that the configured CPU is achieved. Given the presence
of slow commands, the proportional extra time is insignificant to
latency. This also addresses the overload case. At 100% CPU, if the
event loop slows, defrag will run proportionately longer to achieve the
configured CPU utilization.

Optionally, in low CPU situations, there would be little impact in
utilizing more than the configured CPU. We could optionally allow the
timer to pop more often (even with a 0ms delay) and the (tail) latency
impact would not change.

And we add a time limit for the defrag duty cycle to prevent excessive
latency. When latency is already high (indicated by a long time between
calls), we don't want to make it worse by running defrag for too long.

Addressing maintainability:

* The basic code structure can more clearly be organized around a
"cycle".
* Have clear begin/end functions and a set of "stages" to be executed.
* Rather than stages being limited to "kvstore" type data, a cycle
should be more flexible, incorporating the ability to incrementally
perform arbitrary work. This will likely be necessary in the future for
certain module types. It can be used today to address oddballs like
defragOtherGlobals.
* We reduced some of the globals, and reduce some of the coupling.
defrag_later should be removed from serverDb.
* Each stage should begin on a fresh cycle. So if there are
non-time-bounded operations like kvstoreDictLUTDefrag, these would be
less likely to introduce additional latency.


Signed-off-by: Jim Brunner
[brunnerj@amazon.com](mailto:brunnerj@amazon.com)
Signed-off-by: Madelyn Olson
[madelyneolson@gmail.com](mailto:madelyneolson@gmail.com)
Co-authored-by: Madelyn Olson
[madelyneolson@gmail.com](mailto:madelyneolson@gmail.com)

---------

Signed-off-by: Jim Brunner brunnerj@amazon.com
Signed-off-by: Madelyn Olson madelyneolson@gmail.com
Co-authored-by: Madelyn Olson madelyneolson@gmail.com
Co-authored-by: ShooterIT <wangyuancode@163.com>
 2025-02-20 00:05:24 +08:00
guybe7 66df58f961 Do not send NL if replica client is already closed (#13813) 
In case a replica connection was closed mid-RDB, we should not send a \n
to that replica, otherwise, it may reach the replica BEFORE it realizes
that the RDB transfer failed, causing it to treat the \n as if it was
read from the RDB stream
 2025-02-19 15:04:28 +07:00
Rowan Trollope c4f7efc3cd Removing file 2025-02-18 19:38:04 -08:00
Rowan Trollope b6d129dce0 Merge branch 'redis:main' into main 2025-02-19 12:34:19 +09:00
luozongle01 b045fe4e17 Fix overflow on 32-bit systems when calculating idle time for eviction (#13804) 
the `dictGetSignedIntegerVal` function should be used here,
because in some cases (especially on 32-bit systems) long may
be 4 bytes, and the ttl time saved in expires is a unix timestamp
(millisecond value), which is more than 4 bytes. In this case, we may
not be able to get the correct idle time, which may cause eviction
disorder, in other words, keys that should be evicted later may be
evicted earlier.
 2025-02-19 11:01:15 +08:00
Yunxiao Du c5f91abaf7 Fix syntax issue in comments of src/module.c (#13802) 
closes https://github.com/redis/redis/issues/13797, just fix syntax
issue in comments instead of real code.
 2025-02-19 10:58:14 +08:00
Ozan Tezcan 6c202f495c Remove DENYOOM flag from hexpire command (#13800) 
Remove DENYOOM flag from hexpire / hexpireat / hpexpire / hpexpireat
commands.

h(p)expire(at) commands may allocate some memory but it is not that big.
Similary, we don't have DENYOOM flag for EXPIRE command. This change
will align EXPIRE and HEXPIRE commands in this manner.
 2025-02-16 20:07:29 +03:00
Rowan Trollope bf13c84977 ignore *.rdb 2025-02-16 16:37:19 +09:00
Rowan Trollope a266351c0a Update .gitignore 2025-02-16 16:32:33 +09:00
Rowan Trollope 914cfec777 Merge branch 'main' of https://github.com/rowantrollope/vector-sets 2025-02-16 16:31:56 +09:00
Ozan Tezcan e2608478b6 Add HGETDEL, HGETEX and HSETEX hash commands (#13798) 
This PR adds three new hash commands: HGETDEL, HGETEX and HSETEX. These
commands enable user to do multiple operations in one step atomically
e.g. set a hash field and update its TTL with a single command.
Previously, it was only possible to do it by calling hset and hexpire
commands subsequently.

- **HGETDEL command**

  ```
  HGETDEL <key> FIELDS <numfields> field [field ...]
  ```
  
  **Description**  
  Get and delete the value of one or more fields of a given hash key
  
  **Reply**  
Array reply: list of the value associated with each field or nil if the
field doesn’t exist.

- **HGETEX command**

  ```
   HGETEX <key>  
[EX seconds | PX milliseconds | EXAT unix-time-seconds | PXAT
unix-time-milliseconds | PERSIST]
     FIELDS <numfields> field [field ...]
  ```

  **Description**
Get the value of one or more fields of a given hash key, and optionally
set their expiration

  **Options:**
  EX seconds: Set the specified expiration time, in seconds.
  PX milliseconds: Set the specified expiration time, in milliseconds.
EXAT timestamp-seconds: Set the specified Unix time at which the field
will expire, in seconds.
PXAT timestamp-milliseconds: Set the specified Unix time at which the
field will expire, in milliseconds.
  PERSIST: Remove the time to live associated with the field.

  **Reply** 
Array reply: list of the value associated with each field or nil if the
field doesn’t exist.

- **HSETEX command**

  ```
  HSETEX <key>
     [FNX | FXX]
[EX seconds | PX milliseconds | EXAT unix-time-seconds | PXAT
unix-time-milliseconds | KEEPTTL]
     FIELDS <numfields> field value [field value...]
  ```
  **Description**
Set the value of one or more fields of a given hash key, and optionally
set their expiration

  **Options:**
  FNX: Only set the fields if all do not already exist.
  FXX: Only set the fields if all already exist.

  EX seconds: Set the specified expiration time, in seconds.
  PX milliseconds: Set the specified expiration time, in milliseconds.
EXAT timestamp-seconds: Set the specified Unix time at which the field
will expire, in seconds.
PXAT timestamp-milliseconds: Set the specified Unix time at which the
field will expire, in milliseconds.
  KEEPTTL: Retain the time to live associated with the field.

  
Note: If no option is provided, any associated expiration time will be
discarded similar to how SET command behaves.

  **Reply**
  Integer reply: 0 if no fields were set
  Integer reply: 1 if all the fields were set
 2025-02-14 17:13:35 +03:00
Ofir Luzon 57807cd338 Memory Usage command LIST accuracy fix (#13783) 
MEMORY USAGE on a List samples quicklist entries, but does not account
to how many elements are in each sampled node. This can skew the
calculation when the sampled nodes are not balanced.
The fix calculate the average element size in the sampled nodes instead
of the average node size.
 2025-02-14 09:18:47 +08:00
Yuan Wang 7f5f588232 AOF offset info (#13773) 
### Background 
AOF is often used as an effective data recovery method, but now if we
have two AOFs from different nodes, it is hard to learn which one has
latest data. Generally, we determine whose data is more up-to-date by
reading the latest modification time of the AOF file, but because of
replication delay, even if both master and replica write to the AOF at
the same time, the data in the master is more up-to-date (there are
commands that didn't arrive at the replica yet, or a large number of
commands have accumulated on replica side ), so we may make wrong
decision.

### Solution
The replication offset always increments when AOF is enabled even if
there is no replica, we think replication offset is better method to
determine which one has more up-to-date data, whoever has a larger
offset will have newer data, so we add the start replication offset info
for AOF, as bellow.
```
file appendonly.aof.2.base.rdb seq 2 type b
file appendonly.aof.2.incr.aof seq 2 type i startoffset 224
```
And if we close gracefully the AOF file, not a crash, such as
`shutdown`, `kill signal 15` or `config set appendonly no`, we will add
the end replication offset, as bellow.
```
file appendonly.aof.2.base.rdb seq 2 type b
file appendonly.aof.2.incr.aof seq 2 type i startoffset 224 endoffset 532
```

#### Things to pay attention to
- For BASE AOF, we do not add `startoffset` and `endoffset` info, since
we could not know the start replication replication of data, and it is
useless to help us to determine which one has more up-to-date data.
- For AOFs from old version, we also don't add `startoffset` and
`endoffset` info, since we also don't know start replication replication
of them. If we add the start offset from 0, we might make the judgment
even less accurate. For example, if the master has just rewritten the
AOF, its INCR AOF will inevitably be very small. However, if the replica
has not rewritten AOF for a long time, its INCR AOF might be much
larger. By applying the following method, we might make incorrect
decisions, so we still just check timestamp instead of adding offset
info
- If the last INCR AOF has `startoffset` or `endoffset`, we need to
restore `server.master_repl_offset` according to them to avoid the
rollback of the `startoffset` of next INCR AOF. If it has `endoffset`,
we just use this value as `server.master_repl_offset`, and a very
important thing is to remove this information from the manifest file to
avoid the next time we load the manifest file with wrong `endoffset`. If
it only has `startoffset`, we calculate `server.master_repl_offset` by
the `startoffset` plus the file size.

### How to determine which one has more up-to-date data
If AOF has a larger replication offset, it will have more up-to-date
data. The following is how to get AOF offset:

Read the AOF manifest file to obtain information about **the last INCR
AOF**
1. If the last INCR AOF has `endoffset` field, we can directly use the
`endoffset` to present the replication offset of AOF
2. If there is no `endoffset`(such as redis crashes abnormally), but
there is `startoffset` filed of the last INCR AOF, we can get the
replication offset of AOF by `startoffset` plus the file size
3. Finally, if the AOF doesn’t have both `startoffset` and `endoffset`,
maybe from old version, and new version redis has not rewritten AOF yet,
we still need to check the modification timestamp of the last INCR AOF

### TODO
Fix ping causing inconsistency between AOF size and replication
offset in the future PR. Because we increment the replication offset
when sending PING/REPLCONF to the replica but do not write data to the
AOF file, this might cause the starting offset of the AOF file plus its
size to be inconsistent with the actual replication offset.
 2025-02-13 17:31:40 +08:00
Yuan Wang 662cb2fe75 Don't send unnecessary PING to replicas (#13790) 
The reason why master sends PING is to keep the connection with replica
active, so master need not send PING to replicas if already sent
replication stream in the past `repl_ping_slave_period` time.

Now master only sends PINGs and increases `master_repl_offset` if there
is no traffic, so this PR also can reduce the impact of issue in
https://github.com/redis/redis/pull/13773, of course, does not resolve
it completely.
> Fix ping causing inconsistency between AOF size and replication offset
in the future PR. Because we increment the replication offset when
sending PING/REPLCONF to the replica but do not write data to the AOF
file, this might cause the starting offset of the AOF file plus its size
to be inconsistent with the actual replication offset.
 2025-02-13 10:52:19 +08:00
Yuan Wang 87124a38b6 Fix wrongly updating fsynced_reploff_pending when appendfsync=everysecond (#13793) 
```
if (server.aof_fsync == AOF_FSYNC_EVERYSEC &&
    server.aof_last_incr_fsync_offset != server.aof_last_incr_size &&
    server.mstime - server.aof_last_fsync >= 1000 &&
    !(sync_in_progress = aofFsyncInProgress())) {
    goto try_fsync;
```
In https://github.com/redis/redis/pull/12622, when when
appendfsync=everysecond, if redis has written some data to AOF but not
`fsync`, and less than 1 second has passed since the last `fsync `,
redis will won't fsync AOF, but we will update `
fsynced_reploff_pending`, so it cause the `WAITAOF` to return
prematurely.

this bug is introduced in https://github.com/redis/redis/pull/12622,
from 7.4

The bug fix
https://github.com/redis/redis/pull/13793/commits/1bd6688bcae4add51dc829d96776359dfa39b100
is just as follows:
```diff
diff --git a/src/aof.c b/src/aof.c
index 8ccd8d8f8..521b30449 100644
--- a/src/aof.c
+++ b/src/aof.c
@@ -1096,8 +1096,11 @@ void flushAppendOnlyFile(int force) {
              * in which case master_repl_offset will increase but fsynced_reploff_pending won't be updated
              * (because there's no reason, from the AOF POV, to call fsync) and then WAITAOF may wait on
              * the higher offset (which contains data that was only propagated to replicas, and not to AOF) */
-            if (!sync_in_progress && server.aof_fsync != AOF_FSYNC_NO)
+            if (server.aof_last_incr_fsync_offset == server.aof_last_incr_size &&
+                !(sync_in_progress = aofFsyncInProgress()))
+            {
                 atomicSet(server.fsynced_reploff_pending, server.master_repl_offset);
+            }
             return;
```
Additionally, we slightly refactored fsync AOF to make it simpler, as
https://github.com/redis/redis/pull/13793/commits/584f008d1c39b90ae1d4862a313e04e3b426b136
 2025-02-13 10:48:29 +08:00
Yves LeBras 1583d60cd6 Missing --memkeys and --keystats for some options in redis-cli help text (#13794) 
Help text modified for -i, --count, --pattern.
 2025-02-13 08:42:38 +08:00
antirez 8b4bde19b4 README: some rephrasing. 2025-02-09 09:53:09 +01:00
antirez f69090d56b HNSW: Clarify psuedomagic early break condition. 2025-02-08 12:00:59 +01:00
Rowan Trollope f723c65f1b Updated gitignore 
Ignoring movies example files
 2025-02-08 09:14:58 +09:00
Rowan Trollope 9f376fb803 Merge pull request #2 from rowantrollope/main 
Comment fix
 2025-02-08 09:11:51 +09:00
Rowan Trollope b08629a426 Merge branch 'antirez:main' into main 2025-02-08 09:10:44 +09:00
YaacovHazanandYaacovHazan 1cd622bdca Add an API to load default configuration values (#13788) 
Currently we have RedisModule_LoadConfigs which the module is expected
to call during OnLoad which sets the configuration values from the
config queue or it sets the default value.

The problem is that the module might still want to support loading
values from the command line. If we want to give precedence to the
config file values then it means the module needs to set the values
before calling the Load Config function.

The problem is that then the API overrides the variables which were set
from the module command line with default values.

The new API should solve that in the following way.

1.Module registers its configuration parameters with redis 2.Module
calls RedisModule_LoadDefaultConfigs which loads the default values for
all the registered configuration parameters of the module 3.Module sets
the variables internally using the values it got from the command line
4.Module calls RedisModule_LoadConfigs which will set the values based
on the redis configuration file.

This allows for the default values to be set, for the module to override
them and for redis to override what the module wrote. In short it
determines a logical flow and ordering of where the values for the
parameters should come from.

The change done by all these previous commits:
d9134f8f9
7a40fd630
b9361ad5f
83c034855
f164012c1
98be450f1
0f6e3a827
de4e92ac3
49455c43a
c2694fb69
c88f9fe26
855ec46a6
f7353db7e
294492dbf
192799539
a8850a8d3
f35ad8231
a03477349
fd5c32588

Co-authored-by: YaacovHazan <yaacov.hazan@redislabs.com>
 2025-02-06 13:38:07 +02:00
kei-nananddebing.sun d9134f8f95 Update tests/modules/moduleconfigs.c 
missing else clause

Co-authored-by: debing.sun <debing.sun@redis.com>
 2025-02-06 13:16:33 +02:00
jonathan keinan 7a40fd630d * fix comments 2025-02-06 13:16:33 +02:00
jonathan keinan b9361ad5fe * only use new api if override-default was provided as an argument 2025-02-06 13:16:33 +02:00
kei-nanandOran Agra 83c0348553 Apply suggestions from code review 
* apply comment suggestions

Co-authored-by: Oran Agra <oran@redislabs.com>
 2025-02-06 13:16:33 +02:00
kei-nanandnafraf f164012c19 Update tests/unit/moduleapi/moduleconfigs.tcl 
Co-authored-by: nafraf <nafraf@users.noreply.github.com>
 2025-02-06 13:16:33 +02:00
jonathan keinan 98be450f1d * fix typo 2025-02-06 13:16:33 +02:00
jonathan keinan 0f6e3a8273 * improve function documentation 2025-02-06 13:16:33 +02:00
jonathan keinan de4e92ac39 * addressing code review comments 2025-02-06 13:16:33 +02:00
jonathan keinan 49455c43ae * change foo to goo so test will be correct and pass 2025-02-06 13:16:33 +02:00
jonathan keinan c2694fb696 * change config value in test to be different than overwritten value 2025-02-06 13:16:33 +02:00
jonathan keinan c88f9fe26f * update comment 2025-02-06 13:16:33 +02:00
jonathan keinan 855ec46a6a * rename MODULE_ONLOAD_CONFIG to MODULE_NON_DEFAULT_CONFIG 2025-02-06 13:16:33 +02:00
jonathan keinan f7353db7eb * fix test 
* cleanup strval2 on if an error during the OnLoad was encountered.
 2025-02-06 13:16:33 +02:00
jonathan keinan 294492dbf2 * fix tests 
* add some logging to test module
 2025-02-06 13:16:33 +02:00
jonathan keinan 192799539f * register LoadDefaultConfigs 2025-02-06 13:16:33 +02:00
jonathan keinan a8850a8d30 * remove unused variable 2025-02-06 13:16:33 +02:00
jonathan keinan f35ad82314 * add missing newline 2025-02-06 13:16:33 +02:00
jonathan keinan a034773497 * remove redundant module config variable 2025-02-06 13:16:33 +02:00
jonathan keinan fd5c325886 * initial commit 2025-02-06 13:16:33 +02:00
Alexander Gorbulya 17eb33e0c3 Fix typo in repl-ping-replica-period comment in redis.conf (#13782) 
The comment for the `repl-ping-replica-period` option in `redis.conf`
mistakenly refers to `repl_ping_replica_period` (with underscores).
This PR corrects it to use the proper format with dashes, as per the
actual configuration option.
 2025-02-06 10:14:56 +08:00
YaacovHazan 0aeb86d78d Revert "Improve GETRANGE command behavior (#12272)" 
Although the commit #6ceadfb58 improves GETRANGE command behavior,
we can't accept it as we should avoid breaking changes for non-critical bug fixes.

This reverts commit 6ceadfb580.
 2025-02-05 20:49:42 +02:00
YaacovHazan 8afb72a326 Revert "improve performance for scan command when matching data type (#12395)" 
Although the commit #7f0a7f0a6 improves the performance of the SCAN command,
we can't accept it as we should avoid breaking changes for non-critical bug fixes.

This reverts commit 7f0a7f0a69.
 2025-02-05 20:49:42 +02:00
antirez 6c1e55d07c VEMB RAW implemented. 2025-02-05 11:21:06 +01:00
antirez 8206c782b5 Function to get quantization name. 2025-02-05 11:07:35 +01:00
Raz MonsonegoandMeir Shpilraien 04589f90d7 Add internal connection and command mechanism (#13740) 
# PR: Add Mechanism for Internal Commands and Connections in Redis

This PR introduces a mechanism to handle **internal commands and
connections** in Redis. It includes enhancements for command
registration, internal authentication, and observability.

## Key Features

1. **Internal Command Flag**:
   - Introduced a new **module command registration flag**: `internal`.
- Commands marked with `internal` can only be executed by **internal
connections**, AOF loading flows, and master-replica connections.
- For any other connection, these commands will appear as non-existent.

2. **Support for internal authentication added to `AUTH`**:
- Used by depicting the special username `internal connection` with the
right internal password, i.e.,: `AUTH "internal connection"
<internal_secret>`.
- No user-defined ACL username can have this name, since spaces are not
aloud in the ACL parser.
   - Allows connections to authenticate as **internal connections**.
- Authenticated internal connections can execute internal commands
successfully.

4. **Module API for Internal Secret**:
- Added the `RedisModule_GetInternalSecret()` API, that exposes the
internal secret that should be used as the password for the new `AUTH
"internal connection" <password>` command.
- This API enables the modules to authenticate against other shards as
local connections.

## Notes on Behavior

- **ACL validation**:
- Commands dispatched by internal connections bypass ACL validation, to
give the caller full access regardless of the user with which it is
connected.

- **Command Visibility**:
- Internal commands **do not appear** in `COMMAND <subcommand>` and
`MONITOR` for non-internal connections.
- Internal commands **are logged** in the slow log, latency report and
commands' statistics to maintain observability.

- **`RM_Call()` Updates**:
  - **Non-internal connections**:
- Cannot execute internal commands when the command is sent with the `C`
flag (otherwise can).
- Internal connections bypass ACL validations (i.e., run as the
unrestricted user).

- **Internal commands' success**:
- Internal commands succeed upon being sent from either an internal
connection (i.e., authenticated via the new `AUTH "internal connection"
<internal_secret>` API), an AOF loading process, or from a master via
the replication link.
Any other connections that attempt to execute an internal command fail
with the `unknown command` error message raised.

- **`CLIENT LIST` flags**:
  - Added the `I` flag, to indicate that the connection is internal.

- **Lua Scripts**:
   - Prevented internal commands from being executed via Lua scripts.

---------

Co-authored-by: Meir Shpilraien <meir@redis.com>
 2025-02-05 11:48:08 +02:00
Ozan Tezcan 09f8a2f374 Start AOFRW before streaming repl buffer during fullsync (#13758) 
During fullsync, before loading RDB on the replica, we stop aof child to
prevent copy-on-write disaster.
Once rdb is loaded, aof is started again and it will trigger aof
rewrite. With https://github.com/redis/redis/pull/13732 , for rdbchannel
replication, this behavior was changed. Currently, we start aof after
replication buffer is streamed to db. This PR changes it back to start
aof just after rdb is loaded (before repl buffer is streamed)

Both approaches may have pros and cons. If we start aof before streaming
repl buffers, we may still face with copy-on-write issues as repl
buffers potentially include large amount of changes. If we wait until
replication buffer drained, it means we are delaying starting aof
persistence.

Additional changes are introduced as part of this PR:

- Interface change:
Added `mem_replica_full_sync_buffer` field to the `INFO MEMORY` command
reply. During full sync, it shows total memory consumed by accumulated
replication stream buffer on replica. Added same metric to `MEMORY
STATS` command reply as `replica.fullsync.buffer` field.
  
  
- Fixes: 
- Count repl stream buffer size of replica as part of 'memory overhead'
calculation for fields in "INFO MEMORY" and "MEMORY STATS" outputs.
Before this PR, repl buffer was not counted as part of memory overhead
calculation, causing misreports for fields like `used_memory_overhead`
and `used_memory_dataset` in "INFO STATS" and for `overhead.total` field
in "MEMORY STATS" command reply.
- Dismiss replication stream buffers memory of replica in the fork to
reduce COW impact during a fork.
- Fixed a few time sensitive flaky tests, deleted a noop statement,
fixed some comments and fail messages in rdbchannel tests.
 2025-02-04 21:40:18 +03:00
Rowan Trollope 324f861f0e Merge branch 'main' of https://github.com/rowantrollope/vector-sets 2025-02-03 13:27:17 -08:00
Rowan Trollope a50f3b517c typo fix 2025-02-03 13:23:15 -08:00
antirez e6f1667a3d Q8 option for VADD. See README update for info. 2025-02-03 13:05:40 +01:00
antirez ff20d534c6 VSIM: execute on main thread for Lua and MULTI/EXEC. 2025-02-03 11:46:01 +01:00
antirez 337fc3d6fd Don't use threaded VADD in replicas, lua, multi. 2025-02-03 11:28:48 +01:00
Meir Shpilraien (Spielrein) 870b6bd487 Added a shared secret over Redis cluster. (#13763) 
The PR introduces a new shared secret that is shared over all the nodes
on the Redis cluster. The main idea is to leverage the cluster bus to
share a secret between all the nodes such that later the nodes will be
able to authenticate using this secret and send internal commands to
each other (see #13740 for more information about internal commands).

The way the shared secret is chosen is the following:
1. Each node, when start, randomly generate its own internal secret.
2. Each node share its internal secret over the cluster ping messages.
3. If a node gets a ping message with secret smaller then his current
secret, it embrace it.
4. Eventually all nodes should embrace the minimal secret

The converges of the secret is as good as the topology converges.

To extend the ping messages to contain the secret, we leverage the
extension mechanism. Nodes that runs an older Redis version will just
ignore those extensions.

Specific tests were added to verify that eventually all nodes see the
secrets. In addition, a verification was added to the test infra to
verify the secret on `cluster_config_consistent` and to
`assert_cluster_state`.
 2025-02-03 09:54:37 +02:00
Raz Monsonego c688537d49 Add flag for ability of a module context to execute debug commands (#13774) 
This PR adds a flag to the `RM_GetContextFlags` module-API function that
depicts whether the context may execute debug commands, according to
redis's standards.
 2025-02-03 09:52:41 +02:00
Rowan Trollope 285134e43d Comment said: "XXX: check explicitly that ELE was passed, not just size" however this code is in the else block after strcasecmp already was checked for ! ELE, therefore we have already validated that this is an ELE element type. REMOVED comment for clarity. 2025-02-01 10:28:24 +01:00
Rowan Trollope daea83d2cf Moved hardcoded default EF of 100 (during threaded search) to define. 2025-02-01 10:25:58 +01:00
Mingyi Kang e3b9397dfe Bump actions/upload-artifact from 3 to 4 (#13780) 
Update `upload-artifact` from v3 to v4 to avoid the failure of `External
Server Tests` (I encountered this error when opening
[#13779](https://github.com/redis/redis/pull/13779)):

> Error: This request has been automatically failed because it uses a
deprecated version of `actions/upload-artifact: v3`. Learn more:
https://github.blog/changelog/2024-04-16-deprecation-notice-v3-of-the-artifact-actions/
 2025-02-01 14:09:00 +08:00
antirez 31f2b27a05 Digest method: fix calls for order independent hashing. 2025-01-31 20:17:38 +01:00
antirez b61e3021b6 Remove rounding error due to re-normalization on loading. 2025-01-31 19:36:11 +01:00
antirez a71feb6dd8 Replicate VADD with CAS. 2025-01-30 23:26:00 +01:00
antirez 6d1a15987b CLI tool added as example / utility. 2025-01-30 22:44:20 +01:00
Moti Cohen 2bfffe85e9 Fix memleak of SFLUSH experimental command (#13766) 
On flushallSyncBgDone, if client doesn't exists, take care release
sflush struct.
 2025-01-30 13:35:02 +02:00
antirez 182737f3cc Movies plots example. 2025-01-29 12:45:06 +01:00
antirez 7b5fbf7b3f glove-100 example added. 2025-01-29 12:37:59 +01:00
antirez 26e5871c67 Use -latomic only for ARM. 2025-01-28 22:17:01 +01:00
antirez 9771ca726a Link with -latomic in Linux. 2025-01-28 22:12:05 +01:00
antirez 42a8981b46 HNSW: do not denormalize binary vectors. 2025-01-28 10:12:37 +01:00
antirez 5c8097c2de Document VADD EF option. 2025-01-28 08:39:20 +01:00
antirez 5c23f59ee3 HSNW: Fix adding node with binary quants. 2025-01-27 20:03:14 +01:00
antirez 7cfd894f3a Fix binary quants loading. 2025-01-27 19:30:58 +01:00
antirez 31f097d418 VINFO: handle bin quantization. 2025-01-27 17:59:18 +01:00
antirez 33d653e24f First internal release. 2025-01-27 17:24:58 +01:00
Masonanddebing.sun f5e046a730 Update history for ban-list propagation (#13749) 
Update CLUSTER FORGET docs for changes in
https://github.com/redis/redis/pull/10869

Docs PR:
https://github.com/redis/docs/pull/1057

---------

Co-authored-by: debing.sun <debing.sun@redis.com>
 2025-01-27 21:05:37 +08:00
Vitah Linanddebing.sun 5dbcb3e4ab Add codecov for automated code coverage (#13393) 
This PR introduces Codecov to automate code coverage tracking for our
project's tests.
For more information about the Codecov platform, please refer to
https://docs.codecov.com/docs/quick-start

---------

Co-authored-by: debing.sun <debing.sun@redis.com>
 2025-01-27 21:04:11 +08:00
Dustin Rodrigues eb50eb20a5 Fix compile error with TLS_DEBUGGING (#13772) 2025-01-26 10:14:43 +08:00
YaacovHazan c2d3e28540 Update RQE version 7.99.3 (#13767) (#13769)
CI / test-ubuntu-latest (push) Has been cancelled
Details
CI / test-sanitizer-address (push) Has been cancelled
Details
CI / build-debian-old (push) Has been cancelled
Details
CI / build-macos-latest (push) Has been cancelled
Details
CI / build-32bit (push) Has been cancelled
Details
CI / build-libc-malloc (push) Has been cancelled
Details
CI / build-centos-jemalloc (push) Has been cancelled
Details
CI / build-old-chain-jemalloc (push) Has been cancelled
Details
External Server Tests / test-external-standalone (push) Has been cancelled
Details
External Server Tests / test-external-cluster (push) Has been cancelled
Details
External Server Tests / test-external-nodebug (push) Has been cancelled
Details
Spellcheck / Spellcheck (push) Has been cancelled
Details 
Update RQE version 7.99.3
 2025-01-24 09:20:23 +02:00
YaacovHazan 0c1a764074 Merge commit 'dcd0b3d02' into HEAD 2025-01-24 09:15:59 +02:00
debing.sunandoranagra f86575f210 Gradually reduce defrag CPU usage when defragmentation is ineffective (#13752) 
This PR addresses an issue where if a module does not provide a
defragmentation callback, we cannot defragment the fragmentation it
generates. However, the defragmentation process still considers a large
amount of fragmentation to be present, leading to more aggressive
defragmentation efforts that ultimately have no effect.

To mitigate this, the PR introduces a mechanism to gradually reduce the
CPU consumption for defragmentation when the defragmentation
effectiveness is poor. This occurs when the fragmentation rate drops
below 2% and the hit ratio is less than 1%, or when the fragmentation
rate increases by no more than 2%. The CPU consumption will be gradually
decreased until it reaches the minimum threshold defined by
`active-defrag-cycle-min`.

---------

Co-authored-by: oranagra <oran@redislabs.com>
 2025-01-24 11:35:32 +08:00
nafraf dcd0b3d020 Update RQE version 7.99.3 (#13767) 
Update RQE version 7.99.3
 2025-01-23 22:46:50 +02:00
YaacovHazan d88611d36f Redis 8.0 M03 (#13759)
CI / test-ubuntu-latest (push) Has been cancelled
Details
CI / test-sanitizer-address (push) Has been cancelled
Details
CI / build-debian-old (push) Has been cancelled
Details
CI / build-macos-latest (push) Has been cancelled
Details
CI / build-32bit (push) Has been cancelled
Details
CI / build-libc-malloc (push) Has been cancelled
Details
CI / build-centos-jemalloc (push) Has been cancelled
Details
CI / build-old-chain-jemalloc (push) Has been cancelled
Details
External Server Tests / test-external-standalone (push) Has been cancelled
Details
External Server Tests / test-external-cluster (push) Has been cancelled
Details
External Server Tests / test-external-nodebug (push) Has been cancelled
Details
Spellcheck / Spellcheck (push) Has been cancelled
Details
2025-01-20 19:45:10 +02:00
YaacovHazan 4c5a9076d7 Redis 8.0 M03 2025-01-20 16:00:49 +02:00
YaacovHazan 8aab7ca84c Merge remote-tracking branch 'upstream/unstable' into HEAD 2025-01-20 11:06:49 +02:00
YaacovHazanandYaacovHazan 781ccc1bee Update modules with latest version (#13755) 
Update redisbloom, redisjson and redistimeseries versions to 7.99.2

Co-authored-by: YaacovHazan <yaacov.hazan@redislabs.com>
 2025-01-20 10:08:19 +02:00
DvirDukhan ee96a5a6f5 Update RQE version (#13750) 
v7.99.2
 2025-01-16 08:40:19 +02:00
YaacovHazan 9c81f8bd61 Merge remote-tracking branch 'upstream/unstable' into HEAD 2025-01-14 14:01:19 +02:00
debing.sun 0f65806b5b Update info.tcl test to revert client output limits sooner (#13738) 
This PR is based on: https://github.com/valkey-io/valkey/pull/1462

We set the client output buffer limits to 10 bytes, and then execute
info stats which produces more than 10 bytes of output, which can cause
that command to throw an error.

I'm not sure why it wasn't consistently erroring before, might have been
some change related to the ubuntu upgrade though.

failed CI:
https://github.com/redis/redis/actions/runs/12738281720/job/35500381299

------
Co-authored-by: Madelyn Olson
[madelyneolson@gmail.com](mailto:madelyneolson@gmail.com)
 2025-01-14 17:30:18 +08:00
Yuan Wang 5b8b58e472 Fix incorrect parameter type reports (#13744) 
After upgrading of ubuntu 24.04, clang18 can check runtime error: call
to function XXX through pointer to incorrect function type, our daily CI
reports the errors by UndefinedBehaviorSanitizer (UBSan):

https://github.com/redis/redis/actions/runs/12738281720/job/35500380251#step:6:346

now we add generic version of some existing `free` functions to support
to call function through (void*) pointer, actually, they just are the
wrapper functions that will cast the data type and call the
corresponding functions.
 2025-01-14 15:51:05 +08:00
YaacovHazan 342ee426ad Fix LUA garbage collector (CVE-2024-46981) 
Reset GC state before closing the lua VM to prevent user data
to be wrongly freed while still might be used on destructor callbacks.
 2025-01-13 21:20:19 +02:00
YaacovHazan 4a95b3005a Fix Read/Write key pattern selector (CVE-2024-51741) 
The '%' rule must contain one or both of R/W
 2025-01-13 21:20:19 +02:00
+3
Ozan TezcanYuan Wangdebing.sunMoti CohennagleraAmit NaglerMadelyn OlsonBinbinViktor SöderqvistPing XieRan Shidlansikranshidxbasel
 		73a9b916c9 Rdb channel replication (#13732) 
This PR is based on:

https://github.com/redis/redis/pull/12109
https://github.com/valkey-io/valkey/pull/60

Closes: https://github.com/redis/redis/issues/11678

**Motivation**

During a full sync, when master is delivering RDB to the replica,
incoming write commands are kept in a replication buffer in order to be
sent to the replica once RDB delivery is completed. If RDB delivery
takes a long time, it might create memory pressure on master. Also, once
a replica connection accumulates replication data which is larger than
output buffer limits, master will kill replica connection. This may
cause a replication failure.

The main benefit of the rdb channel replication is streaming incoming
commands in parallel to the RDB delivery. This approach shifts
replication stream buffering to the replica and reduces load on master.
We do this by opening another connection for RDB delivery. The main
channel on replica will be receiving replication stream while rdb
channel is receiving the RDB.

This feature also helps to reduce master's main process CPU load. By
opening a dedicated connection for the RDB transfer, the bgsave process
has access to the new connection and it will stream RDB directly to the
replicas. Before this change, due to TLS connection restriction, the
bgsave process was writing RDB bytes to a pipe and the main process was
forwarding
it to the replica. This is no longer necessary, the main process can
avoid these expensive socket read/write syscalls. It also means RDB
delivery to replica will be faster as it avoids this step.

In summary, replication will be faster and master's performance during
full syncs will improve.


**Implementation steps**

1. When replica connects to the master, it sends 'rdb-channel-repl' as
part of capability exchange to let master to know replica supports rdb
channel.
2. When replica lacks sufficient data for PSYNC, master sends
+RDBCHANNELSYNC reply with replica's client id. As the next step, the
replica opens a new connection (rdb-channel) and configures it against
the master with the appropriate capabilities and requirements. It also
sends given client id back to master over rdbchannel, so that master can
associate these channels. (initial replica connection will be referred
as main-channel) Then, replica requests fullsync using the RDB channel.
3. Prior to forking, master attaches the replica's main channel to the
replication backlog to deliver replication stream starting at the
snapshot end offset.
4. The master main process sends replication stream via the main
channel, while the bgsave process sends the RDB directly to the replica
via the rdb-channel. Replica accumulates replication stream in a local
buffer, while the RDB is being loaded into the memory.
5. Once the replica completes loading the rdb, it drops the rdb channel
and streams the accumulated replication stream into the db. Sync is
completed.

**Some details**
- Currently, rdbchannel replication is supported only if
`repl-diskless-sync` is enabled on master. Otherwise, replication will
happen over a single connection as in before.
- On replica, there is a limit to replication stream buffering. Replica
uses a new config `replica-full-sync-buffer-limit` to limit number of
bytes to accumulate. If it is not set, replica inherits
`client-output-buffer-limit <replica>` hard limit config. If we reach
this limit, replica stops accumulating. This is not a failure scenario
though. Further accumulation will happen on master side. Depending on
the configured limits on master, master may kill the replica connection.

**API changes in INFO output:**

1. New replica state: `send_bulk_and_stream`. Indicates full sync is
still in progress for this replica. It is receiving replication stream
and rdb in parallel.
```
slave0:ip=127.0.0.1,port=5002,state=send_bulk_and_stream,offset=0,lag=0
```
Replica state changes in steps:
- First, replica sends psync and receives +RDBCHANNELSYNC
:`state=wait_bgsave`
- After replica connects with rdbchannel and delivery starts:
`state=send_bulk_and_stream`
 - After full sync: `state=online`

2. On replica side, replication stream buffering metrics:
- replica_full_sync_buffer_size: Currently accumulated replication
stream data in bytes.
- replica_full_sync_buffer_peak: Peak number of bytes that this instance
accumulated in the lifetime of the process.

```
replica_full_sync_buffer_size:20485             
replica_full_sync_buffer_peak:1048560
```

**API changes in CLIENT LIST**

In `client list` output, rdbchannel clients will have 'C' flag in
addition to 'S' replica flag:
```
id=11 addr=127.0.0.1:39108 laddr=127.0.0.1:5001 fd=14 name= age=5 idle=5 flags=SC db=0 sub=0 psub=0 ssub=0 multi=-1 watch=0 qbuf=0 qbuf-free=0 argv-mem=0 multi-mem=0 rbs=1024 rbp=0 obl=0 oll=0 omem=0 tot-mem=1920 events=r cmd=psync user=default redir=-1 resp=2 lib-name= lib-ver= io-thread=0
```

**Config changes:**
- `replica-full-sync-buffer-limit`: Controls how much replication data
replica can accumulate during rdbchannel replication. If it is not set,
a value of 0 means replica will inherit `client-output-buffer-limit
<replica>` hard limit config to limit accumulated data.
- `repl-rdb-channel` config is added as a hidden config. This is mostly
for testing as we need to support both rdbchannel replication and the
older single connection replication (to keep compatibility with older
versions and rdbchannel replication will not be enabled if
repl-diskless-sync is not enabled). it affects both the master (not to
respond to rdb channel requests), and the replica (not to declare
capability)

**Internal API changes:**
Changes that were introduced to Redis replication:
- New replication capability is added to replconf command: `capa
rdb-channel-repl`. Indicates replica is capable of rdb channel
replication. Replica sends it when it connects to master along with
other capabilities.
- If replica needs fullsync, master replies `+RDBCHANNELSYNC
<client-id>` to the replica's PSYNC request.
- When replica opens rdbchannel connection, as part of replconf command,
it sends `rdb-channel 1` to let master know this is rdb channel. Also,
it sends `main-ch-client-id <client-id>` as part of replconf command so
master can associate channels.
  
**Testing:**
As rdbchannel replication is enabled by default, we run whole test suite
with it. Though, as we need to support both rdbchannel and single
connection replication, we'll be running some tests twice with
`repl-rdb-channel yes/no` config.

**Replica state diagram**
```
* * Replica state machine *
 *
 * Main channel state
 * ┌───────────────────┐
 * │RECEIVE_PING_REPLY │
 * └────────┬──────────┘
 *          │ +PONG
 * ┌────────▼──────────┐
 * │SEND_HANDSHAKE     │                     RDB channel state
 * └────────┬──────────┘            ┌───────────────────────────────┐
 *          │+OK                ┌───► RDB_CH_SEND_HANDSHAKE         │
 * ┌────────▼──────────┐        │   └──────────────┬────────────────┘
 * │RECEIVE_AUTH_REPLY │        │    REPLCONF main-ch-client-id <clientid>
 * └────────┬──────────┘        │   ┌──────────────▼────────────────┐
 *          │+OK                │   │ RDB_CH_RECEIVE_AUTH_REPLY     │
 * ┌────────▼──────────┐        │   └──────────────┬────────────────┘
 * │RECEIVE_PORT_REPLY │        │                  │ +OK
 * └────────┬──────────┘        │   ┌──────────────▼────────────────┐
 *          │+OK                │   │  RDB_CH_RECEIVE_REPLCONF_REPLY│
 * ┌────────▼──────────┐        │   └──────────────┬────────────────┘
 * │RECEIVE_IP_REPLY   │        │                  │ +OK
 * └────────┬──────────┘        │   ┌──────────────▼────────────────┐
 *          │+OK                │   │ RDB_CH_RECEIVE_FULLRESYNC     │
 * ┌────────▼──────────┐        │   └──────────────┬────────────────┘
 * │RECEIVE_CAPA_REPLY │        │                  │+FULLRESYNC
 * └────────┬──────────┘        │                  │Rdb delivery
 *          │                   │   ┌──────────────▼────────────────┐
 * ┌────────▼──────────┐        │   │ RDB_CH_RDB_LOADING            │
 * │SEND_PSYNC         │        │   └──────────────┬────────────────┘
 * └─┬─────────────────┘        │                  │ Done loading
 *   │PSYNC (use cached-master) │                  │
 * ┌─▼─────────────────┐        │                  │
 * │RECEIVE_PSYNC_REPLY│        │    ┌────────────►│ Replica streams replication
 * └─┬─────────────────┘        │    │             │ buffer into memory
 *   │                          │    │             │
 *   │+RDBCHANNELSYNC client-id │    │             │
 *   ├──────┬───────────────────┘    │             │
 *   │      │ Main channel           │             │
 *   │      │ accumulates repl data  │             │
 *   │   ┌──▼────────────────┐       │     ┌───────▼───────────┐
 *   │   │ REPL_TRANSFER     ├───────┘     │    CONNECTED      │
 *   │   └───────────────────┘             └────▲───▲──────────┘
 *   │                                          │   │
 *   │                                          │   │
 *   │  +FULLRESYNC    ┌───────────────────┐    │   │
 *   ├────────────────► REPL_TRANSFER      ├────┘   │
 *   │                 └───────────────────┘        │
 *   │  +CONTINUE                                   │
 *   └──────────────────────────────────────────────┘
 */
 ```
 -----
 This PR also contains changes and ideas from: 
https://github.com/valkey-io/valkey/pull/837
https://github.com/valkey-io/valkey/pull/1173
https://github.com/valkey-io/valkey/pull/804
https://github.com/valkey-io/valkey/pull/945
https://github.com/valkey-io/valkey/pull/989
---------

Co-authored-by: Yuan Wang <wangyuancode@163.com>
Co-authored-by: debing.sun <debing.sun@redis.com>
Co-authored-by: Moti Cohen <moticless@gmail.com>
Co-authored-by: naglera <anagler123@gmail.com>
Co-authored-by: Amit Nagler <58042354+naglera@users.noreply.github.com>
Co-authored-by: Madelyn Olson <madelyneolson@gmail.com>
Co-authored-by: Binbin <binloveplay1314@qq.com>
Co-authored-by: Viktor Söderqvist <viktor.soderqvist@est.tech>
Co-authored-by: Ping Xie <pingxie@outlook.com>
Co-authored-by: Ran Shidlansik <ranshid@amazon.com>
Co-authored-by: ranshid <88133677+ranshid@users.noreply.github.com>
Co-authored-by: xbasel <103044017+xbasel@users.noreply.github.com>
 2025-01-13 15:09:52 +03:00
Filipe Oliveira (Redis)debing.sunYuan Wang
 		dc0ee51cb1 Refactor Client Write Preparation and Handling (#13721) 
This update refactors prepareClientToWrite by introducing
_prepareClientToWrite for inline checks within network.c file, and
separates replica and non-replica handling for pending replies and
writes (_clientHasPendingRepliesSlave/NonSlave and
_writeToClientSlave/NonSlave).

---------

Co-authored-by: debing.sun <debing.sun@redis.com>
Co-authored-by: Yuan Wang <wangyuancode@163.com>
 2025-01-13 15:40:36 +08:00
debing.sunandYuan Wang 21aee83abd Fix issue with argv not being shrunk (#13698) 
Found by @ShooterIT 

## Describe
If a client first creates a command with a very large number of
parameters, such as 10,000 parameters, the argv will be expanded to
accommodate 10,000. If the subsequent commands have fewer than 10,000
parameters, this argv will continue to be reused and will never be
shrunk.

## Solution
When determining whether it is necessary to rebuild argv, if the length
of the previous argv has already exceeded 1024, we will progressively
create argv regardless.

## Free argv in cron
Add a new condition to determine whether argv needs to be resized in
cron. When the number of parameters exceeds 128, we will resize it
regardless to avoid a single client consuming too much memory. It will
now occupy a maximum of (128 * 8 bytes).

---------

Co-authored-by: Yuan Wang <wangyuancode@163.com>
 2025-01-08 16:12:52 +08:00
debing.sunShooterITmeiravgri
 		08d714d0e5 Fix crash due to cron argv release (#13725) 
Introduced by https://github.com/redis/redis/issues/13521

If the client argv was released due to a timeout before sending the
complete command, `argv_len` will be reset to 0.
When argv is parsed again and resized, requesting a length of 0 may
result in argv being NULL, then leading to a crash.

And fix a bug that `argv_len` is not updated correctly in
`replaceClientCommandVector()`.

---------

Co-authored-by: ShooterIT <wangyuancode@163.com>
Co-authored-by: meiravgri <109056284+meiravgri@users.noreply.github.com>
 2025-01-08 09:57:23 +08:00
RQfreefly 4a12291765 Fix typos in multiple Redis source files (#13716) 2025-01-07 15:35:47 +08:00
Yuan Wang 8e9f5146dd Add reads/writes metrics for IO threads (#13703) 
The main job of the IO thread is read queries and write replies, so
reads/writes metrics can reflect the workload of IO threads, now we also
support this metrics `io_threaded_reads/writes_processed` in detail for
each IO thread.

Of course, to avoid break changes, `io_threaded_reads/writes_processed`
is still there. But before async io thread commit, we may sum the IO
done by the main thread if IO threads are active, but now we only sum
the IO done by IO threads.

Now threads section in `info` command output is as follows:
```
# Threads
io_thread_0:clients=0,reads=0,writes=0
io_thread_1:clients=54,reads=6546940,writes=6546919
io_thread_2:clients=54,reads=6513650,writes=6513625
io_thread_3:clients=54,reads=6396571,writes=6396525
io_thread_4:clients=53,reads=6511120,writes=6511097
io_thread_5:clients=53,reads=6539302,writes=6539280
io_thread_6:clients=53,reads=6502269,writes=6502248
```
 2025-01-06 15:59:02 +08:00
raffertyyuanddebing.sun 04f63d4af7 Fix index error of CRLF when replying with integer-encoded strings (#13711) 
close #13709

Fix the index error of CRLF character for integer-encoded strings
in addReplyBulk function

---------

Co-authored-by: debing.sun <debing.sun@redis.com>
 2024-12-31 21:41:10 +08:00
Yuan WangLipeng ZhuWangyang Guo
 		dc57ee03b1 Do security attack check only when command not found to reduce the critical path (#13702) 
This PR is based on the commits from PR
https://github.com/valkey-io/valkey/pull/1212.

When explored the cycles distribution for main thread with io-threads
enabled. We found this security attack check takes significant time in
main thread, **~3%** cycles were used to do the commands security check
in main thread.

This patch try to completely avoid doing it in the hot path. We can do
it only after we looked up the command and it wasn't found, just before
we call commandCheckExistence.

---------

Co-authored-by: Lipeng Zhu <lipeng.zhu@intel.com>
Co-authored-by: Wangyang Guo <wangyang.guo@intel.com>
 2024-12-26 12:51:44 +08:00
Thalia Archibald 8144019a13 Check length before reading in stringmatchlen (#13690) 
Fixes four cases where `stringmatchlen` could overrun the pattern if it is not
terminated with NUL.

These commits are cherry-picked from my
[fork](https://github.com/thaliaarchi/antirez-stringmatch) which
extracts `stringmatch` as a library and compares it to other projects by
antirez which uses the same matcher.
 2024-12-26 12:37:23 +08:00
Yuan Wang 7665bdc91a Offload lookupCommand into IO threads when threaded IO is enabled (#13696) 
From flame graph, we could see `lookupCommand` in main thread costs much
CPU, so we can let IO threads to perform `lookupCommand`.

To avoid race condition among multiple IO threads, made the following
changes:
- Pause all IO threads when register or unregister commands
- Force a full rehashing of the command table dict when resizing
 2024-12-25 16:03:22 +08:00
Yuan Wangdebing.sunoranagra
 		64a40b20d9 Async IO Threads (#13695) 
## Introduction
Redis introduced IO Thread in 6.0, allowing IO threads to handle client
request reading, command parsing and reply writing, thereby improving
performance. The current IO thread implementation has a few drawbacks.
- The main thread is blocked during IO thread read/write operations and
must wait for all IO threads to complete their current tasks before it
can continue execution. In other words, the entire process is
synchronous. This prevents the efficient utilization of multi-core CPUs
for parallel processing.

- When the number of clients and requests increases moderately, it
causes all IO threads to reach full CPU utilization due to the busy wait
mechanism used by the IO threads. This makes it challenging for us to
determine which part of Redis has reached its bottleneck.

- When IO threads are enabled with TLS and io-threads-do-reads, a
disconnection of a connection with pending data may result in it being
assigned to multiple IO threads simultaneously. This can cause race
conditions and trigger assertion failures. Related issue:
redis#12540

Therefore, we designed an asynchronous IO threads solution. The IO
threads adopt an event-driven model, with the main thread dedicated to
command processing, meanwhile, the IO threads handle client read and
write operations in parallel.

## Implementation
### Overall
As before, we did not change the fact that all client commands must be
executed on the main thread, because Redis was originally designed to be
single-threaded, and processing commands in a multi-threaded manner
would inevitably introduce numerous race and synchronization issues. But
now each IO thread has independent event loop, therefore, IO threads can
use a multiplexing approach to handle client read and write operations,
eliminating the CPU overhead caused by busy-waiting.

the execution process can be briefly described as follows:
the main thread assigns clients to IO threads after accepting
connections, IO threads will notify the main thread when clients
finish reading and parsing queries, then the main thread processes
queries from IO threads and generates replies, IO threads handle
writing reply to clients after receiving clients list from main thread,
and then continue to handle client read and write events.

### Each IO thread has independent event loop
We now assign each IO thread its own event loop. This approach
eliminates the need for the main thread to perform the costly
`epoll_wait` operation for handling connections (except for specific
ones). Instead, the main thread processes requests from the IO threads
and hands them back once completed, fully offloading read and write
events to the IO threads.

Additionally, all TLS operations, including handling pending data, have
been moved entirely to the IO threads. This resolves the issue where
io-threads-do-reads could not be used with TLS.

### Event-notified client queue
To facilitate communication between the IO threads and the main thread,
we designed an event-notified client queue. Each IO thread and the main
thread have two such queues to store clients waiting to be processed.
These queues are also integrated with the event loop to enable handling.
We use pthread_mutex to ensure the safety of queue operations, as well
as data visibility and ordering, and race conditions are minimized, as
each IO thread and the main thread operate on independent queues,
avoiding thread suspension due to lock contention. And we implemented an
event notifier based on `eventfd` or `pipe` to support event-driven
handling.

### Thread safety
Since the main thread and IO threads can execute in parallel, we must
handle data race issues carefully.

**client->flags**
The primary tasks of IO threads are reading and writing, i.e.
`readQueryFromClient` and `writeToClient`. However, IO threads and the
main thread may concurrently modify or access `client->flags`, leading
to potential race conditions. To address this, we introduced an io-flags
variable to record operations performed by IO threads, thereby avoiding
race conditions on `client->flags`.

**Pause IO thread**
In the main thread, we may want to operate data of IO threads, maybe
uninstall event handler, access or operate query/output buffer or resize
event loop, we need a clean and safe context to do that. We pause IO
thread in `IOThreadBeforeSleep`, do some jobs and then resume it. To
avoid thread suspended, we use busy waiting to confirm the target
status. Besides we use atomic variable to make sure memory visibility
and ordering. We introduce these functions to pause/resume IO Threads as
below.
```
pauseIOThread, resumeIOThread
pauseAllIOThreads, resumeAllIOThreads
pauseIOThreadsRange, resumeIOThreadsRange
```
Testing has shown that `pauseIOThread` is highly efficient, allowing the
main thread to execute nearly 200,000 operations per second during
stress tests. Similarly, `pauseAllIOThreads` with 8 IO threads can
handle up to nearly 56,000 operations per second. But operations
performed between pausing and resuming IO threads must be quick;
otherwise, they could cause the IO threads to reach full CPU
utilization.

**freeClient and freeClientAsync**
The main thread may need to terminate a client currently running on an
IO thread, for example, due to ACL rule changes, reaching the output
buffer limit, or evicting a client. In such cases, we need to pause the
IO thread to safely operate on the client.

**maxclients and maxmemory-clients updating**
When adjusting `maxclients`, we need to resize the event loop for all IO
threads. Similarly, when modifying `maxmemory-clients`, we need to
traverse all clients to calculate their memory usage. To ensure safe
operations, we pause all IO threads during these adjustments.

**Client info reading**
The main thread may need to read a client’s fields to generate a
descriptive string, such as for the `CLIENT LIST` command or logging
purposes. In such cases, we need to pause the IO thread handling that
client. If information for all clients needs to be displayed, all IO
threads must be paused.

**Tracking redirect**
Redis supports the tracking feature and can even send invalidation
messages to a connection with a specified ID. But the target client may
be running on IO thread, directly manipulating the client’s output
buffer is not thread-safe, and the IO thread may not be aware that the
client requires a response. In such cases, we pause the IO thread
handling the client, modify the output buffer, and install a write event
handler to ensure proper handling.

**clientsCron**
In the `clientsCron` function, the main thread needs to traverse all
clients to perform operations such as timeout checks, verifying whether
they have reached the soft output buffer limit, resizing the
output/query buffer, or updating memory usage. To safely operate on a
client, the IO thread handling that client must be paused.
If we were to pause the IO thread for each client individually, the
efficiency would be very low. Conversely, pausing all IO threads
simultaneously would be costly, especially when there are many IO
threads, as clientsCron is invoked relatively frequently.
To address this, we adopted a batched approach for pausing IO threads.
At most, 8 IO threads are paused at a time. The operations mentioned
above are only performed on clients running in the paused IO threads,
significantly reducing overhead while maintaining safety.

### Observability
In the current design, the main thread always assigns clients to the IO
thread with the least clients. To clearly observe the number of clients
handled by each IO thread, we added the new section in INFO output. The
`INFO THREADS` section can show the client count for each IO thread.
```
# Threads
io_thread_0:clients=0
io_thread_1:clients=2
io_thread_2:clients=2
```

Additionally, in the `CLIENT LIST` output, we also added a field to
indicate the thread to which each client is assigned.

`id=244 addr=127.0.0.1:41870 laddr=127.0.0.1:6379 ... resp=2 lib-name=
lib-ver= io-thread=1`

## Trade-off
### Special Clients
For certain special types of clients, keeping them running on IO threads
would result in severe race issues that are difficult to resolve.
Therefore, we chose not to offload these clients to the IO threads.

For replica, monitor, subscribe, and tracking clients, main thread may
directly write them a reply when conditions are met. Race issues are
difficult to resolve, so we have them processed in the main thread. This
includes the Lua debug clients as well, since we may operate connection
directly.

For blocking client, after the IO thread reads and parses a command and
hands it over to the main thread, if the client is identified as a
blocking type, it will be remained in the main thread. Once the blocking
operation completes and the reply is generated, the client is
transferred back to the IO thread to send the reply and wait for event
triggers.

### Clients Eviction
To support client eviction, it is necessary to update each client’s
memory usage promptly during operations such as read, write, or command
execution. However, when a client operates on an IO thread, it is not
feasible to update the memory usage immediately due to the risk of data
races. As a result, memory usage can only be updated either in the main
thread while processing commands or in the `ClientsCron` periodically.
The downside of this approach is that updates might experience a delay
of up to one second, which could impact the precision of memory
management for eviction.

To avoid incorrectly evicting clients. We adopted a best-effort
compensation solution, when we decide to eviction a client, we update
its memory usage again before evicting, if the memory used by the client
does not decrease or memory usage bucket is not changed, then we will
evict it, otherwise, not evict it.

However, we have not completely solved this problem. Due to the delay in
memory usage updates, it may lead us to make incorrect decisions about
the need to evict clients.

### Defragment
In the majority of cases we do NOT use the data from argv directly in
the db.
1. key names
We store a copy that we allocate in the main thread, see `sdsdup()` in
`dbAdd()`.
2. hash key and value
We store key as hfield and store value as sds, see `hfieldNew()` and
`sdsdup()` in `hashTypeSet()`.
3. other datatypes
   They don't even use SDS, so there is no reference issues.

But in some cases client the data from argv may be retain by the main
thread.
As a result, during fragmentation cleanup, we need to move allocations
from the IO thread’s arena to the main thread’s arena. We always
allocate new memory in the main thread’s arena, but the memory released
by IO threads may not yet have been reclaimed. This ultimately causes
the fragmentation rate to be higher compared to creating and allocating
entirely within a single thread.
The following cases below will lead to memory allocated by the IO thread
being kept by the main thread.
1. string related command: `append`, `getset`, `mset` and `set`.
If `tryObjectEncoding()` does not change argv, we will keep it directly
in the main thread, see the code in `tryObjectEncoding()`(specifically
`trimStringObjectIfNeeded()`)
2. block related command.
    the key names will be kept in `c->db->blocking_keys`.
3. watch command
    the key names will be kept in `c->db->watched_keys`.
4. [s]subscribe command
    channel name will be kept in `serverPubSubChannels`.
5. script load command
    script will be kept in `server.lua_scripts`.
7. some module API: `RM_RetainString`, `RM_HoldString`

Those issues will be handled in other PRs.

## Testing
### Functional Testing
The commit with enabling IO Threads has passed all TCL tests, but we did
some changes:
**Client query buffer**: In the original code, when using a reusable
query buffer, ownership of the query buffer would be released after the
command was processed. However, with IO threads enabled, the client
transitions from an IO thread to the main thread for processing. This
causes the ownership release to occur earlier than the command
execution. As a result, when IO threads are enabled, the client's
information will never indicate that a shared query buffer is in use.
Therefore, we skip the corresponding query buffer tests in this case.
**Defragment**: Add a new defragmentation test to verify the effect of
io threads on defragmentation.
**Command delay**: For deferred clients in TCL tests, due to clients
being assigned to different threads for execution, delays may occur. To
address this, we introduced conditional waiting: the process proceeds to
the next step only when the `client list` contains the corresponding
commands.

### Sanitizer Testing
The commit passed all TCL tests and reported no errors when compiled
with the `fsanitizer=thread` and `fsanitizer=address` options enabled.
But we made the following modifications: we suppressed the sanitizer
warnings for clients with watched keys when updating `client->flags`, we
think IO threads read `client->flags`, but never modify it or read the
`CLIENT_DIRTY_CAS` bit, main thread just only modifies this bit, so
there is no actual data race.

## Others
### IO thread number
In the new multi-threaded design, the main thread is primarily focused
on command processing to improve performance. Typically, the main thread
does not handle regular client I/O operations but is responsible for
clients such as replication and tracking clients. To avoid breaking
changes, we still consider the main thread as the first IO thread.

When the io-threads configuration is set to a low value (e.g., 2),
performance does not show a significant improvement compared to a
single-threaded setup for simple commands (such as SET or GET), as the
main thread does not consume much CPU for these simple operations. This
results in underutilized multi-core capacity. However, for more complex
commands, having a low number of IO threads may still be beneficial.
Therefore, it’s important to adjust the `io-threads` based on your own
performance tests.

Additionally, you can clearly monitor the CPU utilization of the main
thread and IO threads using `top -H -p $redis_pid`. This allows you to
easily identify where the bottleneck is. If the IO thread is the
bottleneck, increasing the `io-threads` will improve performance. If the
main thread is the bottleneck, the overall performance can only be
scaled by increasing the number of shards or replicas.

---------

Co-authored-by: debing.sun <debing.sun@redis.com>
Co-authored-by: oranagra <oran@redislabs.com>
 2024-12-23 14:16:40 +08:00
Moti Cohen 08c2b276fb Optimize dict no_value also for even addresses (#13683) 
This pull request enhances the no_value flag option in the dict implementation,
which is used to store keys without associated values. Previously, when a key
had an odd memory address and was the only item in a table entry, it could be
directly stored as a pointer without requiring an intermediate dictEntry. With
this update, the optimization has been extended to also handle keys with even
memory addresses in the same manner.
 2024-12-22 14:10:07 +02:00
debing.sunandMadelyn Olson 1f09a55eba Avoid importing memory aligned malloc (#13693) 
This PR is based on the commits from PR
https://github.com/valkey-io/valkey/pull/1442.

We deprecate the usage of classic malloc and free, but under certain
circumstances they might get imported from intrinsics. The original
thought is we should just override malloc and free to use zmalloc and
zfree, but I think we should continue to deprecate it to avoid
accidental imports of allocations.

---------

Co-authored-by: Madelyn Olson <matolson@amazon.com>
 2024-12-20 09:39:14 +08:00
Nugineanddebing.sun 684077682e Fix bug in PFMERGE command (#13672) 
The bug was introduced in #13558 . 

When merging dense hll structures, `hllDenseCompress` writes to wrong
location and the result will be zero. The unit tests didn't cover this
case.

This PR
+ fixes the bug
+ adds `PFDEBUG SIMD (ON|OFF)` for unit tests
+ adds a new TCL test to cover the cases

Synchronized from https://github.com/valkey-io/valkey/pull/1293

---------

Signed-off-by: Xuyang Wang <xuyangwang@link.cuhk.edu.cn>
Co-authored-by: debing.sun <debing.sun@redis.com>
 2024-12-18 14:41:04 +08:00
Filipe Oliveira (Redis)andMoti Cohen f8942f93a6 Avoid unnecessary hfield Creation/Deletion on updates in hashTypeSet. HSET updates improvement of ~10% (#13655) 
This PR eliminates unnecessary creation and destruction of hfield
objects, ensuring only required updates or insertions are performed.
This reduces overhead and improves performance by streamlining field
management in hash dictionaries, particularly in scenarios involving
frequent updates, like the benchmarks in:
-
[memtier_benchmark-100Kkeys-load-hash-50-fields-with-100B-values](https://github.com/redis/redis-benchmarks-specification/blob/main/redis_benchmarks_specification/test-suites/memtier_benchmark-100Kkeys-load-hash-50-fields-with-100B-values.yml)
-
[memtier_benchmark-10Mkeys-load-hash-5-fields-with-100B-values-pipeline-10](https://github.com/redis/redis-benchmarks-specification/blob/main/redis_benchmarks_specification/test-suites/memtier_benchmark-10Mkeys-load-hash-5-fields-with-100B-values-pipeline-10.yml)


To test it we can simply focus on the hfield related tests

```
tclsh tests/test_helper.tcl --single unit/type/hash-field-expire
tclsh tests/test_helper.tcl --single unit/type/hash
tclsh tests/test_helper.tcl --dump-logs --single unit/other
```

Extra check on full CI:
- [x] https://github.com/filipecosta90/redis/actions/runs/12225788759

## microbenchmark results 
16.7% improvement (drop in time) in dictAddNonExistingRaw vs dictAddRaw
```
make REDIS_CFLAGS="-g -fno-omit-frame-pointer -O3 -DREDIS_TEST" -j
$ ./src/redis-server test dict --accurate
(...)
Inserting via dictAddRaw() non existing: 5000000 items in 2592 ms
(...)
Inserting via dictAddNonExistingRaw() non existing: 5000000 items in 2160 ms
```

8% improvement (drop in time) in find (non existing) and adding via
`dictGetHash()+dictFindWithHash()+dictAddNonExistingRaw()` vs
`dictFind()+dictAddRaw()`
```
make REDIS_CFLAGS="-g -fno-omit-frame-pointer -O3 -DREDIS_TEST" -j
$ ./src/redis-server test dict --accurate
(...)
Find() and inserting via dictFind()+dictAddRaw() non existing: 5000000 items in 2983 ms
Find() and inserting via dictGetHash()+dictFindWithHash()+dictAddNonExistingRaw() non existing: 5000000 items in 2740 ms

```

## benchmark results 


To benchmark:

```
pip3 install redis-benchmarks-specification==0.1.250
taskset -c 0 ./src/redis-server --save '' --protected-mode no --daemonize yes
redis-benchmarks-spec-client-runner --tests-regexp ".*load-hash.*" --flushall_on_every_test_start --flushall_on_every_test_end  --cpuset_start_pos 2 --override-memtier-test-time 60
```

Improvements on achievable throughput in:

test | ops/sec unstable (59953d2df6) |
ops/sec this PR (24af7190fdc0ad3c6d5957c17853490990c35dcc) | % change
-- | -- | -- | --
memtier_benchmark-1key-load-hash-1K-fields-with-5B-values | 4097 | 5032
| 22.8%
memtier_benchmark-100Kkeys-load-hash-50-fields-with-100B-values | 37658
| 44688 | 18.7%
memtier_benchmark-100Kkeys-load-hash-50-fields-with-1000B-values | 14736
| 17350 | 17.7%

memtier_benchmark-1Mkeys-load-hash-5-fields-with-1000B-values-pipeline-10
| 131848 | 143485 | 8.8%
memtier_benchmark-1Mkeys-load-hash-hmset-5-fields-with-1000B-values |
82071 | 85681 | 4.4%
memtier_benchmark-1Mkeys-load-hash-5-fields-with-1000B-values | 82882 |
86336 | 4.2%

memtier_benchmark-10Mkeys-load-hash-5-fields-with-100B-values-pipeline-10
| 262502 | 273376 | 4.1%
memtier_benchmark-10Kkeys-load-hash-50-fields-with-10000B-values | 2821
| 2936 | 4.1%

---------

Co-authored-by: Moti Cohen <moticless@gmail.com>
 2024-12-12 19:41:08 +02:00
Moti Cohenanddebing.sun c51c96656b modules API: Add test for ACL check of empty prefix (#13678) 
- Add empty string test for the new API
`RedisModule_ACLCheckKeyPrefixPermissions`.
- Fix order of checks: `(pattern[patternLen - 1] != '*' || patternLen ==
0)`

---------

Co-authored-by: debing.sun <debing.sun@redis.com>
 2024-12-10 09:16:30 +02:00
Moti Cohen 0dd057222b Modules API: new HashFieldMinExpire(). Add flag REDISMODULE_HASH_EXPIRE_TIME to HashGet(). (#13676) 
This PR introduces API to query Expiration time of hash fields.

# New `RedisModule_HashFieldMinExpire()`
For a given hash, retrieves the minimum expiration time across all
fields. If no fields have expiration or if the key is not a hash then
return `REDISMODULE_NO_EXPIRE` (-1).
```
mstime_t RM_HashFieldMinExpire(RedisModuleKey *hash);
```

# Extension to `RedisModule_HashGet()`
Adds a new flag, `REDISMODULE_HASH_EXPIRE_TIME`, to retrieve the
expiration time of a specific hash field. If the field does not exist or
has no expiration, returns `REDISMODULE_NO_EXPIRE`. It is fully
backward-compatible (RM_HashGet retains its original behavior unless the
new flag is used).

Example:
```
mstime_t expiry1, expiry2;
RedisModule_HashGet(mykey, REDISMODULE_HASH_EXPIRE_TIME, "field1", &expiry1, NULL);
RedisModule_HashGet(mykey, REDISMODULE_HASH_EXPIRE_TIME, "field1", &expiry1, "field2", &expiry2, NULL);
```
 2024-12-05 11:14:52 +02:00
Filipe Oliveira (Redis)anddebing.sun 59953d2df6 Improve listpack Handling and Decoding Efficiency: 16.3% improvement on LRANGE command (#13652) 
This PR focused on refining listpack encoding/decoding functions and
optimizing reply handling mechanisms related to it.
Each commit has the measured improvement up until the last accumulated
improvement of 16.3% on
[memtier_benchmark-1key-list-100-elements-lrange-all-elements-pipeline-10](https://github.com/redis/redis-benchmarks-specification/blob/main/redis_benchmarks_specification/test-suites/memtier_benchmark-1key-list-100-elements-lrange-all-elements-pipeline-10.yml)
benchmark.

Connection mode | CE Baseline (Nov 14th)
701f06657d | CE PR #13652 | CE PR vs CE
Unstable
-- | -- | -- | --
TCP | 155696 | 178874 | 14.9%
Unix socket | 169743 | 197428 | 16.3%

To test it we can simply focus on the scan.tcl

```
tclsh tests/test_helper.tcl --single unit/replybufsize
```

### Commit details:
- 2e58d048fd9d1365c56292ebd69f2ed141bfeda1 +
29c6c86c6b96376f807656f203b61d0e7fcb65a4 : Eliminate an indirect memory
access on lpCurrentEncodedSizeBytes and completely avoid passing p*
fully to lpCurrentEncodedSizeBytes + Add lpNextWithBytes helper function
and optimize addListListpackRangeReply

**- Improvement of 3.1%, from 168969.88 ops/sec to 174239.75 ops/sec**
- af52aacff86e2809f3451e2561043d532b9ee223 Refactor lpDecodeBacklen for
loop-based decoding, improving readability and branch efficiency.
**- NO CHANGE. REVERTED in 09f6680ba0d0b5acabca537c651008f0c8ec061b**
    
- 048bfe4edaaf5671f7f06b06d1492f81e6af3e59 +
03e8ff3af70891cbd3d53ca865558976459bb38e : reducing condition checks in
_addReplyToBuffer, inlining it, and avoid entering it when there are
there already entries in the reply list
and check if the reply length exceeds available buffer space before
calling _addReplyToBuffer
**- accumulated Improvement of 12.4%, from 168969.88 ops/sec to
189726.81 ops/sec**

- 9a63d4d6a9fa946505e31ecce4c7796845fc022c: always update the buf_peak
on _addReplyToBufferOrList
**- accumulated Improvement of 14.2%, from 168969.88 ops/sec to 193887
ops/sec**

- b544ade67628a1feaf714d6cfd114930e0c7670b: Introduce
lpEncodeBacklenBytes to avoid any indirect memory access on previous
usage of lpEncodeBacklen(NULL,...). inline lpEncodeBacklenBytes().
**- accumulated Improvement of 16.3%, from 168969.88 ops/sec to
197427.70 ops/sec**

---------

Co-authored-by: debing.sun <debing.sun@redis.com>
 2024-12-04 18:04:37 +08:00
Filipe Oliveira (Redis)andYuan Wang ddafac4c6c Optimize dictFind with prefetching and branch prediction hints (#13646) 
This pull request optimizes the `dictFind` function by adding software
prefetching and branch prediction hints to improve cache efficiency and
reduce memory latency.
It introduces 2 prefetch hints (read/write) that became no-ops in case
the compiler does not support it.

Baseline profiling with Intel VTune indicated that dictFind was
significantly back-end bound, with memory latency accounting for 59.6%
of clockticks, with frequent stalls from DRAM-bound operations due to
cache misses during hash table lookups.

![microarch](https://github.com/user-attachments/assets/9e3cf334-ae6b-4767-b568-713a4ac24e87)

---------

Co-authored-by: Yuan Wang <wangyuancode@163.com>
 2024-12-04 17:16:14 +08:00
Ozan Tezcan 2af69a931a Do not call _dictClear()'s callback for the first 65k items (#13674) 
In https://github.com/redis/redis/pull/13495, we introduced a feature to
reply -LOADING while flushing a large db on a replica.
While `_dictClear()` is in progress, it calls a callback for every 65k
items and we yield back to eventloop to reply -LOADING.

This change has made some tests unstable as those tests don't expect new
-LOADING reply.
One observation, inside `_dictClear()`, we call the callback even if db
has a few keys. Most tests run with small amount of keys. So, each
replication and cluster test has to handle potential -LOADING reply now.

This PR changes this behavior, skips calling callback when `i=0` to
stabilize replication tests.
Callback will be called after the first 65k items. Most tests use less
than 65k keys and they won't get -LOADING reply.
 2024-12-03 09:26:19 +03:00
Moti Cohen 06b144aa09 Modules API: Add RedisModule_ACLCheckKeyPrefixPermissions (#13666) 
This PR introduces a new API function to the Redis Module API:
```
int RedisModule_ACLCheckKeyPrefixPermissions(RedisModuleUser *user, RedisModuleString *prefix, int flags);
```
Purpose:
The function checks if a given user has access permissions to any key
that match a specific prefix. This validation is based on the user’s ACL
permissions and the specified flags.

Note, this prefix-based approach API may fail to detect prefixes that
are individually uncovered but collectively covered by the patterns. For
example the prefix `ID-*` is not fully included in pattern `ID-[0]*` and
is not fully included in pattern `ID-[^0]*` but it is fully included in
the set of patterns `{ID-[0]*, ID-[^0]*}`
 2024-11-28 18:33:58 +02:00
Vitah Linanddebing.sun db33b67d37 Deprecate ubuntu lunar and macos-12 in workflows (#13669) 
1. Ubuntu Lunar reached End of Life on January 25, 2024, so upgrade the
ubuntu version to plucky in action `test-ubuntu-jemalloc-fortify` to
pass the daily CI
2. The macOS-12 environment is deprecated so upgrade macos-12 to
macos-13 in daily CI

---------

Co-authored-by: debing.sun <debing.sun@redis.com>
 2024-11-28 21:59:43 +08:00
Filipe Oliveira (Redis) a106198878 Optimize addReplyBulk on sds/int encoded strings: 2.2% to 4% reduction of CPU Time on GET high pipeline use-cases (#13644) 
### Summary

By profing 1KiB 100% GET's use-case, on high pipeline use-cases, we can
see that addReplyBulk and it's inner calls takes 8.30% of the CPU
cycles. This PR reduces from 2.2% to 4% the CPU time spent on
addReplyBulk. Specifically for GET use-cases, we saw an improvement from
2.7% to 9.1% on the achievable ops/sec

### Improvement

By reducing the duplicate work we can improve by around 2.7% on sds
encoded strings, and around 9% on int encoded strings. This PR does the
following:
- Avoid duplicate sdslen on addReplyBulk() for sds enconded objects
- Avoid duplicate sdigits10() call on int incoded objects on
addReplyBulk()
- avoid final "\r\n" addReplyProto() in the OBJ_ENCODING_INT type on
addReplyBulk

Altogether this improvements results in the following improvement on the
achievable ops/sec :

Encoding | unstable (commit 9906daf5c9) |
this PR | % improvement
-- | -- | -- | --
1KiB Values string SDS encoded | 1478081.88 | 1517635.38 | 2.7%
Values string "1" OBJ_ENCODING_INT | 1521139.36 | 1658876.59 | 9.1%

### CPU Time: Total of addReplyBulk

Encoding | unstable (commit 9906daf5c9) |
this PR | reduction of CPU Time: Total
-- | -- | -- | --
1KiB Values string SDS encoded | 8.30% | 6.10% | 2.2%
Values string "1" OBJ_ENCODING_INT | 7.20% | 3.20% | 4.0%

### To reproduce

Run redis with unix socket enabled
```
taskset -c 0 /root/redis/src/redis-server  --unixsocket /tmp/1.socket --save '' --enable-debug-command local
```

#### 1KiB Values string SDS encoded

Load data
```
taskset -c 2-5 memtier_benchmark  --ratio 1:0 -n allkeys --key-pattern P:P --key-maximum 1000000  --hide-histogram  --pipeline 10 -S /tmp/1.socket

```

Benchmark
```
taskset -c 2-6 memtier_benchmark --ratio 0:1 -c 1 -t 5 --test-time 60 --hide-histogram -d 1000 --pipeline 500  -S /tmp/1.socket --key-maximum 1000000 --json-out-file results.json
```

#### Values string "1" OBJ_ENCODING_INT 

Load data
```
$ taskset -c 2-5 memtier_benchmark  --command "SET __key__ 1" -n allkeys --command-key-pattern P --key-maximum 1000000  --hide-histogram -c 1 -t 1  --pipeline 100 -S /tmp/1.socket

# confirm we have the expected reply and format 
$ redis-cli get memtier-1
"1"

$ redis-cli debug object memtier-1
Value at:0x7f14cec57570 refcount:2147483647 encoding:int serializedlength:2 lru:2861503 lru_seconds_idle:8

```

Benchmark
```
taskset -c 2-6 memtier_benchmark --ratio 0:1 -c 1 -t 5 --test-time 60 --hide-histogram -d 1000 --pipeline 500  -S /tmp/1.socket --key-maximum 1000000 --json-out-file results.json
```
 2024-11-26 16:11:01 +08:00
Ali 05b99c8f4c Fix typo in redis.conf (#12634) 
unnecessarily and repetitive "OR"
 2024-11-22 20:29:17 +08:00
Ozan Tezcananddebing.sun 9ebf80a28c Fix memory leak of jemalloc tcache on function flush command (#13661) 
Starting from https://github.com/redis/redis/pull/13133, we allocate a
jemalloc thread cache and use it for lua vm.
On certain cases, like `script flush` or `function flush` command, we
free the existing thread cache and create a new one.

Though, for `function flush`, we were not actually destroying the
existing thread cache itself. Each call creates a new thread cache on
jemalloc and we leak the previous thread cache instances. Jemalloc
allows maximum 4096 thread cache instances. If we reach this limit,
Redis prints "Failed creating the lua jemalloc tcache" log and abort.

There are other cases that can cause this memory leak, including
replication scenarios when emptyData() is called.

The implication is that it looks like redis `used_memory` is low, but
`allocator_allocated` and RSS remain high.

Co-authored-by: debing.sun <debing.sun@redis.com>
 2024-11-21 14:12:58 +03:00
Moti Cohen 155634502d modules API: Support register unprefixed config parameters (#13656) 
PR #10285 introduced support for modules to register four types of
configurations — Bool, Numeric, String, and Enum. Accessible through the
Redis config file and the CONFIG command.

With this PR, it will be possible to register configuration parameters
without automatically prefixing the parameter names. This provides
greater flexibility in configuration naming, enabling, for instance,
both `bf-initial-size` or `initial-size` to be defined in the module
without automatically prefixing with `<MODULE-NAME>.`. In addition it
will also be possible to create a single additional alias via the same
API. This brings us another step closer to integrate modules into redis
core.

**Example:** Register a configuration parameter `bf-initial-size` with
an alias `initial-size` without the automatic module name prefix, set
with new `REDISMODULE_CONFIG_UNPREFIXED` flag:
```
RedisModule_RegisterBoolConfig(ctx, "bf-initial-size|initial-size", default_val, optflags | REDISMODULE_CONFIG_UNPREFIXED, getfn, setfn, applyfn, privdata);
```
# API changes
Related functions that now support unprefixed configuration flag
(`REDISMODULE_CONFIG_UNPREFIXED`) along with optional alias:
```
RedisModule_RegisterBoolConfig
RedisModule_RegisterEnumConfig
RedisModule_RegisterNumericConfig
RedisModule_RegisterStringConfig
```

# Implementation Details:
`config.c`: On load server configuration, at function
`loadServerConfigFromString()`, it collects all unknown configurations
into `module_configs_queue` dictionary. These may include valid module
configurations or invalid ones. They will be validated later by
`loadModuleConfigs()` against the configurations declared by the loaded
module(s).
`Module.c:` The `ModuleConfig` structure has been modified to store now:
(1) Full configuration name (2) Alias (3) Unprefixed flag status -
ensuring that configurations retain their original registration format
when triggered in notifications.

Added error printout:
This change introduces an error printout for unresolved configurations,
detailing each unresolved parameter detected during startup. The last
line in the output existed prior to this change and has been retained to
systems relies on it:
```
595011:M 18 Nov 2024 08:26:23.616 # Unresolved Configuration(s) Detected:
595011:M 18 Nov 2024 08:26:23.616 #  >>> 'bf-initiel-size 8'
595011:M 18 Nov 2024 08:26:23.616 #  >>> 'search-sizex 32'
595011:M 18 Nov 2024 08:26:23.616 # Module Configuration detected without loadmodule directive or no ApplyConfig call: aborting
```

# Backward Compatibility:
Existing modules will function without modification, as the new
functionality only applies if REDISMODULE_CONFIG_UNPREFIXED is
explicitly set.

# Module vs. Core API Conflict Behavior
The new API allows to modules loading duplication of same configuration
name or same configuration alias, just like redis core configuration
allows (i.e. the users sets two configs with a different value, but
these two configs are actually the same one). Unlike redis core, given a
name and its alias, it doesn't allow have both configuration on load. To
implement it, it is required to modify DS `module_configs_queue` to
reflect the order of their loading and later on, during
`loadModuleConfigs()`, resolve pairs of names and aliases and which one
is the last one to apply. "Relaxing" this limitation can be deferred to
a future update if necessary, but for now, we error in this case.
 2024-11-21 09:55:02 +02:00
Oran Agra 79fd255828 Add Lua VM memory to memory overhead, now that it's part of zmalloc (#13660) 
To complement the work done in #13133.
it added the script VMs memory to be counted as part of zmalloc, but
that means they
should be also counted as part of the non-value overhead.

this commit contains some refactoring to make variable names and
function names less confusing.
it also adds a new field named `script.VMs` into the `MEMORY STATS`
command.

additionally, clear scripts and stats between tests in external mode
(which is related to how this issue was discovered)
 2024-11-21 08:22:17 +02:00
nafrafanddebing.sun 5b84dc9678 Fix module loadex command crash due to invalid config (#13653) 
Fix to https://github.com/redis/redis/issues/13650

providing an invalid config to a module with datatype crashes when redis
tries to unload the module due to the invalid config

---------

Co-authored-by: debing.sun <debing.sun@redis.com>
 2024-11-21 14:14:14 +08:00
debing.sunandOzan Tezcan 701f06657d Reuse c->argv after command execution to reduce memory allocation overhead (#13521) 
inspred by https://github.com/redis/redis/pull/12730

Before this PR, we allocate new memory to store the user command
arguments, however, if the size of the current `c->argv` is larger than
the current command, we can reuse the previously allocated argv to avoid
allocating new memory for the current command.
And we will free `c->argv` in client cron when the client is idle for 2
seconds.

---------

Co-authored-by: Ozan Tezcan <ozantezcan@gmail.com>
 2024-11-14 20:35:31 +08:00
Moti CohenViktor SöderqvistMadelyn OlsonJosiah Carlson
 		cf83803880 CRC64 perf improvements (#13638) 
Improve the performance of crc64 for large batches by processing large
number
of bytes in parallel and combining the results.

---------
Co-authored-by: Viktor Söderqvist <viktor.soderqvist@est.tech>
Co-authored-by: Madelyn Olson <madelyneolson@gmail.com>
Co-authored-by: Josiah Carlson <josiah.carlson@gmail.com>
 2024-11-12 09:21:22 +02:00
Ozan Tezcan 54038811c0 Print command tokens on a crash when hide-user-data-from-log is enabled (#13639) 
If `hide-user-data-from-log` config is enabled, we don't print client
argv in the crashlog to avoid leaking user info.
Though, debugging a crash becomes harder as we don't see the command
arguments causing the crash.

With this PR, we'll be printing command tokens to the log. As we have
command tokens defined in json schema for each command, using this data,
we can find tokens in the client argv.

e.g. 
`SET key value GET EX 10` ---> we'll print `SET * * GET EX *` in the
log.

Modules should introduce their command structure via
`RM_SetCommandInfo()`.
Then, on a crash we'll able to know module command tokens.
 2024-11-11 09:34:18 +03:00
NugineYuan Wangdebing.sun
 		fdeb97629e Optimize PFCOUNT, PFMERGE command by SIMD acceleration (#13558) 
This PR optimizes the performance of HyperLogLog commands (PFCOUNT,
PFMERGE) by adding AVX2 fast paths.

Two AVX2 functions are added for conversion between raw representation
and dense representation. They are 15 ~ 30 times faster than scalar
implementaion. Note that sparse representation is not accelerated.

AVX2 fast paths are enabled when the CPU supports AVX2 (checked at
runtime) and the hyperloglog configuration is default (HLL_REGISTERS ==
16384 && HLL_BITS == 6).

When merging 3 dense hll structures, the benchmark shows a 12x speedup
compared to the scalar version.

```
pfcount key1 key2 key3
pfmerge keyall key1 key2 key3
```

```
======================================================================================================
Type             Ops/sec    Avg. Latency     p50 Latency     p99 Latency   p99.9 Latency       KB/sec 
------------------------------------------------------------------------------------------------------
PFCOUNT-scalar    5570.09        35.89060        32.51100        65.27900        69.11900       299.17
PFCOUNT-avx2     72604.92         2.82072         2.73500         5.50300         7.13500      3899.68
------------------------------------------------------------------------------------------------------
PFMERGE-scalar    7879.13        25.52156        24.19100        46.33500        48.38300       492.45
PFMERGE-avx2    126448.64         1.58120         1.53500         3.08700         4.89500      7903.04
------------------------------------------------------------------------------------------------------

scalar: redis:unstable   9906daf5c9
avx2:   Nugine:hll-simd  02e09f85ac07eace50ebdddd0fd70822f7b9152d 

CPU:    13th Gen Intel® Core™ i9-13900H × 20
Memory: 32.0 GiB
OS:     Ubuntu 22.04.5 LTS
```

Experiment repo: https://github.com/Nugine/redis-hyperloglog
Benchmark script:
https://github.com/Nugine/redis-hyperloglog/blob/main/scripts/memtier.sh
Algorithm:
https://github.com/Nugine/redis-hyperloglog/blob/main/cpp/bench.cpp

resolves #13551

---------

Co-authored-by: Yuan Wang <wangyuancode@163.com>
Co-authored-by: debing.sun <debing.sun@redis.com>
 2024-11-08 15:19:38 +08:00
David Dougherty 9906daf5c9 Update old links for modules-api-ref.md (#13479) 
This PR replaces old .../topics/... links with current links,
specifically for the modules-api-ref.md file and the new automation that
Paolo Lazzari is working on. A few of the topics links have redirects,
but some don't. Best to use updated links.
 2024-11-04 18:18:22 +02:00
guybe7 ded8d993b7 Modules: defrag CB should take robj, not sds (#13627) 
Added a log of the keyname in the test modules to reproduce the problem
(tests crash without the fix)
 2024-10-30 17:32:51 +08:00
Moti Cohen 6437d07b03 Fix memory leak on rdbload error (#13626) 
On RDB load error, if an invalid `expireAt` value is read,
`dupSearchDict` is not released.
 2024-10-30 10:03:31 +02:00
debing.sun 4b29be3f36 Avoid redundant lpGet to boost quicklistCompare (#11533) 
`lpCompare()` in `quicklistCompare()` will call `lpGet()` again, which
would be a waste.
The change will result in a boost for all commands that use
`quicklistCompre()`, including `linsert`, `lpos` and `lrem`.
 2024-10-30 08:45:25 +08:00
Moti Cohen 2ec78d262d Add KEYSIZES section to INFO (#13592) 
This PR adds a new section to the `INFO` command output, called
`keysizes`. This section provides detailed statistics on the
distribution of key sizes for each data type (strings, lists, sets,
hashes and zsets) within the dataset. The distribution is tracked using
a base-2 logarithmic histogram.

# Motivation
Currently, Redis lacks a built-in feature to track key sizes and item
sizes per data type at a granular level. Understanding the distribution
of key sizes is critical for monitoring memory usage and optimizing
performance, particularly in large datasets. This enhancement will allow
users to inspect the size distribution of keys directly from the `INFO`
command, assisting with performance analysis and capacity planning.

# Changes
New Section in `INFO` Command: A new section called `keysizes` has been
added to the `INFO` command output. This section reports a per-database,
per-type histogram of key sizes. It provides insights into how many keys
fall into specific size ranges (represented in powers of 2).

**Example output:**
```
127.0.0.1:6379> INFO keysizes
# Keysizes
db0_distrib_strings_sizes:1=19,2=655,512=100899,1K=31,2K=29,4K=23,8K=16,16K=3,32K=2
db0_distrib_lists_items:1=5784492,32=3558,64=1047,128=676,256=533,512=218,4K=1,8K=42
db0_distrib_sets_items:1=735564=50612,8=21462,64=1365,128=974,2K=292,4K=154,8K=89,
db0_distrib_hashes_items:2=1,4=544,32=141169,64=207329,128=4349,256=136226,1K=1
```
## Future Use Cases:
The key size distribution is collected per slot as well, laying the
groundwork for future enhancements related to Redis Cluster.
 2024-10-29 13:07:26 +02:00
YaacovHazan 0a24bfec13 Redis 8.0 M02 (#13624)
CI / test-ubuntu-latest (push) Has been cancelled
Details
CI / test-sanitizer-address (push) Has been cancelled
Details
CI / build-debian-old (push) Has been cancelled
Details
CI / build-macos-latest (push) Has been cancelled
Details
CI / build-32bit (push) Has been cancelled
Details
CI / build-libc-malloc (push) Has been cancelled
Details
CI / build-centos-jemalloc (push) Has been cancelled
Details
CI / build-old-chain-jemalloc (push) Has been cancelled
Details
External Server Tests / test-external-standalone (push) Has been cancelled
Details
External Server Tests / test-external-cluster (push) Has been cancelled
Details
External Server Tests / test-external-nodebug (push) Has been cancelled
Details
Spellcheck / Spellcheck (push) Has been cancelled
Details
2024-10-28 19:09:18 +02:00
Shockingly Good 611c950293 Fix crash in RM_GetCurrentUserName() when the user isn't accessible (#13619) 
The crash happens whenever the user isn't accessible, for example, it
isn't set for the context (when it is temporary) or in some other cases
like `notifyKeyspaceEvent`. To properly check for the ACL compliance, we
need to get the user name and the user to invoke other APIs. However, it
is not possible if it crashes, and it is impossible to work that around
in the code since we don't know (and **shouldn't know**!) when it is
available and when it is not.
 2024-10-28 21:26:29 +08:00
YaacovHazan 14c48cc5ed Redis 8.0 M02 2024-10-28 14:01:22 +02:00
YaacovHazan b08fd9f989 Merge unstable into 8.0 (#13622) 2024-10-27 17:23:40 +02:00
YaacovHazan dd20aa5160 Merge remote-tracking branch 'upstream/unstable' into HEAD 2024-10-27 13:50:58 +02:00
opt-mandYuan Wang 0a8e546957 Fix get # option in sort command (#13608) 
From 7.4, Redis allows `GET` options in cluster mode when the pattern maps to
the same slot as the key, but GET # pattern that represents key itself is missed.
This commit resolves it, bug report #13607.

---------

Co-authored-by: Yuan Wang <yuan.wang@redis.com>
 2024-10-22 09:55:00 +08:00
debing.sunOzan TezcanShooterIT
 		4f8cdc2a1e Fix compilation on compilers that do not support target attribute (#13609) 
introduced by https://github.com/redis/redis/pull/13359
failure CI on ARM64:
https://github.com/redis/redis-extra-ci/actions/runs/11377893230/job/31652773710

---------

Co-authored-by: Ozan Tezcan <ozantezcan@gmail.com>
Co-authored-by: ShooterIT <wangyuancode@163.com>
 2024-10-18 09:11:23 +08:00
hanhui365debing.sunoranagraNugine
 		3788a055fe Optimize bitcount command by using popcnt (#13359) 
Nowadays popcnt instruction is almost supported by X86 machine, which is
used to calculate "Hamming weight", it can bring much performance boost
in redis bitcount comand.

---------

Signed-off-by: hanhui365(hanhui@hygon.cn)
Co-authored-by: debing.sun <debing.sun@redis.com>
Co-authored-by: oranagra <oran@redislabs.com>
Co-authored-by: Nugine <nugine@foxmail.com>
 2024-10-17 09:13:19 +08:00
Yuan Wang b71a610f5c Clean up .rediscli_history_test temporary file (#13601) 
After running test in local, there will be a file named
`.rediscli_history_test`, and it is not in `.gitignore` file, so this is
considered to have changed the code base. It is a little annoying, this
commit just clean up the temporary file.

We should delete `.rediscli_history_test` in the end since the second
server tests also write somethings into it, to make it corresponding, i
put `set ::env(REDISCLI_HISTFILE) ".rediscli_history_test"` at the
beginning.

Maybe we also can add this file into `.gitignore`
 2024-10-17 09:12:11 +08:00
YaacovHazanandYaacovHazan efcfffc528 Update modules with latest version (#13606) 
Update redisbloom, redisjson and redistimeseries versions to 7.99.1

Co-authored-by: YaacovHazan <yaacov.hazan@redislabs.com>
 2024-10-15 19:58:42 +03:00
paoloredis 99d09c824c Only run redis_docs_sync.yaml on latest release (#13603) 
We only want to trigger the workflow on the documentation repository for
the latest release
 2024-10-15 16:02:11 +03:00
YaacovHazanandYaacovHazan 6c5e263d7b Temporarily hide the new SFLUSH command by marking it as experimental (#13600) 
- Add a new 'EXPERIMENTAL' command flag, which causes the command
generator to skip over it and make the command to be unavailable for
execution
- Skip experimental tests by default
- Move the SFLUSH tests from the old framework to the new one

---------

Co-authored-by: YaacovHazan <yaacov.hazan@redislabs.com>
 2024-10-15 11:02:51 +03:00
debing.sun 3fc7ef8f81 Fix race in stream-cgroups test (#13593) 
failed CI:
https://github.com/redis/redis/actions/runs/11171608362/job/31056659165
https://github.com/redis/redis/actions/runs/11226025974/job/31205787575
 2024-10-12 09:23:19 +08:00
guybe7 a38c29b6c8 Cleanups related to expiry/eviction (#13591) 
1. `dbRandomKey`: excessive call to `dbFindExpires` (will always return
1 if `allvolatile` + anyway called inside `expireIfNeeded`
2. Add `deleteKeyAndPropagate` that is used by both expiry/eviction
3. Change the order of calls in `expireIfNeeded` to save redundant calls
to `keyIsExpired`
4. `expireIfNeeded`: move `OBJ_STATIC_REFCOUNT` to
`deleteKeyAndPropagate`
5. `performEvictions` now uses `deleteEvictedKeyAndPropagate`
6. active-expire: moved `postExecutionUnitOperations` inside
`activeExpireCycleTryExpire`
7. `activeExpireCycleTryExpire`: less indentation + expire a key if `now
== t`
8. rename `lazy_expire_disabled` to `allow_access_expired`
 2024-10-10 16:58:52 +08:00
Oran Agra 472d8a0df5 Prevent pattern matching abuse (CVE-2024-31228) 2024-10-08 20:55:44 +03:00
Oran Agra 8ec5da785b Fix ACL SETUSER Read/Write key pattern selector (CVE-2024-31227) 
The '%' rule must contain one or both of R/W
 2024-10-08 20:55:44 +03:00
Oran Agra 3a2669e8ae Fix lua bit.tohex (CVE-2024-31449) 
INT_MIN value must be explicitly checked, and cannot be negated.
 2024-10-08 20:55:44 +03:00
alonre24 f39e51178e Update target module in search (#13578) 
Update search target path and version from M02
 2024-10-08 13:58:28 +03:00
chx9 5f7d7ce8b0 fix typo in test_helper.tcl (#13576) 
fix typo in test_helper.tcl: even driven => event driven
 2024-10-08 14:15:48 +08:00
Moti Cohen d092d64d7a Add new SFLUSH command to cluster for slot-based FLUSH (#13564) 
This PR introduces a new `SFLUSH` command to cluster mode that allows
partial flushing of nodes based on specified slot ranges. Current
implementation is designed to flush all slots of a shard, but future
extensions could allow for more granular flushing.

**Command Usage:**
`SFLUSH <start-slot> <end-slot> [<start-slot> <end-slot>]* [SYNC|ASYNC]`

This command removes all data from the specified slots, either
synchronously or asynchronously depending on the optional SYNC/ASYNC
argument.

**Functionality:**
Current imp of `SFLUSH` command verifies that the provided slot ranges
are valid and cover all of the node's slots before proceeding. If slots
are partially or incorrectly specified, the command will fail and return
an error, ensuring that all slots of a node must be fully covered for
the flush to proceed.

The function supports both synchronous (default) and asynchronous
flushing. In addition, if possible, SFLUSH SYNC will be run as blocking
ASYNC as an optimization.
 2024-09-29 09:13:21 +03:00
YaacovHazan 30eb6c3210 Merge unstable into 8.0 (#13573) 2024-09-26 14:36:04 +03:00
YaacovHazan 80458d0490 Merge remote-tracking branch 'upstream/unstable' into HEAD 2024-09-26 10:50:12 +03:00
Ozan TezcanLipeng ZhuWangyang Guo
 		99c40ab53d Use hashtable as the default type of temp set object during sunion/sdiff (#13567) 
This PR is based on https://github.com/valkey-io/valkey/pull/996


Currently, for operations like SUNION or SDIFF, temporary set object can
be intset or listpack. Search operation is costly for these encodings.
This patch tries to set the temporary set object as hash table by
default. It also tries to determine correct encoding for the temporary
set object to reduce the unnecessary conversation.

This change is supposed to give performance boost for tests like:
-
[memtier_benchmark-2keys-set-10-100-elements-sdiff](https://github.com/redis/redis-benchmarks-specification/blob/main/redis_benchmarks_specification/test-suites/memtier_benchmark-2keys-set-10-100-elements-sdiff.yml)
66.2% IMPROVEMENT
-
[memtier_benchmark-2keys-set-10-100-elements-sunion](https://github.com/redis/redis-benchmarks-specification/blob/main/redis_benchmarks_specification/test-suites/memtier_benchmark-2keys-set-10-100-elements-sunion.yml)
126.5% IMPROVEMENT

-------
Co-authored-by: Lipeng Zhu <lipeng.zhu@intel.com>
Co-authored-by: Wangyang Guo <wangyang.guo@intel.com>

Co-authored-by: Lipeng Zhu <lipeng.zhu@intel.com>
Co-authored-by: Wangyang Guo <wangyang.guo@intel.com>
 2024-09-25 12:41:17 +03:00
Moti CohenandRayacoo zisong.cw@alibaba-inc.com 26ef28467a Optimize ZUNION[STORE] by avoiding redundant temporary dict usage (#13566) 
This PR is based on valkey-io/valkey#829

Previously, ZUNION and ZUNIONSTORE commands used a temporary accumulator dict
and at the end copied it as-is to dstzset->dict. This PR removes accumulator and directly
stores into dstzset->dict, eliminating the extra copy.

Co-authored-by: Rayacoo zisong.cw@alibaba-inc.com
 2024-09-25 11:55:00 +03:00
Moti Cohen 5f28bd96db Fix race in HFE tests (#13563) 
Test 1 - give more time for expiration
Test 2 - Evaluate expiration time boundaries [+1,+2] before setting expiration [+1]
Test 3 - Avoid race on test HFEs propagated to replica
 2024-09-23 10:30:29 +03:00
debing.sun 438cfed70a Replace wrongly free with zfree in redis-cli (#13560) 
#13258 Incorrect use of free instead of zfree
 2024-09-23 09:40:47 +08:00
Moti Cohen 3a3cacfefa Extend modules API to read also expired keys and subkeys (#13526) 
The PR extends `RedisModule_OpenKey`'s flags to include
`REDISMODULE_OPEN_KEY_ACCESS_EXPIRED`, which allows to access expired
keys.

It also allows to access expired subkeys. Currently relevant only for
hash fields
and has its impact on `RM_HashGet` and `RM_Scan`.
 2024-09-19 20:47:00 +03:00
debing.sun 617909e943 Align the offset in ASCII logo (#13557) 
Since `\\` is only one character, we need to add an extra space to the right.
 2024-09-18 14:42:32 +08:00
adamiBs e9cbfccec6 Support musl Rust Installation in Modules Makefile (#13549) 
This PR introduces the installation of the `musl`-based version of Rust,
in order to support alpine-based runtime environments (Rust is used by
[RedisJSON](https://github.com/RedisJSON/RedisJSON)).
 2024-09-15 20:23:05 +03:00
Filipe Oliveira (Redis)anddebing.sun 7b69183a8d Replace usage of _addReplyLongLongWithPrefix with specific bulk/mbulk functions to reduce condition checks in hotpath. (#13520) 
Instead of adding runtime logic to decide which prefix/shared object to
use when doing the reply we can simply use an inline method to avoid runtime
overhead of condition checks, and also keep the code change small.
Preliminary data show improvements on commands that heavily rely on
bulk/mbulk replies (example of LRANGE).

---------

Co-authored-by: debing.sun <debing.sun@redis.com>
 2024-09-15 21:40:09 +08:00
Filipe Oliveira (Personal)andfcostaoliveira af7fca797a Using fast_float library for faster parsing of 64 decimal strings. (#11884) 
Fixes #8825 

We're using the fast_float library[1] in our (compiled-in)
floating-point fast_float_strtod implementation for faster and more
portable parsing of 64 decimal strings.

The single file fast_float.h is an amalgamation of the entire library,
which can be (re)generated with the amalgamate.py script (from the
fast_float repository) via the command:

```
python3 ./script/amalgamate.py --license=MIT > $REDIS_SRC/deps/fast_float/fast_float.h
```

[1]: https://github.com/fastfloat/fast_float

The used commit from fast_float library was the one from
https://github.com/fastfloat/fast_float/releases/tag/v3.10.1

---------

Co-authored-by: fcostaoliveira <filipe@redis.com>
 2024-09-15 21:37:29 +08:00
Filipe Oliveira (Redis)anddebing.sun 9146ac050b Optimize HSCAN/ZSCAN command in case of listpack encoding: avoid the usage of intermediate list (#13531) 
Similar to #13530 , applied to HSCAN and ZSCAN in case of listpack
encoding.

**Preliminary benchmark results showcase an improvement of 108% on the
achievable ops/sec for ZSCAN and 65% for HSCAN**.

---------

Co-authored-by: debing.sun <debing.sun@redis.com>
 2024-09-13 20:36:19 +08:00
debing.sun ef3a5f58a8 Fix missing initialization of EbucketsIterator->isRax (#13545) 
in https://github.com/redis/redis/pull/13519, when `eb` is empty,
`isRax` is not correctly initialized to 0, which can lead to `ebStop()`
potentially entering the wrong rax branch.
 2024-09-13 17:12:27 +08:00
Filipe Oliveira (Redis) f2f85ba354 Optimize SSCAN command in case of listpack or intset encoding: avoid the usage of intermediate list. From 2N to N iterations (#13530) 
On SSCAN, in case of listpack and intset encoding we actually reply the
entire set, and always reply with the cursor 0.

For those cases, we don't need to accumulate the replies in a list and
can completely avoid the overhead of list appending and then iterating
over the list again -- meaning we do N iterations instead of 2N
iterations over the SET and save intermediate memory as well.

Preliminary benchmarks, `SSCAN set:100 0`, showcased an improvement of
60% as visible bellow on a SET with 100 string elements (listpack
encoded).
 2024-09-12 22:36:54 +08:00
Moti Cohen c115c5230e Add iterator capability to ebuckets DS (#13519) 
Add basic iterator API for ebuckets of start, next, nextBucket and stop.
 2024-09-12 15:02:32 +03:00
Moti Cohen 65a87cb773 Correct spelling error at t_hash.c comment (#13540) 
spell check error : ./src/t_hash.c:1141: RESOTRE ==> RESTORE
 2024-09-12 12:50:44 +03:00
Moti Cohen 9a89e32a95 HFE - Fix key ref by the hash on RENAME/MOVE/SWAPDB/RESTORE (#13539) 
If the hash previously had HFEs (hash-fields with expiration) but later no longer
does, the key ref in the hash might become outdated after a MOVE, COPY,
RENAME or RESTORE operation. These commands maintain the key ref only
if HFEs are present. That is, we can only be sure that key ref is valid as long as the
hash has HFEs.
 2024-09-12 12:40:12 +03:00
YaacovHazan 5fe3e74a38 Redis 8.0 M01 (#13538)
CI / test-ubuntu-latest (push) Has been cancelled
Details
CI / test-sanitizer-address (push) Has been cancelled
Details
CI / build-debian-old (push) Has been cancelled
Details
CI / build-macos-latest (push) Has been cancelled
Details
CI / build-32bit (push) Has been cancelled
Details
CI / build-libc-malloc (push) Has been cancelled
Details
CI / build-centos-jemalloc (push) Has been cancelled
Details
CI / build-old-chain-jemalloc (push) Has been cancelled
Details
External Server Tests / test-external-standalone (push) Has been cancelled
Details
External Server Tests / test-external-cluster (push) Has been cancelled
Details
External Server Tests / test-external-nodebug (push) Has been cancelled
Details
Spellcheck / Spellcheck (push) Has been cancelled
Details 
### New Features in binary distributions

- 7 new data structures: JSON, Time series, Bloom filter, Cuckoo filter,
Count-min sketch, Top-k, t-digest
- Redis scalable query engine (including vector search)

### Potentially breaking changes

- #12272 `GETRANGE` returns an empty bulk when the negative end index is
out of range
- #12395 Optimize `SCAN` command when matching data type

### Bug fixes

- #13510 Fix `RM_RdbLoad` to enable AOF after RDB loading is completed
- #13489 `ACL CAT` - return module commands
- #13476 Fix a race condition in the `cache_memory` of `functionsLibCtx`
- #13473 Fix incorrect lag due to trimming stream via `XTRIM` command
- #13338 Fix incorrect lag field in `XINFO` when tombstone is after the
`last_id` of the consume group
- #13470 On `HDEL` of last field - update the global hash field
expiration data structure
- #13465 Cluster: Pass extensions to node if extension processing is
handled by it
- #13443 Cluster: Ensure validity of myself when loading cluster config
- #13422 Cluster: Fix `CLUSTER SHARDS` command returns empty array

### Modules API

- #13509 New API calls: `RM_DefragAllocRaw`, `RM_DefragFreeRaw`, and
`RM_RegisterDefragCallbacks` - defrag API to allocate and free raw
memory

### Performance and resource utilization improvements

- #13503 Avoid overhead of comparison function pointer calls in listpack
`lpFind`
- #13505 Optimize `STRING` datatype write commands
- #13499 Optimize `SMEMBERS` command
- #13494 Optimize `GEO*` commands reply
- #13490 Optimize `HELLO` command
- #13488 Optimize client query buffer
- #12395 Optimize `SCAN` command when matching data type
- #13529 Optimize `LREM`, `LPOS`, `LINSERT`, and `LINDEX` commands
- #13516 Optimize `LRANGE` and other commands that perform several
writes to client buffers per call
- #13431 Avoid `used_memory` contention when updating from multiple
threads

### Other general improvements

- #13495 Reply `-LOADING` on replica while flushing the db

### CLI tools

- #13411 redis-cli: Fix wrong `dbnum` showed after the client
reconnected

### Notes

- No backward compatibility for replication or persistence.
- Additional distributions, upgrade paths, features, and improvements
will be introduced in upcoming pre-releases.
- With the GA release of 8.0 we will deprecate Redis Stack.
 2024-09-12 11:52:28 +03:00
Oran AgraandUdi Ron 610eb26c11 RED-129256, Fix TOUCH command from script in no-touch mode (#13512) 
When a client in no-touch mode issues a TOUCH command on a key, the
key’s access time should be updated, but in scripts, and module's
RM_Call, it isn’t updated.
Command proc should be matched to the executing client, not the current
client.

Co-authored-by: Udi Ron <udi@speedb.io>
 2024-09-12 11:33:26 +03:00
YaacovHazan 4955375ec7 Redis 8.0 M01 2024-09-12 10:23:36 +03:00
Steveanddebing.sun d265a61438 Avoid cluster.nodes load corruption due to shard-id generation (#13468) 
PR #13428 doesn't fully resolve an issue where corruption errors can
still occur on loading of cluster.nodes file - seen on upgrade where
there were no shard_ids (from old Redis), 7.2.5 loading generated new
random ones, and persisted them to the file before gossip/handshake
could propagate the correct ones (or some other nodes unreachable).
This results in a primary/replica having differing shard_id in the
cluster.nodes and then the server cannot startup - reports corruption.

This PR builds on #13428 by simply ignoring the replica's shard_id in
cluster.nodes (if it exists), and uses the replica's primary's shard_id.
Additional handling was necessary to cover the case where the replica
appears before the primary in cluster.nodes, where it will first use a
generated shard_id for the primary, and then correct after it loads the
primary cluster.nodes entry.

---------

Co-authored-by: debing.sun <debing.sun@redis.com>
 2024-09-12 14:01:09 +08:00
YaacovHazan 406a365f44 Bundle modules for Redis 8.0 M01 2024-09-11 16:40:16 +03:00
debing.sunandoranagra 2dd4cca363 Increment kvstore's non_empty_dicts only on first insert (#13528) 
Found by @oranagra 

Currently, when the size of dict becomes 1, we do not check whether
`delta` is positive or negative.
As a result, `non_empty_dicts` is still incremented when the size of
dict changes from 2 to 1.
We should only increment `non_empty_dicts` when `delta` is positive, as
this indicates the first time an element is inserted into the dict.

---------

Co-authored-by: oranagra <oran@redislabs.com>
 2024-09-11 09:36:01 +08:00
Filipe Oliveira (Redis) bcae770819 Optimize LREM, LPOS, LINSERT, LINDEX: Avoid N-1 sdslen() calls on listTypeEqual (#13529) 
This is a very easy optimization, that avoids duplicate computation of
the object length for LREM, LPOS, LINSERT na LINDEX.

We can see that sdslen takes 7.7% of the total CPU cycles of the
benchmarks.

Function Stack | CPU Time: Total | CPU Time: Self | Module | Function
(Full) | Source File | Start Address
-- | -- | -- | -- | -- | -- | --
listTypeEqual | 15.50% | 2.346s | redis-server | listTypeEqual |
t_list.c | 0x845dd
sdslen | 7.70% | 2.300s | redis-server | sdslen | sds.h | 0x845e4

Preliminary data showcases 4% improvement in the achievable ops/sec of
LPOS in string elements, and 2% in int elements.
 2024-09-10 20:26:36 +08:00
YaacovHazanandYaacovHazan bf802b0764 Add the option to build Redis with modules (#13524) 
A new BUILD_WITH_MODULES flag was added to the Makefile to control
building the module directory.

The new module directory includes a general Makefile that iterates
over each module, fetch a specific version, and build it.

Co-authored-by: YaacovHazan <yaacov.hazan@redislabs.com>
 2024-09-09 15:47:02 +03:00
Ozan Tezcan ac03e3721d Fix flaky replication tests (#13518) 
#13495 introduced a change to reply -LOADING while flushing existing db on a replica. Some of our tests are
 sensitive to this change and do no expect -LOADING reply.

Fixing a couple of tests that fail time to time.
 2024-09-08 12:54:01 +03:00
Filipe Oliveira (Redis)anddebing.sun 31227f4faf Optimize client type check on reply hot code paths (#13516) 
## Proposed improvement

This PR introduces the static inlined function `clientTypeIsSlave` which
is doing only 1 condition check vs 3 checks of `getClientType`, and also
uses the `unlikely` to tell the compiler that the most common outcome is
for the client not to be a slave.
Preliminary data show 3% improvement on the achievable ops/sec on the
specific LRANGE benchmark. After running the entire suite we see up to
5% improvement in 2 tests.
https://github.com/redis/redis/pull/13516#issuecomment-2331326052

## Context

This optimization efforts comes from analyzing the profile info from the
[memtier_benchmark-1key-list-1K-elements-lrange-all-elements](https://github.com/redis/redis-benchmarks-specification/blob/main/redis_benchmarks_specification/test-suites/memtier_benchmark-1key-list-1K-elements-lrange-all-elements.yml)
benchmark.
 
By going over it, we can see that `getClientType` consumes 2% of the cpu
time, strictly to check if the client is a slave (
https://github.com/redis/redis/blob/unstable/src/networking.c#L397 , and
https://github.com/redis/redis/blob/unstable/src/networking.c#L1254 )


Function | CPU Time: Total | CPU Time: Self | Module | Function (Full)
-- | -- | -- | -- | --
_addReplyToBufferOrList->getClientType | 1.20% | 0.728s | redis-server |
getClientType
clientHasPendingReplies->getClientType | 0.80% | 0.482s | redis-server |
getClientType

---------

Co-authored-by: debing.sun <debing.sun@redis.com>
 2024-09-06 10:24:30 +08:00
Max Malekzadeh f6f11f3ef1 Remove outdated "Try Redis" link in README.md (#13498) 2024-09-05 22:04:49 +08:00
Moti Cohen 569584d463 HFE - Simplify logic of HGETALL command (#13425) 2024-09-05 12:48:44 +03:00
debing.sunUri YagelnikMadelyn Olsonoranagra
 		ea3e8b79a1 Introduce reusable query buffer for client reads (#13488) 
This PR is based on the commits from PR
https://github.com/valkey-io/valkey/pull/258,
https://github.com/valkey-io/valkey/pull/593,
https://github.com/valkey-io/valkey/pull/639

This PR optimizes client query buffer handling in Redis by introducing
a reusable query buffer that is used by default for client reads. This
reduces memory usage by ~20KB per client by avoiding allocations for
most clients using short (<16KB) complete commands. For larger or
partial commands, the client still gets its own private buffer.

The primary changes are:

* Adding a reusable query buffer `thread_shared_qb` that clients use by
default.
* Modifying client querybuf initialization and reset logic.
* Freeing idle client query buffers when empty to allow reuse of the
reusable query buffer.
* Master client query buffers are kept private as their contents need to
be preserved for replication stream.
* When nested commands is executed, only the first user uses the reuse
buffer, and subsequent users will still use the private buffer.

In addition to the memory savings, this change shows a 3% improvement in
latency and throughput when running with 1000 active clients.

The memory reduction may also help reduce the need to evict clients when
reaching max memory limit, as the query buffer is the main memory
consumer per client.

This PR is different from https://github.com/valkey-io/valkey/pull/258
1. When a client is in the mid of requiring a reused buffer and
returning it, regardless of whether the query buffer has changed
(expanded), we do not update the reused query buffer in the middle, but
return the reused query buffer (expanded or with data remaining) or
reset it at the end.
2. Adding a new thread variable `thread_shared_qb_used` to avoid
multiple clients requiring the reusable query buffer at the same time.

---------

Signed-off-by: Uri Yagelnik <uriy@amazon.com>
Signed-off-by: Madelyn Olson <matolson@amazon.com>
Co-authored-by: Uri Yagelnik <uriy@amazon.com>
Co-authored-by: Madelyn Olson <madelyneolson@gmail.com>
Co-authored-by: oranagra <oran@redislabs.com>
 2024-09-04 19:10:40 +08:00
debing.sun 74609d44cd Fix set with invalid length causes smembers to hang (#13515) 
After https://github.com/redis/redis/pull/13499, If the length set by
`addReplySetLen()` does not match the actual number of elements in the
reply, it will cause protocol broken and result in the client hanging.
 2024-09-04 17:35:46 +08:00
Ozan Tezcan ea05c6ac47 Fix RM_RdbLoad() to enable AOF after loading is completed (#13510) 
RM_RdbLoad() disables AOF temporarily while loading RDB.
Later, it does not enable it back as it checks AOF state (disabled by then) 
rather than AOF config parameter.

Added a change to restart AOF according to config parameter.
 2024-09-04 11:11:04 +03:00
Filipe Oliveira (Redis)anddebing.sun 05aed4cab9 Optimize SET/INCR*/DECR*/SETRANGE/APPEND by reducing duplicate computation (#13505) 
- Avoid addReplyLongLong (which converts back to string) the value we
already have as a robj, by using addReplyProto + addReply
- Avoid doing dbFind Twice for the same dictEntry on
INCR*/DECR*/SETRANGE/APPEND commands.
- Avoid multiple sdslen calls with the same input on setrangeCommand and
appendCommand
- Introduce setKeyWithDictEntry, which is like setKey(), but accepts an
optional dictEntry input: Avoids the second dictFind in SET command

---------

Co-authored-by: debing.sun <debing.sun@redis.com>
 2024-09-04 14:51:21 +08:00
debing.sunandOzan Tezcan de7f2f87f7 Avoid overhead of comparision function pointer calls in lpFind() (#13503) 
In #13279 (found by @filipecosta90), for custom lookups, we introduce a
comparison function for `lpFind()` to compare entry, but it also
introduces some overhead.

To avoid the overhead of function pointer calls:
1. Extract the lpFindCb() method into a lpFindCbInternal() method that
is easier to inline.
2. Use unlikely to annotate the comparison method, as can only success
once.

---------

Co-authored-by: Ozan Tezcan <ozantezcan@gmail.com>
 2024-09-03 22:53:13 +08:00
Filipe Oliveira (Redis) fb8755a636 changed addReplyHumanLongDouble to addReplyDouble in georadiusGeneric and geoposCommand (#13494) 
# Summary 

- Addresses https://github.com/redis/redis/issues/11565
- Measured improvements of 30% and 37% on the simple use-case (GEOSEARCH
and GEOPOS) (check
https://github.com/redis/redis/pull/13494#issuecomment-2313668934), and
of 66% on a dataset with >60M datapoints and pipeline 10 benchmark.
 2024-09-03 20:54:20 +08:00
Meir Shpilraien (Spielrein) d3d94ccf2e Added new defrag API to allocate and free raw memory. (#13509) 
All the defrag allocations API expects to get a value and replace it, leaving the old value untouchable.
In some cases a value might be shared between multiple keys, in such cases we can not simply replace
it when the defrag callback is called.

To allow support such use cases, the PR adds two new API's to the defrag API:

1. `RM_DefragAllocRaw` - allocate memory base on a given size.
2. `RM_DefragFreeRaw` - Free the given pointer.

Those API's avoid using tcache so they operate just like `RM_DefragAlloc` but allows the user to split
the allocation and the memory free operations into two stages and control when those happen.

In addition the PR adds new API to allow the module to receive notifications when defrag start and end: `RM_RegisterDefragCallbacks`
Those callbacks are the same as `RM_RegisterDefragFunc` but promised to be called and the start
and the end of the defrag process.
 2024-09-03 15:03:19 +03:00
Filipe Oliveira (Redis)anddebing.sun 00a8e72cfc Created specific SMEMBERS command logic which avoids sinterGenericCommand, and minimizes processing and memory overhead (#13499) 
This PR introduces a dedicated implementation for the SMEMBERS command
that avoids using the more generalized sinterGenericCommand function.
By tailoring the logic specifically for SMEMBERS, we reduce unnecessary
processing and memory overheads that were previously incurred by
handling more complex cases like set intersections.

---------

Co-authored-by: debing.sun <debing.sun@redis.com>
 2024-09-03 18:32:43 +08:00
Filipe Oliveira (Redis) a31b516e25 Optimize the HELLO command reply (#13490) 
# Overall improvement

TBD ( current is approximately 6% on the achievable ops/sec), coming
from:

- In case of no module we can skip 1.3% CPU cycles on dict Iterator
creation/deletion
- Use addReplyBulkCBuffer instead of addReplyBulkCString to avoid
runtime strlen overhead within HELLO reply on string constants.

## Optimization 1: In case of no module we can skip 1.3% CPU cycles on
dict Iterator creation/deletion.

## Optimization 2: Use addReplyBulkCBuffer instead of
addReplyBulkCString to avoid runtime strlen overhead within HELLO reply
on string constants.
 2024-09-03 17:27:35 +08:00
cyy-tagandyingyin.chen c77b8f45e9 Fixed variable parameter formatting issues in serverPanic function (#13504) 
Currently aeApiPoll panic does not record error code information. Added
variable parameter formatting to _serverPanic to fix the issue

---------

Co-authored-by: yingyin.chen <15816602944@163.com>
 2024-09-03 15:51:46 +08:00
Ozan Tezcan a7afd1d2b2 Reply LOADING on replica while flushing the db (#13495) 
On a full sync, replica starts discarding existing db. If the existing 
db is huge and flush is happening synchronously, replica may become 
unresponsive. 

Adding a change to yield back to event loop while flushing db on 
a replica. Replica will reply -LOADING in this case. Note that while 
replica is loading the new rdb, it may get an error and start flushing
the partial db. This step may take a long time as well. Similarly, 
replica will reply -LOADING in this case. 

To call processEventsWhileBlocked() and reply -LOADING, we need to do:
- Set connSetReadHandler() null not to process further data from the master
- Set server.loading flag
- Call blockingOperationStarts()

rdbload() already does these steps and calls processEventsWhileBlocked()
while loading the rdb. Added a new call rdbLoadWithEmptyFunc() which 
accepts callback to flush db before loading rdb or when an error 
happens while loading. 

For diskless replication, doing something similar and calling emptyData()
after setting required flags.

Additional changes:
- Allow `appendonly` config change during loading. 
 Config can be changed while loading data on startup or on replication 
 when slave is loading RDB. We allow config change command to update 
 `server.aof_enabled` and then lazily apply config change after loading
 operation is completed.
 
 - Added a test for `replica-lazy-flush` config
 2024-09-03 09:48:44 +03:00
Oran Agra 3fcddfb61f testsuite --dump-logs works on servers started before the test (#13500) 
so far ./runtest --dump-logs used work for servers started within the
test proc.
now it'll also work on servers started outside the test proc scope.
the downside is that these logs can be huge if they served many tests
and not just the failing one.
but for some rare failures, we rather have that than nothing.
this feature isn't enabled y default, but is used by our GH actions.
 2024-08-29 07:27:23 +01:00
CoolThi 3c9f5954b5 Remove variable expired in expireSlaveKeys() to prevent confusing the compiler (#13299) 
This change prevents missed optimization for some compilers:
https://godbolt.org/z/W66h86E13 (the reduced intermediate form in
optimization).
 2024-08-28 21:52:23 +08:00
Raz Monsonego 3b1b1d1486 MOD-7645: Return module commands in ACL CAT (#13489) 
Currently, module commands are not returned for the `ACL CAT <category>`
command, but skipped instead. Since now modules can add ACL categories
they should no longer be skipped.
 2024-08-26 19:25:22 +01:00
paoloredis 60f22ca830 Add redis_docs_sync workflow (#13426) 
This workflow executes on releases. It invokes the `redis_docs_sync`
workflow on `redis/docs`.
 2024-08-21 09:41:14 +01:00
Zihao Linanddebing.sun 6ceadfb580 Improve GETRANGE command behavior (#12272) 
Fixed the issue about GETRANGE and SUBSTR command
return unexpected result caused by the `start` and `end` out of
definition range of string.

---
## break change
Before this PR, when negative `end` was out of range (i.e., end <
-strlen), we would fix it to 0 to get the substring, which also resulted
in the first character still being returned for this kind of out of
range.
After this PR, we ensure that `GETRANGE` returns an empty bulk when the
negative end index is out of range.

Closes #11738

---------

Co-authored-by: debing.sun <debing.sun@redis.com>
 2024-08-20 12:34:43 +08:00
judeng 7f0a7f0a69 improve performance for scan command when matching data type (#12395) 
Move the TYPE filtering to the scan callback so that avoided the
`lookupKey` operation. This is the follow-up to #12209 . In this thread
we introduced two breaking changes：
1. we will not attempt to do lazy expire (delete) a key that was
filtered by not matching the TYPE (like we already do for MATCH
pattern).
2. when the specified key TYPE filter is an unknown type, server will
reply a error immediately instead of doing a full scan that comes back
empty handed.
 2024-08-20 10:47:51 +08:00
Meir Shpilraien (Spielrein) 3264deb24e Avoid used_memory contention when update from multiple threads. (#13431) 
The PR attempt to avoid contention on the `used_memory` global variable
when allocate or free memory from multiple threads at the same time.

Each time a thread is allocating or releasing a memory, it needs to
update the `used_memory` global variable. This update might cause a
contention when done aggressively from multiple threads.

### The solution

Instead of having a single global variable that need to be updated from
multiple thread. We create an array of used_memory, each entry in the
array is updated by a single thread and the main thread summarizes all
the values to accumulate the memory usage.

This solution, though reduces the contention between threads on updating
the `used_memory` global variable, it adds work to the main thread that
need to summarize all the entries at the `used_memory` array. To avoid
increasing the work done by the main thread by too much, we limit the
size of the used memory array to 16. This means that up to 16 threads
can run without any contention between them. If there are more than 16
threads, we will reuse entries on the used_memory array, in this case we
might still have contention between threads, but it will be much less
significant.

Notice, that in order to really avoid contention, the entries in the
`used_memory` array must reside on different cache lines. To achieve
that we create a struct with padding such that its size will be exactly
cache_line size. In addition we make sure the address of the
`used_memory` array will be aligned to cache_line size.

### Benchmark

Some benchmark shows improvement (up to 15%):

| Test Case |Baseline unstable (median obs. +- std.dev)|Comparison
test_used_memory_per_thread_array (median obs. +- std.dev)|% change
(higher-better)| Note |

|-------------------------------------------------------------------------------|------------------------------------------|--------------------------------------------------------------------:|------------------------|------------------------------------|
|memtier_benchmark-1key-list-100-elements-lrange-all-elements | 92657 +-
2.0% (2 datapoints) | 101445|9.5% |IMPROVEMENT |
|memtier_benchmark-1key-list-1K-elements-lrange-all-elements | 14965 +-
1.3% (2 datapoints) | 16296|8.9% |IMPROVEMENT |
|memtier_benchmark-1key-set-10-elements-smembers-pipeline-10 | 431019 +-
5.2% (2 datapoints) | 461039|7.0% |waterline=5.2%. IMPROVEMENT |
|memtier_benchmark-1key-set-100-elements-smembers | 74367 +- 0.0% (2
datapoints) | 80190|7.8% |IMPROVEMENT |
|memtier_benchmark-1key-set-1K-elements-smembers | 11730 +- 0.4% (2
datapoints) | 13519|15.3% |IMPROVEMENT |


Full results:

| Test Case |Baseline unstable (median obs. +- std.dev)|Comparison
test_used_memory_per_thread_array (median obs. +- std.dev)|% change
(higher-better)| Note |

|-------------------------------------------------------------------------------|------------------------------------------|--------------------------------------------------------------------:|------------------------|------------------------------------|
|memtier_benchmark-10Mkeys-load-hash-5-fields-with-1000B-values | 88613
+- 1.0% (2 datapoints) | 88688|0.1% |No Change |

|memtier_benchmark-10Mkeys-load-hash-5-fields-with-1000B-values-pipeline-10
| 124786 +- 1.2% (2 datapoints) | 123671|-0.9% |No Change |
|memtier_benchmark-10Mkeys-load-hash-5-fields-with-100B-values | 122460
+- 1.4% (2 datapoints) | 122990|0.4% |No Change |

|memtier_benchmark-10Mkeys-load-hash-5-fields-with-100B-values-pipeline-10
| 333384 +- 5.1% (2 datapoints) | 319221|-4.2% |waterline=5.1%.
potential REGRESSION|
|memtier_benchmark-10Mkeys-load-hash-5-fields-with-10B-values | 137354
+- 0.3% (2 datapoints) | 138759|1.0% |No Change |

|memtier_benchmark-10Mkeys-load-hash-5-fields-with-10B-values-pipeline-10
| 401261 +- 4.3% (2 datapoints) | 398524|-0.7% |No Change |
|memtier_benchmark-1Mkeys-100B-expire-use-case | 179058 +- 0.4% (2
datapoints) | 180114|0.6% |No Change |
|memtier_benchmark-1Mkeys-10B-expire-use-case | 180390 +- 0.2% (2
datapoints) | 180401|0.0% |No Change |
|memtier_benchmark-1Mkeys-1KiB-expire-use-case | 175993 +- 0.7% (2
datapoints) | 175147|-0.5% |No Change |
|memtier_benchmark-1Mkeys-4KiB-expire-use-case | 165771 +- 0.0% (2
datapoints) | 164434|-0.8% |No Change |
|memtier_benchmark-1Mkeys-bitmap-getbit-pipeline-10 | 931339 +- 2.1% (2
datapoints) | 929487|-0.2% |No Change |
|memtier_benchmark-1Mkeys-generic-exists-pipeline-10 | 999462 +- 0.4% (2
datapoints) | 963226|-3.6% |potential REGRESSION |
|memtier_benchmark-1Mkeys-generic-expire-pipeline-10 | 905333 +- 1.4% (2
datapoints) | 896673|-1.0% |No Change |
|memtier_benchmark-1Mkeys-generic-expireat-pipeline-10 | 885015 +- 1.0%
(2 datapoints) | 865010|-2.3% |No Change |
|memtier_benchmark-1Mkeys-generic-pexpire-pipeline-10 | 897115 +- 1.2%
(2 datapoints) | 887544|-1.1% |No Change |
|memtier_benchmark-1Mkeys-generic-scan-pipeline-10 | 451103 +- 3.2% (2
datapoints) | 465571|3.2% |potential IMPROVEMENT |
|memtier_benchmark-1Mkeys-generic-touch-pipeline-10 | 996809 +- 0.6% (2
datapoints) | 984478|-1.2% |No Change |
|memtier_benchmark-1Mkeys-generic-ttl-pipeline-10 | 979570 +- 1.7% (2
datapoints) | 958752|-2.1% |No Change |

|memtier_benchmark-1Mkeys-hash-hget-hgetall-hkeys-hvals-with-100B-values
| 180888 +- 0.5% (2 datapoints) | 182295|0.8% |No Change |

|memtier_benchmark-1Mkeys-hash-hmget-5-fields-with-100B-values-pipeline-10
| 717881 +- 1.0% (2 datapoints) | 724814|1.0% |No Change |
|memtier_benchmark-1Mkeys-hash-transactions-multi-exec-pipeline-20 |
1055447 +- 0.4% (2 datapoints) | 1065836|1.0% |No Change |
|memtier_benchmark-1Mkeys-lhash-hexists | 164332 +- 0.1% (2 datapoints)
| 163636|-0.4% |No Change |
|memtier_benchmark-1Mkeys-lhash-hincbry | 171674 +- 0.3% (2 datapoints)
| 172737|0.6% |No Change |
|memtier_benchmark-1Mkeys-list-lpop-rpop-with-100B-values | 180904 +-
1.1% (2 datapoints) | 179467|-0.8% |No Change |
|memtier_benchmark-1Mkeys-list-lpop-rpop-with-10B-values | 181746 +-
0.8% (2 datapoints) | 182416|0.4% |No Change |
|memtier_benchmark-1Mkeys-list-lpop-rpop-with-1KiB-values | 182004 +-
0.7% (2 datapoints) | 180237|-1.0% |No Change |
|memtier_benchmark-1Mkeys-load-hash-5-fields-with-1000B-values | 105191
+- 0.9% (2 datapoints) | 105058|-0.1% |No Change |

|memtier_benchmark-1Mkeys-load-hash-5-fields-with-1000B-values-pipeline-10
| 150683 +- 0.9% (2 datapoints) | 153597|1.9% |No Change |
|memtier_benchmark-1Mkeys-load-hash-hmset-5-fields-with-1000B-values |
104122 +- 0.7% (2 datapoints) | 105236|1.1% |No Change |
|memtier_benchmark-1Mkeys-load-list-with-100B-values | 149770 +- 0.9% (2
datapoints) | 150510|0.5% |No Change |
|memtier_benchmark-1Mkeys-load-list-with-10B-values | 165537 +- 1.9% (2
datapoints) | 164329|-0.7% |No Change |
|memtier_benchmark-1Mkeys-load-list-with-1KiB-values | 113315 +- 0.5% (2
datapoints) | 114110|0.7% |No Change |
|memtier_benchmark-1Mkeys-load-stream-1-fields-with-100B-values | 131201
+- 0.7% (2 datapoints) | 129545|-1.3% |No Change |

|memtier_benchmark-1Mkeys-load-stream-1-fields-with-100B-values-pipeline-10
| 352891 +- 2.8% (2 datapoints) | 348338|-1.3% |No Change |
|memtier_benchmark-1Mkeys-load-stream-5-fields-with-100B-values | 104386
+- 0.7% (2 datapoints) | 105796|1.4% |No Change |

|memtier_benchmark-1Mkeys-load-stream-5-fields-with-100B-values-pipeline-10
| 227593 +- 5.5% (2 datapoints) | 218783|-3.9% |waterline=5.5%.
potential REGRESSION|
|memtier_benchmark-1Mkeys-load-string-with-100B-values | 167552 +- 0.2%
(2 datapoints) | 170282|1.6% |No Change |
|memtier_benchmark-1Mkeys-load-string-with-100B-values-pipeline-10 |
646888 +- 0.5% (2 datapoints) | 639680|-1.1% |No Change |
|memtier_benchmark-1Mkeys-load-string-with-10B-values | 174891 +- 0.7%
(2 datapoints) | 174382|-0.3% |No Change |
|memtier_benchmark-1Mkeys-load-string-with-10B-values-pipeline-10 |
749988 +- 5.1% (2 datapoints) | 769986|2.7% |waterline=5.1%. No Change |
|memtier_benchmark-1Mkeys-load-string-with-1KiB-values | 155929 +- 0.1%
(2 datapoints) | 156387|0.3% |No Change |
|memtier_benchmark-1Mkeys-load-zset-with-10-elements-double-score |
92241 +- 0.2% (2 datapoints) | 92189|-0.1% |No Change |
|memtier_benchmark-1Mkeys-load-zset-with-10-elements-int-score | 114328
+- 1.3% (2 datapoints) | 113154|-1.0% |No Change |
|memtier_benchmark-1Mkeys-string-get-100B | 180685 +- 0.2% (2
datapoints) | 180359|-0.2% |No Change |
|memtier_benchmark-1Mkeys-string-get-100B-pipeline-10 | 991291 +- 3.1%
(2 datapoints) | 1020086|2.9% |No Change |
|memtier_benchmark-1Mkeys-string-get-10B | 181183 +- 0.3% (2 datapoints)
| 177868|-1.8% |No Change |
|memtier_benchmark-1Mkeys-string-get-10B-pipeline-10 | 1032554 +- 0.8%
(2 datapoints) | 1023120|-0.9% |No Change |
|memtier_benchmark-1Mkeys-string-get-1KiB | 180479 +- 0.9% (2
datapoints) | 182215|1.0% |No Change |
|memtier_benchmark-1Mkeys-string-get-1KiB-pipeline-10 | 979286 +- 0.9%
(2 datapoints) | 989888|1.1% |No Change |
|memtier_benchmark-1Mkeys-string-mget-1KiB | 121950 +- 0.4% (2
datapoints) | 120996|-0.8% |No Change |
|memtier_benchmark-1key-geo-60M-elements-geodist | 179404 +- 1.0% (2
datapoints) | 181232|1.0% |No Change |
|memtier_benchmark-1key-geo-60M-elements-geodist-pipeline-10 | 1023797
+- 0.5% (2 datapoints) | 1014980|-0.9% |No Change |
|memtier_benchmark-1key-geo-60M-elements-geohash | 180808 +- 1.2% (2
datapoints) | 180606|-0.1% |No Change |
|memtier_benchmark-1key-geo-60M-elements-geohash-pipeline-10 | 1056458
+- 1.6% (2 datapoints) | 1040050|-1.6% |No Change |
|memtier_benchmark-1key-geo-60M-elements-geopos | 181808 +- 0.2% (2
datapoints) | 175945|-3.2% |potential REGRESSION |
|memtier_benchmark-1key-geo-60M-elements-geopos-pipeline-10 | 1038180 +-
3.4% (2 datapoints) | 1033005|-0.5% |No Change |
|memtier_benchmark-1key-geo-60M-elements-geosearch-fromlonlat | 142614
+- 0.3% (2 datapoints) | 144259|1.2% |No Change |
|memtier_benchmark-1key-geo-60M-elements-geosearch-fromlonlat-bybox |
141008 +- 0.4% (2 datapoints) | 139602|-1.0% |No Change |

|memtier_benchmark-1key-geo-60M-elements-geosearch-fromlonlat-pipeline-10
| 560698 +- 0.8% (2 datapoints) | 548806|-2.1% |No Change |
|memtier_benchmark-1key-list-10-elements-lrange-all-elements | 166132 +-
0.9% (2 datapoints) | 170259|2.5% |No Change |
|memtier_benchmark-1key-list-100-elements-lrange-all-elements | 92657 +-
2.0% (2 datapoints) | 101445|9.5% |IMPROVEMENT |
|memtier_benchmark-1key-list-1K-elements-lrange-all-elements | 14965 +-
1.3% (2 datapoints) | 16296|8.9% |IMPROVEMENT |
|memtier_benchmark-1key-pfadd-4KB-values-pipeline-10 | 264156 +- 0.2% (2
datapoints) | 262582|-0.6% |No Change |
|memtier_benchmark-1key-set-10-elements-smembers | 138916 +- 1.7% (2
datapoints) | 138016|-0.6% |No Change |
|memtier_benchmark-1key-set-10-elements-smembers-pipeline-10 | 431019 +-
5.2% (2 datapoints) | 461039|7.0% |waterline=5.2%. IMPROVEMENT |
|memtier_benchmark-1key-set-10-elements-smismember | 173545 +- 1.1% (2
datapoints) | 173488|-0.0% |No Change |
|memtier_benchmark-1key-set-100-elements-smembers | 74367 +- 0.0% (2
datapoints) | 80190|7.8% |IMPROVEMENT |
|memtier_benchmark-1key-set-100-elements-smismember | 155682 +- 1.6% (2
datapoints) | 151367|-2.8% |No Change |
|memtier_benchmark-1key-set-1K-elements-smembers | 11730 +- 0.4% (2
datapoints) | 13519|15.3% |IMPROVEMENT |
|memtier_benchmark-1key-set-200K-elements-sadd-constant | 181070 +- 1.1%
(2 datapoints) | 180214|-0.5% |No Change |
|memtier_benchmark-1key-set-2M-elements-sadd-increasing | 166364 +- 0.1%
(2 datapoints) | 166944|0.3% |No Change |
|memtier_benchmark-1key-zincrby-1M-elements-pipeline-1 | 46071 +- 0.6%
(2 datapoints) | 44979|-2.4% |No Change |
|memtier_benchmark-1key-zrank-1M-elements-pipeline-1 | 48429 +- 0.4% (2
datapoints) | 49265|1.7% |No Change |
|memtier_benchmark-1key-zrem-5M-elements-pipeline-1 | 48528 +- 0.4% (2
datapoints) | 48869|0.7% |No Change |
|memtier_benchmark-1key-zrevrangebyscore-256K-elements-pipeline-1 |
100580 +- 1.5% (2 datapoints) | 101782|1.2% |No Change |
|memtier_benchmark-1key-zrevrank-1M-elements-pipeline-1 | 48621 +- 2.0%
(2 datapoints) | 48473|-0.3% |No Change |
|memtier_benchmark-1key-zset-10-elements-zrange-all-elements | 83485 +-
0.6% (2 datapoints) | 83095|-0.5% |No Change |

|memtier_benchmark-1key-zset-10-elements-zrange-all-elements-long-scores
| 118673 +- 0.8% (2 datapoints) | 118006|-0.6% |No Change |
|memtier_benchmark-1key-zset-100-elements-zrange-all-elements | 19009 +-
1.1% (2 datapoints) | 19293|1.5% |No Change |
|memtier_benchmark-1key-zset-100-elements-zrangebyscore-all-elements |
18957 +- 0.5% (2 datapoints) | 19419|2.4% |No Change |

|memtier_benchmark-1key-zset-100-elements-zrangebyscore-all-elements-long-scores|
171693 +- 0.5% (2 datapoints) | 172432|0.4% |No Change |
|memtier_benchmark-1key-zset-1K-elements-zrange-all-elements | 3566 +-
0.6% (2 datapoints) | 3672|3.0% |No Change |
|memtier_benchmark-1key-zset-1M-elements-zcard-pipeline-10 | 1067713 +-
0.4% (2 datapoints) | 1071550|0.4% |No Change |
|memtier_benchmark-1key-zset-1M-elements-zrevrange-5-elements | 169195
+- 0.7% (2 datapoints) | 169620|0.3% |No Change |
|memtier_benchmark-1key-zset-1M-elements-zscore-pipeline-10 | 914338 +-
0.2% (2 datapoints) | 905540|-1.0% |No Change |
|memtier_benchmark-2keys-lua-eval-hset-expire | 88346 +- 1.7% (2
datapoints) | 87259|-1.2% |No Change |
|memtier_benchmark-2keys-lua-evalsha-hset-expire | 103273 +- 1.2% (2
datapoints) | 102393|-0.9% |No Change |
|memtier_benchmark-2keys-set-10-100-elements-sdiff | 15418 +- 10.9%
UNSTABLE (2 datapoints) | 14369|-6.8% |UNSTABLE (very high variance) |
|memtier_benchmark-2keys-set-10-100-elements-sinter | 83601 +- 3.6% (2
datapoints) | 82508|-1.3% |No Change |
|memtier_benchmark-2keys-set-10-100-elements-sunion | 14942 +- 11.2%
UNSTABLE (2 datapoints) | 14001|-6.3% |UNSTABLE (very high variance) |
|memtier_benchmark-2keys-stream-5-entries-xread-all-entries | 75938 +-
0.4% (2 datapoints) | 76565|0.8% |No Change |
|memtier_benchmark-2keys-stream-5-entries-xread-all-entries-pipeline-10
| 120781 +- 1.1% (2 datapoints) | 119142|-1.4% |No Change |
 2024-08-19 13:22:16 +03:00
debing.sun 6c6489280c Fix a race condition issue in the cache_memory of functionsLibCtx (#13476) 
This is a missing of the PR https://github.com/redis/redis/pull/13383.
We will call `functionsLibCtxClear()` in bio, so we shouldn't touch
`curr_functions_lib_ctx` in it.
 2024-08-19 10:11:45 +08:00
debing.sun 2b88db90aa Fix incorrect lag due to trimming stream via XTRIM command (#13473) 
## Describe
When using the `XTRIM` command to trim a stream, it does not update the
maximal tombstone (`max_deleted_entry_id`). This leads to an issue where
the lag calculation incorrectly assumes that there are no tombstones
after the consumer group's last_id, resulting in an inaccurate lag.

The reason XTRIM doesn't need to update the maximal tombstone is that it
always trims from the beginning of the stream. This means that it
consistently changes the position of the first entry, leading to the
following scenarios:

1) First entry trimmed after maximal tombstone:
If the first entry is trimmed to a position after the maximal tombstone,
all tombstones will be before the first entry, so they won't affect the
consumer group's lag.

2) First entry trimmed before maximal tombstone:
If the first entry is trimmed to a position before the maximal
tombstone, the maximal tombstone will not be updated.

## Solution
Therefore, this PR optimizes the lag calculation by ensuring that when
both the consumer group's last_id and the maximal tombstone are behind
the first entry, the consumer group's lag is always equal to the number
of remaining elements in the stream.

Supplement to PR https://github.com/redis/redis/pull/13338
 2024-08-16 23:13:31 +08:00
debing.sun b94b714f81 Fix error message for XREAD command with wrong parameter (#13474) 
Fixed a missing from #13117.
When the number of streams is incorrect, the error message for `XREAD`
needs to include the '+' symbol.
 2024-08-14 21:40:43 +08:00
Moti Cohen 806459f481 On HDEL last field with expiry, update global HFE DS (#13470) 
Hash field expiration is optimized to avoid frequent update global HFE DS for
each field deletion. Eventually active-expiration will run and update or remove
the hash from global HFE DS gracefully. Nevertheless, statistic "subexpiry"
might reflect wrong number of hashes with HFE to the user if HDEL deletes
the last field with expiration in hash (yet there are more fields without expiration).

Following this change, if HDEL the last field with expiration in the hash then
take care to remove the hash from global HFE DS as well.
 2024-08-11 16:39:03 +03:00
LuMingYinDetectanddebing.sun 3a08819f51 Fix some memory leaks in redis-cli (#13258) 
Fix memory leak related to variable slot_nodes  in the
clusterManagerFixSlotsCoverage() function.

---------

Co-authored-by: debing.sun <debing.sun@redis.com>
 2024-08-08 10:51:33 +08:00
debing.sunandHarkrishn Patro 6f0ddc9d92 Pass extensions to node if extension processing is handled by it (#13465) 
This PR is based on the commits from PR
https://github.com/valkey-io/valkey/pull/52.
Ref: https://github.com/redis/redis/pull/12760
Close https://github.com/redis/redis/issues/13401
This PR will replace https://github.com/redis/redis/pull/13449

Fixes compatibilty of Redis cluster (7.2 - extensions enabled by
default) with older Redis cluster (< 7.0 - extensions not handled) .

With some of the extensions enabled by default in 7.2 version, new nodes
running 7.2 and above start sending out larger clusterbus message
payload including the ping extensions. This caused an incompatibility
with node running engine versions < 7.0. Old nodes (< 7.0) would receive
the payload from new nodes (> 7.2) would observe a payload length
(totlen) > (estlen) and would perform an early exit and won't process
the message.

This fix does the following things:
1. Always set `CLUSTERMSG_FLAG0_EXT_DATA`, because during the meet
phase, we do not know whether the connected node supports ext data, we
need to make sure that it knows and send back its ext data if it has.
2. If another node does not support ext data, we will not send it ext
data to avoid the handshake failure due to the incorrect payload length.

Note: A successful `PING`/`PONG` is required as a sender for a given
node to be marked as `CLUSTERMSG_FLAG0_EXT_DATA` and then extensions
message
will be sent to it. This could cause a slight delay in receiving the
extensions message(s).

---------

Signed-off-by: Harkrishn Patro <harkrisp@amazon.com>
Co-authored-by: Harkrishn Patro <harkrisp@amazon.com>

---------

Signed-off-by: Harkrishn Patro <harkrisp@amazon.com>
Co-authored-by: Harkrishn Patro <harkrisp@amazon.com>
 2024-08-08 10:48:03 +08:00
Anurag BandyopadhyayAnuragkillswitchdebing.sun
 		731f2dc5c7 Make some commets more friendly (#13319) 
Close  #13316

---------

Co-authored-by: Anuragkillswitch <70265851+Anuragkillswitch@users.noreply.github.com>
Co-authored-by: debing.sun <debing.sun@redis.com>
 2024-08-07 01:20:06 +08:00
debing.sun bf643a63c8 Ensure validity of myself as master or replica when loading cluster config (#13443) 
First, we need to ensure that `curmaster` in
`clusterUpdateSlotsConfigWith()` is not NULL in the line
https://github.com/redis/redis/blob/82f00f5179720c8cee6cd650763d184ba943be92/src/cluster_legacy.c#L2320
otherwise, it will crash in the
https://github.com/redis/redis/blob/82f00f5179720c8cee6cd650763d184ba943be92/src/cluster_legacy.c#L2395

So when loading cluster node config, we need to ensure that the
following conditions are met:
1. A node must be at least one of the master or replica.
2. If a node is a replica, its master can't be NULL.
 2024-08-06 20:40:46 +08:00
YaacovHazan e4ddc34463 Keep cluster shards command implementation generic (#13440) 
Make the clusterCommandShards function use only cluster API functions
instead of accessing cluster implementation details.
This way the cluster API implementation doesn't have to have intimate
knowledge of the command reply format, and doesn't need to interact with
the client directly (the addReply function family).
The PR has two commits, one moves the function from cluster_legacy.c to
cluster.c, and the other modifies it's implementation.


**better merge without squashing.**
 2024-08-05 11:08:48 +03:00
Josh Hershberg 6d5d754119 Make cluster shards cmd implementation generic 
This and the previous commit make the cluster
shards command a generic implementation instead of a
specific implementation for each cluster API implementation.
This commit (a) adds functions to the cluster API
and (b) modifies the cluster shards cmd implementation
to use cluster API functions instead of directly
accessing the legacy clustering implementation.

Signed-off-by: Josh Hershberg <yehoshua@redis.com>
 2024-08-05 10:31:40 +03:00
Josh Hershberg e3e631f394 Prep to make cluster shards cmd generic 
This and the next following commit makes the cluster
shards command a generic implementation instead of a
specific implementation for each cluster API implementation.
This commit simply moves the cluster shards implementation
from cluster_legacy.c to cluster.c without changing the
implementation at all. The reason for doing so was to
help with reviewing the changes in the diff.

Signed-off-by: Josh Hershberg <yehoshua@redis.com>
 2024-08-05 10:27:03 +03:00
Zhongxian Pananddebing.sun 6263823e54 Replace bit shift with __builtin_ctzll in HyperLogLog (#13218) 
## Replace bit shift with `__builtin_ctzll` in HyperLogLog

Builtin function `__builtin_ctzll` is more effective than bit shift even
though "in the average case there are high probabilities to find a 1
after a few iterations" mentioned in the source file comment.

---------

Co-authored-by: debing.sun <debing.sun@redis.com>
 2024-08-05 10:51:43 +08:00
Moti Cohen 4dd8b1faa9 Fix HTTL/HPTTL to be NONDETERMINISTIC_OUTPUT (#13461) 
H[P]TTL should be marked as NONDETERMINISTIC_OUTPUT just like [P]TTL.
 2024-08-04 17:42:50 +03:00
Vitah Linanddebing.sun 8038eb3147 Fix wrong dbnum showed in redis-cli after client reconnected (#13411) 
When the server restarts while the CLI is connecting, the reconnection
does not automatically select the previous db.
This may lead users to believe they are still in the previous db, in
fact, they are in db0.

This PR will automatically reset the current dbnum and `cliSelect()`
again when reconnecting.

---------

Co-authored-by: debing.sun <debing.sun@redis.com>
 2024-08-03 12:06:02 +08:00
c8ef 89742a95db Fix typo in hyperloglog.c (#13458) 2024-08-02 07:49:52 +08:00
debing.sun 60e9e630bd Fix CLUSTER SHARDS command returns empty array (#13422) 
Close https://github.com/redis/redis/issues/13414

When the cluster's master node fails and is switched to another node,
the first node in the shard node list (the old master) is no longer
valid.
Add a new method clusterGetMasterFromShard() to obtain the current
master.
 2024-08-02 07:22:13 +08:00
debing.sunandoranagra e750c619b2 Fix some test failures caused by key being deleted due to premature expiration (#13453) 
1. Fix fuzzer test failure when the key was deleted due to expiration
before sending random traffic for the key.

After HFE, when all fields in a hash are expired, the hash might be
deleted due to expiration.

If the key was expired in the mid of `RESTORE` command and sending rand
trafic, `fuzzer` test will fail in the following code because the 'TYPE
key' will return `none` and then throw an exception because it cannot be
found in `$commands`

https://github.com/redis/redis/blob/94b9072e44af0bae8cfe2de5cfa4af7b8e399759/tests/support/util.tcl#L712-L713

This PR adds a `None` check for the reply of `KEY TYPE` command and adds
a print of `err` to avoid false positives in the future.

failed CI:
https://github.com/redis/redis/actions/runs/10127334121/job/28004985388

2. Fix the issue where key was deleted due to expiration before the
`scan.scan_key` command could execute, caused by premature enabling of
`set-active-expire`.

failed CI:
https://github.com/redis/redis/actions/runs/10153722715/job/28077610552

---------

Co-authored-by: oranagra <oran@redislabs.com>
 2024-07-31 08:15:39 +08:00
debing.sun 93fb83b4cb Fix incorrect lag field in XINFO when tombstone is after the last_id of consume group (#13338) 
Fix #13337

Ths PR fixes fixed two bugs that caused lag calculation errors.
1. When the latest tombstone is before the first entry, the tombstone
may stil be after the last id of consume group.
2. When a tombstone is after the last id of consume group, the group's
counter will be invalid, we should caculate the entries_read by using
estimates.
 2024-07-30 22:31:31 +08:00
249 changed files with 34862 additions and 3009 deletions
Expand all files Collapse all files
.github/workflows/ci.yml
+4 -3
View File
@@ -62,7 +62,7 @@ jobs:
- uses: actions/checkout@v4
- name: make
run: |
sudo apt-get update && sudo apt-get install libc6-dev-i386
sudo apt-get update && sudo apt-get install libc6-dev-i386 gcc-multilib g++-multilib
make REDIS_CFLAGS='-Werror' 32bit
build-libc-malloc:
@@ -79,7 +79,7 @@ jobs:
- uses: actions/checkout@v4
- name: make
run: |
dnf -y install which gcc make
dnf -y install which gcc gcc-c++ make
make REDIS_CFLAGS='-Werror'
build-old-chain-jemalloc:
@@ -96,6 +96,7 @@ jobs:
apt-key adv --keyserver keyserver.ubuntu.com --recv-keys 40976EAF437D05B5
apt-key adv --keyserver keyserver.ubuntu.com --recv-keys 3B4FE6ACC0B21F32
apt-get update
apt-get install -y make gcc-4.8
apt-get install -y make gcc-4.8 g++-4.8
update-alternatives --install /usr/bin/gcc gcc /usr/bin/gcc-4.8 100
update-alternatives --install /usr/bin/g++ g++ /usr/bin/g++-4.8 100
make CC=gcc REDIS_CFLAGS='-Werror'
.github/workflows/codecov.yml
+24
View File
@@ -0,0 +1,24 @@
name: "Codecov"
# Enabling on each push is to display the coverage changes in every PR,
# where each PR needs to be compared against the coverage of the head commit
on: [push, pull_request]
jobs:
code-coverage:
runs-on: ubuntu-22.04
steps:
- name: Checkout repository
uses: actions/checkout@v4
- name: Install lcov and run test
run: |
sudo apt-get install lcov
make lcov
- name: Upload coverage reports to Codecov
uses: codecov/codecov-action@v4
with:
token: ${{ secrets.CODECOV_TOKEN }}
file: ./src/redis.info
.github/workflows/daily.yml
+22 -18
View File
@@ -76,7 +76,6 @@ jobs:
if: |
(github.event_name == 'workflow_dispatch' || (github.event_name != 'workflow_dispatch' && github.repository == 'redis/redis')) &&
!contains(github.event.inputs.skipjobs, 'fortify')
container: ubuntu:lunar
timeout-minutes: 14400
steps:
- name: prep
@@ -94,11 +93,10 @@ jobs:
ref: ${{ env.GITHUB_HEAD_REF }}
- name: make
run: |
apt-get update && apt-get install -y make gcc-13
update-alternatives --install /usr/bin/gcc gcc /usr/bin/gcc-13 100
apt-get update && apt-get install -y make gcc g++
make CC=gcc REDIS_CFLAGS='-Werror -DREDIS_TEST -U_FORTIFY_SOURCE -D_FORTIFY_SOURCE=3'
- name: testprep
run: apt-get install -y tcl8.6 tclx procps
run: sudo apt-get install -y tcl8.6 tclx procps
- name: test
if: true && !contains(github.event.inputs.skiptests, 'redis')
run: ./runtest --accurate --verbose --dump-logs ${{github.event.inputs.test_args}}
@@ -211,7 +209,7 @@ jobs:
ref: ${{ env.GITHUB_HEAD_REF }}
- name: make
run: |
sudo apt-get update && sudo apt-get install libc6-dev-i386
sudo apt-get update && sudo apt-get install libc6-dev-i386 g++ gcc-multilib g++-multilib
make 32bit REDIS_CFLAGS='-Werror -DREDIS_TEST'
- name: testprep
run: sudo apt-get install tcl8.6 tclx
@@ -456,7 +454,7 @@ jobs:
- name: testprep
run: |
sudo apt-get update
sudo apt-get install tcl8.6 tclx valgrind -y
sudo apt-get install tcl8.6 tclx valgrind g++ -y
- name: test
if: true && !contains(github.event.inputs.skiptests, 'redis')
run: ./runtest --valgrind --no-latency --verbose --clients 1 --timeout 2400 --dump-logs ${{github.event.inputs.test_args}}
@@ -521,7 +519,7 @@ jobs:
- name: testprep
run: |
sudo apt-get update
sudo apt-get install tcl8.6 tclx valgrind -y
sudo apt-get install tcl8.6 tclx valgrind g++ -y
- name: test
if: true && !contains(github.event.inputs.skiptests, 'redis')
run: ./runtest --valgrind --no-latency --verbose --clients 1 --timeout 2400 --dump-logs ${{github.event.inputs.test_args}}
@@ -634,7 +632,7 @@ jobs:
repository: ${{ env.GITHUB_REPOSITORY }}
ref: ${{ env.GITHUB_HEAD_REF }}
- name: make
run: make SANITIZER=undefined REDIS_CFLAGS='-DREDIS_TEST -Werror' LUA_DEBUG=yes # we (ab)use this flow to also check Lua C API violations
run: make SANITIZER=undefined REDIS_CFLAGS='-DREDIS_TEST -Werror' SKIP_VEC_SETS=yes LUA_DEBUG=yes # we (ab)use this flow to also check Lua C API violations
- name: testprep
run: |
sudo apt-get update
@@ -678,7 +676,7 @@ jobs:
ref: ${{ env.GITHUB_HEAD_REF }}
- name: make
run: |
dnf -y install which gcc make
dnf -y install which gcc make g++
make REDIS_CFLAGS='-Werror'
- name: testprep
run: |
@@ -720,7 +718,7 @@ jobs:
ref: ${{ env.GITHUB_HEAD_REF }}
- name: make
run: |
dnf -y install which gcc make openssl-devel openssl
dnf -y install which gcc make openssl-devel openssl g++
make BUILD_TLS=module REDIS_CFLAGS='-Werror'
- name: testprep
run: |
@@ -767,7 +765,7 @@ jobs:
ref: ${{ env.GITHUB_HEAD_REF }}
- name: make
run: |
dnf -y install which gcc make openssl-devel openssl
dnf -y install which gcc make openssl-devel openssl g++
make BUILD_TLS=module REDIS_CFLAGS='-Werror'
- name: testprep
run: |
@@ -875,7 +873,7 @@ jobs:
build-macos:
strategy:
matrix:
os: [macos-12, macos-14]
os: [macos-13, macos-15]
runs-on: ${{ matrix.os }}
if: |
(github.event_name == 'workflow_dispatch' || (github.event_name != 'workflow_dispatch' && github.repository == 'redis/redis')) &&
@@ -902,11 +900,14 @@ jobs:
run: make REDIS_CFLAGS='-Werror -DREDIS_TEST'
test-freebsd:
runs-on: macos-12
runs-on: macos-13
if: |
(github.event_name == 'workflow_dispatch' || (github.event_name != 'workflow_dispatch' && github.repository == 'redis/redis')) &&
!contains(github.event.inputs.skipjobs, 'freebsd')
timeout-minutes: 14400
env:
CC: clang
CXX: clang++
steps:
- name: prep
if: github.event_name == 'workflow_dispatch'
@@ -925,7 +926,7 @@ jobs:
version: 13.2
shell: bash
run: |
sudo pkg install -y bash gmake lang/tcl86 lang/tclx
sudo pkg install -y bash gmake lang/tcl86 lang/tclx gcc
gmake
./runtest --single unit/keyspace --single unit/auth --single unit/networking --single unit/protocol
@@ -1080,8 +1081,9 @@ jobs:
apt-key adv --keyserver keyserver.ubuntu.com --recv-keys 40976EAF437D05B5
apt-key adv --keyserver keyserver.ubuntu.com --recv-keys 3B4FE6ACC0B21F32
apt-get update
apt-get install -y make gcc-4.8
apt-get install -y make gcc-4.8 g++-4.8
update-alternatives --install /usr/bin/gcc gcc /usr/bin/gcc-4.8 100
update-alternatives --install /usr/bin/g++ g++ /usr/bin/g++-4.8 100
make CC=gcc REDIS_CFLAGS='-Werror'
- name: testprep
run: apt-get install -y tcl tcltls tclx
@@ -1128,9 +1130,10 @@ jobs:
apt-key adv --keyserver keyserver.ubuntu.com --recv-keys 40976EAF437D05B5
apt-key adv --keyserver keyserver.ubuntu.com --recv-keys 3B4FE6ACC0B21F32
apt-get update
apt-get install -y make gcc-4.8 openssl libssl-dev
apt-get install -y make gcc-4.8 g++-4.8 openssl libssl-dev
update-alternatives --install /usr/bin/gcc gcc /usr/bin/gcc-4.8 100
make CC=gcc BUILD_TLS=module REDIS_CFLAGS='-Werror'
update-alternatives --install /usr/bin/g++ g++ /usr/bin/g++-4.8 100
make CC=gcc CXX=g++ BUILD_TLS=module REDIS_CFLAGS='-Werror'
- name: testprep
run: |
apt-get install -y tcl tcltls tclx
@@ -1182,8 +1185,9 @@ jobs:
apt-key adv --keyserver keyserver.ubuntu.com --recv-keys 40976EAF437D05B5
apt-key adv --keyserver keyserver.ubuntu.com --recv-keys 3B4FE6ACC0B21F32
apt-get update
apt-get install -y make gcc-4.8 openssl libssl-dev
apt-get install -y make gcc-4.8 g++-4.8 openssl libssl-dev
update-alternatives --install /usr/bin/gcc gcc /usr/bin/gcc-4.8 100
update-alternatives --install /usr/bin/g++ g++ /usr/bin/g++-4.8 100
make BUILD_TLS=module CC=gcc REDIS_CFLAGS='-Werror'
- name: testprep
run: |
.github/workflows/external.yml
+3 -3
View File
@@ -27,7 +27,7 @@ jobs:
--tags -slow
- name: Archive redis log
if: ${{ failure() }}
uses: actions/upload-artifact@v3
uses: actions/upload-artifact@v4
with:
name: test-external-redis-log
path: external-redis.log
@@ -55,7 +55,7 @@ jobs:
--tags -slow
- name: Archive redis log
if: ${{ failure() }}
uses: actions/upload-artifact@v3
uses: actions/upload-artifact@v4
with:
name: test-external-cluster-log
path: external-redis-cluster.log
@@ -79,7 +79,7 @@ jobs:
--tags "-slow -needs:debug"
- name: Archive redis log
if: ${{ failure() }}
uses: actions/upload-artifact@v3
uses: actions/upload-artifact@v4
with:
name: test-external-redis-nodebug-log
path: external-redis-nodebug.log
.github/workflows/redis_docs_sync.yaml
+35
View File
@@ -0,0 +1,35 @@
name: redis_docs_sync
on:
release:
types: [published]
jobs:
redis_docs_sync:
if: github.repository == 'redis/redis'
runs-on: ubuntu-latest
steps:
- name: Generate a token
id: generate-token
uses: actions/create-github-app-token@v1
with:
app-id: ${{ secrets.DOCS_APP_ID }}
private-key: ${{ secrets.DOCS_APP_PRIVATE_KEY }}
- name: Invoke workflow on redis/docs
env:
GH_TOKEN: ${{ steps.generate-token.outputs.token }}
RELEASE_NAME: ${{ github.event.release.tag_name }}
run: |
LATEST_RELEASE=$(
curl -Ls \
-H "Accept: application/vnd.github+json" \
-H "Authorization: Bearer ${GH_TOKEN}" \
-H "X-GitHub-Api-Version: 2022-11-28" \
https://api.github.com/repos/redis/redis/releases/latest \
| jq -r '.tag_name'
)
if [[ "${LATEST_RELEASE}" == "${RELEASE_NAME}" ]]; then
gh workflow run -R redis/docs redis_docs_sync.yaml -f release="${RELEASE_NAME}"
fi
.gitignore
+1
View File
@@ -30,6 +30,7 @@ deps/lua/src/luac
deps/lua/src/liblua.a
deps/hdr_histogram/libhdrhistogram.a
deps/fpconv/libfpconv.a
deps/fast_float/libfast_float.a
tests/tls/*
.make-*
.prerequisites
00-RELEASENOTES
+293 -11
View File
@@ -1,16 +1,298 @@
Hello! This file is just a placeholder, since this is the "unstable" branch
of Redis, the place where all the development happens.
Redis Community Edition 8.0 release notes
=========================================
There is no release notes for this branch, it gets forked into another branch
every time there is a partial feature freeze in order to eventually create
a new stable release.
==========================================================
8.0-RC1 (v7.9.240) Released Mon 7 Apr 2025 10:00:00 IST
==========================================================
Usually "unstable" is stable enough for you to use it in development environments
however you should never use it in production environments. It is possible
to download the latest stable release here:
This is the first Release Candidate of Redis Community Edition 8.0.
https://download.redis.io/redis-stable.tar.gz
Release Candidates are feature-complete pre-releases. Pre-releases are not suitable for production use.
More information is available at https://redis.io
Happy hacking!
### Headlines
8.0-RC1 includes a new beta data structure - vector set.
### Distributions
- Alpine and Debian Docker images - https://hub.docker.com/_/redis
- Install using snap - see https://github.com/redis/redis-snap
- Install using brew - see https://github.com/redis/homebrew-redis
- Install using RPM and Debian APT - will be added on the GA release
### New Features
- #13915 Vector set - a new data structure [beta]:
Vector set extends the concept of sorted sets to allow the storage and querying of
high-dimensional vector embeddings, enhancing Redis for AI use cases that involve
semantic search and recommendation systems. Vector sets complement the existing
vector search capability in the Redis Query Engine. The vector set data type is
available in beta. We may change, or even break, the features and the API in
future versions. We are open to your feedback as you try out this new data type.
- #13846 Allow detecting incompatibility risks before switching to cluster mode
### Bug fixes
- #13895 RDB Channel replication - replica is online after BGSAVE is done
- #13877 Inconsistency for ShardID in case both master and replica support it
- #13883 Defrag scan may return nothing when type/encoding changes during it
- #13863 `RANDOMKEY` - infinite loop during client pause
- #13853 `SLAVEOF` - crash when clients are blocked on lazy free
- #13632 `XREAD` returns nil while stream is not empty
### Metrics
- #13846 `INFO`: `cluster_incompatible_ops` - number of cluster-incompatible commands
### Configuration parameters
- #13846 `cluster-compatibility-sample-ratio` - sampling ratio (0-100) for checking command compatibility with cluster mode
============================================================
8.0-M04 (v7.9.227) Committed Sun 16 Mar 2025 11:00:00 IST
============================================================
This is the fourth Milestone of Redis Community Edition 8.0.
Milestones are non-feature-complete pre-releases. Pre-releases are not suitable for production use.
Once we reach feature-completeness we will release RC1.
### Headlines
8.0-M04 includes 3 new hash commands, performance improvements, and memory defragmentation improvements.
### Distributions
- Alpine and Debian Docker images - https://hub.docker.com/_/redis
- Install using snap - see https://github.com/redis/redis-snap
- Install using brew - see https://github.com/redis/homebrew-redis
- Install using RPM and Debian APT - will be added on the GA release
### New Features
- #13798 Hash - new commands:
- `HGETDEL` Get and delete the value of one or more fields of a given hash key
- `HGETEX` Get the value of one or more fields of a given hash key, and optionally set their expiration
- `HSETEX` Set the value of one or more fields of a given hash key, and optionally set their expiration
- #13773 Add replication offset to AOF, allowing more robust way to determine which AOF has a more up-to-date data during recovery
- #13740, #13763 shared secret - new mechanism to allow sending internal commands between nodes
### Bug fixes
- #13804 Overflow on 32-bit systems when calculating idle time for eviction
- #13793 `WAITAOF` returns prematurely
- #13800 Remove `DENYOOM` from `HEXPIRE`, `HEXPIREAT`, `HPEXPIRE`, and `HPEXPIREAT`
- #13632 Streams - wrong behavior of `XREAD +` after last entry
### Modules API
- #13788 `RedisModule_LoadDefaultConfigs` - load module configuration values from redis.conf
- #13815 `RM_RegisterDefragFunc2` - support for incremental defragmentation of global module data
- #13816 `RM_DefragRedisModuleDict` - allow modules to defrag `RedisModuleDict`
- #13774 `RM_GetContextFlags` - add a `REDISMODULE_CTX_FLAGS_DEBUG_ENABLED` flag to execute debug commands
### Performance and resource utilization improvements
- #13752 Reduce defrag CPU usage when defragmentation is ineffective
- #13764 Reduce latency when a command is called consecutively
- #13787 Optimize parsing data from clients, specifically multi-bulk (array) data
- #13792 Optimize dictionary lookup by avoiding duplicate key length calculation during comparisons
- #13796 Optimize expiration checks
============================================================
8.0-M03 (v7.9.226) Committed Mon 20 Jan 2025 15:00:00 IST
============================================================
This is the third Milestone of Redis Community Edition 8.0.
Milestones are non-feature-complete pre-releases. Pre-releases are not suitable for production use.
Once we reach feature-completeness we will release RC1.
### Headlines:
8.0-M03 introduces an improved replication mechanism which is more performant and robust, a new I/O threading implementation which enables throughput increase on multi-core environments, and many additional performance improvements. Both Alpine and Debian Docker images are now available on [Docker Hub](https://hub.docker.com/_/redis). A snap and Homebrew distributions are available as well.
### Security fixes
- (CVE-2024-46981) Lua script may lead to remote code execution
- (CVE-2024-51741) Denial-of-service due to malformed ACL selectors
### New Features
- #13695 New I/O threading implementation
- #13732 New replication mechanism
### Bug fixes
- #13653 `MODULE LOADEX` - crash on nonexistent parameter name
- #13661 `FUNCTION FLUSH` - memory leak when using jemalloc
- #13626 Memory leak on failed RDB loading
### Other general improvements
- #13639 When `hide-user-data-from-log` is enabled - also print command tokens on crash
- #13660 Add the Lua VM memory to memory overhead
### New metrics
- #13592 `INFO` - new `KEYSIZES` section includes key size distributions for basic data types
- #13695 `INFO` - new `Threads` section includes I/O threading metrics
### Modules API
- #13666 `RedisModule_ACLCheckKeyPrefixPermissions` - check access permissions to any key matching a given prefix
- #13676 `RedisModule_HashFieldMinExpire` - query the minimum expiration time over all the hash’s fields
- #13676 `RedisModule_HashGet` - new `REDISMODULE_HASH_EXPIRE_TIME` flag - query the field expiration time
- #13656 `RedisModule_RegisterXXXConfig` - allow registering unprefixed configuration parameters
### Configuration parameters
- `replica-full-sync-buffer-limit` - maximum size of accumulated replication stream data on the replica side
- `io-threads-do-reads` is no longer effective. The new I/O threading implementation always use threads for both reads and writes
### Performance and resource utilization improvements
- #13638 Optimize CRC64 performance
- #13521 Optimize commands with large argument count - reuse c->argv after command execution
- #13558 Optimize `PFCOUNT` and `PFMERGE` - SIMD acceleration
- #13644 Optimize `GET` on high pipeline use-cases
- #13646 Optimize `EXISTS` - prefetching and branch prediction hints
- #13652 Optimize `LRANGE` - improve listpack handling and decoding efficiency
- #13655 Optimize `HSET` - avoid unnecessary hash field creation or deletion
- #13721 Optimize `LRANGE` and `HGETALL` - refactor client write preparation and handling
============================================================
8.0-M02 (v7.9.225) Committed Mon 28 Oct 2024 14:00:00 IST
============================================================
This is the second Milestone of Redis Community Edition 8.0.
Milestones are non-feature-complete pre-releases. Pre-releases are not suitable for production use.
Once we reach feature-completeness we will release RC1.
### Headlines:
8.0-M02 introduces significant performance improvements. Both Alpine and Debian Docker images are now available on [Docker Hub](https://hub.docker.com/_/redis). Additional distributions will be introduced in upcoming pre-releases.
### Supported upgrade paths (by replication or persistence) to 8.0-M02
- From previous Redis versions, without modules
The following upgrade paths (by replication or persistence) to 8.0-M02 are not yet tested and will be introduced in upcoming pre-releases:
- From previous Redis versions with modules (RediSearch, RedisJSON, RedisTimeSeries, RedisBloom)
- From Redis Stack 7.2 or 7.4
### Security fixes
- (CVE-2024-31449) Lua library commands may lead to stack overflow and potential RCE.
- (CVE-2024-31227) Potential Denial-of-service due to malformed ACL selectors.
- (CVE-2024-31228) Potential Denial-of-service due to unbounded pattern matching.
### Bug fixes
- #13539 Hash: Fix key ref for a hash that no longer has fields with expiration on `RENAME`/`MOVE`/`SWAPDB`/`RESTORE`
- #13512 Fix `TOUCH` command from a script in no-touch mode
- #13468 Cluster: Fix cluster node config corruption caused by mixing shard-id and non-shard-id versions
- #13608 Cluster: Fix `GET #` option in `SORT` command
### Modules API
- #13526 Extend `RedisModule_OpenKey` to read also expired keys and subkeys
### Performance and resource utilization improvements
- #11884 Optimize `ZADD` and `ZRANGE*` commands
- #13530 Optimize `SSCAN` command in case of listpack or intset encoding
- #13531 Optimize `HSCAN`/`ZSCAN` command in case of listpack encoding
- #13520 Optimize commands that heavily rely on bulk/mbulk replies (example of `LRANGE`)
- #13566 Optimize `ZUNION[STORE]` by avoiding redundant temporary dict usage
- #13567 Optimize `SUNION`/`SDIFF` commands by avoiding redundant temporary dict usage
- #11533 Avoid redundant `lpGet` to boost `quicklistCompare`
- #13412 Reduce redundant call of `prepareClientToWrite` when call `addReply*` continuously
===========================================================
8.0-M01 (v7.9.224) Released Thu 12 Sep 2024 10:00:00 IST
===========================================================
This is the first Milestone of Redis Community Edition 8.0.
Milestones are non-feature-complete pre-releases. Pre-releases are not suitable for production use.
Once we reach feature-completeness we will release RC1.
### Headlines:
Redis 8.0 introduces new data structures: JSON, time series, and 5 probabilistic data structures (previously available as separate Redis modules) and incorporates the enhanced Redis Query Enginer (with vector search).
8.0-M01 is available as a Docker image and can be downloaded from [Docker Hub](https://hub.docker.com/_/redis). Additional distributions will be introduced in upcoming pre-releases.
### Supported upgrade paths (by replication or persistence) to 8.0-M01
- From previous Redis versions, without modules
The following upgrade paths (by replication or persistence) to 8.0-M01 are not yet tested and will be introduced in upcoming pre-releases:
- From previous Redis versions with modules (RediSearch, RedisJSON, RedisTimeSeries, RedisBloom)
- From Redis Stack 7.2 or 7.4
### New Features in binary distributions
- 7 new data structures: JSON, Time series, Bloom filter, Cuckoo filter, Count-min sketch, Top-k, t-digest
- The enhanced Redis Query Engine (with vector search)
### Potentially breaking changes
- #12272 `GETRANGE` returns an empty bulk when the negative end index is out of range
- #12395 Optimize `SCAN` command when matching data type
### Bug fixes
- #13510 Fix `RM_RdbLoad` to enable AOF after RDB loading is completed
- #13489 `ACL CAT` - return module commands
- #13476 Fix a race condition in the `cache_memory` of `functionsLibCtx`
- #13473 Fix incorrect lag due to trimming stream via `XTRIM` command
- #13338 Fix incorrect lag field in `XINFO` when tombstone is after the `last_id` of the consume group
- #13470 On `HDEL` of last field - update the global hash field expiration data structure
- #13465 Cluster: Pass extensions to node if extension processing is handled by it
- #13443 Cluster: Ensure validity of myself when loading cluster config
- #13422 Cluster: Fix `CLUSTER SHARDS` command returns empty array
### Modules API
- #13509 New API calls: `RM_DefragAllocRaw`, `RM_DefragFreeRaw`, and `RM_RegisterDefragCallbacks` - defrag API to allocate and free raw memory
### Performance and resource utilization improvements
- #13503 Avoid overhead of comparison function pointer calls in listpack `lpFind`
- #13505 Optimize `STRING` datatype write commands
- #13499 Optimize `SMEMBERS` command
- #13494 Optimize `GEO*` commands reply
- #13490 Optimize `HELLO` command
- #13488 Optimize client query buffer
- #12395 Optimize `SCAN` command when matching data type
- #13529 Optimize `LREM`, `LPOS`, `LINSERT`, and `LINDEX` commands
- #13516 Optimize `LRANGE` and other commands that perform several writes to client buffers per call
- #13431 Avoid `used_memory` contention when updating from multiple threads
### Other general improvements
- #13495 Reply `-LOADING` on replica while flushing the db
### CLI tools
- #13411 redis-cli: Fix wrong `dbnum` showed after the client reconnected
### Notes
- No backward compatibility for replication or persistence.
- Additional distributions, upgrade paths, features, and improvements will be introduced in upcoming pre-releases.
- With the GA release of 8.0 we will deprecate Redis Stack.
Makefile
+8 -3
View File
@@ -1,11 +1,16 @@
# Top level makefile, the real shit is at src/Makefile
# Top level makefile, the real stuff is at ./src/Makefile and in ./modules/Makefile
SUBDIRS = src
ifeq ($(BUILD_WITH_MODULES), yes)
SUBDIRS += modules
endif
default: all
.DEFAULT:
cd src && $(MAKE) $@
for dir in $(SUBDIRS); do $(MAKE) -C $$dir $@; done
install:
cd src && $(MAKE) $@
for dir in $(SUBDIRS); do $(MAKE) -C $$dir $@; done
.PHONY: install
README.md
+5 -3
View File
@@ -1,3 +1,5 @@
[![codecov](https://codecov.io/github/redis/redis/graph/badge.svg?token=6bVHb5fRuz)](https://codecov.io/github/redis/redis)
This README is just a fast *quick start* document. You can find more detailed documentation at [redis.io](https://redis.io).
What is Redis?
@@ -15,8 +17,8 @@ Another good example is to think of Redis as a more complex version of memcached
If you want to know more, this is a list of selected starting points:
* Introduction to Redis data types. https://redis.io/topics/data-types-intro
* Try Redis directly inside your browser. https://try.redis.io
* Introduction to Redis data types. https://redis.io/docs/latest/develop/data-types/
* The full list of Redis commands. https://redis.io/commands
* There is much more inside the official Redis documentation. https://redis.io/documentation
@@ -494,7 +496,7 @@ Other C files
* `dict.c` is an implementation of a non-blocking hash table which rehashes incrementally.
* `cluster.c` implements the Redis Cluster. Probably a good read only after being very familiar with the rest of the Redis code base. If you want to read `cluster.c` make sure to read the [Redis Cluster specification][4].
[4]: https://redis.io/topics/cluster-spec
[4]: https://redis.io/docs/latest/operate/oss_and_stack/reference/cluster-spec/
Anatomy of a Redis command
---
codecov.yml
+19
View File
@@ -0,0 +1,19 @@
coverage:
status:
patch:
default:
informational: true
project:
default:
informational: true
comment:
require_changes: false
require_head: false
require_base: false
layout: "condensed_header, diff, files"
hide_project_coverage: false
behavior: default
github_checks:
annotations: false
deps/Makefile
Vendored
+7
View File
@@ -42,6 +42,7 @@ distclean:
-(cd jemalloc && [ -f Makefile ] && $(MAKE) distclean) > /dev/null || true
-(cd hdr_histogram && $(MAKE) clean) > /dev/null || true
-(cd fpconv && $(MAKE) clean) > /dev/null || true
-(cd fast_float && $(MAKE) clean) > /dev/null || true
-(rm -f .make-*)
.PHONY: distclean
@@ -74,6 +75,12 @@ fpconv: .make-prerequisites
.PHONY: fpconv
fast_float: .make-prerequisites
@printf '%b %b\n' $(MAKECOLOR)MAKE$(ENDCOLOR) $(BINCOLOR)$@$(ENDCOLOR)
cd fast_float && $(MAKE) libfast_float
.PHONY: fast_float
ifeq ($(uname_S),SunOS)
# Make isinf() available
LUA_CFLAGS= -D__C99FEATURES__=1
deps/fast_float/Makefile
Vendored
+24
View File
@@ -0,0 +1,24 @@
# Fallback to gcc/g++ when $CC or $CXX is not in $PATH.
CC ?= gcc
CXX ?= g++
CFLAGS=-Wall -O3
# This avoids loosing the fastfloat specific compile flags when we override the CFLAGS via the main project
FASTFLOAT_CFLAGS=-std=c++11 -DFASTFLOAT_ALLOWS_LEADING_PLUS
LDFLAGS=
libfast_float: fast_float_strtod.o
$(AR) -r libfast_float.a fast_float_strtod.o
32bit: CFLAGS += -m32
32bit: LDFLAGS += -m32
32bit: libfast_float
fast_float_strtod.o: fast_float_strtod.cpp
$(CXX) $(CFLAGS) $(FASTFLOAT_CFLAGS) -c fast_float_strtod.cpp $(LDFLAGS)
clean:
rm -f *.o
rm -f *.a
rm -f *.h.gch
rm -rf *.dSYM
deps/fast_float/README.md
Vendored
+21
View File
@@ -0,0 +1,21 @@
README for fast_float v6.1.4
----------------------------------------------
We're using the fast_float library[1] in our (compiled-in)
floating-point fast_float_strtod implementation for faster and more
portable parsing of 64 decimal strings.
The single file fast_float.h is an amalgamation of the entire library,
which can be (re)generated with the amalgamate.py script (from the
fast_float repository) via the command
```
git clone https://github.com/fastfloat/fast_float
cd fast_float
git checkout v6.1.4
python3 ./script/amalgamate.py --license=MIT \
> $REDIS_SRC/deps/fast_float/fast_float.h
```
[1]: https://github.com/fastfloat/fast_float
deps/fast_float/fast_float.h
Vendored
+3838
View File
File diff suppressed because it is too large Load Diff
deps/fast_float/fast_float_strtod.cpp
Vendored
+32
View File
@@ -0,0 +1,32 @@
#include "fast_float.h"
#include <iostream>
#include <string>
#include <system_error>
#include <cerrno>
/* Convert NPTR to a double using the fast_float library.
*
* This function behaves similarly to the standard strtod function, converting
* the initial portion of the string pointed to by `nptr` to a `double` value,
* using the fast_float library for high performance. If the conversion fails,
* errno is set to EINVAL error code.
*
* @param nptr A pointer to the null-terminated byte string to be interpreted.
* @param endptr A pointer to a pointer to character. If `endptr` is not NULL,
* it will point to the character after the last character used
* in the conversion.
* @return The converted value as a double. If no valid conversion could
* be performed, returns 0.0.
* If ENDPTR is not NULL, a pointer to the character after the last one used
* in the number is put in *ENDPTR. */
extern "C" double fast_float_strtod(const char *nptr, char **endptr) {
double result = 0.0;
auto answer = fast_float::from_chars(nptr, nptr + strlen(nptr), result);
if (answer.ec != std::errc()) {
errno = EINVAL; // Fallback to for other errors
}
if (endptr != NULL) {
*endptr = (char *)answer.ptr;
}
return result;
}
deps/fast_float/fast_float_strtod.h
Vendored
+15
View File
@@ -0,0 +1,15 @@
#ifndef __FAST_FLOAT_STRTOD_H__
#define __FAST_FLOAT_STRTOD_H__
#if defined(__cplusplus)
extern "C"
{
#endif
double fast_float_strtod(const char *in, char **out);
#if defined(__cplusplus)
}
#endif
#endif /* __FAST_FLOAT_STRTOD_H__ */
deps/hiredis/async.c
Vendored
+1 -1
View File
@@ -478,7 +478,7 @@ static int __redisGetSubscribeCallback(redisAsyncContext *ac, redisReply *reply,
/* Match reply with the expected format of a pushed message.
* The type and number of elements (3 to 4) are specified at:
* https://redis.io/topics/pubsub#format-of-pushed-messages */
* https://redis.io/docs/latest/develop/interact/pubsub/#format-of-pushed-messages */
if ((reply->type == REDIS_REPLY_ARRAY && !(c->flags & REDIS_SUPPORTS_PUSH) && reply->elements >= 3) ||
reply->type == REDIS_REPLY_PUSH) {
assert(reply->element[0]->type == REDIS_REPLY_STRING);
deps/linenoise/linenoise.c
Vendored
+1 -1
View File
@@ -296,7 +296,7 @@ static int isUnsupportedTerm(void) {
return 0;
}
/* Raw mode: 1960 magic shit. */
/* Raw mode: 1960's magic. */
static int enableRawMode(int fd) {
if (getenv("FAKETTY_WITH_PROMPT") != NULL) {
return 0;
deps/lua/src/lua_bit.c
Vendored
+1
View File
@@ -132,6 +132,7 @@ static int bit_tohex(lua_State *L)
const char *hexdigits = "0123456789abcdef";
char buf[8];
int i;
if (n == INT32_MIN) n = INT32_MIN+1;
if (n < 0) { n = -n; hexdigits = "0123456789ABCDEF"; }
if (n > 8) n = 8;
for (i = (int)n; --i >= 0; ) { buf[i] = hexdigits[b & 15]; b >>= 4; }
modules/Makefile
+90
View File
@@ -0,0 +1,90 @@
SUBDIRS = redisjson redistimeseries redisbloom redisearch
define submake
for dir in $(SUBDIRS); do $(MAKE) -C $$dir $(1); done
endef
all: prepare_source
$(call submake,$@)
get_source:
$(call submake,$@)
prepare_source: get_source handle-werrors setup_environment
clean:
$(call submake,$@)
distclean: clean_environment
$(call submake,$@)
pristine:
$(call submake,$@)
install:
$(call submake,$@)
setup_environment: install-rust handle-werrors
clean_environment: uninstall-rust
# Keep all of the Rust stuff in one place
install-rust:
ifeq ($(INSTALL_RUST_TOOLCHAIN),yes)
@RUST_VERSION=1.80.1; \
ARCH="$$(uname -m)"; \
if ldd --version 2>&1 | grep -q musl; then LIBC_TYPE="musl"; else LIBC_TYPE="gnu"; fi; \
echo "Detected architecture: $${ARCH} and libc: $${LIBC_TYPE}"; \
case "$${ARCH}" in \
'x86_64') \
if [ "$${LIBC_TYPE}" = "musl" ]; then \
RUST_INSTALLER="rust-$${RUST_VERSION}-x86_64-unknown-linux-musl"; \
RUST_SHA256="37bbec6a7b9f55fef79c451260766d281a7a5b9d2e65c348bbc241127cf34c8d"; \
else \
RUST_INSTALLER="rust-$${RUST_VERSION}-x86_64-unknown-linux-gnu"; \
RUST_SHA256="85e936d5d36970afb80756fa122edcc99bd72a88155f6bdd514f5d27e778e00a"; \
fi ;; \
'aarch64') \
if [ "$${LIBC_TYPE}" = "musl" ]; then \
RUST_INSTALLER="rust-$${RUST_VERSION}-aarch64-unknown-linux-musl"; \
RUST_SHA256="dd668c2d82f77c5458deb023932600fae633fff8d7f876330e01bc47e9976d17"; \
else \
RUST_INSTALLER="rust-$${RUST_VERSION}-aarch64-unknown-linux-gnu"; \
RUST_SHA256="2e89bad7857711a1c11d017ea28fbfeec54076317763901194f8f5decbac1850"; \
fi ;; \
*) echo >&2 "Unsupported architecture: '$${ARCH}'"; exit 1 ;; \
esac; \
echo "Downloading and installing Rust standalone installer: $${RUST_INSTALLER}"; \
wget --quiet -O $${RUST_INSTALLER}.tar.xz https://static.rust-lang.org/dist/$${RUST_INSTALLER}.tar.xz; \
echo "$${RUST_SHA256} $${RUST_INSTALLER}.tar.xz" | sha256sum -c --quiet || { echo "Rust standalone installer checksum failed!"; exit 1; }; \
tar -xf $${RUST_INSTALLER}.tar.xz; \
(cd $${RUST_INSTALLER} && ./install.sh); \
rm -rf $${RUST_INSTALLER}
endif
uninstall-rust:
ifeq ($(INSTALL_RUST_TOOLCHAIN),yes)
@if [ -x "/usr/local/lib/rustlib/uninstall.sh" ]; then \
echo "Uninstalling Rust using uninstall.sh script"; \
rm -rf ~/.cargo; \
/usr/local/lib/rustlib/uninstall.sh; \
else \
echo "WARNING: Rust toolchain not found or uninstall script is missing."; \
fi
endif
handle-werrors: get_source
ifeq ($(DISABLE_WERRORS),yes)
@echo "Disabling -Werror for all modules"
@for dir in $(SUBDIRS); do \
echo "Processing $$dir"; \
find $$dir/src -type f \
\( -name "Makefile" \
-o -name "*.mk" \
-o -name "CMakeLists.txt" \) \
-exec sed -i 's/-Werror//g' {} +; \
done
endif
.PHONY: all clean distclean install $(SUBDIRS) setup_environment clean_environment install-rust uninstall-rust handle-werrors
modules/common.mk
+51
View File
@@ -0,0 +1,51 @@
PREFIX ?= /usr/local
INSTALL_DIR ?= $(DESTDIR)$(PREFIX)/lib/redis/modules
INSTALL ?= install
# This logic *partially* follows the current module build system. It is a bit awkward and
# should be changed if/when the modules' build process is refactored.
ARCH_MAP_x86_64 := x64
ARCH_MAP_i386 := x86
ARCH_MAP_i686 := x86
ARCH_MAP_aarch64 := arm64v8
ARCH_MAP_arm64 := arm64v8
OS := $(shell uname -s | tr '[:upper:]' '[:lower:]')
ARCH := $(ARCH_MAP_$(shell uname -m))
ifeq ($(ARCH),)
$(error Unrecognized CPU architecture $(shell uname -m))
endif
FULL_VARIANT := $(OS)-$(ARCH)-release
# Common rules for all modules, based on per-module configuration
all: $(TARGET_MODULE)
$(TARGET_MODULE): get_source
$(MAKE) -C $(SRC_DIR)
cp ${TARGET_MODULE} ./
get_source: $(SRC_DIR)/.prepared
$(SRC_DIR)/.prepared:
mkdir -p $(SRC_DIR)
git clone --recursive --depth 1 --branch $(MODULE_VERSION) $(MODULE_REPO) $(SRC_DIR)
touch $@
clean:
-$(MAKE) -C $(SRC_DIR) clean
-rm -f ./*.so
distclean: clean
-$(MAKE) -C $(SRC_DIR) distclean
pristine:
-rm -rf $(SRC_DIR)
install: $(TARGET_MODULE)
mkdir -p $(INSTALL_DIR)
$(INSTALL) -m 0755 -D $(TARGET_MODULE) $(INSTALL_DIR)
.PHONY: all clean distclean pristine install
modules/redisbloom/Makefile
+6
View File
@@ -0,0 +1,6 @@
SRC_DIR = src
MODULE_VERSION = v7.99.90
MODULE_REPO = https://github.com/redisbloom/redisbloom
TARGET_MODULE = $(SRC_DIR)/bin/$(FULL_VARIANT)/redisbloom.so
include ../common.mk
modules/redisearch/Makefile
+7
View File
@@ -0,0 +1,7 @@
SRC_DIR = src
MODULE_VERSION = v7.99.90
MODULE_REPO = https://github.com/redisearch/redisearch
TARGET_MODULE = $(SRC_DIR)/bin/$(FULL_VARIANT)/search-community/redisearch.so
include ../common.mk
modules/redisjson/Makefile
+11
View File
@@ -0,0 +1,11 @@
SRC_DIR = src
MODULE_VERSION = v7.99.90
MODULE_REPO = https://github.com/redisjson/redisjson
TARGET_MODULE = $(SRC_DIR)/bin/$(FULL_VARIANT)/rejson.so
include ../common.mk
$(SRC_DIR)/.cargo_fetched:
cd $(SRC_DIR) && cargo fetch
get_source: $(SRC_DIR)/.cargo_fetched
modules/redistimeseries/Makefile
+6
View File
@@ -0,0 +1,6 @@
SRC_DIR = src
MODULE_VERSION = v7.99.90
MODULE_REPO = https://github.com/redistimeseries/redistimeseries
TARGET_MODULE = $(SRC_DIR)/bin/$(FULL_VARIANT)/redistimeseries.so
include ../common.mk
modules/vector-sets/.gitignore
+11
View File
@@ -0,0 +1,11 @@
__pycache__
misc
*.so
*.xo
*.o
.DS_Store
w2v
word2vec.bin
TODO
*.txt
*.rdb
modules/vector-sets/Makefile
+84
View File
@@ -0,0 +1,84 @@
# Compiler settings
CC = cc
ifdef SANITIZER
ifeq ($(SANITIZER),address)
SAN=-fsanitize=address
else
ifeq ($(SANITIZER),undefined)
SAN=-fsanitize=undefined
else
ifeq ($(SANITIZER),thread)
SAN=-fsanitize=thread
else
$(error "unknown sanitizer=${SANITIZER}")
endif
endif
endif
endif
CFLAGS = -O2 -Wall -Wextra -g $(SAN) -std=c11
LDFLAGS = -lm $(SAN)
# Detect OS
uname_S := $(shell sh -c 'uname -s 2>/dev/null || echo not')
uname_M := $(shell sh -c 'uname -m 2>/dev/null || echo not')
# Shared library compile flags for linux / osx
ifeq ($(uname_S),Linux)
SHOBJ_CFLAGS ?= -W -Wall -fno-common -g -ggdb -std=c11 -O2
SHOBJ_LDFLAGS ?= -shared
ifneq (,$(findstring armv,$(uname_M)))
SHOBJ_LDFLAGS += -latomic
endif
ifneq (,$(findstring aarch64,$(uname_M)))
SHOBJ_LDFLAGS += -latomic
endif
else
SHOBJ_CFLAGS ?= -W -Wall -dynamic -fno-common -g -ggdb -std=c11 -O3
SHOBJ_LDFLAGS ?= -bundle -undefined dynamic_lookup
endif
# OS X 11.x doesn't have /usr/lib/libSystem.dylib and needs an explicit setting.
ifeq ($(uname_S),Darwin)
ifeq ("$(wildcard /usr/lib/libSystem.dylib)","")
LIBS = -L /Library/Developer/CommandLineTools/SDKs/MacOSX.sdk/usr/lib -lsystem
endif
endif
.SUFFIXES: .c .so .xo .o
all: vset.so
.c.xo:
$(CC) -I. $(CFLAGS) $(SHOBJ_CFLAGS) -fPIC -c $< -o $@
vset.xo: ../../src/redismodule.h expr.c
vset.so: vset.xo hnsw.xo cJSON.xo
$(CC) -o $@ $^ $(SHOBJ_LDFLAGS) $(LIBS) $(SAN) -lc
# Example sources / objects
SRCS = hnsw.c w2v.c
OBJS = $(SRCS:.c=.o)
TARGET = w2v
MODULE = vset.so
# Default target
all: $(TARGET) $(MODULE)
# Example linking rule
$(TARGET): $(OBJS)
$(CC) $(OBJS) $(LDFLAGS) -o $(TARGET)
# Compilation rule for object files
%.o: %.c
$(CC) $(CFLAGS) -c $< -o $@
# Clean rule
clean:
rm -f $(TARGET) $(OBJS) *.xo *.so
# Declare phony targets
.PHONY: all clean
modules/vector-sets/README.md
+633
View File
@@ -0,0 +1,633 @@
This module implements Vector Sets for Redis, a new Redis data type similar
to Sorted Sets but having string elements associated to a vector instead of
a score. The fundamental goal of Vector Sets is to make possible adding items,
and later get a subset of the added items that are the most similar to a
specified vector (often a learned embedding), or the most similar to the vector
of an element that is already part of the Vector Set.
Moreover, Vector sets implement optional filtered search capabilities: it is possible to associate attributes to all or to a subset of elements in the set, and then, using the `FILTER` option of the `VSIM` command, to ask for items similar to a given vector but also passing a filter specified as a simple mathematical expression (Like `".year > 1950"` or similar). This means that **you can have vector similarity and scalar filters at the same time**.
## Installation
Build with:
make
Then load the module with the following command line, or by inserting the needed directives in the `redis.conf` file.
./redis-server --loadmodule vset.so
To run tests, I suggest using this:
./redis-server --save "" --enable-debug-command yes
The execute the tests with:
./test.py
## Reference of available commands
**VADD: add items into a vector set**
VADD key [REDUCE dim] FP32|VALUES vector element [CAS] [NOQUANT | Q8 | BIN]
[EF build-exploration-factor] [SETATTR <attributes>] [M <numlinks>]
Add a new element into the vector set specified by the key.
The vector can be provided as FP32 blob of values, or as floating point
numbers as strings, prefixed by the number of elements (3 in the example):
VADD mykey VALUES 3 0.1 1.2 0.5 my-element
Meaning of the options:
`REDUCE` implements random projection, in order to reduce the
dimensionality of the vector. The projection matrix is saved and reloaded
along with the vector set. **Please note that** the `REDUCE` option must be passed immediately before the vector, like in `REDUCE 50 VALUES ...`.
`CAS` performs the operation partially using threads, in a
check-and-set style. The neighbor candidates collection, which is slow, is
performed in the background, while the command is executed in the main thread.
`NOQUANT` forces the vector to be created (in the first VADD call to a given key) without integer 8 quantization, which is otherwise the default.
`BIN` forces the vector to use binary quantization instead of int8. This is much faster and uses less memory, but has impacts on the recall quality.
`Q8` forces the vector to use signed 8 bit quantization. This is the default, and the option only exists in order to make sure to check at insertion time if the vector set is of the same format.
`EF` plays a role in the effort made to find good candidates when connecting the new node to the existing HNSW graph. The default is 200. Using a larger value, may help to have a better recall. To improve the recall it is also possible to increase `EF` during `VSIM` searches.
`SETATTR` associates attributes to the newly created entry or update the entry attributes (if it already exists). It is the same as calling the `VSETATTR` attribute separately, so please check the documentation of that command in the filtered search section of this documentation.
`M` defaults to 16 and is the HNSW famous `M` parameters. It is the maximum number of connections that each node of the graph have with other nodes: more connections mean more memory, but a better ability to explore the graph. Nodes at layer zero (every node exists at least at layer zero) have `M*2` connections, while the other layers only have `M` connections. This means that, for instance, an `M` of 64 will use at least 1024 bytes of memory for each node! That is, `64 links * 2 times * 8 bytes pointers`, and even more, since on average each node has something like 1.33 layers (but the other layers have just `M` connections, instead of `M*2`). If you don't have a recall quality problem, the default is fine, and uses a limited amount of memory.
**VSIM: return elements by vector similarity**
VSIM key [ELE|FP32|VALUES] <vector or element> [WITHSCORES] [COUNT num] [EF search-exploration-factor] [FILTER expression] [FILTER-EF max-filtering-effort] [TRUTH] [NOTHREAD]
The command returns similar vectors, for simplicity (and verbosity) in the following example, instead of providing a vector using FP32 or VALUES (like in `VADD`), we will ask for elements having a vector similar to a given element already in the sorted set:
> VSIM word_embeddings ELE apple
1) "apple"
2) "apples"
3) "pear"
4) "fruit"
5) "berry"
6) "pears"
7) "strawberry"
8) "peach"
9) "potato"
10) "grape"
It is possible to specify a `COUNT` and also to get the similarity score (from 1 to 0, where 1 is identical, 0 is opposite vector) between the query and the returned items.
> VSIM word_embeddings ELE apple WITHSCORES COUNT 3
1) "apple"
2) "0.9998867657923256"
3) "apples"
4) "0.8598527610301971"
5) "pear"
6) "0.8226882219314575"
The `EF` argument is the exploration factor: the higher it is, the slower the command becomes, but the better the index is explored to find nodes that are near to our query. Sensible values are from 50 to 1000.
The `TRUTH` option forces the command to perform a linear scan of all the entries inside the set, without using the graph search inside the HNSW, so it returns the best matching elements (the perfect result set) that can be used in order to easily calculate the recall. Of course the linear scan is `O(N)`, so it is much slower than the `log(N)` (considering a small `COUNT`) provided by the HNSW index.
The `NOTHREAD` option forces the command to execute the search on the data structure in the main thread. Normally `VSIM` spawns a thread instead. This may be useful for benchmarking purposes, or when we work with extremely small vector sets and don't want to pay the cost of spawning a thread. It is possible that in the future this option will be automatically used by Redis when we detect small vector sets. Note that this option blocks the server for all the time needed to complete the command, so it is a source of potential latency issues: if you are in doubt, never use it.
For `FILTER` and `FILTER-EF` options, please check the filtered search section of this documentation.
**VDIM: return the dimension of the vectors inside the vector set**
VDIM keyname
Example:
> VDIM word_embeddings
(integer) 300
Note that in the case of vectors that were populated using the `REDUCE`
option, for random projection, the vector set will report the size of
the projected (reduced) dimension. Yet the user should perform all the
queries using full-size vectors.
**VCARD: return the number of elements in a vector set**
VCARD key
Example:
> VCARD word_embeddings
(integer) 3000000
**VREM: remove elements from vector set**
VREM key element
Example:
> VADD vset VALUES 3 1 0 1 bar
(integer) 1
> VREM vset bar
(integer) 1
> VREM vset bar
(integer) 0
VREM does not perform thumstone / logical deletion, but will actually reclaim
the memory from the vector set, so it is save to add and remove elements
in a vector set in the context of long running applications that continuously
update the same index.
**VEMB: return the approximated vector of an element**
VEMB key element
Example:
> VEMB word_embeddings SQL
1) "0.18208661675453186"
2) "0.08535309880971909"
3) "0.1365649551153183"
4) "-0.16501599550247192"
5) "0.14225517213344574"
... 295 more elements ...
Because vector sets perform insertion time normalization and optional
quantization, the returned vector could be approximated. `VEMB` will take
care to de-quantized and de-normalize the vector before returning it.
It is possible to ask VEMB to return raw data, that is, the internal representation used by the vector: fp32, int8, or a bitmap for binary quantization. This behavior is triggered by the `RAW` option of of VEMB:
VEMB word_embedding apple RAW
In this case the return value of the command is an array of three or more elements:
1. The name of the quantization used, that is one of: "fp32", "bin", "q8".
2. The a string blob containing the raw data, 4 bytes fp32 floats for fp32, a bitmap for binary quants, or int8 bytes array for q8 quants.
3. A float representing the l2 of the vector before normalization. You need to multiply by this vector if you want to de-normalize the value for any reason.
For q8 quantization, an additional elements is also returned: the quantization
range, so the integers from -127 to 127 represent (normalized) components
in the range `-range`, `+range`.
**VLINKS: introspection command that shows neighbors for a node**
VLINKS key element [WITHSCORES]
The command reports the neighbors for each level.
**VINFO: introspection command that shows info about a vector set**
VINFO key
Example:
> VINFO word_embeddings
1) quant-type
2) int8
3) vector-dim
4) (integer) 300
5) size
6) (integer) 3000000
7) max-level
8) (integer) 12
9) vset-uid
10) (integer) 1
11) hnsw-max-node-uid
12) (integer) 3000000
**VSETATTR: associate or remove the JSON attributes of elements**
VSETATTR key element "{... json ...}"
Each element of a vector set can be optionally associated with a JSON string
in order to use the `FILTER` option of `VSIM` to filter elements by scalars
(see the filtered search section for more information). This command can set,
update (if already set) or delete (if you set to an empty string) the
associated JSON attributes of an element.
The command returns 0 if the element or the key don't exist, without
raising an error, otherwise 1 is returned, and the element attributes
are set or updated.
**VGETATTR: retrieve the JSON attributes of elements**
VGETATTR key element
The command returns the JSON attribute associated with an element, or
null if there is no element associated, or no element at all, or no key.
**VRANDMEMBER: return random members from a vector set**
VRANDMEMBER key [count]
Return one or more random elements from a vector set.
The semantics of this command are similar to Redis's native SRANDMEMBER command:
- When called without count, returns a single random element from the set, as a single string (no array reply).
- When called with a positive count, returns up to count distinct random elements (no duplicates).
- When called with a negative count, returns count random elements, potentially with duplicates.
- If the count value is larger than the set size (and positive), only the entire set is returned.
If the key doesn't exist, returns a Null reply if count is not given, or an empty array if a count is provided.
Examples:
> VADD vset VALUES 3 1 0 0 elem1
(integer) 1
> VADD vset VALUES 3 0 1 0 elem2
(integer) 1
> VADD vset VALUES 3 0 0 1 elem3
(integer) 1
# Return a single random element
> VRANDMEMBER vset
"elem2"
# Return 2 distinct random elements
> VRANDMEMBER vset 2
1) "elem1"
2) "elem3"
# Return 3 random elements with possible duplicates
> VRANDMEMBER vset -3
1) "elem2"
2) "elem2"
3) "elem1"
# Return more elements than in the set (returns all elements)
> VRANDMEMBER vset 10
1) "elem1"
2) "elem2"
3) "elem3"
# When key doesn't exist
> VRANDMEMBER nonexistent
(nil)
> VRANDMEMBER nonexistent 3
(empty array)
This command is particularly useful for:
1. Selecting random samples from a vector set for testing or training.
2. Performance testing by retrieving random elements for subsequent similarity searches.
When the user asks for unique elements (positev count) the implementation optimizes for two scenarios:
- For small sample sizes (less than 20% of the set size), it uses a dictionary to avoid duplicates, and performs a real random walk inside the graph.
- For large sample sizes (more than 20% of the set size), it starts from a random node and sequentially traverses the internal list, providing faster performances but not really "random" elements.
The command has `O(N)` worst-case time complexity when requesting many unique elements (it uses linear scanning), or `O(M*log(N))` complexity when the users asks for `M` random elements in a sorted set of `N` elements, with `M` much smaller than `N`.
# Filtered search
Each element of the vector set can be associated with a set of attributes specified as a JSON blob:
> VADD vset VALUES 3 1 1 1 a SETATTR '{"year": 1950}'
(integer) 1
> VADD vset VALUES 3 -1 -1 -1 b SETATTR '{"year": 1951}'
(integer) 1
Specifying an attribute with the `SETATTR` option of `VADD` is exactly equivalent to adding an element and then setting (or updating, if already set) the attributes JSON string. Also the symmetrical `VGETATTR` command returns the attribute associated to a given element.
> VADD vset VALUES 3 0 1 0 c
(integer) 1
> VSETATTR vset c '{"year": 1952}'
(integer) 1
> VGETATTR vset c
"{\"year\": 1952}"
At this point, I may use the FILTER option of VSIM to only ask for the subset of elements that are verified by my expression:
> VSIM vset VALUES 3 0 0 0 FILTER '.year > 1950'
1) "c"
2) "b"
The items will be returned again in order of similarity (most similar first), but only the items with the year field matching the expression is returned.
The expressions are similar to what you would write inside the `if` statement of JavaScript or other familiar programming languages: you can use `and`, `or`, the obvious math operators like `+`, `-`, `/`, `>=`, `<`, ... and so forth (see the expressions section for more info). The selectors of the JSON object attributes start with a dot followed by the name of the key inside the JSON objects.
Elements with invalid JSON or not having a given specified field **are considered as not matching** the expression, but will not generate any error at runtime.
## FILTER expressions capabilities
FILTER expressions allow you to perform complex filtering on vector similarity results using a JavaScript-like syntax. The expression is evaluated against each element's JSON attributes, with only elements that satisfy the expression being included in the results.
### Expression Syntax
Expressions support the following operators and capabilities:
1. **Arithmetic operators**: `+`, `-`, `*`, `/`, `%` (modulo), `**` (exponentiation)
2. **Comparison operators**: `>`, `>=`, `<`, `<=`, `==`, `!=`
3. **Logical operators**: `and`/`&&`, `or`/`||`, `!`/`not`
4. **Containment operator**: `in`
5. **Parentheses** for grouping: `(...)`
### Selector Notation
Attributes are accessed using dot notation:
- `.year` references the "year" attribute
- `.movie.year` would **NOT** reference the "year" field inside a "movie" object, only keys that are at the first level of the JSON object are accessible.
### JSON and expressions data types
Expressions can work with:
- Numbers (dobule precision floats)
- Strings (enclosed in single or double quotes)
- Booleans (no native type: they are represented as 1 for true, 0 for false)
- Arrays (for use with the `in` operator: `value in [1, 2, 3]`)
JSON attributes are converted in this way:
- Numbers will be converted to numbers.
- Strings to strings.
- Booleans to 0 or 1 number.
- Arrays to tuples (for "in" operator), but only if composed of just numbers and strings.
Any other type is ignored, and accessig it will make the expression evaluate to false.
### Examples
```
# Find items from the 1980s
VSIM movies VALUES 3 0.5 0.8 0.2 FILTER '.year >= 1980 and .year < 1990'
# Find action movies with high ratings
VSIM movies VALUES 3 0.5 0.8 0.2 FILTER '.genre == "action" and .rating > 8.0'
# Find movies directed by either Spielberg or Nolan
VSIM movies VALUES 3 0.5 0.8 0.2 FILTER '.director in ["Spielberg", "Nolan"]'
# Complex condition with numerical operations
VSIM movies VALUES 3 0.5 0.8 0.2 FILTER '(.year - 2000) ** 2 < 100 and .rating / 2 > 4'
```
### Error Handling
Elements with any of the following conditions are considered not matching:
- Missing the queried JSON attribute
- Having invalid JSON in their attributes
- Having a JSON value that cannot be converted to the expected type
This behavior allows you to safely filter on optional attributes without generating errors.
### FILTER effort
The `FILTER-EF` option controls the maximum effort spent when filtering vector search results.
When performing vector similarity search with filtering, Vector Sets perform the standard similarity search as they apply the filter expression to each node. Since many results might be filtered out, Vector Sets may need to examine a lot more candidates than the requested `COUNT` to ensure sufficient matching results are returned. Actually, if the elements matching the filter are very rare or if there are less than elements matching than the specified count, this would trigger a full scan of the HNSW graph.
For this reason, by default, the maximum effort is limited to a reasonable amount of nodes explored.
### Modifying the FILTER effort
1. By default, Vector Sets will explore up to `COUNT * 100` candidates to find matching results.
2. You can control this exploration with the `FILTER-EF` parameter.
3. A higher `FILTER-EF` value increases the chances of finding all relevant matches at the cost of increased processing time.
4. A `FILTER-EF` of zero will explore as many nodes as needed in order to actually return the number of elements specified by `COUNT`.
5. Even when a high `FILTER-EF` value is specified **the implementation will do a lot less work** if the elements passing the filter are very common, because of the early stop conditions of the HNSW implementation (once the specified amount of elements is reached and the quality check of the other candidates trigger an early stop).
```
VSIM key [ELE|FP32|VALUES] <vector or element> COUNT 10 FILTER '.year > 2000' FILTER-EF 500
```
In this example, Vector Sets will examine up to 500 potential nodes. Of course if count is reached before exploring 500 nodes, and the quality checks show that it is not possible to make progresses on similarity, the search is ended sooner.
### Performance Considerations
- If you have highly selective filters (few items match), use a higher `FILTER-EF`, or just design your application in order to handle a result set that is smaller than the requested count. Note that anyway the additional elements may be too distant than the query vector.
- For less selective filters, the default should be sufficient.
- Very selective filters with low `FILTER-EF` values may return fewer items than requested.
- Extremely high values may impact performance without significantly improving results.
The optimal `FILTER-EF` value depends on:
1. The selectivity of your filter.
2. The distribution of your data.
3. The required recall quality.
A good practice is to start with the default and increase if needed when you observe fewer results than expected.
### Testing a larg-ish data set
To really see how things work at scale, you can [download](https://antirez.com/word2vec_with_attribs.rdb) the following dataset:
wget https://antirez.com/word2vec_with_attribs.rdb
It contains the 3 million words in Word2Vec having as attribute a JSON with just the length of the word. Because of the length distribution of words in large amounts of texts, where longer words become less and less common, this is ideal to check how filtering behaves with a filter verifying as true with less and less elements in a vector set.
For instance:
> VSIM word_embeddings_bin ele "pasta" FILTER ".len == 6"
1) "pastas"
2) "rotini"
3) "gnocci"
4) "panino"
5) "salads"
6) "breads"
7) "salame"
8) "sauces"
9) "cheese"
10) "fritti"
This will easily retrieve the desired amount of items (`COUNT` is 10 by default) since there are many items of length 6. However:
> VSIM word_embeddings_bin ele "pasta" FILTER ".len == 33"
1) "skinless_boneless_chicken_breasts"
2) "boneless_skinless_chicken_breasts"
3) "Boneless_skinless_chicken_breasts"
This time even if we asked for 10 items, we only get 3, since the default filter effort will be `10*100 = 1000`. We can tune this giving the effort in an explicit way, with the risk of our query being slower, of course:
> VSIM word_embeddings_bin ele "pasta" FILTER ".len == 33" FILTER-EF 10000
1) "skinless_boneless_chicken_breasts"
2) "boneless_skinless_chicken_breasts"
3) "Boneless_skinless_chicken_breasts"
4) "mozzarella_feta_provolone_cheddar"
5) "Greatfood.com_R_www.greatfood.com"
6) "Pepperidge_Farm_Goldfish_crackers"
7) "Prosecuted_Mobsters_Rebuilt_Dying"
8) "Crispy_Snacker_Sandwiches_Popcorn"
9) "risultati_delle_partite_disputate"
10) "Peppermint_Mocha_Twist_Gingersnap"
This time we get all the ten items, even if the last one will be quite far from our query vector. We encourage to experiment with this test dataset in order to understand better the dynamics of the implementation and the natural tradeoffs of filtered search.
**Keep in mind** that by default, Redis Vector Sets will try to avoid a likely very useless huge scan of the HNSW graph, and will be more happy to return few or no elements at all, since this is almost always what the user actually wants in the context of retrieving *similar* items to the query.
# Single Instance Scalability and Latency
Vector Sets implement a threading model that allows Redis to handle many concurrent requests: by default `VSIM` is always threaded, and `VADD` is not (but can be partially threaded using the `CAS` option). This section explains how the threading and locking mechanisms work, and what to expect in terms of performance.
## Threading Model
- The `VSIM` command runs in a separate thread by default, allowing Redis to continue serving other commands.
- A maximum of 32 threads can run concurrently (defined by `HNSW_MAX_THREADS`).
- When this limit is reached, additional `VSIM` requests are queued - Redis remains responsive, no latency event is generated.
- The `VADD` command with the `CAS` option also leverages threading for the computation-heavy candidate search phase, but the insertion itself is performed in the main thread. `VADD` always runs in a sub-millisecond time, so this is not a source of latency, but having too many hundreds of writes per second can be challenging to handle with a single instance. Please, look at the next section about multiple instances scalability.
- Commands run within Lua scripts, MULTI/EXEC blocks, or from replication are executed in the main thread to ensure consistency.
```
> VSIM vset VALUES 3 1 1 1 FILTER '.year > 2000' # This runs in a thread.
> VADD vset VALUES 3 1 1 1 element CAS # Candidate search runs in a thread.
```
## Locking Mechanism
Vector Sets use a read/write locking mechanism to coordinate access:
- Reads (`VSIM`, `VEMB`, etc.) acquire a read lock, allowing multiple concurrent reads.
- Writes (`VADD`, `VREM`, etc.) acquire a write lock, temporarily blocking all reads.
- When a write lock is requested while reads are in progress, the write operation waits for all reads to complete.
- Once a write lock is granted, all reads are blocked until the write completes.
- Each thread has a dedicated slot for tracking visited nodes during graph traversal, avoiding contention. This improves performances but limits the maximum number of concurrent threads, since each node has a memory cost proportional to the number of slots.
## DEL latency
Deleting a very large vector set (millions of elements) can cause latency spikes, as deletion rebuilds connections between nodes. This may change in the future.
The deletion latency is most noticeable when using `DEL` on a key containing a large vector set or when the key expires.
## Performance Characteristics
- Search operations (`VSIM`) scale almost linearly with the number of CPU cores available, up to the thread limit. You can expect a Vector Set composed of million of items associated with components of dimension 300, with the default int8 quantization, to deliver around 50k VSIM operations per second in a single host.
- Insertion operations (`VADD`) are more computationally expensive than searches, and can't be threaded: expect much lower throughput, in the range of a few thousands inserts per second.
- Binary quantization offers significantly faster search performance at the cost of some recall quality, while int8 quantization, the default, seems to have very small impacts on recall quality, while it significantly improves performances and space efficiency.
- The `EF` parameter has a major impact on both search quality and performance - higher values mean better recall but slower searches.
- Graph traversal time scales logarithmically with the number of elements, making Vector Sets efficient even with millions of vectors
## Loading / Saving performances
Vector Sets are able to serialize on disk the graph structure as it is in memory, so loading back the data does not need to rebuild the HNSW graph. This means that Redis can load millions of items per minute. For instance 3 million items with 300 components vectors can be loaded back into memory into around 15 seconds.
# Scaling vector sets to multiple instances
The fundamental way vector sets can be scaled to very large data sets
and to many Redis instances is that a given very large set of vectors
can be partitioned into N different Redis keys, that can also live into
different Redis instances.
For instance, I could add my elements into `key0`, `key1`, `key2`, by hashing
the item in some way, like doing `crc32(item)%3`, effectively splitting
the dataset into three different parts. However once I want all the vectors
of my dataset near to a given query vector, I could simply perform the
`VSIM` command against all the three keys, merging the results by
score (so the commands must be called using the `WITHSCORES` option) on
the client side: once the union of the results are ordered by the
similarity score, the query is equivalent to having a single key `key1+2+3`
containing all the items.
There are a few interesting facts to note about this pattern:
1. It is possible to have a logical sorted set that is as big as the sum of all the Redis instances we are using.
2. Deletion operations remain simple, we can hash the key and select the key where our item belongs.
3. However, even if I use 10 different Redis instances, I'm not going to reach 10x the **read** operations per second, compared to using a single server: for each logical query, I need to query all the instances. Yet, smaller graphs are faster to navigate, so there is some win even from the point of view of CPU usage.
4. Insertions, so **write** queries, will be scaled linearly: I can add N items against N instances at the same time, splitting the insertion load evenly. This is very important since vector sets, being based on HNSW data structures, are slower to add items than to query similar items, by a very big factor.
5. While it cannot guarantee always the best results, with proper timeout management this system may be considered *highly available*, since if a subset of N instances are reachable, I'll be still be able to return similar items to my query vector.
Notably, this pattern can be implemented in a way that avoids paying the sum of the round trip time with all the servers: it is possible to send the queries at the same time to all the instances, so that latency will be equal the slower reply out of of the N servers queries.
# Optimizing memory usage
Vector Sets, or better, HNSWs, the underlying data structure used by Vector Sets, combined with the features provided by the Vector Sets themselves (quantization, random projection, filtering, ...) form an implementation that has a non-trivial space of parameters that can be tuned. Despite to the complexity of the implementation and of vector similarity problems, here there is a list of simple ideas that can drive the user to pick the best settings:
* 8 bit quantization (the default) is almost always a win. It reduces the memory usage of vectors by a factor of 4, yet the performance penalty in terms of recall is minimal. It also reduces insertion and search time by around 2 times or more.
* Binary quantization is much more extreme: it makes vector sets a lot faster, but increases the recall error in a sensible way, for instance from 95% to 80% if all the parameters remain the same. Yet, the speedup is really big, and the memory usage of vectors, compaerd to full precision vectors, 32 times smaller.
* Vectors memory usage are not the only responsible for Vector Set high memory usage per entry: nodes contain, on average `M*2 + M*0.33` pointers, where M is by default 16 (but can be tuned in `VADD`, see the `M` option). Also each node has the string item and the optional JSON attributes: those should be as small as possible in order to avoid contributing more to the memory usage.
* The `M` parameter should be increased to 32 or more only when a near perfect recall is really needed.
* It is possible to gain space (less memory usage) sacrificing time (more CPU time) by using a low `M` (the default of 16, for instance) and a high `EF` (the effort parameter of `VSIM`) in order to scan the graph more deeply.
* When memory usage is seriosu concern, and there is the suspect the vectors we are storing don't contain as much information - at least for our use case - to justify the number of components they feature, random projection (the `REDUCE` option of `VADD`) could be tested to see if dimensionality reduction is possible with acceptable precision loss.
## Random projection tradeoffs
Sometimes learned vectors are not as information dense as we could guess, that
is there are components having similar meanings in the space, and components
having values that don't really represent features that matter in our use case.
At the same time, certain vectors are very big, 1024 components or more. In this cases, it is possible to use the random projection feature of Redis Vector Sets in order to reduce both space (less RAM used) and space (more operstions per second). The feature is accessible via the `REDUCE` option of the `VADD` command. However, keep in mind that you need to test how much reduction impacts the performances of your vectors in term of recall and quality of the results you get back.
## What is a random projection?
The concept of Random Projection is relatively simple to grasp. For instance, a projection that turns a 100 components vector into a 10 components vector will perform a different linear transformation between the 100 components and each of the target 10 components. Please note that *each of the target components* will get some random amount of all the 100 original components. It is mathematically proved that this process results in a vector space where elements still have similar distances among them, but still some information will get lost.
## Examples of projections and loss of precision
To show you a bit of a extreme case, let's take Word2Vec 3 million items and compress them from 300 to 100, 50 and 25 components vectors. Then, we check the recall compared to the ground truth against each of the vector sets produced in this way (using different `REDUCE` parameters of `VADD`). This is the result, obtained asking for the top 10 elements.
```
----------------------------------------------------------------------
Key Average Recall % Std Dev
----------------------------------------------------------------------
word_embeddings_int8 95.98 12.14
^ This is the same key used for ground truth, but without TRUTH option
word_embeddings_reduced_100 40.20 20.13
word_embeddings_reduced_50 24.42 16.89
word_embeddings_reduced_25 14.31 9.99
```
Here the dimensionality reduction we are using is quite extreme: from 300 to 100 means that 66.6% of the original information is lost. The recall drops from 96% to 40%, down to 24% and 14% for even more extreme dimension reduction.
Reducing the dimension of vectors that are already relatively small, like the above example, of 300 components, will provide only relatively small memory savings, especially because by default Vector Sets use `int8` quantization, that will use only one byte per component:
```
> MEMORY USAGE word_embeddings_int8
(integer) 3107002888
> MEMORY USAGE word_embeddings_reduced_100
(integer) 2507122888
```
Of course going, for example, from 2048 component vectors to 1024 would provide a much more sensible memory saving, even with the `int8` quantization used by Vector Sets, assuming the recall loss is acceptable. Other than the memory saving, there is also the reduction in CPU time, translating to more operations per second.
Another thing to note is that, with certain embedding models, binary quantization (that offers a 8x reduction of memory usage compared to 8 bit quants, and a very big speedup in computation) performs much better than reducing the dimension of vectors of the same amount via random projections:
```
word_embeddings_bin 35.48 19.78
```
Here in the same test did above: we have a 35% recall which is not too far than the 40% obtained with a random projection from 300 to 100 components. However, while the first technique reduces the size by 3 times, the size reduced of binary quantization is by 8 times.
```
> memory usage word_embeddings_bin
(integer) 2327002888
```
In this specific case the key uses JSON attributes and has a graph connection overhead that is much bigger than the 300 bits each vector takes, but, as already said, for big vectors (1024 components, for instance) or for lower values of `M` (see `VADD`, the `M` parameter connects the level of connectivity, so it changes the amount of pointers used per node) the memory saving is much stronger.
# Vector Sets troubleshooting and understandability
## Debugging poor recall or unexpected results
Vector graphs and similarity queries pose many challenges mainly due to the following three problems:
1. The error due to the approximated nature of Vector Sets is hard to evaluate.
2. The error added by the quantization is often depends on the exact vector space (the embedding we are using **and** how far apart the elements we represent into such embeddings are).
3. We live in the illusion that learned embeddings capture the best similarity possible among elements, which is obviously not always true, and highly application dependent.
The only way to debug such problems, is the ability to inspect step by step what is happening inside our application, and the structure of the HNSW graph itself. To do so, we suggest to consider the following tools:
1. The `TRUTH` option of the `VSIM` command is able to return the ground truth of the most similar elements, without using the HNSW graph, but doing a linear scan.
2. The `VLINKS` command allows to explore the graph to see if the connections among nodes make sense, and to investigate why a given node may be more isolated than expected. Such command can also be used in a different way, when we want very fast "similar items" without paying the HNSW traversal time. It exploits the fact that we have a direct reference from each element in our vector set to each node in our HNSW graph.
3. The `WITHSCORES` option, in the supported commands, return a value that is directly related to the *cosine similarity* between the query and the items vectors, the interval of the similarity is simply rescaled from the -1, 1 original range to 0, 1, otherwise the metric is identical.
## Clients, latency and bandwidth usage
During Vector Sets testing, we discovered that often clients introduce considerable latecy and CPU usage (in the client side, not in Redis) for two main reasons:
1. Often the serialization to `VALUES ... list of floats ...` can be very slow.
2. The vector payload of floats represented as strings is very large, resulting in high bandwidth usage and latency, compared to other Redis commands.
Switching from `VALUES` to `FP32` as a method for transmitting vectors may easily provide 10-20x speedups.
# Known bugs
* Replication code is pretty much untested, and very vanilla (replicating the commands verbatim).
# Implementation details
Vector sets are based on the `hnsw.c` implementation of the HNSW data structure with extensions for speed and functionality.
The main features are:
* Proper nodes deletion with relinking.
* 8 bits and binary quantization.
* Threaded queries.
* Filtered search with predicate callback.
modules/vector-sets/cJSON.c
+3164
View File
File diff suppressed because it is too large Load Diff
modules/vector-sets/cJSON.h
+306
View File
@@ -0,0 +1,306 @@
/*
Copyright (c) 2009-2017 Dave Gamble and cJSON contributors
Permission is hereby granted, free of charge, to any person obtaining a copy
of this software and associated documentation files (the "Software"), to deal
in the Software without restriction, including without limitation the rights
to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
copies of the Software, and to permit persons to whom the Software is
furnished to do so, subject to the following conditions:
The above copyright notice and this permission notice shall be included in
all copies or substantial portions of the Software.
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
THE SOFTWARE.
*/
#ifndef cJSON__h
#define cJSON__h
#ifdef __cplusplus
extern "C"
{
#endif
#if !defined(__WINDOWS__) && (defined(WIN32) || defined(WIN64) || defined(_MSC_VER) || defined(_WIN32))
#define __WINDOWS__
#endif
#ifdef __WINDOWS__
/* When compiling for windows, we specify a specific calling convention to avoid issues where we are being called from a project with a different default calling convention. For windows you have 3 define options:
CJSON_HIDE_SYMBOLS - Define this in the case where you don't want to ever dllexport symbols
CJSON_EXPORT_SYMBOLS - Define this on library build when you want to dllexport symbols (default)
CJSON_IMPORT_SYMBOLS - Define this if you want to dllimport symbol
For *nix builds that support visibility attribute, you can define similar behavior by
setting default visibility to hidden by adding
-fvisibility=hidden (for gcc)
or
-xldscope=hidden (for sun cc)
to CFLAGS
then using the CJSON_API_VISIBILITY flag to "export" the same symbols the way CJSON_EXPORT_SYMBOLS does
*/
#define CJSON_CDECL __cdecl
#define CJSON_STDCALL __stdcall
/* export symbols by default, this is necessary for copy pasting the C and header file */
#if !defined(CJSON_HIDE_SYMBOLS) && !defined(CJSON_IMPORT_SYMBOLS) && !defined(CJSON_EXPORT_SYMBOLS)
#define CJSON_EXPORT_SYMBOLS
#endif
#if defined(CJSON_HIDE_SYMBOLS)
#define CJSON_PUBLIC(type) type CJSON_STDCALL
#elif defined(CJSON_EXPORT_SYMBOLS)
#define CJSON_PUBLIC(type) __declspec(dllexport) type CJSON_STDCALL
#elif defined(CJSON_IMPORT_SYMBOLS)
#define CJSON_PUBLIC(type) __declspec(dllimport) type CJSON_STDCALL
#endif
#else /* !__WINDOWS__ */
#define CJSON_CDECL
#define CJSON_STDCALL
#if (defined(__GNUC__) || defined(__SUNPRO_CC) || defined (__SUNPRO_C)) && defined(CJSON_API_VISIBILITY)
#define CJSON_PUBLIC(type) __attribute__((visibility("default"))) type
#else
#define CJSON_PUBLIC(type) type
#endif
#endif
/* project version */
#define CJSON_VERSION_MAJOR 1
#define CJSON_VERSION_MINOR 7
#define CJSON_VERSION_PATCH 18
#include <stddef.h>
/* cJSON Types: */
#define cJSON_Invalid (0)
#define cJSON_False (1 << 0)
#define cJSON_True (1 << 1)
#define cJSON_NULL (1 << 2)
#define cJSON_Number (1 << 3)
#define cJSON_String (1 << 4)
#define cJSON_Array (1 << 5)
#define cJSON_Object (1 << 6)
#define cJSON_Raw (1 << 7) /* raw json */
#define cJSON_IsReference 256
#define cJSON_StringIsConst 512
/* The cJSON structure: */
typedef struct cJSON
{
/* next/prev allow you to walk array/object chains. Alternatively, use GetArraySize/GetArrayItem/GetObjectItem */
struct cJSON *next;
struct cJSON *prev;
/* An array or object item will have a child pointer pointing to a chain of the items in the array/object. */
struct cJSON *child;
/* The type of the item, as above. */
int type;
/* The item's string, if type==cJSON_String and type == cJSON_Raw */
char *valuestring;
/* writing to valueint is DEPRECATED, use cJSON_SetNumberValue instead */
int valueint;
/* The item's number, if type==cJSON_Number */
double valuedouble;
/* The item's name string, if this item is the child of, or is in the list of subitems of an object. */
char *string;
} cJSON;
typedef struct cJSON_Hooks
{
/* malloc/free are CDECL on Windows regardless of the default calling convention of the compiler, so ensure the hooks allow passing those functions directly. */
void *(CJSON_CDECL *malloc_fn)(size_t sz);
void (CJSON_CDECL *free_fn)(void *ptr);
} cJSON_Hooks;
typedef int cJSON_bool;
/* Limits how deeply nested arrays/objects can be before cJSON rejects to parse them.
* This is to prevent stack overflows. */
#ifndef CJSON_NESTING_LIMIT
#define CJSON_NESTING_LIMIT 1000
#endif
/* Limits the length of circular references can be before cJSON rejects to parse them.
* This is to prevent stack overflows. */
#ifndef CJSON_CIRCULAR_LIMIT
#define CJSON_CIRCULAR_LIMIT 10000
#endif
/* returns the version of cJSON as a string */
CJSON_PUBLIC(const char*) cJSON_Version(void);
/* Supply malloc, realloc and free functions to cJSON */
CJSON_PUBLIC(void) cJSON_InitHooks(cJSON_Hooks* hooks);
/* Memory Management: the caller is always responsible to free the results from all variants of cJSON_Parse (with cJSON_Delete) and cJSON_Print (with stdlib free, cJSON_Hooks.free_fn, or cJSON_free as appropriate). The exception is cJSON_PrintPreallocated, where the caller has full responsibility of the buffer. */
/* Supply a block of JSON, and this returns a cJSON object you can interrogate. */
CJSON_PUBLIC(cJSON *) cJSON_Parse(const char *value);
CJSON_PUBLIC(cJSON *) cJSON_ParseWithLength(const char *value, size_t buffer_length);
/* ParseWithOpts allows you to require (and check) that the JSON is null terminated, and to retrieve the pointer to the final byte parsed. */
/* If you supply a ptr in return_parse_end and parsing fails, then return_parse_end will contain a pointer to the error so will match cJSON_GetErrorPtr(). */
CJSON_PUBLIC(cJSON *) cJSON_ParseWithOpts(const char *value, const char **return_parse_end, cJSON_bool require_null_terminated);
CJSON_PUBLIC(cJSON *) cJSON_ParseWithLengthOpts(const char *value, size_t buffer_length, const char **return_parse_end, cJSON_bool require_null_terminated);
/* Render a cJSON entity to text for transfer/storage. */
CJSON_PUBLIC(char *) cJSON_Print(const cJSON *item);
/* Render a cJSON entity to text for transfer/storage without any formatting. */
CJSON_PUBLIC(char *) cJSON_PrintUnformatted(const cJSON *item);
/* Render a cJSON entity to text using a buffered strategy. prebuffer is a guess at the final size. guessing well reduces reallocation. fmt=0 gives unformatted, =1 gives formatted */
CJSON_PUBLIC(char *) cJSON_PrintBuffered(const cJSON *item, int prebuffer, cJSON_bool fmt);
/* Render a cJSON entity to text using a buffer already allocated in memory with given length. Returns 1 on success and 0 on failure. */
/* NOTE: cJSON is not always 100% accurate in estimating how much memory it will use, so to be safe allocate 5 bytes more than you actually need */
CJSON_PUBLIC(cJSON_bool) cJSON_PrintPreallocated(cJSON *item, char *buffer, const int length, const cJSON_bool format);
/* Delete a cJSON entity and all subentities. */
CJSON_PUBLIC(void) cJSON_Delete(cJSON *item);
/* Returns the number of items in an array (or object). */
CJSON_PUBLIC(int) cJSON_GetArraySize(const cJSON *array);
/* Retrieve item number "index" from array "array". Returns NULL if unsuccessful. */
CJSON_PUBLIC(cJSON *) cJSON_GetArrayItem(const cJSON *array, int index);
/* Get item "string" from object. Case insensitive. */
CJSON_PUBLIC(cJSON *) cJSON_GetObjectItem(const cJSON * const object, const char * const string);
CJSON_PUBLIC(cJSON *) cJSON_GetObjectItemCaseSensitive(const cJSON * const object, const char * const string);
CJSON_PUBLIC(cJSON_bool) cJSON_HasObjectItem(const cJSON *object, const char *string);
/* For analysing failed parses. This returns a pointer to the parse error. You'll probably need to look a few chars back to make sense of it. Defined when cJSON_Parse() returns 0. 0 when cJSON_Parse() succeeds. */
CJSON_PUBLIC(const char *) cJSON_GetErrorPtr(void);
/* Check item type and return its value */
CJSON_PUBLIC(char *) cJSON_GetStringValue(const cJSON * const item);
CJSON_PUBLIC(double) cJSON_GetNumberValue(const cJSON * const item);
/* These functions check the type of an item */
CJSON_PUBLIC(cJSON_bool) cJSON_IsInvalid(const cJSON * const item);
CJSON_PUBLIC(cJSON_bool) cJSON_IsFalse(const cJSON * const item);
CJSON_PUBLIC(cJSON_bool) cJSON_IsTrue(const cJSON * const item);
CJSON_PUBLIC(cJSON_bool) cJSON_IsBool(const cJSON * const item);
CJSON_PUBLIC(cJSON_bool) cJSON_IsNull(const cJSON * const item);
CJSON_PUBLIC(cJSON_bool) cJSON_IsNumber(const cJSON * const item);
CJSON_PUBLIC(cJSON_bool) cJSON_IsString(const cJSON * const item);
CJSON_PUBLIC(cJSON_bool) cJSON_IsArray(const cJSON * const item);
CJSON_PUBLIC(cJSON_bool) cJSON_IsObject(const cJSON * const item);
CJSON_PUBLIC(cJSON_bool) cJSON_IsRaw(const cJSON * const item);
/* These calls create a cJSON item of the appropriate type. */
CJSON_PUBLIC(cJSON *) cJSON_CreateNull(void);
CJSON_PUBLIC(cJSON *) cJSON_CreateTrue(void);
CJSON_PUBLIC(cJSON *) cJSON_CreateFalse(void);
CJSON_PUBLIC(cJSON *) cJSON_CreateBool(cJSON_bool boolean);
CJSON_PUBLIC(cJSON *) cJSON_CreateNumber(double num);
CJSON_PUBLIC(cJSON *) cJSON_CreateString(const char *string);
/* raw json */
CJSON_PUBLIC(cJSON *) cJSON_CreateRaw(const char *raw);
CJSON_PUBLIC(cJSON *) cJSON_CreateArray(void);
CJSON_PUBLIC(cJSON *) cJSON_CreateObject(void);
/* Create a string where valuestring references a string so
* it will not be freed by cJSON_Delete */
CJSON_PUBLIC(cJSON *) cJSON_CreateStringReference(const char *string);
/* Create an object/array that only references it's elements so
* they will not be freed by cJSON_Delete */
CJSON_PUBLIC(cJSON *) cJSON_CreateObjectReference(const cJSON *child);
CJSON_PUBLIC(cJSON *) cJSON_CreateArrayReference(const cJSON *child);
/* These utilities create an Array of count items.
* The parameter count cannot be greater than the number of elements in the number array, otherwise array access will be out of bounds.*/
CJSON_PUBLIC(cJSON *) cJSON_CreateIntArray(const int *numbers, int count);
CJSON_PUBLIC(cJSON *) cJSON_CreateFloatArray(const float *numbers, int count);
CJSON_PUBLIC(cJSON *) cJSON_CreateDoubleArray(const double *numbers, int count);
CJSON_PUBLIC(cJSON *) cJSON_CreateStringArray(const char *const *strings, int count);
/* Append item to the specified array/object. */
CJSON_PUBLIC(cJSON_bool) cJSON_AddItemToArray(cJSON *array, cJSON *item);
CJSON_PUBLIC(cJSON_bool) cJSON_AddItemToObject(cJSON *object, const char *string, cJSON *item);
/* Use this when string is definitely const (i.e. a literal, or as good as), and will definitely survive the cJSON object.
* WARNING: When this function was used, make sure to always check that (item->type & cJSON_StringIsConst) is zero before
* writing to `item->string` */
CJSON_PUBLIC(cJSON_bool) cJSON_AddItemToObjectCS(cJSON *object, const char *string, cJSON *item);
/* Append reference to item to the specified array/object. Use this when you want to add an existing cJSON to a new cJSON, but don't want to corrupt your existing cJSON. */
CJSON_PUBLIC(cJSON_bool) cJSON_AddItemReferenceToArray(cJSON *array, cJSON *item);
CJSON_PUBLIC(cJSON_bool) cJSON_AddItemReferenceToObject(cJSON *object, const char *string, cJSON *item);
/* Remove/Detach items from Arrays/Objects. */
CJSON_PUBLIC(cJSON *) cJSON_DetachItemViaPointer(cJSON *parent, cJSON * const item);
CJSON_PUBLIC(cJSON *) cJSON_DetachItemFromArray(cJSON *array, int which);
CJSON_PUBLIC(void) cJSON_DeleteItemFromArray(cJSON *array, int which);
CJSON_PUBLIC(cJSON *) cJSON_DetachItemFromObject(cJSON *object, const char *string);
CJSON_PUBLIC(cJSON *) cJSON_DetachItemFromObjectCaseSensitive(cJSON *object, const char *string);
CJSON_PUBLIC(void) cJSON_DeleteItemFromObject(cJSON *object, const char *string);
CJSON_PUBLIC(void) cJSON_DeleteItemFromObjectCaseSensitive(cJSON *object, const char *string);
/* Update array items. */
CJSON_PUBLIC(cJSON_bool) cJSON_InsertItemInArray(cJSON *array, int which, cJSON *newitem); /* Shifts pre-existing items to the right. */
CJSON_PUBLIC(cJSON_bool) cJSON_ReplaceItemViaPointer(cJSON * const parent, cJSON * const item, cJSON * replacement);
CJSON_PUBLIC(cJSON_bool) cJSON_ReplaceItemInArray(cJSON *array, int which, cJSON *newitem);
CJSON_PUBLIC(cJSON_bool) cJSON_ReplaceItemInObject(cJSON *object,const char *string,cJSON *newitem);
CJSON_PUBLIC(cJSON_bool) cJSON_ReplaceItemInObjectCaseSensitive(cJSON *object,const char *string,cJSON *newitem);
/* Duplicate a cJSON item */
CJSON_PUBLIC(cJSON *) cJSON_Duplicate(const cJSON *item, cJSON_bool recurse);
/* Duplicate will create a new, identical cJSON item to the one you pass, in new memory that will
* need to be released. With recurse!=0, it will duplicate any children connected to the item.
* The item->next and ->prev pointers are always zero on return from Duplicate. */
/* Recursively compare two cJSON items for equality. If either a or b is NULL or invalid, they will be considered unequal.
* case_sensitive determines if object keys are treated case sensitive (1) or case insensitive (0) */
CJSON_PUBLIC(cJSON_bool) cJSON_Compare(const cJSON * const a, const cJSON * const b, const cJSON_bool case_sensitive);
/* Minify a strings, remove blank characters(such as ' ', '\t', '\r', '\n') from strings.
* The input pointer json cannot point to a read-only address area, such as a string constant,
* but should point to a readable and writable address area. */
CJSON_PUBLIC(void) cJSON_Minify(char *json);
/* Helper functions for creating and adding items to an object at the same time.
* They return the added item or NULL on failure. */
CJSON_PUBLIC(cJSON*) cJSON_AddNullToObject(cJSON * const object, const char * const name);
CJSON_PUBLIC(cJSON*) cJSON_AddTrueToObject(cJSON * const object, const char * const name);
CJSON_PUBLIC(cJSON*) cJSON_AddFalseToObject(cJSON * const object, const char * const name);
CJSON_PUBLIC(cJSON*) cJSON_AddBoolToObject(cJSON * const object, const char * const name, const cJSON_bool boolean);
CJSON_PUBLIC(cJSON*) cJSON_AddNumberToObject(cJSON * const object, const char * const name, const double number);
CJSON_PUBLIC(cJSON*) cJSON_AddStringToObject(cJSON * const object, const char * const name, const char * const string);
CJSON_PUBLIC(cJSON*) cJSON_AddRawToObject(cJSON * const object, const char * const name, const char * const raw);
CJSON_PUBLIC(cJSON*) cJSON_AddObjectToObject(cJSON * const object, const char * const name);
CJSON_PUBLIC(cJSON*) cJSON_AddArrayToObject(cJSON * const object, const char * const name);
/* When assigning an integer value, it needs to be propagated to valuedouble too. */
#define cJSON_SetIntValue(object, number) ((object) ? (object)->valueint = (object)->valuedouble = (number) : (number))
/* helper for the cJSON_SetNumberValue macro */
CJSON_PUBLIC(double) cJSON_SetNumberHelper(cJSON *object, double number);
#define cJSON_SetNumberValue(object, number) ((object != NULL) ? cJSON_SetNumberHelper(object, (double)number) : (number))
/* Change the valuestring of a cJSON_String object, only takes effect when type of object is cJSON_String */
CJSON_PUBLIC(char*) cJSON_SetValuestring(cJSON *object, const char *valuestring);
/* If the object is not a boolean type this does nothing and returns cJSON_Invalid else it returns the new type*/
#define cJSON_SetBoolValue(object, boolValue) ( \
(object != NULL && ((object)->type & (cJSON_False|cJSON_True))) ? \
(object)->type=((object)->type &(~(cJSON_False|cJSON_True)))|((boolValue)?cJSON_True:cJSON_False) : \
cJSON_Invalid\
)
/* Macro for iterating over an array or object */
#define cJSON_ArrayForEach(element, array) for(element = (array != NULL) ? (array)->child : NULL; element != NULL; element = element->next)
/* malloc/free objects using the malloc/free functions that have been set with cJSON_InitHooks */
CJSON_PUBLIC(void *) cJSON_malloc(size_t size);
CJSON_PUBLIC(void) cJSON_free(void *object);
#ifdef __cplusplus
}
#endif
#endif
modules/vector-sets/examples/cli-tool/.gitignore
+1
View File
@@ -0,0 +1 @@
venv
modules/vector-sets/examples/cli-tool/README.md
+44
View File
@@ -0,0 +1,44 @@
This tool is similar to redis-cli (but very basic) but allows
to specify arguments that are expanded as vectors by calling
ollama to get the embedding.
Whatever is passed as !"foo bar" gets expanded into
VALUES ... embedding ...
You must have ollama running with the mxbai-emb-large model
already installed for this to work.
Example:
redis> KEYS *
1) food_items
2) glove_embeddings_bin
3) many_movies_mxbai-embed-large_BIN
4) many_movies_mxbai-embed-large_NOQUANT
5) word_embeddings
6) word_embeddings_bin
7) glove_embeddings_fp32
redis> VSIM food_items !"drinks with fruit"
1) (Fruit)Juices,Lemonade,100ml,50 cal,210 kJ
2) (Fruit)Juices,Limeade,100ml,128 cal,538 kJ
3) CannedFruit,Canned Fruit Cocktail,100g,81 cal,340 kJ
4) (Fruit)Juices,Energy-Drink,100ml,87 cal,365 kJ
5) Fruits,Lime,100g,30 cal,126 kJ
6) (Fruit)Juices,Coconut Water,100ml,19 cal,80 kJ
7) Fruits,Lemon,100g,29 cal,122 kJ
8) (Fruit)Juices,Clamato,100ml,60 cal,252 kJ
9) Fruits,Fruit salad,100g,50 cal,210 kJ
10) (Fruit)Juices,Capri-Sun,100ml,41 cal,172 kJ
redis> vsim food_items !"barilla"
1) Pasta&Noodles,Spirelli,100g,367 cal,1541 kJ
2) Pasta&Noodles,Farfalle,100g,358 cal,1504 kJ
3) Pasta&Noodles,Capellini,100g,353 cal,1483 kJ
4) Pasta&Noodles,Spaetzle,100g,368 cal,1546 kJ
5) Pasta&Noodles,Cappelletti,100g,164 cal,689 kJ
6) Pasta&Noodles,Penne,100g,351 cal,1474 kJ
7) Pasta&Noodles,Shells,100g,353 cal,1483 kJ
8) Pasta&Noodles,Linguine,100g,357 cal,1499 kJ
9) Pasta&Noodles,Rotini,100g,353 cal,1483 kJ
10) Pasta&Noodles,Rigatoni,100g,353 cal,1483 kJ
modules/vector-sets/examples/cli-tool/cli.py
Executable
+146
View File
@@ -0,0 +1,146 @@
#
# Copyright (c) 2009-Present, Redis Ltd.
# All rights reserved.
#
# Licensed under your choice of the Redis Source Available License 2.0
# (RSALv2) or the Server Side Public License v1 (SSPLv1).
#
#!/usr/bin/env python3
import redis
import requests
import re
import shlex
from prompt_toolkit import PromptSession
from prompt_toolkit.history import InMemoryHistory
def get_embedding(text):
"""Get embedding from local Ollama API"""
url = "http://localhost:11434/api/embeddings"
payload = {
"model": "mxbai-embed-large",
"prompt": text
}
try:
response = requests.post(url, json=payload)
response.raise_for_status()
return response.json()['embedding']
except requests.exceptions.RequestException as e:
raise Exception(f"Failed to get embedding: {str(e)}")
def process_embedding_patterns(text):
"""Process !"text" and !!"text" patterns in the command"""
def replace_with_embedding(match):
text = match.group(1)
embedding = get_embedding(text)
return f"VALUES {len(embedding)} {' '.join(map(str, embedding))}"
def replace_with_embedding_and_text(match):
text = match.group(1)
embedding = get_embedding(text)
# Return both the embedding values and the original text as next argument
return f'VALUES {len(embedding)} {" ".join(map(str, embedding))} "{text}"'
# First handle !!"text" pattern (must be done before !"text")
text = re.sub(r'!!"([^"]*)"', replace_with_embedding_and_text, text)
# Then handle !"text" pattern
text = re.sub(r'!"([^"]*)"', replace_with_embedding, text)
return text
def parse_command(command):
"""Parse command respecting quoted strings"""
try:
# Use shlex to properly handle quoted strings
return shlex.split(command)
except ValueError as e:
raise Exception(f"Invalid command syntax: {str(e)}")
def format_response(response):
"""Format the response to match Redis protocol style"""
if response is None:
return "(nil)"
elif isinstance(response, bool):
return "+OK" if response else "(error) Operation failed"
elif isinstance(response, (list, set)):
if not response:
return "(empty list or set)"
return "\n".join(f"{i+1}) {item}" for i, item in enumerate(response))
elif isinstance(response, int):
return f"(integer) {response}"
else:
return str(response)
def main():
# Default connection to localhost:6379
r = redis.Redis(host='localhost', port=6379, decode_responses=True)
try:
# Test connection
r.ping()
print("Connected to Redis. Type your commands (CTRL+D to exit):")
print("Special syntax:")
print(" !\"text\" - Replace with embedding")
print(" !!\"text\" - Replace with embedding and append text as value")
print(" \"text\" - Quote strings containing spaces")
except redis.ConnectionError:
print("Error: Could not connect to Redis server")
return
# Setup prompt session with history
session = PromptSession(history=InMemoryHistory())
# Main loop
while True:
try:
# Read input with line editing support
command = session.prompt("redis> ")
# Skip empty commands
if not command.strip():
continue
# Process any embedding patterns before parsing
try:
processed_command = process_embedding_patterns(command)
except Exception as e:
print(f"(error) Embedding processing failed: {str(e)}")
continue
# Parse the command respecting quoted strings
try:
parts = parse_command(processed_command)
except Exception as e:
print(f"(error) {str(e)}")
continue
if not parts:
continue
cmd = parts[0].lower()
args = parts[1:]
# Execute command
try:
method = getattr(r, cmd, None)
if method is not None:
result = method(*args)
else:
# Use execute_command for unknown commands
result = r.execute_command(cmd, *args)
print(format_response(result))
except AttributeError:
print(f"(error) Unknown command '{cmd}'")
except EOFError:
print("\nGoodbye!")
break
except KeyboardInterrupt:
continue # Allow Ctrl+C to clear current line
except redis.RedisError as e:
print(f"(error) {str(e)}")
except Exception as e:
print(f"(error) {str(e)}")
if __name__ == "__main__":
main()
modules/vector-sets/examples/glove-100/README
+3
View File
@@ -0,0 +1,3 @@
wget http://ann-benchmarks.com/glove-100-angular.hdf5
python insert.py
python recall.py (use --k <count> optionally, default top-10)
modules/vector-sets/examples/glove-100/insert.py
+55
View File
@@ -0,0 +1,55 @@
#
# Copyright (c) 2009-Present, Redis Ltd.
# All rights reserved.
#
# Licensed under your choice of the Redis Source Available License 2.0
# (RSALv2) or the Server Side Public License v1 (SSPLv1).
#
import h5py
import redis
from tqdm import tqdm
# Initialize Redis connection
redis_client = redis.Redis(host='localhost', port=6379, decode_responses=True, encoding='utf-8')
def add_to_redis(index, embedding):
"""Add embedding to Redis using VADD command"""
args = ["VADD", "glove_embeddings", "VALUES", "100"] # 100 is vector dimension
args.extend(map(str, embedding))
args.append(f"{index}") # Using index as identifier since we don't have words
args.append("EF")
args.append("200")
# args.append("NOQUANT")
# args.append("BIN")
redis_client.execute_command(*args)
def main():
with h5py.File('glove-100-angular.hdf5', 'r') as f:
# Get the train dataset
train_vectors = f['train']
total_vectors = train_vectors.shape[0]
print(f"Starting to process {total_vectors} vectors...")
# Process in batches to avoid memory issues
batch_size = 1000
for i in tqdm(range(0, total_vectors, batch_size)):
batch_end = min(i + batch_size, total_vectors)
batch = train_vectors[i:batch_end]
for j, vector in enumerate(batch):
try:
current_index = i + j
add_to_redis(current_index, vector)
except Exception as e:
print(f"Error processing vector {current_index}: {str(e)}")
continue
if (i + batch_size) % 10000 == 0:
print(f"Processed {i + batch_size} vectors")
if __name__ == "__main__":
main()
modules/vector-sets/examples/glove-100/recall.py
+86
View File
@@ -0,0 +1,86 @@
#
# Copyright (c) 2009-Present, Redis Ltd.
# All rights reserved.
#
# Licensed under your choice of the Redis Source Available License 2.0
# (RSALv2) or the Server Side Public License v1 (SSPLv1).
#
import h5py
import redis
import numpy as np
from tqdm import tqdm
import argparse
# Initialize Redis connection
redis_client = redis.Redis(host='localhost', port=6379, decode_responses=True, encoding='utf-8')
def get_redis_neighbors(query_vector, k):
"""Get nearest neighbors using Redis VSIM command"""
args = ["VSIM", "glove_embeddings_bin", "VALUES", "100"]
args.extend(map(str, query_vector))
args.extend(["COUNT", str(k)])
args.extend(["EF", 100])
if False:
print(args)
exit(1)
results = redis_client.execute_command(*args)
return [int(res) for res in results]
def calculate_recall(ground_truth, predicted, k):
"""Calculate recall@k"""
relevant = set(ground_truth[:k])
retrieved = set(predicted[:k])
return len(relevant.intersection(retrieved)) / len(relevant)
def main():
parser = argparse.ArgumentParser(description='Evaluate Redis VSIM recall')
parser.add_argument('--k', type=int, default=10, help='Number of neighbors to evaluate (default: 10)')
parser.add_argument('--batch', type=int, default=100, help='Progress update frequency (default: 100)')
args = parser.parse_args()
k = args.k
batch_size = args.batch
with h5py.File('glove-100-angular.hdf5', 'r') as f:
test_vectors = f['test'][:]
ground_truth_neighbors = f['neighbors'][:]
num_queries = len(test_vectors)
recalls = []
print(f"Evaluating recall@{k} for {num_queries} test queries...")
for i in tqdm(range(num_queries)):
try:
# Get Redis results
redis_neighbors = get_redis_neighbors(test_vectors[i], k)
# Get ground truth for this query
true_neighbors = ground_truth_neighbors[i]
# Calculate recall
recall = calculate_recall(true_neighbors, redis_neighbors, k)
recalls.append(recall)
if (i + 1) % batch_size == 0:
current_avg_recall = np.mean(recalls)
print(f"Current average recall@{k} after {i+1} queries: {current_avg_recall:.4f}")
except Exception as e:
print(f"Error processing query {i}: {str(e)}")
continue
final_recall = np.mean(recalls)
print("\nFinal Results:")
print(f"Average recall@{k}: {final_recall:.4f}")
print(f"Total queries evaluated: {len(recalls)}")
# Save detailed results
with open(f'recall_evaluation_results_k{k}.txt', 'w') as f:
f.write(f"Average recall@{k}: {final_recall:.4f}\n")
f.write(f"Total queries evaluated: {len(recalls)}\n")
f.write(f"Individual query recalls: {recalls}\n")
if __name__ == "__main__":
main()
modules/vector-sets/examples/movies/.gitignore
+2
View File
@@ -0,0 +1,2 @@
mpst_full_data.csv
partition.json
modules/vector-sets/examples/movies/README
+30
View File
@@ -0,0 +1,30 @@
This example maps long form movies plots to movies titles.
It will create fp32 and binary vectors (the two extremes).
1. Install ollama, and install the embedding model "mxbai-embed-large"
2. Download mpst_full_data.csv from https://www.kaggle.com/datasets/cryptexcode/mpst-movie-plot-synopses-with-tags
3. python insert.py
127.0.0.1:6379> VSIM many_movies_mxbai-embed-large_NOQUANT ELE "The Matrix"
1) "The Matrix"
2) "The Matrix Reloaded"
3) "The Matrix Revolutions"
4) "Commando"
5) "Avatar"
6) "Forbidden Planet"
7) "Terminator Salvation"
8) "Mandroid"
9) "The Omega Code"
10) "Coherence"
127.0.0.1:6379> VSIM many_movies_mxbai-embed-large_BIN ELE "The Matrix"
1) "The Matrix"
2) "The Matrix Reloaded"
3) "The Matrix Revolutions"
4) "The Omega Code"
5) "Forbidden Planet"
6) "Avatar"
7) "John Carter"
8) "System Shock 2"
9) "Coherence"
10) "Tomorrowland"
modules/vector-sets/examples/movies/insert.py
+56
View File
@@ -0,0 +1,56 @@
#
# Copyright (c) 2009-Present, Redis Ltd.
# All rights reserved.
#
# Licensed under your choice of the Redis Source Available License 2.0
# (RSALv2) or the Server Side Public License v1 (SSPLv1).
#
import csv
import requests
import redis
ModelName="mxbai-embed-large"
# Initialize Redis connection, setting encoding to utf-8
redis_client = redis.Redis(host='localhost', port=6379, decode_responses=True, encoding='utf-8')
def get_embedding(text):
"""Get embedding from local API"""
url = "http://localhost:11434/api/embeddings"
payload = {
"model": ModelName,
"prompt": "Represent this movie plot and genre: "+text
}
response = requests.post(url, json=payload)
return response.json()['embedding']
def add_to_redis(title, embedding, quant_type):
"""Add embedding to Redis using VADD command"""
args = ["VADD", "many_movies_"+ModelName+"_"+quant_type, "VALUES", str(len(embedding))]
args.extend(map(str, embedding))
args.append(title)
args.append(quant_type)
redis_client.execute_command(*args)
def main():
with open('mpst_full_data.csv', 'r', encoding='utf-8') as file:
reader = csv.DictReader(file)
for movie in reader:
try:
text_to_embed = f"{movie['title']} {movie['plot_synopsis']} {movie['tags']}"
print(f"Getting embedding for: {movie['title']}")
embedding = get_embedding(text_to_embed)
add_to_redis(movie['title'], embedding, "BIN")
add_to_redis(movie['title'], embedding, "NOQUANT")
print(f"Successfully processed: {movie['title']}")
except Exception as e:
print(f"Error processing {movie['title']}: {str(e)}")
continue
if __name__ == "__main__":
main()
modules/vector-sets/expr.c
+999
View File
@@ -0,0 +1,999 @@
/* Filtering of objects based on simple expressions.
* This powers the FILTER option of Vector Sets, but it is otherwise
* general code to be used when we want to tell if a given object (with fields)
* passes or fails a given test for scalars, strings, ...
*
* Copyright (c) 2009-Present, Redis Ltd.
* All rights reserved.
*
* Licensed under your choice of the Redis Source Available License 2.0
* (RSALv2) or the Server Side Public License v1 (SSPLv1).
* Originally authored by: Salvatore Sanfilippo.
*/
#include <stdio.h>
#include <stdlib.h>
#include <ctype.h>
#include <math.h>
#include <string.h>
#include "cJSON.h"
#ifdef TEST_MAIN
#define RedisModule_Alloc malloc
#define RedisModule_Realloc realloc
#define RedisModule_Free free
#define RedisModule_Strdup strdup
#endif
#define EXPR_TOKEN_EOF 0
#define EXPR_TOKEN_NUM 1
#define EXPR_TOKEN_STR 2
#define EXPR_TOKEN_TUPLE 3
#define EXPR_TOKEN_SELECTOR 4
#define EXPR_TOKEN_OP 5
#define EXPR_OP_OPAREN 0 /* ( */
#define EXPR_OP_CPAREN 1 /* ) */
#define EXPR_OP_NOT 2 /* ! */
#define EXPR_OP_POW 3 /* ** */
#define EXPR_OP_MULT 4 /* * */
#define EXPR_OP_DIV 5 /* / */
#define EXPR_OP_MOD 6 /* % */
#define EXPR_OP_SUM 7 /* + */
#define EXPR_OP_DIFF 8 /* - */
#define EXPR_OP_GT 9 /* > */
#define EXPR_OP_GTE 10 /* >= */
#define EXPR_OP_LT 11 /* < */
#define EXPR_OP_LTE 12 /* <= */
#define EXPR_OP_EQ 13 /* == */
#define EXPR_OP_NEQ 14 /* != */
#define EXPR_OP_IN 15 /* in */
#define EXPR_OP_AND 16 /* and */
#define EXPR_OP_OR 17 /* or */
/* This structure represents a token in our expression. It's either
* literals like 4, "foo", or operators like "+", "-", "and", or
* json selectors, that start with a dot: ".age", ".properties.somearray[1]" */
typedef struct exprtoken {
int refcount; // Reference counting for memory reclaiming.
int token_type; // Token type of the just parsed token.
int offset; // Chars offset in expression.
union {
double num; // Value for EXPR_TOKEN_NUM.
struct {
char *start; // String pointer for EXPR_TOKEN_STR / SELECTOR.
size_t len; // String len for EXPR_TOKEN_STR / SELECTOR.
char *heapstr; // True if we have a private allocation for this
// string. When possible, it just references to the
// string expression we compiled, exprstate->expr.
} str;
int opcode; // Opcode ID for EXPR_TOKEN_OP.
struct {
struct exprtoken **ele;
size_t len;
} tuple; // Tuples are like [1, 2, 3] for "in" operator.
};
} exprtoken;
/* Simple stack of expr tokens. This is used both to represent the stack
* of values and the stack of operands during VM execution. */
typedef struct exprstack {
exprtoken **items;
int numitems;
int allocsize;
} exprstack;
typedef struct exprstate {
char *expr; /* Expression string to compile. Note that
* expression token strings point directly to this
* string. */
char *p; // Current position inside 'expr', while parsing.
// Virtual machine state.
exprstack values_stack;
exprstack ops_stack; // Operator stack used during compilation.
exprstack tokens; // Expression processed into a sequence of tokens.
exprstack program; // Expression compiled into opcodes and values.
} exprstate;
/* Valid operators. */
struct {
char *opname;
int oplen;
int opcode;
int precedence;
int arity;
} ExprOptable[] = {
{"(", 1, EXPR_OP_OPAREN, 7, 0},
{")", 1, EXPR_OP_CPAREN, 7, 0},
{"!", 1, EXPR_OP_NOT, 6, 1},
{"not", 3, EXPR_OP_NOT, 6, 1},
{"**", 2, EXPR_OP_POW, 5, 2},
{"*", 1, EXPR_OP_MULT, 4, 2},
{"/", 1, EXPR_OP_DIV, 4, 2},
{"%", 1, EXPR_OP_MOD, 4, 2},
{"+", 1, EXPR_OP_SUM, 3, 2},
{"-", 1, EXPR_OP_DIFF, 3, 2},
{">", 1, EXPR_OP_GT, 2, 2},
{">=", 2, EXPR_OP_GTE, 2, 2},
{"<", 1, EXPR_OP_LT, 2, 2},
{"<=", 2, EXPR_OP_LTE, 2, 2},
{"==", 2, EXPR_OP_EQ, 2, 2},
{"!=", 2, EXPR_OP_NEQ, 2, 2},
{"in", 2, EXPR_OP_IN, 2, 2},
{"and", 3, EXPR_OP_AND, 1, 2},
{"&&", 2, EXPR_OP_AND, 1, 2},
{"or", 2, EXPR_OP_OR, 0, 2},
{"||", 2, EXPR_OP_OR, 0, 2},
{NULL, 0, 0, 0, 0} // Terminator.
};
#define EXPR_OP_SPECIALCHARS "+-*%/!()<>=|&"
#define EXPR_SELECTOR_SPECIALCHARS "_-"
/* ================================ Expr token ============================== */
/* Return an heap allocated token of the specified type, setting the
* reference count to 1. */
exprtoken *exprNewToken(int type) {
exprtoken *t = RedisModule_Alloc(sizeof(exprtoken));
memset(t,0,sizeof(*t));
t->token_type = type;
t->refcount = 1;
return t;
}
/* Generic free token function, can be used to free stack allocated
* objects (in this case the pointer itself will not be freed) or
* heap allocated objects. See the wrappers below. */
void exprTokenRelease(exprtoken *t) {
if (t == NULL) return;
if (t->refcount <= 0) {
printf("exprTokenRelease() against a token with refcount %d!\n"
"Aborting program execution\n",
t->refcount);
exit(1);
}
t->refcount--;
if (t->refcount > 0) return;
// We reached refcount 0: free the object.
if (t->token_type == EXPR_TOKEN_STR) {
if (t->str.heapstr != NULL) RedisModule_Free(t->str.heapstr);
} else if (t->token_type == EXPR_TOKEN_TUPLE) {
for (size_t j = 0; j < t->tuple.len; j++)
exprTokenRelease(t->tuple.ele[j]);
if (t->tuple.ele) RedisModule_Free(t->tuple.ele);
}
RedisModule_Free(t);
}
void exprTokenRetain(exprtoken *t) {
t->refcount++;
}
/* ============================== Stack handling ============================ */
#include <stdlib.h>
#include <string.h>
#define EXPR_STACK_INITIAL_SIZE 16
/* Initialize a new expression stack. */
void exprStackInit(exprstack *stack) {
stack->items = RedisModule_Alloc(sizeof(exprtoken*) * EXPR_STACK_INITIAL_SIZE);
stack->numitems = 0;
stack->allocsize = EXPR_STACK_INITIAL_SIZE;
}
/* Push a token pointer onto the stack. Does not increment the refcount
* of the token: it is up to the caller doing this. */
void exprStackPush(exprstack *stack, exprtoken *token) {
/* Check if we need to grow the stack. */
if (stack->numitems == stack->allocsize) {
size_t newsize = stack->allocsize * 2;
exprtoken **newitems =
RedisModule_Realloc(stack->items, sizeof(exprtoken*) * newsize);
stack->items = newitems;
stack->allocsize = newsize;
}
stack->items[stack->numitems] = token;
stack->numitems++;
}
/* Pop a token pointer from the stack. Return NULL if the stack is
* empty. Does NOT recrement the refcount of the token, it's up to the
* caller to do so, as the new owner of the reference. */
exprtoken *exprStackPop(exprstack *stack) {
if (stack->numitems == 0) return NULL;
stack->numitems--;
return stack->items[stack->numitems];
}
/* Just return the last element pushed, without consuming it nor altering
* the reference count. */
exprtoken *exprStackPeek(exprstack *stack) {
if (stack->numitems == 0) return NULL;
return stack->items[stack->numitems-1];
}
/* Free the stack structure state, including the items it contains, that are
* assumed to be heap allocated. The passed pointer itself is not freed. */
void exprStackFree(exprstack *stack) {
for (int j = 0; j < stack->numitems; j++)
exprTokenRelease(stack->items[j]);
RedisModule_Free(stack->items);
}
/* Just reset the stack removing all the items, but leaving it in a state
* that makes it still usable for new elements. */
void exprStackReset(exprstack *stack) {
for (int j = 0; j < stack->numitems; j++)
exprTokenRelease(stack->items[j]);
stack->numitems = 0;
}
/* =========================== Expression compilation ======================= */
void exprConsumeSpaces(exprstate *es) {
while(es->p[0] && isspace(es->p[0])) es->p++;
}
/* Parse an operator, trying to match the longer match in the
* operators table. */
exprtoken *exprParseOperator(exprstate *es) {
exprtoken *t = exprNewToken(EXPR_TOKEN_OP);
char *start = es->p;
while(es->p[0] &&
(isalpha(es->p[0]) ||
strchr(EXPR_OP_SPECIALCHARS,es->p[0]) != NULL))
{
es->p++;
}
int matchlen = es->p - start;
int bestlen = 0;
int j;
// Find the longest matching operator.
for (j = 0; ExprOptable[j].opname != NULL; j++) {
if (ExprOptable[j].oplen > matchlen) continue;
if (memcmp(ExprOptable[j].opname, start, ExprOptable[j].oplen) != 0)
{
continue;
}
if (ExprOptable[j].oplen > bestlen) {
t->opcode = ExprOptable[j].opcode;
bestlen = ExprOptable[j].oplen;
}
}
if (bestlen == 0) {
exprTokenRelease(t);
return NULL;
} else {
es->p = start + bestlen;
}
return t;
}
// Valid selector charset.
static int is_selector_char(int c) {
return (isalpha(c) ||
isdigit(c) ||
strchr(EXPR_SELECTOR_SPECIALCHARS,c) != NULL);
}
/* Parse selectors, they start with a dot and can have alphanumerical
* or few special chars. */
exprtoken *exprParseSelector(exprstate *es) {
exprtoken *t = exprNewToken(EXPR_TOKEN_SELECTOR);
es->p++; // Skip dot.
char *start = es->p;
while(es->p[0] && is_selector_char(es->p[0])) es->p++;
int matchlen = es->p - start;
t->str.start = start;
t->str.len = matchlen;
return t;
}
exprtoken *exprParseNumber(exprstate *es) {
exprtoken *t = exprNewToken(EXPR_TOKEN_NUM);
char num[64];
int idx = 0;
while(isdigit(es->p[0]) || es->p[0] == '.' || es->p[0] == 'e' ||
es->p[0] == 'E' || (idx == 0 && es->p[0] == '-'))
{
if (idx >= (int)sizeof(num)-1) {
exprTokenRelease(t);
return NULL;
}
num[idx++] = es->p[0];
es->p++;
}
num[idx] = 0;
char *endptr;
t->num = strtod(num, &endptr);
if (*endptr != '\0') {
exprTokenRelease(t);
return NULL;
}
return t;
}
exprtoken *exprParseString(exprstate *es) {
char quote = es->p[0]; /* Store the quote type (' or "). */
es->p++; /* Skip opening quote. */
exprtoken *t = exprNewToken(EXPR_TOKEN_STR);
t->str.start = es->p;
while(es->p[0] != '\0') {
if (es->p[0] == '\\' && es->p[1] != '\0') {
es->p += 2; // Skip escaped char.
continue;
}
if (es->p[0] == quote) {
t->str.len = es->p - t->str.start;
es->p++; // Skip closing quote.
return t;
}
es->p++;
}
/* If we reach here, string was not terminated. */
exprTokenRelease(t);
return NULL;
}
/* Parse a tuple of the form [1, "foo", 42]. No nested tuples are
* supported. This type is useful mostly to be used with the "IN"
* operator. */
exprtoken *exprParseTuple(exprstate *es) {
exprtoken *t = exprNewToken(EXPR_TOKEN_TUPLE);
t->tuple.ele = NULL;
t->tuple.len = 0;
es->p++; /* Skip opening '['. */
size_t allocated = 0;
while(1) {
exprConsumeSpaces(es);
/* Check for empty tuple or end. */
if (es->p[0] == ']') {
es->p++;
break;
}
/* Grow tuple array if needed. */
if (t->tuple.len == allocated) {
size_t newsize = allocated == 0 ? 4 : allocated * 2;
exprtoken **newele = RedisModule_Realloc(t->tuple.ele,
sizeof(exprtoken*) * newsize);
t->tuple.ele = newele;
allocated = newsize;
}
/* Parse tuple element. */
exprtoken *ele = NULL;
if (isdigit(es->p[0]) || es->p[0] == '-') {
ele = exprParseNumber(es);
} else if (es->p[0] == '"' || es->p[0] == '\'') {
ele = exprParseString(es);
} else {
exprTokenRelease(t);
return NULL;
}
/* Error parsing number/string? */
if (ele == NULL) {
exprTokenRelease(t);
return NULL;
}
/* Store element if no error was detected. */
t->tuple.ele[t->tuple.len] = ele;
t->tuple.len++;
/* Check for next element. */
exprConsumeSpaces(es);
if (es->p[0] == ']') {
es->p++;
break;
}
if (es->p[0] != ',') {
exprTokenRelease(t);
return NULL;
}
es->p++; /* Skip comma. */
}
return t;
}
/* Deallocate the object returned by exprCompile(). */
void exprFree(exprstate *es) {
if (es == NULL) return;
/* Free the original expression string. */
if (es->expr) RedisModule_Free(es->expr);
/* Free all stacks. */
exprStackFree(&es->values_stack);
exprStackFree(&es->ops_stack);
exprStackFree(&es->tokens);
exprStackFree(&es->program);
/* Free the state object itself. */
RedisModule_Free(es);
}
/* Split the provided expression into a stack of tokens. Returns
* 0 on success, 1 on error. */
int exprTokenize(exprstate *es, int *errpos) {
/* Main parsing loop. */
while(1) {
exprConsumeSpaces(es);
/* Set a flag to see if we can consider the - part of the
* number, or an operator. */
int minus_is_number = 0; // By default is an operator.
exprtoken *last = exprStackPeek(&es->tokens);
if (last == NULL) {
/* If we are at the start of an expression, the minus is
* considered a number. */
minus_is_number = 1;
} else if (last->token_type == EXPR_TOKEN_OP &&
last->opcode != EXPR_OP_CPAREN)
{
/* Also, if the previous token was an operator, the minus
* is considered a number, unless the previous operator is
* a closing parens. In such case it's like (...) -5, or alike
* and we want to emit an operator. */
minus_is_number = 1;
}
/* Parse based on the current character. */
exprtoken *current = NULL;
if (*es->p == '\0') {
current = exprNewToken(EXPR_TOKEN_EOF);
} else if (isdigit(*es->p) ||
(minus_is_number && *es->p == '-' && isdigit(es->p[1])))
{
current = exprParseNumber(es);
} else if (*es->p == '"' || *es->p == '\'') {
current = exprParseString(es);
} else if (*es->p == '.' && is_selector_char(es->p[1])) {
current = exprParseSelector(es);
} else if (isalpha(*es->p) || strchr(EXPR_OP_SPECIALCHARS, *es->p)) {
current = exprParseOperator(es);
} else if (*es->p == '[') {
current = exprParseTuple(es);
}
if (current == NULL) {
if (errpos) *errpos = es->p - es->expr;
return 1; // Syntax Error.
}
/* Push the current token to tokens stack. */
exprStackPush(&es->tokens, current);
if (current->token_type == EXPR_TOKEN_EOF) break;
}
return 0;
}
/* Helper function to get operator precedence from the operator table. */
int exprGetOpPrecedence(int opcode) {
for (int i = 0; ExprOptable[i].opname != NULL; i++) {
if (ExprOptable[i].opcode == opcode)
return ExprOptable[i].precedence;
}
return -1;
}
/* Helper function to get operator arity from the operator table. */
int exprGetOpArity(int opcode) {
for (int i = 0; ExprOptable[i].opname != NULL; i++) {
if (ExprOptable[i].opcode == opcode)
return ExprOptable[i].arity;
}
return -1;
}
/* Process an operator during compilation. Returns 0 on success, 1 on error.
* This function will retain a reference of the operator 'op' in case it
* is pushed on the operators stack. */
int exprProcessOperator(exprstate *es, exprtoken *op, int *stack_items, int *errpos) {
if (op->opcode == EXPR_OP_OPAREN) {
// This is just a marker for us. Do nothing.
exprStackPush(&es->ops_stack, op);
exprTokenRetain(op);
return 0;
}
if (op->opcode == EXPR_OP_CPAREN) {
/* Process operators until we find the matching opening parenthesis. */
while (1) {
exprtoken *top_op = exprStackPop(&es->ops_stack);
if (top_op == NULL) {
if (errpos) *errpos = op->offset;
return 1;
}
if (top_op->opcode == EXPR_OP_OPAREN) {
/* Open parethesis found. Our work finished. */
exprTokenRelease(top_op);
return 0;
}
int arity = exprGetOpArity(top_op->opcode);
if (*stack_items < arity) {
exprTokenRelease(top_op);
if (errpos) *errpos = top_op->offset;
return 1;
}
/* Move the operator on the program stack. */
exprStackPush(&es->program, top_op);
*stack_items = *stack_items - arity + 1;
}
}
int curr_prec = exprGetOpPrecedence(op->opcode);
/* Process operators with higher or equal precedence. */
while (1) {
exprtoken *top_op = exprStackPeek(&es->ops_stack);
if (top_op == NULL || top_op->opcode == EXPR_OP_OPAREN) break;
int top_prec = exprGetOpPrecedence(top_op->opcode);
if (top_prec < curr_prec) break;
/* Special case for **: only pop if precedence is strictly higher
* so that the operator is right associative, that is:
* 2 ** 3 ** 2 is evaluated as 2 ** (3 ** 2) == 512 instead
* of (2 ** 3) ** 2 == 64. */
if (op->opcode == EXPR_OP_POW && top_prec <= curr_prec) break;
/* Pop and add to program. */
top_op = exprStackPop(&es->ops_stack);
int arity = exprGetOpArity(top_op->opcode);
if (*stack_items < arity) {
exprTokenRelease(top_op);
if (errpos) *errpos = top_op->offset;
return 1;
}
/* Move to the program stack. */
exprStackPush(&es->program, top_op);
*stack_items = *stack_items - arity + 1;
}
/* Push current operator. */
exprStackPush(&es->ops_stack, op);
exprTokenRetain(op);
return 0;
}
/* Compile the expression into a set of push-value and exec-operator
* that exprRun() can execute. The function returns an expstate object
* that can be used for execution of the program. On error, NULL
* is returned, and optionally the position of the error into the
* expression is returned by reference. */
exprstate *exprCompile(char *expr, int *errpos) {
/* Initialize expression state. */
exprstate *es = RedisModule_Alloc(sizeof(exprstate));
es->expr = RedisModule_Strdup(expr);
es->p = es->expr;
/* Initialize all stacks. */
exprStackInit(&es->values_stack);
exprStackInit(&es->ops_stack);
exprStackInit(&es->tokens);
exprStackInit(&es->program);
/* Tokenization. */
if (exprTokenize(es, errpos)) {
exprFree(es);
return NULL;
}
/* Compile the expression into a sequence of operations. */
int stack_items = 0; // Track # of items that would be on the stack
// during execution. This way we can detect arity
// issues at compile time.
/* Process each token. */
for (int i = 0; i < es->tokens.numitems; i++) {
exprtoken *token = es->tokens.items[i];
if (token->token_type == EXPR_TOKEN_EOF) break;
/* Handle values (numbers, strings, selectors). */
if (token->token_type == EXPR_TOKEN_NUM ||
token->token_type == EXPR_TOKEN_STR ||
token->token_type == EXPR_TOKEN_TUPLE ||
token->token_type == EXPR_TOKEN_SELECTOR)
{
exprStackPush(&es->program, token);
exprTokenRetain(token);
stack_items++;
continue;
}
/* Handle operators. */
if (token->token_type == EXPR_TOKEN_OP) {
if (exprProcessOperator(es, token, &stack_items, errpos)) {
exprFree(es);
return NULL;
}
continue;
}
}
/* Process remaining operators on the stack. */
while (es->ops_stack.numitems > 0) {
exprtoken *op = exprStackPop(&es->ops_stack);
if (op->opcode == EXPR_OP_OPAREN) {
if (errpos) *errpos = op->offset;
exprTokenRelease(op);
exprFree(es);
return NULL;
}
int arity = exprGetOpArity(op->opcode);
if (stack_items < arity) {
if (errpos) *errpos = op->offset;
exprTokenRelease(op);
exprFree(es);
return NULL;
}
exprStackPush(&es->program, op);
stack_items = stack_items - arity + 1;
}
/* Verify that exactly one value would remain on the stack after
* execution. We could also check that such value is a number, but this
* would make the code more complex without much gains. */
if (stack_items != 1) {
if (errpos) {
/* Point to the last token's offset for error reporting. */
exprtoken *last = es->tokens.items[es->tokens.numitems - 1];
*errpos = last->offset;
}
exprFree(es);
return NULL;
}
return es;
}
/* ============================ Expression execution ======================== */
/* Convert a token to its numeric value. For strings we attempt to parse them
* as numbers, returning 0 if conversion fails. */
double exprTokenToNum(exprtoken *t) {
char buf[128];
if (t->token_type == EXPR_TOKEN_NUM) {
return t->num;
} else if (t->token_type == EXPR_TOKEN_STR && t->str.len < sizeof(buf)) {
memcpy(buf, t->str.start, t->str.len);
buf[t->str.len] = '\0';
char *endptr;
double val = strtod(buf, &endptr);
return *endptr == '\0' ? val : 0;
} else {
return 0;
}
}
/* Convert object to true/false (0 or 1) */
double exprTokenToBool(exprtoken *t) {
if (t->token_type == EXPR_TOKEN_NUM) {
return t->num != 0;
} else if (t->token_type == EXPR_TOKEN_STR && t->str.len == 0) {
return 0; // Empty string are false, like in Javascript.
} else {
return 1; // Every non numerical type is true.
}
}
/* Compare two tokens. Returns true if they are equal. */
int exprTokensEqual(exprtoken *a, exprtoken *b) {
// If both are strings, do string comparison.
if (a->token_type == EXPR_TOKEN_STR && b->token_type == EXPR_TOKEN_STR) {
return a->str.len == b->str.len &&
memcmp(a->str.start, b->str.start, a->str.len) == 0;
}
// If both are numbers, do numeric comparison.
if (a->token_type == EXPR_TOKEN_NUM && b->token_type == EXPR_TOKEN_NUM) {
return a->num == b->num;
}
// Mixed types - convert to numbers and compare.
return exprTokenToNum(a) == exprTokenToNum(b);
}
/* Convert a json object to an expression token. There is only
* limited support for JSON arrays: they must be composed of
* just numbers and strings. Returns NULL if the JSON object
* cannot be converted. */
exprtoken *exprJsonToToken(cJSON *js) {
if (cJSON_IsNumber(js)) {
exprtoken *obj = exprNewToken(EXPR_TOKEN_NUM);
obj->num = cJSON_GetNumberValue(js);
return obj;
} else if (cJSON_IsString(js)) {
exprtoken *obj = exprNewToken(EXPR_TOKEN_STR);
char *strval = cJSON_GetStringValue(js);
obj->str.heapstr = RedisModule_Strdup(strval);
obj->str.start = obj->str.heapstr;
obj->str.len = strlen(obj->str.heapstr);
return obj;
} else if (cJSON_IsBool(js)) {
exprtoken *obj = exprNewToken(EXPR_TOKEN_NUM);
obj->num = cJSON_IsTrue(js);
return obj;
} else if (cJSON_IsArray(js)) {
// First, scan the array to ensure it only
// contains strings and numbers. Otherwise the
// expression will evaluate to false.
int array_size = cJSON_GetArraySize(js);
for (int j = 0; j < array_size; j++) {
cJSON *item = cJSON_GetArrayItem(js, j);
if (!cJSON_IsNumber(item) && !cJSON_IsString(item)) return NULL;
}
// Create a tuple token for the array.
exprtoken *obj = exprNewToken(EXPR_TOKEN_TUPLE);
obj->tuple.len = array_size;
obj->tuple.ele = NULL;
if (obj->tuple.len == 0) return obj; // No elements, already ok.
obj->tuple.ele =
RedisModule_Alloc(sizeof(exprtoken*) * obj->tuple.len);
// Convert each array element to a token.
for (size_t j = 0; j < obj->tuple.len; j++) {
cJSON *item = cJSON_GetArrayItem(js, j);
if (cJSON_IsNumber(item)) {
exprtoken *eleToken = exprNewToken(EXPR_TOKEN_NUM);
eleToken->num = cJSON_GetNumberValue(item);
obj->tuple.ele[j] = eleToken;
} else if (cJSON_IsString(item)) {
exprtoken *eleToken = exprNewToken(EXPR_TOKEN_STR);
char *strval = cJSON_GetStringValue(item);
eleToken->str.heapstr = RedisModule_Strdup(strval);
eleToken->str.start = eleToken->str.heapstr;
eleToken->str.len = strlen(eleToken->str.heapstr);
obj->tuple.ele[j] = eleToken;
}
}
return obj;
}
return NULL; // No conversion possible for this type.
}
/* Execute the compiled expression program. Returns 1 if the final stack value
* evaluates to true, 0 otherwise. Also returns 0 if any selector callback
* fails. */
int exprRun(exprstate *es, char *json, size_t json_len) {
exprStackReset(&es->values_stack);
cJSON *parsed_json = NULL;
// Execute each instruction in the program.
for (int i = 0; i < es->program.numitems; i++) {
exprtoken *t = es->program.items[i];
// Handle selectors by calling the callback.
if (t->token_type == EXPR_TOKEN_SELECTOR) {
if (json != NULL) {
cJSON *attrib = NULL;
if (parsed_json == NULL) {
parsed_json = cJSON_ParseWithLength(json,json_len);
// Will be left to NULL if the above fails.
}
if (parsed_json) {
char item_name[128];
if (t->str.len > 0 && t->str.len < sizeof(item_name)) {
memcpy(item_name,t->str.start,t->str.len);
item_name[t->str.len] = 0;
attrib = cJSON_GetObjectItem(parsed_json,item_name);
}
/* Fill the token according to the JSON type stored
* at the attribute. */
if (attrib) {
exprtoken *obj = exprJsonToToken(attrib);
if (obj) {
exprStackPush(&es->values_stack, obj);
continue;
}
}
}
}
// Selector not found or JSON object not convertible to
// expression tokens. Evaluate the expression to false.
if (parsed_json) cJSON_Delete(parsed_json);
return 0;
}
// Push non-operator values directly onto the stack.
if (t->token_type != EXPR_TOKEN_OP) {
exprStackPush(&es->values_stack, t);
exprTokenRetain(t);
continue;
}
// Handle operators.
exprtoken *result = exprNewToken(EXPR_TOKEN_NUM);
// Pop operands - we know we have enough from compile-time checks.
exprtoken *b = exprStackPop(&es->values_stack);
exprtoken *a = NULL;
if (exprGetOpArity(t->opcode) == 2) {
a = exprStackPop(&es->values_stack);
}
switch(t->opcode) {
case EXPR_OP_NOT:
result->num = exprTokenToBool(b) == 0 ? 1 : 0;
break;
case EXPR_OP_POW: {
double base = exprTokenToNum(a);
double exp = exprTokenToNum(b);
result->num = pow(base, exp);
break;
}
case EXPR_OP_MULT:
result->num = exprTokenToNum(a) * exprTokenToNum(b);
break;
case EXPR_OP_DIV:
result->num = exprTokenToNum(a) / exprTokenToNum(b);
break;
case EXPR_OP_MOD: {
double va = exprTokenToNum(a);
double vb = exprTokenToNum(b);
result->num = fmod(va, vb);
break;
}
case EXPR_OP_SUM:
result->num = exprTokenToNum(a) + exprTokenToNum(b);
break;
case EXPR_OP_DIFF:
result->num = exprTokenToNum(a) - exprTokenToNum(b);
break;
case EXPR_OP_GT:
result->num = exprTokenToNum(a) > exprTokenToNum(b) ? 1 : 0;
break;
case EXPR_OP_GTE:
result->num = exprTokenToNum(a) >= exprTokenToNum(b) ? 1 : 0;
break;
case EXPR_OP_LT:
result->num = exprTokenToNum(a) < exprTokenToNum(b) ? 1 : 0;
break;
case EXPR_OP_LTE:
result->num = exprTokenToNum(a) <= exprTokenToNum(b) ? 1 : 0;
break;
case EXPR_OP_EQ:
result->num = exprTokensEqual(a, b) ? 1 : 0;
break;
case EXPR_OP_NEQ:
result->num = !exprTokensEqual(a, b) ? 1 : 0;
break;
case EXPR_OP_IN: {
// For 'in' operator, b must be a tuple.
result->num = 0; // Default to false.
if (b->token_type == EXPR_TOKEN_TUPLE) {
for (size_t j = 0; j < b->tuple.len; j++) {
if (exprTokensEqual(a, b->tuple.ele[j])) {
result->num = 1; // Found a match.
break;
}
}
}
break;
}
case EXPR_OP_AND:
result->num =
exprTokenToBool(a) != 0 && exprTokenToBool(b) != 0 ? 1 : 0;
break;
case EXPR_OP_OR:
result->num =
exprTokenToBool(a) != 0 || exprTokenToBool(b) != 0 ? 1 : 0;
break;
default:
// Do nothing: we don't want runtime errors.
break;
}
// Free operands and push result.
if (a) exprTokenRelease(a);
exprTokenRelease(b);
exprStackPush(&es->values_stack, result);
}
if (parsed_json) cJSON_Delete(parsed_json);
// Get final result from stack.
exprtoken *final = exprStackPop(&es->values_stack);
if (final == NULL) return 0;
// Convert result to boolean.
int retval = exprTokenToBool(final);
exprTokenRelease(final);
return retval;
}
/* ============================ Simple test main ============================ */
#ifdef TEST_MAIN
void exprPrintToken(exprtoken *t) {
switch(t->token_type) {
case EXPR_TOKEN_EOF:
printf("EOF");
break;
case EXPR_TOKEN_NUM:
printf("NUM:%g", t->num);
break;
case EXPR_TOKEN_STR:
printf("STR:\"%.*s\"", (int)t->str.len, t->str.start);
break;
case EXPR_TOKEN_SELECTOR:
printf("SEL:%.*s", (int)t->str.len, t->str.start);
break;
case EXPR_TOKEN_OP:
printf("OP:");
for (int i = 0; ExprOptable[i].opname != NULL; i++) {
if (ExprOptable[i].opcode == t->opcode) {
printf("%s", ExprOptable[i].opname);
break;
}
}
break;
default:
printf("UNKNOWN");
break;
}
}
void exprPrintStack(exprstack *stack, const char *name) {
printf("%s (%d items):", name, stack->numitems);
for (int j = 0; j < stack->numitems; j++) {
printf(" ");
exprPrintToken(stack->items[j]);
}
printf("\n");
}
int main(int argc, char **argv) {
char *testexpr = "(5+2)*3 and .year > 1980 and 'foo' == 'foo'";
char *testjson = "{\"year\": 1984, \"name\": \"The Matrix\"}";
if (argc >= 2) testexpr = argv[1];
if (argc >= 3) testjson = argv[2];
printf("Compiling expression: %s\n", testexpr);
int errpos = 0;
exprstate *es = exprCompile(testexpr,&errpos);
if (es == NULL) {
printf("Compilation failed near \"...%s\"\n", testexpr+errpos);
return 1;
}
exprPrintStack(&es->tokens, "Tokens");
exprPrintStack(&es->program, "Program");
printf("Running against object: %s\n", testjson);
int result = exprRun(es,testjson,strlen(testjson));
printf("Result1: %s\n", result ? "True" : "False");
result = exprRun(es,testjson,strlen(testjson));
printf("Result2: %s\n", result ? "True" : "False");
exprFree(es);
return 0;
}
#endif
modules/vector-sets/hnsw.c
+2722
View File
File diff suppressed because it is too large Load Diff
modules/vector-sets/hnsw.h
+188
View File
@@ -0,0 +1,188 @@
/*
* HNSW (Hierarchical Navigable Small World) Implementation
* Based on the paper by Yu. A. Malkov, D. A. Yashunin
*
* Copyright (c) 2009-Present, Redis Ltd.
* All rights reserved.
*
* Licensed under your choice of the Redis Source Available License 2.0
* (RSALv2) or the Server Side Public License v1 (SSPLv1).
* Originally authored by: Salvatore Sanfilippo.
*/
#ifndef HNSW_H
#define HNSW_H
#include <pthread.h>
#include <stdatomic.h>
#define HNSW_DEFAULT_M 16 /* Used when 0 is given at creation time. */
#define HNSW_MIN_M 4 /* Probably even too low already. */
#define HNSW_MAX_M 4096 /* Safeguard sanity limit. */
#define HNSW_MAX_THREADS 32 /* Maximum number of concurrent threads */
/* Quantization types you can enable at creation time in hnsw_new() */
#define HNSW_QUANT_NONE 0 // No quantization.
#define HNSW_QUANT_Q8 1 // Q8 quantization.
#define HNSW_QUANT_BIN 2 // Binary quantization.
/* Layer structure for HNSW nodes. Each node will have from one to a few
* of this depending on its level. */
typedef struct {
struct hnswNode **links; /* Array of neighbors for this layer */
uint32_t num_links; /* Number of used links */
uint32_t max_links; /* Maximum links for this layer. We may
* reallocate the node in very particular
* conditions in order to allow linking of
* new inserted nodes, so this may change
* dynamically and be > M*2 for a small set of
* nodes. */
float worst_distance; /* Distance to the worst neighbor */
uint32_t worst_idx; /* Index of the worst neighbor */
} hnswNodeLayer;
/* Node structure for HNSW graph */
typedef struct hnswNode {
uint32_t level; /* Node's maximum level */
uint64_t id; /* Unique identifier, may be useful in order to
* have a bitmap of visited notes to use as
* alternative to epoch / visited_epoch.
* Also used in serialization in order to retain
* links specifying IDs. */
void *vector; /* The vector, quantized or not. */
float quants_range; /* Quantization range for this vector:
* min/max values will be in the range
* -quants_range, +quants_range */
float l2; /* L2 before normalization. */
/* Last time (epoch) this node was visited. We need one per thread.
* This avoids having a different data structure where we track
* visited nodes, but costs memory per node. */
uint64_t visited_epoch[HNSW_MAX_THREADS];
void *value; /* Associated value */
struct hnswNode *prev, *next; /* Prev/Next node in the list starting at
* HNSW->head. */
/* Links (and links info) per each layer. Note that this is part
* of the node allocation to be more cache friendly: reliable 3% speedup
* on Apple silicon, and does not make anything more complex. */
hnswNodeLayer layers[];
} hnswNode;
struct HNSW;
/* It is possible to navigate an HNSW with a cursor that guarantees
* visiting all the elements that remain in the HNSW from the start to the
* end of the process (but not the new ones, so that the process will
* eventually finish). Check hnsw_cursor_init(), hnsw_cursor_next() and
* hnsw_cursor_free(). */
typedef struct hnswCursor {
struct HNSW *index; // Reference to the index of this cursor.
hnswNode *current; // Element to report when hnsw_cursor_next() is called.
struct hnswCursor *next; // Next cursor active.
} hnswCursor;
/* Main HNSW index structure */
typedef struct HNSW {
hnswNode *enter_point; /* Entry point for the graph */
uint32_t M; /* M as in the paper: layer 0 has M*2 max
neighbors (M populated at insertion time)
while all the other layers have M neighbors. */
uint32_t max_level; /* Current maximum level in the graph */
uint32_t vector_dim; /* Dimensionality of stored vectors */
uint64_t node_count; /* Total number of nodes */
_Atomic uint64_t last_id; /* Last node ID used */
uint64_t current_epoch[HNSW_MAX_THREADS]; /* Current epoch for visit tracking */
hnswNode *head; /* Linked list of nodes. Last first */
/* We have two locks here:
* 1. A global_lock that is used to perform write operations blocking all
* the readers.
* 2. One mutex per epoch slot, in order for read operations to acquire
* a lock on a specific slot to use epochs tracking of visited nodes. */
pthread_rwlock_t global_lock; /* Global read-write lock */
pthread_mutex_t slot_locks[HNSW_MAX_THREADS]; /* Per-slot locks */
_Atomic uint32_t next_slot; /* Next thread slot to try */
_Atomic uint64_t version; /* Version for optimistic concurrency, this is
* incremented on deletions and entry point
* updates. */
uint32_t quant_type; /* Quantization used. HNSW_QUANT_... */
hnswCursor *cursors;
} HNSW;
/* Serialized node. This structure is used as return value of
* hnsw_serialize_node(). */
typedef struct hnswSerNode {
void *vector;
uint32_t vector_size;
uint64_t *params;
uint32_t params_count;
} hnswSerNode;
/* Insert preparation context */
typedef struct InsertContext InsertContext;
/* Core HNSW functions */
HNSW *hnsw_new(uint32_t vector_dim, uint32_t quant_type, uint32_t m);
void hnsw_free(HNSW *index,void(*free_value)(void*value));
void hnsw_node_free(hnswNode *node);
void hnsw_print_stats(HNSW *index);
hnswNode *hnsw_insert(HNSW *index, const float *vector, const int8_t *qvector,
float qrange, uint64_t id, void *value, int ef);
int hnsw_search(HNSW *index, const float *query, uint32_t k,
hnswNode **neighbors, float *distances, uint32_t slot,
int query_vector_is_normalized);
int hnsw_search_with_filter
(HNSW *index, const float *query_vector, uint32_t k,
hnswNode **neighbors, float *distances, uint32_t slot,
int query_vector_is_normalized,
int (*filter_callback)(void *value, void *privdata),
void *filter_privdata, uint32_t max_candidates);
void hnsw_get_node_vector(HNSW *index, hnswNode *node, float *vec);
int hnsw_delete_node(HNSW *index, hnswNode *node, void(*free_value)(void*value));
hnswNode *hnsw_random_node(HNSW *index, int slot);
/* Thread safety functions. */
int hnsw_acquire_read_slot(HNSW *index);
void hnsw_release_read_slot(HNSW *index, int slot);
/* Optimistic insertion API. */
InsertContext *hnsw_prepare_insert(HNSW *index, const float *vector, const int8_t *qvector, float qrange, uint64_t id, int ef);
hnswNode *hnsw_try_commit_insert(HNSW *index, InsertContext *ctx, void *value);
void hnsw_free_insert_context(InsertContext *ctx);
/* Serialization. */
hnswSerNode *hnsw_serialize_node(HNSW *index, hnswNode *node);
void hnsw_free_serialized_node(hnswSerNode *sn);
hnswNode *hnsw_insert_serialized(HNSW *index, void *vector, uint64_t *params, uint32_t params_len, void *value);
int hnsw_deserialize_index(HNSW *index);
// Helper function in case the user wants to directly copy
// the vector bytes.
uint32_t hnsw_quants_bytes(HNSW *index);
/* Cursors. */
hnswCursor *hnsw_cursor_init(HNSW *index);
void hnsw_cursor_free(hnswCursor *cursor);
hnswNode *hnsw_cursor_next(hnswCursor *cursor);
int hnsw_cursor_acquire_lock(hnswCursor *cursor);
void hnsw_cursor_release_lock(hnswCursor *cursor);
/* Allocator selection. */
void hnsw_set_allocator(void (*free_ptr)(void*), void *(*malloc_ptr)(size_t),
void *(*realloc_ptr)(void*, size_t));
/* Testing. */
int hnsw_validate_graph(HNSW *index, uint64_t *connected_nodes, int *reciprocal_links);
void hnsw_test_graph_recall(HNSW *index, int test_ef, int verbose);
float hnsw_distance(HNSW *index, hnswNode *a, hnswNode *b);
int hnsw_ground_truth_with_filter
(HNSW *index, const float *query_vector, uint32_t k,
hnswNode **neighbors, float *distances, uint32_t slot,
int query_vector_is_normalized,
int (*filter_callback)(void *value, void *privdata),
void *filter_privdata);
#endif /* HNSW_H */
modules/vector-sets/test.py
Executable
+230
View File
@@ -0,0 +1,230 @@
#!/usr/bin/env python3
#
# Vector set tests.
# A Redis instance should be running in the default port.
#
# Copyright (c) 2009-Present, Redis Ltd.
# All rights reserved.
#
# Licensed under your choice of the Redis Source Available License 2.0
# (RSALv2) or the Server Side Public License v1 (SSPLv1).
#
#!/usr/bin/env python3
import redis
import random
import struct
import math
import time
import sys
import os
import importlib
import inspect
from typing import List, Tuple, Optional
from dataclasses import dataclass
def colored(text: str, color: str) -> str:
colors = {
'red': '\033[91m',
'green': '\033[92m'
}
reset = '\033[0m'
return f"{colors.get(color, '')}{text}{reset}"
@dataclass
class VectorData:
vectors: List[List[float]]
names: List[str]
def find_k_nearest(self, query_vector: List[float], k: int) -> List[Tuple[str, float]]:
"""Find k-nearest neighbors using the same scoring as Redis VSIM WITHSCORES."""
similarities = []
query_norm = math.sqrt(sum(x*x for x in query_vector))
if query_norm == 0:
return []
for i, vec in enumerate(self.vectors):
vec_norm = math.sqrt(sum(x*x for x in vec))
if vec_norm == 0:
continue
dot_product = sum(a*b for a,b in zip(query_vector, vec))
cosine_sim = dot_product / (query_norm * vec_norm)
distance = 1.0 - cosine_sim
redis_similarity = 1.0 - (distance/2.0)
similarities.append((self.names[i], redis_similarity))
similarities.sort(key=lambda x: x[1], reverse=True)
return similarities[:k]
def generate_random_vector(dim: int) -> List[float]:
"""Generate a random normalized vector."""
vec = [random.gauss(0, 1) for _ in range(dim)]
norm = math.sqrt(sum(x*x for x in vec))
return [x/norm for x in vec]
def fill_redis_with_vectors(r: redis.Redis, key: str, count: int, dim: int,
with_reduce: Optional[int] = None) -> VectorData:
"""Fill Redis with random vectors and return a VectorData object for verification."""
vectors = []
names = []
r.delete(key)
for i in range(count):
vec = generate_random_vector(dim)
name = f"{key}:item:{i}"
vectors.append(vec)
names.append(name)
vec_bytes = struct.pack(f'{dim}f', *vec)
args = [key]
if with_reduce:
args.extend(['REDUCE', with_reduce])
args.extend(['FP32', vec_bytes, name])
r.execute_command('VADD', *args)
return VectorData(vectors=vectors, names=names)
class TestCase:
def __init__(self):
self.error_msg = None
self.error_details = None
self.test_key = f"test:{self.__class__.__name__.lower()}"
# Primary Redis instance (default port)
self.redis = redis.Redis()
# Replica Redis instance (port 6380)
self.replica = redis.Redis(port=6380)
# Replication status
self.replication_setup = False
def setup(self):
self.redis.delete(self.test_key)
def teardown(self):
self.redis.delete(self.test_key)
def setup_replication(self) -> bool:
"""
Setup replication between primary and replica Redis instances.
Returns True if replication is successfully established, False otherwise.
"""
# Configure replica to replicate from primary
self.replica.execute_command('REPLICAOF', '127.0.0.1', 6379)
# Wait for replication to be established
max_attempts = 10
for attempt in range(max_attempts):
# Check replication info
repl_info = self.replica.info('replication')
# Check if replication is established
if (repl_info.get('role') == 'slave' and
repl_info.get('master_host') == '127.0.0.1' and
repl_info.get('master_port') == 6379 and
repl_info.get('master_link_status') == 'up'):
self.replication_setup = True
return True
# Wait before next attempt
time.sleep(0.5)
# If we get here, replication wasn't established
self.error_msg = "Failed to establish replication between primary and replica"
return False
def test(self):
raise NotImplementedError("Subclasses must implement test method")
def run(self):
try:
self.setup()
self.test()
return True
except AssertionError as e:
self.error_msg = str(e)
import traceback
self.error_details = traceback.format_exc()
return False
except Exception as e:
self.error_msg = f"Unexpected error: {str(e)}"
import traceback
self.error_details = traceback.format_exc()
return False
finally:
self.teardown()
def getname(self):
"""Each test class should override this to provide its name"""
return self.__class__.__name__
def estimated_runtime(self):
""""Each test class should override this if it takes a significant amount of time to run. Default is 100ms"""
return 0.1
def find_test_classes():
test_classes = []
tests_dir = 'tests'
if not os.path.exists(tests_dir):
return []
for file in os.listdir(tests_dir):
if file.endswith('.py'):
module_name = f"tests.{file[:-3]}"
try:
module = importlib.import_module(module_name)
for name, obj in inspect.getmembers(module):
if inspect.isclass(obj) and obj.__name__ != 'TestCase' and hasattr(obj, 'test'):
test_classes.append(obj())
except Exception as e:
print(f"Error loading {file}: {e}")
return test_classes
def run_tests():
print("================================================\n"+
"Make sure to have Redis running in the localhost\n"+
"with --enable-debug-command yes\n"+
"Both primary (6379) and replica (6380) instances\n"+
"================================================\n")
tests = find_test_classes()
if not tests:
print("No tests found!")
return
# Sort tests by estimated runtime
tests.sort(key=lambda t: t.estimated_runtime())
passed = 0
total = len(tests)
for test in tests:
print(f"{test.getname()}: ", end="")
sys.stdout.flush()
start_time = time.time()
success = test.run()
duration = time.time() - start_time
if success:
print(colored("OK", "green"), f"({duration:.2f}s)")
passed += 1
else:
print(colored("ERR", "red"), f"({duration:.2f}s)")
print(f"Error: {test.error_msg}")
if test.error_details:
print("\nTraceback:")
print(test.error_details)
print("\n" + "="*50)
print(f"\nTest Summary: {passed}/{total} tests passed")
if passed == total:
print(colored("\nALL TESTS PASSED!", "green"))
else:
print(colored(f"\n{total-passed} TESTS FAILED!", "red"))
if __name__ == "__main__":
run_tests()
modules/vector-sets/tests/basic_commands.py
+21
View File
@@ -0,0 +1,21 @@
from test import TestCase, generate_random_vector
import struct
class BasicCommands(TestCase):
def getname(self):
return "VADD, VDIM, VCARD basic usage"
def test(self):
# Test VADD
vec = generate_random_vector(4)
vec_bytes = struct.pack('4f', *vec)
result = self.redis.execute_command('VADD', self.test_key, 'FP32', vec_bytes, f'{self.test_key}:item:1')
assert result == 1, "VADD should return 1 for first item"
# Test VDIM
dim = self.redis.execute_command('VDIM', self.test_key)
assert dim == 4, f"VDIM should return 4, got {dim}"
# Test VCARD
card = self.redis.execute_command('VCARD', self.test_key)
assert card == 1, f"VCARD should return 1, got {card}"
modules/vector-sets/tests/basic_similarity.py
+35
View File
@@ -0,0 +1,35 @@
from test import TestCase
class BasicSimilarity(TestCase):
def getname(self):
return "VSIM reported distance makes sense with 4D vectors"
def test(self):
# Add two very similar vectors, one different
vec1 = [1, 0, 0, 0]
vec2 = [0.99, 0.01, 0, 0]
vec3 = [0.1, 1, -1, 0.5]
# Add vectors using VALUES format
self.redis.execute_command('VADD', self.test_key, 'VALUES', 4,
*[str(x) for x in vec1], f'{self.test_key}:item:1')
self.redis.execute_command('VADD', self.test_key, 'VALUES', 4,
*[str(x) for x in vec2], f'{self.test_key}:item:2')
self.redis.execute_command('VADD', self.test_key, 'VALUES', 4,
*[str(x) for x in vec3], f'{self.test_key}:item:3')
# Query similarity with vec1
result = self.redis.execute_command('VSIM', self.test_key, 'VALUES', 4,
*[str(x) for x in vec1], 'WITHSCORES')
# Convert results to dictionary
results_dict = {}
for i in range(0, len(result), 2):
key = result[i].decode()
score = float(result[i+1])
results_dict[key] = score
# Verify results
assert results_dict[f'{self.test_key}:item:1'] > 0.99, "Self-similarity should be very high"
assert results_dict[f'{self.test_key}:item:2'] > 0.99, "Similar vector should have high similarity"
assert results_dict[f'{self.test_key}:item:3'] < 0.8, "Not very similar vector should have low similarity"
modules/vector-sets/tests/concurrent_vadd_cas_del_vsim.py
+156
View File
@@ -0,0 +1,156 @@
from test import TestCase, generate_random_vector
import threading
import time
import struct
class ThreadingStressTest(TestCase):
def getname(self):
return "Concurrent VADD/DEL/VSIM operations stress test"
def estimated_runtime(self):
return 10 # Test runs for 10 seconds
def test(self):
# Constants - easy to modify if needed
NUM_VADD_THREADS = 10
NUM_VSIM_THREADS = 1
NUM_DEL_THREADS = 1
TEST_DURATION = 10 # seconds
VECTOR_DIM = 100
DEL_INTERVAL = 1 # seconds
# Shared flags and state
stop_event = threading.Event()
error_list = []
error_lock = threading.Lock()
def log_error(thread_name, error):
with error_lock:
error_list.append(f"{thread_name}: {error}")
def vadd_worker(thread_id):
"""Thread function to perform VADD operations"""
thread_name = f"VADD-{thread_id}"
try:
vector_count = 0
while not stop_event.is_set():
try:
# Generate random vector
vec = generate_random_vector(VECTOR_DIM)
vec_bytes = struct.pack(f'{VECTOR_DIM}f', *vec)
# Add vector with CAS option
self.redis.execute_command(
'VADD',
self.test_key,
'FP32',
vec_bytes,
f'{self.test_key}:item:{thread_id}:{vector_count}',
'CAS'
)
vector_count += 1
# Small sleep to reduce CPU pressure
if vector_count % 10 == 0:
time.sleep(0.001)
except Exception as e:
log_error(thread_name, f"Error: {str(e)}")
time.sleep(0.1) # Slight backoff on error
except Exception as e:
log_error(thread_name, f"Thread error: {str(e)}")
def del_worker():
"""Thread function that deletes the key periodically"""
thread_name = "DEL"
try:
del_count = 0
while not stop_event.is_set():
try:
# Sleep first, then delete
time.sleep(DEL_INTERVAL)
if stop_event.is_set():
break
self.redis.delete(self.test_key)
del_count += 1
except Exception as e:
log_error(thread_name, f"Error: {str(e)}")
except Exception as e:
log_error(thread_name, f"Thread error: {str(e)}")
def vsim_worker(thread_id):
"""Thread function to perform VSIM operations"""
thread_name = f"VSIM-{thread_id}"
try:
search_count = 0
while not stop_event.is_set():
try:
# Generate query vector
query_vec = generate_random_vector(VECTOR_DIM)
query_str = [str(x) for x in query_vec]
# Perform similarity search
args = ['VSIM', self.test_key, 'VALUES', VECTOR_DIM]
args.extend(query_str)
args.extend(['COUNT', 10])
self.redis.execute_command(*args)
search_count += 1
# Small sleep to reduce CPU pressure
if search_count % 10 == 0:
time.sleep(0.005)
except Exception as e:
# Don't log empty array errors, as they're expected when key doesn't exist
if "empty array" not in str(e).lower():
log_error(thread_name, f"Error: {str(e)}")
time.sleep(0.1) # Slight backoff on error
except Exception as e:
log_error(thread_name, f"Thread error: {str(e)}")
# Start all threads
threads = []
# VADD threads
for i in range(NUM_VADD_THREADS):
thread = threading.Thread(target=vadd_worker, args=(i,))
thread.start()
threads.append(thread)
# DEL threads
for _ in range(NUM_DEL_THREADS):
thread = threading.Thread(target=del_worker)
thread.start()
threads.append(thread)
# VSIM threads
for i in range(NUM_VSIM_THREADS):
thread = threading.Thread(target=vsim_worker, args=(i,))
thread.start()
threads.append(thread)
# Let the test run for the specified duration
time.sleep(TEST_DURATION)
# Signal all threads to stop
stop_event.set()
# Wait for threads to finish
for thread in threads:
thread.join(timeout=2.0)
# Check if Redis is still responsive
try:
ping_result = self.redis.ping()
assert ping_result, "Redis did not respond to PING after stress test"
except Exception as e:
assert False, f"Redis connection failed after stress test: {str(e)}"
# Report any errors for diagnosis, but don't fail the test unless PING fails
if error_list:
error_count = len(error_list)
print(f"\nEncountered {error_count} errors during stress test.")
print("First 5 errors:")
for error in error_list[:5]:
print(f"- {error}")
modules/vector-sets/tests/concurrent_vsim_and_del.py
+48
View File
@@ -0,0 +1,48 @@
from test import TestCase, fill_redis_with_vectors, generate_random_vector
import threading, time
class ConcurrentVSIMAndDEL(TestCase):
def getname(self):
return "Concurrent VSIM and DEL operations"
def estimated_runtime(self):
return 2
def test(self):
# Fill the key with 5000 random vectors
dim = 128
count = 5000
fill_redis_with_vectors(self.redis, self.test_key, count, dim)
# List to store results from threads
thread_results = []
def vsim_thread():
"""Thread function to perform VSIM operations until the key is deleted"""
while True:
query_vec = generate_random_vector(dim)
result = self.redis.execute_command('VSIM', self.test_key, 'VALUES', dim,
*[str(x) for x in query_vec], 'COUNT', 10)
if not result:
# Empty array detected, key is deleted
thread_results.append(True)
break
# Start multiple threads to perform VSIM operations
threads = []
for _ in range(4): # Start 4 threads
t = threading.Thread(target=vsim_thread)
t.start()
threads.append(t)
# Delete the key while threads are still running
time.sleep(1)
self.redis.delete(self.test_key)
# Wait for all threads to finish (they will exit once they detect the key is deleted)
for t in threads:
t.join()
# Verify that all threads detected an empty array or error
assert len(thread_results) == len(threads), "Not all threads detected the key deletion"
assert all(thread_results), "Some threads did not detect an empty array or error after DEL"
modules/vector-sets/tests/debug_digest.py
+39
View File
@@ -0,0 +1,39 @@
from test import TestCase, generate_random_vector
import struct
class DebugDigestTest(TestCase):
def getname(self):
return "[regression] DEBUG DIGEST-VALUE with attributes"
def test(self):
# Generate random vectors
vec1 = generate_random_vector(4)
vec2 = generate_random_vector(4)
vec_bytes1 = struct.pack('4f', *vec1)
vec_bytes2 = struct.pack('4f', *vec2)
# Add vectors to the key, one with attribute, one without
self.redis.execute_command('VADD', self.test_key, 'FP32', vec_bytes1, f'{self.test_key}:item:1')
self.redis.execute_command('VADD', self.test_key, 'FP32', vec_bytes2, f'{self.test_key}:item:2', 'SETATTR', '{"color":"red"}')
# Call DEBUG DIGEST-VALUE on the key
try:
digest1 = self.redis.execute_command('DEBUG', 'DIGEST-VALUE', self.test_key)
assert digest1 is not None, "DEBUG DIGEST-VALUE should return a value"
# Change attribute and verify digest changes
self.redis.execute_command('VSETATTR', self.test_key, f'{self.test_key}:item:2', '{"color":"blue"}')
digest2 = self.redis.execute_command('DEBUG', 'DIGEST-VALUE', self.test_key)
assert digest2 is not None, "DEBUG DIGEST-VALUE should return a value after attribute change"
assert digest1 != digest2, "Digest should change when an attribute is modified"
# Remove attribute and verify digest changes again
self.redis.execute_command('VSETATTR', self.test_key, f'{self.test_key}:item:2', '')
digest3 = self.redis.execute_command('DEBUG', 'DIGEST-VALUE', self.test_key)
assert digest3 is not None, "DEBUG DIGEST-VALUE should return a value after attribute removal"
assert digest2 != digest3, "Digest should change when an attribute is removed"
except Exception as e:
raise AssertionError(f"DEBUG DIGEST-VALUE command failed: {str(e)}")
modules/vector-sets/tests/deletion.py
+173
View File
@@ -0,0 +1,173 @@
from test import TestCase, fill_redis_with_vectors, generate_random_vector
import random
"""
A note about this test:
It was experimentally tried to modify hnsw.c in order to
avoid calling hnsw_reconnect_nodes(). In this case, the test
fails very often with EF set to 250, while it hardly
fails at all with the same parameters if hnsw_reconnect_nodes()
is called.
Note that for the nature of the test (it is very strict) it can
still fail from time to time, without this signaling any
actual bug.
"""
class VREM(TestCase):
def getname(self):
return "Deletion and graph state after deletion"
def estimated_runtime(self):
return 2.0
def format_neighbors_with_scores(self, links_result, old_links=None, items_to_remove=None):
"""Format neighbors with their similarity scores and status indicators"""
if not links_result:
return "No neighbors"
output = []
for level, neighbors in enumerate(links_result):
level_num = len(links_result) - level - 1
output.append(f"Level {level_num}:")
# Get neighbors and scores
neighbors_with_scores = []
for i in range(0, len(neighbors), 2):
neighbor = neighbors[i].decode() if isinstance(neighbors[i], bytes) else neighbors[i]
score = float(neighbors[i+1]) if i+1 < len(neighbors) else None
status = ""
# For old links, mark deleted ones
if items_to_remove and neighbor in items_to_remove:
status = " [lost]"
# For new links, mark newly added ones
elif old_links is not None:
# Check if this neighbor was in the old links at this level
was_present = False
if old_links and level < len(old_links):
old_neighbors = [n.decode() if isinstance(n, bytes) else n
for n in old_links[level]]
was_present = neighbor in old_neighbors
if not was_present:
status = " [gained]"
if score is not None:
neighbors_with_scores.append(f"{len(neighbors_with_scores)+1}. {neighbor} ({score:.6f}){status}")
else:
neighbors_with_scores.append(f"{len(neighbors_with_scores)+1}. {neighbor}{status}")
output.extend([" " + n for n in neighbors_with_scores])
return "\n".join(output)
def test(self):
# 1. Fill server with random elements
dim = 128
count = 5000
data = fill_redis_with_vectors(self.redis, self.test_key, count, dim)
# 2. Do VSIM to get 200 items
query_vec = generate_random_vector(dim)
results = self.redis.execute_command('VSIM', self.test_key, 'VALUES', dim,
*[str(x) for x in query_vec],
'COUNT', 200, 'WITHSCORES')
# Convert results to list of (item, score) pairs, sorted by score
items = []
for i in range(0, len(results), 2):
item = results[i].decode()
score = float(results[i+1])
items.append((item, score))
items.sort(key=lambda x: x[1], reverse=True) # Sort by similarity
# Store the graph structure for all items before deletion
neighbors_before = {}
for item, _ in items:
links = self.redis.execute_command('VLINKS', self.test_key, item, 'WITHSCORES')
if links: # Some items might not have links
neighbors_before[item] = links
# 3. Remove 100 random items
items_to_remove = set(item for item, _ in random.sample(items, 100))
# Keep track of top 10 non-removed items
top_remaining = []
for item, score in items:
if item not in items_to_remove:
top_remaining.append((item, score))
if len(top_remaining) == 10:
break
# Remove the items
for item in items_to_remove:
result = self.redis.execute_command('VREM', self.test_key, item)
assert result == 1, f"VREM failed to remove {item}"
# 4. Do VSIM again with same vector
new_results = self.redis.execute_command('VSIM', self.test_key, 'VALUES', dim,
*[str(x) for x in query_vec],
'COUNT', 200, 'WITHSCORES',
'EF', 500)
# Convert new results to dict of item -> score
new_scores = {}
for i in range(0, len(new_results), 2):
item = new_results[i].decode()
score = float(new_results[i+1])
new_scores[item] = score
failure = False
failed_item = None
failed_reason = None
# 5. Verify all top 10 non-removed items are still found with similar scores
for item, old_score in top_remaining:
if item not in new_scores:
failure = True
failed_item = item
failed_reason = "missing"
break
new_score = new_scores[item]
if abs(new_score - old_score) >= 0.01:
failure = True
failed_item = item
failed_reason = f"score changed: {old_score:.6f} -> {new_score:.6f}"
break
if failure:
print("\nTest failed!")
print(f"Problem with item: {failed_item} ({failed_reason})")
print("\nOriginal neighbors (with similarity scores):")
if failed_item in neighbors_before:
print(self.format_neighbors_with_scores(
neighbors_before[failed_item],
items_to_remove=items_to_remove))
else:
print("No neighbors found in original graph")
print("\nCurrent neighbors (with similarity scores):")
current_links = self.redis.execute_command('VLINKS', self.test_key,
failed_item, 'WITHSCORES')
if current_links:
print(self.format_neighbors_with_scores(
current_links,
old_links=neighbors_before.get(failed_item)))
else:
print("No neighbors in current graph")
print("\nOriginal results (top 20):")
for item, score in items[:20]:
deleted = "[deleted]" if item in items_to_remove else ""
print(f"{item}: {score:.6f} {deleted}")
print("\nNew results after removal (top 20):")
new_items = []
for i in range(0, len(new_results), 2):
item = new_results[i].decode()
score = float(new_results[i+1])
new_items.append((item, score))
new_items.sort(key=lambda x: x[1], reverse=True)
for item, score in new_items[:20]:
print(f"{item}: {score:.6f}")
raise AssertionError(f"Test failed: Problem with item {failed_item} ({failed_reason}). *** IMPORTANT *** This test may fail from time to time without indicating that there is a bug. However normally it should pass. The fact is that it's a quite extreme test where we destroy 50% of nodes of top results and still expect perfect recall, with vectors that are very hostile because of the distribution used.")
modules/vector-sets/tests/dimension_validation.py
+67
View File
@@ -0,0 +1,67 @@
from test import TestCase, generate_random_vector
import struct
import redis.exceptions
class DimensionValidation(TestCase):
def getname(self):
return "[regression] Dimension Validation with Projection"
def estimated_runtime(self):
return 0.5
def test(self):
# Test scenario 1: Create a set with projection
original_dim = 100
reduced_dim = 50
# Create the initial vector and set with projection
vec1 = generate_random_vector(original_dim)
vec1_bytes = struct.pack(f'{original_dim}f', *vec1)
# Add first vector with projection
result = self.redis.execute_command('VADD', self.test_key,
'REDUCE', reduced_dim,
'FP32', vec1_bytes, f'{self.test_key}:item:1')
assert result == 1, "First VADD with REDUCE should return 1"
# Check VINFO returns the correct projection information
info = self.redis.execute_command('VINFO', self.test_key)
info_map = {k.decode('utf-8'): v for k, v in zip(info[::2], info[1::2])}
assert 'vector-dim' in info_map, "VINFO should contain vector-dim"
assert info_map['vector-dim'] == reduced_dim, f"Expected reduced dimension {reduced_dim}, got {info['vector-dim']}"
assert 'projection-input-dim' in info_map, "VINFO should contain projection-input-dim"
assert info_map['projection-input-dim'] == original_dim, f"Expected original dimension {original_dim}, got {info['projection-input-dim']}"
# Test scenario 2: Try adding a mismatched vector - should fail
wrong_dim = 80
wrong_vec = generate_random_vector(wrong_dim)
wrong_vec_bytes = struct.pack(f'{wrong_dim}f', *wrong_vec)
# This should fail with dimension mismatch error
try:
self.redis.execute_command('VADD', self.test_key,
'REDUCE', reduced_dim,
'FP32', wrong_vec_bytes, f'{self.test_key}:item:2')
assert False, "VADD with wrong dimension should fail"
except redis.exceptions.ResponseError as e:
assert "Input dimension mismatch for projection" in str(e), f"Expected dimension mismatch error, got: {e}"
# Test scenario 3: Add a correctly-sized vector
vec2 = generate_random_vector(original_dim)
vec2_bytes = struct.pack(f'{original_dim}f', *vec2)
# This should succeed
result = self.redis.execute_command('VADD', self.test_key,
'REDUCE', reduced_dim,
'FP32', vec2_bytes, f'{self.test_key}:item:3')
assert result == 1, "VADD with correct dimensions should succeed"
# Check VSIM also validates input dimensions
wrong_query = generate_random_vector(wrong_dim)
try:
self.redis.execute_command('VSIM', self.test_key,
'VALUES', wrong_dim, *[str(x) for x in wrong_query],
'COUNT', 10)
assert False, "VSIM with wrong dimension should fail"
except redis.exceptions.ResponseError as e:
assert "Input dimension mismatch for projection" in str(e), f"Expected dimension mismatch error in VSIM, got: {e}"
modules/vector-sets/tests/evict_empty.py
+27
View File
@@ -0,0 +1,27 @@
from test import TestCase, generate_random_vector
import struct
class VREM_LastItemDeletesKey(TestCase):
def getname(self):
return "VREM last item deletes key"
def test(self):
# Generate a random vector
vec = generate_random_vector(4)
vec_bytes = struct.pack('4f', *vec)
# Add the vector to the key
result = self.redis.execute_command('VADD', self.test_key, 'FP32', vec_bytes, f'{self.test_key}:item:1')
assert result == 1, "VADD should return 1 for first item"
# Verify the key exists
exists = self.redis.exists(self.test_key)
assert exists == 1, "Key should exist after VADD"
# Remove the item
result = self.redis.execute_command('VREM', self.test_key, f'{self.test_key}:item:1')
assert result == 1, "VREM should return 1 for successful removal"
# Verify the key no longer exists
exists = self.redis.exists(self.test_key)
assert exists == 0, "Key should no longer exist after VREM of last item"
modules/vector-sets/tests/filter_expr.py
+177
View File
@@ -0,0 +1,177 @@
from test import TestCase
class VSIMFilterExpressions(TestCase):
def getname(self):
return "VSIM FILTER expressions basic functionality"
def test(self):
# Create a small set of vectors with different attributes
# Basic vectors for testing - all orthogonal for clear results
vec1 = [1, 0, 0, 0]
vec2 = [0, 1, 0, 0]
vec3 = [0, 0, 1, 0]
vec4 = [0, 0, 0, 1]
vec5 = [0.5, 0.5, 0, 0]
# Add vectors with various attributes
self.redis.execute_command('VADD', self.test_key, 'VALUES', 4,
*[str(x) for x in vec1], f'{self.test_key}:item:1')
self.redis.execute_command('VSETATTR', self.test_key, f'{self.test_key}:item:1',
'{"age": 25, "name": "Alice", "active": true, "scores": [85, 90, 95], "city": "New York"}')
self.redis.execute_command('VADD', self.test_key, 'VALUES', 4,
*[str(x) for x in vec2], f'{self.test_key}:item:2')
self.redis.execute_command('VSETATTR', self.test_key, f'{self.test_key}:item:2',
'{"age": 30, "name": "Bob", "active": false, "scores": [70, 75, 80], "city": "Boston"}')
self.redis.execute_command('VADD', self.test_key, 'VALUES', 4,
*[str(x) for x in vec3], f'{self.test_key}:item:3')
self.redis.execute_command('VSETATTR', self.test_key, f'{self.test_key}:item:3',
'{"age": 35, "name": "Charlie", "scores": [60, 65, 70], "city": "Seattle"}')
self.redis.execute_command('VADD', self.test_key, 'VALUES', 4,
*[str(x) for x in vec4], f'{self.test_key}:item:4')
# Item 4 has no attribute at all
self.redis.execute_command('VADD', self.test_key, 'VALUES', 4,
*[str(x) for x in vec5], f'{self.test_key}:item:5')
self.redis.execute_command('VSETATTR', self.test_key, f'{self.test_key}:item:5',
'invalid json') # Intentionally malformed JSON
# Test 1: Basic equality with numbers
result = self.redis.execute_command('VSIM', self.test_key, 'VALUES', 4,
*[str(x) for x in vec1],
'FILTER', '.age == 25')
assert len(result) == 1, "Expected 1 result for age == 25"
assert result[0].decode() == f'{self.test_key}:item:1', "Expected item:1 for age == 25"
# Test 2: Greater than
result = self.redis.execute_command('VSIM', self.test_key, 'VALUES', 4,
*[str(x) for x in vec1],
'FILTER', '.age > 25')
assert len(result) == 2, "Expected 2 results for age > 25"
# Test 3: Less than or equal
result = self.redis.execute_command('VSIM', self.test_key, 'VALUES', 4,
*[str(x) for x in vec1],
'FILTER', '.age <= 30')
assert len(result) == 2, "Expected 2 results for age <= 30"
# Test 4: String equality
result = self.redis.execute_command('VSIM', self.test_key, 'VALUES', 4,
*[str(x) for x in vec1],
'FILTER', '.name == "Alice"')
assert len(result) == 1, "Expected 1 result for name == Alice"
assert result[0].decode() == f'{self.test_key}:item:1', "Expected item:1 for name == Alice"
# Test 5: String inequality
result = self.redis.execute_command('VSIM', self.test_key, 'VALUES', 4,
*[str(x) for x in vec1],
'FILTER', '.name != "Alice"')
assert len(result) == 2, "Expected 2 results for name != Alice"
# Test 6: Boolean value
result = self.redis.execute_command('VSIM', self.test_key, 'VALUES', 4,
*[str(x) for x in vec1],
'FILTER', '.active')
assert len(result) == 1, "Expected 1 result for .active being true"
# Test 7: Logical AND
result = self.redis.execute_command('VSIM', self.test_key, 'VALUES', 4,
*[str(x) for x in vec1],
'FILTER', '.age > 20 and .age < 30')
assert len(result) == 1, "Expected 1 result for 20 < age < 30"
assert result[0].decode() == f'{self.test_key}:item:1', "Expected item:1 for 20 < age < 30"
# Test 8: Logical OR
result = self.redis.execute_command('VSIM', self.test_key, 'VALUES', 4,
*[str(x) for x in vec1],
'FILTER', '.age < 30 or .age > 35')
assert len(result) == 1, "Expected 1 result for age < 30 or age > 35"
# Test 9: Logical NOT
result = self.redis.execute_command('VSIM', self.test_key, 'VALUES', 4,
*[str(x) for x in vec1],
'FILTER', '!(.age == 25)')
assert len(result) == 2, "Expected 2 results for NOT(age == 25)"
# Test 10: The "in" operator with array
result = self.redis.execute_command('VSIM', self.test_key, 'VALUES', 4,
*[str(x) for x in vec1],
'FILTER', '.age in [25, 35]')
assert len(result) == 2, "Expected 2 results for age in [25, 35]"
# Test 11: The "in" operator with strings in array
result = self.redis.execute_command('VSIM', self.test_key, 'VALUES', 4,
*[str(x) for x in vec1],
'FILTER', '.name in ["Alice", "David"]')
assert len(result) == 1, "Expected 1 result for name in [Alice, David]"
# Test 12: Arithmetic operations - addition
result = self.redis.execute_command('VSIM', self.test_key, 'VALUES', 4,
*[str(x) for x in vec1],
'FILTER', '.age + 10 > 40')
assert len(result) == 1, "Expected 1 result for age + 10 > 40"
# Test 13: Arithmetic operations - multiplication
result = self.redis.execute_command('VSIM', self.test_key, 'VALUES', 4,
*[str(x) for x in vec1],
'FILTER', '.age * 2 > 60')
assert len(result) == 1, "Expected 1 result for age * 2 > 60"
# Test 14: Arithmetic operations - division
result = self.redis.execute_command('VSIM', self.test_key, 'VALUES', 4,
*[str(x) for x in vec1],
'FILTER', '.age / 5 == 5')
assert len(result) == 1, "Expected 1 result for age / 5 == 5"
# Test 15: Arithmetic operations - modulo
result = self.redis.execute_command('VSIM', self.test_key, 'VALUES', 4,
*[str(x) for x in vec1],
'FILTER', '.age % 2 == 0')
assert len(result) == 1, "Expected 1 result for age % 2 == 0"
# Test 16: Power operator
result = self.redis.execute_command('VSIM', self.test_key, 'VALUES', 4,
*[str(x) for x in vec1],
'FILTER', '.age ** 2 > 900')
assert len(result) == 1, "Expected 1 result for age^2 > 900"
# Test 17: Missing attribute (should exclude items missing that attribute)
result = self.redis.execute_command('VSIM', self.test_key, 'VALUES', 4,
*[str(x) for x in vec1],
'FILTER', '.missing_field == "value"')
assert len(result) == 0, "Expected 0 results for missing_field == value"
# Test 18: No attribute set at all
result = self.redis.execute_command('VSIM', self.test_key, 'VALUES', 4,
*[str(x) for x in vec1],
'FILTER', '.any_field')
assert f'{self.test_key}:item:4' not in [item.decode() for item in result], "Item with no attribute should be excluded"
# Test 19: Malformed JSON
result = self.redis.execute_command('VSIM', self.test_key, 'VALUES', 4,
*[str(x) for x in vec1],
'FILTER', '.any_field')
assert f'{self.test_key}:item:5' not in [item.decode() for item in result], "Item with malformed JSON should be excluded"
# Test 20: Complex expression combining multiple operators
result = self.redis.execute_command('VSIM', self.test_key, 'VALUES', 4,
*[str(x) for x in vec1],
'FILTER', '(.age > 20 and .age < 40) and (.city == "Boston" or .city == "New York")')
assert len(result) == 2, "Expected 2 results for the complex expression"
expected_items = [f'{self.test_key}:item:1', f'{self.test_key}:item:2']
assert set([item.decode() for item in result]) == set(expected_items), "Expected item:1 and item:2 for the complex expression"
# Test 21: Parentheses to control operator precedence
result = self.redis.execute_command('VSIM', self.test_key, 'VALUES', 4,
*[str(x) for x in vec1],
'FILTER', '.age > (20 + 10)')
assert len(result) == 1, "Expected 1 result for age > (20 + 10)"
# Test 22: Array access (arrays evaluate to true)
result = self.redis.execute_command('VSIM', self.test_key, 'VALUES', 4,
*[str(x) for x in vec1],
'FILTER', '.scores')
assert len(result) == 3, "Expected 3 results for .scores (arrays evaluate to true)"
modules/vector-sets/tests/filter_int.py
+668
View File
@@ -0,0 +1,668 @@
from test import TestCase, generate_random_vector
import struct
import random
import math
import json
import time
class VSIMFilterAdvanced(TestCase):
def getname(self):
return "VSIM FILTER comprehensive functionality testing"
def estimated_runtime(self):
return 15 # This test might take up to 15 seconds for the large dataset
def setup(self):
super().setup()
self.dim = 32 # Vector dimension
self.count = 5000 # Number of vectors for large tests
self.small_count = 50 # Number of vectors for small/quick tests
# Categories for attributes
self.categories = ["electronics", "furniture", "clothing", "books", "food"]
self.cities = ["New York", "London", "Tokyo", "Paris", "Berlin", "Sydney", "Toronto", "Singapore"]
self.price_ranges = [(10, 50), (50, 200), (200, 1000), (1000, 5000)]
self.years = list(range(2000, 2025))
def create_attributes(self, index):
"""Create realistic attributes for a vector"""
category = random.choice(self.categories)
city = random.choice(self.cities)
min_price, max_price = random.choice(self.price_ranges)
price = round(random.uniform(min_price, max_price), 2)
year = random.choice(self.years)
in_stock = random.random() > 0.3 # 70% chance of being in stock
rating = round(random.uniform(1, 5), 1)
views = int(random.expovariate(1/1000)) # Exponential distribution for page views
tags = random.sample(["popular", "sale", "new", "limited", "exclusive", "clearance"],
k=random.randint(0, 3))
# Add some specific patterns for testing
# Every 10th item has a specific property combination for testing
is_premium = (index % 10 == 0)
# Create attributes dictionary
attrs = {
"id": index,
"category": category,
"location": city,
"price": price,
"year": year,
"in_stock": in_stock,
"rating": rating,
"views": views,
"tags": tags
}
if is_premium:
attrs["is_premium"] = True
attrs["special_features"] = ["premium", "warranty", "support"]
# Add sub-categories for more complex filters
if category == "electronics":
attrs["subcategory"] = random.choice(["phones", "computers", "cameras", "audio"])
elif category == "furniture":
attrs["subcategory"] = random.choice(["chairs", "tables", "sofas", "beds"])
elif category == "clothing":
attrs["subcategory"] = random.choice(["shirts", "pants", "dresses", "shoes"])
# Add some intentionally missing fields for testing
if random.random() > 0.9: # 10% chance of missing price
del attrs["price"]
# Some items have promotion field
if random.random() > 0.7: # 30% chance of having a promotion
attrs["promotion"] = random.choice(["discount", "bundle", "gift"])
# Create invalid JSON for a small percentage of vectors
if random.random() > 0.98: # 2% chance of having invalid JSON
return "{{invalid json}}"
return json.dumps(attrs)
def create_vectors_with_attributes(self, key, count):
"""Create vectors and add attributes to them"""
vectors = []
names = []
attribute_map = {} # To store attributes for verification
# Create vectors
for i in range(count):
vec = generate_random_vector(self.dim)
vectors.append(vec)
name = f"{key}:item:{i}"
names.append(name)
# Add to Redis
vec_bytes = struct.pack(f'{self.dim}f', *vec)
self.redis.execute_command('VADD', key, 'FP32', vec_bytes, name)
# Create and add attributes
attrs = self.create_attributes(i)
self.redis.execute_command('VSETATTR', key, name, attrs)
# Store attributes for later verification
try:
attribute_map[name] = json.loads(attrs) if '{' in attrs else None
except json.JSONDecodeError:
attribute_map[name] = None
return vectors, names, attribute_map
def filter_linear_search(self, vectors, names, query_vector, filter_expr, attribute_map, k=10):
"""Perform a linear search with filtering for verification"""
similarities = []
query_norm = math.sqrt(sum(x*x for x in query_vector))
if query_norm == 0:
return []
for i, vec in enumerate(vectors):
name = names[i]
attributes = attribute_map.get(name)
# Skip if doesn't match filter
if not self.matches_filter(attributes, filter_expr):
continue
vec_norm = math.sqrt(sum(x*x for x in vec))
if vec_norm == 0:
continue
dot_product = sum(a*b for a,b in zip(query_vector, vec))
cosine_sim = dot_product / (query_norm * vec_norm)
distance = 1.0 - cosine_sim
redis_similarity = 1.0 - (distance/2.0)
similarities.append((name, redis_similarity))
similarities.sort(key=lambda x: x[1], reverse=True)
return similarities[:k]
def matches_filter(self, attributes, filter_expr):
"""Filter matching for verification - uses Python eval to handle complex expressions"""
if attributes is None:
return False # No attributes or invalid JSON
# Replace JSON path selectors with Python dictionary access
py_expr = filter_expr
# Handle `.field` notation (replace with attributes['field'])
i = 0
while i < len(py_expr):
if py_expr[i] == '.' and (i == 0 or not py_expr[i-1].isalnum()):
# Find the end of the selector (stops at operators or whitespace)
j = i + 1
while j < len(py_expr) and (py_expr[j].isalnum() or py_expr[j] == '_'):
j += 1
if j > i + 1: # Found a valid selector
field = py_expr[i+1:j]
# Use a safe access pattern that returns a default value based on context
py_expr = py_expr[:i] + f"attributes.get('{field}')" + py_expr[j:]
i = i + len(f"attributes.get('{field}')")
else:
i += 1
else:
i += 1
# Convert not operator if needed
py_expr = py_expr.replace('!', ' not ')
try:
# Custom evaluation that handles exceptions for missing fields
# by returning False for the entire expression
# Split the expression on logical operators
parts = []
for op in [' and ', ' or ']:
if op in py_expr:
parts = py_expr.split(op)
break
if not parts: # No logical operators found
parts = [py_expr]
# Try to evaluate each part - if any part fails,
# the whole expression should fail
try:
result = eval(py_expr, {"attributes": attributes})
return bool(result)
except (TypeError, AttributeError):
# This typically happens when trying to compare None with
# numbers or other types, or when an attribute doesn't exist
return False
except Exception as e:
print(f"Error evaluating filter expression '{filter_expr}' as '{py_expr}': {e}")
return False
except Exception as e:
print(f"Error evaluating filter expression '{filter_expr}' as '{py_expr}': {e}")
return False
def safe_decode(self,item):
return item.decode() if isinstance(item, bytes) else item
def calculate_recall(self, redis_results, linear_results, k=10):
"""Calculate recall (percentage of correct results retrieved)"""
redis_set = set(self.safe_decode(item) for item in redis_results)
linear_set = set(item[0] for item in linear_results[:k])
if not linear_set:
return 1.0 # If no linear results, consider it perfect recall
intersection = redis_set.intersection(linear_set)
return len(intersection) / len(linear_set)
def test_recall_with_filter(self, filter_expr, ef=500, filter_ef=None):
"""Test recall for a given filter expression"""
# Create query vector
query_vec = generate_random_vector(self.dim)
# First, get ground truth using linear scan
linear_results = self.filter_linear_search(
self.vectors, self.names, query_vec, filter_expr, self.attribute_map, k=50)
# Calculate true selectivity from ground truth
true_selectivity = len(linear_results) / len(self.names) if self.names else 0
# Perform Redis search with filter
cmd_args = ['VSIM', self.test_key, 'VALUES', self.dim]
cmd_args.extend([str(x) for x in query_vec])
cmd_args.extend(['COUNT', 50, 'WITHSCORES', 'EF', ef, 'FILTER', filter_expr])
if filter_ef:
cmd_args.extend(['FILTER-EF', filter_ef])
start_time = time.time()
redis_results = self.redis.execute_command(*cmd_args)
query_time = time.time() - start_time
# Convert Redis results to dict
redis_items = {}
for i in range(0, len(redis_results), 2):
key = redis_results[i].decode() if isinstance(redis_results[i], bytes) else redis_results[i]
score = float(redis_results[i+1])
redis_items[key] = score
# Calculate metrics
recall = self.calculate_recall(redis_items.keys(), linear_results)
selectivity = len(redis_items) / len(self.names) if redis_items else 0
# Compare against the true selectivity from linear scan
assert abs(selectivity - true_selectivity) < 0.1, \
f"Redis selectivity {selectivity:.3f} differs significantly from ground truth {true_selectivity:.3f}"
# We expect high recall for standard parameters
if ef >= 500 and (filter_ef is None or filter_ef >= 1000):
try:
assert recall >= 0.7, \
f"Low recall {recall:.2f} for filter '{filter_expr}'"
except AssertionError as e:
# Get items found in each set
redis_items_set = set(redis_items.keys())
linear_items_set = set(item[0] for item in linear_results)
# Find items in each set
only_in_redis = redis_items_set - linear_items_set
only_in_linear = linear_items_set - redis_items_set
in_both = redis_items_set & linear_items_set
# Build comprehensive debug message
debug = f"\nGround Truth: {len(linear_results)} matching items (total vectors: {len(self.vectors)})"
debug += f"\nRedis Found: {len(redis_items)} items with FILTER-EF: {filter_ef or 'default'}"
debug += f"\nItems in both sets: {len(in_both)} (recall: {recall:.4f})"
debug += f"\nItems only in Redis: {len(only_in_redis)}"
debug += f"\nItems only in Ground Truth: {len(only_in_linear)}"
# Show some example items from each set with their scores
if only_in_redis:
debug += "\n\nTOP 5 ITEMS ONLY IN REDIS:"
sorted_redis = sorted([(k, v) for k, v in redis_items.items()], key=lambda x: x[1], reverse=True)
for i, (item, score) in enumerate(sorted_redis[:5]):
if item in only_in_redis:
debug += f"\n {i+1}. {item} (Score: {score:.4f})"
# Show attribute that should match filter
attr = self.attribute_map.get(item)
if attr:
debug += f" - Attrs: {attr.get('category', 'N/A')}, Price: {attr.get('price', 'N/A')}"
if only_in_linear:
debug += "\n\nTOP 5 ITEMS ONLY IN GROUND TRUTH:"
for i, (item, score) in enumerate(linear_results[:5]):
if item in only_in_linear:
debug += f"\n {i+1}. {item} (Score: {score:.4f})"
# Show attribute that should match filter
attr = self.attribute_map.get(item)
if attr:
debug += f" - Attrs: {attr.get('category', 'N/A')}, Price: {attr.get('price', 'N/A')}"
# Help identify parsing issues
debug += "\n\nPARSING CHECK:"
debug += f"\nRedis command: VSIM {self.test_key} VALUES {self.dim} [...] FILTER '{filter_expr}'"
# Check for WITHSCORES handling issues
if len(redis_results) > 0 and len(redis_results) % 2 == 0:
debug += f"\nRedis returned {len(redis_results)} items (looks like item,score pairs)"
debug += f"\nFirst few results: {redis_results[:4]}"
# Check the filter implementation
debug += "\n\nFILTER IMPLEMENTATION CHECK:"
debug += f"\nFilter expression: '{filter_expr}'"
debug += "\nSample attribute matches from attribute_map:"
count_matching = 0
for i, (name, attrs) in enumerate(self.attribute_map.items()):
if attrs and self.matches_filter(attrs, filter_expr):
count_matching += 1
if i < 3: # Show first 3 matches
debug += f"\n - {name}: {attrs}"
debug += f"\nTotal items matching filter in attribute_map: {count_matching}"
# Check if results array handling could be wrong
debug += "\n\nRESULT ARRAYS CHECK:"
if len(linear_results) >= 1:
debug += f"\nlinear_results[0]: {linear_results[0]}"
if isinstance(linear_results[0], tuple) and len(linear_results[0]) == 2:
debug += " (correct tuple format: (name, score))"
else:
debug += " (UNEXPECTED FORMAT!)"
# Debug sort order
debug += "\n\nSORTING CHECK:"
if len(linear_results) >= 2:
debug += f"\nGround truth first item score: {linear_results[0][1]}"
debug += f"\nGround truth second item score: {linear_results[1][1]}"
debug += f"\nCorrectly sorted by similarity? {linear_results[0][1] >= linear_results[1][1]}"
# Re-raise with detailed information
raise AssertionError(str(e) + debug)
return recall, selectivity, query_time, len(redis_items)
def test(self):
print(f"\nRunning comprehensive VSIM FILTER tests...")
# Create a larger dataset for testing
print(f"Creating dataset with {self.count} vectors and attributes...")
self.vectors, self.names, self.attribute_map = self.create_vectors_with_attributes(
self.test_key, self.count)
# ==== 1. Recall and Precision Testing ====
print("Testing recall for various filters...")
# Test basic filters with different selectivity
results = {}
results["category"] = self.test_recall_with_filter('.category == "electronics"')
results["price_high"] = self.test_recall_with_filter('.price > 1000')
results["in_stock"] = self.test_recall_with_filter('.in_stock')
results["rating"] = self.test_recall_with_filter('.rating >= 4')
results["complex1"] = self.test_recall_with_filter('.category == "electronics" and .price < 500')
print("Filter | Recall | Selectivity | Time (ms) | Results")
print("----------------------------------------------------")
for name, (recall, selectivity, time_ms, count) in results.items():
print(f"{name:7} | {recall:.3f} | {selectivity:.3f} | {time_ms*1000:.1f} | {count}")
# ==== 2. Filter Selectivity Performance ====
print("\nTesting filter selectivity performance...")
# High selectivity (very few matches)
high_sel_recall, _, high_sel_time, _ = self.test_recall_with_filter('.is_premium')
# Medium selectivity
med_sel_recall, _, med_sel_time, _ = self.test_recall_with_filter('.price > 100 and .price < 1000')
# Low selectivity (many matches)
low_sel_recall, _, low_sel_time, _ = self.test_recall_with_filter('.year > 2000')
print(f"High selectivity recall: {high_sel_recall:.3f}, time: {high_sel_time*1000:.1f}ms")
print(f"Med selectivity recall: {med_sel_recall:.3f}, time: {med_sel_time*1000:.1f}ms")
print(f"Low selectivity recall: {low_sel_recall:.3f}, time: {low_sel_time*1000:.1f}ms")
# ==== 3. FILTER-EF Parameter Testing ====
print("\nTesting FILTER-EF parameter...")
# Test with different FILTER-EF values
filter_expr = '.category == "electronics" and .price > 200'
ef_values = [100, 500, 2000, 5000]
print("FILTER-EF | Recall | Time (ms)")
print("-----------------------------")
for filter_ef in ef_values:
recall, _, query_time, _ = self.test_recall_with_filter(
filter_expr, ef=500, filter_ef=filter_ef)
print(f"{filter_ef:9} | {recall:.3f} | {query_time*1000:.1f}")
# Assert that higher FILTER-EF generally gives better recall
low_ef_recall, _, _, _ = self.test_recall_with_filter(filter_expr, filter_ef=100)
high_ef_recall, _, _, _ = self.test_recall_with_filter(filter_expr, filter_ef=5000)
# This might not always be true due to randomness, but generally holds
# We use a softer assertion to avoid flaky tests
assert high_ef_recall >= low_ef_recall * 0.8, \
f"Higher FILTER-EF should generally give better recall: {high_ef_recall:.3f} vs {low_ef_recall:.3f}"
# ==== 4. Complex Filter Expressions ====
print("\nTesting complex filter expressions...")
# Test a variety of complex expressions
complex_filters = [
'.price > 100 and (.category == "electronics" or .category == "furniture")',
'(.rating > 4 and .in_stock) or (.price < 50 and .views > 1000)',
'.category in ["electronics", "clothing"] and .price > 200 and .rating >= 3',
'(.category == "electronics" and .subcategory == "phones") or (.category == "furniture" and .price > 1000)',
'.year > 2010 and !(.price < 100) and .in_stock'
]
print("Expression | Results | Time (ms)")
print("-----------------------------")
for i, expr in enumerate(complex_filters):
try:
_, _, query_time, result_count = self.test_recall_with_filter(expr)
print(f"Complex {i+1} | {result_count:7} | {query_time*1000:.1f}")
except Exception as e:
print(f"Complex {i+1} | Error: {str(e)}")
# ==== 5. Attribute Type Testing ====
print("\nTesting different attribute types...")
type_filters = [
('.price > 500', "Numeric"),
('.category == "books"', "String equality"),
('.in_stock', "Boolean"),
('.tags in ["sale", "new"]', "Array membership"),
('.rating * 2 > 8', "Arithmetic")
]
for expr, type_name in type_filters:
try:
_, _, query_time, result_count = self.test_recall_with_filter(expr)
print(f"{type_name:16} | {expr:30} | {result_count:5} results | {query_time*1000:.1f}ms")
except Exception as e:
print(f"{type_name:16} | {expr:30} | Error: {str(e)}")
# ==== 6. Filter + Count Interaction ====
print("\nTesting COUNT parameter with filters...")
filter_expr = '.category == "electronics"'
counts = [5, 20, 100]
for count in counts:
query_vec = generate_random_vector(self.dim)
cmd_args = ['VSIM', self.test_key, 'VALUES', self.dim]
cmd_args.extend([str(x) for x in query_vec])
cmd_args.extend(['COUNT', count, 'WITHSCORES', 'FILTER', filter_expr])
results = self.redis.execute_command(*cmd_args)
result_count = len(results) // 2 # Divide by 2 because WITHSCORES returns pairs
# We expect result count to be at most the requested count
assert result_count <= count, f"Got {result_count} results with COUNT {count}"
print(f"COUNT {count:3} | Got {result_count:3} results")
# ==== 7. Edge Cases ====
print("\nTesting edge cases...")
# Test with no matching items
no_match_expr = '.category == "nonexistent_category"'
results = self.redis.execute_command('VSIM', self.test_key, 'VALUES', self.dim,
*[str(x) for x in generate_random_vector(self.dim)],
'FILTER', no_match_expr)
assert len(results) == 0, f"Expected 0 results for non-matching filter, got {len(results)}"
print(f"No matching items: {len(results)} results (expected 0)")
# Test with invalid filter syntax
try:
self.redis.execute_command('VSIM', self.test_key, 'VALUES', self.dim,
*[str(x) for x in generate_random_vector(self.dim)],
'FILTER', '.category === "books"') # Triple equals is invalid
assert False, "Expected error for invalid filter syntax"
except:
print("Invalid filter syntax correctly raised an error")
# Test with extremely long complex expression
long_expr = ' and '.join([f'.rating > {i/10}' for i in range(10)])
try:
results = self.redis.execute_command('VSIM', self.test_key, 'VALUES', self.dim,
*[str(x) for x in generate_random_vector(self.dim)],
'FILTER', long_expr)
print(f"Long expression: {len(results)} results")
except Exception as e:
print(f"Long expression error: {str(e)}")
print("\nComprehensive VSIM FILTER tests completed successfully")
class VSIMFilterSelectivityTest(TestCase):
def getname(self):
return "VSIM FILTER selectivity performance benchmark"
def estimated_runtime(self):
return 8 # This test might take up to 8 seconds
def setup(self):
super().setup()
self.dim = 32
self.count = 10000
self.test_key = f"{self.test_key}:selectivity" # Use a different key
def create_vector_with_age_attribute(self, name, age):
"""Create a vector with a specific age attribute"""
vec = generate_random_vector(self.dim)
vec_bytes = struct.pack(f'{self.dim}f', *vec)
self.redis.execute_command('VADD', self.test_key, 'FP32', vec_bytes, name)
self.redis.execute_command('VSETATTR', self.test_key, name, json.dumps({"age": age}))
def test(self):
print("\nRunning VSIM FILTER selectivity benchmark...")
# Create a dataset where we control the exact selectivity
print(f"Creating controlled dataset with {self.count} vectors...")
# Create vectors with age attributes from 1 to 100
for i in range(self.count):
age = (i % 100) + 1 # Ages from 1 to 100
name = f"{self.test_key}:item:{i}"
self.create_vector_with_age_attribute(name, age)
# Create a query vector
query_vec = generate_random_vector(self.dim)
# Test filters with different selectivities
selectivities = [0.01, 0.05, 0.10, 0.25, 0.50, 0.75, 0.99]
results = []
print("\nSelectivity | Filter | Results | Time (ms)")
print("--------------------------------------------------")
for target_selectivity in selectivities:
# Calculate age threshold for desired selectivity
# For example, age <= 10 gives 10% selectivity
age_threshold = int(target_selectivity * 100)
filter_expr = f'.age <= {age_threshold}'
# Run query and measure time
start_time = time.time()
cmd_args = ['VSIM', self.test_key, 'VALUES', self.dim]
cmd_args.extend([str(x) for x in query_vec])
cmd_args.extend(['COUNT', 100, 'FILTER', filter_expr])
results = self.redis.execute_command(*cmd_args)
query_time = time.time() - start_time
actual_selectivity = len(results) / min(100, int(target_selectivity * self.count))
print(f"{target_selectivity:.2f} | {filter_expr:15} | {len(results):7} | {query_time*1000:.1f}")
# Add assertion to ensure reasonable performance for different selectivities
# For very selective queries (1%), we might need more exploration
if target_selectivity <= 0.05:
# For very selective queries, ensure we can find some results
assert len(results) > 0, f"No results found for {filter_expr}"
else:
# For less selective queries, performance should be reasonable
assert query_time < 1.0, f"Query too slow: {query_time:.3f}s for {filter_expr}"
print("\nSelectivity benchmark completed successfully")
class VSIMFilterComparisonTest(TestCase):
def getname(self):
return "VSIM FILTER EF parameter comparison"
def estimated_runtime(self):
return 8 # This test might take up to 8 seconds
def setup(self):
super().setup()
self.dim = 32
self.count = 5000
self.test_key = f"{self.test_key}:efparams" # Use a different key
def create_dataset(self):
"""Create a dataset with specific attribute patterns for testing FILTER-EF"""
vectors = []
names = []
# Create vectors with category and quality score attributes
for i in range(self.count):
vec = generate_random_vector(self.dim)
name = f"{self.test_key}:item:{i}"
# Add vector to Redis
vec_bytes = struct.pack(f'{self.dim}f', *vec)
self.redis.execute_command('VADD', self.test_key, 'FP32', vec_bytes, name)
# Create attributes - we want a very selective filter
# Only 2% of items have category=premium AND quality>90
category = "premium" if random.random() < 0.1 else random.choice(["standard", "economy", "basic"])
quality = random.randint(1, 100)
attrs = {
"id": i,
"category": category,
"quality": quality
}
self.redis.execute_command('VSETATTR', self.test_key, name, json.dumps(attrs))
vectors.append(vec)
names.append(name)
return vectors, names
def test(self):
print("\nRunning VSIM FILTER-EF parameter comparison...")
# Create dataset
vectors, names = self.create_dataset()
# Create a selective filter that matches ~2% of items
filter_expr = '.category == "premium" and .quality > 90'
# Create query vector
query_vec = generate_random_vector(self.dim)
# Test different FILTER-EF values
ef_values = [50, 100, 500, 1000, 5000]
results = []
print("\nFILTER-EF | Results | Time (ms) | Notes")
print("---------------------------------------")
baseline_count = None
for ef in ef_values:
# Run query and measure time
start_time = time.time()
cmd_args = ['VSIM', self.test_key, 'VALUES', self.dim]
cmd_args.extend([str(x) for x in query_vec])
cmd_args.extend(['COUNT', 100, 'FILTER', filter_expr, 'FILTER-EF', ef])
query_results = self.redis.execute_command(*cmd_args)
query_time = time.time() - start_time
# Set baseline for comparison
if baseline_count is None:
baseline_count = len(query_results)
recall_rate = len(query_results) / max(1, baseline_count) if baseline_count > 0 else 1.0
notes = ""
if ef == 5000:
notes = "Baseline"
elif recall_rate < 0.5:
notes = "Low recall!"
print(f"{ef:9} | {len(query_results):7} | {query_time*1000:.1f} | {notes}")
results.append((ef, len(query_results), query_time))
# If we have enough results at highest EF, check that recall improves with higher EF
if results[-1][1] >= 5: # At least 5 results for highest EF
# Extract result counts
result_counts = [r[1] for r in results]
# The last result (highest EF) should typically find more results than the first (lowest EF)
# but we use a soft assertion to avoid flaky tests
assert result_counts[-1] >= result_counts[0], \
f"Higher FILTER-EF should find at least as many results: {result_counts[-1]} vs {result_counts[0]}"
print("\nFILTER-EF parameter comparison completed successfully")
modules/vector-sets/tests/large_scale.py
+56
View File
@@ -0,0 +1,56 @@
from test import TestCase, fill_redis_with_vectors, generate_random_vector
import random
class LargeScale(TestCase):
def getname(self):
return "Large Scale Comparison"
def estimated_runtime(self):
return 10
def test(self):
dim = 300
count = 20000
k = 50
# Fill Redis and get reference data for comparison
random.seed(42) # Make test deterministic
data = fill_redis_with_vectors(self.redis, self.test_key, count, dim)
# Generate query vector
query_vec = generate_random_vector(dim)
# Get results from Redis with good exploration factor
redis_raw = self.redis.execute_command('VSIM', self.test_key, 'VALUES', dim,
*[str(x) for x in query_vec],
'COUNT', k, 'WITHSCORES', 'EF', 500)
# Convert Redis results to dict
redis_results = {}
for i in range(0, len(redis_raw), 2):
key = redis_raw[i].decode()
score = float(redis_raw[i+1])
redis_results[key] = score
# Get results from linear scan
linear_results = data.find_k_nearest(query_vec, k)
linear_items = {name: score for name, score in linear_results}
# Compare overlap
redis_set = set(redis_results.keys())
linear_set = set(linear_items.keys())
overlap = len(redis_set & linear_set)
# If test fails, print comparison for debugging
if overlap < k * 0.7:
data.print_comparison({'items': redis_results, 'query_vector': query_vec}, k)
assert overlap >= k * 0.7, \
f"Expected at least 70% overlap in top {k} results, got {overlap/k*100:.1f}%"
# Verify scores for common items
for item in redis_set & linear_set:
redis_score = redis_results[item]
linear_score = linear_items[item]
assert abs(redis_score - linear_score) < 0.01, \
f"Score mismatch for {item}: Redis={redis_score:.3f} Linear={linear_score:.3f}"
modules/vector-sets/tests/memory_usage.py
+36
View File
@@ -0,0 +1,36 @@
from test import TestCase, generate_random_vector
import struct
class MemoryUsageTest(TestCase):
def getname(self):
return "[regression] MEMORY USAGE with attributes"
def test(self):
# Generate random vectors
vec1 = generate_random_vector(4)
vec2 = generate_random_vector(4)
vec_bytes1 = struct.pack('4f', *vec1)
vec_bytes2 = struct.pack('4f', *vec2)
# Add vectors to the key, one with attribute, one without
self.redis.execute_command('VADD', self.test_key, 'FP32', vec_bytes1, f'{self.test_key}:item:1')
self.redis.execute_command('VADD', self.test_key, 'FP32', vec_bytes2, f'{self.test_key}:item:2', 'SETATTR', '{"color":"red"}')
# Get memory usage for the key
try:
memory_usage = self.redis.execute_command('MEMORY', 'USAGE', self.test_key)
# If we got here without exception, the command worked
assert memory_usage > 0, "MEMORY USAGE should return a positive value"
# Add more attributes to increase complexity
self.redis.execute_command('VSETATTR', self.test_key, f'{self.test_key}:item:1', '{"color":"blue","size":10}')
# Check memory usage again
new_memory_usage = self.redis.execute_command('MEMORY', 'USAGE', self.test_key)
assert new_memory_usage > 0, "MEMORY USAGE should still return a positive value after setting attributes"
# Memory usage should be higher after adding attributes
assert new_memory_usage > memory_usage, "Memory usage increase after adding attributes"
except Exception as e:
raise AssertionError(f"MEMORY USAGE command failed: {str(e)}")
modules/vector-sets/tests/node_update.py
+85
View File
@@ -0,0 +1,85 @@
from test import TestCase, generate_random_vector
import struct
import math
import random
class VectorUpdateAndClusters(TestCase):
def getname(self):
return "VADD vector update with cluster relocation"
def estimated_runtime(self):
return 2.0 # Should take around 2 seconds
def generate_cluster_vector(self, base_vec, noise=0.1):
"""Generate a vector that's similar to base_vec with some noise."""
vec = [x + random.gauss(0, noise) for x in base_vec]
# Normalize
norm = math.sqrt(sum(x*x for x in vec))
return [x/norm for x in vec]
def test(self):
dim = 128
vectors_per_cluster = 5000
# Create two very different base vectors for our clusters
cluster1_base = generate_random_vector(dim)
cluster2_base = [-x for x in cluster1_base] # Opposite direction
# Add vectors from first cluster
for i in range(vectors_per_cluster):
vec = self.generate_cluster_vector(cluster1_base)
vec_bytes = struct.pack(f'{dim}f', *vec)
self.redis.execute_command('VADD', self.test_key, 'FP32', vec_bytes,
f'{self.test_key}:cluster1:{i}')
# Add vectors from second cluster
for i in range(vectors_per_cluster):
vec = self.generate_cluster_vector(cluster2_base)
vec_bytes = struct.pack(f'{dim}f', *vec)
self.redis.execute_command('VADD', self.test_key, 'FP32', vec_bytes,
f'{self.test_key}:cluster2:{i}')
# Pick a test vector from cluster1
test_key = f'{self.test_key}:cluster1:0'
# Verify it's in cluster1 using VSIM
initial_vec = self.generate_cluster_vector(cluster1_base)
results = self.redis.execute_command('VSIM', self.test_key, 'VALUES', dim,
*[str(x) for x in initial_vec],
'COUNT', 100, 'WITHSCORES')
# Count how many cluster1 items are in top results
cluster1_count = sum(1 for i in range(0, len(results), 2)
if b'cluster1' in results[i])
assert cluster1_count > 80, "Initial clustering check failed"
# Now update the test vector to be in cluster2
new_vec = self.generate_cluster_vector(cluster2_base, noise=0.05)
vec_bytes = struct.pack(f'{dim}f', *new_vec)
self.redis.execute_command('VADD', self.test_key, 'FP32', vec_bytes, test_key)
# Verify the embedding was actually updated using VEMB
emb_result = self.redis.execute_command('VEMB', self.test_key, test_key)
updated_vec = [float(x) for x in emb_result]
# Verify updated vector matches what we inserted
dot_product = sum(a*b for a,b in zip(updated_vec, new_vec))
similarity = dot_product / (math.sqrt(sum(x*x for x in updated_vec)) *
math.sqrt(sum(x*x for x in new_vec)))
assert similarity > 0.9, "Vector was not properly updated"
# Verify it's now in cluster2 using VSIM
results = self.redis.execute_command('VSIM', self.test_key, 'VALUES', dim,
*[str(x) for x in cluster2_base],
'COUNT', 100, 'WITHSCORES')
# Verify our updated vector is among top results
found = False
for i in range(0, len(results), 2):
if results[i].decode() == test_key:
found = True
similarity = float(results[i+1])
assert similarity > 0.80, f"Updated vector has low similarity: {similarity}"
break
assert found, "Updated vector not found in cluster2 proximity"
modules/vector-sets/tests/persistence.py
+83
View File
@@ -0,0 +1,83 @@
from test import TestCase, fill_redis_with_vectors, generate_random_vector
import random
class HNSWPersistence(TestCase):
def getname(self):
return "HNSW Persistence"
def estimated_runtime(self):
return 30
def _verify_results(self, key, dim, query_vec, reduced_dim=None):
"""Run a query and return results dict"""
k = 10
args = ['VSIM', key]
if reduced_dim:
args.extend(['VALUES', dim])
args.extend([str(x) for x in query_vec])
else:
args.extend(['VALUES', dim])
args.extend([str(x) for x in query_vec])
args.extend(['COUNT', k, 'WITHSCORES'])
results = self.redis.execute_command(*args)
results_dict = {}
for i in range(0, len(results), 2):
key = results[i].decode()
score = float(results[i+1])
results_dict[key] = score
return results_dict
def test(self):
# Setup dimensions
dim = 128
reduced_dim = 32
count = 5000
random.seed(42)
# Create two datasets - one normal and one with dimension reduction
normal_data = fill_redis_with_vectors(self.redis, f"{self.test_key}:normal", count, dim)
projected_data = fill_redis_with_vectors(self.redis, f"{self.test_key}:projected",
count, dim, reduced_dim)
# Generate query vectors we'll use before and after reload
query_vec_normal = generate_random_vector(dim)
query_vec_projected = generate_random_vector(dim)
# Get initial results for both sets
initial_normal = self._verify_results(f"{self.test_key}:normal",
dim, query_vec_normal)
initial_projected = self._verify_results(f"{self.test_key}:projected",
dim, query_vec_projected, reduced_dim)
# Force Redis to save and reload the dataset
self.redis.execute_command('DEBUG', 'RELOAD')
# Verify results after reload
reloaded_normal = self._verify_results(f"{self.test_key}:normal",
dim, query_vec_normal)
reloaded_projected = self._verify_results(f"{self.test_key}:projected",
dim, query_vec_projected, reduced_dim)
# Verify normal vectors results
assert len(initial_normal) == len(reloaded_normal), \
"Normal vectors: Result count mismatch before/after reload"
for key in initial_normal:
assert key in reloaded_normal, f"Normal vectors: Missing item after reload: {key}"
assert abs(initial_normal[key] - reloaded_normal[key]) < 0.0001, \
f"Normal vectors: Score mismatch for {key}: " + \
f"before={initial_normal[key]:.6f}, after={reloaded_normal[key]:.6f}"
# Verify projected vectors results
assert len(initial_projected) == len(reloaded_projected), \
"Projected vectors: Result count mismatch before/after reload"
for key in initial_projected:
assert key in reloaded_projected, \
f"Projected vectors: Missing item after reload: {key}"
assert abs(initial_projected[key] - reloaded_projected[key]) < 0.0001, \
f"Projected vectors: Score mismatch for {key}: " + \
f"before={initial_projected[key]:.6f}, after={reloaded_projected[key]:.6f}"
modules/vector-sets/tests/reduce.py
+71
View File
@@ -0,0 +1,71 @@
from test import TestCase, fill_redis_with_vectors, generate_random_vector
class Reduce(TestCase):
def getname(self):
return "Dimension Reduction"
def estimated_runtime(self):
return 0.2
def test(self):
original_dim = 100
reduced_dim = 80
count = 1000
k = 50 # Number of nearest neighbors to check
# Fill Redis with vectors using REDUCE and get reference data
data = fill_redis_with_vectors(self.redis, self.test_key, count, original_dim, reduced_dim)
# Verify dimension is reduced
dim = self.redis.execute_command('VDIM', self.test_key)
assert dim == reduced_dim, f"Expected dimension {reduced_dim}, got {dim}"
# Generate query vector and get nearest neighbors using Redis
query_vec = generate_random_vector(original_dim)
redis_raw = self.redis.execute_command('VSIM', self.test_key, 'VALUES',
original_dim, *[str(x) for x in query_vec],
'COUNT', k, 'WITHSCORES')
# Convert Redis results to dict
redis_results = {}
for i in range(0, len(redis_raw), 2):
key = redis_raw[i].decode()
score = float(redis_raw[i+1])
redis_results[key] = score
# Get results from linear scan with original vectors
linear_results = data.find_k_nearest(query_vec, k)
linear_items = {name: score for name, score in linear_results}
# Compare overlap between reduced and non-reduced results
redis_set = set(redis_results.keys())
linear_set = set(linear_items.keys())
overlap = len(redis_set & linear_set)
overlap_ratio = overlap / k
# With random projection, we expect some loss of accuracy but should
# maintain at least some similarity structure.
# Note that gaussian distribution is the worse with this test, so
# in real world practice, things will be better.
min_expected_overlap = 0.1 # At least 10% overlap in top-k
assert overlap_ratio >= min_expected_overlap, \
f"Dimension reduction lost too much structure. Only {overlap_ratio*100:.1f}% overlap in top {k}"
# For items that appear in both results, scores should be reasonably correlated
common_items = redis_set & linear_set
for item in common_items:
redis_score = redis_results[item]
linear_score = linear_items[item]
# Allow for some deviation due to dimensionality reduction
assert abs(redis_score - linear_score) < 0.2, \
f"Score mismatch too high for {item}: Redis={redis_score:.3f} Linear={linear_score:.3f}"
# If test fails, print comparison for debugging
if overlap_ratio < min_expected_overlap:
print("\nLow overlap in results. Details:")
print("\nTop results from linear scan (original vectors):")
for name, score in linear_results:
print(f"{name}: {score:.3f}")
print("\nTop results from Redis (reduced vectors):")
for item, score in sorted(redis_results.items(), key=lambda x: x[1], reverse=True):
print(f"{item}: {score:.3f}")
modules/vector-sets/tests/replication.py
+92
View File
@@ -0,0 +1,92 @@
from test import TestCase, generate_random_vector
import struct
import random
import time
class ComprehensiveReplicationTest(TestCase):
def getname(self):
return "Comprehensive Replication Test with mixed operations"
def estimated_runtime(self):
# This test will take longer than the default 100ms
return 20.0 # 20 seconds estimate
def test(self):
# Setup replication between primary and replica
assert self.setup_replication(), "Failed to setup replication"
# Test parameters
num_vectors = 5000
vector_dim = 8
delete_probability = 0.1
cas_probability = 0.3
# Keep track of added items for potential deletion
added_items = []
# Add vectors and occasionally delete
for i in range(num_vectors):
# Generate a random vector
vec = generate_random_vector(vector_dim)
vec_bytes = struct.pack(f'{vector_dim}f', *vec)
item_name = f"{self.test_key}:item:{i}"
# Decide whether to use CAS or not
use_cas = random.random() < cas_probability
if use_cas and added_items:
# Get an existing item for CAS reference (if available)
cas_item = random.choice(added_items)
try:
# Add with CAS
result = self.redis.execute_command('VADD', self.test_key, 'FP32', vec_bytes,
item_name, 'CAS')
# Only add to our list if actually added (CAS might fail)
if result == 1:
added_items.append(item_name)
except Exception as e:
print(f" CAS VADD failed: {e}")
else:
try:
# Add without CAS
result = self.redis.execute_command('VADD', self.test_key, 'FP32', vec_bytes, item_name)
# Only add to our list if actually added
if result == 1:
added_items.append(item_name)
except Exception as e:
print(f" VADD failed: {e}")
# Randomly delete items (with 10% probability)
if random.random() < delete_probability and added_items:
try:
# Select a random item to delete
item_to_delete = random.choice(added_items)
# Delete the item using VREM (not VDEL)
self.redis.execute_command('VREM', self.test_key, item_to_delete)
# Remove from our list
added_items.remove(item_to_delete)
except Exception as e:
print(f" VREM failed: {e}")
# Allow time for replication to complete
time.sleep(2.0)
# Verify final VCARD matches
primary_card = self.redis.execute_command('VCARD', self.test_key)
replica_card = self.replica.execute_command('VCARD', self.test_key)
assert primary_card == replica_card, f"Final VCARD mismatch: primary={primary_card}, replica={replica_card}"
# Verify VDIM matches
primary_dim = self.redis.execute_command('VDIM', self.test_key)
replica_dim = self.replica.execute_command('VDIM', self.test_key)
assert primary_dim == replica_dim, f"VDIM mismatch: primary={primary_dim}, replica={replica_dim}"
# Verify digests match using DEBUG DIGEST
primary_digest = self.redis.execute_command('DEBUG', 'DIGEST-VALUE', self.test_key)
replica_digest = self.replica.execute_command('DEBUG', 'DIGEST-VALUE', self.test_key)
assert primary_digest == replica_digest, f"Digest mismatch: primary={primary_digest}, replica={replica_digest}"
# Print summary
print(f"\n Added and maintained {len(added_items)} vectors with dimension {vector_dim}")
print(f" Final vector count: {primary_card}")
print(f" Final digest: {primary_digest[0].decode()}")
modules/vector-sets/tests/vadd_cas.py
+98
View File
@@ -0,0 +1,98 @@
from test import TestCase, generate_random_vector
import threading
import struct
import math
import time
import random
from typing import List, Dict
class ConcurrentCASTest(TestCase):
def getname(self):
return "Concurrent VADD with CAS"
def estimated_runtime(self):
return 1.5
def worker(self, vectors: List[List[float]], start_idx: int, end_idx: int,
dim: int, results: Dict[str, bool]):
"""Worker thread that adds a subset of vectors using VADD CAS"""
for i in range(start_idx, end_idx):
vec = vectors[i]
name = f"{self.test_key}:item:{i}"
vec_bytes = struct.pack(f'{dim}f', *vec)
# Try to add the vector with CAS
try:
result = self.redis.execute_command('VADD', self.test_key, 'FP32',
vec_bytes, name, 'CAS')
results[name] = (result == 1) # Store if it was actually added
except Exception as e:
results[name] = False
print(f"Error adding {name}: {e}")
def verify_vector_similarity(self, vec1: List[float], vec2: List[float]) -> float:
"""Calculate cosine similarity between two vectors"""
dot_product = sum(a*b for a,b in zip(vec1, vec2))
norm1 = math.sqrt(sum(x*x for x in vec1))
norm2 = math.sqrt(sum(x*x for x in vec2))
return dot_product / (norm1 * norm2) if norm1 > 0 and norm2 > 0 else 0
def test(self):
# Test parameters
dim = 128
total_vectors = 5000
num_threads = 8
vectors_per_thread = total_vectors // num_threads
# Generate all vectors upfront
random.seed(42) # For reproducibility
vectors = [generate_random_vector(dim) for _ in range(total_vectors)]
# Prepare threads and results dictionary
threads = []
results = {} # Will store success/failure for each vector
# Launch threads
for i in range(num_threads):
start_idx = i * vectors_per_thread
end_idx = start_idx + vectors_per_thread if i < num_threads-1 else total_vectors
thread = threading.Thread(target=self.worker,
args=(vectors, start_idx, end_idx, dim, results))
threads.append(thread)
thread.start()
# Wait for all threads to complete
for thread in threads:
thread.join()
# Verify cardinality
card = self.redis.execute_command('VCARD', self.test_key)
assert card == total_vectors, \
f"Expected {total_vectors} elements, but found {card}"
# Verify each vector
num_verified = 0
for i in range(total_vectors):
name = f"{self.test_key}:item:{i}"
# Verify the item was successfully added
assert results[name], f"Vector {name} was not successfully added"
# Get the stored vector
stored_vec_raw = self.redis.execute_command('VEMB', self.test_key, name)
stored_vec = [float(x) for x in stored_vec_raw]
# Verify vector dimensions
assert len(stored_vec) == dim, \
f"Stored vector dimension mismatch for {name}: {len(stored_vec)} != {dim}"
# Calculate similarity with original vector
similarity = self.verify_vector_similarity(vectors[i], stored_vec)
assert similarity > 0.99, \
f"Low similarity ({similarity}) for {name}"
num_verified += 1
# Final verification
assert num_verified == total_vectors, \
f"Only verified {num_verified} out of {total_vectors} vectors"
modules/vector-sets/tests/vemb.py
+41
View File
@@ -0,0 +1,41 @@
from test import TestCase
import struct
import math
class VEMB(TestCase):
def getname(self):
return "VEMB Command"
def test(self):
dim = 4
# Add same vector in both formats
vec = [1, 0, 0, 0]
norm = math.sqrt(sum(x*x for x in vec))
vec = [x/norm for x in vec] # Normalize the vector
# Add using FP32
vec_bytes = struct.pack(f'{dim}f', *vec)
self.redis.execute_command('VADD', self.test_key, 'FP32', vec_bytes, f'{self.test_key}:item:1')
# Add using VALUES
self.redis.execute_command('VADD', self.test_key, 'VALUES', dim,
*[str(x) for x in vec], f'{self.test_key}:item:2')
# Get both back with VEMB
result1 = self.redis.execute_command('VEMB', self.test_key, f'{self.test_key}:item:1')
result2 = self.redis.execute_command('VEMB', self.test_key, f'{self.test_key}:item:2')
retrieved_vec1 = [float(x) for x in result1]
retrieved_vec2 = [float(x) for x in result2]
# Compare both vectors with original (allow for small quantization errors)
for i in range(dim):
assert abs(vec[i] - retrieved_vec1[i]) < 0.01, \
f"FP32 vector component {i} mismatch: expected {vec[i]}, got {retrieved_vec1[i]}"
assert abs(vec[i] - retrieved_vec2[i]) < 0.01, \
f"VALUES vector component {i} mismatch: expected {vec[i]}, got {retrieved_vec2[i]}"
# Test non-existent item
result = self.redis.execute_command('VEMB', self.test_key, 'nonexistent')
assert result is None, "Non-existent item should return nil"
modules/vector-sets/tests/vrandmember.py
+55
View File
@@ -0,0 +1,55 @@
from test import TestCase, generate_random_vector, fill_redis_with_vectors
import struct
class VRANDMEMBERTest(TestCase):
def getname(self):
return "VRANDMEMBER basic functionality"
def test(self):
# Test with empty key
result = self.redis.execute_command('VRANDMEMBER', self.test_key)
assert result is None, "VRANDMEMBER on non-existent key should return NULL"
result = self.redis.execute_command('VRANDMEMBER', self.test_key, 5)
assert isinstance(result, list) and len(result) == 0, "VRANDMEMBER with count on non-existent key should return empty array"
# Fill with vectors
dim = 4
count = 100
data = fill_redis_with_vectors(self.redis, self.test_key, count, dim)
# Test single random member
result = self.redis.execute_command('VRANDMEMBER', self.test_key)
assert result is not None, "VRANDMEMBER should return a random member"
assert result.decode() in data.names, "Random member should be in the set"
# Test multiple unique members (positive count)
positive_count = 10
result = self.redis.execute_command('VRANDMEMBER', self.test_key, positive_count)
assert isinstance(result, list), "VRANDMEMBER with positive count should return an array"
assert len(result) == positive_count, f"Should return {positive_count} members"
# Check for uniqueness
decoded_results = [r.decode() for r in result]
assert len(decoded_results) == len(set(decoded_results)), "Results should be unique with positive count"
for item in decoded_results:
assert item in data.names, "All returned items should be in the set"
# Test more members than in the set
result = self.redis.execute_command('VRANDMEMBER', self.test_key, count + 10)
assert len(result) == count, "Should return only the available members when asking for more than exist"
# Test with duplicates (negative count)
negative_count = -20
result = self.redis.execute_command('VRANDMEMBER', self.test_key, negative_count)
assert isinstance(result, list), "VRANDMEMBER with negative count should return an array"
assert len(result) == abs(negative_count), f"Should return {abs(negative_count)} members"
# Check that all returned elements are valid
decoded_results = [r.decode() for r in result]
for item in decoded_results:
assert item in data.names, "All returned items should be in the set"
# Test with count = 0 (edge case)
result = self.redis.execute_command('VRANDMEMBER', self.test_key, 0)
assert isinstance(result, list) and len(result) == 0, "VRANDMEMBER with count=0 should return empty array"
modules/vector-sets/vset.c
+1985
View File
File diff suppressed because it is too large Load Diff
modules/vector-sets/w2v.c
+514
View File
@@ -0,0 +1,514 @@
/*
* HNSW (Hierarchical Navigable Small World) Implementation
* Based on the paper by Yu. A. Malkov, D. A. Yashunin
*
* Copyright (c) 2009-Present, Redis Ltd.
* All rights reserved.
*
* Licensed under your choice of the Redis Source Available License 2.0
* (RSALv2) or the Server Side Public License v1 (SSPLv1).
* Originally authored by: Salvatore Sanfilippo
*/
#include <stdio.h>
#include <stdlib.h>
#include <string.h>
#include <sys/time.h>
#include <time.h>
#include <stdint.h>
#include <pthread.h>
#include <stdatomic.h>
#include <math.h>
#include "hnsw.h"
/* Get current time in milliseconds */
uint64_t ms_time(void) {
struct timeval tv;
gettimeofday(&tv, NULL);
return (uint64_t)tv.tv_sec * 1000 + (tv.tv_usec / 1000);
}
/* Implementation of the recall test with random vectors. */
void test_recall(HNSW *index, int ef) {
const int num_test_vectors = 10000;
const int k = 100; // Number of nearest neighbors to find.
if (ef < k) ef = k;
// Add recall distribution counters (2% bins from 0-100%).
int recall_bins[50] = {0};
// Create array to store vectors for mixing.
int num_source_vectors = 1000; // Enough, since we mix them.
float **source_vectors = malloc(sizeof(float*) * num_source_vectors);
if (!source_vectors) {
printf("Failed to allocate memory for source vectors\n");
return;
}
// Allocate memory for each source vector.
for (int i = 0; i < num_source_vectors; i++) {
source_vectors[i] = malloc(sizeof(float) * 300);
if (!source_vectors[i]) {
printf("Failed to allocate memory for source vector %d\n", i);
// Clean up already allocated vectors.
for (int j = 0; j < i; j++) free(source_vectors[j]);
free(source_vectors);
return;
}
}
/* Populate source vectors from the index, we just scan the
* first N items. */
int source_count = 0;
hnswNode *current = index->head;
while (current && source_count < num_source_vectors) {
hnsw_get_node_vector(index, current, source_vectors[source_count]);
source_count++;
current = current->next;
}
if (source_count < num_source_vectors) {
printf("Warning: Only found %d nodes for source vectors\n",
source_count);
num_source_vectors = source_count;
}
// Allocate memory for test vector.
float *test_vector = malloc(sizeof(float) * 300);
if (!test_vector) {
printf("Failed to allocate memory for test vector\n");
for (int i = 0; i < num_source_vectors; i++) {
free(source_vectors[i]);
}
free(source_vectors);
return;
}
// Allocate memory for results.
hnswNode **hnsw_results = malloc(sizeof(hnswNode*) * ef);
hnswNode **linear_results = malloc(sizeof(hnswNode*) * ef);
float *hnsw_distances = malloc(sizeof(float) * ef);
float *linear_distances = malloc(sizeof(float) * ef);
if (!hnsw_results || !linear_results || !hnsw_distances || !linear_distances) {
printf("Failed to allocate memory for results\n");
if (hnsw_results) free(hnsw_results);
if (linear_results) free(linear_results);
if (hnsw_distances) free(hnsw_distances);
if (linear_distances) free(linear_distances);
for (int i = 0; i < num_source_vectors; i++) free(source_vectors[i]);
free(source_vectors);
free(test_vector);
return;
}
// Initialize random seed.
srand(time(NULL));
// Perform recall test.
printf("\nPerforming recall test with EF=%d on %d random vectors...\n",
ef, num_test_vectors);
double total_recall = 0.0;
for (int t = 0; t < num_test_vectors; t++) {
// Create a random vector by mixing 3 existing vectors.
float weights[3] = {0.0};
int src_indices[3] = {0};
// Generate random weights.
float weight_sum = 0.0;
for (int i = 0; i < 3; i++) {
weights[i] = (float)rand() / RAND_MAX;
weight_sum += weights[i];
src_indices[i] = rand() % num_source_vectors;
}
// Normalize weights.
for (int i = 0; i < 3; i++) weights[i] /= weight_sum;
// Mix vectors.
memset(test_vector, 0, sizeof(float) * 300);
for (int i = 0; i < 3; i++) {
for (int j = 0; j < 300; j++) {
test_vector[j] +=
weights[i] * source_vectors[src_indices[i]][j];
}
}
// Perform HNSW search with the specified EF parameter.
int slot = hnsw_acquire_read_slot(index);
int hnsw_found = hnsw_search(index, test_vector, ef, hnsw_results, hnsw_distances, slot, 0);
// Perform linear search (ground truth).
int linear_found = hnsw_ground_truth_with_filter(index, test_vector, ef, linear_results, linear_distances, slot, 0, NULL, NULL);
hnsw_release_read_slot(index, slot);
// Calculate recall for this query (intersection size / k).
if (hnsw_found > k) hnsw_found = k;
if (linear_found > k) linear_found = k;
int intersection_count = 0;
for (int i = 0; i < linear_found; i++) {
for (int j = 0; j < hnsw_found; j++) {
if (linear_results[i] == hnsw_results[j]) {
intersection_count++;
break;
}
}
}
double recall = (double)intersection_count / linear_found;
total_recall += recall;
// Add to distribution bins (2% steps)
int bin_index = (int)(recall * 50);
if (bin_index >= 50) bin_index = 49; // Handle 100% recall case
recall_bins[bin_index]++;
// Show progress.
if ((t+1) % 1000 == 0 || t == num_test_vectors-1) {
printf("Processed %d/%d queries, current avg recall: %.2f%%\n",
t+1, num_test_vectors, (total_recall / (t+1)) * 100);
}
}
// Calculate and print final average recall.
double avg_recall = (total_recall / num_test_vectors) * 100;
printf("\nRecall Test Results:\n");
printf("Average recall@%d (EF=%d): %.2f%%\n", k, ef, avg_recall);
// Print recall distribution histogram.
printf("\nRecall Distribution (2%% bins):\n");
printf("================================\n");
// Find the maximum bin count for scaling.
int max_count = 0;
for (int i = 0; i < 50; i++) {
if (recall_bins[i] > max_count) max_count = recall_bins[i];
}
// Scale factor for histogram (max 50 chars wide)
const int max_bars = 50;
double scale = (max_count > max_bars) ? (double)max_bars / max_count : 1.0;
// Print the histogram.
for (int i = 0; i < 50; i++) {
int bar_len = (int)(recall_bins[i] * scale);
printf("%3d%%-%-3d%% | %-6d |", i*2, (i+1)*2, recall_bins[i]);
for (int j = 0; j < bar_len; j++) printf("#");
printf("\n");
}
// Cleanup.
free(hnsw_results);
free(linear_results);
free(hnsw_distances);
free(linear_distances);
free(test_vector);
for (int i = 0; i < num_source_vectors; i++) free(source_vectors[i]);
free(source_vectors);
}
/* Example usage in main() */
int w2v_single_thread(int m_param, int quantization, uint64_t numele, int massdel, int self_recall, int recall_ef) {
/* Create index */
HNSW *index = hnsw_new(300, quantization, m_param);
float v[300];
uint16_t wlen;
FILE *fp = fopen("word2vec.bin","rb");
if (fp == NULL) {
perror("word2vec.bin file missing");
exit(1);
}
unsigned char header[8];
fread(header,8,1,fp); // Skip header
uint64_t id = 0;
uint64_t start_time = ms_time();
char *word = NULL;
hnswNode *search_node = NULL;
while(id < numele) {
if (fread(&wlen,2,1,fp) == 0) break;
word = malloc(wlen+1);
fread(word,wlen,1,fp);
word[wlen] = 0;
fread(v,300*sizeof(float),1,fp);
// Plain API that acquires a write lock for the whole time.
hnswNode *added = hnsw_insert(index, v, NULL, 0, id++, word, 200);
if (!strcmp(word,"banana")) search_node = added;
if (!(id % 10000)) printf("%llu added\n", (unsigned long long)id);
}
uint64_t elapsed = ms_time() - start_time;
fclose(fp);
printf("%llu words added (%llu words/sec), last word: %s\n",
(unsigned long long)index->node_count,
(unsigned long long)id*1000/elapsed, word);
/* Search query */
if (search_node == NULL) search_node = index->head;
hnsw_get_node_vector(index,search_node,v);
hnswNode *neighbors[10];
float distances[10];
int found, j;
start_time = ms_time();
for (j = 0; j < 20000; j++)
found = hnsw_search(index, v, 10, neighbors, distances, 0, 0);
elapsed = ms_time() - start_time;
printf("%d searches performed (%llu searches/sec), nodes found: %d\n",
j, (unsigned long long)j*1000/elapsed, found);
if (found > 0) {
printf("Found %d neighbors:\n", found);
for (int i = 0; i < found; i++) {
printf("Node ID: %llu, distance: %f, word: %s\n",
(unsigned long long)neighbors[i]->id,
distances[i], (char*)neighbors[i]->value);
}
}
// Self-recall test (ability to find the node by its own vector).
if (self_recall) {
hnsw_print_stats(index);
hnsw_test_graph_recall(index,200,0);
}
// Recall test with random vectors.
if (recall_ef > 0) {
test_recall(index, recall_ef);
}
uint64_t connected_nodes;
int reciprocal_links;
hnsw_validate_graph(index, &connected_nodes, &reciprocal_links);
if (massdel) {
int remove_perc = 95;
printf("\nRemoving %d%% of nodes...\n", remove_perc);
uint64_t initial_nodes = index->node_count;
hnswNode *current = index->head;
while (current && index->node_count > initial_nodes*(100-remove_perc)/100) {
hnswNode *next = current->next;
hnsw_delete_node(index,current,free);
current = next;
// In order to don't remove only contiguous nodes, from time
// skip a node.
if (current && !(random() % remove_perc)) current = current->next;
}
printf("%llu nodes left\n", (unsigned long long)index->node_count);
// Test again.
hnsw_validate_graph(index, &connected_nodes, &reciprocal_links);
hnsw_test_graph_recall(index,200,0);
}
hnsw_free(index,free);
return 0;
}
struct threadContext {
pthread_mutex_t FileAccessMutex;
uint64_t numele;
_Atomic uint64_t SearchesDone;
_Atomic uint64_t id;
FILE *fp;
HNSW *index;
float *search_vector;
};
// Note that in practical terms inserting with many concurrent threads
// may be *slower* and not faster, because there is a lot of
// contention. So this is more a robustness test than anything else.
//
// The optimistic commit API goal is actually to exploit the ability to
// add faster when there are many concurrent reads.
void *threaded_insert(void *ctxptr) {
struct threadContext *ctx = ctxptr;
char *word;
float v[300];
uint16_t wlen;
while(1) {
pthread_mutex_lock(&ctx->FileAccessMutex);
if (fread(&wlen,2,1,ctx->fp) == 0) break;
pthread_mutex_unlock(&ctx->FileAccessMutex);
word = malloc(wlen+1);
fread(word,wlen,1,ctx->fp);
word[wlen] = 0;
fread(v,300*sizeof(float),1,ctx->fp);
// Check-and-set API that performs the costly scan for similar
// nodes concurrently with other read threads, and finally
// applies the check if the graph wasn't modified.
InsertContext *ic;
uint64_t next_id = ctx->id++;
ic = hnsw_prepare_insert(ctx->index, v, NULL, 0, next_id, 200);
if (hnsw_try_commit_insert(ctx->index, ic, word) == NULL) {
// This time try locking since the start.
hnsw_insert(ctx->index, v, NULL, 0, next_id, word, 200);
}
if (next_id >= ctx->numele) break;
if (!((next_id+1) % 10000))
printf("%llu added\n", (unsigned long long)next_id+1);
}
return NULL;
}
void *threaded_search(void *ctxptr) {
struct threadContext *ctx = ctxptr;
/* Search query */
hnswNode *neighbors[10];
float distances[10];
int found = 0;
uint64_t last_id = 0;
while(ctx->id < 1000000) {
int slot = hnsw_acquire_read_slot(ctx->index);
found = hnsw_search(ctx->index, ctx->search_vector, 10, neighbors, distances, slot, 0);
hnsw_release_read_slot(ctx->index,slot);
last_id = ++ctx->id;
}
if (found > 0 && last_id == 1000000) {
printf("Found %d neighbors:\n", found);
for (int i = 0; i < found; i++) {
printf("Node ID: %llu, distance: %f, word: %s\n",
(unsigned long long)neighbors[i]->id,
distances[i], (char*)neighbors[i]->value);
}
}
return NULL;
}
int w2v_multi_thread(int m_param, int numthreads, int quantization, uint64_t numele) {
/* Create index */
struct threadContext ctx;
ctx.index = hnsw_new(300, quantization, m_param);
ctx.fp = fopen("word2vec.bin","rb");
if (ctx.fp == NULL) {
perror("word2vec.bin file missing");
exit(1);
}
unsigned char header[8];
fread(header,8,1,ctx.fp); // Skip header
pthread_mutex_init(&ctx.FileAccessMutex,NULL);
uint64_t start_time = ms_time();
ctx.id = 0;
ctx.numele = numele;
pthread_t threads[numthreads];
for (int j = 0; j < numthreads; j++)
pthread_create(&threads[j], NULL, threaded_insert, &ctx);
// Wait for all the threads to terminate adding items.
for (int j = 0; j < numthreads; j++)
pthread_join(threads[j],NULL);
uint64_t elapsed = ms_time() - start_time;
fclose(ctx.fp);
// Obtain the last word.
hnswNode *node = ctx.index->head;
char *word = node->value;
// We will search this last inserted word in the next test.
// Let's save its embedding.
ctx.search_vector = malloc(sizeof(float)*300);
hnsw_get_node_vector(ctx.index,node,ctx.search_vector);
printf("%llu words added (%llu words/sec), last word: %s\n",
(unsigned long long)ctx.index->node_count,
(unsigned long long)ctx.id*1000/elapsed, word);
/* Search query */
start_time = ms_time();
ctx.id = 0; // We will use this atomic field to stop at N queries done.
for (int j = 0; j < numthreads; j++)
pthread_create(&threads[j], NULL, threaded_search, &ctx);
// Wait for all the threads to terminate searching.
for (int j = 0; j < numthreads; j++)
pthread_join(threads[j],NULL);
elapsed = ms_time() - start_time;
printf("%llu searches performed (%llu searches/sec)\n",
(unsigned long long)ctx.id,
(unsigned long long)ctx.id*1000/elapsed);
hnsw_print_stats(ctx.index);
uint64_t connected_nodes;
int reciprocal_links;
hnsw_validate_graph(ctx.index, &connected_nodes, &reciprocal_links);
printf("%llu connected nodes. Links all reciprocal: %d\n",
(unsigned long long)connected_nodes, reciprocal_links);
hnsw_free(ctx.index,free);
return 0;
}
int main(int argc, char **argv) {
int quantization = HNSW_QUANT_NONE;
int numthreads = 0;
uint64_t numele = 20000;
int m_param = 0; // Default value (0 means use HNSW_DEFAULT_M)
/* This you can enable in single thread mode for testing: */
int massdel = 0; // If true, does the mass deletion test.
int self_recall = 0; // If true, does the self-recall test.
int recall_ef = 0; // If not 0, does the recall test with this EF value.
for (int j = 1; j < argc; j++) {
int moreargs = argc-j-1;
if (!strcasecmp(argv[j],"--quant")) {
quantization = HNSW_QUANT_Q8;
} else if (!strcasecmp(argv[j],"--bin")) {
quantization = HNSW_QUANT_BIN;
} else if (!strcasecmp(argv[j],"--mass-del")) {
massdel = 1;
} else if (!strcasecmp(argv[j],"--self-recall")) {
self_recall = 1;
} else if (moreargs >= 1 && !strcasecmp(argv[j],"--recall")) {
recall_ef = atoi(argv[j+1]);
j++;
} else if (moreargs >= 1 && !strcasecmp(argv[j],"--threads")) {
numthreads = atoi(argv[j+1]);
j++;
} else if (moreargs >= 1 && !strcasecmp(argv[j],"--numele")) {
numele = strtoll(argv[j+1],NULL,0);
j++;
if (numele < 1) numele = 1;
} else if (moreargs >= 1 && !strcasecmp(argv[j],"--m")) {
m_param = atoi(argv[j+1]);
j++;
} else if (!strcasecmp(argv[j],"--help")) {
printf("%s [--quant] [--bin] [--thread <count>] [--numele <count>] [--m <count>] [--mass-del] [--self-recall] [--recall <ef>]\n", argv[0]);
exit(0);
} else {
printf("Unrecognized option or wrong number of arguments: %s\n", argv[j]);
exit(1);
}
}
if (quantization == HNSW_QUANT_NONE) {
printf("You can enable quantization with --quant\n");
}
if (numthreads > 0) {
w2v_multi_thread(m_param, numthreads, quantization, numele);
} else {
printf("Single thread execution. Use --threads 4 for concurrent API\n");
w2v_single_thread(m_param, quantization, numele, massdel, self_recall, recall_ef);
}
}
redis-full.conf
+376
View File
@@ -0,0 +1,376 @@
include redis.conf
loadmodule ./modules/redisbloom/redisbloom.so
loadmodule ./modules/redisearch/redisearch.so
loadmodule ./modules/redisjson/rejson.so
loadmodule ./modules/redistimeseries/redistimeseries.so
############################## QUERY ENGINE CONFIG ############################
# Keep numeric ranges in numeric tree parent nodes of leafs for `x` generations.
# numeric, valid range: [0, 2], default: 0
#
# search-_numeric-ranges-parents 0
# The number of iterations to run while performing background indexing
# before we call usleep(1) (sleep for 1 micro-second) and make sure that we
# allow redis to process other commands.
# numeric, valid range: [1, UINT32_MAX], default: 100
#
# search-bg-index-sleep-gap 100
# The default dialect used in search queries.
# numeric, valid range: [1, 4], default: 1
#
# search-default-dialect 1
# the fork gc will only start to clean when the number of not cleaned document
# will exceed this threshold.
# numeric, valid range: [1, LLONG_MAX], default: 100
#
# search-fork-gc-clean-threshold 100
# interval (in seconds) in which to retry running the forkgc after failure.
# numeric, valid range: [1, LLONG_MAX], default: 5
#
# search-fork-gc-retry-interval 5
# interval (in seconds) in which to run the fork gc (relevant only when fork
# gc is used).
# numeric, valid range: [1, LLONG_MAX], default: 30
#
# search-fork-gc-run-interval 30
# the amount of seconds for the fork GC to sleep before exiting.
# numeric, valid range: [0, LLONG_MAX], default: 0
#
# search-fork-gc-sleep-before-exit 0
# Scan this many documents at a time during every GC iteration.
# numeric, valid range: [1, LLONG_MAX], default: 100
#
# search-gc-scan-size 100
# Max number of cursors for a given index that can be opened inside of a shard.
# numeric, valid range: [0, LLONG_MAX], default: 128
#
# search-index-cursor-limit 128
# Maximum number of results from ft.aggregate command.
# numeric, valid range: [0, (1ULL << 31)], default: 1ULL << 31
#
# search-max-aggregate-results 2147483648
# Maximum prefix expansions to be used in a query.
# numeric, valid range: [1, LLONG_MAX], default: 200
#
# search-max-prefix-expansions 200
# Maximum runtime document table size (for this process).
# numeric, valid range: [1, 100000000], default: 1000000
#
# search-max-doctablesize 1000000
# max idle time allowed to be set for cursor, setting it high might cause
# high memory consumption.
# numeric, valid range: [1, LLONG_MAX], default: 300000
#
# search-cursor-max-idle 300000
# Maximum number of results from ft.search command.
# numeric, valid range: [0, 1ULL << 31], default: 1000000
#
# search-max-search-results 1000000
# Number of worker threads to use for background tasks when the server is
# in an operation event.
# numeric, valid range: [1, 16], default: 4
#
# search-min-operation-workers 4
# Minimum length of term to be considered for phonetic matching.
# numeric, valid range: [1, LLONG_MAX], default: 3
#
# search-min-phonetic-term-len 3
# the minimum prefix for expansions (`*`).
# numeric, valid range: [1, LLONG_MAX], default: 2
#
# search-min-prefix 2
# the minimum word length to stem.
# numeric, valid range: [2, UINT32_MAX], default: 4
#
# search-min-stem-len 4
# Delta used to increase positional offsets between array
# slots for multi text values.
# Can control the level of separation between phrases in different
# array slots (related to the SLOP parameter of ft.search command)"
# numeric, valid range: [1, UINT32_MAX], default: 100
#
# search-multi-text-slop 100
# Used for setting the buffer limit threshold for vector similarity tiered
# HNSW index, so that if we are using WORKERS for indexing, and the
# number of vectors waiting in the buffer to be indexed exceeds this limit,
# we insert new vectors directly into HNSW.
# numeric, valid range: [0, LLONG_MAX], default: 1024
#
# search-tiered-hnsw-buffer-limit 1024
# Query timeout.
# numeric, valid range: [1, LLONG_MAX], default: 500
#
# search-timeout 500
# minimum number of iterators in a union from which the iterator will
# will switch to heap-based implementation.
# numeric, valid range: [1, LLONG_MAX], default: 20
# switch to heap based implementation.
#
# search-union-iterator-heap 20
# The maximum memory resize for vector similarity indexes (in bytes).
# numeric, valid range: [0, UINT32_MAX], default: 0
#
# search-vss-max-resize 0
# Number of worker threads to use for query processing and background tasks.
# numeric, valid range: [0, 16], default: 0
# This configuration also affects the number of connections per shard.
#
# search-workers 0
# The number of high priority tasks to be executed at any given time by the
# worker thread pool, before executing low priority tasks. After this number
# of high priority tasks are being executed, the worker thread pool will
# execute high and low priority tasks alternately.
# numeric, valid range: [0, LLONG_MAX], default: 1
#
# search-workers-priority-bias-threshold 1
# Load extension scoring/expansion module. Immutable.
# string, default: ""
#
# search-ext-load ""
# Path to Chinese dictionary configuration file (for Chinese tokenization). Immutable.
# string, default: ""
#
# search-friso-ini ""
# Action to perform when search timeout is exceeded (choose RETURN or FAIL).
# enum, valid values: ["return", "fail"], default: "fail"
#
# search-on-timeout fail
# Determine whether some index resources are free on a second thread.
# bool, default: yes
#
# search-_free-resource-on-thread yes
# Enable legacy compression of double to float.
# bool, default: no
#
# search-_numeric-compress no
# Disable print of time for ft.profile. For testing only.
# bool, default: yes
#
# search-_print-profile-clock yes
# Intersection iterator orders the children iterators by their relative estimated
# number of results in ascending order, so that if we see first iterators with
# a lower count of results we will skip a larger number of results, which
# translates into faster iteration. If this flag is set, we use this
# optimization in a way where union iterators are being factorize by the number
# of their own children, so that we sort by the number of children times the
# overall estimated number of results instead.
# bool, default: no
#
# search-_prioritize-intersect-union-children no
# Set to run without memory pools.
# bool, default: no
#
# search-no-mem-pools no
# Disable garbage collection (for this process).
# bool, default: no
#
# search-no-gc no
# Enable commands filter which optimize indexing on partial hash updates.
# bool, default: no
#
# search-partial-indexed-docs no
# Disable compression for DocID inverted index. Boost CPU performance.
# bool, default: no
#
# search-raw-docid-encoding no
# Number of search threads in the coordinator thread pool.
# numeric, valid range: [1, LLONG_MAX], default: 20
#
# search-threads 20
# Timeout for topology validation (in milliseconds). After this timeout,
# any pending requests will be processed, even if the topology is not fully connected.
# numeric, valid range: [0, LLONG_MAX], default: 30000
#
# search-topology-validation-timeout 30000
############################## TIME SERIES CONFIG #############################
# The maximal number of per-shard threads for cross-key queries when using cluster mode
# (TS.MRANGE, TS.MREVRANGE, TS.MGET, and TS.QUERYINDEX).
# Note: increasing this value may either increase or decrease the performance.
# integer, valid range: [1..16], default: 3
# This is a load-time configuration parameter.
#
# ts-num-threads 3
# Default compaction rules for newly created key with TS.ADD, TS.INCRBY, and TS.DECRBY.
# Has no effect on keys created with TS.CREATE.
# This default value is applied to each new time series upon its creation.
# string, see documentation for rules format, default: no compaction rules
#
# ts-compaction-policy ""
# Default chunk encoding for automatically-created compacted time series.
# This default value is applied to each new compacted time series automatically
# created when ts-compaction-policy is specified.
# valid values: COMPRESSED, UNCOMPRESSED, default: COMPRESSED
#
# ts-encoding COMPRESSED
# Default retention period, in milliseconds. 0 means no expiration.
# This default value is applied to each new time series upon its creation.
# If ts-compaction-policy is specified - it is overridden for created
# compactions as specified in ts-compaction-policy.
# integer, valid range: [0 .. LLONG_MAX], default: 0
#
# ts-retention-policy 0
# Default policy for handling insertion (TS.ADD and TS.MADD) of multiple
# samples with identical timestamps.
# This default value is applied to each new time series upon its creation.
# string, valid values: BLOCK, FIRST, LAST, MIN, MAX, SUM, default: BLOCK
#
# ts-duplicate-policy BLOCK
# Default initial allocation size, in bytes, for the data part of each new chunk
# This default value is applied to each new time series upon its creation.
# integer, valid range: [48 .. 1048576]; must be a multiple of 8, default: 4096
#
# ts-chunk-size-bytes 4096
# Default values for newly created time series.
# Many sensors report data periodically. Often, the difference between the measured
# value and the previous measured value is negligible and related to random noise
# or to measurement accuracy limitations. In such situations it may be preferable
# not to add the new measurement to the time series.
# A new sample is considered a duplicate and is ignored if the following conditions are met:
# - The time series is not a compaction;
# - The time series' DUPLICATE_POLICY IS LAST;
# - The sample is added in-order (timestamp >= max_timestamp);
# - The difference of the current timestamp from the previous timestamp
# (timestamp - max_timestamp) is less than or equal to ts-ignore-max-time-diff
# - The absolute value difference of the current value from the value at the previous maximum timestamp
# (abs(value - value_at_max_timestamp) is less than or equal to ts-ignore-max-val-diff.
# where max_timestamp is the timestamp of the sample with the largest timestamp in the time series,
# and value_at_max_timestamp is the value at max_timestamp.
# ts-ignore-max-time-diff: integer, valid range: [0 .. LLONG_MAX], default: 0
# ts-ignore-max-val-diff: double, Valid range: [0 .. DBL_MAX], default: 0
#
# ts-ignore-max-time-diff 0
# ts-ignore-max-val-diff 0
########################### BLOOM FILTERS CONFIG ##############################
# Defaults values for new Bloom filters created with BF.ADD, BF.MADD, BF.INSERT, and BF.RESERVE
# These defaults are applied to each new Bloom filter upon its creation.
# Error ratio
# The desired probability for false positives.
# For a false positive rate of 0.1% (1 in 1000) - the value should be 0.001.
# double, Valid range: (0 .. 1), value greater than 0.25 is treated as 0.25, default: 0.01
#
# bf-error-rate 0.01
# Initial capacity
# The number of entries intended to be added to the filter.
# integer, valid range: [1 .. 1GB], default: 100
#
# bf-initial-size 100
# Expansion factor
# When capacity is reached, an additional sub-filter is created.
# The size of the new sub-filter is the size of the last sub-filter multiplied
# by expansion.
# integer, [0 .. 32768]. 0 is equivalent to NONSCALING. default: 2
#
# bf-expansion-factor 2
########################### CUCKOO FILTERS CONFIG #############################
# Defaults values for new Cuckoo filters created with
# CF.ADD, CF.ADDNX, CF.INSERT, CF.INSERTNX, and CF.RESERVE
# These defaults are applied to each new Cuckoo filter upon its creation.
# Initial capacity
# A filter will likely not fill up to 100% of its capacity.
# Make sure to reserve extra capacity if you want to avoid expansions.
# value is rounded to the next 2^n integer.
# integer, valid range: [2*cf-bucket-size .. 1GB], default: 1024
#
# cf-initial-size 1024
# Number of items in each bucket
# The minimal false positive rate is 2/255 ~ 0.78% when bucket size of 1 is used.
# Larger buckets increase the error rate linearly, but improve the fill rate.
# integer, valid range: [1 .. 255], default: 2
#
# cf-bucket-size 2
# Maximum iterations
# Number of attempts to swap items between buckets before declaring filter
# as full and creating an additional filter.
# A lower value improves performance. A higher value improves fill rate.
# integer, Valid range: [1 .. 65535], default: 20
#
# cf-max-iterations 20
# Expansion factor
# When a new filter is created, its size is the size of the current filter
# multiplied by this factor.
# integer, Valid range: [0 .. 32768], 0 is equivalent to NONSCALING, default: 1
#
# cf-expansion-factor 1
# Maximum expansions
# integer, Valid range: [1 .. 65536], default: 32
#
# cf-max-expansions 32
################################## SECURITY ###################################
#
# The following is a list of command categories and their meanings:
#
# * search - Query engine related.
# * json - Data type: JSON related.
# * timeseries - Data type: time series related.
# * bloom - Data type: Bloom filter related.
# * cuckoo - Data type: cuckoo filter related.
# * topk - Data type: top-k related.
# * cms - Data type: count-min sketch related.
# * tdigest - Data type: t-digest related.
redis.conf
+49 -27
View File
@@ -668,7 +668,7 @@ repl-diskless-sync-max-replicas 0
repl-diskless-load disabled
# Master send PINGs to its replicas in a predefined interval. It's possible to
# change this interval with the repl_ping_replica_period option. The default
# change this interval with the repl-ping-replica-period option. The default
# value is 10 seconds.
#
# repl-ping-replica-period 10
@@ -727,6 +727,24 @@ repl-disable-tcp-nodelay no
#
# repl-backlog-ttl 3600
# During a fullsync, the master may decide to send both the RDB file and the
# replication stream to the replica in parallel. This approach shifts the
# responsibility of buffering the replication stream to the replica during the
# fullsync process. The replica accumulates the replication stream data until
# the RDB file is fully loaded. Once the RDB delivery is completed and
# successfully loaded, the replica begins processing and applying the
# accumulated replication data to the db. The configuration below controls how
# much replication data the replica can accumulate during a fullsync.
#
# When the replica reaches this limit, it will stop accumulating further data.
# At this point, additional data accumulation may occur on the master side
# depending on the 'client-output-buffer-limit <replica>' config of master.
#
# A value of 0 means replica inherits hard limit of
# 'client-output-buffer-limit <replica>' config to limit accumulation size.
#
# replica-full-sync-buffer-limit 0
# The replica priority is an integer number published by Redis in the INFO
# output. It is used by Redis Sentinel in order to select a replica to promote
# into a master if the master is no longer working correctly.
@@ -838,7 +856,7 @@ replica-priority 100
# this is used in order to send invalidation messages to clients. Please
# check this page to understand more about the feature:
#
# https://redis.io/topics/client-side-caching
# https://redis.io/docs/latest/develop/use/client-side-caching/
#
# When tracking is enabled for a client, all the read only queries are assumed
# to be cached: this will force Redis to store information in the invalidation
@@ -1016,7 +1034,7 @@ replica-priority 100
# * stream - Data type: streams related.
#
# For more information about ACL configuration please refer to
# the Redis web site at https://redis.io/topics/acl
# the Redis web site at https://redis.io/docs/latest/operate/oss_and_stack/management/security/acl/
# ACL LOG
#
@@ -1291,38 +1309,27 @@ lazyfree-lazy-user-flush no
# in different I/O threads. Since especially writing is so slow, normally
# Redis users use pipelining in order to speed up the Redis performances per
# core, and spawn multiple instances in order to scale more. Using I/O
# threads it is possible to easily speedup two times Redis without resorting
# threads it is possible to easily speedup several times Redis without resorting
# to pipelining nor sharding of the instance.
#
# By default threading is disabled, we suggest enabling it only in machines
# that have at least 4 or more cores, leaving at least one spare core.
# Using more than 8 threads is unlikely to help much. We also recommend using
# threaded I/O only if you actually have performance problems, with Redis
# instances being able to use a quite big percentage of CPU time, otherwise
# there is no point in using this feature.
# We also recommend using threaded I/O only if you actually have performance
# problems, with Redis instances being able to use a quite big percentage of
# CPU time, otherwise there is no point in using this feature.
#
# So for instance if you have a four cores boxes, try to use 2 or 3 I/O
# threads, if you have a 8 cores, try to use 6 threads. In order to
# So for instance if you have a four cores boxes, try to use 3 I/O
# threads, if you have a 8 cores, try to use 7 threads. In order to
# enable I/O threads use the following configuration directive:
#
# io-threads 4
#
# Setting io-threads to 1 will just use the main thread as usual.
# When I/O threads are enabled, we only use threads for writes, that is
# to thread the write(2) syscall and transfer the client buffers to the
# socket. However it is also possible to enable threading of reads and
# protocol parsing using the following configuration directive, by setting
# it to yes:
# When I/O threads are enabled, we not only use threads for writes, that
# is to thread the write(2) syscall and transfer the client buffers to the
# socket, but also use threads for reads and protocol parsing.
#
# io-threads-do-reads no
#
# Usually threading reads doesn't help much.
#
# NOTE 1: This configuration directive cannot be changed at runtime via
# CONFIG SET. Also, this feature currently does not work when SSL is
# enabled.
#
# NOTE 2: If you want to test the Redis speedup using redis-benchmark, make
# NOTE: If you want to test the Redis speedup using redis-benchmark, make
# sure you also run the benchmark itself in threaded mode, using the
# --threads option to match the number of Redis threads, otherwise you'll not
# be able to notice the improvements.
@@ -1362,7 +1369,7 @@ oom-score-adj-values 0 200 800
#################### KERNEL transparent hugepage CONTROL ######################
# Usually the kernel Transparent Huge Pages control is set to "madvise" or
# or "never" by default (/sys/kernel/mm/transparent_hugepage/enabled), in which
# "never" by default (/sys/kernel/mm/transparent_hugepage/enabled), in which
# case this config has no effect. On systems in which it is set to "always",
# redis will attempt to disable it specifically for the redis process in order
# to avoid latency problems specifically with fork(2) and CoW.
@@ -1393,7 +1400,7 @@ disable-thp yes
# restarting the server can lead to data loss. A conversion needs to be done
# by setting it via CONFIG command on a live server first.
#
# Please check https://redis.io/topics/persistence for more information.
# Please check https://redis.io/docs/latest/operate/oss_and_stack/management/persistence/ for more information.
appendonly no
@@ -1776,6 +1783,21 @@ aof-timestamp-enabled no
#
# cluster-preferred-endpoint-type ip
# This configuration defines the sampling ratio (0-100) for checking command
# compatibility in cluster mode. When a command is executed, it is sampled at
# the specified ratio to determine if it complies with Redis cluster constraints,
# such as cross-slot restrictions.
#
# - A value of 0 means no commands are sampled for compatibility checks.
# - A value of 100 means all commands are checked.
# - Intermediate values (e.g., 10) mean that approximately 10% of the commands
# are randomly selected for compatibility verification.
#
# Higher sampling ratios may introduce additional performance overhead, especially
# under high QPS. The default value is 0 (no sampling).
#
# cluster-compatibility-sample-ratio 0
# In order to setup your cluster make sure to read the documentation
# available at https://redis.io web site.
@@ -1880,7 +1902,7 @@ latency-monitor-threshold 0
############################# EVENT NOTIFICATION ##############################
# Redis can notify Pub/Sub clients about events happening in the key space.
# This feature is documented at https://redis.io/topics/notifications
# This feature is documented at https://redis.io/docs/latest/develop/use/keyspace-notifications/
#
# For instance if keyspace events notification is enabled, and a client
# performs a DEL operation on key "foo" stored in the Database 0, two
runtest-moduleapi
+1
View File
@@ -56,4 +56,5 @@ $TCLSH tests/test_helper.tcl \
--single unit/moduleapi/moduleauth \
--single unit/moduleapi/rdbloadsave \
--single unit/moduleapi/crash \
--single unit/moduleapi/internalsecret \
"${@}"
sentinel.conf
+3 -3
View File
@@ -133,7 +133,7 @@ sentinel monitor mymaster 127.0.0.1 6379 2
sentinel down-after-milliseconds mymaster 30000
# IMPORTANT NOTE: starting with Redis 6.2 ACL capability is supported for
# Sentinel mode, please refer to the Redis website https://redis.io/topics/acl
# Sentinel mode, please refer to the Redis website https://redis.io/docs/latest/operate/oss_and_stack/management/security/acl/
# for more details.
# Sentinel's ACL users are defined in the following format:
@@ -145,7 +145,7 @@ sentinel down-after-milliseconds mymaster 30000
# user worker +@admin +@connection ~* on >ffa9203c493aa99
#
# For more information about ACL configuration please refer to the Redis
# website at https://redis.io/topics/acl and redis server configuration
# website at https://redis.io/docs/latest/operate/oss_and_stack/management/security/acl/ and redis server configuration
# template redis.conf.
# ACL LOG
@@ -174,7 +174,7 @@ acllog-max-len 128
# so Sentinel will try to authenticate with the same password to all the
# other Sentinels. So you need to configure all your Sentinels in a given
# group with the same "requirepass" password. Check the following documentation
# for more info: https://redis.io/topics/sentinel
# for more info: https://redis.io/docs/latest/operate/oss_and_stack/management/sentinel/
#
# IMPORTANT NOTE: starting with Redis 6.2 "requirepass" is a compatibility
# layer on top of the ACL system. The option effect will be just setting
src/Makefile
+21 -12
View File
@@ -34,7 +34,7 @@ endif
ifneq ($(OPTIMIZATION),-O0)
OPTIMIZATION+=-fno-omit-frame-pointer
endif
DEPENDENCY_TARGETS=hiredis linenoise lua hdr_histogram fpconv
DEPENDENCY_TARGETS=hiredis linenoise lua hdr_histogram fpconv fast_float
NODEPS:=clean distclean
# Default settings
@@ -52,6 +52,7 @@ endif
WARN=-Wall -W -Wno-missing-field-initializers -Werror=deprecated-declarations -Wstrict-prototypes
OPT=$(OPTIMIZATION)
SKIP_VEC_SETS?=no
# Detect if the compiler supports C11 _Atomic.
# NUMBER_SIGN_CHAR is a workaround to support both GNU Make 4.3 and older versions.
NUMBER_SIGN_CHAR := \#
@@ -61,6 +62,7 @@ C11_ATOMIC := $(shell sh -c 'echo "$(NUMBER_SIGN_CHAR)include <stdatomic.h>" > f
ifeq ($(C11_ATOMIC),yes)
STD+=-std=gnu11
else
SKIP_VEC_SETS=yes
STD+=-std=c99
endif
@@ -127,7 +129,7 @@ endif
FINAL_CFLAGS=$(STD) $(WARN) $(OPT) $(DEBUG) $(CFLAGS) $(REDIS_CFLAGS)
FINAL_LDFLAGS=$(LDFLAGS) $(OPT) $(REDIS_LDFLAGS) $(DEBUG)
FINAL_LIBS=-lm
FINAL_LIBS=-lm -lstdc++
DEBUG=-g -ggdb
# Linux ARM32 needs -latomic at linking time
@@ -235,7 +237,7 @@ ifdef OPENSSL_PREFIX
endif
# Include paths to dependencies
FINAL_CFLAGS+= -I../deps/hiredis -I../deps/linenoise -I../deps/lua/src -I../deps/hdr_histogram -I../deps/fpconv
FINAL_CFLAGS+= -I../deps/hiredis -I../deps/linenoise -I../deps/lua/src -I../deps/hdr_histogram -I../deps/fpconv -I../deps/fast_float
# Determine systemd support and/or build preference (defaulting to auto-detection)
BUILD_WITH_SYSTEMD=no
@@ -315,6 +317,12 @@ ifeq ($(BUILD_TLS),module)
TLS_MODULE_CFLAGS+=-DUSE_OPENSSL=$(BUILD_MODULE) $(OPENSSL_CFLAGS) -DBUILD_TLS_MODULE=$(BUILD_MODULE)
endif
ifneq ($(SKIP_VEC_SETS),yes)
vpath %.c ../modules/vector-sets
REDIS_VEC_SETS_OBJ=hnsw.o cJSON.o vset.o
FINAL_CFLAGS+=-DINCLUDE_VEC_SETS=1
endif
ifndef V
define MAKE_INSTALL
@printf ' %b %b\n' $(LINKCOLOR)INSTALL$(ENDCOLOR) $(BINCOLOR)$(1)$(ENDCOLOR) 1>&2
@@ -354,14 +362,14 @@ endif
REDIS_SERVER_NAME=redis-server$(PROG_SUFFIX)
REDIS_SENTINEL_NAME=redis-sentinel$(PROG_SUFFIX)
REDIS_SERVER_OBJ=threads_mngr.o adlist.o quicklist.o ae.o anet.o dict.o ebuckets.o mstr.o kvstore.o server.o sds.o zmalloc.o lzf_c.o lzf_d.o pqsort.o zipmap.o sha1.o ziplist.o release.o networking.o util.o object.o db.o replication.o rdb.o t_string.o t_list.o t_set.o t_zset.o t_hash.o config.o aof.o pubsub.o multi.o debug.o sort.o intset.o syncio.o cluster.o cluster_legacy.o crc16.o endianconv.o slowlog.o eval.o bio.o rio.o rand.o memtest.o syscheck.o crcspeed.o crc64.o bitops.o sentinel.o notify.o setproctitle.o blocked.o hyperloglog.o latency.o sparkline.o redis-check-rdb.o redis-check-aof.o geo.o lazyfree.o module.o evict.o expire.o geohash.o geohash_helper.o childinfo.o defrag.o siphash.o rax.o t_stream.o listpack.o localtime.o lolwut.o lolwut5.o lolwut6.o acl.o tracking.o socket.o tls.o sha256.o timeout.o setcpuaffinity.o monotonic.o mt19937-64.o resp_parser.o call_reply.o script_lua.o script.o functions.o function_lua.o commands.o strl.o connection.o unix.o logreqres.o
REDIS_SERVER_OBJ=threads_mngr.o adlist.o quicklist.o ae.o anet.o dict.o ebuckets.o eventnotifier.o iothread.o mstr.o kvstore.o server.o sds.o zmalloc.o lzf_c.o lzf_d.o pqsort.o zipmap.o sha1.o ziplist.o release.o networking.o util.o object.o db.o replication.o rdb.o t_string.o t_list.o t_set.o t_zset.o t_hash.o config.o aof.o pubsub.o multi.o debug.o sort.o intset.o syncio.o cluster.o cluster_legacy.o crc16.o endianconv.o slowlog.o eval.o bio.o rio.o rand.o memtest.o syscheck.o crcspeed.o crccombine.o crc64.o bitops.o sentinel.o notify.o setproctitle.o blocked.o hyperloglog.o latency.o sparkline.o redis-check-rdb.o redis-check-aof.o geo.o lazyfree.o module.o evict.o expire.o geohash.o geohash_helper.o childinfo.o defrag.o siphash.o rax.o t_stream.o listpack.o localtime.o lolwut.o lolwut5.o lolwut6.o acl.o tracking.o socket.o tls.o sha256.o timeout.o setcpuaffinity.o monotonic.o mt19937-64.o resp_parser.o call_reply.o script_lua.o script.o functions.o function_lua.o commands.o strl.o connection.o unix.o logreqres.o
REDIS_CLI_NAME=redis-cli$(PROG_SUFFIX)
REDIS_CLI_OBJ=anet.o adlist.o dict.o redis-cli.o zmalloc.o release.o ae.o redisassert.o crcspeed.o crc64.o siphash.o crc16.o monotonic.o cli_common.o mt19937-64.o strl.o cli_commands.o
REDIS_CLI_OBJ=anet.o adlist.o dict.o redis-cli.o zmalloc.o release.o ae.o redisassert.o crcspeed.o crccombine.o crc64.o siphash.o crc16.o monotonic.o cli_common.o mt19937-64.o strl.o cli_commands.o
REDIS_BENCHMARK_NAME=redis-benchmark$(PROG_SUFFIX)
REDIS_BENCHMARK_OBJ=ae.o anet.o redis-benchmark.o adlist.o dict.o zmalloc.o redisassert.o release.o crcspeed.o crc64.o siphash.o crc16.o monotonic.o cli_common.o mt19937-64.o strl.o
REDIS_BENCHMARK_OBJ=ae.o anet.o redis-benchmark.o adlist.o dict.o zmalloc.o redisassert.o release.o crcspeed.o crccombine.o crc64.o siphash.o crc16.o monotonic.o cli_common.o mt19937-64.o strl.o
REDIS_CHECK_RDB_NAME=redis-check-rdb$(PROG_SUFFIX)
REDIS_CHECK_AOF_NAME=redis-check-aof$(PROG_SUFFIX)
ALL_SOURCES=$(sort $(patsubst %.o,%.c,$(REDIS_SERVER_OBJ) $(REDIS_CLI_OBJ) $(REDIS_BENCHMARK_OBJ)))
ALL_SOURCES=$(sort $(patsubst %.o,%.c,$(REDIS_SERVER_OBJ) $(REDIS_VEC_SETS_OBJ) $(REDIS_CLI_OBJ) $(REDIS_BENCHMARK_OBJ)))
all: $(REDIS_SERVER_NAME) $(REDIS_SENTINEL_NAME) $(REDIS_CLI_NAME) $(REDIS_BENCHMARK_NAME) $(REDIS_CHECK_RDB_NAME) $(REDIS_CHECK_AOF_NAME) $(TLS_MODULE)
@echo ""
@@ -408,8 +416,8 @@ ifneq ($(strip $(PREV_FINAL_LDFLAGS)), $(strip $(FINAL_LDFLAGS)))
endif
# redis-server
$(REDIS_SERVER_NAME): $(REDIS_SERVER_OBJ)
$(REDIS_LD) -o $@ $^ ../deps/hiredis/libhiredis.a ../deps/lua/src/liblua.a ../deps/hdr_histogram/libhdrhistogram.a ../deps/fpconv/libfpconv.a $(FINAL_LIBS)
$(REDIS_SERVER_NAME): $(REDIS_SERVER_OBJ) $(REDIS_VEC_SETS_OBJ)
$(REDIS_LD) -o $@ $^ ../deps/hiredis/libhiredis.a ../deps/lua/src/liblua.a ../deps/hdr_histogram/libhdrhistogram.a ../deps/fpconv/libfpconv.a ../deps/fast_float/libfast_float.a $(FINAL_LIBS)
# redis-sentinel
$(REDIS_SENTINEL_NAME): $(REDIS_SERVER_NAME)
@@ -435,7 +443,7 @@ $(REDIS_CLI_NAME): $(REDIS_CLI_OBJ)
$(REDIS_BENCHMARK_NAME): $(REDIS_BENCHMARK_OBJ)
$(REDIS_LD) -o $@ $^ ../deps/hiredis/libhiredis.a ../deps/hdr_histogram/libhdrhistogram.a $(FINAL_LIBS) $(TLS_CLIENT_LIBS)
DEP = $(REDIS_SERVER_OBJ:%.o=%.d) $(REDIS_CLI_OBJ:%.o=%.d) $(REDIS_BENCHMARK_OBJ:%.o=%.d)
DEP = $(REDIS_SERVER_OBJ:%.o=%.d) $(REDIS_VEC_SETS_OBJ:%.o=%.d) $(REDIS_CLI_OBJ:%.o=%.d) $(REDIS_BENCHMARK_OBJ:%.o=%.d)
-include $(DEP)
# Because the jemalloc.h header is generated as a part of the jemalloc build,
@@ -487,8 +495,9 @@ test-cluster: $(REDIS_SERVER_NAME) $(REDIS_CLI_NAME)
check: test
lcov:
@lcov --version
$(MAKE) gcov
@(set -e; cd ..; ./runtest --clients 1)
@(set -e; cd ..; ./runtest)
@geninfo -o redis.info .
@genhtml --legend -o lcov-html redis.info
@@ -501,7 +510,7 @@ bench: $(REDIS_BENCHMARK_NAME)
@echo ""
@echo "WARNING: if it fails under Linux you probably need to install libc6-dev-i386"
@echo ""
$(MAKE) CFLAGS="-m32" LDFLAGS="-m32"
$(MAKE) CFLAGS="-m32" LDFLAGS="-m32" SKIP_VEC_SETS="yes"
gcov:
$(MAKE) REDIS_CFLAGS="-fprofile-arcs -ftest-coverage -DCOVERAGE_TEST" REDIS_LDFLAGS="-fprofile-arcs -ftest-coverage"
src/acl.c
+67 -9
View File
@@ -7,6 +7,7 @@
*/
#include "server.h"
#include "cluster.h"
#include "sha256.h"
#include <fcntl.h>
#include <ctype.h>
@@ -277,7 +278,7 @@ int ACLListMatchSds(void *a, void *b) {
/* Method to free list elements from ACL users password/patterns lists. */
void ACLListFreeSds(void *item) {
sdsfree(item);
sdsfreegeneric(item);
}
/* Method to duplicate list elements from ACL users password/patterns lists. */
@@ -469,6 +470,11 @@ void ACLFreeUser(user *u) {
zfree(u);
}
/* Generic version of ACLFreeUser. */
void ACLFreeUserGeneric(void *u) {
ACLFreeUser((user *)u);
}
/* When a user is deleted we need to cycle the active
* connections in order to kill all the pending ones that
* are authenticated with such user. */
@@ -1061,6 +1067,7 @@ int ACLSetSelector(aclSelector *selector, const char* op, size_t oplen) {
int flags = 0;
size_t offset = 1;
if (op[0] == '%') {
int perm_ok = 1;
for (; offset < oplen; offset++) {
if (toupper(op[offset]) == 'R' && !(flags & ACL_READ_PERMISSION)) {
flags |= ACL_READ_PERMISSION;
@@ -1070,10 +1077,14 @@ int ACLSetSelector(aclSelector *selector, const char* op, size_t oplen) {
offset++;
break;
} else {
errno = EINVAL;
return C_ERR;
perm_ok = 0;
break;
}
}
if (!flags || !perm_ok) {
errno = EINVAL;
return C_ERR;
}
} else {
flags = ACL_ALL_PERMISSION;
}
@@ -1577,14 +1588,22 @@ static int ACLSelectorCheckKey(aclSelector *selector, const char *key, int keyle
if (keyspec_flags & CMD_KEY_DELETE) key_flags |= ACL_WRITE_PERMISSION;
if (keyspec_flags & CMD_KEY_UPDATE) key_flags |= ACL_WRITE_PERMISSION;
/* Is given key represent a prefix of a set of keys */
int prefix = keyspec_flags & CMD_KEY_PREFIX;
/* Test this key against every pattern. */
while((ln = listNext(&li))) {
keyPattern *pattern = listNodeValue(ln);
if ((pattern->flags & key_flags) != key_flags)
continue;
size_t plen = sdslen(pattern->pattern);
if (stringmatchlen(pattern->pattern,plen,key,keylen,0))
return ACL_OK;
if (prefix) {
if (prefixmatch(pattern->pattern,plen,key,keylen,0))
return ACL_OK;
} else {
if (stringmatchlen(pattern->pattern, plen, key, keylen, 0))
return ACL_OK;
}
}
return ACL_DENIED_KEY;
}
@@ -2446,12 +2465,12 @@ sds ACLLoadFromFile(const char *filename) {
}
if (user_channels)
raxFreeWithCallback(user_channels, (void(*)(void*))listRelease);
raxFreeWithCallback(old_users,(void(*)(void*))ACLFreeUser);
raxFreeWithCallback(user_channels, listReleaseGeneric);
raxFreeWithCallback(old_users, ACLFreeUserGeneric);
sdsfree(errors);
return NULL;
} else {
raxFreeWithCallback(Users,(void(*)(void*))ACLFreeUser);
raxFreeWithCallback(Users, ACLFreeUserGeneric);
Users = old_users;
errors = sdscat(errors,"WARNING: ACL errors detected, no change to the previously active ACL rules was performed");
return errors;
@@ -2762,7 +2781,6 @@ void aclCatWithFlags(client *c, dict *commands, uint64_t cflag, int *arraylen) {
while ((de = dictNext(di)) != NULL) {
struct redisCommand *cmd = dictGetVal(de);
if (cmd->flags & CMD_MODULE) continue;
if (cmd->acl_categories & cflag) {
addReplyBulkCBuffer(c, cmd->fullname, sdslen(cmd->fullname));
(*arraylen)++;
@@ -3177,6 +3195,38 @@ void addReplyCommandCategories(client *c, struct redisCommand *cmd) {
setDeferredSetLen(c, flaglen, flagcount);
}
/* When successful, initiates an internal connection, that is able to execute
* internal commands (see CMD_INTERNAL). */
static void internalAuth(client *c) {
if (server.cluster == NULL) {
addReplyError(c, "Cannot authenticate as an internal connection on non-cluster instances");
return;
}
sds password = c->argv[2]->ptr;
/* Get internal secret. */
size_t len = -1;
const char *internal_secret = clusterGetSecret(&len);
if (sdslen(password) != len) {
addReplyError(c, "-WRONGPASS invalid internal password");
return;
}
if (!time_independent_strcmp((char *)internal_secret, (char *)password, len)) {
c->flags |= CLIENT_INTERNAL;
/* No further authentication is needed. */
c->authenticated = 1;
/* Set the user to the unrestricted user, if it is not already set (default). */
if (c->user != NULL) {
c->user = NULL;
moduleNotifyUserChanged(c);
}
addReply(c, shared.ok);
} else {
addReplyError(c, "-WRONGPASS invalid internal password");
}
}
/* AUTH <password>
* AUTH <username> <password> (Redis >= 6.0 form)
*
@@ -3210,6 +3260,14 @@ void authCommand(client *c) {
username = c->argv[1];
password = c->argv[2];
redactClientCommandArgument(c, 2);
/* Handle internal authentication commands.
* Note: No user-defined ACL user can have this username (no spaces
* allowed), thus no conflicts with ACL possible. */
if (!strcmp(username->ptr, "internal connection")) {
internalAuth(c);
return;
}
}
robj *err = NULL;
src/adlist.c
+5
View File
@@ -61,6 +61,11 @@ void listRelease(list *list)
zfree(list);
}
/* Generic version of listRelease. */
void listReleaseGeneric(void *list) {
listRelease((struct list*)list);
}
/* Add a new node to the list, to head, containing the specified 'value'
* pointer as value.
*
src/adlist.h
+1
View File
@@ -51,6 +51,7 @@ typedef struct list {
/* Prototypes */
list *listCreate(void);
void listRelease(list *list);
void listReleaseGeneric(void *list);
void listEmpty(list *list);
list *listAddNodeHead(list *list, void *value);
list *listAddNodeTail(list *list, void *value);
src/ae.c
+29 -12
View File
@@ -42,7 +42,7 @@
#endif
#endif
#define INITIAL_EVENT 1024
aeEventLoop *aeCreateEventLoop(int setsize) {
aeEventLoop *eventLoop;
int i;
@@ -50,8 +50,9 @@ aeEventLoop *aeCreateEventLoop(int setsize) {
monotonicInit(); /* just in case the calling app didn't initialize */
if ((eventLoop = zmalloc(sizeof(*eventLoop))) == NULL) goto err;
eventLoop->events = zmalloc(sizeof(aeFileEvent)*setsize);
eventLoop->fired = zmalloc(sizeof(aeFiredEvent)*setsize);
eventLoop->nevents = setsize < INITIAL_EVENT ? setsize : INITIAL_EVENT;
eventLoop->events = zmalloc(sizeof(aeFileEvent)*eventLoop->nevents);
eventLoop->fired = zmalloc(sizeof(aeFiredEvent)*eventLoop->nevents);
if (eventLoop->events == NULL || eventLoop->fired == NULL) goto err;
eventLoop->setsize = setsize;
eventLoop->timeEventHead = NULL;
@@ -61,10 +62,11 @@ aeEventLoop *aeCreateEventLoop(int setsize) {
eventLoop->beforesleep = NULL;
eventLoop->aftersleep = NULL;
eventLoop->flags = 0;
memset(eventLoop->privdata, 0, sizeof(eventLoop->privdata));
if (aeApiCreate(eventLoop) == -1) goto err;
/* Events with mask == AE_NONE are not set. So let's initialize the
* vector with it. */
for (i = 0; i < setsize; i++)
for (i = 0; i < eventLoop->nevents; i++)
eventLoop->events[i].mask = AE_NONE;
return eventLoop;
@@ -102,20 +104,19 @@ void aeSetDontWait(aeEventLoop *eventLoop, int noWait) {
*
* Otherwise AE_OK is returned and the operation is successful. */
int aeResizeSetSize(aeEventLoop *eventLoop, int setsize) {
int i;
if (setsize == eventLoop->setsize) return AE_OK;
if (eventLoop->maxfd >= setsize) return AE_ERR;
if (aeApiResize(eventLoop,setsize) == -1) return AE_ERR;
eventLoop->events = zrealloc(eventLoop->events,sizeof(aeFileEvent)*setsize);
eventLoop->fired = zrealloc(eventLoop->fired,sizeof(aeFiredEvent)*setsize);
eventLoop->setsize = setsize;
/* Make sure that if we created new slots, they are initialized with
* an AE_NONE mask. */
for (i = eventLoop->maxfd+1; i < setsize; i++)
eventLoop->events[i].mask = AE_NONE;
/* If the current allocated space is larger than the requested size,
* we need to shrink it to the requested size. */
if (setsize < eventLoop->nevents) {
eventLoop->events = zrealloc(eventLoop->events,sizeof(aeFileEvent)*setsize);
eventLoop->fired = zrealloc(eventLoop->fired,sizeof(aeFiredEvent)*setsize);
eventLoop->nevents = setsize;
}
return AE_OK;
}
@@ -147,6 +148,22 @@ int aeCreateFileEvent(aeEventLoop *eventLoop, int fd, int mask,
errno = ERANGE;
return AE_ERR;
}
/* Resize the events and fired arrays if the file
* descriptor exceeds the current number of events. */
if (unlikely(fd >= eventLoop->nevents)) {
int newnevents = eventLoop->nevents;
newnevents = (newnevents * 2 > fd + 1) ? newnevents * 2 : fd + 1;
newnevents = (newnevents > eventLoop->setsize) ? eventLoop->setsize : newnevents;
eventLoop->events = zrealloc(eventLoop->events, sizeof(aeFileEvent) * newnevents);
eventLoop->fired = zrealloc(eventLoop->fired, sizeof(aeFiredEvent) * newnevents);
/* Initialize new slots with an AE_NONE mask */
for (int i = eventLoop->nevents; i < newnevents; i++)
eventLoop->events[i].mask = AE_NONE;
eventLoop->nevents = newnevents;
}
aeFileEvent *fe = &eventLoop->events[fd];
if (aeApiAddEvent(eventLoop, fd, mask) == -1)
src/ae.h
+2
View File
@@ -79,6 +79,7 @@ typedef struct aeEventLoop {
int maxfd; /* highest file descriptor currently registered */
int setsize; /* max number of file descriptors tracked */
long long timeEventNextId;
int nevents; /* Size of Registered events */
aeFileEvent *events; /* Registered events */
aeFiredEvent *fired; /* Fired events */
aeTimeEvent *timeEventHead;
@@ -87,6 +88,7 @@ typedef struct aeEventLoop {
aeBeforeSleepProc *beforesleep;
aeBeforeSleepProc *aftersleep;
int flags;
void *privdata[2];
} aeEventLoop;
/* Prototypes */
src/aof.c
+128 -32
View File
@@ -30,6 +30,13 @@ aofManifest *aofLoadManifestFromFile(sds am_filepath);
void aofManifestFreeAndUpdate(aofManifest *am);
void aof_background_fsync_and_close(int fd);
/* When we call 'startAppendOnly', we will create a temp INCR AOF, and rename
* it to the real INCR AOF name when the AOFRW is done, so if want to know the
* accurate start offset of the INCR AOF, we need to record it when we create
* the temp INCR AOF. This variable is used to record the start offset, and
* set the start offset of the real INCR AOF when the AOFRW is done. */
static long long tempIncAofStartReplOffset = 0;
/* ----------------------------------------------------------------------------
* AOF Manifest file implementation.
*
@@ -73,10 +80,15 @@ void aof_background_fsync_and_close(int fd);
#define AOF_MANIFEST_KEY_FILE_NAME "file"
#define AOF_MANIFEST_KEY_FILE_SEQ "seq"
#define AOF_MANIFEST_KEY_FILE_TYPE "type"
#define AOF_MANIFEST_KEY_FILE_STARTOFFSET "startoffset"
#define AOF_MANIFEST_KEY_FILE_ENDOFFSET "endoffset"
/* Create an empty aofInfo. */
aofInfo *aofInfoCreate(void) {
return zcalloc(sizeof(aofInfo));
aofInfo *ai = zcalloc(sizeof(aofInfo));
ai->start_offset = -1;
ai->end_offset = -1;
return ai;
}
/* Free the aofInfo structure (pointed to by ai) and its embedded file_name. */
@@ -93,6 +105,8 @@ aofInfo *aofInfoDup(aofInfo *orig) {
ai->file_name = sdsdup(orig->file_name);
ai->file_seq = orig->file_seq;
ai->file_type = orig->file_type;
ai->start_offset = orig->start_offset;
ai->end_offset = orig->end_offset;
return ai;
}
@@ -105,10 +119,19 @@ sds aofInfoFormat(sds buf, aofInfo *ai) {
if (sdsneedsrepr(ai->file_name))
filename_repr = sdscatrepr(sdsempty(), ai->file_name, sdslen(ai->file_name));
sds ret = sdscatprintf(buf, "%s %s %s %lld %s %c\n",
sds ret = sdscatprintf(buf, "%s %s %s %lld %s %c",
AOF_MANIFEST_KEY_FILE_NAME, filename_repr ? filename_repr : ai->file_name,
AOF_MANIFEST_KEY_FILE_SEQ, ai->file_seq,
AOF_MANIFEST_KEY_FILE_TYPE, ai->file_type);
if (ai->start_offset != -1) {
ret = sdscatprintf(ret, " %s %lld", AOF_MANIFEST_KEY_FILE_STARTOFFSET, ai->start_offset);
if (ai->end_offset != -1) {
ret = sdscatprintf(ret, " %s %lld", AOF_MANIFEST_KEY_FILE_ENDOFFSET, ai->end_offset);
}
}
ret = sdscatlen(ret, "\n", 1);
sdsfree(filename_repr);
return ret;
@@ -304,6 +327,10 @@ aofManifest *aofLoadManifestFromFile(sds am_filepath) {
ai->file_seq = atoll(argv[i+1]);
} else if (!strcasecmp(argv[i], AOF_MANIFEST_KEY_FILE_TYPE)) {
ai->file_type = (argv[i+1])[0];
} else if (!strcasecmp(argv[i], AOF_MANIFEST_KEY_FILE_STARTOFFSET)) {
ai->start_offset = atoll(argv[i+1]);
} else if (!strcasecmp(argv[i], AOF_MANIFEST_KEY_FILE_ENDOFFSET)) {
ai->end_offset = atoll(argv[i+1]);
}
/* else if (!strcasecmp(argv[i], AOF_MANIFEST_KEY_OTHER)) {} */
}
@@ -433,12 +460,13 @@ sds getNewBaseFileNameAndMarkPreAsHistory(aofManifest *am) {
* for example:
* appendonly.aof.1.incr.aof
*/
sds getNewIncrAofName(aofManifest *am) {
sds getNewIncrAofName(aofManifest *am, long long start_reploff) {
aofInfo *ai = aofInfoCreate();
ai->file_type = AOF_FILE_TYPE_INCR;
ai->file_name = sdscatprintf(sdsempty(), "%s.%lld%s%s", server.aof_filename,
++am->curr_incr_file_seq, INCR_FILE_SUFFIX, AOF_FORMAT_SUFFIX);
ai->file_seq = am->curr_incr_file_seq;
ai->start_offset = start_reploff;
listAddNodeTail(am->incr_aof_list, ai);
am->dirty = 1;
return ai->file_name;
@@ -456,7 +484,7 @@ sds getLastIncrAofName(aofManifest *am) {
/* If 'incr_aof_list' is empty, just create a new one. */
if (!listLength(am->incr_aof_list)) {
return getNewIncrAofName(am);
return getNewIncrAofName(am, server.master_repl_offset);
}
/* Or return the last one. */
@@ -781,10 +809,11 @@ int openNewIncrAofForAppend(void) {
if (server.aof_state == AOF_WAIT_REWRITE) {
/* Use a temporary INCR AOF file to accumulate data during AOF_WAIT_REWRITE. */
new_aof_name = getTempIncrAofName();
tempIncAofStartReplOffset = server.master_repl_offset;
} else {
/* Dup a temp aof_manifest to modify. */
temp_am = aofManifestDup(server.aof_manifest);
new_aof_name = sdsdup(getNewIncrAofName(temp_am));
new_aof_name = sdsdup(getNewIncrAofName(temp_am, server.master_repl_offset));
}
sds new_aof_filepath = makePath(server.aof_dirname, new_aof_name);
newfd = open(new_aof_filepath, O_WRONLY|O_TRUNC|O_CREAT, 0644);
@@ -833,6 +862,50 @@ cleanup:
return C_ERR;
}
/* When we close gracefully the AOF file, we have the chance to persist the
* end replication offset of current INCR AOF. */
void updateCurIncrAofEndOffset(void) {
if (server.aof_state != AOF_ON) return;
serverAssert(server.aof_manifest != NULL);
if (listLength(server.aof_manifest->incr_aof_list) == 0) return;
aofInfo *ai = listNodeValue(listLast(server.aof_manifest->incr_aof_list));
ai->end_offset = server.master_repl_offset;
server.aof_manifest->dirty = 1;
/* It doesn't matter if the persistence fails since this information is not
* critical, we can get an approximate value by start offset plus file size. */
persistAofManifest(server.aof_manifest);
}
/* After loading AOF data, we need to update the `server.master_repl_offset`
* based on the information of the last INCR AOF, to avoid the rollback of
* the start offset of new INCR AOF. */
void updateReplOffsetAndResetEndOffset(void) {
if (server.aof_state != AOF_ON) return;
serverAssert(server.aof_manifest != NULL);
/* If the INCR file has an end offset, we directly use it, and clear it
* to avoid the next time we load the manifest file, we will use the same
* offset, but the real offset may have advanced. */
if (listLength(server.aof_manifest->incr_aof_list) == 0) return;
aofInfo *ai = listNodeValue(listLast(server.aof_manifest->incr_aof_list));
if (ai->end_offset != -1) {
server.master_repl_offset = ai->end_offset;
ai->end_offset = -1;
server.aof_manifest->dirty = 1;
/* We must update the end offset of INCR file correctly, otherwise we
* may keep wrong information in the manifest file, since we continue
* to append data to the same INCR file. */
if (persistAofManifest(server.aof_manifest) != AOF_OK)
exit(1);
} else {
/* If the INCR file doesn't have an end offset, we need to calculate
* the replication offset by the start offset plus the file size. */
server.master_repl_offset = (ai->start_offset == -1 ? 0 : ai->start_offset) +
getAppendOnlyFileSize(ai->file_name, NULL);
}
}
/* Whether to limit the execution of Background AOF rewrite.
*
* At present, if AOFRW fails, redis will automatically retry. If it continues
@@ -938,6 +1011,7 @@ void stopAppendOnly(void) {
server.aof_last_fsync = server.mstime;
}
close(server.aof_fd);
updateCurIncrAofEndOffset();
server.aof_fd = -1;
server.aof_selected_db = -1;
@@ -997,6 +1071,29 @@ int startAppendOnly(void) {
return C_OK;
}
void startAppendOnlyWithRetry(void) {
unsigned int tries, max_tries = 10;
for (tries = 0; tries < max_tries; ++tries) {
if (startAppendOnly() == C_OK)
break;
serverLog(LL_WARNING, "Failed to enable AOF! Trying it again in one second.");
sleep(1);
}
if (tries == max_tries) {
serverLog(LL_WARNING, "FATAL: AOF can't be turned on. Exiting now.");
exit(1);
}
}
/* Called after "appendonly" config is changed. */
void applyAppendOnlyConfig(void) {
if (!server.aof_enabled && server.aof_state != AOF_OFF) {
stopAppendOnly();
} else if (server.aof_enabled && server.aof_state == AOF_OFF) {
startAppendOnlyWithRetry();
}
}
/* This is a wrapper to the write syscall in order to retry on short writes
* or if the syscall gets interrupted. It could look strange that we retry
* on short writes given that we are writing to a block device: normally if
@@ -1048,35 +1145,34 @@ void flushAppendOnlyFile(int force) {
mstime_t latency;
if (sdslen(server.aof_buf) == 0) {
/* Check if we need to do fsync even the aof buffer is empty,
* because previously in AOF_FSYNC_EVERYSEC mode, fsync is
* called only when aof buffer is not empty, so if users
* stop write commands before fsync called in one second,
* the data in page cache cannot be flushed in time. */
if (server.aof_fsync == AOF_FSYNC_EVERYSEC &&
server.aof_last_incr_fsync_offset != server.aof_last_incr_size &&
server.mstime - server.aof_last_fsync >= 1000 &&
!(sync_in_progress = aofFsyncInProgress())) {
goto try_fsync;
/* Check if we need to do fsync even the aof buffer is empty,
* the reason is described in the previous AOF_FSYNC_EVERYSEC block,
* and AOF_FSYNC_ALWAYS is also checked here to handle a case where
* aof_fsync is changed from everysec to always. */
} else if (server.aof_fsync == AOF_FSYNC_ALWAYS &&
server.aof_last_incr_fsync_offset != server.aof_last_incr_size)
{
goto try_fsync;
} else {
if (server.aof_last_incr_fsync_offset == server.aof_last_incr_size) {
/* All data is fsync'd already: Update fsynced_reploff_pending just in case.
* This is needed to avoid a WAITAOF hang in case a module used RM_Call with the NO_AOF flag,
* in which case master_repl_offset will increase but fsynced_reploff_pending won't be updated
* (because there's no reason, from the AOF POV, to call fsync) and then WAITAOF may wait on
* the higher offset (which contains data that was only propagated to replicas, and not to AOF) */
if (!sync_in_progress && server.aof_fsync != AOF_FSYNC_NO)
* This is needed to avoid a WAITAOF hang in case a module used RM_Call
* with the NO_AOF flag, in which case master_repl_offset will increase but
* fsynced_reploff_pending won't be updated (because there's no reason, from
* the AOF POV, to call fsync) and then WAITAOF may wait on the higher offset
* (which contains data that was only propagated to replicas, and not to AOF) */
if (!aofFsyncInProgress())
atomicSet(server.fsynced_reploff_pending, server.master_repl_offset);
return;
} else {
/* Check if we need to do fsync even the aof buffer is empty,
* because previously in AOF_FSYNC_EVERYSEC mode, fsync is
* called only when aof buffer is not empty, so if users
* stop write commands before fsync called in one second,
* the data in page cache cannot be flushed in time. */
if (server.aof_fsync == AOF_FSYNC_EVERYSEC &&
server.mstime - server.aof_last_fsync >= 1000 &&
!(sync_in_progress = aofFsyncInProgress()))
goto try_fsync;
/* Check if we need to do fsync even the aof buffer is empty,
* the reason is described in the previous AOF_FSYNC_EVERYSEC block,
* and AOF_FSYNC_ALWAYS is also checked here to handle a case where
* aof_fsync is changed from everysec to always. */
if (server.aof_fsync == AOF_FSYNC_ALWAYS)
goto try_fsync;
}
return;
}
if (server.aof_fsync == AOF_FSYNC_EVERYSEC)
@@ -2642,7 +2738,7 @@ void backgroundRewriteDoneHandler(int exitcode, int bysignal) {
sds temp_incr_aof_name = getTempIncrAofName();
sds temp_incr_filepath = makePath(server.aof_dirname, temp_incr_aof_name);
/* Get next new incr aof name. */
sds new_incr_filename = getNewIncrAofName(temp_am);
sds new_incr_filename = getNewIncrAofName(temp_am, tempIncAofStartReplOffset);
new_incr_filepath = makePath(server.aof_dirname, new_incr_filename);
latencyStartMonitor(latency);
if (rename(temp_incr_filepath, new_incr_filepath) == -1) {
src/asciilogo.h
+1 -1
View File
@@ -10,7 +10,7 @@ const char *ascii_logo =
" _._ \n"
" _.-``__ ''-._ \n"
" _.-`` `. `_. ''-._ Redis Community Edition \n"
" .-`` .-```. ```\\/ _.,_ ''-._ %s (%s/%d) %s bit\n"
" .-`` .-```. ```\\/ _.,_ ''-._ %s (%s/%d) %s bit\n"
" ( ' , .-` | `, ) Running in %s mode\n"
" |`-._`-...-` __...-.``-._|'` _.-'| Port: %d\n"
" | `-._ `._ / _.-' | PID: %ld\n"
src/atomicvar.h
+1 -1
View File
@@ -32,7 +32,7 @@
* (if the flag was 0 -> set to 1, if it's already 1 -> do nothing, but the final result is that the flag is set),
* and also it has a full barrier (__sync_lock_test_and_set has acquire barrier).
*
* NOTE2: Unlike other atomic type, which aren't guaranteed to be lock free, c11 atmoic_flag does.
* NOTE2: Unlike other atomic type, which aren't guaranteed to be lock free, c11 atomic_flag does.
* To check whether a type is lock free, atomic_is_lock_free() can be used.
* It can be considered to limit the flag type to atomic_flag to improve performance.
*
src/bio.c
+6 -2
View File
@@ -81,6 +81,7 @@ static int job_comp_pipe[2]; /* Pipe used to awake the event loop */
typedef struct bio_comp_item {
comp_fn *func; /* callback after completion job will be processed */
uint64_t arg; /* user data to be passed to the function */
void *ptr; /* user pointer to be passed to the function */
} bio_comp_item;
/* This structure represents a background Job. It is only used locally to this
@@ -110,6 +111,7 @@ typedef union bio_job {
int type; /* header */
comp_fn *fn; /* callback. Handover to main thread to cb as notify for job completion */
uint64_t arg; /* callback arguments */
void *ptr; /* callback pointer */
} comp_rq;
} bio_job;
@@ -200,7 +202,7 @@ void bioCreateLazyFreeJob(lazy_free_fn free_fn, int arg_count, ...) {
bioSubmitJob(BIO_LAZY_FREE, job);
}
void bioCreateCompRq(bio_worker_t assigned_worker, comp_fn *func, uint64_t user_data) {
void bioCreateCompRq(bio_worker_t assigned_worker, comp_fn *func, uint64_t user_data, void *user_ptr) {
int type;
switch (assigned_worker) {
case BIO_WORKER_CLOSE_FILE:
@@ -219,6 +221,7 @@ void bioCreateCompRq(bio_worker_t assigned_worker, comp_fn *func, uint64_t user_
bio_job *job = zmalloc(sizeof(*job));
job->comp_rq.fn = func;
job->comp_rq.arg = user_data;
job->comp_rq.ptr = user_ptr;
bioSubmitJob(type, job);
}
@@ -339,6 +342,7 @@ void *bioProcessBackgroundJobs(void *arg) {
bio_comp_item *comp_rsp = zmalloc(sizeof(bio_comp_item));
comp_rsp->func = job->comp_rq.fn;
comp_rsp->arg = job->comp_rq.arg;
comp_rsp->ptr = job->comp_rq.ptr;
/* just write it to completion job responses */
pthread_mutex_lock(&bio_mutex_comp);
@@ -432,7 +436,7 @@ void bioPipeReadJobCompList(aeEventLoop *el, int fd, void *privdata, int mask) {
listNode *ln = listFirst(tmp_list);
bio_comp_item *rsp = ln->value;
listDelNode(tmp_list, ln);
rsp->func(rsp->arg);
rsp->func(rsp->arg, rsp->ptr);
zfree(rsp);
}
listRelease(tmp_list);
src/bio.h
+2 -2
View File
@@ -10,7 +10,7 @@
#define __BIO_H
typedef void lazy_free_fn(void *args[]);
typedef void comp_fn(uint64_t user_data);
typedef void comp_fn(uint64_t user_data, void *user_ptr);
typedef enum bio_worker_t {
BIO_WORKER_CLOSE_FILE = 0,
@@ -40,7 +40,7 @@ void bioCreateCloseJob(int fd, int need_fsync, int need_reclaim_cache);
void bioCreateCloseAofJob(int fd, long long offset, int need_reclaim_cache);
void bioCreateFsyncJob(int fd, long long offset, int need_reclaim_cache);
void bioCreateLazyFreeJob(lazy_free_fn free_fn, int arg_count, ...);
void bioCreateCompRq(bio_worker_t assigned_worker, comp_fn *func, uint64_t user_data);
void bioCreateCompRq(bio_worker_t assigned_worker, comp_fn *func, uint64_t user_data, void *user_ptr);
#endif
src/bitops.c
+72 -17
View File
@@ -16,18 +16,49 @@
/* Count number of bits set in the binary array pointed by 's' and long
* 'count' bytes. The implementation of this function is required to
* work with an input string length up to 512 MB or more (server.proto_max_bulk_len) */
ATTRIBUTE_TARGET_POPCNT
long long redisPopcount(void *s, long count) {
long long bits = 0;
unsigned char *p = s;
uint32_t *p4;
#if defined(HAVE_POPCNT)
int use_popcnt = __builtin_cpu_supports("popcnt"); /* Check if CPU supports POPCNT instruction. */
#else
int use_popcnt = 0; /* Assume CPU does not support POPCNT if
* __builtin_cpu_supports() is not available. */
#endif
static const unsigned char bitsinbyte[256] = {0,1,1,2,1,2,2,3,1,2,2,3,2,3,3,4,1,2,2,3,2,3,3,4,2,3,3,4,3,4,4,5,1,2,2,3,2,3,3,4,2,3,3,4,3,4,4,5,2,3,3,4,3,4,4,5,3,4,4,5,4,5,5,6,1,2,2,3,2,3,3,4,2,3,3,4,3,4,4,5,2,3,3,4,3,4,4,5,3,4,4,5,4,5,5,6,2,3,3,4,3,4,4,5,3,4,4,5,4,5,5,6,3,4,4,5,4,5,5,6,4,5,5,6,5,6,6,7,1,2,2,3,2,3,3,4,2,3,3,4,3,4,4,5,2,3,3,4,3,4,4,5,3,4,4,5,4,5,5,6,2,3,3,4,3,4,4,5,3,4,4,5,4,5,5,6,3,4,4,5,4,5,5,6,4,5,5,6,5,6,6,7,2,3,3,4,3,4,4,5,3,4,4,5,4,5,5,6,3,4,4,5,4,5,5,6,4,5,5,6,5,6,6,7,3,4,4,5,4,5,5,6,4,5,5,6,5,6,6,7,4,5,5,6,5,6,6,7,5,6,6,7,6,7,7,8};
/* Count initial bytes not aligned to 32 bit. */
while((unsigned long)p & 3 && count) {
/* Count initial bytes not aligned to 64-bit when using the POPCNT instruction,
* otherwise align to 32-bit. */
int align = use_popcnt ? 7 : 3;
while ((unsigned long)p & align && count) {
bits += bitsinbyte[*p++];
count--;
}
if (likely(use_popcnt)) {
/* Use separate counters to make the CPU think there are no
* dependencies between these popcnt operations. */
uint64_t cnt[4];
memset(cnt, 0, sizeof(cnt));
/* Count bits 32 bytes at a time by using popcnt.
* Unroll the loop to avoid the overhead of a single popcnt per iteration,
* allowing the CPU to extract more instruction-level parallelism.
* Reference: https://danluu.com/assembly-intrinsics/ */
while (count >= 32) {
cnt[0] += __builtin_popcountll(*(uint64_t*)(p));
cnt[1] += __builtin_popcountll(*(uint64_t*)(p + 8));
cnt[2] += __builtin_popcountll(*(uint64_t*)(p + 16));
cnt[3] += __builtin_popcountll(*(uint64_t*)(p + 24));
count -= 32;
p += 32;
}
bits += cnt[0] + cnt[1] + cnt[2] + cnt[3];
goto remain;
}
/* Count bits 28 bytes at a time */
p4 = (uint32_t*)p;
while(count>=28) {
@@ -64,8 +95,10 @@ long long redisPopcount(void *s, long count) {
((aux6 + (aux6 >> 4)) & 0x0F0F0F0F) +
((aux7 + (aux7 >> 4)) & 0x0F0F0F0F))* 0x01010101) >> 24;
}
/* Count the remaining bytes. */
p = (unsigned char*)p4;
remain:
/* Count the remaining bytes. */
while(count--) bits += bitsinbyte[*p++];
return bits;
}
@@ -456,22 +489,27 @@ int getBitfieldTypeFromArgument(client *c, robj *o, int *sign, int *bits) {
* bits to a string object. The command creates or pad with zeroes the string
* so that the 'maxbit' bit can be addressed. The object is finally
* returned. Otherwise if the key holds a wrong type NULL is returned and
* an error is sent to the client. */
robj *lookupStringForBitCommand(client *c, uint64_t maxbit, int *dirty) {
* an error is sent to the client.
*
* (Must provide all the arguments to the function)
*/
static robj *lookupStringForBitCommand(client *c, uint64_t maxbit,
size_t *strOldSize, size_t *strGrowSize)
{
size_t byte = maxbit >> 3;
robj *o = lookupKeyWrite(c->db,c->argv[1]);
if (checkType(c,o,OBJ_STRING)) return NULL;
if (dirty) *dirty = 0;
if (o == NULL) {
o = createObject(OBJ_STRING,sdsnewlen(NULL, byte+1));
dbAdd(c->db,c->argv[1],o);
if (dirty) *dirty = 1;
*strGrowSize = byte + 1;
*strOldSize = 0;
} else {
o = dbUnshareStringValue(c->db,c->argv[1],o);
size_t oldlen = sdslen(o->ptr);
*strOldSize = sdslen(o->ptr);
o->ptr = sdsgrowzero(o->ptr,byte+1);
if (dirty && oldlen != sdslen(o->ptr)) *dirty = 1;
*strGrowSize = sdslen(o->ptr) - *strOldSize;
}
return o;
}
@@ -528,8 +566,9 @@ void setbitCommand(client *c) {
return;
}
int dirty;
if ((o = lookupStringForBitCommand(c,bitoffset,&dirty)) == NULL) return;
size_t strOldSize, strGrowSize;
if ((o = lookupStringForBitCommand(c,bitoffset,&strOldSize,&strGrowSize)) == NULL)
return;
/* Get current values */
byte = bitoffset >> 3;
@@ -540,7 +579,7 @@ void setbitCommand(client *c) {
/* Either it is newly created, changed length, or the bit changes before and after.
* Note that the bitval here is actually a decimal number.
* So we need to use `!!` to convert it to 0 or 1 for comparison. */
if (dirty || (!!bitval != on)) {
if (strGrowSize || (!!bitval != on)) {
/* Update byte with new bit value. */
byteval &= ~(1 << bit);
byteval |= ((on & 0x1) << bit);
@@ -548,6 +587,13 @@ void setbitCommand(client *c) {
signalModifiedKey(c,c->db,c->argv[1]);
notifyKeyspaceEvent(NOTIFY_STRING,"setbit",c->argv[1],c->db->id);
server.dirty++;
/* If this is not a new key (old size not 0) and size changed, then
* update the keysizes histogram. Otherwise, the histogram already
* updated in lookupStringForBitCommand() by calling dbAdd(). */
if ((strOldSize > 0) && (strGrowSize != 0))
updateKeysizesHist(c->db, getKeySlot(c->argv[1]->ptr), OBJ_STRING,
strOldSize, strOldSize + strGrowSize);
}
/* Return original value. */
@@ -1032,7 +1078,8 @@ struct bitfieldOp {
void bitfieldGeneric(client *c, int flags) {
robj *o;
uint64_t bitoffset;
int j, numops = 0, changes = 0, dirty = 0;
int j, numops = 0, changes = 0;
size_t strOldSize, strGrowSize = 0;
struct bitfieldOp *ops = NULL; /* Array of ops to execute at end. */
int owtype = BFOVERFLOW_WRAP; /* Overflow type. */
int readonly = 1;
@@ -1126,7 +1173,7 @@ void bitfieldGeneric(client *c, int flags) {
/* Lookup by making room up to the farthest bit reached by
* this operation. */
if ((o = lookupStringForBitCommand(c,
highest_write_offset,&dirty)) == NULL) {
highest_write_offset,&strOldSize,&strGrowSize)) == NULL) {
zfree(ops);
return;
}
@@ -1176,7 +1223,7 @@ void bitfieldGeneric(client *c, int flags) {
setSignedBitfield(o->ptr,thisop->offset,
thisop->bits,newval);
if (dirty || (oldval != newval))
if (strGrowSize || (oldval != newval))
changes++;
} else {
addReplyNull(c);
@@ -1210,7 +1257,7 @@ void bitfieldGeneric(client *c, int flags) {
setUnsignedBitfield(o->ptr,thisop->offset,
thisop->bits,newval);
if (dirty || (oldval != newval))
if (strGrowSize || (oldval != newval))
changes++;
} else {
addReplyNull(c);
@@ -1253,6 +1300,14 @@ void bitfieldGeneric(client *c, int flags) {
}
if (changes) {
/* If this is not a new key (old size not 0) and size changed, then
* update the keysizes histogram. Otherwise, the histogram already
* updated in lookupStringForBitCommand() by calling dbAdd(). */
if ((strOldSize > 0) && (strGrowSize != 0))
updateKeysizesHist(c->db, getKeySlot(c->argv[1]->ptr), OBJ_STRING,
strOldSize, strOldSize + strGrowSize);
signalModifiedKey(c,c->db,c->argv[1]);
notifyKeyspaceEvent(NOTIFY_STRING,"setbit",c->argv[1],c->db->id);
server.dirty += changes;
src/blocked.c
+1
View File
@@ -270,6 +270,7 @@ void disconnectAllBlockedClients(void) {
if (c->bstate.btype == BLOCKED_LAZYFREE) {
addReply(c, shared.ok); /* No reason lazy-free to fail */
updateStatsOnUnblock(c, 0, 0, 0);
c->flags &= ~CLIENT_PENDING_COMMAND;
unblockClient(c, 1);
} else {
src/call_reply.c
+1 -1
View File
@@ -533,7 +533,7 @@ CallReply *callReplyCreateError(sds reply, void *private_data) {
sdsfree(reply);
}
list *deferred_error_list = listCreate();
listSetFreeMethod(deferred_error_list, (void (*)(void*))sdsfree);
listSetFreeMethod(deferred_error_list, sdsfreegeneric);
listAddNodeTail(deferred_error_list, sdsnew(err_buff));
return callReplyCreate(err_buff, deferred_error_list, private_data);
}
src/cluster.c
+245 -1
View File
@@ -317,7 +317,7 @@ migrateCachedSocket* migrateGetSocket(client *c, robj *host, robj *port, long ti
}
/* Create the connection */
conn = connCreate(connTypeOfCluster());
conn = connCreate(server.el, connTypeOfCluster());
if (connBlockingConnect(conn, host->ptr, atoi(port->ptr), timeout)
!= C_OK) {
addReplyError(c,"-IOERR error or timeout connecting to the client");
@@ -783,6 +783,130 @@ unsigned int countKeysInSlot(unsigned int slot) {
return kvstoreDictSize(server.db->keys, slot);
}
/* Add detailed information of a node to the output buffer of the given client. */
void addNodeDetailsToShardReply(client *c, clusterNode *node) {
int reply_count = 0;
char *hostname;
void *node_replylen = addReplyDeferredLen(c);
addReplyBulkCString(c, "id");
addReplyBulkCBuffer(c, clusterNodeGetName(node), CLUSTER_NAMELEN);
reply_count++;
if (clusterNodeTcpPort(node)) {
addReplyBulkCString(c, "port");
addReplyLongLong(c, clusterNodeTcpPort(node));
reply_count++;
}
if (clusterNodeTlsPort(node)) {
addReplyBulkCString(c, "tls-port");
addReplyLongLong(c, clusterNodeTlsPort(node));
reply_count++;
}
addReplyBulkCString(c, "ip");
addReplyBulkCString(c, clusterNodeIp(node));
reply_count++;
addReplyBulkCString(c, "endpoint");
addReplyBulkCString(c, clusterNodePreferredEndpoint(node));
reply_count++;
hostname = clusterNodeHostname(node);
if (hostname != NULL && *hostname != '\0') {
addReplyBulkCString(c, "hostname");
addReplyBulkCString(c, hostname);
reply_count++;
}
long long node_offset;
if (clusterNodeIsMyself(node)) {
node_offset = clusterNodeIsSlave(node) ? replicationGetSlaveOffset() : server.master_repl_offset;
} else {
node_offset = clusterNodeReplOffset(node);
}
addReplyBulkCString(c, "role");
addReplyBulkCString(c, clusterNodeIsSlave(node) ? "replica" : "master");
reply_count++;
addReplyBulkCString(c, "replication-offset");
addReplyLongLong(c, node_offset);
reply_count++;
addReplyBulkCString(c, "health");
const char *health_msg = NULL;
if (clusterNodeIsFailing(node)) {
health_msg = "fail";
} else if (clusterNodeIsSlave(node) && node_offset == 0) {
health_msg = "loading";
} else {
health_msg = "online";
}
addReplyBulkCString(c, health_msg);
reply_count++;
setDeferredMapLen(c, node_replylen, reply_count);
}
static clusterNode *clusterGetMasterFromShard(void *shard_handle) {
clusterNode *n = NULL;
void *node_it = clusterShardHandleGetNodeIterator(shard_handle);
while((n = clusterShardNodeIteratorNext(node_it)) != NULL) {
if (!clusterNodeIsFailing(n)) {
break;
}
}
clusterShardNodeIteratorFree(node_it);
if (!n) return NULL;
return clusterNodeGetMaster(n);
}
/* Add the shard reply of a single shard based off the given primary node. */
void addShardReplyForClusterShards(client *c, void *shard_handle) {
serverAssert(clusterGetShardNodeCount(shard_handle) > 0);
addReplyMapLen(c, 2);
addReplyBulkCString(c, "slots");
/* Use slot_info_pairs from the primary only */
clusterNode *master_node = clusterGetMasterFromShard(shard_handle);
if (master_node && clusterNodeHasSlotInfo(master_node)) {
serverAssert((clusterNodeSlotInfoCount(master_node) % 2) == 0);
addReplyArrayLen(c, clusterNodeSlotInfoCount(master_node));
for (int i = 0; i < clusterNodeSlotInfoCount(master_node); i++)
addReplyLongLong(c, (unsigned long)clusterNodeSlotInfoEntry(master_node, i));
} else {
/* If no slot info pair is provided, the node owns no slots */
addReplyArrayLen(c, 0);
}
addReplyBulkCString(c, "nodes");
addReplyArrayLen(c, clusterGetShardNodeCount(shard_handle));
void *node_it = clusterShardHandleGetNodeIterator(shard_handle);
for (clusterNode *n = clusterShardNodeIteratorNext(node_it); n != NULL; n = clusterShardNodeIteratorNext(node_it)) {
addNodeDetailsToShardReply(c, n);
clusterFreeNodesSlotsInfo(n);
}
clusterShardNodeIteratorFree(node_it);
}
/* Add to the output buffer of the given client, an array of slot (start, end)
* pair owned by the shard, also the primary and set of replica(s) along with
* information about each node. */
void clusterCommandShards(client *c) {
addReplyArrayLen(c, clusterGetShardCount());
/* This call will add slot_info_pairs to all nodes */
clusterGenNodesSlotsInfo(0);
dictIterator *shard_it = clusterGetShardIterator();
for(void *shard_handle = clusterNextShardHandle(shard_it); shard_handle != NULL; shard_handle = clusterNextShardHandle(shard_it)) {
addShardReplyForClusterShards(c, shard_handle);
}
clusterFreeShardIterator(shard_it);
}
void clusterCommandHelp(client *c) {
const char *help[] = {
"COUNTKEYSINSLOT <slot>",
@@ -1434,6 +1558,126 @@ void readonlyCommand(client *c) {
addReply(c,shared.ok);
}
void replySlotsFlushAndFree(client *c, SlotsFlush *sflush) {
addReplyArrayLen(c, sflush->numRanges);
for (int i = 0 ; i < sflush->numRanges ; i++) {
addReplyArrayLen(c, 2);
addReplyLongLong(c, sflush->ranges[i].first);
addReplyLongLong(c, sflush->ranges[i].last);
}
zfree(sflush);
}
/* Partially flush destination DB in a cluster node, based on the slot range.
*
* Usage: SFLUSH <start-slot> <end slot> [<start-slot> <end slot>]* [SYNC|ASYNC]
*
* This is an initial implementation of SFLUSH (slots flush) which is limited to
* flushing a single shard as a whole, but in the future the same command may be
* used to partially flush a shard based on hash slots. Currently only if provided
* slots cover entirely the slots of a node, the node will be flushed and the
* return value will be pairs of slot ranges. Otherwise, a single empty set will
* be returned. If possible, SFLUSH SYNC will be run as blocking ASYNC as an
* optimization.
*/
void sflushCommand(client *c) {
int flags = EMPTYDB_NO_FLAGS, argc = c->argc;
if (server.cluster_enabled == 0) {
addReplyError(c,"This instance has cluster support disabled");
return;
}
/* check if last argument is SYNC or ASYNC */
if (!strcasecmp(c->argv[c->argc-1]->ptr,"sync")) {
flags = EMPTYDB_NO_FLAGS;
argc--;
} else if (!strcasecmp(c->argv[c->argc-1]->ptr,"async")) {
flags = EMPTYDB_ASYNC;
argc--;
} else if (server.lazyfree_lazy_user_flush) {
flags = EMPTYDB_ASYNC;
}
/* parse the slot range */
if (argc % 2 == 0) {
addReplyErrorArity(c);
return;
}
/* Verify <first, last> slot pairs are valid and not overlapping */
long long j, first, last;
unsigned char slotsToFlushRq[CLUSTER_SLOTS] = {0};
for (j = 1; j < argc; j += 2) {
/* check if the first slot is valid */
if (getLongLongFromObject(c->argv[j], &first) != C_OK || first < 0 || first >= CLUSTER_SLOTS) {
addReplyError(c,"Invalid or out of range slot");
return;
}
/* check if the last slot is valid */
if (getLongLongFromObject(c->argv[j+1], &last) != C_OK || last < 0 || last >= CLUSTER_SLOTS) {
addReplyError(c,"Invalid or out of range slot");
return;
}
if (first > last) {
addReplyErrorFormat(c,"start slot number %lld is greater than end slot number %lld", first, last);
return;
}
/* Mark the slots in slotsToFlushRq[] */
for (int i = first; i <= last; i++) {
if (slotsToFlushRq[i]) {
addReplyErrorFormat(c, "Slot %d specified multiple times", i);
return;
}
slotsToFlushRq[i] = 1;
}
}
/* Verify slotsToFlushRq[] covers ALL slots of myNode. */
clusterNode *myNode = getMyClusterNode();
/* During iteration trace also the slot range pairs and save in SlotsFlush.
* It is allocated on heap since there is a chance that FLUSH SYNC will be
* running as blocking ASYNC and only later reply with slot ranges */
int capacity = 32; /* Initial capacity */
SlotsFlush *sflush = zmalloc(sizeof(SlotsFlush) + sizeof(SlotRange) * capacity);
sflush->numRanges = 0;
int inSlotRange = 0;
for (int i = 0; i < CLUSTER_SLOTS; i++) {
if (myNode == getNodeBySlot(i)) {
if (!slotsToFlushRq[i]) {
addReplySetLen(c, 0); /* Not all slots of mynode got covered. See sflushCommand() comment. */
zfree(sflush);
return;
}
if (!inSlotRange) { /* If start another slot range */
sflush->ranges[sflush->numRanges].first = i;
inSlotRange = 1;
}
} else {
if (inSlotRange) { /* If end another slot range */
sflush->ranges[sflush->numRanges++].last = i - 1;
inSlotRange = 0;
/* If reached 'sflush' capacity, double the capacity */
if (sflush->numRanges >= capacity) {
capacity *= 2;
sflush = zrealloc(sflush, sizeof(SlotsFlush) + sizeof(SlotRange) * capacity);
}
}
}
}
/* Update last pair if last cluster slot is also end of last range */
if (inSlotRange) sflush->ranges[sflush->numRanges++].last = CLUSTER_SLOTS - 1;
/* Flush selected slots. If not flush as blocking async, then reply immediately */
if (flushCommandCommon(c, FLUSH_TYPE_SLOTS, flags, sflush) == 0)
replySlotsFlushAndFree(c, sflush);
}
/* The READWRITE command just clears the READONLY command state. */
void readwriteCommand(client *c) {
if (server.cluster_enabled == 0) {
src/cluster.h
+21 -1
View File
@@ -97,7 +97,7 @@ int clusterManualFailoverTimeLimit(void);
void clusterCommandSlots(client * c);
void clusterCommandMyId(client *c);
void clusterCommandMyShardId(client *c);
void clusterCommandShards(client *c);
sds clusterGenNodeDescription(client *c, clusterNode *node, int tls_primary);
int clusterNodeCoversSlot(clusterNode *n, int slot);
@@ -131,6 +131,7 @@ char *clusterNodeHostname(clusterNode *node);
const char *clusterNodePreferredEndpoint(clusterNode *n);
long long clusterNodeReplOffset(clusterNode *node);
clusterNode *clusterLookupNode(const char *name, int length);
const char *clusterGetSecret(size_t *len);
/* functions with shared implementations */
clusterNode *getNodeByQuery(client *c, struct redisCommand *cmd, robj **argv, int argc, int *hashslot, uint64_t cmd_flags, int *error_code);
@@ -142,4 +143,23 @@ int isValidAuxString(char *s, unsigned int length);
void migrateCommand(client *c);
void clusterCommand(client *c);
ConnectionType *connTypeOfCluster(void);
void clusterGenNodesSlotsInfo(int filter);
void clusterFreeNodesSlotsInfo(clusterNode *n);
int clusterNodeSlotInfoCount(clusterNode *n);
uint16_t clusterNodeSlotInfoEntry(clusterNode *n, int idx);
int clusterNodeHasSlotInfo(clusterNode *n);
int clusterGetShardCount(void);
void *clusterGetShardIterator(void);
void *clusterNextShardHandle(void *shard_iterator);
void clusterFreeShardIterator(void *shard_iterator);
int clusterGetShardNodeCount(void *shard);
void *clusterShardHandleGetNodeIterator(void *shard);
clusterNode *clusterShardNodeIteratorNext(void *node_iterator);
void clusterShardNodeIteratorFree(void *node_iterator);
clusterNode *clusterShardNodeFirst(void *shard);
int clusterNodeTcpPort(clusterNode *node);
int clusterNodeTlsPort(clusterNode *node);
#endif /* __CLUSTER_H */
src/cluster_legacy.c
+144 -141
View File
@@ -2,8 +2,13 @@
* Copyright (c) 2009-Present, Redis Ltd.
* All rights reserved.
*
* Copyright (c) 2024-present, Valkey contributors.
* All rights reserved.
*
* Licensed under your choice of the Redis Source Available License 2.0
* (RSALv2) or the Server Side Public License v1 (SSPLv1).
*
* Portions of this file are available under BSD3 terms; see REDISCONTRIBUTIONS for more information.
*/
/*
@@ -88,6 +93,7 @@ int auxTlsPortPresent(clusterNode *n);
static void clusterBuildMessageHdr(clusterMsg *hdr, int type, size_t msglen);
void freeClusterLink(clusterLink *link);
int verifyClusterNodeId(const char *name, int length);
static void updateShardId(clusterNode *node, const char *shard_id);
int getNodeDefaultClientPort(clusterNode *n) {
return server.tls_cluster ? n->tls_port : n->tcp_port;
@@ -198,12 +204,11 @@ int auxShardIdSetter(clusterNode *n, void *value, int length) {
return C_ERR;
}
memcpy(n->shard_id, value, CLUSTER_NAMELEN);
/* if n already has replicas, make sure they all agree
* on the shard id */
/* if n already has replicas, make sure they all use
* the primary shard id */
for (int i = 0; i < n->numslaves; i++) {
if (memcmp(n->slaves[i]->shard_id, n->shard_id, CLUSTER_NAMELEN) != 0) {
return C_ERR;
}
if (memcmp(n->slaves[i]->shard_id, n->shard_id, CLUSTER_NAMELEN) != 0)
updateShardId(n->slaves[i], n->shard_id);
}
clusterAddNodeToShard(value, n);
return C_OK;
@@ -545,18 +550,12 @@ int clusterLoadConfig(char *filename) {
clusterAddNode(master);
}
/* shard_id can be absent if we are loading a nodes.conf generated
* by an older version of Redis; we should follow the primary's
* shard_id in this case */
if (auxFieldHandlers[af_shard_id].isPresent(n) == 0) {
memcpy(n->shard_id, master->shard_id, CLUSTER_NAMELEN);
clusterAddNodeToShard(master->shard_id, n);
} else if (clusterGetNodesInMyShard(master) != NULL &&
memcmp(master->shard_id, n->shard_id, CLUSTER_NAMELEN) != 0)
{
/* If the primary has been added to a shard, make sure this
* node has the same persisted shard id as the primary. */
goto fmterr;
}
* by an older version of Redis;
* ignore replica's shard_id in the file, only use the primary's.
* If replica precedes primary in file, it will be corrected
* later by the auxShardIdSetter */
memcpy(n->shard_id, master->shard_id, CLUSTER_NAMELEN);
clusterAddNodeToShard(master->shard_id, n);
n->slaveof = master;
clusterNodeAddSlave(master,n);
} else if (auxFieldHandlers[af_shard_id].isPresent(n) == 0) {
@@ -634,6 +633,8 @@ int clusterLoadConfig(char *filename) {
}
/* Config sanity check */
if (server.cluster->myself == NULL) goto fmterr;
if (!(myself->flags & (CLUSTER_NODE_MASTER | CLUSTER_NODE_SLAVE))) goto fmterr;
if (nodeIsSlave(myself) && myself->slaveof == NULL) goto fmterr;
zfree(line);
fclose(fp);
@@ -901,22 +902,33 @@ static void updateAnnouncedHumanNodename(clusterNode *node, char *new) {
clusterDoBeforeSleep(CLUSTER_TODO_SAVE_CONFIG);
}
static void assignShardIdToNode(clusterNode *node, const char *shard_id, int flag) {
clusterRemoveNodeFromShard(node);
memcpy(node->shard_id, shard_id, CLUSTER_NAMELEN);
clusterAddNodeToShard(shard_id, node);
clusterDoBeforeSleep(flag);
}
static void updateShardId(clusterNode *node, const char *shard_id) {
if (shard_id && memcmp(node->shard_id, shard_id, CLUSTER_NAMELEN) != 0) {
clusterRemoveNodeFromShard(node);
memcpy(node->shard_id, shard_id, CLUSTER_NAMELEN);
clusterAddNodeToShard(shard_id, node);
clusterDoBeforeSleep(CLUSTER_TODO_SAVE_CONFIG);
}
if (shard_id && myself != node && myself->slaveof == node) {
if (memcmp(myself->shard_id, shard_id, CLUSTER_NAMELEN) != 0) {
/* shard-id can diverge right after a rolling upgrade
* from pre-7.2 releases */
clusterRemoveNodeFromShard(myself);
memcpy(myself->shard_id, shard_id, CLUSTER_NAMELEN);
clusterAddNodeToShard(shard_id, myself);
clusterDoBeforeSleep(CLUSTER_TODO_SAVE_CONFIG|CLUSTER_TODO_FSYNC_CONFIG);
/* We always make our best effort to keep the shard-id consistent
* between the master and its replicas:
*
* 1. When updating the master's shard-id, we simultaneously update the
* shard-id of all its replicas to ensure consistency.
* 2. When updating replica's shard-id, if it differs from its master's shard-id,
* we discard this replica's shard-id and continue using master's shard-id.
* This applies even if the master does not support shard-id, in which
* case we rely on the master's randomly generated shard-id. */
if (node->slaveof == NULL) {
assignShardIdToNode(node, shard_id, CLUSTER_TODO_SAVE_CONFIG);
for (int i = 0; i < clusterNodeNumSlaves(node); i++) {
clusterNode *slavenode = clusterNodeGetSlave(node, i);
if (memcmp(slavenode->shard_id, shard_id, CLUSTER_NAMELEN) != 0)
assignShardIdToNode(slavenode, shard_id, CLUSTER_TODO_SAVE_CONFIG|CLUSTER_TODO_FSYNC_CONFIG);
}
} else if (memcmp(node->slaveof->shard_id, shard_id, CLUSTER_NAMELEN) == 0) {
assignShardIdToNode(node, shard_id, CLUSTER_TODO_SAVE_CONFIG);
}
}
}
@@ -1012,6 +1024,8 @@ void clusterInit(void) {
clusterUpdateMyselfIp();
clusterUpdateMyselfHostname();
clusterUpdateMyselfHumanNodename();
getRandomHexChars(server.cluster->internal_secret, CLUSTER_INTERNALSECRETLEN);
}
void clusterInitLast(void) {
@@ -1244,7 +1258,7 @@ void clusterAcceptHandler(aeEventLoop *el, int fd, void *privdata, int mask) {
return;
}
connection *conn = connCreateAccepted(connTypeOfCluster(), cfd, &require_auth);
connection *conn = connCreateAccepted(server.el, connTypeOfCluster(), cfd, &require_auth);
/* Make sure connection is not in an error state */
if (connGetState(conn) != CONN_STATE_ACCEPTING) {
@@ -1561,6 +1575,14 @@ clusterNode *clusterLookupNode(const char *name, int length) {
return dictGetVal(de);
}
const char *clusterGetSecret(size_t *len) {
if (!server.cluster) {
return NULL;
}
*len = CLUSTER_INTERNALSECRETLEN;
return server.cluster->internal_secret;
}
/* Get all the nodes in my shard.
* Note that the list returned is not computed on the fly
* via slaveof; rather, it is maintained permanently to
@@ -2485,6 +2507,10 @@ uint32_t getShardIdPingExtSize(void) {
return getAlignedPingExtSize(sizeof(clusterMsgPingExtShardId));
}
uint32_t getInternalSecretPingExtSize(void) {
return getAlignedPingExtSize(sizeof(clusterMsgPingExtInternalSecret));
}
uint32_t getForgottenNodeExtSize(void) {
return getAlignedPingExtSize(sizeof(clusterMsgPingExtForgottenNode));
}
@@ -2576,10 +2602,18 @@ uint32_t writePingExt(clusterMsg *hdr, int gossipcount) {
totlen += getShardIdPingExtSize();
extensions++;
/* Populate insternal secret */
if (cursor != NULL) {
clusterMsgPingExtInternalSecret *ext = preparePingExt(cursor, CLUSTERMSG_EXT_TYPE_INTERNALSECRET, getInternalSecretPingExtSize());
memcpy(ext->internal_secret, server.cluster->internal_secret, CLUSTER_INTERNALSECRETLEN);
/* Move the write cursor */
cursor = nextPingExt(cursor);
}
totlen += getInternalSecretPingExtSize();
extensions++;
if (hdr != NULL) {
if (extensions != 0) {
hdr->mflags[0] |= CLUSTERMSG_FLAG0_EXT_DATA;
}
hdr->extensions = htons(extensions);
}
@@ -2619,9 +2653,14 @@ void clusterProcessPingExtensions(clusterMsg *hdr, clusterLink *link) {
} else if (type == CLUSTERMSG_EXT_TYPE_SHARDID) {
clusterMsgPingExtShardId *shardid_ext = (clusterMsgPingExtShardId *) &(ext->ext[0].shard_id);
ext_shardid = shardid_ext->shard_id;
} else if (type == CLUSTERMSG_EXT_TYPE_INTERNALSECRET) {
clusterMsgPingExtInternalSecret *internal_secret_ext = (clusterMsgPingExtInternalSecret *) &(ext->ext[0].internal_secret);
if (memcmp(server.cluster->internal_secret, internal_secret_ext->internal_secret, CLUSTER_INTERNALSECRETLEN) > 0 ) {
memcpy(server.cluster->internal_secret, internal_secret_ext->internal_secret, CLUSTER_INTERNALSECRETLEN);
}
} else {
/* Unknown type, we will ignore it but log what happened. */
serverLog(LL_WARNING, "Received unknown extension type %d", type);
serverLog(LL_VERBOSE, "Received unknown extension type %d", type);
}
/* We know this will be valid since we validated it ahead of time */
@@ -2769,6 +2808,9 @@ int clusterProcessPacket(clusterLink *link) {
}
sender = getNodeFromLinkAndMsg(link, hdr);
if (sender && (hdr->mflags[0] & CLUSTERMSG_FLAG0_EXT_DATA)) {
sender->flags |= CLUSTER_NODE_EXTENSIONS_SUPPORTED;
}
/* Update the last time we saw any data from this node. We
* use this in order to avoid detecting a timeout from a node that
@@ -3534,6 +3576,8 @@ static void clusterBuildMessageHdr(clusterMsg *hdr, int type, size_t msglen) {
/* Set the message flags. */
if (clusterNodeIsMaster(myself) && server.cluster->mf_end)
hdr->mflags[0] |= CLUSTERMSG_FLAG0_PAUSED;
hdr->mflags[0] |= CLUSTERMSG_FLAG0_EXT_DATA; /* Always make other nodes know that
* this node supports extension data. */
hdr->totlen = htonl(msglen);
}
@@ -3612,7 +3656,9 @@ void clusterSendPing(clusterLink *link, int type) {
* to put inside the packet. */
estlen = sizeof(clusterMsg) - sizeof(union clusterMsgData);
estlen += (sizeof(clusterMsgDataGossip)*(wanted + pfail_wanted));
estlen += writePingExt(NULL, 0);
if (link->node && nodeSupportsExtensions(link->node)) {
estlen += writePingExt(NULL, 0);
}
/* Note: clusterBuildMessageHdr() expects the buffer to be always at least
* sizeof(clusterMsg) or more. */
if (estlen < (int)sizeof(clusterMsg)) estlen = sizeof(clusterMsg);
@@ -3682,7 +3728,9 @@ void clusterSendPing(clusterLink *link, int type) {
/* Compute the actual total length and send! */
uint32_t totlen = 0;
totlen += writePingExt(hdr, gossipcount);
if (link->node && nodeSupportsExtensions(link->node)) {
totlen += writePingExt(hdr, gossipcount);
}
totlen += sizeof(clusterMsg)-sizeof(union clusterMsgData);
totlen += (sizeof(clusterMsgDataGossip)*gossipcount);
serverAssert(gossipcount < USHRT_MAX);
@@ -4559,7 +4607,7 @@ static int clusterNodeCronHandleReconnect(clusterNode *node, mstime_t handshake_
if (node->link == NULL) {
clusterLink *link = createClusterLink(node);
link->conn = connCreate(connTypeOfCluster());
link->conn = connCreate(server.el, connTypeOfCluster());
connSetPrivateData(link->conn, link);
if (connConnect(link->conn, node->ip, node->cport, server.bind_source_addr,
clusterLinkConnectHandler) == C_ERR) {
@@ -5582,113 +5630,68 @@ void clusterUpdateSlots(client *c, unsigned char *slots, int del) {
}
}
/* Add detailed information of a node to the output buffer of the given client. */
void addNodeDetailsToShardReply(client *c, clusterNode *node) {
int reply_count = 0;
void *node_replylen = addReplyDeferredLen(c);
addReplyBulkCString(c, "id");
addReplyBulkCBuffer(c, node->name, CLUSTER_NAMELEN);
reply_count++;
if (node->tcp_port) {
addReplyBulkCString(c, "port");
addReplyLongLong(c, node->tcp_port);
reply_count++;
}
if (node->tls_port) {
addReplyBulkCString(c, "tls-port");
addReplyLongLong(c, node->tls_port);
reply_count++;
}
addReplyBulkCString(c, "ip");
addReplyBulkCString(c, node->ip);
reply_count++;
addReplyBulkCString(c, "endpoint");
addReplyBulkCString(c, clusterNodePreferredEndpoint(node));
reply_count++;
if (sdslen(node->hostname) != 0) {
addReplyBulkCString(c, "hostname");
addReplyBulkCBuffer(c, node->hostname, sdslen(node->hostname));
reply_count++;
}
long long node_offset;
if (node->flags & CLUSTER_NODE_MYSELF) {
node_offset = nodeIsSlave(node) ? replicationGetSlaveOffset() : server.master_repl_offset;
} else {
node_offset = node->repl_offset;
}
addReplyBulkCString(c, "role");
addReplyBulkCString(c, nodeIsSlave(node) ? "replica" : "master");
reply_count++;
addReplyBulkCString(c, "replication-offset");
addReplyLongLong(c, node_offset);
reply_count++;
addReplyBulkCString(c, "health");
const char *health_msg = NULL;
if (nodeFailed(node)) {
health_msg = "fail";
} else if (nodeIsSlave(node) && node_offset == 0) {
health_msg = "loading";
} else {
health_msg = "online";
}
addReplyBulkCString(c, health_msg);
reply_count++;
setDeferredMapLen(c, node_replylen, reply_count);
int clusterGetShardCount(void) {
return dictSize(server.cluster->shards);
}
/* Add the shard reply of a single shard based off the given primary node. */
void addShardReplyForClusterShards(client *c, list *nodes) {
serverAssert(listLength(nodes) > 0);
clusterNode *n = listNodeValue(listFirst(nodes));
addReplyMapLen(c, 2);
addReplyBulkCString(c, "slots");
/* Use slot_info_pairs from the primary only */
n = clusterNodeGetMaster(n);
if (n->slot_info_pairs != NULL) {
serverAssert((n->slot_info_pairs_count % 2) == 0);
addReplyArrayLen(c, n->slot_info_pairs_count);
for (int i = 0; i < n->slot_info_pairs_count; i++)
addReplyLongLong(c, (unsigned long)n->slot_info_pairs[i]);
} else {
/* If no slot info pair is provided, the node owns no slots */
addReplyArrayLen(c, 0);
}
addReplyBulkCString(c, "nodes");
addReplyArrayLen(c, listLength(nodes));
listIter li;
listRewind(nodes, &li);
for (listNode *ln = listNext(&li); ln != NULL; ln = listNext(&li)) {
clusterNode *n = listNodeValue(ln);
addNodeDetailsToShardReply(c, n);
clusterFreeNodesSlotsInfo(n);
}
void *clusterGetShardIterator(void) {
return dictGetSafeIterator(server.cluster->shards);
}
/* Add to the output buffer of the given client, an array of slot (start, end)
* pair owned by the shard, also the primary and set of replica(s) along with
* information about each node. */
void clusterCommandShards(client *c) {
addReplyArrayLen(c, dictSize(server.cluster->shards));
/* This call will add slot_info_pairs to all nodes */
clusterGenNodesSlotsInfo(0);
dictIterator *di = dictGetSafeIterator(server.cluster->shards);
for(dictEntry *de = dictNext(di); de != NULL; de = dictNext(di)) {
addShardReplyForClusterShards(c, dictGetVal(de));
}
dictReleaseIterator(di);
void *clusterNextShardHandle(void *shard_iterator) {
dictEntry *de = dictNext(shard_iterator);
if(de == NULL) return NULL;
return dictGetVal(de);
}
void clusterFreeShardIterator(void *shard_iterator) {
dictReleaseIterator(shard_iterator);
}
int clusterNodeHasSlotInfo(clusterNode *n) {
return n->slot_info_pairs != NULL;
}
int clusterNodeSlotInfoCount(clusterNode *n) {
return n->slot_info_pairs_count;
}
uint16_t clusterNodeSlotInfoEntry(clusterNode *n, int idx) {
return n->slot_info_pairs[idx];
}
int clusterGetShardNodeCount(void *shard) {
return listLength((list*)shard);
}
void *clusterShardHandleGetNodeIterator(void *shard) {
listIter *li = zmalloc(sizeof(listIter));
listRewind((list*)shard, li);
return li;
}
void clusterShardNodeIteratorFree(void *node_iterator) {
zfree(node_iterator);
}
clusterNode *clusterShardNodeIteratorNext(void *node_iterator) {
listNode *item = listNext((listIter*)node_iterator);
if (item == NULL) return NULL;
return listNodeValue(item);
}
clusterNode *clusterShardNodeFirst(void *shard) {
listNode *item = listFirst((list*)shard);
if (item == NULL) return NULL;
return listNodeValue(item);
}
int clusterNodeTcpPort(clusterNode *node) {
return node->tcp_port;
}
int clusterNodeTlsPort(clusterNode *node) {
return node->tls_port;
}
sds genClusterInfoString(void) {
src/cluster_legacy.h
+23
View File
@@ -1,3 +1,16 @@
/*
* Copyright (c) 2009-Present, Redis Ltd.
* All rights reserved.
*
* Copyright (c) 2024-present, Valkey contributors.
* All rights reserved.
*
* Licensed under your choice of the Redis Source Available License 2.0
* (RSALv2) or the Server Side Public License v1 (SSPLv1).
*
* Portions of this file are available under BSD3 terms; see REDISCONTRIBUTIONS for more information.
*/
#ifndef CLUSTER_LEGACY_H
#define CLUSTER_LEGACY_H
@@ -51,6 +64,7 @@ typedef struct clusterLink {
#define CLUSTER_NODE_MEET 128 /* Send a MEET message to this node */
#define CLUSTER_NODE_MIGRATE_TO 256 /* Master eligible for replica migration. */
#define CLUSTER_NODE_NOFAILOVER 512 /* Slave will not try to failover. */
#define CLUSTER_NODE_EXTENSIONS_SUPPORTED 1024 /* This node supports extensions. */
#define CLUSTER_NODE_NULL_NAME "\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000"
#define nodeIsSlave(n) ((n)->flags & CLUSTER_NODE_SLAVE)
@@ -59,6 +73,7 @@ typedef struct clusterLink {
#define nodeTimedOut(n) ((n)->flags & CLUSTER_NODE_PFAIL)
#define nodeFailed(n) ((n)->flags & CLUSTER_NODE_FAIL)
#define nodeCantFailover(n) ((n)->flags & CLUSTER_NODE_NOFAILOVER)
#define nodeSupportsExtensions(n) ((n)->flags & CLUSTER_NODE_EXTENSIONS_SUPPORTED)
/* This structure represent elements of node->fail_reports. */
typedef struct clusterNodeFailReport {
@@ -133,10 +148,12 @@ typedef enum {
CLUSTERMSG_EXT_TYPE_HUMAN_NODENAME,
CLUSTERMSG_EXT_TYPE_FORGOTTEN_NODE,
CLUSTERMSG_EXT_TYPE_SHARDID,
CLUSTERMSG_EXT_TYPE_INTERNALSECRET,
} clusterMsgPingtypes;
/* Helper function for making sure extensions are eight byte aligned. */
#define EIGHT_BYTE_ALIGN(size) ((((size) + 7) / 8) * 8)
#define CLUSTER_INTERNALSECRETLEN 40 /* sha1 hex length */
typedef struct {
char hostname[1]; /* The announced hostname, ends with \0. */
@@ -157,6 +174,10 @@ typedef struct {
char shard_id[CLUSTER_NAMELEN]; /* The shard_id, 40 bytes fixed. */
} clusterMsgPingExtShardId;
typedef struct {
char internal_secret[CLUSTER_INTERNALSECRETLEN]; /* Current shard internal secret */
} clusterMsgPingExtInternalSecret;
typedef struct {
uint32_t length; /* Total length of this extension message (including this header) */
uint16_t type; /* Type of this extension message (see clusterMsgPingExtTypes) */
@@ -166,6 +187,7 @@ typedef struct {
clusterMsgPingExtHumanNodename human_nodename;
clusterMsgPingExtForgottenNode forgotten_node;
clusterMsgPingExtShardId shard_id;
clusterMsgPingExtInternalSecret internal_secret;
} ext[]; /* Actual extension information, formatted so that the data is 8
* byte aligned, regardless of its content. */
} clusterMsgPingExt;
@@ -318,6 +340,7 @@ struct clusterState {
clusterNode *migrating_slots_to[CLUSTER_SLOTS];
clusterNode *importing_slots_from[CLUSTER_SLOTS];
clusterNode *slots[CLUSTER_SLOTS];
char internal_secret[CLUSTER_INTERNALSECRETLEN];
/* The following fields are used to take the slave state on elections. */
mstime_t failover_auth_time; /* Time of previous or next election. */
int failover_auth_count; /* Number of votes received so far. */
src/commands.def
+146 -10
View File
@@ -1239,6 +1239,9 @@ commandHistory CLIENT_LIST_History[] = {
{"6.2.0","Added `argv-mem`, `tot-mem`, `laddr` and `redir` fields and the optional `ID` filter."},
{"7.0.0","Added `resp`, `multi-mem`, `rbs` and `rbp` fields."},
{"7.0.3","Added `ssub` field."},
{"7.2.0","Added `lib-name` and `lib-ver` fields."},
{"7.4.0","Added `watch` field."},
{"8.0.0","Added `io-thread` field."},
};
#endif
@@ -1546,7 +1549,7 @@ struct COMMAND_STRUCT CLIENT_Subcommands[] = {
{MAKE_CMD("id","Returns the unique client ID of the connection.","O(1)","5.0.0",CMD_DOC_NONE,NULL,NULL,"connection",COMMAND_GROUP_CONNECTION,CLIENT_ID_History,0,CLIENT_ID_Tips,0,clientCommand,2,CMD_NOSCRIPT|CMD_LOADING|CMD_STALE|CMD_SENTINEL,ACL_CATEGORY_CONNECTION,CLIENT_ID_Keyspecs,0,NULL,0)},
{MAKE_CMD("info","Returns information about the connection.","O(1)","6.2.0",CMD_DOC_NONE,NULL,NULL,"connection",COMMAND_GROUP_CONNECTION,CLIENT_INFO_History,0,CLIENT_INFO_Tips,1,clientCommand,2,CMD_NOSCRIPT|CMD_LOADING|CMD_STALE|CMD_SENTINEL,ACL_CATEGORY_CONNECTION,CLIENT_INFO_Keyspecs,0,NULL,0)},
{MAKE_CMD("kill","Terminates open connections.","O(N) where N is the number of client connections","2.4.0",CMD_DOC_NONE,NULL,NULL,"connection",COMMAND_GROUP_CONNECTION,CLIENT_KILL_History,6,CLIENT_KILL_Tips,0,clientCommand,-3,CMD_ADMIN|CMD_NOSCRIPT|CMD_LOADING|CMD_STALE|CMD_SENTINEL,ACL_CATEGORY_CONNECTION,CLIENT_KILL_Keyspecs,0,NULL,1),.args=CLIENT_KILL_Args},
{MAKE_CMD("list","Lists open connections.","O(N) where N is the number of client connections","2.4.0",CMD_DOC_NONE,NULL,NULL,"connection",COMMAND_GROUP_CONNECTION,CLIENT_LIST_History,6,CLIENT_LIST_Tips,1,clientCommand,-2,CMD_ADMIN|CMD_NOSCRIPT|CMD_LOADING|CMD_STALE|CMD_SENTINEL,ACL_CATEGORY_CONNECTION,CLIENT_LIST_Keyspecs,0,NULL,2),.args=CLIENT_LIST_Args},
{MAKE_CMD("list","Lists open connections.","O(N) where N is the number of client connections","2.4.0",CMD_DOC_NONE,NULL,NULL,"connection",COMMAND_GROUP_CONNECTION,CLIENT_LIST_History,9,CLIENT_LIST_Tips,1,clientCommand,-2,CMD_ADMIN|CMD_NOSCRIPT|CMD_LOADING|CMD_STALE|CMD_SENTINEL,ACL_CATEGORY_CONNECTION,CLIENT_LIST_Keyspecs,0,NULL,2),.args=CLIENT_LIST_Args},
{MAKE_CMD("no-evict","Sets the client eviction mode of the connection.","O(1)","7.0.0",CMD_DOC_NONE,NULL,NULL,"connection",COMMAND_GROUP_CONNECTION,CLIENT_NO_EVICT_History,0,CLIENT_NO_EVICT_Tips,0,clientCommand,3,CMD_ADMIN|CMD_NOSCRIPT|CMD_LOADING|CMD_STALE|CMD_SENTINEL,ACL_CATEGORY_CONNECTION,CLIENT_NO_EVICT_Keyspecs,0,NULL,1),.args=CLIENT_NO_EVICT_Args},
{MAKE_CMD("no-touch","Controls whether commands sent by the client affect the LRU/LFU of accessed keys.","O(1)","7.2.0",CMD_DOC_NONE,NULL,NULL,"connection",COMMAND_GROUP_CONNECTION,CLIENT_NO_TOUCH_History,0,CLIENT_NO_TOUCH_Tips,0,clientCommand,3,CMD_NOSCRIPT|CMD_LOADING|CMD_STALE,ACL_CATEGORY_CONNECTION,CLIENT_NO_TOUCH_Keyspecs,0,NULL,1),.args=CLIENT_NO_TOUCH_Args},
{MAKE_CMD("pause","Suspends commands processing.","O(1)","3.0.0",CMD_DOC_NONE,NULL,NULL,"connection",COMMAND_GROUP_CONNECTION,CLIENT_PAUSE_History,1,CLIENT_PAUSE_Tips,0,clientCommand,-3,CMD_ADMIN|CMD_NOSCRIPT|CMD_LOADING|CMD_STALE|CMD_SENTINEL,ACL_CATEGORY_CONNECTION,CLIENT_PAUSE_Keyspecs,0,NULL,2),.args=CLIENT_PAUSE_Args},
@@ -3467,6 +3470,78 @@ struct COMMAND_ARG HGETALL_Args[] = {
{MAKE_ARG("key",ARG_TYPE_KEY,0,NULL,NULL,NULL,CMD_ARG_NONE,0,NULL)},
};
/********** HGETDEL ********************/
#ifndef SKIP_CMD_HISTORY_TABLE
/* HGETDEL history */
#define HGETDEL_History NULL
#endif
#ifndef SKIP_CMD_TIPS_TABLE
/* HGETDEL tips */
#define HGETDEL_Tips NULL
#endif
#ifndef SKIP_CMD_KEY_SPECS_TABLE
/* HGETDEL key specs */
keySpec HGETDEL_Keyspecs[1] = {
{NULL,CMD_KEY_RW|CMD_KEY_ACCESS|CMD_KEY_DELETE,KSPEC_BS_INDEX,.bs.index={1},KSPEC_FK_RANGE,.fk.range={0,1,0}}
};
#endif
/* HGETDEL fields argument table */
struct COMMAND_ARG HGETDEL_fields_Subargs[] = {
{MAKE_ARG("numfields",ARG_TYPE_INTEGER,-1,NULL,NULL,NULL,CMD_ARG_NONE,0,NULL)},
{MAKE_ARG("field",ARG_TYPE_STRING,-1,NULL,NULL,NULL,CMD_ARG_MULTIPLE,0,NULL)},
};
/* HGETDEL argument table */
struct COMMAND_ARG HGETDEL_Args[] = {
{MAKE_ARG("key",ARG_TYPE_KEY,0,NULL,NULL,NULL,CMD_ARG_NONE,0,NULL)},
{MAKE_ARG("fields",ARG_TYPE_BLOCK,-1,"FIELDS",NULL,NULL,CMD_ARG_NONE,2,NULL),.subargs=HGETDEL_fields_Subargs},
};
/********** HGETEX ********************/
#ifndef SKIP_CMD_HISTORY_TABLE
/* HGETEX history */
#define HGETEX_History NULL
#endif
#ifndef SKIP_CMD_TIPS_TABLE
/* HGETEX tips */
#define HGETEX_Tips NULL
#endif
#ifndef SKIP_CMD_KEY_SPECS_TABLE
/* HGETEX key specs */
keySpec HGETEX_Keyspecs[1] = {
{"RW and UPDATE because it changes the TTL",CMD_KEY_RW|CMD_KEY_ACCESS|CMD_KEY_UPDATE,KSPEC_BS_INDEX,.bs.index={1},KSPEC_FK_RANGE,.fk.range={0,1,0}}
};
#endif
/* HGETEX expiration argument table */
struct COMMAND_ARG HGETEX_expiration_Subargs[] = {
{MAKE_ARG("seconds",ARG_TYPE_INTEGER,-1,"EX",NULL,NULL,CMD_ARG_NONE,0,NULL)},
{MAKE_ARG("milliseconds",ARG_TYPE_INTEGER,-1,"PX",NULL,NULL,CMD_ARG_NONE,0,NULL)},
{MAKE_ARG("unix-time-seconds",ARG_TYPE_UNIX_TIME,-1,"EXAT",NULL,NULL,CMD_ARG_NONE,0,NULL)},
{MAKE_ARG("unix-time-milliseconds",ARG_TYPE_UNIX_TIME,-1,"PXAT",NULL,NULL,CMD_ARG_NONE,0,NULL)},
{MAKE_ARG("persist",ARG_TYPE_PURE_TOKEN,-1,"PERSIST",NULL,NULL,CMD_ARG_NONE,0,NULL)},
};
/* HGETEX fields argument table */
struct COMMAND_ARG HGETEX_fields_Subargs[] = {
{MAKE_ARG("numfields",ARG_TYPE_INTEGER,-1,NULL,NULL,NULL,CMD_ARG_NONE,0,NULL)},
{MAKE_ARG("field",ARG_TYPE_STRING,-1,NULL,NULL,NULL,CMD_ARG_MULTIPLE,0,NULL)},
};
/* HGETEX argument table */
struct COMMAND_ARG HGETEX_Args[] = {
{MAKE_ARG("key",ARG_TYPE_KEY,0,NULL,NULL,NULL,CMD_ARG_NONE,0,NULL)},
{MAKE_ARG("expiration",ARG_TYPE_ONEOF,-1,NULL,NULL,NULL,CMD_ARG_OPTIONAL,5,NULL),.subargs=HGETEX_expiration_Subargs},
{MAKE_ARG("fields",ARG_TYPE_BLOCK,-1,"FIELDS",NULL,NULL,CMD_ARG_NONE,2,NULL),.subargs=HGETEX_fields_Subargs},
};
/********** HINCRBY ********************/
#ifndef SKIP_CMD_HISTORY_TABLE
@@ -3778,7 +3853,9 @@ struct COMMAND_ARG HPEXPIRETIME_Args[] = {
#ifndef SKIP_CMD_TIPS_TABLE
/* HPTTL tips */
#define HPTTL_Tips NULL
const char *HPTTL_Tips[] = {
"nondeterministic_output",
};
#endif
#ifndef SKIP_CMD_KEY_SPECS_TABLE
@@ -3896,6 +3973,60 @@ struct COMMAND_ARG HSET_Args[] = {
{MAKE_ARG("data",ARG_TYPE_BLOCK,-1,NULL,NULL,NULL,CMD_ARG_MULTIPLE,2,NULL),.subargs=HSET_data_Subargs},
};
/********** HSETEX ********************/
#ifndef SKIP_CMD_HISTORY_TABLE
/* HSETEX history */
#define HSETEX_History NULL
#endif
#ifndef SKIP_CMD_TIPS_TABLE
/* HSETEX tips */
#define HSETEX_Tips NULL
#endif
#ifndef SKIP_CMD_KEY_SPECS_TABLE
/* HSETEX key specs */
keySpec HSETEX_Keyspecs[1] = {
{NULL,CMD_KEY_RW|CMD_KEY_UPDATE,KSPEC_BS_INDEX,.bs.index={1},KSPEC_FK_RANGE,.fk.range={0,1,0}}
};
#endif
/* HSETEX condition argument table */
struct COMMAND_ARG HSETEX_condition_Subargs[] = {
{MAKE_ARG("fnx",ARG_TYPE_PURE_TOKEN,-1,"FNX",NULL,NULL,CMD_ARG_NONE,0,NULL)},
{MAKE_ARG("fxx",ARG_TYPE_PURE_TOKEN,-1,"FXX",NULL,NULL,CMD_ARG_NONE,0,NULL)},
};
/* HSETEX expiration argument table */
struct COMMAND_ARG HSETEX_expiration_Subargs[] = {
{MAKE_ARG("seconds",ARG_TYPE_INTEGER,-1,"EX",NULL,NULL,CMD_ARG_NONE,0,NULL)},
{MAKE_ARG("milliseconds",ARG_TYPE_INTEGER,-1,"PX",NULL,NULL,CMD_ARG_NONE,0,NULL)},
{MAKE_ARG("unix-time-seconds",ARG_TYPE_UNIX_TIME,-1,"EXAT",NULL,NULL,CMD_ARG_NONE,0,NULL)},
{MAKE_ARG("unix-time-milliseconds",ARG_TYPE_UNIX_TIME,-1,"PXAT",NULL,NULL,CMD_ARG_NONE,0,NULL)},
{MAKE_ARG("keepttl",ARG_TYPE_PURE_TOKEN,-1,"KEEPTTL",NULL,NULL,CMD_ARG_NONE,0,NULL)},
};
/* HSETEX fields data argument table */
struct COMMAND_ARG HSETEX_fields_data_Subargs[] = {
{MAKE_ARG("field",ARG_TYPE_STRING,-1,NULL,NULL,NULL,CMD_ARG_NONE,0,NULL)},
{MAKE_ARG("value",ARG_TYPE_STRING,-1,NULL,NULL,NULL,CMD_ARG_NONE,0,NULL)},
};
/* HSETEX fields argument table */
struct COMMAND_ARG HSETEX_fields_Subargs[] = {
{MAKE_ARG("numfields",ARG_TYPE_INTEGER,-1,NULL,NULL,NULL,CMD_ARG_NONE,0,NULL)},
{MAKE_ARG("data",ARG_TYPE_BLOCK,-1,NULL,NULL,NULL,CMD_ARG_MULTIPLE,2,NULL),.subargs=HSETEX_fields_data_Subargs},
};
/* HSETEX argument table */
struct COMMAND_ARG HSETEX_Args[] = {
{MAKE_ARG("key",ARG_TYPE_KEY,0,NULL,NULL,NULL,CMD_ARG_NONE,0,NULL)},
{MAKE_ARG("condition",ARG_TYPE_ONEOF,-1,NULL,NULL,NULL,CMD_ARG_OPTIONAL,2,NULL),.subargs=HSETEX_condition_Subargs},
{MAKE_ARG("expiration",ARG_TYPE_ONEOF,-1,NULL,NULL,NULL,CMD_ARG_OPTIONAL,5,NULL),.subargs=HSETEX_expiration_Subargs},
{MAKE_ARG("fields",ARG_TYPE_BLOCK,-1,"FIELDS",NULL,NULL,CMD_ARG_NONE,2,NULL),.subargs=HSETEX_fields_Subargs},
};
/********** HSETNX ********************/
#ifndef SKIP_CMD_HISTORY_TABLE
@@ -3956,7 +4087,9 @@ struct COMMAND_ARG HSTRLEN_Args[] = {
#ifndef SKIP_CMD_TIPS_TABLE
/* HTTL tips */
#define HTTL_Tips NULL
const char *HTTL_Tips[] = {
"nondeterministic_output",
};
#endif
#ifndef SKIP_CMD_KEY_SPECS_TABLE
@@ -11029,11 +11162,13 @@ struct COMMAND_STRUCT redisCommandTable[] = {
/* hash */
{MAKE_CMD("hdel","Deletes one or more fields and their values from a hash. Deletes the hash if no fields remain.","O(N) where N is the number of fields to be removed.","2.0.0",CMD_DOC_NONE,NULL,NULL,"hash",COMMAND_GROUP_HASH,HDEL_History,1,HDEL_Tips,0,hdelCommand,-3,CMD_WRITE|CMD_FAST,ACL_CATEGORY_HASH,HDEL_Keyspecs,1,NULL,2),.args=HDEL_Args},
{MAKE_CMD("hexists","Determines whether a field exists in a hash.","O(1)","2.0.0",CMD_DOC_NONE,NULL,NULL,"hash",COMMAND_GROUP_HASH,HEXISTS_History,0,HEXISTS_Tips,0,hexistsCommand,3,CMD_READONLY|CMD_FAST,ACL_CATEGORY_HASH,HEXISTS_Keyspecs,1,NULL,2),.args=HEXISTS_Args},
{MAKE_CMD("hexpire","Set expiry for hash field using relative time to expire (seconds)","O(N) where N is the number of specified fields","7.4.0",CMD_DOC_NONE,NULL,NULL,"hash",COMMAND_GROUP_HASH,HEXPIRE_History,0,HEXPIRE_Tips,0,hexpireCommand,-6,CMD_WRITE|CMD_DENYOOM|CMD_FAST,ACL_CATEGORY_HASH,HEXPIRE_Keyspecs,1,NULL,4),.args=HEXPIRE_Args},
{MAKE_CMD("hexpireat","Set expiry for hash field using an absolute Unix timestamp (seconds)","O(N) where N is the number of specified fields","7.4.0",CMD_DOC_NONE,NULL,NULL,"hash",COMMAND_GROUP_HASH,HEXPIREAT_History,0,HEXPIREAT_Tips,0,hexpireatCommand,-6,CMD_WRITE|CMD_DENYOOM|CMD_FAST,ACL_CATEGORY_HASH,HEXPIREAT_Keyspecs,1,NULL,4),.args=HEXPIREAT_Args},
{MAKE_CMD("hexpire","Set expiry for hash field using relative time to expire (seconds)","O(N) where N is the number of specified fields","7.4.0",CMD_DOC_NONE,NULL,NULL,"hash",COMMAND_GROUP_HASH,HEXPIRE_History,0,HEXPIRE_Tips,0,hexpireCommand,-6,CMD_WRITE|CMD_FAST,ACL_CATEGORY_HASH,HEXPIRE_Keyspecs,1,NULL,4),.args=HEXPIRE_Args},
{MAKE_CMD("hexpireat","Set expiry for hash field using an absolute Unix timestamp (seconds)","O(N) where N is the number of specified fields","7.4.0",CMD_DOC_NONE,NULL,NULL,"hash",COMMAND_GROUP_HASH,HEXPIREAT_History,0,HEXPIREAT_Tips,0,hexpireatCommand,-6,CMD_WRITE|CMD_FAST,ACL_CATEGORY_HASH,HEXPIREAT_Keyspecs,1,NULL,4),.args=HEXPIREAT_Args},
{MAKE_CMD("hexpiretime","Returns the expiration time of a hash field as a Unix timestamp, in seconds.","O(N) where N is the number of specified fields","7.4.0",CMD_DOC_NONE,NULL,NULL,"hash",COMMAND_GROUP_HASH,HEXPIRETIME_History,0,HEXPIRETIME_Tips,0,hexpiretimeCommand,-5,CMD_READONLY|CMD_FAST,ACL_CATEGORY_HASH,HEXPIRETIME_Keyspecs,1,NULL,2),.args=HEXPIRETIME_Args},
{MAKE_CMD("hget","Returns the value of a field in a hash.","O(1)","2.0.0",CMD_DOC_NONE,NULL,NULL,"hash",COMMAND_GROUP_HASH,HGET_History,0,HGET_Tips,0,hgetCommand,3,CMD_READONLY|CMD_FAST,ACL_CATEGORY_HASH,HGET_Keyspecs,1,NULL,2),.args=HGET_Args},
{MAKE_CMD("hgetall","Returns all fields and values in a hash.","O(N) where N is the size of the hash.","2.0.0",CMD_DOC_NONE,NULL,NULL,"hash",COMMAND_GROUP_HASH,HGETALL_History,0,HGETALL_Tips,1,hgetallCommand,2,CMD_READONLY,ACL_CATEGORY_HASH,HGETALL_Keyspecs,1,NULL,1),.args=HGETALL_Args},
{MAKE_CMD("hgetdel","Returns the value of a field and deletes it from the hash.","O(N) where N is the number of specified fields","8.0.0",CMD_DOC_NONE,NULL,NULL,"hash",COMMAND_GROUP_HASH,HGETDEL_History,0,HGETDEL_Tips,0,hgetdelCommand,-5,CMD_WRITE|CMD_FAST,ACL_CATEGORY_HASH,HGETDEL_Keyspecs,1,NULL,2),.args=HGETDEL_Args},
{MAKE_CMD("hgetex","Get the value of one or more fields of a given hash key, and optionally set their expiration.","O(N) where N is the number of specified fields","8.0.0",CMD_DOC_NONE,NULL,NULL,"hash",COMMAND_GROUP_HASH,HGETEX_History,0,HGETEX_Tips,0,hgetexCommand,-5,CMD_WRITE|CMD_FAST,ACL_CATEGORY_HASH,HGETEX_Keyspecs,1,NULL,3),.args=HGETEX_Args},
{MAKE_CMD("hincrby","Increments the integer value of a field in a hash by a number. Uses 0 as initial value if the field doesn't exist.","O(1)","2.0.0",CMD_DOC_NONE,NULL,NULL,"hash",COMMAND_GROUP_HASH,HINCRBY_History,0,HINCRBY_Tips,0,hincrbyCommand,4,CMD_WRITE|CMD_DENYOOM|CMD_FAST,ACL_CATEGORY_HASH,HINCRBY_Keyspecs,1,NULL,3),.args=HINCRBY_Args},
{MAKE_CMD("hincrbyfloat","Increments the floating point value of a field by a number. Uses 0 as initial value if the field doesn't exist.","O(1)","2.6.0",CMD_DOC_NONE,NULL,NULL,"hash",COMMAND_GROUP_HASH,HINCRBYFLOAT_History,0,HINCRBYFLOAT_Tips,0,hincrbyfloatCommand,4,CMD_WRITE|CMD_DENYOOM|CMD_FAST,ACL_CATEGORY_HASH,HINCRBYFLOAT_Keyspecs,1,NULL,3),.args=HINCRBYFLOAT_Args},
{MAKE_CMD("hkeys","Returns all fields in a hash.","O(N) where N is the size of the hash.","2.0.0",CMD_DOC_NONE,NULL,NULL,"hash",COMMAND_GROUP_HASH,HKEYS_History,0,HKEYS_Tips,1,hkeysCommand,2,CMD_READONLY,ACL_CATEGORY_HASH,HKEYS_Keyspecs,1,NULL,1),.args=HKEYS_Args},
@@ -11041,16 +11176,17 @@ struct COMMAND_STRUCT redisCommandTable[] = {
{MAKE_CMD("hmget","Returns the values of all fields in a hash.","O(N) where N is the number of fields being requested.","2.0.0",CMD_DOC_NONE,NULL,NULL,"hash",COMMAND_GROUP_HASH,HMGET_History,0,HMGET_Tips,0,hmgetCommand,-3,CMD_READONLY|CMD_FAST,ACL_CATEGORY_HASH,HMGET_Keyspecs,1,NULL,2),.args=HMGET_Args},
{MAKE_CMD("hmset","Sets the values of multiple fields.","O(N) where N is the number of fields being set.","2.0.0",CMD_DOC_DEPRECATED,"`HSET` with multiple field-value pairs","4.0.0","hash",COMMAND_GROUP_HASH,HMSET_History,0,HMSET_Tips,0,hsetCommand,-4,CMD_WRITE|CMD_DENYOOM|CMD_FAST,ACL_CATEGORY_HASH,HMSET_Keyspecs,1,NULL,2),.args=HMSET_Args},
{MAKE_CMD("hpersist","Removes the expiration time for each specified field","O(N) where N is the number of specified fields","7.4.0",CMD_DOC_NONE,NULL,NULL,"hash",COMMAND_GROUP_HASH,HPERSIST_History,0,HPERSIST_Tips,0,hpersistCommand,-5,CMD_WRITE|CMD_FAST,ACL_CATEGORY_HASH,HPERSIST_Keyspecs,1,NULL,2),.args=HPERSIST_Args},
{MAKE_CMD("hpexpire","Set expiry for hash field using relative time to expire (milliseconds)","O(N) where N is the number of specified fields","7.4.0",CMD_DOC_NONE,NULL,NULL,"hash",COMMAND_GROUP_HASH,HPEXPIRE_History,0,HPEXPIRE_Tips,0,hpexpireCommand,-6,CMD_WRITE|CMD_DENYOOM|CMD_FAST,ACL_CATEGORY_HASH,HPEXPIRE_Keyspecs,1,NULL,4),.args=HPEXPIRE_Args},
{MAKE_CMD("hpexpireat","Set expiry for hash field using an absolute Unix timestamp (milliseconds)","O(N) where N is the number of specified fields","7.4.0",CMD_DOC_NONE,NULL,NULL,"hash",COMMAND_GROUP_HASH,HPEXPIREAT_History,0,HPEXPIREAT_Tips,0,hpexpireatCommand,-6,CMD_WRITE|CMD_DENYOOM|CMD_FAST,ACL_CATEGORY_HASH,HPEXPIREAT_Keyspecs,1,NULL,4),.args=HPEXPIREAT_Args},
{MAKE_CMD("hpexpire","Set expiry for hash field using relative time to expire (milliseconds)","O(N) where N is the number of specified fields","7.4.0",CMD_DOC_NONE,NULL,NULL,"hash",COMMAND_GROUP_HASH,HPEXPIRE_History,0,HPEXPIRE_Tips,0,hpexpireCommand,-6,CMD_WRITE|CMD_FAST,ACL_CATEGORY_HASH,HPEXPIRE_Keyspecs,1,NULL,4),.args=HPEXPIRE_Args},
{MAKE_CMD("hpexpireat","Set expiry for hash field using an absolute Unix timestamp (milliseconds)","O(N) where N is the number of specified fields","7.4.0",CMD_DOC_NONE,NULL,NULL,"hash",COMMAND_GROUP_HASH,HPEXPIREAT_History,0,HPEXPIREAT_Tips,0,hpexpireatCommand,-6,CMD_WRITE|CMD_FAST,ACL_CATEGORY_HASH,HPEXPIREAT_Keyspecs,1,NULL,4),.args=HPEXPIREAT_Args},
{MAKE_CMD("hpexpiretime","Returns the expiration time of a hash field as a Unix timestamp, in msec.","O(N) where N is the number of specified fields","7.4.0",CMD_DOC_NONE,NULL,NULL,"hash",COMMAND_GROUP_HASH,HPEXPIRETIME_History,0,HPEXPIRETIME_Tips,0,hpexpiretimeCommand,-5,CMD_READONLY|CMD_FAST,ACL_CATEGORY_HASH,HPEXPIRETIME_Keyspecs,1,NULL,2),.args=HPEXPIRETIME_Args},
{MAKE_CMD("hpttl","Returns the TTL in milliseconds of a hash field.","O(N) where N is the number of specified fields","7.4.0",CMD_DOC_NONE,NULL,NULL,"hash",COMMAND_GROUP_HASH,HPTTL_History,0,HPTTL_Tips,0,hpttlCommand,-5,CMD_READONLY|CMD_FAST,ACL_CATEGORY_HASH,HPTTL_Keyspecs,1,NULL,2),.args=HPTTL_Args},
{MAKE_CMD("hpttl","Returns the TTL in milliseconds of a hash field.","O(N) where N is the number of specified fields","7.4.0",CMD_DOC_NONE,NULL,NULL,"hash",COMMAND_GROUP_HASH,HPTTL_History,0,HPTTL_Tips,1,hpttlCommand,-5,CMD_READONLY|CMD_FAST,ACL_CATEGORY_HASH,HPTTL_Keyspecs,1,NULL,2),.args=HPTTL_Args},
{MAKE_CMD("hrandfield","Returns one or more random fields from a hash.","O(N) where N is the number of fields returned","6.2.0",CMD_DOC_NONE,NULL,NULL,"hash",COMMAND_GROUP_HASH,HRANDFIELD_History,0,HRANDFIELD_Tips,1,hrandfieldCommand,-2,CMD_READONLY,ACL_CATEGORY_HASH,HRANDFIELD_Keyspecs,1,NULL,2),.args=HRANDFIELD_Args},
{MAKE_CMD("hscan","Iterates over fields and values of a hash.","O(1) for every call. O(N) for a complete iteration, including enough command calls for the cursor to return back to 0. N is the number of elements inside the collection.","2.8.0",CMD_DOC_NONE,NULL,NULL,"hash",COMMAND_GROUP_HASH,HSCAN_History,0,HSCAN_Tips,1,hscanCommand,-3,CMD_READONLY,ACL_CATEGORY_HASH,HSCAN_Keyspecs,1,NULL,5),.args=HSCAN_Args},
{MAKE_CMD("hset","Creates or modifies the value of a field in a hash.","O(1) for each field/value pair added, so O(N) to add N field/value pairs when the command is called with multiple field/value pairs.","2.0.0",CMD_DOC_NONE,NULL,NULL,"hash",COMMAND_GROUP_HASH,HSET_History,1,HSET_Tips,0,hsetCommand,-4,CMD_WRITE|CMD_DENYOOM|CMD_FAST,ACL_CATEGORY_HASH,HSET_Keyspecs,1,NULL,2),.args=HSET_Args},
{MAKE_CMD("hsetex","Set the value of one or more fields of a given hash key, and optionally set their expiration.","O(N) where N is the number of fields being set.","8.0.0",CMD_DOC_NONE,NULL,NULL,"hash",COMMAND_GROUP_HASH,HSETEX_History,0,HSETEX_Tips,0,hsetexCommand,-6,CMD_WRITE|CMD_DENYOOM|CMD_FAST,ACL_CATEGORY_HASH,HSETEX_Keyspecs,1,NULL,4),.args=HSETEX_Args},
{MAKE_CMD("hsetnx","Sets the value of a field in a hash only when the field doesn't exist.","O(1)","2.0.0",CMD_DOC_NONE,NULL,NULL,"hash",COMMAND_GROUP_HASH,HSETNX_History,0,HSETNX_Tips,0,hsetnxCommand,4,CMD_WRITE|CMD_DENYOOM|CMD_FAST,ACL_CATEGORY_HASH,HSETNX_Keyspecs,1,NULL,3),.args=HSETNX_Args},
{MAKE_CMD("hstrlen","Returns the length of the value of a field.","O(1)","3.2.0",CMD_DOC_NONE,NULL,NULL,"hash",COMMAND_GROUP_HASH,HSTRLEN_History,0,HSTRLEN_Tips,0,hstrlenCommand,3,CMD_READONLY|CMD_FAST,ACL_CATEGORY_HASH,HSTRLEN_Keyspecs,1,NULL,2),.args=HSTRLEN_Args},
{MAKE_CMD("httl","Returns the TTL in seconds of a hash field.","O(N) where N is the number of specified fields","7.4.0",CMD_DOC_NONE,NULL,NULL,"hash",COMMAND_GROUP_HASH,HTTL_History,0,HTTL_Tips,0,httlCommand,-5,CMD_READONLY|CMD_FAST,ACL_CATEGORY_HASH,HTTL_Keyspecs,1,NULL,2),.args=HTTL_Args},
{MAKE_CMD("httl","Returns the TTL in seconds of a hash field.","O(N) where N is the number of specified fields","7.4.0",CMD_DOC_NONE,NULL,NULL,"hash",COMMAND_GROUP_HASH,HTTL_History,0,HTTL_Tips,1,httlCommand,-5,CMD_READONLY|CMD_FAST,ACL_CATEGORY_HASH,HTTL_Keyspecs,1,NULL,2),.args=HTTL_Args},
{MAKE_CMD("hvals","Returns all values in a hash.","O(N) where N is the size of the hash.","2.0.0",CMD_DOC_NONE,NULL,NULL,"hash",COMMAND_GROUP_HASH,HVALS_History,0,HVALS_Tips,1,hvalsCommand,2,CMD_READONLY,ACL_CATEGORY_HASH,HVALS_Keyspecs,1,NULL,1),.args=HVALS_Args},
/* hyperloglog */
{MAKE_CMD("pfadd","Adds elements to a HyperLogLog key. Creates the key if it doesn't exist.","O(1) to add every element.","2.8.9",CMD_DOC_NONE,NULL,NULL,"hyperloglog",COMMAND_GROUP_HYPERLOGLOG,PFADD_History,0,PFADD_Tips,0,pfaddCommand,-2,CMD_WRITE|CMD_DENYOOM|CMD_FAST,ACL_CATEGORY_HYPERLOGLOG,PFADD_Keyspecs,1,NULL,2),.args=PFADD_Args},
@@ -11141,7 +11277,7 @@ struct COMMAND_STRUCT redisCommandTable[] = {
{MAKE_CMD("sintercard","Returns the number of members of the intersect of multiple sets.","O(N*M) worst case where N is the cardinality of the smallest set and M is the number of sets.","7.0.0",CMD_DOC_NONE,NULL,NULL,"set",COMMAND_GROUP_SET,SINTERCARD_History,0,SINTERCARD_Tips,0,sinterCardCommand,-3,CMD_READONLY,ACL_CATEGORY_SET,SINTERCARD_Keyspecs,1,sintercardGetKeys,3),.args=SINTERCARD_Args},
{MAKE_CMD("sinterstore","Stores the intersect of multiple sets in a key.","O(N*M) worst case where N is the cardinality of the smallest set and M is the number of sets.","1.0.0",CMD_DOC_NONE,NULL,NULL,"set",COMMAND_GROUP_SET,SINTERSTORE_History,0,SINTERSTORE_Tips,0,sinterstoreCommand,-3,CMD_WRITE|CMD_DENYOOM,ACL_CATEGORY_SET,SINTERSTORE_Keyspecs,2,NULL,2),.args=SINTERSTORE_Args},
{MAKE_CMD("sismember","Determines whether a member belongs to a set.","O(1)","1.0.0",CMD_DOC_NONE,NULL,NULL,"set",COMMAND_GROUP_SET,SISMEMBER_History,0,SISMEMBER_Tips,0,sismemberCommand,3,CMD_READONLY|CMD_FAST,ACL_CATEGORY_SET,SISMEMBER_Keyspecs,1,NULL,2),.args=SISMEMBER_Args},
{MAKE_CMD("smembers","Returns all members of a set.","O(N) where N is the set cardinality.","1.0.0",CMD_DOC_NONE,NULL,NULL,"set",COMMAND_GROUP_SET,SMEMBERS_History,0,SMEMBERS_Tips,1,sinterCommand,2,CMD_READONLY,ACL_CATEGORY_SET,SMEMBERS_Keyspecs,1,NULL,1),.args=SMEMBERS_Args},
{MAKE_CMD("smembers","Returns all members of a set.","O(N) where N is the set cardinality.","1.0.0",CMD_DOC_NONE,NULL,NULL,"set",COMMAND_GROUP_SET,SMEMBERS_History,0,SMEMBERS_Tips,1,smembersCommand,2,CMD_READONLY,ACL_CATEGORY_SET,SMEMBERS_Keyspecs,1,NULL,1),.args=SMEMBERS_Args},
{MAKE_CMD("smismember","Determines whether multiple members belong to a set.","O(N) where N is the number of elements being checked for membership","6.2.0",CMD_DOC_NONE,NULL,NULL,"set",COMMAND_GROUP_SET,SMISMEMBER_History,0,SMISMEMBER_Tips,0,smismemberCommand,-3,CMD_READONLY|CMD_FAST,ACL_CATEGORY_SET,SMISMEMBER_Keyspecs,1,NULL,2),.args=SMISMEMBER_Args},
{MAKE_CMD("smove","Moves a member from one set to another.","O(1)","1.0.0",CMD_DOC_NONE,NULL,NULL,"set",COMMAND_GROUP_SET,SMOVE_History,0,SMOVE_Tips,0,smoveCommand,4,CMD_WRITE|CMD_FAST,ACL_CATEGORY_SET,SMOVE_Keyspecs,2,NULL,3),.args=SMOVE_Args},
{MAKE_CMD("spop","Returns one or more random members from a set after removing them. Deletes the set if the last member was popped.","Without the count argument O(1), otherwise O(N) where N is the value of the passed count.","1.0.0",CMD_DOC_NONE,NULL,NULL,"set",COMMAND_GROUP_SET,SPOP_History,1,SPOP_Tips,1,spopCommand,-2,CMD_WRITE|CMD_FAST,ACL_CATEGORY_SET,SPOP_Keyspecs,1,NULL,2),.args=SPOP_Args},
src/commands/client-list.json
+12
View File
@@ -31,6 +31,18 @@
[
"7.0.3",
"Added `ssub` field."
],
[
"7.2.0",
"Added `lib-name` and `lib-ver` fields."
],
[
"7.4.0",
"Added `watch` field."
],
[
"8.0.0",
"Added `io-thread` field."
]
],
"command_flags": [
src/commands/hexpire.json
-1
View File
@@ -9,7 +9,6 @@
"history": [],
"command_flags": [
"WRITE",
"DENYOOM",
"FAST"
],
"acl_categories": [
src/commands/hexpireat.json
-1
View File
@@ -9,7 +9,6 @@
"history": [],
"command_flags": [
"WRITE",
"DENYOOM",
"FAST"
],
"acl_categories": [
src/commands/hgetdel.json
+78
View File
@@ -0,0 +1,78 @@
{
"HGETDEL": {
"summary": "Returns the value of a field and deletes it from the hash.",
"complexity": "O(N) where N is the number of specified fields",
"group": "hash",
"since": "8.0.0",
"arity": -5,
"function": "hgetdelCommand",
"history": [],
"command_flags": [
"WRITE",
"FAST"
],
"acl_categories": [
"HASH"
],
"key_specs": [
{
"flags": [
"RW",
"ACCESS",
"DELETE"
],
"begin_search": {
"index": {
"pos": 1
}
},
"find_keys": {
"range": {
"lastkey": 0,
"step": 1,
"limit": 0
}
}
}
],
"reply_schema": {
"description": "List of values associated with the given fields, in the same order as they are requested.",
"type": "array",
"minItems": 1,
"items": {
"oneOf": [
{
"type": "string"
},
{
"type": "null"
}
]
}
},
"arguments": [
{
"name": "key",
"type": "key",
"key_spec_index": 0
},
{
"name": "fields",
"token": "FIELDS",
"type": "block",
"arguments": [
{
"name": "numfields",
"type": "integer"
},
{
"name": "field",
"type": "string",
"multiple": true
}
]
}
]
}
}
src/commands/hgetex.json
+111
View File
@@ -0,0 +1,111 @@
{
"HGETEX": {
"summary": "Get the value of one or more fields of a given hash key, and optionally set their expiration.",
"complexity": "O(N) where N is the number of specified fields",
"group": "hash",
"since": "8.0.0",
"arity": -5,
"function": "hgetexCommand",
"history": [],
"command_flags": [
"WRITE",
"FAST"
],
"acl_categories": [
"HASH"
],
"key_specs": [
{
"notes": "RW and UPDATE because it changes the TTL",
"flags": [
"RW",
"ACCESS",
"UPDATE"
],
"begin_search": {
"index": {
"pos": 1
}
},
"find_keys": {
"range": {
"lastkey": 0,
"step": 1,
"limit": 0
}
}
}
],
"reply_schema": {
"description": "List of values associated with the given fields, in the same order as they are requested.",
"type": "array",
"minItems": 1,
"items": {
"oneOf": [
{
"type": "string"
},
{
"type": "null"
}
]
}
},
"arguments": [
{
"name": "key",
"type": "key",
"key_spec_index": 0
},
{
"name": "expiration",
"type": "oneof",
"optional": true,
"arguments": [
{
"name": "seconds",
"type": "integer",
"token": "EX"
},
{
"name": "milliseconds",
"type": "integer",
"token": "PX"
},
{
"name": "unix-time-seconds",
"type": "unix-time",
"token": "EXAT"
},
{
"name": "unix-time-milliseconds",
"type": "unix-time",
"token": "PXAT"
},
{
"name": "persist",
"type": "pure-token",
"token": "PERSIST"
}
]
},
{
"name": "fields",
"token": "FIELDS",
"type": "block",
"arguments": [
{
"name": "numfields",
"type": "integer"
},
{
"name": "field",
"type": "string",
"multiple": true
}
]
}
]
}
}
src/commands/hpexpire.json
-1
View File
@@ -9,7 +9,6 @@
"history": [],
"command_flags": [
"WRITE",
"DENYOOM",
"FAST"
],
"acl_categories": [
src/commands/hpexpireat.json
-1
View File
@@ -9,7 +9,6 @@
"history": [],
"command_flags": [
"WRITE",
"DENYOOM",
"FAST"
],
"acl_categories": [
src/commands/hpttl.json
+3
View File
@@ -14,6 +14,9 @@
"acl_categories": [
"HASH"
],
"command_tips": [
"NONDETERMINISTIC_OUTPUT"
],
"key_specs": [
{
"flags": [
src/commands/hsetex.json
+132
View File
@@ -0,0 +1,132 @@
{
"HSETEX": {
"summary": "Set the value of one or more fields of a given hash key, and optionally set their expiration.",
"complexity": "O(N) where N is the number of fields being set.",
"group": "hash",
"since": "8.0.0",
"arity": -6,
"function": "hsetexCommand",
"command_flags": [
"WRITE",
"DENYOOM",
"FAST"
],
"acl_categories": [
"HASH"
],
"key_specs": [
{
"flags": [
"RW",
"UPDATE"
],
"begin_search": {
"index": {
"pos": 1
}
},
"find_keys": {
"range": {
"lastkey": 0,
"step": 1,
"limit": 0
}
}
}
],
"reply_schema": {
"oneOf": [
{
"description": "No field was set (due to FXX or FNX flags).",
"const": 0
},
{
"description": "All the fields were set.",
"const": 1
}
]
},
"arguments": [
{
"name": "key",
"type": "key",
"key_spec_index": 0
},
{
"name": "condition",
"type": "oneof",
"optional": true,
"arguments": [
{
"name": "fnx",
"type": "pure-token",
"token": "FNX"
},
{
"name": "fxx",
"type": "pure-token",
"token": "FXX"
}
]
},
{
"name": "expiration",
"type": "oneof",
"optional": true,
"arguments": [
{
"name": "seconds",
"type": "integer",
"token": "EX"
},
{
"name": "milliseconds",
"type": "integer",
"token": "PX"
},
{
"name": "unix-time-seconds",
"type": "unix-time",
"token": "EXAT"
},
{
"name": "unix-time-milliseconds",
"type": "unix-time",
"token": "PXAT"
},
{
"name": "keepttl",
"type": "pure-token",
"token": "KEEPTTL"
}
]
},
{
"name": "fields",
"token": "FIELDS",
"type": "block",
"arguments": [
{
"name": "numfields",
"type": "integer"
},
{
"name": "data",
"type": "block",
"multiple": true,
"arguments": [
{
"name": "field",
"type": "string"
},
{
"name": "value",
"type": "string"
}
]
}
]
}
]
}
}
src/commands/httl.json
+3
View File
@@ -14,6 +14,9 @@
"acl_categories": [
"HASH"
],
"command_tips": [
"NONDETERMINISTIC_OUTPUT"
],
"key_specs": [
{
"flags": [
src/commands/memory-stats.json
+6
View File
@@ -29,6 +29,9 @@
"replication.backlog": {
"type": "integer"
},
"replica.fullsync.buffer": {
"type": "integer"
},
"clients.slaves": {
"type": "integer"
},
@@ -44,6 +47,9 @@
"lua.caches": {
"type": "integer"
},
"script.VMs": {
"type": "integer"
},
"functions.caches": {
"type": "integer"
},
src/commands/sflush.json
+75
View File
@@ -0,0 +1,75 @@
{
"SFLUSH": {
"summary": "Remove all keys from selected range of slots.",
"complexity": "O(N)+O(k) where N is the number of keys and k is the number of slots.",
"group": "server",
"since": "8.0.0",
"arity": -3,
"function": "sflushCommand",
"command_flags": [
"WRITE",
"EXPERIMENTAL"
],
"acl_categories": [
"KEYSPACE",
"DANGEROUS"
],
"command_tips": [
],
"reply_schema": {
"description": "List of slot ranges",
"type": "array",
"minItems": 0,
"maxItems": 4294967295,
"items": {
"type": "array",
"minItems": 2,
"maxItems": 2,
"items": [
{
"description": "start slot number",
"type": "integer"
},
{
"description": "end slot number",
"type": "integer"
}
]
}
},
"arguments": [
{
"name": "data",
"type": "block",
"multiple": true,
"arguments": [
{
"name": "slot-start",
"type": "integer"
},
{
"name": "slot-last",
"type": "integer"
}
]
},
{
"name": "flush-type",
"type": "oneof",
"optional": true,
"arguments": [
{
"name": "async",
"type": "pure-token",
"token": "ASYNC"
},
{
"name": "sync",
"type": "pure-token",
"token": "SYNC"
}
]
}
]
}
}
src/commands/smembers.json
+1 -1
View File
@@ -5,7 +5,7 @@
"group": "set",
"since": "1.0.0",
"arity": 2,
"function": "sinterCommand",
"function": "smembersCommand",
"command_flags": [
"READONLY"
],

Some files were not shown because too many files have changed in this diff Show More