Moti Cohenanddebing.sun
33fc0fbfae
* For replica sake, rewrite commands `H*EXPIRE*` , `HSETF`, `HGETF` to have absolute unix time in msec. * On active-expiration of field, propagate HDEL to replica (`propagateHashFieldDeletion()`) * On lazy-expiration, propagate HDEL to replica (`hashTypeGetValue()` now calls `hashTypeDelete()`. It also takes care to call `propagateHashFieldDeletion()`). * Fix `H*EXPIRE*` command such that if it gets flag `LT` and it doesn’t have any expiration on the field then it will considered as valid condition. Note, replicas doesn’t make any active expiration, and should avoid lazy expiration. On `hashTypeGetValue()` it doesn't check expiration (As long as the master didn’t request to delete the field, it is valid) TODO: * Attach `dbid` to HASH metadata. See [here](https://github.com/redis/redis/pull/13209#discussion_r1593385850) --------- Co-authored-by: debing.sun <debing.sun@redis.com>
230 lines
8.9 KiB
Tcl
230 lines
8.9 KiB
Tcl
start_server {tags {"psync2 external:skip"}} {
|
|
start_server {} {
|
|
start_server {} {
|
|
set master [srv 0 client]
|
|
set master_host [srv 0 host]
|
|
set master_port [srv 0 port]
|
|
|
|
set replica [srv -1 client]
|
|
set replica_host [srv -1 host]
|
|
set replica_port [srv -1 port]
|
|
|
|
set sub_replica [srv -2 client]
|
|
|
|
# Make sure the server saves an RDB on shutdown
|
|
$master config set save "3600 1"
|
|
|
|
# Because we will test partial resync later, we don't want a timeout to cause
|
|
# the master-replica disconnect, then the extra reconnections will break the
|
|
# sync_partial_ok stat test
|
|
$master config set repl-timeout 3600
|
|
$replica config set repl-timeout 3600
|
|
$sub_replica config set repl-timeout 3600
|
|
|
|
# Avoid PINGs
|
|
$master config set repl-ping-replica-period 3600
|
|
$master config rewrite
|
|
|
|
# Build replication chain
|
|
$replica replicaof $master_host $master_port
|
|
$sub_replica replicaof $replica_host $replica_port
|
|
|
|
wait_for_condition 50 100 {
|
|
[status $replica master_link_status] eq {up} &&
|
|
[status $sub_replica master_link_status] eq {up}
|
|
} else {
|
|
fail "Replication not started."
|
|
}
|
|
|
|
test "PSYNC2: Partial resync after Master restart using RDB aux fields when offset is 0" {
|
|
assert {[status $master master_repl_offset] == 0}
|
|
|
|
set replid [status $master master_replid]
|
|
$replica config resetstat
|
|
|
|
catch {
|
|
restart_server 0 true false true now
|
|
set master [srv 0 client]
|
|
}
|
|
wait_for_condition 50 1000 {
|
|
[status $replica master_link_status] eq {up} &&
|
|
[status $sub_replica master_link_status] eq {up}
|
|
} else {
|
|
fail "Replicas didn't sync after master restart"
|
|
}
|
|
|
|
# Make sure master restore replication info correctly
|
|
assert {[status $master master_replid] != $replid}
|
|
assert {[status $master master_repl_offset] == 0}
|
|
assert {[status $master master_replid2] eq $replid}
|
|
assert {[status $master second_repl_offset] == 1}
|
|
|
|
# Make sure master set replication backlog correctly
|
|
assert {[status $master repl_backlog_active] == 1}
|
|
assert {[status $master repl_backlog_first_byte_offset] == 1}
|
|
assert {[status $master repl_backlog_histlen] == 0}
|
|
|
|
# Partial resync after Master restart
|
|
assert {[status $master sync_partial_ok] == 1}
|
|
assert {[status $replica sync_partial_ok] == 1}
|
|
}
|
|
|
|
# Generate some data
|
|
createComplexDataset $master 1000
|
|
|
|
test "PSYNC2: Partial resync after Master restart using RDB aux fields with data" {
|
|
wait_for_condition 500 100 {
|
|
[status $master master_repl_offset] == [status $replica master_repl_offset] &&
|
|
[status $master master_repl_offset] == [status $sub_replica master_repl_offset]
|
|
} else {
|
|
fail "Replicas and master offsets were unable to match *exactly*."
|
|
}
|
|
|
|
set replid [status $master master_replid]
|
|
set offset [status $master master_repl_offset]
|
|
$replica config resetstat
|
|
|
|
catch {
|
|
# SHUTDOWN NOW ensures master doesn't send GETACK to replicas before
|
|
# shutting down which would affect the replication offset.
|
|
restart_server 0 true false true now
|
|
set master [srv 0 client]
|
|
}
|
|
wait_for_condition 50 1000 {
|
|
[status $replica master_link_status] eq {up} &&
|
|
[status $sub_replica master_link_status] eq {up}
|
|
} else {
|
|
fail "Replicas didn't sync after master restart"
|
|
}
|
|
|
|
# Make sure master restore replication info correctly
|
|
assert {[status $master master_replid] != $replid}
|
|
assert {[status $master master_repl_offset] == $offset}
|
|
assert {[status $master master_replid2] eq $replid}
|
|
assert {[status $master second_repl_offset] == [expr $offset+1]}
|
|
|
|
# Make sure master set replication backlog correctly
|
|
assert {[status $master repl_backlog_active] == 1}
|
|
assert {[status $master repl_backlog_first_byte_offset] == [expr $offset+1]}
|
|
assert {[status $master repl_backlog_histlen] == 0}
|
|
|
|
# Partial resync after Master restart
|
|
assert {[status $master sync_partial_ok] == 1}
|
|
assert {[status $replica sync_partial_ok] == 1}
|
|
}
|
|
|
|
test "PSYNC2: Partial resync after Master restart using RDB aux fields with expire" {
|
|
$master debug set-active-expire 0
|
|
for {set j 0} {$j < 1024} {incr j} {
|
|
$master select [expr $j%16]
|
|
$master set $j somevalue px 10
|
|
}
|
|
|
|
after 20
|
|
|
|
# Wait until master has received ACK from replica. If the master thinks
|
|
# that any replica is lagging when it shuts down, master would send
|
|
# GETACK to the replicas, affecting the replication offset.
|
|
set offset [status $master master_repl_offset]
|
|
wait_for_condition 500 100 {
|
|
[string match "*slave0:*,offset=$offset,*" [$master info replication]] &&
|
|
$offset == [status $replica master_repl_offset] &&
|
|
$offset == [status $sub_replica master_repl_offset]
|
|
} else {
|
|
show_cluster_status
|
|
fail "Replicas and master offsets were unable to match *exactly*."
|
|
}
|
|
|
|
set offset [status $master master_repl_offset]
|
|
$replica config resetstat
|
|
|
|
catch {
|
|
# Unlike the test above, here we use SIGTERM, which behaves
|
|
# differently compared to SHUTDOWN NOW if there are lagging
|
|
# replicas. This is just to increase coverage and let each test use
|
|
# a different shutdown approach. In this case there are no lagging
|
|
# replicas though.
|
|
restart_server 0 true false
|
|
set master [srv 0 client]
|
|
}
|
|
wait_for_condition 50 1000 {
|
|
[status $replica master_link_status] eq {up} &&
|
|
[status $sub_replica master_link_status] eq {up}
|
|
} else {
|
|
fail "Replicas didn't sync after master restart"
|
|
}
|
|
|
|
set expired_offset [status $master repl_backlog_histlen]
|
|
# Stale keys expired and master_repl_offset grows correctly
|
|
assert {[status $master rdb_last_load_keys_expired] == 1024}
|
|
assert {[status $master master_repl_offset] == [expr $offset+$expired_offset]}
|
|
|
|
# Partial resync after Master restart
|
|
assert {[status $master sync_partial_ok] == 1}
|
|
assert {[status $replica sync_partial_ok] == 1}
|
|
|
|
set digest [$master debug digest]
|
|
assert {$digest eq [$replica debug digest]}
|
|
assert {$digest eq [$sub_replica debug digest]}
|
|
}
|
|
|
|
test "PSYNC2: Full resync after Master restart when too many key expired" {
|
|
$master config set repl-backlog-size 16384
|
|
$master config rewrite
|
|
|
|
$master debug set-active-expire 0
|
|
# Make sure replication backlog is full and will be trimmed.
|
|
for {set j 0} {$j < 2048} {incr j} {
|
|
$master select [expr $j%16]
|
|
$master set $j somevalue px 10
|
|
}
|
|
|
|
##### hash-field-expiration
|
|
# Hashes of type OBJ_ENCODING_LISTPACK_EX won't be discarded during
|
|
# RDB load, even if they are expired.
|
|
$master hset myhash1 f1 v1 f2 v2 f3 v3
|
|
$master hpexpire myhash1 10 FIELDS 3 f1 f2 f3
|
|
# Hashes of type RDB_TYPE_HASH_METADATA will be discarded during RDB load.
|
|
$master config set hash-max-listpack-entries 0
|
|
$master hset myhash2 f1 v1 f2 v2
|
|
$master hpexpire myhash2 10 FIELDS 2 f1 f2
|
|
$master config set hash-max-listpack-entries 1
|
|
|
|
after 20
|
|
|
|
wait_for_condition 500 100 {
|
|
[status $master master_repl_offset] == [status $replica master_repl_offset] &&
|
|
[status $master master_repl_offset] == [status $sub_replica master_repl_offset]
|
|
} else {
|
|
fail "Replicas and master offsets were unable to match *exactly*."
|
|
}
|
|
|
|
$replica config resetstat
|
|
|
|
catch {
|
|
# Unlike the test above, here we use SIGTERM. This is just to
|
|
# increase coverage and let each test use a different shutdown
|
|
# approach.
|
|
restart_server 0 true false
|
|
set master [srv 0 client]
|
|
}
|
|
wait_for_condition 50 1000 {
|
|
[status $replica master_link_status] eq {up} &&
|
|
[status $sub_replica master_link_status] eq {up}
|
|
} else {
|
|
fail "Replicas didn't sync after master restart"
|
|
}
|
|
|
|
# Replication backlog is full
|
|
assert {[status $master repl_backlog_first_byte_offset] > [status $master second_repl_offset]}
|
|
assert {[status $master sync_partial_ok] == 0}
|
|
assert {[status $master sync_full] == 1}
|
|
assert {[status $master rdb_last_load_keys_expired] == 2048}
|
|
assert {[status $replica sync_full] == 1}
|
|
|
|
set digest [$master debug digest]
|
|
assert {$digest eq [$replica debug digest]}
|
|
assert {$digest eq [$sub_replica debug digest]}
|
|
}
|
|
}}}
|