Ignore RM_Call deny-oom flag if maxmemory is zero (#11319)

If a command gets an OOM response and then if we set maxmemory to zero
to disable the limit, server.pre_command_oom_state never gets updated
and it stays true. As RM_Call() calls with "respect deny-oom" flag checks
server.pre_command_oom_state, all calls will fail with OOM.

Added server.maxmemory check in RM_Call() to process deny-oom flag
only if maxmemory is configured.

(cherry picked from commit 18920813a9)

src/module.c

@@ -5886,7 +5886,7 @@ RedisModuleCallReply *RM_Call(RedisModuleCtx *ctx, const char *cmdname, const ch
         }
     }
 
-    if (flags & REDISMODULE_ARGV_RESPECT_DENY_OOM) {
+    if (flags & REDISMODULE_ARGV_RESPECT_DENY_OOM && server.maxmemory) {
         if (cmd_flags & CMD_DENYOOM) {
             int oom_state;
             if (ctx->flags & REDISMODULE_CTX_THREAD_SAFE) {

tests/unit/moduleapi/misc.tcl

@@ -183,6 +183,21 @@ start_server {tags {"modules"}} {
         r config set maxmemory 0
     } {OK} {needs:config-maxmemory}
 
+    test {rm_call clear OOM} {
+        r config set maxmemory 1
+
+        # verify rm_call fails with OOM
+        assert_error {OOM *} {
+            r test.rm_call_flags M set x 1
+        }
+
+        # clear OOM state
+        r config set maxmemory 0
+
+        # test set command is allowed
+        r test.rm_call_flags M set x 1
+    } {OK} {needs:config-maxmemory}
+
     test {rm_call OOM Eval} {
         r config set maxmemory 1
         r config set maxmemory-policy volatile-lru