Improve cmd_flags for script/functions in RM_Call (#11159)

When RM_Call was used with `M` (reject OOM), `W` (reject writes),
as well as `S` (rejecting stale or write commands in "Script mode"),
it would have only checked the command flags, but not the declared
script flag in case it's a command that runs a script.

Refactoring: extracts out similar code in server.c's processCommand
to be usable in RM_Call as well.

tests/unit/moduleapi/misc.tcl

@@ -173,6 +173,29 @@ start_server {tags {"modules"}} {
         r config set maxmemory 0
     } {OK} {needs:config-maxmemory}
 
+    test {rm_call OOM Eval} {
+        r config set maxmemory 1
+        r config set maxmemory-policy volatile-lru
+
+        # use the M flag without allow-oom shebang flag
+        assert_error {OOM *} {
+            r test.rm_call_flags M eval {#!lua
+                redis.call('set','x',1)
+                return 1
+            } 1 x
+        }
+
+        # add the M flag with allow-oom shebang flag
+        assert_equal {1} [
+            r test.rm_call_flags M eval {#!lua flags=allow-oom
+                redis.call('set','x',1)
+                return 1
+            } 1 x
+        ]
+
+        r config set maxmemory 0
+    } {OK} {needs:config-maxmemory}
+
     test {rm_call write flag} {
         # add the W flag
         assert_error {ERR Write command 'set' was called while write is not allowed.} {