Cluster mode draft PR #2

ClusterRedisExample.py

@@ -0,0 +1,45 @@
+from redis import RedisCluster as Redis
+from redis.cluster import ClusterNode
+
+host = 'localhost'
+startup_nodes = [ClusterNode(host, 16379), ClusterNode(host, 16380)]
+
+# from_url examples
+rc_url = Redis.from_url("redis://localhost:16379/0")
+print(rc_url.cluster_slots())
+print(rc_url.ping(Redis.PRIMARIES))
+print(rc_url.ping(Redis.REPLICAS))
+print(rc_url.ping(Redis.RANDOM))
+print(rc_url.ping(Redis.ALL_NODES))
+print(rc_url.execute_command("STRALGO", "LCS", "STRINGS", "string1",
+                             "string2",
+                             target_nodes=rc_url.get_random_node()))
+print(rc_url.client_list())
+print(rc_url.set('foo', 'bar1'))
+print(rc_url.mget('{bar}1', '{bar}2'))
+print(rc_url.set('zzzsdfsdf', 'bar2'))
+print(rc_url.keyslot('bar'))
+print(rc_url.set('{000}', 'bar3'))
+print(f"get_nodes: {rc_url.get_nodes()}")
+print(rc_url.get('foo'))
+print(rc_url.keys())
+# rc = Redis(host=host, port=6379)
+rc = Redis(startup_nodes=startup_nodes, decode_responses=True)
+print(rc.get('{000}'))
+print(rc.keys())
+print(rc.cluster_save_config(rc.get_primaries()))
+print(rc.cluster_save_config(rc.get_node(host=host, port=16379)))
+
+# READONLY examples
+rc_readonly = Redis(startup_nodes=startup_nodes, read_from_replicas=True,
+                    debug=True)
+rc_readonly.set('bar', 'foo')
+for i in range(0, 4):
+    # Assigning the read command to the slot's servers in a Round-Robin manner
+    print(rc_readonly.get('bar'))
+# set command would be directed only to the slot's primary node
+# reset READONLY flag
+print(rc_readonly.readwrite())
+for i in range(0, 4):
+    # now the get command would be directed only to the slot's primary node
+    print(rc_readonly.get('bar'))

README.md

@@ -941,8 +941,228 @@ C 3
 
 ### Cluster Mode
 
-redis-py does not currently support [Cluster
-Mode](https://redis.io/topics/cluster-tutorial).
+redis-py is now supports cluster mode and provides a client for
+[Redis Cluster](<https://redis.io/topics/cluster-tutorial>).
+
+The cluster client is based on [redis-py-cluster](https://github.com/Grokzen/redis-py-cluster)
+by Grokzen, with a lot of added and 
+changed functionality.
+
+**Create RedisCluster:**
+
+Connecting redis-py to the Redis Cluster instance(s) is easy.
+RedisCluster requires at least one node to discover the whole cluster nodes,
+and there is multiple ways of creating a RedisCluster instance:
+
+- Use the 'host' and 'port' arguments:
+
+``` pycon
+    >>> from redis.cluster import RedisCluster as Redis
+    >>> rc = Redis(host='localhost', port=6379)
+    >>> print(rc.get_nodes())
+    [[host=127.0.0.1,port=6379,name=127.0.0.1:6379,server_type=primary,redis_connection=Redis<ConnectionPool<Connection<host=127.0.0.1,port=6379,db=0>>>], [host=127.0.0.1,port=6378,name=127.0.0.1:6378,server_type=primary,redis_connection=Redis<ConnectionPool<Connection<host=127.0.0.1,port=6378,db=0>>>], [host=127.0.0.1,port=6377,name=127.0.0.1:6377,server_type=replica,redis_connection=Redis<ConnectionPool<Connection<host=127.0.0.1,port=6377,db=0>>>]]
+```
+- Use Redis URL:
+
+``` pycon
+    >>> from redis.cluster import RedisCluster as Redis
+    >>> rc = Redis.from_url("redis://localhost:6379/0")
+```
+
+- Use ClusterNode(s):
+
+``` pycon
+    >>> from redis.cluster import RedisCluster as Redis
+    >>> from redis.cluster import ClusterNode
+    >>> nodes = [ClusterNode('localhost', 6379), ClusterNode('localhost', 6378)]
+    >>> rc = Redis(startup_nodes=nodes)
+```
+
+When a RedisCluster instance is being created it first attempts to establish a
+connection to one of the provided startup nodes. If none of the startup nodes
+are reachable, a 'RedisClusterException' will be thrown.
+After a connection to the one of the cluster's nodes is established, the
+RedisCluster instance will be initialized with 3 caches:
+a slots cache which maps each of the 16384 slots to the node/s handling them,
+a nodes cache that contains ClusterNode objects (name, host, port, redis connection)
+for all of the cluster's nodes, and a commands cache contains all the server
+supported commands that were retrieved using the Redis 'COMMAND' output.
+
+RedisCluster instance can be directly used to execute Redis commands. When a
+command is being executed through the cluster instance, the target node(s) will
+be internally determined. When using a key-based command, the target node will
+be the node that holds the key's slot.
+Cluster management commands or other cluster commands have predefined node
+group targets (all-primaries, all-nodes, random-node, all-replicas), which are
+outlined in the command’s function documentation.
+For example, ‘KEYS’ command will be sent to all primaries and return all keys
+in the cluster, and ‘CLUSTER NODES’ command will be sent to a random node.
+Other management commands will require you to pass the target node/s to execute
+the command on.
+
+``` pycon
+    >>> # target-nodes: the node that holds 'foo1's key slot
+    >>> rc.set('foo1', 'bar1')
+    >>> # target-nodes: the node that holds 'foo2's key slot
+    >>> rc.set('foo2', 'bar2')
+    >>> # target-nodes: the node that holds 'foo1's key slot
+    >>> print(rc.get('foo1'))
+    b'bar'
+    >>> # target-nodes: all-primaries
+    >>> print(rc.keys())
+    [b'foo1', b'foo2']
+    >>> # target-nodes: all-nodes
+    >>> rc.flushall()
+```
+
+**Specifying Target Nodes:**
+
+As mentioned above, some RedisCluster commands will require you to provide the
+target node/s that you want to execute the command on, and in other cases, the
+target node will be determined by the client itself. That being said, ALL
+RedisCluster commands can be executed against a specific node or a group of
+nodes by passing the command kwarg `target_nodes`.
+The best practice is to specify target nodes using RedisCluster class's node
+flags: PRIMARIES, REPLICAS, ALL_NODES, RANDOM. When a nodes flag is passed
+along with a command, it will be internally resolved to the relevant node/s.
+If the nodes topology of the cluster changes during the execution of a command,
+the client will be able to resolve the nodes flag again with the new topology
+and attempt to retry executing the command.
+
+``` pycon
+    >>> from redis.cluster import RedisCluster as Redis
+    >>> # run cluster-meet command on all of the cluster's nodes
+    >>> rc.cluster_meet(Redis.ALL_NODES, '127.0.0.1', 6379)
+    >>> # ping all replicas
+    >>> rc.ping(Redis.REPLICAS)
+    >>> # ping a specific node
+    >>> rc.ping(Redis.RANDOM)
+    >>> # ping all nodes in the cluster, default command behavior
+    >>> rc.ping()
+    >>> # execute bgsave in all primaries
+    >>> rc.bgsave(Redis.PRIMARIES)
+```
+
+You could also pass ClusterNodes directly if you want to execute a command on a
+specific node / node group that isn't addressed by the nodes flag. However, if
+the command execution fails due to cluster topology changes, a retry attempt
+will not be made, since the passed target node/s may no longer be valid, and
+the relevant cluster or connection error will be returned.
+
+``` pycon
+    >>> node = rc.get_node('localhost', 6379)
+    >>> # Get the keys only for that specific node
+    >>> rc.keys(node)
+    >>> # get Redis info from a subset of primaries
+    >>> subset_primaries = [node for node in rc.get_primaries() if node.port > 6378]
+    >>> rc.info(subset_primaries)
+```
+
+In addition, you can use the RedisCluster instance to obtain the Redis instance
+of a specific node and execute commands on that node directly. The Redis client,
+however, cannot handle cluster failures and retries.
+
+``` pycon
+    >>> cluster_node = rc.get_node(host='localhost', port=6379)
+    >>> print(cluster_node)
+    [host=127.0.0.1,port=6379,name=127.0.0.1:6379,server_type=primary,redis_connection=Redis<ConnectionPool<Connection<host=127.0.0.1,port=6379,db=0>>>]
+    >>> r = cluster_node.redis_connection
+    >>> r.client_list()
+    [{'id': '276', 'addr': '127.0.0.1:64108', 'fd': '16', 'name': '', 'age': '0', 'idle': '0', 'flags': 'N', 'db': '0', 'sub': '0', 'psub': '0', 'multi': '-1', 'qbuf': '26', 'qbuf-free': '32742', 'argv-mem': '10', 'obl': '0', 'oll': '0', 'omem': '0', 'tot-mem': '54298', 'events': 'r', 'cmd': 'client', 'user': 'default'}]
+    >>> # Get the keys only for that specific node
+    >>> r.keys()
+    [b'foo1']
+```
+
+**Multi-key commands:**
+
+Redis supports multi-key commands in Cluster Mode, such as Set type unions or
+intersections, mset and mget, as long as the keys all hash to the same slot.
+By using RedisCluster client, you can use the known functions (e.g. mget, mset)
+to perform an atomic multi-key operation. However, you must ensure all keys are
+mapped to the same slot, otherwise a RedisClusterException will be thrown.
+Redis Cluster implements a concept called hash tags that can be used in order
+to force certain keys to be stored in the same hash slot, see 
+[Keys hash tag](https://redis.io/topics/cluster-spec#keys-hash-tags).
+You can also use nonatomic for some of the multikey operations, and pass keys
+that aren't mapped to the same slot. The client will then map the keys to the
+relevant slots, sending the commands to the slots' node owners. Non-atomic
+operations batch the keys according to their hash value, and then each batch is
+sent separately to the slot's owner.
+
+``` pycon
+    #  Atomic operations can be used when all keys are mapped to the same slot
+    >>> rc.mset({'{foo}1': 'bar1', '{foo}2': 'bar2'})
+    >>> rc.mget('{foo}1', '{foo}2')
+    [b'bar1', b'bar2']
+    # Non-atomic multi-key operations splits the keys into different slots
+    >>> rc.mset_nonatomic({'foo': 'value1', 'bar': 'value2', 'zzz': 'value3')
+    >>> rc.mget_nonatomic('foo', 'bar', 'zzz')
+    [b'value1', b'value2', b'value3']
+```
+
+**Cluster PubSub:**
+
+When a ClusterPubSub instance is created without specifying a node, a single
+node will be transparently chosen for the pubsub connection on the
+first command execution. The node will be determined by:
+ 1. Hashing the channel name in the request to find its keyslot
+ 2. Selecting a node that handles the keyslot: If read_from_replicas is
+    set to true, a replica can be selected.
+
+*Known limitations with pubsub:*
+
+Pattern subscribe and publish do not work properly because if we hash a pattern 
+like fo* we will get a keyslot for that string but there is a endless   
+possibilities of channel names based on that pattern that we can’t know in 
+advance. This feature is not limited but the commands is not recommended to use 
+right now. 
+See [redis-py-cluster documentaion](https://redis-py-cluster.readthedocs.io/en/stable/pubsub.html) 
+ for more.
+
+``` pycon
+    >>> p1 = rc.pubsub()
+    # p1 connection will be set to the node that holds 'foo' keyslot
+    >>> p1.subscribe('foo')
+    # p2 connection will be set to node 'localhost:6379'
+    >>> p2 = rc.pubsub(rc.get_node('localhost', 6379))
+```
+
+**Read Only Mode**
+
+By default, Redis Cluster always returns MOVE redirection response on accessing 
+a replica node. You can overcome this limitation and scale read commands with 
+READONLY mode. 
+
+To enable READONLY mode pass read_from_replicas=True to RedisCluster 
+constructor. When set to true, read commands will be assigned between the
+primary and its replications in a Round-Robin manner. 
+
+You could also enable READONLY mode in runtime by running readonly() method,  
+or disable it with readwrite().
+
+``` pycon
+    >>> from cluster import RedisCluster as Redis
+    # Use 'debug' mode to print the node that the command is executed on
+    >>> rc_readonly = Redis(startup_nodes=startup_nodes, 
+                    read_from_replicas=True, debug=True)
+    >>> rc_readonly.set('{foo}1', 'bar1')
+    >>> for i in range(0, 4):
+            # Assigns read command to the slot's hosts in a Round-Robin manner
+    >>>     rc_readonly.get('{foo}1')
+    # set command would be directed only to the slot's primary node
+    >>> rc_readonly.set('{foo}2', 'bar2')
+    # reset READONLY flag
+    >>> rc_readonly.readwrite()
+    # now the get command would be directed only to the slot's primary node
+    >>> rc_readonly.get('{foo}1')
+```
+
+
+
+See [Redis Cluster tutorial](https://redis.io/topics/cluster-tutorial) and
+[Redis Cluster specifications](https://redis.io/topics/cluster-spec)
+to learn more about Redis Cluster.
 
 ### Author
 

docker/base/Dockerfile.cluster

@@ -0,0 +1,8 @@
+FROM redis:6.2.6-buster
+
+COPY ../cluster/create_cluster.sh /create_cluster.sh
+RUN chmod +x /create_cluster.sh
+
+EXPOSE 16379 16380 16381 16382 16383 16384
+
+CMD [ "/create_cluster.sh"]

docker/cluster/create_cluster.sh

@@ -0,0 +1,21 @@
+#! /bin/bash
+mkdir -p /nodes
+echo -n > /nodes/nodemap
+for PORT in $(seq 16379 16384); do
+  mkdir -p /nodes/$PORT
+  if [[ -e /redis.conf ]]; then
+    cp /redis.conf /nodes/$PORT/redis.conf
+  else
+    touch /nodes/$PORT/redis.conf
+  fi
+  cat << EOF >> /nodes/$PORT/redis.conf
+port $PORT
+daemonize yes
+logfile /redis.log
+dir /nodes/$PORT
+EOF
+  redis-server /nodes/$PORT/redis.conf
+  echo 127.0.0.1:$PORT >> /nodes/nodemap
+done
+echo yes | redis-cli --cluster create $(seq -f 127.0.0.1:%g 16379 16384) --cluster-replicas 1
+tail -f /redis.log

docker/cluster/redis.conf

@@ -0,0 +1,3 @@
+# Redis Cluster config file will be shared across all nodes.
+# Dont pass node-unique arguments (e.g. port, dir).
+cluster-enabled yes

redis/__init__.py

@@ -1,4 +1,5 @@
 from redis.client import Redis, StrictRedis
+from redis.cluster import RedisCluster
 from redis.connection import (
     BlockingConnectionPool,
     ConnectionPool,
@@ -49,6 +50,7 @@ def int_or_str(value):
     'PubSubError',
     'ReadOnlyError',
     'Redis',
+    'RedisCluster',
     'RedisError',
     'ResponseError',
     'SSLConnection',