diff --git a/ChannelsDrawShareSite/__pycache__/settings.cpython-36.pyc b/ChannelsDrawShareSite/__pycache__/settings.cpython-36.pyc index 2b1d6fd..847175a 100644 Binary files a/ChannelsDrawShareSite/__pycache__/settings.cpython-36.pyc and b/ChannelsDrawShareSite/__pycache__/settings.cpython-36.pyc differ diff --git a/ChannelsDrawShareSite/settings.py b/ChannelsDrawShareSite/settings.py index 8938113..9f11d03 100644 --- a/ChannelsDrawShareSite/settings.py +++ b/ChannelsDrawShareSite/settings.py @@ -77,12 +77,18 @@ CHANNEL_LAYERS = { 'default': { 'BACKEND': 'channels_redis.core.RedisChannelLayer', 'CONFIG': { - # "hosts": [('127.0.0.1', 6379)], - "hosts": [os.environ.get('REDIS_URL', 'redis://localhost:6379')], + "hosts": [('127.0.0.1', 6379)], }, }, } +CACHES = { + "default": { + "BACKEND": "redis_cache.RedisCache", + "LOCATION": os.environ.get('REDIS_URL'), + } +} + # Database # https://docs.djangoproject.com/en/2.1/ref/settings/#databases diff --git a/drawshare/__pycache__/routing.cpython-36.pyc b/drawshare/__pycache__/routing.cpython-36.pyc index b8c9861..ac5b10c 100644 Binary files a/drawshare/__pycache__/routing.cpython-36.pyc and b/drawshare/__pycache__/routing.cpython-36.pyc differ diff --git a/drawshare/templates/drawshare/room.html b/drawshare/templates/drawshare/room.html index ad6e2b5..0753945 100644 --- a/drawshare/templates/drawshare/room.html +++ b/drawshare/templates/drawshare/room.html @@ -4,6 +4,7 @@ DrawShare Room + + \ No newline at end of file diff --git a/requirements.txt b/requirements.txt index 58abcd2..2fe3ec8 100644 --- a/requirements.txt +++ b/requirements.txt @@ -17,6 +17,7 @@ incremental==17.5.0 msgpack==0.5.6 PyHamcrest==1.9.0 pytz==2018.5 +redis==2.10.6 selenium==3.14.1 six==1.11.0 Twisted==18.9.0 diff --git a/server.key b/server.key new file mode 100644 index 0000000..68bf8b0 --- /dev/null +++ b/server.key @@ -0,0 +1,15 @@ +-----BEGIN RSA PRIVATE KEY----- +MIICXgIBAAKBgQDmHNjYTNLxOfkMQlHg45ESmylF6z3Z7TSI6hjAhS2WxkEH3y3K +Pum78sslz/k/hATpdh2YSTkrs0ojLkK5pejJno903rS5lP8ExGEdSHWAsFIhTBVv +fHg93JdgSGy3Sygt+H3IOxIkPi5vIXjpw3VTZ7EVZoIIWSF27uw7uWTKZwIDAQAB +AoGBAKs5Gc1Q1ME0DvGlQ4GgMylyFPL2yM4op6ec8RAHyNVg7bCqy0qrJ4Z3cdvP +9bniRTlmbz0KdyTiQq8M1A+JuT0rilcEaP+F9Z4nLx6dCmKmsN88j2HBZQDoG/gV +O2uFHmZF3tGxgs+hrZClaPmCL1M2HcTeCxo7GIgy9iwmMpuhAkEA+cVs2ON8U+GK +iwKxw5I9XxE/4EKEKxtLp+MX7IWLO/JTlujFLsacpE219YgNMgNAgQD9Mce+pyK6 +fBPAdovEMQJBAOvZ602YkHDYKQX2cN93Ky8s08O4mIFoxvwaXIWb5f/x23jA7XsM +w5DCFE5VWzyMglKvIkqj1A9Jw0f1iwYgShcCQQCWAbwdhoJk3lAWrMeWbX3uWq3C +QjCeswX9DqaPpqS4nBEX0TSboyzwgLuHeu5x2wIieDWYcB5Qwsq9Oh+dEtQBAkAK +VP6Y5KEXQHDzoOsq7vaGV4ljXpfXu3ZUHveEpuK5hqfdr1338QQ0ODxZfiXEDke7 +RY7UBD9K+ClE4r3XY9y7AkEAp1YQ5P59G61HsYIbOIz0+0Dy9lKzmQT4EIo3MNzC +SKGkYER8bgzC7ZDEIkMXywRQFI3yrJ9IjxjXbLEo2ES6VA== +-----END RSA PRIVATE KEY----- diff --git a/venv/lib/python3.6/site-packages/redis-2.10.6.dist-info/DESCRIPTION.rst b/venv/lib/python3.6/site-packages/redis-2.10.6.dist-info/DESCRIPTION.rst new file mode 100644 index 0000000..bf785a3 --- /dev/null +++ b/venv/lib/python3.6/site-packages/redis-2.10.6.dist-info/DESCRIPTION.rst @@ -0,0 +1,691 @@ +redis-py +======== + +The Python interface to the Redis key-value store. + +.. image:: https://secure.travis-ci.org/andymccurdy/redis-py.png?branch=master + :target: http://travis-ci.org/andymccurdy/redis-py + +Installation +------------ + +redis-py requires a running Redis server. See `Redis's quickstart +`_ for installation instructions. + +To install redis-py, simply: + +.. code-block:: bash + + $ sudo pip install redis + +or alternatively (you really should be using pip though): + +.. code-block:: bash + + $ sudo easy_install redis + +or from source: + +.. code-block:: bash + + $ sudo python setup.py install + + +Getting Started +--------------- + +.. code-block:: pycon + + >>> import redis + >>> r = redis.StrictRedis(host='localhost', port=6379, db=0) + >>> r.set('foo', 'bar') + True + >>> r.get('foo') + 'bar' + +API Reference +------------- + +The `official Redis command documentation `_ does a +great job of explaining each command in detail. redis-py exposes two client +classes that implement these commands. The StrictRedis class attempts to adhere +to the official command syntax. There are a few exceptions: + +* **SELECT**: Not implemented. See the explanation in the Thread Safety section + below. +* **DEL**: 'del' is a reserved keyword in the Python syntax. Therefore redis-py + uses 'delete' instead. +* **CONFIG GET|SET**: These are implemented separately as config_get or config_set. +* **MULTI/EXEC**: These are implemented as part of the Pipeline class. The + pipeline is wrapped with the MULTI and EXEC statements by default when it + is executed, which can be disabled by specifying transaction=False. + See more about Pipelines below. +* **SUBSCRIBE/LISTEN**: Similar to pipelines, PubSub is implemented as a separate + class as it places the underlying connection in a state where it can't + execute non-pubsub commands. Calling the pubsub method from the Redis client + will return a PubSub instance where you can subscribe to channels and listen + for messages. You can only call PUBLISH from the Redis client (see + `this comment on issue #151 + `_ + for details). +* **SCAN/SSCAN/HSCAN/ZSCAN**: The \*SCAN commands are implemented as they + exist in the Redis documentation. In addition, each command has an equivilant + iterator method. These are purely for convenience so the user doesn't have + to keep track of the cursor while iterating. Use the + scan_iter/sscan_iter/hscan_iter/zscan_iter methods for this behavior. + +In addition to the changes above, the Redis class, a subclass of StrictRedis, +overrides several other commands to provide backwards compatibility with older +versions of redis-py: + +* **LREM**: Order of 'num' and 'value' arguments reversed such that 'num' can + provide a default value of zero. +* **ZADD**: Redis specifies the 'score' argument before 'value'. These were swapped + accidentally when being implemented and not discovered until after people + were already using it. The Redis class expects \*args in the form of: + `name1, score1, name2, score2, ...` +* **SETEX**: Order of 'time' and 'value' arguments reversed. + + +More Detail +----------- + +Connection Pools +^^^^^^^^^^^^^^^^ + +Behind the scenes, redis-py uses a connection pool to manage connections to +a Redis server. By default, each Redis instance you create will in turn create +its own connection pool. You can override this behavior and use an existing +connection pool by passing an already created connection pool instance to the +connection_pool argument of the Redis class. You may choose to do this in order +to implement client side sharding or have finer grain control of how +connections are managed. + +.. code-block:: pycon + + >>> pool = redis.ConnectionPool(host='localhost', port=6379, db=0) + >>> r = redis.Redis(connection_pool=pool) + +Connections +^^^^^^^^^^^ + +ConnectionPools manage a set of Connection instances. redis-py ships with two +types of Connections. The default, Connection, is a normal TCP socket based +connection. The UnixDomainSocketConnection allows for clients running on the +same device as the server to connect via a unix domain socket. To use a +UnixDomainSocketConnection connection, simply pass the unix_socket_path +argument, which is a string to the unix domain socket file. Additionally, make +sure the unixsocket parameter is defined in your redis.conf file. It's +commented out by default. + +.. code-block:: pycon + + >>> r = redis.Redis(unix_socket_path='/tmp/redis.sock') + +You can create your own Connection subclasses as well. This may be useful if +you want to control the socket behavior within an async framework. To +instantiate a client class using your own connection, you need to create +a connection pool, passing your class to the connection_class argument. +Other keyword parameters you pass to the pool will be passed to the class +specified during initialization. + +.. code-block:: pycon + + >>> pool = redis.ConnectionPool(connection_class=YourConnectionClass, + your_arg='...', ...) + +Parsers +^^^^^^^ + +Parser classes provide a way to control how responses from the Redis server +are parsed. redis-py ships with two parser classes, the PythonParser and the +HiredisParser. By default, redis-py will attempt to use the HiredisParser if +you have the hiredis module installed and will fallback to the PythonParser +otherwise. + +Hiredis is a C library maintained by the core Redis team. Pieter Noordhuis was +kind enough to create Python bindings. Using Hiredis can provide up to a +10x speed improvement in parsing responses from the Redis server. The +performance increase is most noticeable when retrieving many pieces of data, +such as from LRANGE or SMEMBERS operations. + +Hiredis is available on PyPI, and can be installed via pip or easy_install +just like redis-py. + +.. code-block:: bash + + $ pip install hiredis + +or + +.. code-block:: bash + + $ easy_install hiredis + +Response Callbacks +^^^^^^^^^^^^^^^^^^ + +The client class uses a set of callbacks to cast Redis responses to the +appropriate Python type. There are a number of these callbacks defined on +the Redis client class in a dictionary called RESPONSE_CALLBACKS. + +Custom callbacks can be added on a per-instance basis using the +set_response_callback method. This method accepts two arguments: a command +name and the callback. Callbacks added in this manner are only valid on the +instance the callback is added to. If you want to define or override a callback +globally, you should make a subclass of the Redis client and add your callback +to its RESPONSE_CALLBACKS class dictionary. + +Response callbacks take at least one parameter: the response from the Redis +server. Keyword arguments may also be accepted in order to further control +how to interpret the response. These keyword arguments are specified during the +command's call to execute_command. The ZRANGE implementation demonstrates the +use of response callback keyword arguments with its "withscores" argument. + +Thread Safety +^^^^^^^^^^^^^ + +Redis client instances can safely be shared between threads. Internally, +connection instances are only retrieved from the connection pool during +command execution, and returned to the pool directly after. Command execution +never modifies state on the client instance. + +However, there is one caveat: the Redis SELECT command. The SELECT command +allows you to switch the database currently in use by the connection. That +database remains selected until another is selected or until the connection is +closed. This creates an issue in that connections could be returned to the pool +that are connected to a different database. + +As a result, redis-py does not implement the SELECT command on client +instances. If you use multiple Redis databases within the same application, you +should create a separate client instance (and possibly a separate connection +pool) for each database. + +It is not safe to pass PubSub or Pipeline objects between threads. + +Pipelines +^^^^^^^^^ + +Pipelines are a subclass of the base Redis class that provide support for +buffering multiple commands to the server in a single request. They can be used +to dramatically increase the performance of groups of commands by reducing the +number of back-and-forth TCP packets between the client and server. + +Pipelines are quite simple to use: + +.. code-block:: pycon + + >>> r = redis.Redis(...) + >>> r.set('bing', 'baz') + >>> # Use the pipeline() method to create a pipeline instance + >>> pipe = r.pipeline() + >>> # The following SET commands are buffered + >>> pipe.set('foo', 'bar') + >>> pipe.get('bing') + >>> # the EXECUTE call sends all buffered commands to the server, returning + >>> # a list of responses, one for each command. + >>> pipe.execute() + [True, 'baz'] + +For ease of use, all commands being buffered into the pipeline return the +pipeline object itself. Therefore calls can be chained like: + +.. code-block:: pycon + + >>> pipe.set('foo', 'bar').sadd('faz', 'baz').incr('auto_number').execute() + [True, True, 6] + +In addition, pipelines can also ensure the buffered commands are executed +atomically as a group. This happens by default. If you want to disable the +atomic nature of a pipeline but still want to buffer commands, you can turn +off transactions. + +.. code-block:: pycon + + >>> pipe = r.pipeline(transaction=False) + +A common issue occurs when requiring atomic transactions but needing to +retrieve values in Redis prior for use within the transaction. For instance, +let's assume that the INCR command didn't exist and we need to build an atomic +version of INCR in Python. + +The completely naive implementation could GET the value, increment it in +Python, and SET the new value back. However, this is not atomic because +multiple clients could be doing this at the same time, each getting the same +value from GET. + +Enter the WATCH command. WATCH provides the ability to monitor one or more keys +prior to starting a transaction. If any of those keys change prior the +execution of that transaction, the entire transaction will be canceled and a +WatchError will be raised. To implement our own client-side INCR command, we +could do something like this: + +.. code-block:: pycon + + >>> with r.pipeline() as pipe: + ... while 1: + ... try: + ... # put a WATCH on the key that holds our sequence value + ... pipe.watch('OUR-SEQUENCE-KEY') + ... # after WATCHing, the pipeline is put into immediate execution + ... # mode until we tell it to start buffering commands again. + ... # this allows us to get the current value of our sequence + ... current_value = pipe.get('OUR-SEQUENCE-KEY') + ... next_value = int(current_value) + 1 + ... # now we can put the pipeline back into buffered mode with MULTI + ... pipe.multi() + ... pipe.set('OUR-SEQUENCE-KEY', next_value) + ... # and finally, execute the pipeline (the set command) + ... pipe.execute() + ... # if a WatchError wasn't raised during execution, everything + ... # we just did happened atomically. + ... break + ... except WatchError: + ... # another client must have changed 'OUR-SEQUENCE-KEY' between + ... # the time we started WATCHing it and the pipeline's execution. + ... # our best bet is to just retry. + ... continue + +Note that, because the Pipeline must bind to a single connection for the +duration of a WATCH, care must be taken to ensure that the connection is +returned to the connection pool by calling the reset() method. If the +Pipeline is used as a context manager (as in the example above) reset() +will be called automatically. Of course you can do this the manual way by +explicitly calling reset(): + +.. code-block:: pycon + + >>> pipe = r.pipeline() + >>> while 1: + ... try: + ... pipe.watch('OUR-SEQUENCE-KEY') + ... ... + ... pipe.execute() + ... break + ... except WatchError: + ... continue + ... finally: + ... pipe.reset() + +A convenience method named "transaction" exists for handling all the +boilerplate of handling and retrying watch errors. It takes a callable that +should expect a single parameter, a pipeline object, and any number of keys to +be WATCHed. Our client-side INCR command above can be written like this, +which is much easier to read: + +.. code-block:: pycon + + >>> def client_side_incr(pipe): + ... current_value = pipe.get('OUR-SEQUENCE-KEY') + ... next_value = int(current_value) + 1 + ... pipe.multi() + ... pipe.set('OUR-SEQUENCE-KEY', next_value) + >>> + >>> r.transaction(client_side_incr, 'OUR-SEQUENCE-KEY') + [True] + +Publish / Subscribe +^^^^^^^^^^^^^^^^^^^ + +redis-py includes a `PubSub` object that subscribes to channels and listens +for new messages. Creating a `PubSub` object is easy. + +.. code-block:: pycon + + >>> r = redis.StrictRedis(...) + >>> p = r.pubsub() + +Once a `PubSub` instance is created, channels and patterns can be subscribed +to. + +.. code-block:: pycon + + >>> p.subscribe('my-first-channel', 'my-second-channel', ...) + >>> p.psubscribe('my-*', ...) + +The `PubSub` instance is now subscribed to those channels/patterns. The +subscription confirmations can be seen by reading messages from the `PubSub` +instance. + +.. code-block:: pycon + + >>> p.get_message() + {'pattern': None, 'type': 'subscribe', 'channel': 'my-second-channel', 'data': 1L} + >>> p.get_message() + {'pattern': None, 'type': 'subscribe', 'channel': 'my-first-channel', 'data': 2L} + >>> p.get_message() + {'pattern': None, 'type': 'psubscribe', 'channel': 'my-*', 'data': 3L} + +Every message read from a `PubSub` instance will be a dictionary with the +following keys. + +* **type**: One of the following: 'subscribe', 'unsubscribe', 'psubscribe', + 'punsubscribe', 'message', 'pmessage' +* **channel**: The channel [un]subscribed to or the channel a message was + published to +* **pattern**: The pattern that matched a published message's channel. Will be + `None` in all cases except for 'pmessage' types. +* **data**: The message data. With [un]subscribe messages, this value will be + the number of channels and patterns the connection is currently subscribed + to. With [p]message messages, this value will be the actual published + message. + +Let's send a message now. + +.. code-block:: pycon + + # the publish method returns the number matching channel and pattern + # subscriptions. 'my-first-channel' matches both the 'my-first-channel' + # subscription and the 'my-*' pattern subscription, so this message will + # be delivered to 2 channels/patterns + >>> r.publish('my-first-channel', 'some data') + 2 + >>> p.get_message() + {'channel': 'my-first-channel', 'data': 'some data', 'pattern': None, 'type': 'message'} + >>> p.get_message() + {'channel': 'my-first-channel', 'data': 'some data', 'pattern': 'my-*', 'type': 'pmessage'} + +Unsubscribing works just like subscribing. If no arguments are passed to +[p]unsubscribe, all channels or patterns will be unsubscribed from. + +.. code-block:: pycon + + >>> p.unsubscribe() + >>> p.punsubscribe('my-*') + >>> p.get_message() + {'channel': 'my-second-channel', 'data': 2L, 'pattern': None, 'type': 'unsubscribe'} + >>> p.get_message() + {'channel': 'my-first-channel', 'data': 1L, 'pattern': None, 'type': 'unsubscribe'} + >>> p.get_message() + {'channel': 'my-*', 'data': 0L, 'pattern': None, 'type': 'punsubscribe'} + +redis-py also allows you to register callback functions to handle published +messages. Message handlers take a single argument, the message, which is a +dictionary just like the examples above. To subscribe to a channel or pattern +with a message handler, pass the channel or pattern name as a keyword argument +with its value being the callback function. + +When a message is read on a channel or pattern with a message handler, the +message dictionary is created and passed to the message handler. In this case, +a `None` value is returned from get_message() since the message was already +handled. + +.. code-block:: pycon + + >>> def my_handler(message): + ... print 'MY HANDLER: ', message['data'] + >>> p.subscribe(**{'my-channel': my_handler}) + # read the subscribe confirmation message + >>> p.get_message() + {'pattern': None, 'type': 'subscribe', 'channel': 'my-channel', 'data': 1L} + >>> r.publish('my-channel', 'awesome data') + 1 + # for the message handler to work, we need tell the instance to read data. + # this can be done in several ways (read more below). we'll just use + # the familiar get_message() function for now + >>> message = p.get_message() + MY HANDLER: awesome data + # note here that the my_handler callback printed the string above. + # `message` is None because the message was handled by our handler. + >>> print message + None + +If your application is not interested in the (sometimes noisy) +subscribe/unsubscribe confirmation messages, you can ignore them by passing +`ignore_subscribe_messages=True` to `r.pubsub()`. This will cause all +subscribe/unsubscribe messages to be read, but they won't bubble up to your +application. + +.. code-block:: pycon + + >>> p = r.pubsub(ignore_subscribe_messages=True) + >>> p.subscribe('my-channel') + >>> p.get_message() # hides the subscribe message and returns None + >>> r.publish('my-channel') + 1 + >>> p.get_message() + {'channel': 'my-channel', 'data': 'my data', 'pattern': None, 'type': 'message'} + +There are three different strategies for reading messages. + +The examples above have been using `pubsub.get_message()`. Behind the scenes, +`get_message()` uses the system's 'select' module to quickly poll the +connection's socket. If there's data available to be read, `get_message()` will +read it, format the message and return it or pass it to a message handler. If +there's no data to be read, `get_message()` will immediately return None. This +makes it trivial to integrate into an existing event loop inside your +application. + +.. code-block:: pycon + + >>> while True: + >>> message = p.get_message() + >>> if message: + >>> # do something with the message + >>> time.sleep(0.001) # be nice to the system :) + +Older versions of redis-py only read messages with `pubsub.listen()`. listen() +is a generator that blocks until a message is available. If your application +doesn't need to do anything else but receive and act on messages received from +redis, listen() is an easy way to get up an running. + +.. code-block:: pycon + + >>> for message in p.listen(): + ... # do something with the message + +The third option runs an event loop in a separate thread. +`pubsub.run_in_thread()` creates a new thread and starts the event loop. The +thread object is returned to the caller of `run_in_thread()`. The caller can +use the `thread.stop()` method to shut down the event loop and thread. Behind +the scenes, this is simply a wrapper around `get_message()` that runs in a +separate thread, essentially creating a tiny non-blocking event loop for you. +`run_in_thread()` takes an optional `sleep_time` argument. If specified, the +event loop will call `time.sleep()` with the value in each iteration of the +loop. + +Note: Since we're running in a separate thread, there's no way to handle +messages that aren't automatically handled with registered message handlers. +Therefore, redis-py prevents you from calling `run_in_thread()` if you're +subscribed to patterns or channels that don't have message handlers attached. + +.. code-block:: pycon + + >>> p.subscribe(**{'my-channel': my_handler}) + >>> thread = p.run_in_thread(sleep_time=0.001) + # the event loop is now running in the background processing messages + # when it's time to shut it down... + >>> thread.stop() + +A PubSub object adheres to the same encoding semantics as the client instance +it was created from. Any channel or pattern that's unicode will be encoded +using the `charset` specified on the client before being sent to Redis. If the +client's `decode_responses` flag is set the False (the default), the +'channel', 'pattern' and 'data' values in message dictionaries will be byte +strings (str on Python 2, bytes on Python 3). If the client's +`decode_responses` is True, then the 'channel', 'pattern' and 'data' values +will be automatically decoded to unicode strings using the client's `charset`. + +PubSub objects remember what channels and patterns they are subscribed to. In +the event of a disconnection such as a network error or timeout, the +PubSub object will re-subscribe to all prior channels and patterns when +reconnecting. Messages that were published while the client was disconnected +cannot be delivered. When you're finished with a PubSub object, call its +`.close()` method to shutdown the connection. + +.. code-block:: pycon + + >>> p = r.pubsub() + >>> ... + >>> p.close() + + +The PUBSUB set of subcommands CHANNELS, NUMSUB and NUMPAT are also +supported: + +.. code-block:: pycon + + >>> r.pubsub_channels() + ['foo', 'bar'] + >>> r.pubsub_numsub('foo', 'bar') + [('foo', 9001), ('bar', 42)] + >>> r.pubsub_numsub('baz') + [('baz', 0)] + >>> r.pubsub_numpat() + 1204 + + +LUA Scripting +^^^^^^^^^^^^^ + +redis-py supports the EVAL, EVALSHA, and SCRIPT commands. However, there are +a number of edge cases that make these commands tedious to use in real world +scenarios. Therefore, redis-py exposes a Script object that makes scripting +much easier to use. + +To create a Script instance, use the `register_script` function on a client +instance passing the LUA code as the first argument. `register_script` returns +a Script instance that you can use throughout your code. + +The following trivial LUA script accepts two parameters: the name of a key and +a multiplier value. The script fetches the value stored in the key, multiplies +it with the multiplier value and returns the result. + +.. code-block:: pycon + + >>> r = redis.StrictRedis() + >>> lua = """ + ... local value = redis.call('GET', KEYS[1]) + ... value = tonumber(value) + ... return value * ARGV[1]""" + >>> multiply = r.register_script(lua) + +`multiply` is now a Script instance that is invoked by calling it like a +function. Script instances accept the following optional arguments: + +* **keys**: A list of key names that the script will access. This becomes the + KEYS list in LUA. +* **args**: A list of argument values. This becomes the ARGV list in LUA. +* **client**: A redis-py Client or Pipeline instance that will invoke the + script. If client isn't specified, the client that intiially + created the Script instance (the one that `register_script` was + invoked from) will be used. + +Continuing the example from above: + +.. code-block:: pycon + + >>> r.set('foo', 2) + >>> multiply(keys=['foo'], args=[5]) + 10 + +The value of key 'foo' is set to 2. When multiply is invoked, the 'foo' key is +passed to the script along with the multiplier value of 5. LUA executes the +script and returns the result, 10. + +Script instances can be executed using a different client instance, even one +that points to a completely different Redis server. + +.. code-block:: pycon + + >>> r2 = redis.StrictRedis('redis2.example.com') + >>> r2.set('foo', 3) + >>> multiply(keys=['foo'], args=[5], client=r2) + 15 + +The Script object ensures that the LUA script is loaded into Redis's script +cache. In the event of a NOSCRIPT error, it will load the script and retry +executing it. + +Script objects can also be used in pipelines. The pipeline instance should be +passed as the client argument when calling the script. Care is taken to ensure +that the script is registered in Redis's script cache just prior to pipeline +execution. + +.. code-block:: pycon + + >>> pipe = r.pipeline() + >>> pipe.set('foo', 5) + >>> multiply(keys=['foo'], args=[5], client=pipe) + >>> pipe.execute() + [True, 25] + +Sentinel support +^^^^^^^^^^^^^^^^ + +redis-py can be used together with `Redis Sentinel `_ +to discover Redis nodes. You need to have at least one Sentinel daemon running +in order to use redis-py's Sentinel support. + +Connecting redis-py to the Sentinel instance(s) is easy. You can use a +Sentinel connection to discover the master and slaves network addresses: + +.. code-block:: pycon + + >>> from redis.sentinel import Sentinel + >>> sentinel = Sentinel([('localhost', 26379)], socket_timeout=0.1) + >>> sentinel.discover_master('mymaster') + ('127.0.0.1', 6379) + >>> sentinel.discover_slaves('mymaster') + [('127.0.0.1', 6380)] + +You can also create Redis client connections from a Sentinel instance. You can +connect to either the master (for write operations) or a slave (for read-only +operations). + +.. code-block:: pycon + + >>> master = sentinel.master_for('mymaster', socket_timeout=0.1) + >>> slave = sentinel.slave_for('mymaster', socket_timeout=0.1) + >>> master.set('foo', 'bar') + >>> slave.get('foo') + 'bar' + +The master and slave objects are normal StrictRedis instances with their +connection pool bound to the Sentinel instance. When a Sentinel backed client +attempts to establish a connection, it first queries the Sentinel servers to +determine an appropriate host to connect to. If no server is found, +a MasterNotFoundError or SlaveNotFoundError is raised. Both exceptions are +subclasses of ConnectionError. + +When trying to connect to a slave client, the Sentinel connection pool will +iterate over the list of slaves until it finds one that can be connected to. +If no slaves can be connected to, a connection will be established with the +master. + +See `Guidelines for Redis clients with support for Redis Sentinel +`_ to learn more about Redis Sentinel. + +Scan Iterators +^^^^^^^^^^^^^^ + +The \*SCAN commands introduced in Redis 2.8 can be cumbersome to use. While +these commands are fully supported, redis-py also exposes the following methods +that return Python iterators for convenience: `scan_iter`, `hscan_iter`, +`sscan_iter` and `zscan_iter`. + +.. code-block:: pycon + + >>> for key, value in (('A', '1'), ('B', '2'), ('C', '3')): + ... r.set(key, value) + >>> for key in r.scan_iter(): + ... print key, r.get(key) + A 1 + B 2 + C 3 + +Author +^^^^^^ + +redis-py is developed and maintained by Andy McCurdy (sedrik@gmail.com). +It can be found here: http://github.com/andymccurdy/redis-py + +Special thanks to: + +* Ludovico Magnocavallo, author of the original Python Redis client, from + which some of the socket code is still used. +* Alexander Solovyov for ideas on the generic response callback system. +* Paul Hubbard for initial packaging support. + + + diff --git a/venv/lib/python3.6/site-packages/redis-2.10.6.dist-info/INSTALLER b/venv/lib/python3.6/site-packages/redis-2.10.6.dist-info/INSTALLER new file mode 100644 index 0000000..a1b589e --- /dev/null +++ b/venv/lib/python3.6/site-packages/redis-2.10.6.dist-info/INSTALLER @@ -0,0 +1 @@ +pip diff --git a/venv/lib/python3.6/site-packages/redis-2.10.6.dist-info/METADATA b/venv/lib/python3.6/site-packages/redis-2.10.6.dist-info/METADATA new file mode 100644 index 0000000..9b5fa65 --- /dev/null +++ b/venv/lib/python3.6/site-packages/redis-2.10.6.dist-info/METADATA @@ -0,0 +1,716 @@ +Metadata-Version: 2.0 +Name: redis +Version: 2.10.6 +Summary: Python client for Redis key-value store +Home-page: http://github.com/andymccurdy/redis-py +Author: Andy McCurdy +Author-email: sedrik@gmail.com +License: MIT +Keywords: Redis,key-value store +Platform: UNKNOWN +Classifier: Development Status :: 5 - Production/Stable +Classifier: Environment :: Console +Classifier: Intended Audience :: Developers +Classifier: License :: OSI Approved :: MIT License +Classifier: Operating System :: OS Independent +Classifier: Programming Language :: Python +Classifier: Programming Language :: Python :: 2 +Classifier: Programming Language :: Python :: 2.6 +Classifier: Programming Language :: Python :: 2.7 +Classifier: Programming Language :: Python :: 3 +Classifier: Programming Language :: Python :: 3.3 +Classifier: Programming Language :: Python :: 3.4 +Classifier: Programming Language :: Python :: 3.5 +Classifier: Programming Language :: Python :: 3.6 + +redis-py +======== + +The Python interface to the Redis key-value store. + +.. image:: https://secure.travis-ci.org/andymccurdy/redis-py.png?branch=master + :target: http://travis-ci.org/andymccurdy/redis-py + +Installation +------------ + +redis-py requires a running Redis server. See `Redis's quickstart +`_ for installation instructions. + +To install redis-py, simply: + +.. code-block:: bash + + $ sudo pip install redis + +or alternatively (you really should be using pip though): + +.. code-block:: bash + + $ sudo easy_install redis + +or from source: + +.. code-block:: bash + + $ sudo python setup.py install + + +Getting Started +--------------- + +.. code-block:: pycon + + >>> import redis + >>> r = redis.StrictRedis(host='localhost', port=6379, db=0) + >>> r.set('foo', 'bar') + True + >>> r.get('foo') + 'bar' + +API Reference +------------- + +The `official Redis command documentation `_ does a +great job of explaining each command in detail. redis-py exposes two client +classes that implement these commands. The StrictRedis class attempts to adhere +to the official command syntax. There are a few exceptions: + +* **SELECT**: Not implemented. See the explanation in the Thread Safety section + below. +* **DEL**: 'del' is a reserved keyword in the Python syntax. Therefore redis-py + uses 'delete' instead. +* **CONFIG GET|SET**: These are implemented separately as config_get or config_set. +* **MULTI/EXEC**: These are implemented as part of the Pipeline class. The + pipeline is wrapped with the MULTI and EXEC statements by default when it + is executed, which can be disabled by specifying transaction=False. + See more about Pipelines below. +* **SUBSCRIBE/LISTEN**: Similar to pipelines, PubSub is implemented as a separate + class as it places the underlying connection in a state where it can't + execute non-pubsub commands. Calling the pubsub method from the Redis client + will return a PubSub instance where you can subscribe to channels and listen + for messages. You can only call PUBLISH from the Redis client (see + `this comment on issue #151 + `_ + for details). +* **SCAN/SSCAN/HSCAN/ZSCAN**: The \*SCAN commands are implemented as they + exist in the Redis documentation. In addition, each command has an equivilant + iterator method. These are purely for convenience so the user doesn't have + to keep track of the cursor while iterating. Use the + scan_iter/sscan_iter/hscan_iter/zscan_iter methods for this behavior. + +In addition to the changes above, the Redis class, a subclass of StrictRedis, +overrides several other commands to provide backwards compatibility with older +versions of redis-py: + +* **LREM**: Order of 'num' and 'value' arguments reversed such that 'num' can + provide a default value of zero. +* **ZADD**: Redis specifies the 'score' argument before 'value'. These were swapped + accidentally when being implemented and not discovered until after people + were already using it. The Redis class expects \*args in the form of: + `name1, score1, name2, score2, ...` +* **SETEX**: Order of 'time' and 'value' arguments reversed. + + +More Detail +----------- + +Connection Pools +^^^^^^^^^^^^^^^^ + +Behind the scenes, redis-py uses a connection pool to manage connections to +a Redis server. By default, each Redis instance you create will in turn create +its own connection pool. You can override this behavior and use an existing +connection pool by passing an already created connection pool instance to the +connection_pool argument of the Redis class. You may choose to do this in order +to implement client side sharding or have finer grain control of how +connections are managed. + +.. code-block:: pycon + + >>> pool = redis.ConnectionPool(host='localhost', port=6379, db=0) + >>> r = redis.Redis(connection_pool=pool) + +Connections +^^^^^^^^^^^ + +ConnectionPools manage a set of Connection instances. redis-py ships with two +types of Connections. The default, Connection, is a normal TCP socket based +connection. The UnixDomainSocketConnection allows for clients running on the +same device as the server to connect via a unix domain socket. To use a +UnixDomainSocketConnection connection, simply pass the unix_socket_path +argument, which is a string to the unix domain socket file. Additionally, make +sure the unixsocket parameter is defined in your redis.conf file. It's +commented out by default. + +.. code-block:: pycon + + >>> r = redis.Redis(unix_socket_path='/tmp/redis.sock') + +You can create your own Connection subclasses as well. This may be useful if +you want to control the socket behavior within an async framework. To +instantiate a client class using your own connection, you need to create +a connection pool, passing your class to the connection_class argument. +Other keyword parameters you pass to the pool will be passed to the class +specified during initialization. + +.. code-block:: pycon + + >>> pool = redis.ConnectionPool(connection_class=YourConnectionClass, + your_arg='...', ...) + +Parsers +^^^^^^^ + +Parser classes provide a way to control how responses from the Redis server +are parsed. redis-py ships with two parser classes, the PythonParser and the +HiredisParser. By default, redis-py will attempt to use the HiredisParser if +you have the hiredis module installed and will fallback to the PythonParser +otherwise. + +Hiredis is a C library maintained by the core Redis team. Pieter Noordhuis was +kind enough to create Python bindings. Using Hiredis can provide up to a +10x speed improvement in parsing responses from the Redis server. The +performance increase is most noticeable when retrieving many pieces of data, +such as from LRANGE or SMEMBERS operations. + +Hiredis is available on PyPI, and can be installed via pip or easy_install +just like redis-py. + +.. code-block:: bash + + $ pip install hiredis + +or + +.. code-block:: bash + + $ easy_install hiredis + +Response Callbacks +^^^^^^^^^^^^^^^^^^ + +The client class uses a set of callbacks to cast Redis responses to the +appropriate Python type. There are a number of these callbacks defined on +the Redis client class in a dictionary called RESPONSE_CALLBACKS. + +Custom callbacks can be added on a per-instance basis using the +set_response_callback method. This method accepts two arguments: a command +name and the callback. Callbacks added in this manner are only valid on the +instance the callback is added to. If you want to define or override a callback +globally, you should make a subclass of the Redis client and add your callback +to its RESPONSE_CALLBACKS class dictionary. + +Response callbacks take at least one parameter: the response from the Redis +server. Keyword arguments may also be accepted in order to further control +how to interpret the response. These keyword arguments are specified during the +command's call to execute_command. The ZRANGE implementation demonstrates the +use of response callback keyword arguments with its "withscores" argument. + +Thread Safety +^^^^^^^^^^^^^ + +Redis client instances can safely be shared between threads. Internally, +connection instances are only retrieved from the connection pool during +command execution, and returned to the pool directly after. Command execution +never modifies state on the client instance. + +However, there is one caveat: the Redis SELECT command. The SELECT command +allows you to switch the database currently in use by the connection. That +database remains selected until another is selected or until the connection is +closed. This creates an issue in that connections could be returned to the pool +that are connected to a different database. + +As a result, redis-py does not implement the SELECT command on client +instances. If you use multiple Redis databases within the same application, you +should create a separate client instance (and possibly a separate connection +pool) for each database. + +It is not safe to pass PubSub or Pipeline objects between threads. + +Pipelines +^^^^^^^^^ + +Pipelines are a subclass of the base Redis class that provide support for +buffering multiple commands to the server in a single request. They can be used +to dramatically increase the performance of groups of commands by reducing the +number of back-and-forth TCP packets between the client and server. + +Pipelines are quite simple to use: + +.. code-block:: pycon + + >>> r = redis.Redis(...) + >>> r.set('bing', 'baz') + >>> # Use the pipeline() method to create a pipeline instance + >>> pipe = r.pipeline() + >>> # The following SET commands are buffered + >>> pipe.set('foo', 'bar') + >>> pipe.get('bing') + >>> # the EXECUTE call sends all buffered commands to the server, returning + >>> # a list of responses, one for each command. + >>> pipe.execute() + [True, 'baz'] + +For ease of use, all commands being buffered into the pipeline return the +pipeline object itself. Therefore calls can be chained like: + +.. code-block:: pycon + + >>> pipe.set('foo', 'bar').sadd('faz', 'baz').incr('auto_number').execute() + [True, True, 6] + +In addition, pipelines can also ensure the buffered commands are executed +atomically as a group. This happens by default. If you want to disable the +atomic nature of a pipeline but still want to buffer commands, you can turn +off transactions. + +.. code-block:: pycon + + >>> pipe = r.pipeline(transaction=False) + +A common issue occurs when requiring atomic transactions but needing to +retrieve values in Redis prior for use within the transaction. For instance, +let's assume that the INCR command didn't exist and we need to build an atomic +version of INCR in Python. + +The completely naive implementation could GET the value, increment it in +Python, and SET the new value back. However, this is not atomic because +multiple clients could be doing this at the same time, each getting the same +value from GET. + +Enter the WATCH command. WATCH provides the ability to monitor one or more keys +prior to starting a transaction. If any of those keys change prior the +execution of that transaction, the entire transaction will be canceled and a +WatchError will be raised. To implement our own client-side INCR command, we +could do something like this: + +.. code-block:: pycon + + >>> with r.pipeline() as pipe: + ... while 1: + ... try: + ... # put a WATCH on the key that holds our sequence value + ... pipe.watch('OUR-SEQUENCE-KEY') + ... # after WATCHing, the pipeline is put into immediate execution + ... # mode until we tell it to start buffering commands again. + ... # this allows us to get the current value of our sequence + ... current_value = pipe.get('OUR-SEQUENCE-KEY') + ... next_value = int(current_value) + 1 + ... # now we can put the pipeline back into buffered mode with MULTI + ... pipe.multi() + ... pipe.set('OUR-SEQUENCE-KEY', next_value) + ... # and finally, execute the pipeline (the set command) + ... pipe.execute() + ... # if a WatchError wasn't raised during execution, everything + ... # we just did happened atomically. + ... break + ... except WatchError: + ... # another client must have changed 'OUR-SEQUENCE-KEY' between + ... # the time we started WATCHing it and the pipeline's execution. + ... # our best bet is to just retry. + ... continue + +Note that, because the Pipeline must bind to a single connection for the +duration of a WATCH, care must be taken to ensure that the connection is +returned to the connection pool by calling the reset() method. If the +Pipeline is used as a context manager (as in the example above) reset() +will be called automatically. Of course you can do this the manual way by +explicitly calling reset(): + +.. code-block:: pycon + + >>> pipe = r.pipeline() + >>> while 1: + ... try: + ... pipe.watch('OUR-SEQUENCE-KEY') + ... ... + ... pipe.execute() + ... break + ... except WatchError: + ... continue + ... finally: + ... pipe.reset() + +A convenience method named "transaction" exists for handling all the +boilerplate of handling and retrying watch errors. It takes a callable that +should expect a single parameter, a pipeline object, and any number of keys to +be WATCHed. Our client-side INCR command above can be written like this, +which is much easier to read: + +.. code-block:: pycon + + >>> def client_side_incr(pipe): + ... current_value = pipe.get('OUR-SEQUENCE-KEY') + ... next_value = int(current_value) + 1 + ... pipe.multi() + ... pipe.set('OUR-SEQUENCE-KEY', next_value) + >>> + >>> r.transaction(client_side_incr, 'OUR-SEQUENCE-KEY') + [True] + +Publish / Subscribe +^^^^^^^^^^^^^^^^^^^ + +redis-py includes a `PubSub` object that subscribes to channels and listens +for new messages. Creating a `PubSub` object is easy. + +.. code-block:: pycon + + >>> r = redis.StrictRedis(...) + >>> p = r.pubsub() + +Once a `PubSub` instance is created, channels and patterns can be subscribed +to. + +.. code-block:: pycon + + >>> p.subscribe('my-first-channel', 'my-second-channel', ...) + >>> p.psubscribe('my-*', ...) + +The `PubSub` instance is now subscribed to those channels/patterns. The +subscription confirmations can be seen by reading messages from the `PubSub` +instance. + +.. code-block:: pycon + + >>> p.get_message() + {'pattern': None, 'type': 'subscribe', 'channel': 'my-second-channel', 'data': 1L} + >>> p.get_message() + {'pattern': None, 'type': 'subscribe', 'channel': 'my-first-channel', 'data': 2L} + >>> p.get_message() + {'pattern': None, 'type': 'psubscribe', 'channel': 'my-*', 'data': 3L} + +Every message read from a `PubSub` instance will be a dictionary with the +following keys. + +* **type**: One of the following: 'subscribe', 'unsubscribe', 'psubscribe', + 'punsubscribe', 'message', 'pmessage' +* **channel**: The channel [un]subscribed to or the channel a message was + published to +* **pattern**: The pattern that matched a published message's channel. Will be + `None` in all cases except for 'pmessage' types. +* **data**: The message data. With [un]subscribe messages, this value will be + the number of channels and patterns the connection is currently subscribed + to. With [p]message messages, this value will be the actual published + message. + +Let's send a message now. + +.. code-block:: pycon + + # the publish method returns the number matching channel and pattern + # subscriptions. 'my-first-channel' matches both the 'my-first-channel' + # subscription and the 'my-*' pattern subscription, so this message will + # be delivered to 2 channels/patterns + >>> r.publish('my-first-channel', 'some data') + 2 + >>> p.get_message() + {'channel': 'my-first-channel', 'data': 'some data', 'pattern': None, 'type': 'message'} + >>> p.get_message() + {'channel': 'my-first-channel', 'data': 'some data', 'pattern': 'my-*', 'type': 'pmessage'} + +Unsubscribing works just like subscribing. If no arguments are passed to +[p]unsubscribe, all channels or patterns will be unsubscribed from. + +.. code-block:: pycon + + >>> p.unsubscribe() + >>> p.punsubscribe('my-*') + >>> p.get_message() + {'channel': 'my-second-channel', 'data': 2L, 'pattern': None, 'type': 'unsubscribe'} + >>> p.get_message() + {'channel': 'my-first-channel', 'data': 1L, 'pattern': None, 'type': 'unsubscribe'} + >>> p.get_message() + {'channel': 'my-*', 'data': 0L, 'pattern': None, 'type': 'punsubscribe'} + +redis-py also allows you to register callback functions to handle published +messages. Message handlers take a single argument, the message, which is a +dictionary just like the examples above. To subscribe to a channel or pattern +with a message handler, pass the channel or pattern name as a keyword argument +with its value being the callback function. + +When a message is read on a channel or pattern with a message handler, the +message dictionary is created and passed to the message handler. In this case, +a `None` value is returned from get_message() since the message was already +handled. + +.. code-block:: pycon + + >>> def my_handler(message): + ... print 'MY HANDLER: ', message['data'] + >>> p.subscribe(**{'my-channel': my_handler}) + # read the subscribe confirmation message + >>> p.get_message() + {'pattern': None, 'type': 'subscribe', 'channel': 'my-channel', 'data': 1L} + >>> r.publish('my-channel', 'awesome data') + 1 + # for the message handler to work, we need tell the instance to read data. + # this can be done in several ways (read more below). we'll just use + # the familiar get_message() function for now + >>> message = p.get_message() + MY HANDLER: awesome data + # note here that the my_handler callback printed the string above. + # `message` is None because the message was handled by our handler. + >>> print message + None + +If your application is not interested in the (sometimes noisy) +subscribe/unsubscribe confirmation messages, you can ignore them by passing +`ignore_subscribe_messages=True` to `r.pubsub()`. This will cause all +subscribe/unsubscribe messages to be read, but they won't bubble up to your +application. + +.. code-block:: pycon + + >>> p = r.pubsub(ignore_subscribe_messages=True) + >>> p.subscribe('my-channel') + >>> p.get_message() # hides the subscribe message and returns None + >>> r.publish('my-channel') + 1 + >>> p.get_message() + {'channel': 'my-channel', 'data': 'my data', 'pattern': None, 'type': 'message'} + +There are three different strategies for reading messages. + +The examples above have been using `pubsub.get_message()`. Behind the scenes, +`get_message()` uses the system's 'select' module to quickly poll the +connection's socket. If there's data available to be read, `get_message()` will +read it, format the message and return it or pass it to a message handler. If +there's no data to be read, `get_message()` will immediately return None. This +makes it trivial to integrate into an existing event loop inside your +application. + +.. code-block:: pycon + + >>> while True: + >>> message = p.get_message() + >>> if message: + >>> # do something with the message + >>> time.sleep(0.001) # be nice to the system :) + +Older versions of redis-py only read messages with `pubsub.listen()`. listen() +is a generator that blocks until a message is available. If your application +doesn't need to do anything else but receive and act on messages received from +redis, listen() is an easy way to get up an running. + +.. code-block:: pycon + + >>> for message in p.listen(): + ... # do something with the message + +The third option runs an event loop in a separate thread. +`pubsub.run_in_thread()` creates a new thread and starts the event loop. The +thread object is returned to the caller of `run_in_thread()`. The caller can +use the `thread.stop()` method to shut down the event loop and thread. Behind +the scenes, this is simply a wrapper around `get_message()` that runs in a +separate thread, essentially creating a tiny non-blocking event loop for you. +`run_in_thread()` takes an optional `sleep_time` argument. If specified, the +event loop will call `time.sleep()` with the value in each iteration of the +loop. + +Note: Since we're running in a separate thread, there's no way to handle +messages that aren't automatically handled with registered message handlers. +Therefore, redis-py prevents you from calling `run_in_thread()` if you're +subscribed to patterns or channels that don't have message handlers attached. + +.. code-block:: pycon + + >>> p.subscribe(**{'my-channel': my_handler}) + >>> thread = p.run_in_thread(sleep_time=0.001) + # the event loop is now running in the background processing messages + # when it's time to shut it down... + >>> thread.stop() + +A PubSub object adheres to the same encoding semantics as the client instance +it was created from. Any channel or pattern that's unicode will be encoded +using the `charset` specified on the client before being sent to Redis. If the +client's `decode_responses` flag is set the False (the default), the +'channel', 'pattern' and 'data' values in message dictionaries will be byte +strings (str on Python 2, bytes on Python 3). If the client's +`decode_responses` is True, then the 'channel', 'pattern' and 'data' values +will be automatically decoded to unicode strings using the client's `charset`. + +PubSub objects remember what channels and patterns they are subscribed to. In +the event of a disconnection such as a network error or timeout, the +PubSub object will re-subscribe to all prior channels and patterns when +reconnecting. Messages that were published while the client was disconnected +cannot be delivered. When you're finished with a PubSub object, call its +`.close()` method to shutdown the connection. + +.. code-block:: pycon + + >>> p = r.pubsub() + >>> ... + >>> p.close() + + +The PUBSUB set of subcommands CHANNELS, NUMSUB and NUMPAT are also +supported: + +.. code-block:: pycon + + >>> r.pubsub_channels() + ['foo', 'bar'] + >>> r.pubsub_numsub('foo', 'bar') + [('foo', 9001), ('bar', 42)] + >>> r.pubsub_numsub('baz') + [('baz', 0)] + >>> r.pubsub_numpat() + 1204 + + +LUA Scripting +^^^^^^^^^^^^^ + +redis-py supports the EVAL, EVALSHA, and SCRIPT commands. However, there are +a number of edge cases that make these commands tedious to use in real world +scenarios. Therefore, redis-py exposes a Script object that makes scripting +much easier to use. + +To create a Script instance, use the `register_script` function on a client +instance passing the LUA code as the first argument. `register_script` returns +a Script instance that you can use throughout your code. + +The following trivial LUA script accepts two parameters: the name of a key and +a multiplier value. The script fetches the value stored in the key, multiplies +it with the multiplier value and returns the result. + +.. code-block:: pycon + + >>> r = redis.StrictRedis() + >>> lua = """ + ... local value = redis.call('GET', KEYS[1]) + ... value = tonumber(value) + ... return value * ARGV[1]""" + >>> multiply = r.register_script(lua) + +`multiply` is now a Script instance that is invoked by calling it like a +function. Script instances accept the following optional arguments: + +* **keys**: A list of key names that the script will access. This becomes the + KEYS list in LUA. +* **args**: A list of argument values. This becomes the ARGV list in LUA. +* **client**: A redis-py Client or Pipeline instance that will invoke the + script. If client isn't specified, the client that intiially + created the Script instance (the one that `register_script` was + invoked from) will be used. + +Continuing the example from above: + +.. code-block:: pycon + + >>> r.set('foo', 2) + >>> multiply(keys=['foo'], args=[5]) + 10 + +The value of key 'foo' is set to 2. When multiply is invoked, the 'foo' key is +passed to the script along with the multiplier value of 5. LUA executes the +script and returns the result, 10. + +Script instances can be executed using a different client instance, even one +that points to a completely different Redis server. + +.. code-block:: pycon + + >>> r2 = redis.StrictRedis('redis2.example.com') + >>> r2.set('foo', 3) + >>> multiply(keys=['foo'], args=[5], client=r2) + 15 + +The Script object ensures that the LUA script is loaded into Redis's script +cache. In the event of a NOSCRIPT error, it will load the script and retry +executing it. + +Script objects can also be used in pipelines. The pipeline instance should be +passed as the client argument when calling the script. Care is taken to ensure +that the script is registered in Redis's script cache just prior to pipeline +execution. + +.. code-block:: pycon + + >>> pipe = r.pipeline() + >>> pipe.set('foo', 5) + >>> multiply(keys=['foo'], args=[5], client=pipe) + >>> pipe.execute() + [True, 25] + +Sentinel support +^^^^^^^^^^^^^^^^ + +redis-py can be used together with `Redis Sentinel `_ +to discover Redis nodes. You need to have at least one Sentinel daemon running +in order to use redis-py's Sentinel support. + +Connecting redis-py to the Sentinel instance(s) is easy. You can use a +Sentinel connection to discover the master and slaves network addresses: + +.. code-block:: pycon + + >>> from redis.sentinel import Sentinel + >>> sentinel = Sentinel([('localhost', 26379)], socket_timeout=0.1) + >>> sentinel.discover_master('mymaster') + ('127.0.0.1', 6379) + >>> sentinel.discover_slaves('mymaster') + [('127.0.0.1', 6380)] + +You can also create Redis client connections from a Sentinel instance. You can +connect to either the master (for write operations) or a slave (for read-only +operations). + +.. code-block:: pycon + + >>> master = sentinel.master_for('mymaster', socket_timeout=0.1) + >>> slave = sentinel.slave_for('mymaster', socket_timeout=0.1) + >>> master.set('foo', 'bar') + >>> slave.get('foo') + 'bar' + +The master and slave objects are normal StrictRedis instances with their +connection pool bound to the Sentinel instance. When a Sentinel backed client +attempts to establish a connection, it first queries the Sentinel servers to +determine an appropriate host to connect to. If no server is found, +a MasterNotFoundError or SlaveNotFoundError is raised. Both exceptions are +subclasses of ConnectionError. + +When trying to connect to a slave client, the Sentinel connection pool will +iterate over the list of slaves until it finds one that can be connected to. +If no slaves can be connected to, a connection will be established with the +master. + +See `Guidelines for Redis clients with support for Redis Sentinel +`_ to learn more about Redis Sentinel. + +Scan Iterators +^^^^^^^^^^^^^^ + +The \*SCAN commands introduced in Redis 2.8 can be cumbersome to use. While +these commands are fully supported, redis-py also exposes the following methods +that return Python iterators for convenience: `scan_iter`, `hscan_iter`, +`sscan_iter` and `zscan_iter`. + +.. code-block:: pycon + + >>> for key, value in (('A', '1'), ('B', '2'), ('C', '3')): + ... r.set(key, value) + >>> for key in r.scan_iter(): + ... print key, r.get(key) + A 1 + B 2 + C 3 + +Author +^^^^^^ + +redis-py is developed and maintained by Andy McCurdy (sedrik@gmail.com). +It can be found here: http://github.com/andymccurdy/redis-py + +Special thanks to: + +* Ludovico Magnocavallo, author of the original Python Redis client, from + which some of the socket code is still used. +* Alexander Solovyov for ideas on the generic response callback system. +* Paul Hubbard for initial packaging support. + + + diff --git a/venv/lib/python3.6/site-packages/redis-2.10.6.dist-info/RECORD b/venv/lib/python3.6/site-packages/redis-2.10.6.dist-info/RECORD new file mode 100644 index 0000000..f709200 --- /dev/null +++ b/venv/lib/python3.6/site-packages/redis-2.10.6.dist-info/RECORD @@ -0,0 +1,23 @@ +redis/__init__.py,sha256=T_2AHL1MVSp3exzkAVG94Gl23RUmwFwqWFJydtUvTUw,902 +redis/_compat.py,sha256=eKl3fmyoqeFTlF5UAHfJWF2OHrVTt4H_R6atDte6j_w,5651 +redis/client.py,sha256=-yM1rSXnZdUezG4gjtJO_QLM-kEd7pUXMQOR2ECZ1TQ,111936 +redis/connection.py,sha256=L1y_0lFkHVWmhpM8-bsGU3lRsxoW1LjDDpcDRt9PqgY,40079 +redis/exceptions.py,sha256=cNISHVuXY5HtIaMGdrIhAj40MK-T04lRUopJ_77AI0Y,1224 +redis/lock.py,sha256=ndqMMNbtlW_ZO6nmIAKPDN8PrQNUleCxWqXruL-Ma3s,10563 +redis/sentinel.py,sha256=JzcteCz11LE_a10lbYI05ppKCYD0TWHK_XyEBOlC5u0,11944 +redis/utils.py,sha256=yTyLWUi60KTfw4U4nWhbGvQdNpQb9Wpdf_Qcx_lUJJU,666 +redis-2.10.6.dist-info/DESCRIPTION.rst,sha256=oPSD23YPuAgZymCrw3BAAoiE7upOJn-w8u9ARBvtNzE,26862 +redis-2.10.6.dist-info/METADATA,sha256=hjB7LTGk5ZBkSLiqEO-HZ1sNBHUgUZIzDYk5VP9Ushc,27799 +redis-2.10.6.dist-info/RECORD,, +redis-2.10.6.dist-info/WHEEL,sha256=o2k-Qa-RMNIJmUdIc7KU6VWR_ErNRbWNlxDIpl7lm34,110 +redis-2.10.6.dist-info/metadata.json,sha256=52ZBqvBiI_S7MHC8f7lkGOnqBqXZv9U-xAbHW_h66jM,1090 +redis-2.10.6.dist-info/top_level.txt,sha256=OMAefszlde6ZoOtlM35AWzpRIrwtcqAMHGlRit-w2-4,6 +redis-2.10.6.dist-info/INSTALLER,sha256=zuuue4knoyJ-UwPPXg8fezS7VCrXJQrAP7zeNuwvFQg,4 +redis/__pycache__/_compat.cpython-36.pyc,, +redis/__pycache__/utils.cpython-36.pyc,, +redis/__pycache__/lock.cpython-36.pyc,, +redis/__pycache__/connection.cpython-36.pyc,, +redis/__pycache__/client.cpython-36.pyc,, +redis/__pycache__/__init__.cpython-36.pyc,, +redis/__pycache__/sentinel.cpython-36.pyc,, +redis/__pycache__/exceptions.cpython-36.pyc,, diff --git a/venv/lib/python3.6/site-packages/redis-2.10.6.dist-info/WHEEL b/venv/lib/python3.6/site-packages/redis-2.10.6.dist-info/WHEEL new file mode 100644 index 0000000..8b6dd1b --- /dev/null +++ b/venv/lib/python3.6/site-packages/redis-2.10.6.dist-info/WHEEL @@ -0,0 +1,6 @@ +Wheel-Version: 1.0 +Generator: bdist_wheel (0.29.0) +Root-Is-Purelib: true +Tag: py2-none-any +Tag: py3-none-any + diff --git a/venv/lib/python3.6/site-packages/redis-2.10.6.dist-info/metadata.json b/venv/lib/python3.6/site-packages/redis-2.10.6.dist-info/metadata.json new file mode 100644 index 0000000..c1b7318 --- /dev/null +++ b/venv/lib/python3.6/site-packages/redis-2.10.6.dist-info/metadata.json @@ -0,0 +1 @@ +{"classifiers": ["Development Status :: 5 - Production/Stable", "Environment :: Console", "Intended Audience :: Developers", "License :: OSI Approved :: MIT License", "Operating System :: OS Independent", "Programming Language :: Python", "Programming Language :: Python :: 2", "Programming Language :: Python :: 2.6", "Programming Language :: Python :: 2.7", "Programming Language :: Python :: 3", "Programming Language :: Python :: 3.3", "Programming Language :: Python :: 3.4", "Programming Language :: Python :: 3.5", "Programming Language :: Python :: 3.6"], "extensions": {"python.details": {"contacts": [{"email": "sedrik@gmail.com", "name": "Andy McCurdy", "role": "author"}], "document_names": {"description": "DESCRIPTION.rst"}, "project_urls": {"Home": "http://github.com/andymccurdy/redis-py"}}}, "generator": "bdist_wheel (0.29.0)", "keywords": ["Redis", "key-value", "store"], "license": "MIT", "metadata_version": "2.0", "name": "redis", "summary": "Python client for Redis key-value store", "test_requires": [{"requires": ["mock", "pytest (>=2.5.0)"]}], "version": "2.10.6"} \ No newline at end of file diff --git a/venv/lib/python3.6/site-packages/redis-2.10.6.dist-info/top_level.txt b/venv/lib/python3.6/site-packages/redis-2.10.6.dist-info/top_level.txt new file mode 100644 index 0000000..7800f0f --- /dev/null +++ b/venv/lib/python3.6/site-packages/redis-2.10.6.dist-info/top_level.txt @@ -0,0 +1 @@ +redis diff --git a/venv/lib/python3.6/site-packages/redis/__init__.py b/venv/lib/python3.6/site-packages/redis/__init__.py new file mode 100644 index 0000000..6607155 --- /dev/null +++ b/venv/lib/python3.6/site-packages/redis/__init__.py @@ -0,0 +1,34 @@ +from redis.client import Redis, StrictRedis +from redis.connection import ( + BlockingConnectionPool, + ConnectionPool, + Connection, + SSLConnection, + UnixDomainSocketConnection +) +from redis.utils import from_url +from redis.exceptions import ( + AuthenticationError, + BusyLoadingError, + ConnectionError, + DataError, + InvalidResponse, + PubSubError, + ReadOnlyError, + RedisError, + ResponseError, + TimeoutError, + WatchError +) + + +__version__ = '2.10.6' +VERSION = tuple(map(int, __version__.split('.'))) + +__all__ = [ + 'Redis', 'StrictRedis', 'ConnectionPool', 'BlockingConnectionPool', + 'Connection', 'SSLConnection', 'UnixDomainSocketConnection', 'from_url', + 'AuthenticationError', 'BusyLoadingError', 'ConnectionError', 'DataError', + 'InvalidResponse', 'PubSubError', 'ReadOnlyError', 'RedisError', + 'ResponseError', 'TimeoutError', 'WatchError' +] diff --git a/venv/lib/python3.6/site-packages/redis/__pycache__/__init__.cpython-36.pyc b/venv/lib/python3.6/site-packages/redis/__pycache__/__init__.cpython-36.pyc new file mode 100644 index 0000000..cb4e6c8 Binary files /dev/null and b/venv/lib/python3.6/site-packages/redis/__pycache__/__init__.cpython-36.pyc differ diff --git a/venv/lib/python3.6/site-packages/redis/__pycache__/_compat.cpython-36.pyc b/venv/lib/python3.6/site-packages/redis/__pycache__/_compat.cpython-36.pyc new file mode 100644 index 0000000..c925a9b Binary files /dev/null and b/venv/lib/python3.6/site-packages/redis/__pycache__/_compat.cpython-36.pyc differ diff --git a/venv/lib/python3.6/site-packages/redis/__pycache__/client.cpython-36.pyc b/venv/lib/python3.6/site-packages/redis/__pycache__/client.cpython-36.pyc new file mode 100644 index 0000000..6681b68 Binary files /dev/null and b/venv/lib/python3.6/site-packages/redis/__pycache__/client.cpython-36.pyc differ diff --git a/venv/lib/python3.6/site-packages/redis/__pycache__/connection.cpython-36.pyc b/venv/lib/python3.6/site-packages/redis/__pycache__/connection.cpython-36.pyc new file mode 100644 index 0000000..faaa7d5 Binary files /dev/null and b/venv/lib/python3.6/site-packages/redis/__pycache__/connection.cpython-36.pyc differ diff --git a/venv/lib/python3.6/site-packages/redis/__pycache__/exceptions.cpython-36.pyc b/venv/lib/python3.6/site-packages/redis/__pycache__/exceptions.cpython-36.pyc new file mode 100644 index 0000000..ccf04e8 Binary files /dev/null and b/venv/lib/python3.6/site-packages/redis/__pycache__/exceptions.cpython-36.pyc differ diff --git a/venv/lib/python3.6/site-packages/redis/__pycache__/lock.cpython-36.pyc b/venv/lib/python3.6/site-packages/redis/__pycache__/lock.cpython-36.pyc new file mode 100644 index 0000000..a9fa9b0 Binary files /dev/null and b/venv/lib/python3.6/site-packages/redis/__pycache__/lock.cpython-36.pyc differ diff --git a/venv/lib/python3.6/site-packages/redis/__pycache__/sentinel.cpython-36.pyc b/venv/lib/python3.6/site-packages/redis/__pycache__/sentinel.cpython-36.pyc new file mode 100644 index 0000000..bac2e86 Binary files /dev/null and b/venv/lib/python3.6/site-packages/redis/__pycache__/sentinel.cpython-36.pyc differ diff --git a/venv/lib/python3.6/site-packages/redis/__pycache__/utils.cpython-36.pyc b/venv/lib/python3.6/site-packages/redis/__pycache__/utils.cpython-36.pyc new file mode 100644 index 0000000..89f0b08 Binary files /dev/null and b/venv/lib/python3.6/site-packages/redis/__pycache__/utils.cpython-36.pyc differ diff --git a/venv/lib/python3.6/site-packages/redis/_compat.py b/venv/lib/python3.6/site-packages/redis/_compat.py new file mode 100644 index 0000000..307f3cc --- /dev/null +++ b/venv/lib/python3.6/site-packages/redis/_compat.py @@ -0,0 +1,198 @@ +"""Internal module for Python 2 backwards compatibility.""" +import errno +import sys + +try: + InterruptedError = InterruptedError +except: + InterruptedError = OSError + +# For Python older than 3.5, retry EINTR. +if sys.version_info[0] < 3 or (sys.version_info[0] == 3 and + sys.version_info[1] < 5): + # Adapted from https://bugs.python.org/review/23863/patch/14532/54418 + import socket + import time + import errno + + from select import select as _select + + def select(rlist, wlist, xlist, timeout): + while True: + try: + return _select(rlist, wlist, xlist, timeout) + except InterruptedError as e: + # Python 2 does not define InterruptedError, instead + # try to catch an OSError with errno == EINTR == 4. + if getattr(e, 'errno', None) == getattr(errno, 'EINTR', 4): + continue + raise + + # Wrapper for handling interruptable system calls. + def _retryable_call(s, func, *args, **kwargs): + # Some modules (SSL) use the _fileobject wrapper directly and + # implement a smaller portion of the socket interface, thus we + # need to let them continue to do so. + timeout, deadline = None, 0.0 + attempted = False + try: + timeout = s.gettimeout() + except AttributeError: + pass + + if timeout: + deadline = time.time() + timeout + + try: + while True: + if attempted and timeout: + now = time.time() + if now >= deadline: + raise socket.error(errno.EWOULDBLOCK, "timed out") + else: + # Overwrite the timeout on the socket object + # to take into account elapsed time. + s.settimeout(deadline - now) + try: + attempted = True + return func(*args, **kwargs) + except socket.error as e: + if e.args[0] == errno.EINTR: + continue + raise + finally: + # Set the existing timeout back for future + # calls. + if timeout: + s.settimeout(timeout) + + def recv(sock, *args, **kwargs): + return _retryable_call(sock, sock.recv, *args, **kwargs) + + def recv_into(sock, *args, **kwargs): + return _retryable_call(sock, sock.recv_into, *args, **kwargs) + +else: # Python 3.5 and above automatically retry EINTR + from select import select + + def recv(sock, *args, **kwargs): + return sock.recv(*args, **kwargs) + + def recv_into(sock, *args, **kwargs): + return sock.recv_into(*args, **kwargs) + +if sys.version_info[0] < 3: + from urllib import unquote + from urlparse import parse_qs, urlparse + from itertools import imap, izip + from string import letters as ascii_letters + from Queue import Queue + try: + from cStringIO import StringIO as BytesIO + except ImportError: + from StringIO import StringIO as BytesIO + + # special unicode handling for python2 to avoid UnicodeDecodeError + def safe_unicode(obj, *args): + """ return the unicode representation of obj """ + try: + return unicode(obj, *args) + except UnicodeDecodeError: + # obj is byte string + ascii_text = str(obj).encode('string_escape') + return unicode(ascii_text) + + def iteritems(x): + return x.iteritems() + + def iterkeys(x): + return x.iterkeys() + + def itervalues(x): + return x.itervalues() + + def nativestr(x): + return x if isinstance(x, str) else x.encode('utf-8', 'replace') + + def u(x): + return x.decode() + + def b(x): + return x + + def next(x): + return x.next() + + def byte_to_chr(x): + return x + + unichr = unichr + xrange = xrange + basestring = basestring + unicode = unicode + bytes = str + long = long +else: + from urllib.parse import parse_qs, unquote, urlparse + from io import BytesIO + from string import ascii_letters + from queue import Queue + + def iteritems(x): + return iter(x.items()) + + def iterkeys(x): + return iter(x.keys()) + + def itervalues(x): + return iter(x.values()) + + def byte_to_chr(x): + return chr(x) + + def nativestr(x): + return x if isinstance(x, str) else x.decode('utf-8', 'replace') + + def u(x): + return x + + def b(x): + return x.encode('latin-1') if not isinstance(x, bytes) else x + + next = next + unichr = chr + imap = map + izip = zip + xrange = range + basestring = str + unicode = str + safe_unicode = str + bytes = bytes + long = int + +try: # Python 3 + from queue import LifoQueue, Empty, Full +except ImportError: + from Queue import Empty, Full + try: # Python 2.6 - 2.7 + from Queue import LifoQueue + except ImportError: # Python 2.5 + from Queue import Queue + # From the Python 2.7 lib. Python 2.5 already extracted the core + # methods to aid implementating different queue organisations. + + class LifoQueue(Queue): + "Override queue methods to implement a last-in first-out queue." + + def _init(self, maxsize): + self.maxsize = maxsize + self.queue = [] + + def _qsize(self, len=len): + return len(self.queue) + + def _put(self, item): + self.queue.append(item) + + def _get(self): + return self.queue.pop() diff --git a/venv/lib/python3.6/site-packages/redis/client.py b/venv/lib/python3.6/site-packages/redis/client.py new file mode 100644 index 0000000..6383e55 --- /dev/null +++ b/venv/lib/python3.6/site-packages/redis/client.py @@ -0,0 +1,2950 @@ +from __future__ import with_statement +from itertools import chain +import datetime +import sys +import warnings +import time +import threading +import time as mod_time +import hashlib +from redis._compat import (b, basestring, bytes, imap, iteritems, iterkeys, + itervalues, izip, long, nativestr, unicode, + safe_unicode) +from redis.connection import (ConnectionPool, UnixDomainSocketConnection, + SSLConnection, Token) +from redis.lock import Lock, LuaLock +from redis.exceptions import ( + ConnectionError, + DataError, + ExecAbortError, + NoScriptError, + PubSubError, + RedisError, + ResponseError, + TimeoutError, + WatchError, +) + +SYM_EMPTY = b('') + + +def list_or_args(keys, args): + # returns a single list combining keys and args + try: + iter(keys) + # a string or bytes instance can be iterated, but indicates + # keys wasn't passed as a list + if isinstance(keys, (basestring, bytes)): + keys = [keys] + except TypeError: + keys = [keys] + if args: + keys.extend(args) + return keys + + +def timestamp_to_datetime(response): + "Converts a unix timestamp to a Python datetime object" + if not response: + return None + try: + response = int(response) + except ValueError: + return None + return datetime.datetime.fromtimestamp(response) + + +def string_keys_to_dict(key_string, callback): + return dict.fromkeys(key_string.split(), callback) + + +def dict_merge(*dicts): + merged = {} + for d in dicts: + merged.update(d) + return merged + + +def parse_debug_object(response): + "Parse the results of Redis's DEBUG OBJECT command into a Python dict" + # The 'type' of the object is the first item in the response, but isn't + # prefixed with a name + response = nativestr(response) + response = 'type:' + response + response = dict([kv.split(':') for kv in response.split()]) + + # parse some expected int values from the string response + # note: this cmd isn't spec'd so these may not appear in all redis versions + int_fields = ('refcount', 'serializedlength', 'lru', 'lru_seconds_idle') + for field in int_fields: + if field in response: + response[field] = int(response[field]) + + return response + + +def parse_object(response, infotype): + "Parse the results of an OBJECT command" + if infotype in ('idletime', 'refcount'): + return int_or_none(response) + return response + + +def parse_info(response): + "Parse the result of Redis's INFO command into a Python dict" + info = {} + response = nativestr(response) + + def get_value(value): + if ',' not in value or '=' not in value: + try: + if '.' in value: + return float(value) + else: + return int(value) + except ValueError: + return value + else: + sub_dict = {} + for item in value.split(','): + k, v = item.rsplit('=', 1) + sub_dict[k] = get_value(v) + return sub_dict + + for line in response.splitlines(): + if line and not line.startswith('#'): + if line.find(':') != -1: + key, value = line.split(':', 1) + info[key] = get_value(value) + else: + # if the line isn't splittable, append it to the "__raw__" key + info.setdefault('__raw__', []).append(line) + + return info + + +SENTINEL_STATE_TYPES = { + 'can-failover-its-master': int, + 'config-epoch': int, + 'down-after-milliseconds': int, + 'failover-timeout': int, + 'info-refresh': int, + 'last-hello-message': int, + 'last-ok-ping-reply': int, + 'last-ping-reply': int, + 'last-ping-sent': int, + 'master-link-down-time': int, + 'master-port': int, + 'num-other-sentinels': int, + 'num-slaves': int, + 'o-down-time': int, + 'pending-commands': int, + 'parallel-syncs': int, + 'port': int, + 'quorum': int, + 'role-reported-time': int, + 's-down-time': int, + 'slave-priority': int, + 'slave-repl-offset': int, + 'voted-leader-epoch': int +} + + +def parse_sentinel_state(item): + result = pairs_to_dict_typed(item, SENTINEL_STATE_TYPES) + flags = set(result['flags'].split(',')) + for name, flag in (('is_master', 'master'), ('is_slave', 'slave'), + ('is_sdown', 's_down'), ('is_odown', 'o_down'), + ('is_sentinel', 'sentinel'), + ('is_disconnected', 'disconnected'), + ('is_master_down', 'master_down')): + result[name] = flag in flags + return result + + +def parse_sentinel_master(response): + return parse_sentinel_state(imap(nativestr, response)) + + +def parse_sentinel_masters(response): + result = {} + for item in response: + state = parse_sentinel_state(imap(nativestr, item)) + result[state['name']] = state + return result + + +def parse_sentinel_slaves_and_sentinels(response): + return [parse_sentinel_state(imap(nativestr, item)) for item in response] + + +def parse_sentinel_get_master(response): + return response and (response[0], int(response[1])) or None + + +def pairs_to_dict(response): + "Create a dict given a list of key/value pairs" + it = iter(response) + return dict(izip(it, it)) + + +def pairs_to_dict_typed(response, type_info): + it = iter(response) + result = {} + for key, value in izip(it, it): + if key in type_info: + try: + value = type_info[key](value) + except: + # if for some reason the value can't be coerced, just use + # the string value + pass + result[key] = value + return result + + +def zset_score_pairs(response, **options): + """ + If ``withscores`` is specified in the options, return the response as + a list of (value, score) pairs + """ + if not response or not options.get('withscores'): + return response + score_cast_func = options.get('score_cast_func', float) + it = iter(response) + return list(izip(it, imap(score_cast_func, it))) + + +def sort_return_tuples(response, **options): + """ + If ``groups`` is specified, return the response as a list of + n-element tuples with n being the value found in options['groups'] + """ + if not response or not options['groups']: + return response + n = options['groups'] + return list(izip(*[response[i::n] for i in range(n)])) + + +def int_or_none(response): + if response is None: + return None + return int(response) + + +def float_or_none(response): + if response is None: + return None + return float(response) + + +def bool_ok(response): + return nativestr(response) == 'OK' + + +def parse_client_list(response, **options): + clients = [] + for c in nativestr(response).splitlines(): + clients.append(dict([pair.split('=') for pair in c.split(' ')])) + return clients + + +def parse_config_get(response, **options): + response = [nativestr(i) if i is not None else None for i in response] + return response and pairs_to_dict(response) or {} + + +def parse_scan(response, **options): + cursor, r = response + return long(cursor), r + + +def parse_hscan(response, **options): + cursor, r = response + return long(cursor), r and pairs_to_dict(r) or {} + + +def parse_zscan(response, **options): + score_cast_func = options.get('score_cast_func', float) + cursor, r = response + it = iter(r) + return long(cursor), list(izip(it, imap(score_cast_func, it))) + + +def parse_slowlog_get(response, **options): + return [{ + 'id': item[0], + 'start_time': int(item[1]), + 'duration': int(item[2]), + 'command': b(' ').join(item[3]) + } for item in response] + + +def parse_cluster_info(response, **options): + return dict([line.split(':') for line in response.splitlines() if line]) + + +def _parse_node_line(line): + line_items = line.split(' ') + node_id, addr, flags, master_id, ping, pong, epoch, \ + connected = line.split(' ')[:8] + slots = [sl.split('-') for sl in line_items[8:]] + node_dict = { + 'node_id': node_id, + 'flags': flags, + 'master_id': master_id, + 'last_ping_sent': ping, + 'last_pong_rcvd': pong, + 'epoch': epoch, + 'slots': slots, + 'connected': True if connected == 'connected' else False + } + return addr, node_dict + + +def parse_cluster_nodes(response, **options): + raw_lines = response + if isinstance(response, basestring): + raw_lines = response.splitlines() + return dict([_parse_node_line(line) for line in raw_lines]) + + +def parse_georadius_generic(response, **options): + if options['store'] or options['store_dist']: + # `store` and `store_diff` cant be combined + # with other command arguments. + return response + + if type(response) != list: + response_list = [response] + else: + response_list = response + + if not options['withdist'] and not options['withcoord']\ + and not options['withhash']: + # just a bunch of places + return [nativestr(r) for r in response_list] + + cast = { + 'withdist': float, + 'withcoord': lambda ll: (float(ll[0]), float(ll[1])), + 'withhash': int + } + + # zip all output results with each casting functino to get + # the properly native Python value. + f = [nativestr] + f += [cast[o] for o in ['withdist', 'withhash', 'withcoord'] if options[o]] + return [ + list(map(lambda fv: fv[0](fv[1]), zip(f, r))) for r in response_list + ] + + +def parse_pubsub_numsub(response, **options): + return list(zip(response[0::2], response[1::2])) + + +class StrictRedis(object): + """ + Implementation of the Redis protocol. + + This abstract class provides a Python interface to all Redis commands + and an implementation of the Redis protocol. + + Connection and Pipeline derive from this, implementing how + the commands are sent and received to the Redis server + """ + RESPONSE_CALLBACKS = dict_merge( + string_keys_to_dict( + 'AUTH EXISTS EXPIRE EXPIREAT HEXISTS HMSET MOVE MSETNX PERSIST ' + 'PSETEX RENAMENX SISMEMBER SMOVE SETEX SETNX', + bool + ), + string_keys_to_dict( + 'BITCOUNT BITPOS DECRBY DEL GETBIT HDEL HLEN HSTRLEN INCRBY ' + 'LINSERT LLEN LPUSHX PFADD PFCOUNT RPUSHX SADD SCARD SDIFFSTORE ' + 'SETBIT SETRANGE SINTERSTORE SREM STRLEN SUNIONSTORE ZADD ZCARD ' + 'ZLEXCOUNT ZREM ZREMRANGEBYLEX ZREMRANGEBYRANK ZREMRANGEBYSCORE ' + 'GEOADD', + int + ), + string_keys_to_dict( + 'INCRBYFLOAT HINCRBYFLOAT GEODIST', + float + ), + string_keys_to_dict( + # these return OK, or int if redis-server is >=1.3.4 + 'LPUSH RPUSH', + lambda r: isinstance(r, (long, int)) and r or nativestr(r) == 'OK' + ), + string_keys_to_dict('SORT', sort_return_tuples), + string_keys_to_dict('ZSCORE ZINCRBY', float_or_none), + string_keys_to_dict( + 'FLUSHALL FLUSHDB LSET LTRIM MSET PFMERGE RENAME ' + 'SAVE SELECT SHUTDOWN SLAVEOF WATCH UNWATCH', + bool_ok + ), + string_keys_to_dict('BLPOP BRPOP', lambda r: r and tuple(r) or None), + string_keys_to_dict( + 'SDIFF SINTER SMEMBERS SUNION', + lambda r: r and set(r) or set() + ), + string_keys_to_dict( + 'ZRANGE ZRANGEBYSCORE ZREVRANGE ZREVRANGEBYSCORE', + zset_score_pairs + ), + string_keys_to_dict('ZRANK ZREVRANK', int_or_none), + string_keys_to_dict('BGREWRITEAOF BGSAVE', lambda r: True), + { + 'CLIENT GETNAME': lambda r: r and nativestr(r), + 'CLIENT KILL': bool_ok, + 'CLIENT LIST': parse_client_list, + 'CLIENT SETNAME': bool_ok, + 'CONFIG GET': parse_config_get, + 'CONFIG RESETSTAT': bool_ok, + 'CONFIG SET': bool_ok, + 'DEBUG OBJECT': parse_debug_object, + 'HGETALL': lambda r: r and pairs_to_dict(r) or {}, + 'HSCAN': parse_hscan, + 'INFO': parse_info, + 'LASTSAVE': timestamp_to_datetime, + 'OBJECT': parse_object, + 'PING': lambda r: nativestr(r) == 'PONG', + 'RANDOMKEY': lambda r: r and r or None, + 'SCAN': parse_scan, + 'SCRIPT EXISTS': lambda r: list(imap(bool, r)), + 'SCRIPT FLUSH': bool_ok, + 'SCRIPT KILL': bool_ok, + 'SCRIPT LOAD': nativestr, + 'SENTINEL GET-MASTER-ADDR-BY-NAME': parse_sentinel_get_master, + 'SENTINEL MASTER': parse_sentinel_master, + 'SENTINEL MASTERS': parse_sentinel_masters, + 'SENTINEL MONITOR': bool_ok, + 'SENTINEL REMOVE': bool_ok, + 'SENTINEL SENTINELS': parse_sentinel_slaves_and_sentinels, + 'SENTINEL SET': bool_ok, + 'SENTINEL SLAVES': parse_sentinel_slaves_and_sentinels, + 'SET': lambda r: r and nativestr(r) == 'OK', + 'SLOWLOG GET': parse_slowlog_get, + 'SLOWLOG LEN': int, + 'SLOWLOG RESET': bool_ok, + 'SSCAN': parse_scan, + 'TIME': lambda x: (int(x[0]), int(x[1])), + 'ZSCAN': parse_zscan, + 'CLUSTER ADDSLOTS': bool_ok, + 'CLUSTER COUNT-FAILURE-REPORTS': lambda x: int(x), + 'CLUSTER COUNTKEYSINSLOT': lambda x: int(x), + 'CLUSTER DELSLOTS': bool_ok, + 'CLUSTER FAILOVER': bool_ok, + 'CLUSTER FORGET': bool_ok, + 'CLUSTER INFO': parse_cluster_info, + 'CLUSTER KEYSLOT': lambda x: int(x), + 'CLUSTER MEET': bool_ok, + 'CLUSTER NODES': parse_cluster_nodes, + 'CLUSTER REPLICATE': bool_ok, + 'CLUSTER RESET': bool_ok, + 'CLUSTER SAVECONFIG': bool_ok, + 'CLUSTER SET-CONFIG-EPOCH': bool_ok, + 'CLUSTER SETSLOT': bool_ok, + 'CLUSTER SLAVES': parse_cluster_nodes, + 'GEOPOS': lambda r: list(map(lambda ll: (float(ll[0]), + float(ll[1])) + if ll is not None else None, r)), + 'GEOHASH': lambda r: list(map(nativestr, r)), + 'GEORADIUS': parse_georadius_generic, + 'GEORADIUSBYMEMBER': parse_georadius_generic, + 'PUBSUB NUMSUB': parse_pubsub_numsub, + } + ) + + @classmethod + def from_url(cls, url, db=None, **kwargs): + """ + Return a Redis client object configured from the given URL, which must + use either `the ``redis://`` scheme + `_ for RESP + connections or the ``unix://`` scheme for Unix domain sockets. + + For example:: + + redis://[:password]@localhost:6379/0 + unix://[:password]@/path/to/socket.sock?db=0 + + There are several ways to specify a database number. The parse function + will return the first specified option: + 1. A ``db`` querystring option, e.g. redis://localhost?db=0 + 2. If using the redis:// scheme, the path argument of the url, e.g. + redis://localhost/0 + 3. The ``db`` argument to this function. + + If none of these options are specified, db=0 is used. + + Any additional querystring arguments and keyword arguments will be + passed along to the ConnectionPool class's initializer. In the case + of conflicting arguments, querystring arguments always win. + """ + connection_pool = ConnectionPool.from_url(url, db=db, **kwargs) + return cls(connection_pool=connection_pool) + + def __init__(self, host='localhost', port=6379, + db=0, password=None, socket_timeout=None, + socket_connect_timeout=None, + socket_keepalive=None, socket_keepalive_options=None, + connection_pool=None, unix_socket_path=None, + encoding='utf-8', encoding_errors='strict', + charset=None, errors=None, + decode_responses=False, retry_on_timeout=False, + ssl=False, ssl_keyfile=None, ssl_certfile=None, + ssl_cert_reqs=None, ssl_ca_certs=None, + max_connections=None): + if not connection_pool: + if charset is not None: + warnings.warn(DeprecationWarning( + '"charset" is deprecated. Use "encoding" instead')) + encoding = charset + if errors is not None: + warnings.warn(DeprecationWarning( + '"errors" is deprecated. Use "encoding_errors" instead')) + encoding_errors = errors + + kwargs = { + 'db': db, + 'password': password, + 'socket_timeout': socket_timeout, + 'encoding': encoding, + 'encoding_errors': encoding_errors, + 'decode_responses': decode_responses, + 'retry_on_timeout': retry_on_timeout, + 'max_connections': max_connections + } + # based on input, setup appropriate connection args + if unix_socket_path is not None: + kwargs.update({ + 'path': unix_socket_path, + 'connection_class': UnixDomainSocketConnection + }) + else: + # TCP specific options + kwargs.update({ + 'host': host, + 'port': port, + 'socket_connect_timeout': socket_connect_timeout, + 'socket_keepalive': socket_keepalive, + 'socket_keepalive_options': socket_keepalive_options, + }) + + if ssl: + kwargs.update({ + 'connection_class': SSLConnection, + 'ssl_keyfile': ssl_keyfile, + 'ssl_certfile': ssl_certfile, + 'ssl_cert_reqs': ssl_cert_reqs, + 'ssl_ca_certs': ssl_ca_certs, + }) + connection_pool = ConnectionPool(**kwargs) + self.connection_pool = connection_pool + self._use_lua_lock = None + + self.response_callbacks = self.__class__.RESPONSE_CALLBACKS.copy() + + def __repr__(self): + return "%s<%s>" % (type(self).__name__, repr(self.connection_pool)) + + def set_response_callback(self, command, callback): + "Set a custom Response Callback" + self.response_callbacks[command] = callback + + def pipeline(self, transaction=True, shard_hint=None): + """ + Return a new pipeline object that can queue multiple commands for + later execution. ``transaction`` indicates whether all commands + should be executed atomically. Apart from making a group of operations + atomic, pipelines are useful for reducing the back-and-forth overhead + between the client and server. + """ + return StrictPipeline( + self.connection_pool, + self.response_callbacks, + transaction, + shard_hint) + + def transaction(self, func, *watches, **kwargs): + """ + Convenience method for executing the callable `func` as a transaction + while watching all keys specified in `watches`. The 'func' callable + should expect a single argument which is a Pipeline object. + """ + shard_hint = kwargs.pop('shard_hint', None) + value_from_callable = kwargs.pop('value_from_callable', False) + watch_delay = kwargs.pop('watch_delay', None) + with self.pipeline(True, shard_hint) as pipe: + while 1: + try: + if watches: + pipe.watch(*watches) + func_value = func(pipe) + exec_value = pipe.execute() + return func_value if value_from_callable else exec_value + except WatchError: + if watch_delay is not None and watch_delay > 0: + time.sleep(watch_delay) + continue + + def lock(self, name, timeout=None, sleep=0.1, blocking_timeout=None, + lock_class=None, thread_local=True): + """ + Return a new Lock object using key ``name`` that mimics + the behavior of threading.Lock. + + If specified, ``timeout`` indicates a maximum life for the lock. + By default, it will remain locked until release() is called. + + ``sleep`` indicates the amount of time to sleep per loop iteration + when the lock is in blocking mode and another client is currently + holding the lock. + + ``blocking_timeout`` indicates the maximum amount of time in seconds to + spend trying to acquire the lock. A value of ``None`` indicates + continue trying forever. ``blocking_timeout`` can be specified as a + float or integer, both representing the number of seconds to wait. + + ``lock_class`` forces the specified lock implementation. + + ``thread_local`` indicates whether the lock token is placed in + thread-local storage. By default, the token is placed in thread local + storage so that a thread only sees its token, not a token set by + another thread. Consider the following timeline: + + time: 0, thread-1 acquires `my-lock`, with a timeout of 5 seconds. + thread-1 sets the token to "abc" + time: 1, thread-2 blocks trying to acquire `my-lock` using the + Lock instance. + time: 5, thread-1 has not yet completed. redis expires the lock + key. + time: 5, thread-2 acquired `my-lock` now that it's available. + thread-2 sets the token to "xyz" + time: 6, thread-1 finishes its work and calls release(). if the + token is *not* stored in thread local storage, then + thread-1 would see the token value as "xyz" and would be + able to successfully release the thread-2's lock. + + In some use cases it's necessary to disable thread local storage. For + example, if you have code where one thread acquires a lock and passes + that lock instance to a worker thread to release later. If thread + local storage isn't disabled in this case, the worker thread won't see + the token set by the thread that acquired the lock. Our assumption + is that these cases aren't common and as such default to using + thread local storage. """ + if lock_class is None: + if self._use_lua_lock is None: + # the first time .lock() is called, determine if we can use + # Lua by attempting to register the necessary scripts + try: + LuaLock.register_scripts(self) + self._use_lua_lock = True + except ResponseError: + self._use_lua_lock = False + lock_class = self._use_lua_lock and LuaLock or Lock + return lock_class(self, name, timeout=timeout, sleep=sleep, + blocking_timeout=blocking_timeout, + thread_local=thread_local) + + def pubsub(self, **kwargs): + """ + Return a Publish/Subscribe object. With this object, you can + subscribe to channels and listen for messages that get published to + them. + """ + return PubSub(self.connection_pool, **kwargs) + + # COMMAND EXECUTION AND PROTOCOL PARSING + def execute_command(self, *args, **options): + "Execute a command and return a parsed response" + pool = self.connection_pool + command_name = args[0] + connection = pool.get_connection(command_name, **options) + try: + connection.send_command(*args) + return self.parse_response(connection, command_name, **options) + except (ConnectionError, TimeoutError) as e: + connection.disconnect() + if not connection.retry_on_timeout and isinstance(e, TimeoutError): + raise + connection.send_command(*args) + return self.parse_response(connection, command_name, **options) + finally: + pool.release(connection) + + def parse_response(self, connection, command_name, **options): + "Parses a response from the Redis server" + response = connection.read_response() + if command_name in self.response_callbacks: + return self.response_callbacks[command_name](response, **options) + return response + + # SERVER INFORMATION + def bgrewriteaof(self): + "Tell the Redis server to rewrite the AOF file from data in memory." + return self.execute_command('BGREWRITEAOF') + + def bgsave(self): + """ + Tell the Redis server to save its data to disk. Unlike save(), + this method is asynchronous and returns immediately. + """ + return self.execute_command('BGSAVE') + + def client_kill(self, address): + "Disconnects the client at ``address`` (ip:port)" + return self.execute_command('CLIENT KILL', address) + + def client_list(self): + "Returns a list of currently connected clients" + return self.execute_command('CLIENT LIST') + + def client_getname(self): + "Returns the current connection name" + return self.execute_command('CLIENT GETNAME') + + def client_setname(self, name): + "Sets the current connection name" + return self.execute_command('CLIENT SETNAME', name) + + def config_get(self, pattern="*"): + "Return a dictionary of configuration based on the ``pattern``" + return self.execute_command('CONFIG GET', pattern) + + def config_set(self, name, value): + "Set config item ``name`` with ``value``" + return self.execute_command('CONFIG SET', name, value) + + def config_resetstat(self): + "Reset runtime statistics" + return self.execute_command('CONFIG RESETSTAT') + + def config_rewrite(self): + "Rewrite config file with the minimal change to reflect running config" + return self.execute_command('CONFIG REWRITE') + + def dbsize(self): + "Returns the number of keys in the current database" + return self.execute_command('DBSIZE') + + def debug_object(self, key): + "Returns version specific meta information about a given key" + return self.execute_command('DEBUG OBJECT', key) + + def echo(self, value): + "Echo the string back from the server" + return self.execute_command('ECHO', value) + + def flushall(self): + "Delete all keys in all databases on the current host" + return self.execute_command('FLUSHALL') + + def flushdb(self): + "Delete all keys in the current database" + return self.execute_command('FLUSHDB') + + def info(self, section=None): + """ + Returns a dictionary containing information about the Redis server + + The ``section`` option can be used to select a specific section + of information + + The section option is not supported by older versions of Redis Server, + and will generate ResponseError + """ + if section is None: + return self.execute_command('INFO') + else: + return self.execute_command('INFO', section) + + def lastsave(self): + """ + Return a Python datetime object representing the last time the + Redis database was saved to disk + """ + return self.execute_command('LASTSAVE') + + def object(self, infotype, key): + "Return the encoding, idletime, or refcount about the key" + return self.execute_command('OBJECT', infotype, key, infotype=infotype) + + def ping(self): + "Ping the Redis server" + return self.execute_command('PING') + + def save(self): + """ + Tell the Redis server to save its data to disk, + blocking until the save is complete + """ + return self.execute_command('SAVE') + + def sentinel(self, *args): + "Redis Sentinel's SENTINEL command." + warnings.warn( + DeprecationWarning('Use the individual sentinel_* methods')) + + def sentinel_get_master_addr_by_name(self, service_name): + "Returns a (host, port) pair for the given ``service_name``" + return self.execute_command('SENTINEL GET-MASTER-ADDR-BY-NAME', + service_name) + + def sentinel_master(self, service_name): + "Returns a dictionary containing the specified masters state." + return self.execute_command('SENTINEL MASTER', service_name) + + def sentinel_masters(self): + "Returns a list of dictionaries containing each master's state." + return self.execute_command('SENTINEL MASTERS') + + def sentinel_monitor(self, name, ip, port, quorum): + "Add a new master to Sentinel to be monitored" + return self.execute_command('SENTINEL MONITOR', name, ip, port, quorum) + + def sentinel_remove(self, name): + "Remove a master from Sentinel's monitoring" + return self.execute_command('SENTINEL REMOVE', name) + + def sentinel_sentinels(self, service_name): + "Returns a list of sentinels for ``service_name``" + return self.execute_command('SENTINEL SENTINELS', service_name) + + def sentinel_set(self, name, option, value): + "Set Sentinel monitoring parameters for a given master" + return self.execute_command('SENTINEL SET', name, option, value) + + def sentinel_slaves(self, service_name): + "Returns a list of slaves for ``service_name``" + return self.execute_command('SENTINEL SLAVES', service_name) + + def shutdown(self): + "Shutdown the server" + try: + self.execute_command('SHUTDOWN') + except ConnectionError: + # a ConnectionError here is expected + return + raise RedisError("SHUTDOWN seems to have failed.") + + def slaveof(self, host=None, port=None): + """ + Set the server to be a replicated slave of the instance identified + by the ``host`` and ``port``. If called without arguments, the + instance is promoted to a master instead. + """ + if host is None and port is None: + return self.execute_command('SLAVEOF', Token.get_token('NO'), + Token.get_token('ONE')) + return self.execute_command('SLAVEOF', host, port) + + def slowlog_get(self, num=None): + """ + Get the entries from the slowlog. If ``num`` is specified, get the + most recent ``num`` items. + """ + args = ['SLOWLOG GET'] + if num is not None: + args.append(num) + return self.execute_command(*args) + + def slowlog_len(self): + "Get the number of items in the slowlog" + return self.execute_command('SLOWLOG LEN') + + def slowlog_reset(self): + "Remove all items in the slowlog" + return self.execute_command('SLOWLOG RESET') + + def time(self): + """ + Returns the server time as a 2-item tuple of ints: + (seconds since epoch, microseconds into this second). + """ + return self.execute_command('TIME') + + def wait(self, num_replicas, timeout): + """ + Redis synchronous replication + That returns the number of replicas that processed the query when + we finally have at least ``num_replicas``, or when the ``timeout`` was + reached. + """ + return self.execute_command('WAIT', num_replicas, timeout) + + # BASIC KEY COMMANDS + def append(self, key, value): + """ + Appends the string ``value`` to the value at ``key``. If ``key`` + doesn't already exist, create it with a value of ``value``. + Returns the new length of the value at ``key``. + """ + return self.execute_command('APPEND', key, value) + + def bitcount(self, key, start=None, end=None): + """ + Returns the count of set bits in the value of ``key``. Optional + ``start`` and ``end`` paramaters indicate which bytes to consider + """ + params = [key] + if start is not None and end is not None: + params.append(start) + params.append(end) + elif (start is not None and end is None) or \ + (end is not None and start is None): + raise RedisError("Both start and end must be specified") + return self.execute_command('BITCOUNT', *params) + + def bitop(self, operation, dest, *keys): + """ + Perform a bitwise operation using ``operation`` between ``keys`` and + store the result in ``dest``. + """ + return self.execute_command('BITOP', operation, dest, *keys) + + def bitpos(self, key, bit, start=None, end=None): + """ + Return the position of the first bit set to 1 or 0 in a string. + ``start`` and ``end`` difines search range. The range is interpreted + as a range of bytes and not a range of bits, so start=0 and end=2 + means to look at the first three bytes. + """ + if bit not in (0, 1): + raise RedisError('bit must be 0 or 1') + params = [key, bit] + + start is not None and params.append(start) + + if start is not None and end is not None: + params.append(end) + elif start is None and end is not None: + raise RedisError("start argument is not set, " + "when end is specified") + return self.execute_command('BITPOS', *params) + + def decr(self, name, amount=1): + """ + Decrements the value of ``key`` by ``amount``. If no key exists, + the value will be initialized as 0 - ``amount`` + """ + return self.execute_command('DECRBY', name, amount) + + def delete(self, *names): + "Delete one or more keys specified by ``names``" + return self.execute_command('DEL', *names) + + def __delitem__(self, name): + self.delete(name) + + def dump(self, name): + """ + Return a serialized version of the value stored at the specified key. + If key does not exist a nil bulk reply is returned. + """ + return self.execute_command('DUMP', name) + + def exists(self, name): + "Returns a boolean indicating whether key ``name`` exists" + return self.execute_command('EXISTS', name) + __contains__ = exists + + def expire(self, name, time): + """ + Set an expire flag on key ``name`` for ``time`` seconds. ``time`` + can be represented by an integer or a Python timedelta object. + """ + if isinstance(time, datetime.timedelta): + time = time.seconds + time.days * 24 * 3600 + return self.execute_command('EXPIRE', name, time) + + def expireat(self, name, when): + """ + Set an expire flag on key ``name``. ``when`` can be represented + as an integer indicating unix time or a Python datetime object. + """ + if isinstance(when, datetime.datetime): + when = int(mod_time.mktime(when.timetuple())) + return self.execute_command('EXPIREAT', name, when) + + def get(self, name): + """ + Return the value at key ``name``, or None if the key doesn't exist + """ + return self.execute_command('GET', name) + + def __getitem__(self, name): + """ + Return the value at key ``name``, raises a KeyError if the key + doesn't exist. + """ + value = self.get(name) + if value is not None: + return value + raise KeyError(name) + + def getbit(self, name, offset): + "Returns a boolean indicating the value of ``offset`` in ``name``" + return self.execute_command('GETBIT', name, offset) + + def getrange(self, key, start, end): + """ + Returns the substring of the string value stored at ``key``, + determined by the offsets ``start`` and ``end`` (both are inclusive) + """ + return self.execute_command('GETRANGE', key, start, end) + + def getset(self, name, value): + """ + Sets the value at key ``name`` to ``value`` + and returns the old value at key ``name`` atomically. + """ + return self.execute_command('GETSET', name, value) + + def incr(self, name, amount=1): + """ + Increments the value of ``key`` by ``amount``. If no key exists, + the value will be initialized as ``amount`` + """ + return self.execute_command('INCRBY', name, amount) + + def incrby(self, name, amount=1): + """ + Increments the value of ``key`` by ``amount``. If no key exists, + the value will be initialized as ``amount`` + """ + + # An alias for ``incr()``, because it is already implemented + # as INCRBY redis command. + return self.incr(name, amount) + + def incrbyfloat(self, name, amount=1.0): + """ + Increments the value at key ``name`` by floating ``amount``. + If no key exists, the value will be initialized as ``amount`` + """ + return self.execute_command('INCRBYFLOAT', name, amount) + + def keys(self, pattern='*'): + "Returns a list of keys matching ``pattern``" + return self.execute_command('KEYS', pattern) + + def mget(self, keys, *args): + """ + Returns a list of values ordered identically to ``keys`` + """ + args = list_or_args(keys, args) + return self.execute_command('MGET', *args) + + def mset(self, *args, **kwargs): + """ + Sets key/values based on a mapping. Mapping can be supplied as a single + dictionary argument or as kwargs. + """ + if args: + if len(args) != 1 or not isinstance(args[0], dict): + raise RedisError('MSET requires **kwargs or a single dict arg') + kwargs.update(args[0]) + items = [] + for pair in iteritems(kwargs): + items.extend(pair) + return self.execute_command('MSET', *items) + + def msetnx(self, *args, **kwargs): + """ + Sets key/values based on a mapping if none of the keys are already set. + Mapping can be supplied as a single dictionary argument or as kwargs. + Returns a boolean indicating if the operation was successful. + """ + if args: + if len(args) != 1 or not isinstance(args[0], dict): + raise RedisError('MSETNX requires **kwargs or a single ' + 'dict arg') + kwargs.update(args[0]) + items = [] + for pair in iteritems(kwargs): + items.extend(pair) + return self.execute_command('MSETNX', *items) + + def move(self, name, db): + "Moves the key ``name`` to a different Redis database ``db``" + return self.execute_command('MOVE', name, db) + + def persist(self, name): + "Removes an expiration on ``name``" + return self.execute_command('PERSIST', name) + + def pexpire(self, name, time): + """ + Set an expire flag on key ``name`` for ``time`` milliseconds. + ``time`` can be represented by an integer or a Python timedelta + object. + """ + if isinstance(time, datetime.timedelta): + ms = int(time.microseconds / 1000) + time = (time.seconds + time.days * 24 * 3600) * 1000 + ms + return self.execute_command('PEXPIRE', name, time) + + def pexpireat(self, name, when): + """ + Set an expire flag on key ``name``. ``when`` can be represented + as an integer representing unix time in milliseconds (unix time * 1000) + or a Python datetime object. + """ + if isinstance(when, datetime.datetime): + ms = int(when.microsecond / 1000) + when = int(mod_time.mktime(when.timetuple())) * 1000 + ms + return self.execute_command('PEXPIREAT', name, when) + + def psetex(self, name, time_ms, value): + """ + Set the value of key ``name`` to ``value`` that expires in ``time_ms`` + milliseconds. ``time_ms`` can be represented by an integer or a Python + timedelta object + """ + if isinstance(time_ms, datetime.timedelta): + ms = int(time_ms.microseconds / 1000) + time_ms = (time_ms.seconds + time_ms.days * 24 * 3600) * 1000 + ms + return self.execute_command('PSETEX', name, time_ms, value) + + def pttl(self, name): + "Returns the number of milliseconds until the key ``name`` will expire" + return self.execute_command('PTTL', name) + + def randomkey(self): + "Returns the name of a random key" + return self.execute_command('RANDOMKEY') + + def rename(self, src, dst): + """ + Rename key ``src`` to ``dst`` + """ + return self.execute_command('RENAME', src, dst) + + def renamenx(self, src, dst): + "Rename key ``src`` to ``dst`` if ``dst`` doesn't already exist" + return self.execute_command('RENAMENX', src, dst) + + def restore(self, name, ttl, value, replace=False): + """ + Create a key using the provided serialized value, previously obtained + using DUMP. + """ + params = [name, ttl, value] + if replace: + params.append('REPLACE') + return self.execute_command('RESTORE', *params) + + def set(self, name, value, ex=None, px=None, nx=False, xx=False): + """ + Set the value at key ``name`` to ``value`` + + ``ex`` sets an expire flag on key ``name`` for ``ex`` seconds. + + ``px`` sets an expire flag on key ``name`` for ``px`` milliseconds. + + ``nx`` if set to True, set the value at key ``name`` to ``value`` only + if it does not exist. + + ``xx`` if set to True, set the value at key ``name`` to ``value`` only + if it already exists. + """ + pieces = [name, value] + if ex is not None: + pieces.append('EX') + if isinstance(ex, datetime.timedelta): + ex = ex.seconds + ex.days * 24 * 3600 + pieces.append(ex) + if px is not None: + pieces.append('PX') + if isinstance(px, datetime.timedelta): + ms = int(px.microseconds / 1000) + px = (px.seconds + px.days * 24 * 3600) * 1000 + ms + pieces.append(px) + + if nx: + pieces.append('NX') + if xx: + pieces.append('XX') + return self.execute_command('SET', *pieces) + + def __setitem__(self, name, value): + self.set(name, value) + + def setbit(self, name, offset, value): + """ + Flag the ``offset`` in ``name`` as ``value``. Returns a boolean + indicating the previous value of ``offset``. + """ + value = value and 1 or 0 + return self.execute_command('SETBIT', name, offset, value) + + def setex(self, name, time, value): + """ + Set the value of key ``name`` to ``value`` that expires in ``time`` + seconds. ``time`` can be represented by an integer or a Python + timedelta object. + """ + if isinstance(time, datetime.timedelta): + time = time.seconds + time.days * 24 * 3600 + return self.execute_command('SETEX', name, time, value) + + def setnx(self, name, value): + "Set the value of key ``name`` to ``value`` if key doesn't exist" + return self.execute_command('SETNX', name, value) + + def setrange(self, name, offset, value): + """ + Overwrite bytes in the value of ``name`` starting at ``offset`` with + ``value``. If ``offset`` plus the length of ``value`` exceeds the + length of the original value, the new value will be larger than before. + If ``offset`` exceeds the length of the original value, null bytes + will be used to pad between the end of the previous value and the start + of what's being injected. + + Returns the length of the new string. + """ + return self.execute_command('SETRANGE', name, offset, value) + + def strlen(self, name): + "Return the number of bytes stored in the value of ``name``" + return self.execute_command('STRLEN', name) + + def substr(self, name, start, end=-1): + """ + Return a substring of the string at key ``name``. ``start`` and ``end`` + are 0-based integers specifying the portion of the string to return. + """ + return self.execute_command('SUBSTR', name, start, end) + + def touch(self, *args): + """ + Alters the last access time of a key(s) ``*args``. A key is ignored + if it does not exist. + """ + return self.execute_command('TOUCH', *args) + + def ttl(self, name): + "Returns the number of seconds until the key ``name`` will expire" + return self.execute_command('TTL', name) + + def type(self, name): + "Returns the type of key ``name``" + return self.execute_command('TYPE', name) + + def watch(self, *names): + """ + Watches the values at keys ``names``, or None if the key doesn't exist + """ + warnings.warn(DeprecationWarning('Call WATCH from a Pipeline object')) + + def unwatch(self): + """ + Unwatches the value at key ``name``, or None of the key doesn't exist + """ + warnings.warn( + DeprecationWarning('Call UNWATCH from a Pipeline object')) + + # LIST COMMANDS + def blpop(self, keys, timeout=0): + """ + LPOP a value off of the first non-empty list + named in the ``keys`` list. + + If none of the lists in ``keys`` has a value to LPOP, then block + for ``timeout`` seconds, or until a value gets pushed on to one + of the lists. + + If timeout is 0, then block indefinitely. + """ + if timeout is None: + timeout = 0 + if isinstance(keys, basestring): + keys = [keys] + else: + keys = list(keys) + keys.append(timeout) + return self.execute_command('BLPOP', *keys) + + def brpop(self, keys, timeout=0): + """ + RPOP a value off of the first non-empty list + named in the ``keys`` list. + + If none of the lists in ``keys`` has a value to RPOP, then block + for ``timeout`` seconds, or until a value gets pushed on to one + of the lists. + + If timeout is 0, then block indefinitely. + """ + if timeout is None: + timeout = 0 + if isinstance(keys, basestring): + keys = [keys] + else: + keys = list(keys) + keys.append(timeout) + return self.execute_command('BRPOP', *keys) + + def brpoplpush(self, src, dst, timeout=0): + """ + Pop a value off the tail of ``src``, push it on the head of ``dst`` + and then return it. + + This command blocks until a value is in ``src`` or until ``timeout`` + seconds elapse, whichever is first. A ``timeout`` value of 0 blocks + forever. + """ + if timeout is None: + timeout = 0 + return self.execute_command('BRPOPLPUSH', src, dst, timeout) + + def lindex(self, name, index): + """ + Return the item from list ``name`` at position ``index`` + + Negative indexes are supported and will return an item at the + end of the list + """ + return self.execute_command('LINDEX', name, index) + + def linsert(self, name, where, refvalue, value): + """ + Insert ``value`` in list ``name`` either immediately before or after + [``where``] ``refvalue`` + + Returns the new length of the list on success or -1 if ``refvalue`` + is not in the list. + """ + return self.execute_command('LINSERT', name, where, refvalue, value) + + def llen(self, name): + "Return the length of the list ``name``" + return self.execute_command('LLEN', name) + + def lpop(self, name): + "Remove and return the first item of the list ``name``" + return self.execute_command('LPOP', name) + + def lpush(self, name, *values): + "Push ``values`` onto the head of the list ``name``" + return self.execute_command('LPUSH', name, *values) + + def lpushx(self, name, value): + "Push ``value`` onto the head of the list ``name`` if ``name`` exists" + return self.execute_command('LPUSHX', name, value) + + def lrange(self, name, start, end): + """ + Return a slice of the list ``name`` between + position ``start`` and ``end`` + + ``start`` and ``end`` can be negative numbers just like + Python slicing notation + """ + return self.execute_command('LRANGE', name, start, end) + + def lrem(self, name, count, value): + """ + Remove the first ``count`` occurrences of elements equal to ``value`` + from the list stored at ``name``. + + The count argument influences the operation in the following ways: + count > 0: Remove elements equal to value moving from head to tail. + count < 0: Remove elements equal to value moving from tail to head. + count = 0: Remove all elements equal to value. + """ + return self.execute_command('LREM', name, count, value) + + def lset(self, name, index, value): + "Set ``position`` of list ``name`` to ``value``" + return self.execute_command('LSET', name, index, value) + + def ltrim(self, name, start, end): + """ + Trim the list ``name``, removing all values not within the slice + between ``start`` and ``end`` + + ``start`` and ``end`` can be negative numbers just like + Python slicing notation + """ + return self.execute_command('LTRIM', name, start, end) + + def rpop(self, name): + "Remove and return the last item of the list ``name``" + return self.execute_command('RPOP', name) + + def rpoplpush(self, src, dst): + """ + RPOP a value off of the ``src`` list and atomically LPUSH it + on to the ``dst`` list. Returns the value. + """ + return self.execute_command('RPOPLPUSH', src, dst) + + def rpush(self, name, *values): + "Push ``values`` onto the tail of the list ``name``" + return self.execute_command('RPUSH', name, *values) + + def rpushx(self, name, value): + "Push ``value`` onto the tail of the list ``name`` if ``name`` exists" + return self.execute_command('RPUSHX', name, value) + + def sort(self, name, start=None, num=None, by=None, get=None, + desc=False, alpha=False, store=None, groups=False): + """ + Sort and return the list, set or sorted set at ``name``. + + ``start`` and ``num`` allow for paging through the sorted data + + ``by`` allows using an external key to weight and sort the items. + Use an "*" to indicate where in the key the item value is located + + ``get`` allows for returning items from external keys rather than the + sorted data itself. Use an "*" to indicate where int he key + the item value is located + + ``desc`` allows for reversing the sort + + ``alpha`` allows for sorting lexicographically rather than numerically + + ``store`` allows for storing the result of the sort into + the key ``store`` + + ``groups`` if set to True and if ``get`` contains at least two + elements, sort will return a list of tuples, each containing the + values fetched from the arguments to ``get``. + + """ + if (start is not None and num is None) or \ + (num is not None and start is None): + raise RedisError("``start`` and ``num`` must both be specified") + + pieces = [name] + if by is not None: + pieces.append(Token.get_token('BY')) + pieces.append(by) + if start is not None and num is not None: + pieces.append(Token.get_token('LIMIT')) + pieces.append(start) + pieces.append(num) + if get is not None: + # If get is a string assume we want to get a single value. + # Otherwise assume it's an interable and we want to get multiple + # values. We can't just iterate blindly because strings are + # iterable. + if isinstance(get, basestring): + pieces.append(Token.get_token('GET')) + pieces.append(get) + else: + for g in get: + pieces.append(Token.get_token('GET')) + pieces.append(g) + if desc: + pieces.append(Token.get_token('DESC')) + if alpha: + pieces.append(Token.get_token('ALPHA')) + if store is not None: + pieces.append(Token.get_token('STORE')) + pieces.append(store) + + if groups: + if not get or isinstance(get, basestring) or len(get) < 2: + raise DataError('when using "groups" the "get" argument ' + 'must be specified and contain at least ' + 'two keys') + + options = {'groups': len(get) if groups else None} + return self.execute_command('SORT', *pieces, **options) + + # SCAN COMMANDS + def scan(self, cursor=0, match=None, count=None): + """ + Incrementally return lists of key names. Also return a cursor + indicating the scan position. + + ``match`` allows for filtering the keys by pattern + + ``count`` allows for hint the minimum number of returns + """ + pieces = [cursor] + if match is not None: + pieces.extend([Token.get_token('MATCH'), match]) + if count is not None: + pieces.extend([Token.get_token('COUNT'), count]) + return self.execute_command('SCAN', *pieces) + + def scan_iter(self, match=None, count=None): + """ + Make an iterator using the SCAN command so that the client doesn't + need to remember the cursor position. + + ``match`` allows for filtering the keys by pattern + + ``count`` allows for hint the minimum number of returns + """ + cursor = '0' + while cursor != 0: + cursor, data = self.scan(cursor=cursor, match=match, count=count) + for item in data: + yield item + + def sscan(self, name, cursor=0, match=None, count=None): + """ + Incrementally return lists of elements in a set. Also return a cursor + indicating the scan position. + + ``match`` allows for filtering the keys by pattern + + ``count`` allows for hint the minimum number of returns + """ + pieces = [name, cursor] + if match is not None: + pieces.extend([Token.get_token('MATCH'), match]) + if count is not None: + pieces.extend([Token.get_token('COUNT'), count]) + return self.execute_command('SSCAN', *pieces) + + def sscan_iter(self, name, match=None, count=None): + """ + Make an iterator using the SSCAN command so that the client doesn't + need to remember the cursor position. + + ``match`` allows for filtering the keys by pattern + + ``count`` allows for hint the minimum number of returns + """ + cursor = '0' + while cursor != 0: + cursor, data = self.sscan(name, cursor=cursor, + match=match, count=count) + for item in data: + yield item + + def hscan(self, name, cursor=0, match=None, count=None): + """ + Incrementally return key/value slices in a hash. Also return a cursor + indicating the scan position. + + ``match`` allows for filtering the keys by pattern + + ``count`` allows for hint the minimum number of returns + """ + pieces = [name, cursor] + if match is not None: + pieces.extend([Token.get_token('MATCH'), match]) + if count is not None: + pieces.extend([Token.get_token('COUNT'), count]) + return self.execute_command('HSCAN', *pieces) + + def hscan_iter(self, name, match=None, count=None): + """ + Make an iterator using the HSCAN command so that the client doesn't + need to remember the cursor position. + + ``match`` allows for filtering the keys by pattern + + ``count`` allows for hint the minimum number of returns + """ + cursor = '0' + while cursor != 0: + cursor, data = self.hscan(name, cursor=cursor, + match=match, count=count) + for item in data.items(): + yield item + + def zscan(self, name, cursor=0, match=None, count=None, + score_cast_func=float): + """ + Incrementally return lists of elements in a sorted set. Also return a + cursor indicating the scan position. + + ``match`` allows for filtering the keys by pattern + + ``count`` allows for hint the minimum number of returns + + ``score_cast_func`` a callable used to cast the score return value + """ + pieces = [name, cursor] + if match is not None: + pieces.extend([Token.get_token('MATCH'), match]) + if count is not None: + pieces.extend([Token.get_token('COUNT'), count]) + options = {'score_cast_func': score_cast_func} + return self.execute_command('ZSCAN', *pieces, **options) + + def zscan_iter(self, name, match=None, count=None, + score_cast_func=float): + """ + Make an iterator using the ZSCAN command so that the client doesn't + need to remember the cursor position. + + ``match`` allows for filtering the keys by pattern + + ``count`` allows for hint the minimum number of returns + + ``score_cast_func`` a callable used to cast the score return value + """ + cursor = '0' + while cursor != 0: + cursor, data = self.zscan(name, cursor=cursor, match=match, + count=count, + score_cast_func=score_cast_func) + for item in data: + yield item + + # SET COMMANDS + def sadd(self, name, *values): + "Add ``value(s)`` to set ``name``" + return self.execute_command('SADD', name, *values) + + def scard(self, name): + "Return the number of elements in set ``name``" + return self.execute_command('SCARD', name) + + def sdiff(self, keys, *args): + "Return the difference of sets specified by ``keys``" + args = list_or_args(keys, args) + return self.execute_command('SDIFF', *args) + + def sdiffstore(self, dest, keys, *args): + """ + Store the difference of sets specified by ``keys`` into a new + set named ``dest``. Returns the number of keys in the new set. + """ + args = list_or_args(keys, args) + return self.execute_command('SDIFFSTORE', dest, *args) + + def sinter(self, keys, *args): + "Return the intersection of sets specified by ``keys``" + args = list_or_args(keys, args) + return self.execute_command('SINTER', *args) + + def sinterstore(self, dest, keys, *args): + """ + Store the intersection of sets specified by ``keys`` into a new + set named ``dest``. Returns the number of keys in the new set. + """ + args = list_or_args(keys, args) + return self.execute_command('SINTERSTORE', dest, *args) + + def sismember(self, name, value): + "Return a boolean indicating if ``value`` is a member of set ``name``" + return self.execute_command('SISMEMBER', name, value) + + def smembers(self, name): + "Return all members of the set ``name``" + return self.execute_command('SMEMBERS', name) + + def smove(self, src, dst, value): + "Move ``value`` from set ``src`` to set ``dst`` atomically" + return self.execute_command('SMOVE', src, dst, value) + + def spop(self, name): + "Remove and return a random member of set ``name``" + return self.execute_command('SPOP', name) + + def srandmember(self, name, number=None): + """ + If ``number`` is None, returns a random member of set ``name``. + + If ``number`` is supplied, returns a list of ``number`` random + memebers of set ``name``. Note this is only available when running + Redis 2.6+. + """ + args = (number is not None) and [number] or [] + return self.execute_command('SRANDMEMBER', name, *args) + + def srem(self, name, *values): + "Remove ``values`` from set ``name``" + return self.execute_command('SREM', name, *values) + + def sunion(self, keys, *args): + "Return the union of sets specified by ``keys``" + args = list_or_args(keys, args) + return self.execute_command('SUNION', *args) + + def sunionstore(self, dest, keys, *args): + """ + Store the union of sets specified by ``keys`` into a new + set named ``dest``. Returns the number of keys in the new set. + """ + args = list_or_args(keys, args) + return self.execute_command('SUNIONSTORE', dest, *args) + + # SORTED SET COMMANDS + def zadd(self, name, *args, **kwargs): + """ + Set any number of score, element-name pairs to the key ``name``. Pairs + can be specified in two ways: + + As *args, in the form of: score1, name1, score2, name2, ... + or as **kwargs, in the form of: name1=score1, name2=score2, ... + + The following example would add four values to the 'my-key' key: + redis.zadd('my-key', 1.1, 'name1', 2.2, 'name2', name3=3.3, name4=4.4) + """ + pieces = [] + if args: + if len(args) % 2 != 0: + raise RedisError("ZADD requires an equal number of " + "values and scores") + pieces.extend(args) + for pair in iteritems(kwargs): + pieces.append(pair[1]) + pieces.append(pair[0]) + return self.execute_command('ZADD', name, *pieces) + + def zcard(self, name): + "Return the number of elements in the sorted set ``name``" + return self.execute_command('ZCARD', name) + + def zcount(self, name, min, max): + """ + Returns the number of elements in the sorted set at key ``name`` with + a score between ``min`` and ``max``. + """ + return self.execute_command('ZCOUNT', name, min, max) + + def zincrby(self, name, value, amount=1): + "Increment the score of ``value`` in sorted set ``name`` by ``amount``" + return self.execute_command('ZINCRBY', name, amount, value) + + def zinterstore(self, dest, keys, aggregate=None): + """ + Intersect multiple sorted sets specified by ``keys`` into + a new sorted set, ``dest``. Scores in the destination will be + aggregated based on the ``aggregate``, or SUM if none is provided. + """ + return self._zaggregate('ZINTERSTORE', dest, keys, aggregate) + + def zlexcount(self, name, min, max): + """ + Return the number of items in the sorted set ``name`` between the + lexicographical range ``min`` and ``max``. + """ + return self.execute_command('ZLEXCOUNT', name, min, max) + + def zrange(self, name, start, end, desc=False, withscores=False, + score_cast_func=float): + """ + Return a range of values from sorted set ``name`` between + ``start`` and ``end`` sorted in ascending order. + + ``start`` and ``end`` can be negative, indicating the end of the range. + + ``desc`` a boolean indicating whether to sort the results descendingly + + ``withscores`` indicates to return the scores along with the values. + The return type is a list of (value, score) pairs + + ``score_cast_func`` a callable used to cast the score return value + """ + if desc: + return self.zrevrange(name, start, end, withscores, + score_cast_func) + pieces = ['ZRANGE', name, start, end] + if withscores: + pieces.append(Token.get_token('WITHSCORES')) + options = { + 'withscores': withscores, + 'score_cast_func': score_cast_func + } + return self.execute_command(*pieces, **options) + + def zrangebylex(self, name, min, max, start=None, num=None): + """ + Return the lexicographical range of values from sorted set ``name`` + between ``min`` and ``max``. + + If ``start`` and ``num`` are specified, then return a slice of the + range. + """ + if (start is not None and num is None) or \ + (num is not None and start is None): + raise RedisError("``start`` and ``num`` must both be specified") + pieces = ['ZRANGEBYLEX', name, min, max] + if start is not None and num is not None: + pieces.extend([Token.get_token('LIMIT'), start, num]) + return self.execute_command(*pieces) + + def zrevrangebylex(self, name, max, min, start=None, num=None): + """ + Return the reversed lexicographical range of values from sorted set + ``name`` between ``max`` and ``min``. + + If ``start`` and ``num`` are specified, then return a slice of the + range. + """ + if (start is not None and num is None) or \ + (num is not None and start is None): + raise RedisError("``start`` and ``num`` must both be specified") + pieces = ['ZREVRANGEBYLEX', name, max, min] + if start is not None and num is not None: + pieces.extend([Token.get_token('LIMIT'), start, num]) + return self.execute_command(*pieces) + + def zrangebyscore(self, name, min, max, start=None, num=None, + withscores=False, score_cast_func=float): + """ + Return a range of values from the sorted set ``name`` with scores + between ``min`` and ``max``. + + If ``start`` and ``num`` are specified, then return a slice + of the range. + + ``withscores`` indicates to return the scores along with the values. + The return type is a list of (value, score) pairs + + `score_cast_func`` a callable used to cast the score return value + """ + if (start is not None and num is None) or \ + (num is not None and start is None): + raise RedisError("``start`` and ``num`` must both be specified") + pieces = ['ZRANGEBYSCORE', name, min, max] + if start is not None and num is not None: + pieces.extend([Token.get_token('LIMIT'), start, num]) + if withscores: + pieces.append(Token.get_token('WITHSCORES')) + options = { + 'withscores': withscores, + 'score_cast_func': score_cast_func + } + return self.execute_command(*pieces, **options) + + def zrank(self, name, value): + """ + Returns a 0-based value indicating the rank of ``value`` in sorted set + ``name`` + """ + return self.execute_command('ZRANK', name, value) + + def zrem(self, name, *values): + "Remove member ``values`` from sorted set ``name``" + return self.execute_command('ZREM', name, *values) + + def zremrangebylex(self, name, min, max): + """ + Remove all elements in the sorted set ``name`` between the + lexicographical range specified by ``min`` and ``max``. + + Returns the number of elements removed. + """ + return self.execute_command('ZREMRANGEBYLEX', name, min, max) + + def zremrangebyrank(self, name, min, max): + """ + Remove all elements in the sorted set ``name`` with ranks between + ``min`` and ``max``. Values are 0-based, ordered from smallest score + to largest. Values can be negative indicating the highest scores. + Returns the number of elements removed + """ + return self.execute_command('ZREMRANGEBYRANK', name, min, max) + + def zremrangebyscore(self, name, min, max): + """ + Remove all elements in the sorted set ``name`` with scores + between ``min`` and ``max``. Returns the number of elements removed. + """ + return self.execute_command('ZREMRANGEBYSCORE', name, min, max) + + def zrevrange(self, name, start, end, withscores=False, + score_cast_func=float): + """ + Return a range of values from sorted set ``name`` between + ``start`` and ``end`` sorted in descending order. + + ``start`` and ``end`` can be negative, indicating the end of the range. + + ``withscores`` indicates to return the scores along with the values + The return type is a list of (value, score) pairs + + ``score_cast_func`` a callable used to cast the score return value + """ + pieces = ['ZREVRANGE', name, start, end] + if withscores: + pieces.append(Token.get_token('WITHSCORES')) + options = { + 'withscores': withscores, + 'score_cast_func': score_cast_func + } + return self.execute_command(*pieces, **options) + + def zrevrangebyscore(self, name, max, min, start=None, num=None, + withscores=False, score_cast_func=float): + """ + Return a range of values from the sorted set ``name`` with scores + between ``min`` and ``max`` in descending order. + + If ``start`` and ``num`` are specified, then return a slice + of the range. + + ``withscores`` indicates to return the scores along with the values. + The return type is a list of (value, score) pairs + + ``score_cast_func`` a callable used to cast the score return value + """ + if (start is not None and num is None) or \ + (num is not None and start is None): + raise RedisError("``start`` and ``num`` must both be specified") + pieces = ['ZREVRANGEBYSCORE', name, max, min] + if start is not None and num is not None: + pieces.extend([Token.get_token('LIMIT'), start, num]) + if withscores: + pieces.append(Token.get_token('WITHSCORES')) + options = { + 'withscores': withscores, + 'score_cast_func': score_cast_func + } + return self.execute_command(*pieces, **options) + + def zrevrank(self, name, value): + """ + Returns a 0-based value indicating the descending rank of + ``value`` in sorted set ``name`` + """ + return self.execute_command('ZREVRANK', name, value) + + def zscore(self, name, value): + "Return the score of element ``value`` in sorted set ``name``" + return self.execute_command('ZSCORE', name, value) + + def zunionstore(self, dest, keys, aggregate=None): + """ + Union multiple sorted sets specified by ``keys`` into + a new sorted set, ``dest``. Scores in the destination will be + aggregated based on the ``aggregate``, or SUM if none is provided. + """ + return self._zaggregate('ZUNIONSTORE', dest, keys, aggregate) + + def _zaggregate(self, command, dest, keys, aggregate=None): + pieces = [command, dest, len(keys)] + if isinstance(keys, dict): + keys, weights = iterkeys(keys), itervalues(keys) + else: + weights = None + pieces.extend(keys) + if weights: + pieces.append(Token.get_token('WEIGHTS')) + pieces.extend(weights) + if aggregate: + pieces.append(Token.get_token('AGGREGATE')) + pieces.append(aggregate) + return self.execute_command(*pieces) + + # HYPERLOGLOG COMMANDS + def pfadd(self, name, *values): + "Adds the specified elements to the specified HyperLogLog." + return self.execute_command('PFADD', name, *values) + + def pfcount(self, *sources): + """ + Return the approximated cardinality of + the set observed by the HyperLogLog at key(s). + """ + return self.execute_command('PFCOUNT', *sources) + + def pfmerge(self, dest, *sources): + "Merge N different HyperLogLogs into a single one." + return self.execute_command('PFMERGE', dest, *sources) + + # HASH COMMANDS + def hdel(self, name, *keys): + "Delete ``keys`` from hash ``name``" + return self.execute_command('HDEL', name, *keys) + + def hexists(self, name, key): + "Returns a boolean indicating if ``key`` exists within hash ``name``" + return self.execute_command('HEXISTS', name, key) + + def hget(self, name, key): + "Return the value of ``key`` within the hash ``name``" + return self.execute_command('HGET', name, key) + + def hgetall(self, name): + "Return a Python dict of the hash's name/value pairs" + return self.execute_command('HGETALL', name) + + def hincrby(self, name, key, amount=1): + "Increment the value of ``key`` in hash ``name`` by ``amount``" + return self.execute_command('HINCRBY', name, key, amount) + + def hincrbyfloat(self, name, key, amount=1.0): + """ + Increment the value of ``key`` in hash ``name`` by floating ``amount`` + """ + return self.execute_command('HINCRBYFLOAT', name, key, amount) + + def hkeys(self, name): + "Return the list of keys within hash ``name``" + return self.execute_command('HKEYS', name) + + def hlen(self, name): + "Return the number of elements in hash ``name``" + return self.execute_command('HLEN', name) + + def hset(self, name, key, value): + """ + Set ``key`` to ``value`` within hash ``name`` + Returns 1 if HSET created a new field, otherwise 0 + """ + return self.execute_command('HSET', name, key, value) + + def hsetnx(self, name, key, value): + """ + Set ``key`` to ``value`` within hash ``name`` if ``key`` does not + exist. Returns 1 if HSETNX created a field, otherwise 0. + """ + return self.execute_command('HSETNX', name, key, value) + + def hmset(self, name, mapping): + """ + Set key to value within hash ``name`` for each corresponding + key and value from the ``mapping`` dict. + """ + if not mapping: + raise DataError("'hmset' with 'mapping' of length 0") + items = [] + for pair in iteritems(mapping): + items.extend(pair) + return self.execute_command('HMSET', name, *items) + + def hmget(self, name, keys, *args): + "Returns a list of values ordered identically to ``keys``" + args = list_or_args(keys, args) + return self.execute_command('HMGET', name, *args) + + def hvals(self, name): + "Return the list of values within hash ``name``" + return self.execute_command('HVALS', name) + + def hstrlen(self, name, key): + """ + Return the number of bytes stored in the value of ``key`` + within hash ``name`` + """ + return self.execute_command('HSTRLEN', name, key) + + def publish(self, channel, message): + """ + Publish ``message`` on ``channel``. + Returns the number of subscribers the message was delivered to. + """ + return self.execute_command('PUBLISH', channel, message) + + def pubsub_channels(self, pattern='*'): + """ + Return a list of channels that have at least one subscriber + """ + return self.execute_command('PUBSUB CHANNELS', pattern) + + def pubsub_numpat(self): + """ + Returns the number of subscriptions to patterns + """ + return self.execute_command('PUBSUB NUMPAT') + + def pubsub_numsub(self, *args): + """ + Return a list of (channel, number of subscribers) tuples + for each channel given in ``*args`` + """ + return self.execute_command('PUBSUB NUMSUB', *args) + + def cluster(self, cluster_arg, *args): + return self.execute_command('CLUSTER %s' % cluster_arg.upper(), *args) + + def eval(self, script, numkeys, *keys_and_args): + """ + Execute the Lua ``script``, specifying the ``numkeys`` the script + will touch and the key names and argument values in ``keys_and_args``. + Returns the result of the script. + + In practice, use the object returned by ``register_script``. This + function exists purely for Redis API completion. + """ + return self.execute_command('EVAL', script, numkeys, *keys_and_args) + + def evalsha(self, sha, numkeys, *keys_and_args): + """ + Use the ``sha`` to execute a Lua script already registered via EVAL + or SCRIPT LOAD. Specify the ``numkeys`` the script will touch and the + key names and argument values in ``keys_and_args``. Returns the result + of the script. + + In practice, use the object returned by ``register_script``. This + function exists purely for Redis API completion. + """ + return self.execute_command('EVALSHA', sha, numkeys, *keys_and_args) + + def script_exists(self, *args): + """ + Check if a script exists in the script cache by specifying the SHAs of + each script as ``args``. Returns a list of boolean values indicating if + if each already script exists in the cache. + """ + return self.execute_command('SCRIPT EXISTS', *args) + + def script_flush(self): + "Flush all scripts from the script cache" + return self.execute_command('SCRIPT FLUSH') + + def script_kill(self): + "Kill the currently executing Lua script" + return self.execute_command('SCRIPT KILL') + + def script_load(self, script): + "Load a Lua ``script`` into the script cache. Returns the SHA." + return self.execute_command('SCRIPT LOAD', script) + + def register_script(self, script): + """ + Register a Lua ``script`` specifying the ``keys`` it will touch. + Returns a Script object that is callable and hides the complexity of + deal with scripts, keys, and shas. This is the preferred way to work + with Lua scripts. + """ + return Script(self, script) + + # GEO COMMANDS + def geoadd(self, name, *values): + """ + Add the specified geospatial items to the specified key identified + by the ``name`` argument. The Geospatial items are given as ordered + members of the ``values`` argument, each item or place is formed by + the triad longitude, latitude and name. + """ + if len(values) % 3 != 0: + raise RedisError("GEOADD requires places with lon, lat and name" + " values") + return self.execute_command('GEOADD', name, *values) + + def geodist(self, name, place1, place2, unit=None): + """ + Return the distance between ``place1`` and ``place2`` members of the + ``name`` key. + The units must be one of the following : m, km mi, ft. By default + meters are used. + """ + pieces = [name, place1, place2] + if unit and unit not in ('m', 'km', 'mi', 'ft'): + raise RedisError("GEODIST invalid unit") + elif unit: + pieces.append(unit) + return self.execute_command('GEODIST', *pieces) + + def geohash(self, name, *values): + """ + Return the geo hash string for each item of ``values`` members of + the specified key identified by the ``name``argument. + """ + return self.execute_command('GEOHASH', name, *values) + + def geopos(self, name, *values): + """ + Return the positions of each item of ``values`` as members of + the specified key identified by the ``name``argument. Each position + is represented by the pairs lon and lat. + """ + return self.execute_command('GEOPOS', name, *values) + + def georadius(self, name, longitude, latitude, radius, unit=None, + withdist=False, withcoord=False, withhash=False, count=None, + sort=None, store=None, store_dist=None): + """ + Return the members of the specified key identified by the + ``name`` argument which are within the borders of the area specified + with the ``latitude`` and ``longitude`` location and the maximum + distance from the center specified by the ``radius`` value. + + The units must be one of the following : m, km mi, ft. By default + + ``withdist`` indicates to return the distances of each place. + + ``withcoord`` indicates to return the latitude and longitude of + each place. + + ``withhash`` indicates to return the geohash string of each place. + + ``count`` indicates to return the number of elements up to N. + + ``sort`` indicates to return the places in a sorted way, ASC for + nearest to fairest and DESC for fairest to nearest. + + ``store`` indicates to save the places names in a sorted set named + with a specific key, each element of the destination sorted set is + populated with the score got from the original geo sorted set. + + ``store_dist`` indicates to save the places names in a sorted set + named with a specific key, instead of ``store`` the sorted set + destination score is set with the distance. + """ + return self._georadiusgeneric('GEORADIUS', + name, longitude, latitude, radius, + unit=unit, withdist=withdist, + withcoord=withcoord, withhash=withhash, + count=count, sort=sort, store=store, + store_dist=store_dist) + + def georadiusbymember(self, name, member, radius, unit=None, + withdist=False, withcoord=False, withhash=False, + count=None, sort=None, store=None, store_dist=None): + """ + This command is exactly like ``georadius`` with the sole difference + that instead of taking, as the center of the area to query, a longitude + and latitude value, it takes the name of a member already existing + inside the geospatial index represented by the sorted set. + """ + return self._georadiusgeneric('GEORADIUSBYMEMBER', + name, member, radius, unit=unit, + withdist=withdist, withcoord=withcoord, + withhash=withhash, count=count, + sort=sort, store=store, + store_dist=store_dist) + + def _georadiusgeneric(self, command, *args, **kwargs): + pieces = list(args) + if kwargs['unit'] and kwargs['unit'] not in ('m', 'km', 'mi', 'ft'): + raise RedisError("GEORADIUS invalid unit") + elif kwargs['unit']: + pieces.append(kwargs['unit']) + else: + pieces.append('m',) + + for token in ('withdist', 'withcoord', 'withhash'): + if kwargs[token]: + pieces.append(Token(token.upper())) + + if kwargs['count']: + pieces.extend([Token('COUNT'), kwargs['count']]) + + if kwargs['sort'] and kwargs['sort'] not in ('ASC', 'DESC'): + raise RedisError("GEORADIUS invalid sort") + elif kwargs['sort']: + pieces.append(Token(kwargs['sort'])) + + if kwargs['store'] and kwargs['store_dist']: + raise RedisError("GEORADIUS store and store_dist cant be set" + " together") + + if kwargs['store']: + pieces.extend([Token('STORE'), kwargs['store']]) + + if kwargs['store_dist']: + pieces.extend([Token('STOREDIST'), kwargs['store_dist']]) + + return self.execute_command(command, *pieces, **kwargs) + + +class Redis(StrictRedis): + """ + Provides backwards compatibility with older versions of redis-py that + changed arguments to some commands to be more Pythonic, sane, or by + accident. + """ + + # Overridden callbacks + RESPONSE_CALLBACKS = dict_merge( + StrictRedis.RESPONSE_CALLBACKS, + { + 'TTL': lambda r: r >= 0 and r or None, + 'PTTL': lambda r: r >= 0 and r or None, + } + ) + + def pipeline(self, transaction=True, shard_hint=None): + """ + Return a new pipeline object that can queue multiple commands for + later execution. ``transaction`` indicates whether all commands + should be executed atomically. Apart from making a group of operations + atomic, pipelines are useful for reducing the back-and-forth overhead + between the client and server. + """ + return Pipeline( + self.connection_pool, + self.response_callbacks, + transaction, + shard_hint) + + def setex(self, name, value, time): + """ + Set the value of key ``name`` to ``value`` that expires in ``time`` + seconds. ``time`` can be represented by an integer or a Python + timedelta object. + """ + if isinstance(time, datetime.timedelta): + time = time.seconds + time.days * 24 * 3600 + return self.execute_command('SETEX', name, time, value) + + def lrem(self, name, value, num=0): + """ + Remove the first ``num`` occurrences of elements equal to ``value`` + from the list stored at ``name``. + + The ``num`` argument influences the operation in the following ways: + num > 0: Remove elements equal to value moving from head to tail. + num < 0: Remove elements equal to value moving from tail to head. + num = 0: Remove all elements equal to value. + """ + return self.execute_command('LREM', name, num, value) + + def zadd(self, name, *args, **kwargs): + """ + NOTE: The order of arguments differs from that of the official ZADD + command. For backwards compatability, this method accepts arguments + in the form of name1, score1, name2, score2, while the official Redis + documents expects score1, name1, score2, name2. + + If you're looking to use the standard syntax, consider using the + StrictRedis class. See the API Reference section of the docs for more + information. + + Set any number of element-name, score pairs to the key ``name``. Pairs + can be specified in two ways: + + As *args, in the form of: name1, score1, name2, score2, ... + or as **kwargs, in the form of: name1=score1, name2=score2, ... + + The following example would add four values to the 'my-key' key: + redis.zadd('my-key', 'name1', 1.1, 'name2', 2.2, name3=3.3, name4=4.4) + """ + pieces = [] + if args: + if len(args) % 2 != 0: + raise RedisError("ZADD requires an equal number of " + "values and scores") + pieces.extend(reversed(args)) + for pair in iteritems(kwargs): + pieces.append(pair[1]) + pieces.append(pair[0]) + return self.execute_command('ZADD', name, *pieces) + + +class PubSub(object): + """ + PubSub provides publish, subscribe and listen support to Redis channels. + + After subscribing to one or more channels, the listen() method will block + until a message arrives on one of the subscribed channels. That message + will be returned and it's safe to start listening again. + """ + PUBLISH_MESSAGE_TYPES = ('message', 'pmessage') + UNSUBSCRIBE_MESSAGE_TYPES = ('unsubscribe', 'punsubscribe') + + def __init__(self, connection_pool, shard_hint=None, + ignore_subscribe_messages=False): + self.connection_pool = connection_pool + self.shard_hint = shard_hint + self.ignore_subscribe_messages = ignore_subscribe_messages + self.connection = None + # we need to know the encoding options for this connection in order + # to lookup channel and pattern names for callback handlers. + self.encoder = self.connection_pool.get_encoder() + self.reset() + + def __del__(self): + try: + # if this object went out of scope prior to shutting down + # subscriptions, close the connection manually before + # returning it to the connection pool + self.reset() + except Exception: + pass + + def reset(self): + if self.connection: + self.connection.disconnect() + self.connection.clear_connect_callbacks() + self.connection_pool.release(self.connection) + self.connection = None + self.channels = {} + self.patterns = {} + + def close(self): + self.reset() + + def on_connect(self, connection): + "Re-subscribe to any channels and patterns previously subscribed to" + # NOTE: for python3, we can't pass bytestrings as keyword arguments + # so we need to decode channel/pattern names back to unicode strings + # before passing them to [p]subscribe. + if self.channels: + channels = {} + for k, v in iteritems(self.channels): + channels[self.encoder.decode(k, force=True)] = v + self.subscribe(**channels) + if self.patterns: + patterns = {} + for k, v in iteritems(self.patterns): + patterns[self.encoder.decode(k, force=True)] = v + self.psubscribe(**patterns) + + @property + def subscribed(self): + "Indicates if there are subscriptions to any channels or patterns" + return bool(self.channels or self.patterns) + + def execute_command(self, *args, **kwargs): + "Execute a publish/subscribe command" + + # NOTE: don't parse the response in this function -- it could pull a + # legitimate message off the stack if the connection is already + # subscribed to one or more channels + + if self.connection is None: + self.connection = self.connection_pool.get_connection( + 'pubsub', + self.shard_hint + ) + # register a callback that re-subscribes to any channels we + # were listening to when we were disconnected + self.connection.register_connect_callback(self.on_connect) + connection = self.connection + self._execute(connection, connection.send_command, *args) + + def _execute(self, connection, command, *args): + try: + return command(*args) + except (ConnectionError, TimeoutError) as e: + connection.disconnect() + if not connection.retry_on_timeout and isinstance(e, TimeoutError): + raise + # Connect manually here. If the Redis server is down, this will + # fail and raise a ConnectionError as desired. + connection.connect() + # the ``on_connect`` callback should haven been called by the + # connection to resubscribe us to any channels and patterns we were + # previously listening to + return command(*args) + + def parse_response(self, block=True, timeout=0): + "Parse the response from a publish/subscribe command" + connection = self.connection + if connection is None: + raise RuntimeError( + 'pubsub connection not set: ' + 'did you forget to call subscribe() or psubscribe()?') + if not block and not connection.can_read(timeout=timeout): + return None + return self._execute(connection, connection.read_response) + + def _normalize_keys(self, data): + """ + normalize channel/pattern names to be either bytes or strings + based on whether responses are automatically decoded. this saves us + from coercing the value for each message coming in. + """ + encode = self.encoder.encode + decode = self.encoder.decode + return dict([(decode(encode(k)), v) for k, v in iteritems(data)]) + + def psubscribe(self, *args, **kwargs): + """ + Subscribe to channel patterns. Patterns supplied as keyword arguments + expect a pattern name as the key and a callable as the value. A + pattern's callable will be invoked automatically when a message is + received on that pattern rather than producing a message via + ``listen()``. + """ + if args: + args = list_or_args(args[0], args[1:]) + new_patterns = dict.fromkeys(args) + new_patterns.update(kwargs) + ret_val = self.execute_command('PSUBSCRIBE', *iterkeys(new_patterns)) + # update the patterns dict AFTER we send the command. we don't want to + # subscribe twice to these patterns, once for the command and again + # for the reconnection. + self.patterns.update(self._normalize_keys(new_patterns)) + return ret_val + + def punsubscribe(self, *args): + """ + Unsubscribe from the supplied patterns. If empy, unsubscribe from + all patterns. + """ + if args: + args = list_or_args(args[0], args[1:]) + return self.execute_command('PUNSUBSCRIBE', *args) + + def subscribe(self, *args, **kwargs): + """ + Subscribe to channels. Channels supplied as keyword arguments expect + a channel name as the key and a callable as the value. A channel's + callable will be invoked automatically when a message is received on + that channel rather than producing a message via ``listen()`` or + ``get_message()``. + """ + if args: + args = list_or_args(args[0], args[1:]) + new_channels = dict.fromkeys(args) + new_channels.update(kwargs) + ret_val = self.execute_command('SUBSCRIBE', *iterkeys(new_channels)) + # update the channels dict AFTER we send the command. we don't want to + # subscribe twice to these channels, once for the command and again + # for the reconnection. + self.channels.update(self._normalize_keys(new_channels)) + return ret_val + + def unsubscribe(self, *args): + """ + Unsubscribe from the supplied channels. If empty, unsubscribe from + all channels + """ + if args: + args = list_or_args(args[0], args[1:]) + return self.execute_command('UNSUBSCRIBE', *args) + + def listen(self): + "Listen for messages on channels this client has been subscribed to" + while self.subscribed: + response = self.handle_message(self.parse_response(block=True)) + if response is not None: + yield response + + def get_message(self, ignore_subscribe_messages=False, timeout=0): + """ + Get the next message if one is available, otherwise None. + + If timeout is specified, the system will wait for `timeout` seconds + before returning. Timeout should be specified as a floating point + number. + """ + response = self.parse_response(block=False, timeout=timeout) + if response: + return self.handle_message(response, ignore_subscribe_messages) + return None + + def handle_message(self, response, ignore_subscribe_messages=False): + """ + Parses a pub/sub message. If the channel or pattern was subscribed to + with a message handler, the handler is invoked instead of a parsed + message being returned. + """ + message_type = nativestr(response[0]) + if message_type == 'pmessage': + message = { + 'type': message_type, + 'pattern': response[1], + 'channel': response[2], + 'data': response[3] + } + else: + message = { + 'type': message_type, + 'pattern': None, + 'channel': response[1], + 'data': response[2] + } + + # if this is an unsubscribe message, remove it from memory + if message_type in self.UNSUBSCRIBE_MESSAGE_TYPES: + subscribed_dict = None + if message_type == 'punsubscribe': + subscribed_dict = self.patterns + else: + subscribed_dict = self.channels + try: + del subscribed_dict[message['channel']] + except KeyError: + pass + + if message_type in self.PUBLISH_MESSAGE_TYPES: + # if there's a message handler, invoke it + handler = None + if message_type == 'pmessage': + handler = self.patterns.get(message['pattern'], None) + else: + handler = self.channels.get(message['channel'], None) + if handler: + handler(message) + return None + else: + # this is a subscribe/unsubscribe message. ignore if we don't + # want them + if ignore_subscribe_messages or self.ignore_subscribe_messages: + return None + + return message + + def run_in_thread(self, sleep_time=0, daemon=False): + for channel, handler in iteritems(self.channels): + if handler is None: + raise PubSubError("Channel: '%s' has no handler registered") + for pattern, handler in iteritems(self.patterns): + if handler is None: + raise PubSubError("Pattern: '%s' has no handler registered") + + thread = PubSubWorkerThread(self, sleep_time, daemon=daemon) + thread.start() + return thread + + +class PubSubWorkerThread(threading.Thread): + def __init__(self, pubsub, sleep_time, daemon=False): + super(PubSubWorkerThread, self).__init__() + self.daemon = daemon + self.pubsub = pubsub + self.sleep_time = sleep_time + self._running = False + + def run(self): + if self._running: + return + self._running = True + pubsub = self.pubsub + sleep_time = self.sleep_time + while pubsub.subscribed: + pubsub.get_message(ignore_subscribe_messages=True, + timeout=sleep_time) + pubsub.close() + self._running = False + + def stop(self): + # stopping simply unsubscribes from all channels and patterns. + # the unsubscribe responses that are generated will short circuit + # the loop in run(), calling pubsub.close() to clean up the connection + self.pubsub.unsubscribe() + self.pubsub.punsubscribe() + + +class BasePipeline(object): + """ + Pipelines provide a way to transmit multiple commands to the Redis server + in one transmission. This is convenient for batch processing, such as + saving all the values in a list to Redis. + + All commands executed within a pipeline are wrapped with MULTI and EXEC + calls. This guarantees all commands executed in the pipeline will be + executed atomically. + + Any command raising an exception does *not* halt the execution of + subsequent commands in the pipeline. Instead, the exception is caught + and its instance is placed into the response list returned by execute(). + Code iterating over the response list should be able to deal with an + instance of an exception as a potential value. In general, these will be + ResponseError exceptions, such as those raised when issuing a command + on a key of a different datatype. + """ + + UNWATCH_COMMANDS = set(('DISCARD', 'EXEC', 'UNWATCH')) + + def __init__(self, connection_pool, response_callbacks, transaction, + shard_hint): + self.connection_pool = connection_pool + self.connection = None + self.response_callbacks = response_callbacks + self.transaction = transaction + self.shard_hint = shard_hint + + self.watching = False + self.reset() + + def __enter__(self): + return self + + def __exit__(self, exc_type, exc_value, traceback): + self.reset() + + def __del__(self): + try: + self.reset() + except Exception: + pass + + def __len__(self): + return len(self.command_stack) + + def reset(self): + self.command_stack = [] + self.scripts = set() + # make sure to reset the connection state in the event that we were + # watching something + if self.watching and self.connection: + try: + # call this manually since our unwatch or + # immediate_execute_command methods can call reset() + self.connection.send_command('UNWATCH') + self.connection.read_response() + except ConnectionError: + # disconnect will also remove any previous WATCHes + self.connection.disconnect() + # clean up the other instance attributes + self.watching = False + self.explicit_transaction = False + # we can safely return the connection to the pool here since we're + # sure we're no longer WATCHing anything + if self.connection: + self.connection_pool.release(self.connection) + self.connection = None + + def multi(self): + """ + Start a transactional block of the pipeline after WATCH commands + are issued. End the transactional block with `execute`. + """ + if self.explicit_transaction: + raise RedisError('Cannot issue nested calls to MULTI') + if self.command_stack: + raise RedisError('Commands without an initial WATCH have already ' + 'been issued') + self.explicit_transaction = True + + def execute_command(self, *args, **kwargs): + if (self.watching or args[0] == 'WATCH') and \ + not self.explicit_transaction: + return self.immediate_execute_command(*args, **kwargs) + return self.pipeline_execute_command(*args, **kwargs) + + def immediate_execute_command(self, *args, **options): + """ + Execute a command immediately, but don't auto-retry on a + ConnectionError if we're already WATCHing a variable. Used when + issuing WATCH or subsequent commands retrieving their values but before + MULTI is called. + """ + command_name = args[0] + conn = self.connection + # if this is the first call, we need a connection + if not conn: + conn = self.connection_pool.get_connection(command_name, + self.shard_hint) + self.connection = conn + try: + conn.send_command(*args) + return self.parse_response(conn, command_name, **options) + except (ConnectionError, TimeoutError) as e: + conn.disconnect() + if not conn.retry_on_timeout and isinstance(e, TimeoutError): + raise + # if we're not already watching, we can safely retry the command + try: + if not self.watching: + conn.send_command(*args) + return self.parse_response(conn, command_name, **options) + except ConnectionError: + # the retry failed so cleanup. + conn.disconnect() + self.reset() + raise + + def pipeline_execute_command(self, *args, **options): + """ + Stage a command to be executed when execute() is next called + + Returns the current Pipeline object back so commands can be + chained together, such as: + + pipe = pipe.set('foo', 'bar').incr('baz').decr('bang') + + At some other point, you can then run: pipe.execute(), + which will execute all commands queued in the pipe. + """ + self.command_stack.append((args, options)) + return self + + def _execute_transaction(self, connection, commands, raise_on_error): + cmds = chain([(('MULTI', ), {})], commands, [(('EXEC', ), {})]) + all_cmds = connection.pack_commands([args for args, _ in cmds]) + connection.send_packed_command(all_cmds) + errors = [] + + # parse off the response for MULTI + # NOTE: we need to handle ResponseErrors here and continue + # so that we read all the additional command messages from + # the socket + try: + self.parse_response(connection, '_') + except ResponseError: + errors.append((0, sys.exc_info()[1])) + + # and all the other commands + for i, command in enumerate(commands): + try: + self.parse_response(connection, '_') + except ResponseError: + ex = sys.exc_info()[1] + self.annotate_exception(ex, i + 1, command[0]) + errors.append((i, ex)) + + # parse the EXEC. + try: + response = self.parse_response(connection, '_') + except ExecAbortError: + if self.explicit_transaction: + self.immediate_execute_command('DISCARD') + if errors: + raise errors[0][1] + raise sys.exc_info()[1] + + if response is None: + raise WatchError("Watched variable changed.") + + # put any parse errors into the response + for i, e in errors: + response.insert(i, e) + + if len(response) != len(commands): + self.connection.disconnect() + raise ResponseError("Wrong number of response items from " + "pipeline execution") + + # find any errors in the response and raise if necessary + if raise_on_error: + self.raise_first_error(commands, response) + + # We have to run response callbacks manually + data = [] + for r, cmd in izip(response, commands): + if not isinstance(r, Exception): + args, options = cmd + command_name = args[0] + if command_name in self.response_callbacks: + r = self.response_callbacks[command_name](r, **options) + data.append(r) + return data + + def _execute_pipeline(self, connection, commands, raise_on_error): + # build up all commands into a single request to increase network perf + all_cmds = connection.pack_commands([args for args, _ in commands]) + connection.send_packed_command(all_cmds) + + response = [] + for args, options in commands: + try: + response.append( + self.parse_response(connection, args[0], **options)) + except ResponseError: + response.append(sys.exc_info()[1]) + + if raise_on_error: + self.raise_first_error(commands, response) + return response + + def raise_first_error(self, commands, response): + for i, r in enumerate(response): + if isinstance(r, ResponseError): + self.annotate_exception(r, i + 1, commands[i][0]) + raise r + + def annotate_exception(self, exception, number, command): + cmd = safe_unicode(' ').join(imap(safe_unicode, command)) + msg = unicode('Command # %d (%s) of pipeline caused error: %s') % ( + number, cmd, safe_unicode(exception.args[0])) + exception.args = (msg,) + exception.args[1:] + + def parse_response(self, connection, command_name, **options): + result = StrictRedis.parse_response( + self, connection, command_name, **options) + if command_name in self.UNWATCH_COMMANDS: + self.watching = False + elif command_name == 'WATCH': + self.watching = True + return result + + def load_scripts(self): + # make sure all scripts that are about to be run on this pipeline exist + scripts = list(self.scripts) + immediate = self.immediate_execute_command + shas = [s.sha for s in scripts] + # we can't use the normal script_* methods because they would just + # get buffered in the pipeline. + exists = immediate('SCRIPT EXISTS', *shas) + if not all(exists): + for s, exist in izip(scripts, exists): + if not exist: + s.sha = immediate('SCRIPT LOAD', s.script) + + def execute(self, raise_on_error=True): + "Execute all the commands in the current pipeline" + stack = self.command_stack + if not stack: + return [] + if self.scripts: + self.load_scripts() + if self.transaction or self.explicit_transaction: + execute = self._execute_transaction + else: + execute = self._execute_pipeline + + conn = self.connection + if not conn: + conn = self.connection_pool.get_connection('MULTI', + self.shard_hint) + # assign to self.connection so reset() releases the connection + # back to the pool after we're done + self.connection = conn + + try: + return execute(conn, stack, raise_on_error) + except (ConnectionError, TimeoutError) as e: + conn.disconnect() + if not conn.retry_on_timeout and isinstance(e, TimeoutError): + raise + # if we were watching a variable, the watch is no longer valid + # since this connection has died. raise a WatchError, which + # indicates the user should retry his transaction. If this is more + # than a temporary failure, the WATCH that the user next issues + # will fail, propegating the real ConnectionError + if self.watching: + raise WatchError("A ConnectionError occured on while watching " + "one or more keys") + # otherwise, it's safe to retry since the transaction isn't + # predicated on any state + return execute(conn, stack, raise_on_error) + finally: + self.reset() + + def watch(self, *names): + "Watches the values at keys ``names``" + if self.explicit_transaction: + raise RedisError('Cannot issue a WATCH after a MULTI') + return self.execute_command('WATCH', *names) + + def unwatch(self): + "Unwatches all previously specified keys" + return self.watching and self.execute_command('UNWATCH') or True + + +class StrictPipeline(BasePipeline, StrictRedis): + "Pipeline for the StrictRedis class" + pass + + +class Pipeline(BasePipeline, Redis): + "Pipeline for the Redis class" + pass + + +class Script(object): + "An executable Lua script object returned by ``register_script``" + + def __init__(self, registered_client, script): + self.registered_client = registered_client + self.script = script + # Precalculate and store the SHA1 hex digest of the script. + + if isinstance(script, basestring): + # We need the encoding from the client in order to generate an + # accurate byte representation of the script + encoder = registered_client.connection_pool.get_encoder() + script = encoder.encode(script) + self.sha = hashlib.sha1(script).hexdigest() + + def __call__(self, keys=[], args=[], client=None): + "Execute the script, passing any required ``args``" + if client is None: + client = self.registered_client + args = tuple(keys) + tuple(args) + # make sure the Redis server knows about the script + if isinstance(client, BasePipeline): + # Make sure the pipeline can register the script before executing. + client.scripts.add(self) + try: + return client.evalsha(self.sha, len(keys), *args) + except NoScriptError: + # Maybe the client is pointed to a differnet server than the client + # that created this instance? + # Overwrite the sha just in case there was a discrepancy. + self.sha = client.script_load(self.script) + return client.evalsha(self.sha, len(keys), *args) diff --git a/venv/lib/python3.6/site-packages/redis/connection.py b/venv/lib/python3.6/site-packages/redis/connection.py new file mode 100644 index 0000000..c8f0513 --- /dev/null +++ b/venv/lib/python3.6/site-packages/redis/connection.py @@ -0,0 +1,1117 @@ +from __future__ import with_statement +from distutils.version import StrictVersion +from itertools import chain +import os +import socket +import sys +import threading +import warnings + +try: + import ssl + ssl_available = True +except ImportError: + ssl_available = False + +from redis._compat import (b, xrange, imap, byte_to_chr, unicode, bytes, long, + BytesIO, nativestr, basestring, iteritems, + LifoQueue, Empty, Full, urlparse, parse_qs, + recv, recv_into, select, unquote) +from redis.exceptions import ( + RedisError, + ConnectionError, + TimeoutError, + BusyLoadingError, + ResponseError, + InvalidResponse, + AuthenticationError, + NoScriptError, + ExecAbortError, + ReadOnlyError +) +from redis.utils import HIREDIS_AVAILABLE +if HIREDIS_AVAILABLE: + import hiredis + + hiredis_version = StrictVersion(hiredis.__version__) + HIREDIS_SUPPORTS_CALLABLE_ERRORS = \ + hiredis_version >= StrictVersion('0.1.3') + HIREDIS_SUPPORTS_BYTE_BUFFER = \ + hiredis_version >= StrictVersion('0.1.4') + + if not HIREDIS_SUPPORTS_BYTE_BUFFER: + msg = ("redis-py works best with hiredis >= 0.1.4. You're running " + "hiredis %s. Please consider upgrading." % hiredis.__version__) + warnings.warn(msg) + + HIREDIS_USE_BYTE_BUFFER = True + # only use byte buffer if hiredis supports it and the Python version + # is >= 2.7 + if not HIREDIS_SUPPORTS_BYTE_BUFFER or ( + sys.version_info[0] == 2 and sys.version_info[1] < 7): + HIREDIS_USE_BYTE_BUFFER = False + +SYM_STAR = b('*') +SYM_DOLLAR = b('$') +SYM_CRLF = b('\r\n') +SYM_EMPTY = b('') + +SERVER_CLOSED_CONNECTION_ERROR = "Connection closed by server." + + +class Token(object): + """ + Literal strings in Redis commands, such as the command names and any + hard-coded arguments are wrapped in this class so we know not to apply + and encoding rules on them. + """ + + _cache = {} + + @classmethod + def get_token(cls, value): + "Gets a cached token object or creates a new one if not already cached" + + # Use try/except because after running for a short time most tokens + # should already be cached + try: + return cls._cache[value] + except KeyError: + token = Token(value) + cls._cache[value] = token + return token + + def __init__(self, value): + if isinstance(value, Token): + value = value.value + self.value = value + self.encoded_value = b(value) + + def __repr__(self): + return self.value + + def __str__(self): + return self.value + + +class Encoder(object): + "Encode strings to bytes and decode bytes to strings" + + def __init__(self, encoding, encoding_errors, decode_responses): + self.encoding = encoding + self.encoding_errors = encoding_errors + self.decode_responses = decode_responses + + def encode(self, value): + "Return a bytestring representation of the value" + if isinstance(value, Token): + return value.encoded_value + elif isinstance(value, bytes): + return value + elif isinstance(value, (int, long)): + value = b(str(value)) + elif isinstance(value, float): + value = b(repr(value)) + elif not isinstance(value, basestring): + # an object we don't know how to deal with. default to unicode() + value = unicode(value) + if isinstance(value, unicode): + value = value.encode(self.encoding, self.encoding_errors) + return value + + def decode(self, value, force=False): + "Return a unicode string from the byte representation" + if (self.decode_responses or force) and isinstance(value, bytes): + value = value.decode(self.encoding, self.encoding_errors) + return value + + +class BaseParser(object): + EXCEPTION_CLASSES = { + 'ERR': { + 'max number of clients reached': ConnectionError + }, + 'EXECABORT': ExecAbortError, + 'LOADING': BusyLoadingError, + 'NOSCRIPT': NoScriptError, + 'READONLY': ReadOnlyError, + } + + def parse_error(self, response): + "Parse an error response" + error_code = response.split(' ')[0] + if error_code in self.EXCEPTION_CLASSES: + response = response[len(error_code) + 1:] + exception_class = self.EXCEPTION_CLASSES[error_code] + if isinstance(exception_class, dict): + exception_class = exception_class.get(response, ResponseError) + return exception_class(response) + return ResponseError(response) + + +class SocketBuffer(object): + def __init__(self, socket, socket_read_size): + self._sock = socket + self.socket_read_size = socket_read_size + self._buffer = BytesIO() + # number of bytes written to the buffer from the socket + self.bytes_written = 0 + # number of bytes read from the buffer + self.bytes_read = 0 + + @property + def length(self): + return self.bytes_written - self.bytes_read + + def _read_from_socket(self, length=None): + socket_read_size = self.socket_read_size + buf = self._buffer + buf.seek(self.bytes_written) + marker = 0 + + try: + while True: + data = recv(self._sock, socket_read_size) + # an empty string indicates the server shutdown the socket + if isinstance(data, bytes) and len(data) == 0: + raise socket.error(SERVER_CLOSED_CONNECTION_ERROR) + buf.write(data) + data_length = len(data) + self.bytes_written += data_length + marker += data_length + + if length is not None and length > marker: + continue + break + except socket.timeout: + raise TimeoutError("Timeout reading from socket") + except socket.error: + e = sys.exc_info()[1] + raise ConnectionError("Error while reading from socket: %s" % + (e.args,)) + + def read(self, length): + length = length + 2 # make sure to read the \r\n terminator + # make sure we've read enough data from the socket + if length > self.length: + self._read_from_socket(length - self.length) + + self._buffer.seek(self.bytes_read) + data = self._buffer.read(length) + self.bytes_read += len(data) + + # purge the buffer when we've consumed it all so it doesn't + # grow forever + if self.bytes_read == self.bytes_written: + self.purge() + + return data[:-2] + + def readline(self): + buf = self._buffer + buf.seek(self.bytes_read) + data = buf.readline() + while not data.endswith(SYM_CRLF): + # there's more data in the socket that we need + self._read_from_socket() + buf.seek(self.bytes_read) + data = buf.readline() + + self.bytes_read += len(data) + + # purge the buffer when we've consumed it all so it doesn't + # grow forever + if self.bytes_read == self.bytes_written: + self.purge() + + return data[:-2] + + def purge(self): + self._buffer.seek(0) + self._buffer.truncate() + self.bytes_written = 0 + self.bytes_read = 0 + + def close(self): + try: + self.purge() + self._buffer.close() + except: + # issue #633 suggests the purge/close somehow raised a + # BadFileDescriptor error. Perhaps the client ran out of + # memory or something else? It's probably OK to ignore + # any error being raised from purge/close since we're + # removing the reference to the instance below. + pass + self._buffer = None + self._sock = None + + +class PythonParser(BaseParser): + "Plain Python parsing class" + def __init__(self, socket_read_size): + self.socket_read_size = socket_read_size + self.encoder = None + self._sock = None + self._buffer = None + + def __del__(self): + try: + self.on_disconnect() + except Exception: + pass + + def on_connect(self, connection): + "Called when the socket connects" + self._sock = connection._sock + self._buffer = SocketBuffer(self._sock, self.socket_read_size) + self.encoder = connection.encoder + + def on_disconnect(self): + "Called when the socket disconnects" + if self._sock is not None: + self._sock.close() + self._sock = None + if self._buffer is not None: + self._buffer.close() + self._buffer = None + self.encoder = None + + def can_read(self): + return self._buffer and bool(self._buffer.length) + + def read_response(self): + response = self._buffer.readline() + if not response: + raise ConnectionError(SERVER_CLOSED_CONNECTION_ERROR) + + byte, response = byte_to_chr(response[0]), response[1:] + + if byte not in ('-', '+', ':', '$', '*'): + raise InvalidResponse("Protocol Error: %s, %s" % + (str(byte), str(response))) + + # server returned an error + if byte == '-': + response = nativestr(response) + error = self.parse_error(response) + # if the error is a ConnectionError, raise immediately so the user + # is notified + if isinstance(error, ConnectionError): + raise error + # otherwise, we're dealing with a ResponseError that might belong + # inside a pipeline response. the connection's read_response() + # and/or the pipeline's execute() will raise this error if + # necessary, so just return the exception instance here. + return error + # single value + elif byte == '+': + pass + # int value + elif byte == ':': + response = long(response) + # bulk response + elif byte == '$': + length = int(response) + if length == -1: + return None + response = self._buffer.read(length) + # multi-bulk response + elif byte == '*': + length = int(response) + if length == -1: + return None + response = [self.read_response() for i in xrange(length)] + if isinstance(response, bytes): + response = self.encoder.decode(response) + return response + + +class HiredisParser(BaseParser): + "Parser class for connections using Hiredis" + def __init__(self, socket_read_size): + if not HIREDIS_AVAILABLE: + raise RedisError("Hiredis is not installed") + self.socket_read_size = socket_read_size + + if HIREDIS_USE_BYTE_BUFFER: + self._buffer = bytearray(socket_read_size) + + def __del__(self): + try: + self.on_disconnect() + except Exception: + pass + + def on_connect(self, connection): + self._sock = connection._sock + kwargs = { + 'protocolError': InvalidResponse, + 'replyError': self.parse_error, + } + + # hiredis < 0.1.3 doesn't support functions that create exceptions + if not HIREDIS_SUPPORTS_CALLABLE_ERRORS: + kwargs['replyError'] = ResponseError + + if connection.encoder.decode_responses: + kwargs['encoding'] = connection.encoder.encoding + self._reader = hiredis.Reader(**kwargs) + self._next_response = False + + def on_disconnect(self): + self._sock = None + self._reader = None + self._next_response = False + + def can_read(self): + if not self._reader: + raise ConnectionError(SERVER_CLOSED_CONNECTION_ERROR) + + if self._next_response is False: + self._next_response = self._reader.gets() + return self._next_response is not False + + def read_response(self): + if not self._reader: + raise ConnectionError(SERVER_CLOSED_CONNECTION_ERROR) + + # _next_response might be cached from a can_read() call + if self._next_response is not False: + response = self._next_response + self._next_response = False + return response + + response = self._reader.gets() + socket_read_size = self.socket_read_size + while response is False: + try: + if HIREDIS_USE_BYTE_BUFFER: + bufflen = recv_into(self._sock, self._buffer) + if bufflen == 0: + raise socket.error(SERVER_CLOSED_CONNECTION_ERROR) + else: + buffer = recv(self._sock, socket_read_size) + # an empty string indicates the server shutdown the socket + if not isinstance(buffer, bytes) or len(buffer) == 0: + raise socket.error(SERVER_CLOSED_CONNECTION_ERROR) + except socket.timeout: + raise TimeoutError("Timeout reading from socket") + except socket.error: + e = sys.exc_info()[1] + raise ConnectionError("Error while reading from socket: %s" % + (e.args,)) + if HIREDIS_USE_BYTE_BUFFER: + self._reader.feed(self._buffer, 0, bufflen) + else: + self._reader.feed(buffer) + response = self._reader.gets() + # if an older version of hiredis is installed, we need to attempt + # to convert ResponseErrors to their appropriate types. + if not HIREDIS_SUPPORTS_CALLABLE_ERRORS: + if isinstance(response, ResponseError): + response = self.parse_error(response.args[0]) + elif isinstance(response, list) and response and \ + isinstance(response[0], ResponseError): + response[0] = self.parse_error(response[0].args[0]) + # if the response is a ConnectionError or the response is a list and + # the first item is a ConnectionError, raise it as something bad + # happened + if isinstance(response, ConnectionError): + raise response + elif isinstance(response, list) and response and \ + isinstance(response[0], ConnectionError): + raise response[0] + return response + + +if HIREDIS_AVAILABLE: + DefaultParser = HiredisParser +else: + DefaultParser = PythonParser + + +class Connection(object): + "Manages TCP communication to and from a Redis server" + description_format = "Connection" + + def __init__(self, host='localhost', port=6379, db=0, password=None, + socket_timeout=None, socket_connect_timeout=None, + socket_keepalive=False, socket_keepalive_options=None, + retry_on_timeout=False, encoding='utf-8', + encoding_errors='strict', decode_responses=False, + parser_class=DefaultParser, socket_read_size=65536): + self.pid = os.getpid() + self.host = host + self.port = int(port) + self.db = db + self.password = password + self.socket_timeout = socket_timeout + self.socket_connect_timeout = socket_connect_timeout or socket_timeout + self.socket_keepalive = socket_keepalive + self.socket_keepalive_options = socket_keepalive_options or {} + self.retry_on_timeout = retry_on_timeout + self.encoder = Encoder(encoding, encoding_errors, decode_responses) + self._sock = None + self._parser = parser_class(socket_read_size=socket_read_size) + self._description_args = { + 'host': self.host, + 'port': self.port, + 'db': self.db, + } + self._connect_callbacks = [] + + def __repr__(self): + return self.description_format % self._description_args + + def __del__(self): + try: + self.disconnect() + except Exception: + pass + + def register_connect_callback(self, callback): + self._connect_callbacks.append(callback) + + def clear_connect_callbacks(self): + self._connect_callbacks = [] + + def connect(self): + "Connects to the Redis server if not already connected" + if self._sock: + return + try: + sock = self._connect() + except socket.timeout: + raise TimeoutError("Timeout connecting to server") + except socket.error: + e = sys.exc_info()[1] + raise ConnectionError(self._error_message(e)) + + self._sock = sock + try: + self.on_connect() + except RedisError: + # clean up after any error in on_connect + self.disconnect() + raise + + # run any user callbacks. right now the only internal callback + # is for pubsub channel/pattern resubscription + for callback in self._connect_callbacks: + callback(self) + + def _connect(self): + "Create a TCP socket connection" + # we want to mimic what socket.create_connection does to support + # ipv4/ipv6, but we want to set options prior to calling + # socket.connect() + err = None + for res in socket.getaddrinfo(self.host, self.port, 0, + socket.SOCK_STREAM): + family, socktype, proto, canonname, socket_address = res + sock = None + try: + sock = socket.socket(family, socktype, proto) + # TCP_NODELAY + sock.setsockopt(socket.IPPROTO_TCP, socket.TCP_NODELAY, 1) + + # TCP_KEEPALIVE + if self.socket_keepalive: + sock.setsockopt(socket.SOL_SOCKET, socket.SO_KEEPALIVE, 1) + for k, v in iteritems(self.socket_keepalive_options): + sock.setsockopt(socket.SOL_TCP, k, v) + + # set the socket_connect_timeout before we connect + sock.settimeout(self.socket_connect_timeout) + + # connect + sock.connect(socket_address) + + # set the socket_timeout now that we're connected + sock.settimeout(self.socket_timeout) + return sock + + except socket.error as _: + err = _ + if sock is not None: + sock.close() + + if err is not None: + raise err + raise socket.error("socket.getaddrinfo returned an empty list") + + def _error_message(self, exception): + # args for socket.error can either be (errno, "message") + # or just "message" + if len(exception.args) == 1: + return "Error connecting to %s:%s. %s." % \ + (self.host, self.port, exception.args[0]) + else: + return "Error %s connecting to %s:%s. %s." % \ + (exception.args[0], self.host, self.port, exception.args[1]) + + def on_connect(self): + "Initialize the connection, authenticate and select a database" + self._parser.on_connect(self) + + # if a password is specified, authenticate + if self.password: + self.send_command('AUTH', self.password) + if nativestr(self.read_response()) != 'OK': + raise AuthenticationError('Invalid Password') + + # if a database is specified, switch to it + if self.db: + self.send_command('SELECT', self.db) + if nativestr(self.read_response()) != 'OK': + raise ConnectionError('Invalid Database') + + def disconnect(self): + "Disconnects from the Redis server" + self._parser.on_disconnect() + if self._sock is None: + return + try: + self._sock.shutdown(socket.SHUT_RDWR) + self._sock.close() + except socket.error: + pass + self._sock = None + + def send_packed_command(self, command): + "Send an already packed command to the Redis server" + if not self._sock: + self.connect() + try: + if isinstance(command, str): + command = [command] + for item in command: + self._sock.sendall(item) + except socket.timeout: + self.disconnect() + raise TimeoutError("Timeout writing to socket") + except socket.error: + e = sys.exc_info()[1] + self.disconnect() + if len(e.args) == 1: + errno, errmsg = 'UNKNOWN', e.args[0] + else: + errno = e.args[0] + errmsg = e.args[1] + raise ConnectionError("Error %s while writing to socket. %s." % + (errno, errmsg)) + except: + self.disconnect() + raise + + def send_command(self, *args): + "Pack and send a command to the Redis server" + self.send_packed_command(self.pack_command(*args)) + + def can_read(self, timeout=0): + "Poll the socket to see if there's data that can be read." + sock = self._sock + if not sock: + self.connect() + sock = self._sock + return self._parser.can_read() or \ + bool(select([sock], [], [], timeout)[0]) + + def read_response(self): + "Read the response from a previously sent command" + try: + response = self._parser.read_response() + except: + self.disconnect() + raise + if isinstance(response, ResponseError): + raise response + return response + + def pack_command(self, *args): + "Pack a series of arguments into the Redis protocol" + output = [] + # the client might have included 1 or more literal arguments in + # the command name, e.g., 'CONFIG GET'. The Redis server expects these + # arguments to be sent separately, so split the first argument + # manually. All of these arguements get wrapped in the Token class + # to prevent them from being encoded. + command = args[0] + if ' ' in command: + args = tuple([Token.get_token(s) + for s in command.split()]) + args[1:] + else: + args = (Token.get_token(command),) + args[1:] + + buff = SYM_EMPTY.join( + (SYM_STAR, b(str(len(args))), SYM_CRLF)) + + for arg in imap(self.encoder.encode, args): + # to avoid large string mallocs, chunk the command into the + # output list if we're sending large values + if len(buff) > 6000 or len(arg) > 6000: + buff = SYM_EMPTY.join( + (buff, SYM_DOLLAR, b(str(len(arg))), SYM_CRLF)) + output.append(buff) + output.append(arg) + buff = SYM_CRLF + else: + buff = SYM_EMPTY.join((buff, SYM_DOLLAR, b(str(len(arg))), + SYM_CRLF, arg, SYM_CRLF)) + output.append(buff) + return output + + def pack_commands(self, commands): + "Pack multiple commands into the Redis protocol" + output = [] + pieces = [] + buffer_length = 0 + + for cmd in commands: + for chunk in self.pack_command(*cmd): + pieces.append(chunk) + buffer_length += len(chunk) + + if buffer_length > 6000: + output.append(SYM_EMPTY.join(pieces)) + buffer_length = 0 + pieces = [] + + if pieces: + output.append(SYM_EMPTY.join(pieces)) + return output + + +class SSLConnection(Connection): + description_format = "SSLConnection" + + def __init__(self, ssl_keyfile=None, ssl_certfile=None, ssl_cert_reqs=None, + ssl_ca_certs=None, **kwargs): + if not ssl_available: + raise RedisError("Python wasn't built with SSL support") + + super(SSLConnection, self).__init__(**kwargs) + + self.keyfile = ssl_keyfile + self.certfile = ssl_certfile + if ssl_cert_reqs is None: + ssl_cert_reqs = ssl.CERT_NONE + elif isinstance(ssl_cert_reqs, basestring): + CERT_REQS = { + 'none': ssl.CERT_NONE, + 'optional': ssl.CERT_OPTIONAL, + 'required': ssl.CERT_REQUIRED + } + if ssl_cert_reqs not in CERT_REQS: + raise RedisError( + "Invalid SSL Certificate Requirements Flag: %s" % + ssl_cert_reqs) + ssl_cert_reqs = CERT_REQS[ssl_cert_reqs] + self.cert_reqs = ssl_cert_reqs + self.ca_certs = ssl_ca_certs + + def _connect(self): + "Wrap the socket with SSL support" + sock = super(SSLConnection, self)._connect() + sock = ssl.wrap_socket(sock, + cert_reqs=self.cert_reqs, + keyfile=self.keyfile, + certfile=self.certfile, + ca_certs=self.ca_certs) + return sock + + +class UnixDomainSocketConnection(Connection): + description_format = "UnixDomainSocketConnection" + + def __init__(self, path='', db=0, password=None, + socket_timeout=None, encoding='utf-8', + encoding_errors='strict', decode_responses=False, + retry_on_timeout=False, + parser_class=DefaultParser, socket_read_size=65536): + self.pid = os.getpid() + self.path = path + self.db = db + self.password = password + self.socket_timeout = socket_timeout + self.retry_on_timeout = retry_on_timeout + self.encoder = Encoder(encoding, encoding_errors, decode_responses) + self._sock = None + self._parser = parser_class(socket_read_size=socket_read_size) + self._description_args = { + 'path': self.path, + 'db': self.db, + } + self._connect_callbacks = [] + + def _connect(self): + "Create a Unix domain socket connection" + sock = socket.socket(socket.AF_UNIX, socket.SOCK_STREAM) + sock.settimeout(self.socket_timeout) + sock.connect(self.path) + return sock + + def _error_message(self, exception): + # args for socket.error can either be (errno, "message") + # or just "message" + if len(exception.args) == 1: + return "Error connecting to unix socket: %s. %s." % \ + (self.path, exception.args[0]) + else: + return "Error %s connecting to unix socket: %s. %s." % \ + (exception.args[0], self.path, exception.args[1]) + + +FALSE_STRINGS = ('0', 'F', 'FALSE', 'N', 'NO') + + +def to_bool(value): + if value is None or value == '': + return None + if isinstance(value, basestring) and value.upper() in FALSE_STRINGS: + return False + return bool(value) + + +URL_QUERY_ARGUMENT_PARSERS = { + 'socket_timeout': float, + 'socket_connect_timeout': float, + 'socket_keepalive': to_bool, + 'retry_on_timeout': to_bool +} + + +class ConnectionPool(object): + "Generic connection pool" + @classmethod + def from_url(cls, url, db=None, decode_components=False, **kwargs): + """ + Return a connection pool configured from the given URL. + + For example:: + + redis://[:password]@localhost:6379/0 + rediss://[:password]@localhost:6379/0 + unix://[:password]@/path/to/socket.sock?db=0 + + Three URL schemes are supported: + + - ```redis://`` + `_ creates a + normal TCP socket connection + - ```rediss://`` + `_ creates a + SSL wrapped TCP socket connection + - ``unix://`` creates a Unix Domain Socket connection + + There are several ways to specify a database number. The parse function + will return the first specified option: + 1. A ``db`` querystring option, e.g. redis://localhost?db=0 + 2. If using the redis:// scheme, the path argument of the url, e.g. + redis://localhost/0 + 3. The ``db`` argument to this function. + + If none of these options are specified, db=0 is used. + + The ``decode_components`` argument allows this function to work with + percent-encoded URLs. If this argument is set to ``True`` all ``%xx`` + escapes will be replaced by their single-character equivalents after + the URL has been parsed. This only applies to the ``hostname``, + ``path``, and ``password`` components. + + Any additional querystring arguments and keyword arguments will be + passed along to the ConnectionPool class's initializer. The querystring + arguments ``socket_connect_timeout`` and ``socket_timeout`` if supplied + are parsed as float values. The arguments ``socket_keepalive`` and + ``retry_on_timeout`` are parsed to boolean values that accept + True/False, Yes/No values to indicate state. Invalid types cause a + ``UserWarning`` to be raised. In the case of conflicting arguments, + querystring arguments always win. + """ + url_string = url + url = urlparse(url) + qs = '' + + # in python2.6, custom URL schemes don't recognize querystring values + # they're left as part of the url.path. + if '?' in url.path and not url.query: + # chop the querystring including the ? off the end of the url + # and reparse it. + qs = url.path.split('?', 1)[1] + url = urlparse(url_string[:-(len(qs) + 1)]) + else: + qs = url.query + + url_options = {} + + for name, value in iteritems(parse_qs(qs)): + if value and len(value) > 0: + parser = URL_QUERY_ARGUMENT_PARSERS.get(name) + if parser: + try: + url_options[name] = parser(value[0]) + except (TypeError, ValueError): + warnings.warn(UserWarning( + "Invalid value for `%s` in connection URL." % name + )) + else: + url_options[name] = value[0] + + if decode_components: + password = unquote(url.password) if url.password else None + path = unquote(url.path) if url.path else None + hostname = unquote(url.hostname) if url.hostname else None + else: + password = url.password + path = url.path + hostname = url.hostname + + # We only support redis:// and unix:// schemes. + if url.scheme == 'unix': + url_options.update({ + 'password': password, + 'path': path, + 'connection_class': UnixDomainSocketConnection, + }) + + else: + url_options.update({ + 'host': hostname, + 'port': int(url.port or 6379), + 'password': password, + }) + + # If there's a path argument, use it as the db argument if a + # querystring value wasn't specified + if 'db' not in url_options and path: + try: + url_options['db'] = int(path.replace('/', '')) + except (AttributeError, ValueError): + pass + + if url.scheme == 'rediss': + url_options['connection_class'] = SSLConnection + + # last shot at the db value + url_options['db'] = int(url_options.get('db', db or 0)) + + # update the arguments from the URL values + kwargs.update(url_options) + + # backwards compatability + if 'charset' in kwargs: + warnings.warn(DeprecationWarning( + '"charset" is deprecated. Use "encoding" instead')) + kwargs['encoding'] = kwargs.pop('charset') + if 'errors' in kwargs: + warnings.warn(DeprecationWarning( + '"errors" is deprecated. Use "encoding_errors" instead')) + kwargs['encoding_errors'] = kwargs.pop('errors') + + return cls(**kwargs) + + def __init__(self, connection_class=Connection, max_connections=None, + **connection_kwargs): + """ + Create a connection pool. If max_connections is set, then this + object raises redis.ConnectionError when the pool's limit is reached. + + By default, TCP connections are created unless connection_class is + specified. Use redis.UnixDomainSocketConnection for unix sockets. + + Any additional keyword arguments are passed to the constructor of + connection_class. + """ + max_connections = max_connections or 2 ** 31 + if not isinstance(max_connections, (int, long)) or max_connections < 0: + raise ValueError('"max_connections" must be a positive integer') + + self.connection_class = connection_class + self.connection_kwargs = connection_kwargs + self.max_connections = max_connections + + self.reset() + + def __repr__(self): + return "%s<%s>" % ( + type(self).__name__, + self.connection_class.description_format % self.connection_kwargs, + ) + + def reset(self): + self.pid = os.getpid() + self._created_connections = 0 + self._available_connections = [] + self._in_use_connections = set() + self._check_lock = threading.Lock() + + def _checkpid(self): + if self.pid != os.getpid(): + with self._check_lock: + if self.pid == os.getpid(): + # another thread already did the work while we waited + # on the lock. + return + self.disconnect() + self.reset() + + def get_connection(self, command_name, *keys, **options): + "Get a connection from the pool" + self._checkpid() + try: + connection = self._available_connections.pop() + except IndexError: + connection = self.make_connection() + self._in_use_connections.add(connection) + return connection + + def get_encoder(self): + "Return an encoder based on encoding settings" + kwargs = self.connection_kwargs + return Encoder( + encoding=kwargs.get('encoding', 'utf-8'), + encoding_errors=kwargs.get('encoding_errors', 'strict'), + decode_responses=kwargs.get('decode_responses', False) + ) + + def make_connection(self): + "Create a new connection" + if self._created_connections >= self.max_connections: + raise ConnectionError("Too many connections") + self._created_connections += 1 + return self.connection_class(**self.connection_kwargs) + + def release(self, connection): + "Releases the connection back to the pool" + self._checkpid() + if connection.pid != self.pid: + return + self._in_use_connections.remove(connection) + self._available_connections.append(connection) + + def disconnect(self): + "Disconnects all connections in the pool" + all_conns = chain(self._available_connections, + self._in_use_connections) + for connection in all_conns: + connection.disconnect() + + +class BlockingConnectionPool(ConnectionPool): + """ + Thread-safe blocking connection pool:: + + >>> from redis.client import Redis + >>> client = Redis(connection_pool=BlockingConnectionPool()) + + It performs the same function as the default + ``:py:class: ~redis.connection.ConnectionPool`` implementation, in that, + it maintains a pool of reusable connections that can be shared by + multiple redis clients (safely across threads if required). + + The difference is that, in the event that a client tries to get a + connection from the pool when all of connections are in use, rather than + raising a ``:py:class: ~redis.exceptions.ConnectionError`` (as the default + ``:py:class: ~redis.connection.ConnectionPool`` implementation does), it + makes the client wait ("blocks") for a specified number of seconds until + a connection becomes available. + + Use ``max_connections`` to increase / decrease the pool size:: + + >>> pool = BlockingConnectionPool(max_connections=10) + + Use ``timeout`` to tell it either how many seconds to wait for a connection + to become available, or to block forever: + + # Block forever. + >>> pool = BlockingConnectionPool(timeout=None) + + # Raise a ``ConnectionError`` after five seconds if a connection is + # not available. + >>> pool = BlockingConnectionPool(timeout=5) + """ + def __init__(self, max_connections=50, timeout=20, + connection_class=Connection, queue_class=LifoQueue, + **connection_kwargs): + + self.queue_class = queue_class + self.timeout = timeout + super(BlockingConnectionPool, self).__init__( + connection_class=connection_class, + max_connections=max_connections, + **connection_kwargs) + + def reset(self): + self.pid = os.getpid() + self._check_lock = threading.Lock() + + # Create and fill up a thread safe queue with ``None`` values. + self.pool = self.queue_class(self.max_connections) + while True: + try: + self.pool.put_nowait(None) + except Full: + break + + # Keep a list of actual connection instances so that we can + # disconnect them later. + self._connections = [] + + def make_connection(self): + "Make a fresh connection." + connection = self.connection_class(**self.connection_kwargs) + self._connections.append(connection) + return connection + + def get_connection(self, command_name, *keys, **options): + """ + Get a connection, blocking for ``self.timeout`` until a connection + is available from the pool. + + If the connection returned is ``None`` then creates a new connection. + Because we use a last-in first-out queue, the existing connections + (having been returned to the pool after the initial ``None`` values + were added) will be returned before ``None`` values. This means we only + create new connections when we need to, i.e.: the actual number of + connections will only increase in response to demand. + """ + # Make sure we haven't changed process. + self._checkpid() + + # Try and get a connection from the pool. If one isn't available within + # self.timeout then raise a ``ConnectionError``. + connection = None + try: + connection = self.pool.get(block=True, timeout=self.timeout) + except Empty: + # Note that this is not caught by the redis client and will be + # raised unless handled by application code. If you want never to + raise ConnectionError("No connection available.") + + # If the ``connection`` is actually ``None`` then that's a cue to make + # a new connection to add to the pool. + if connection is None: + connection = self.make_connection() + + return connection + + def release(self, connection): + "Releases the connection back to the pool." + # Make sure we haven't changed process. + self._checkpid() + if connection.pid != self.pid: + return + + # Put the connection back into the pool. + try: + self.pool.put_nowait(connection) + except Full: + # perhaps the pool has been reset() after a fork? regardless, + # we don't want this connection + pass + + def disconnect(self): + "Disconnects all connections in the pool." + for connection in self._connections: + connection.disconnect() diff --git a/venv/lib/python3.6/site-packages/redis/exceptions.py b/venv/lib/python3.6/site-packages/redis/exceptions.py new file mode 100644 index 0000000..a8518c7 --- /dev/null +++ b/venv/lib/python3.6/site-packages/redis/exceptions.py @@ -0,0 +1,71 @@ +"Core exceptions raised by the Redis client" +from redis._compat import unicode + + +class RedisError(Exception): + pass + + +# python 2.5 doesn't implement Exception.__unicode__. Add it here to all +# our exception types +if not hasattr(RedisError, '__unicode__'): + def __unicode__(self): + if isinstance(self.args[0], unicode): + return self.args[0] + return unicode(self.args[0]) + RedisError.__unicode__ = __unicode__ + + +class AuthenticationError(RedisError): + pass + + +class ConnectionError(RedisError): + pass + + +class TimeoutError(RedisError): + pass + + +class BusyLoadingError(ConnectionError): + pass + + +class InvalidResponse(RedisError): + pass + + +class ResponseError(RedisError): + pass + + +class DataError(RedisError): + pass + + +class PubSubError(RedisError): + pass + + +class WatchError(RedisError): + pass + + +class NoScriptError(ResponseError): + pass + + +class ExecAbortError(ResponseError): + pass + + +class ReadOnlyError(ResponseError): + pass + + +class LockError(RedisError, ValueError): + "Errors acquiring or releasing a lock" + # NOTE: For backwards compatability, this class derives from ValueError. + # This was originally chosen to behave like threading.Lock. + pass diff --git a/venv/lib/python3.6/site-packages/redis/lock.py b/venv/lib/python3.6/site-packages/redis/lock.py new file mode 100644 index 0000000..90f0e7a --- /dev/null +++ b/venv/lib/python3.6/site-packages/redis/lock.py @@ -0,0 +1,272 @@ +import threading +import time as mod_time +import uuid +from redis.exceptions import LockError, WatchError +from redis.utils import dummy +from redis._compat import b + + +class Lock(object): + """ + A shared, distributed Lock. Using Redis for locking allows the Lock + to be shared across processes and/or machines. + + It's left to the user to resolve deadlock issues and make sure + multiple clients play nicely together. + """ + def __init__(self, redis, name, timeout=None, sleep=0.1, + blocking=True, blocking_timeout=None, thread_local=True): + """ + Create a new Lock instance named ``name`` using the Redis client + supplied by ``redis``. + + ``timeout`` indicates a maximum life for the lock. + By default, it will remain locked until release() is called. + ``timeout`` can be specified as a float or integer, both representing + the number of seconds to wait. + + ``sleep`` indicates the amount of time to sleep per loop iteration + when the lock is in blocking mode and another client is currently + holding the lock. + + ``blocking`` indicates whether calling ``acquire`` should block until + the lock has been acquired or to fail immediately, causing ``acquire`` + to return False and the lock not being acquired. Defaults to True. + Note this value can be overridden by passing a ``blocking`` + argument to ``acquire``. + + ``blocking_timeout`` indicates the maximum amount of time in seconds to + spend trying to acquire the lock. A value of ``None`` indicates + continue trying forever. ``blocking_timeout`` can be specified as a + float or integer, both representing the number of seconds to wait. + + ``thread_local`` indicates whether the lock token is placed in + thread-local storage. By default, the token is placed in thread local + storage so that a thread only sees its token, not a token set by + another thread. Consider the following timeline: + + time: 0, thread-1 acquires `my-lock`, with a timeout of 5 seconds. + thread-1 sets the token to "abc" + time: 1, thread-2 blocks trying to acquire `my-lock` using the + Lock instance. + time: 5, thread-1 has not yet completed. redis expires the lock + key. + time: 5, thread-2 acquired `my-lock` now that it's available. + thread-2 sets the token to "xyz" + time: 6, thread-1 finishes its work and calls release(). if the + token is *not* stored in thread local storage, then + thread-1 would see the token value as "xyz" and would be + able to successfully release the thread-2's lock. + + In some use cases it's necessary to disable thread local storage. For + example, if you have code where one thread acquires a lock and passes + that lock instance to a worker thread to release later. If thread + local storage isn't disabled in this case, the worker thread won't see + the token set by the thread that acquired the lock. Our assumption + is that these cases aren't common and as such default to using + thread local storage. + """ + self.redis = redis + self.name = name + self.timeout = timeout + self.sleep = sleep + self.blocking = blocking + self.blocking_timeout = blocking_timeout + self.thread_local = bool(thread_local) + self.local = threading.local() if self.thread_local else dummy() + self.local.token = None + if self.timeout and self.sleep > self.timeout: + raise LockError("'sleep' must be less than 'timeout'") + + def __enter__(self): + # force blocking, as otherwise the user would have to check whether + # the lock was actually acquired or not. + self.acquire(blocking=True) + return self + + def __exit__(self, exc_type, exc_value, traceback): + self.release() + + def acquire(self, blocking=None, blocking_timeout=None): + """ + Use Redis to hold a shared, distributed lock named ``name``. + Returns True once the lock is acquired. + + If ``blocking`` is False, always return immediately. If the lock + was acquired, return True, otherwise return False. + + ``blocking_timeout`` specifies the maximum number of seconds to + wait trying to acquire the lock. + """ + sleep = self.sleep + token = b(uuid.uuid1().hex) + if blocking is None: + blocking = self.blocking + if blocking_timeout is None: + blocking_timeout = self.blocking_timeout + stop_trying_at = None + if blocking_timeout is not None: + stop_trying_at = mod_time.time() + blocking_timeout + while 1: + if self.do_acquire(token): + self.local.token = token + return True + if not blocking: + return False + if stop_trying_at is not None and mod_time.time() > stop_trying_at: + return False + mod_time.sleep(sleep) + + def do_acquire(self, token): + if self.redis.setnx(self.name, token): + if self.timeout: + # convert to milliseconds + timeout = int(self.timeout * 1000) + self.redis.pexpire(self.name, timeout) + return True + return False + + def release(self): + "Releases the already acquired lock" + expected_token = self.local.token + if expected_token is None: + raise LockError("Cannot release an unlocked lock") + self.local.token = None + self.do_release(expected_token) + + def do_release(self, expected_token): + name = self.name + + def execute_release(pipe): + lock_value = pipe.get(name) + if lock_value != expected_token: + raise LockError("Cannot release a lock that's no longer owned") + pipe.delete(name) + + self.redis.transaction(execute_release, name) + + def extend(self, additional_time): + """ + Adds more time to an already acquired lock. + + ``additional_time`` can be specified as an integer or a float, both + representing the number of seconds to add. + """ + if self.local.token is None: + raise LockError("Cannot extend an unlocked lock") + if self.timeout is None: + raise LockError("Cannot extend a lock with no timeout") + return self.do_extend(additional_time) + + def do_extend(self, additional_time): + pipe = self.redis.pipeline() + pipe.watch(self.name) + lock_value = pipe.get(self.name) + if lock_value != self.local.token: + raise LockError("Cannot extend a lock that's no longer owned") + expiration = pipe.pttl(self.name) + if expiration is None or expiration < 0: + # Redis evicted the lock key between the previous get() and now + # we'll handle this when we call pexpire() + expiration = 0 + pipe.multi() + pipe.pexpire(self.name, expiration + int(additional_time * 1000)) + + try: + response = pipe.execute() + except WatchError: + # someone else acquired the lock + raise LockError("Cannot extend a lock that's no longer owned") + if not response[0]: + # pexpire returns False if the key doesn't exist + raise LockError("Cannot extend a lock that's no longer owned") + return True + + +class LuaLock(Lock): + """ + A lock implementation that uses Lua scripts rather than pipelines + and watches. + """ + lua_acquire = None + lua_release = None + lua_extend = None + + # KEYS[1] - lock name + # ARGV[1] - token + # ARGV[2] - timeout in milliseconds + # return 1 if lock was acquired, otherwise 0 + LUA_ACQUIRE_SCRIPT = """ + if redis.call('setnx', KEYS[1], ARGV[1]) == 1 then + if ARGV[2] ~= '' then + redis.call('pexpire', KEYS[1], ARGV[2]) + end + return 1 + end + return 0 + """ + + # KEYS[1] - lock name + # ARGS[1] - token + # return 1 if the lock was released, otherwise 0 + LUA_RELEASE_SCRIPT = """ + local token = redis.call('get', KEYS[1]) + if not token or token ~= ARGV[1] then + return 0 + end + redis.call('del', KEYS[1]) + return 1 + """ + + # KEYS[1] - lock name + # ARGS[1] - token + # ARGS[2] - additional milliseconds + # return 1 if the locks time was extended, otherwise 0 + LUA_EXTEND_SCRIPT = """ + local token = redis.call('get', KEYS[1]) + if not token or token ~= ARGV[1] then + return 0 + end + local expiration = redis.call('pttl', KEYS[1]) + if not expiration then + expiration = 0 + end + if expiration < 0 then + return 0 + end + redis.call('pexpire', KEYS[1], expiration + ARGV[2]) + return 1 + """ + + def __init__(self, *args, **kwargs): + super(LuaLock, self).__init__(*args, **kwargs) + LuaLock.register_scripts(self.redis) + + @classmethod + def register_scripts(cls, redis): + if cls.lua_acquire is None: + cls.lua_acquire = redis.register_script(cls.LUA_ACQUIRE_SCRIPT) + if cls.lua_release is None: + cls.lua_release = redis.register_script(cls.LUA_RELEASE_SCRIPT) + if cls.lua_extend is None: + cls.lua_extend = redis.register_script(cls.LUA_EXTEND_SCRIPT) + + def do_acquire(self, token): + timeout = self.timeout and int(self.timeout * 1000) or '' + return bool(self.lua_acquire(keys=[self.name], + args=[token, timeout], + client=self.redis)) + + def do_release(self, expected_token): + if not bool(self.lua_release(keys=[self.name], + args=[expected_token], + client=self.redis)): + raise LockError("Cannot release a lock that's no longer owned") + + def do_extend(self, additional_time): + additional_time = int(additional_time * 1000) + if not bool(self.lua_extend(keys=[self.name], + args=[self.local.token, additional_time], + client=self.redis)): + raise LockError("Cannot extend a lock that's no longer owned") + return True diff --git a/venv/lib/python3.6/site-packages/redis/sentinel.py b/venv/lib/python3.6/site-packages/redis/sentinel.py new file mode 100644 index 0000000..518fec5 --- /dev/null +++ b/venv/lib/python3.6/site-packages/redis/sentinel.py @@ -0,0 +1,297 @@ +import os +import random +import weakref + +from redis.client import StrictRedis +from redis.connection import ConnectionPool, Connection +from redis.exceptions import (ConnectionError, ResponseError, ReadOnlyError, + TimeoutError) +from redis._compat import iteritems, nativestr, xrange + + +class MasterNotFoundError(ConnectionError): + pass + + +class SlaveNotFoundError(ConnectionError): + pass + + +class SentinelManagedConnection(Connection): + def __init__(self, **kwargs): + self.connection_pool = kwargs.pop('connection_pool') + super(SentinelManagedConnection, self).__init__(**kwargs) + + def __repr__(self): + pool = self.connection_pool + s = '%s' % (type(self).__name__, pool.service_name) + if self.host: + host_info = ',host=%s,port=%s' % (self.host, self.port) + s = s % host_info + return s + + def connect_to(self, address): + self.host, self.port = address + super(SentinelManagedConnection, self).connect() + if self.connection_pool.check_connection: + self.send_command('PING') + if nativestr(self.read_response()) != 'PONG': + raise ConnectionError('PING failed') + + def connect(self): + if self._sock: + return # already connected + if self.connection_pool.is_master: + self.connect_to(self.connection_pool.get_master_address()) + else: + for slave in self.connection_pool.rotate_slaves(): + try: + return self.connect_to(slave) + except ConnectionError: + continue + raise SlaveNotFoundError # Never be here + + def read_response(self): + try: + return super(SentinelManagedConnection, self).read_response() + except ReadOnlyError: + if self.connection_pool.is_master: + # When talking to a master, a ReadOnlyError when likely + # indicates that the previous master that we're still connected + # to has been demoted to a slave and there's a new master. + # calling disconnect will force the connection to re-query + # sentinel during the next connect() attempt. + self.disconnect() + raise ConnectionError('The previous master is now a slave') + raise + + +class SentinelConnectionPool(ConnectionPool): + """ + Sentinel backed connection pool. + + If ``check_connection`` flag is set to True, SentinelManagedConnection + sends a PING command right after establishing the connection. + """ + + def __init__(self, service_name, sentinel_manager, **kwargs): + kwargs['connection_class'] = kwargs.get( + 'connection_class', SentinelManagedConnection) + self.is_master = kwargs.pop('is_master', True) + self.check_connection = kwargs.pop('check_connection', False) + super(SentinelConnectionPool, self).__init__(**kwargs) + self.connection_kwargs['connection_pool'] = weakref.proxy(self) + self.service_name = service_name + self.sentinel_manager = sentinel_manager + + def __repr__(self): + return "%s>> from redis.sentinel import Sentinel + >>> sentinel = Sentinel([('localhost', 26379)], socket_timeout=0.1) + >>> master = sentinel.master_for('mymaster', socket_timeout=0.1) + >>> master.set('foo', 'bar') + >>> slave = sentinel.slave_for('mymaster', socket_timeout=0.1) + >>> slave.get('foo') + 'bar' + + ``sentinels`` is a list of sentinel nodes. Each node is represented by + a pair (hostname, port). + + ``min_other_sentinels`` defined a minimum number of peers for a sentinel. + When querying a sentinel, if it doesn't meet this threshold, responses + from that sentinel won't be considered valid. + + ``sentinel_kwargs`` is a dictionary of connection arguments used when + connecting to sentinel instances. Any argument that can be passed to + a normal Redis connection can be specified here. If ``sentinel_kwargs`` is + not specified, any socket_timeout and socket_keepalive options specified + in ``connection_kwargs`` will be used. + + ``connection_kwargs`` are keyword arguments that will be used when + establishing a connection to a Redis server. + """ + + def __init__(self, sentinels, min_other_sentinels=0, sentinel_kwargs=None, + **connection_kwargs): + # if sentinel_kwargs isn't defined, use the socket_* options from + # connection_kwargs + if sentinel_kwargs is None: + sentinel_kwargs = dict([(k, v) + for k, v in iteritems(connection_kwargs) + if k.startswith('socket_') + ]) + self.sentinel_kwargs = sentinel_kwargs + + self.sentinels = [StrictRedis(hostname, port, **self.sentinel_kwargs) + for hostname, port in sentinels] + self.min_other_sentinels = min_other_sentinels + self.connection_kwargs = connection_kwargs + + def __repr__(self): + sentinel_addresses = [] + for sentinel in self.sentinels: + sentinel_addresses.append('%s:%s' % ( + sentinel.connection_pool.connection_kwargs['host'], + sentinel.connection_pool.connection_kwargs['port'], + )) + return '%s' % ( + type(self).__name__, + ','.join(sentinel_addresses)) + + def check_master_state(self, state, service_name): + if not state['is_master'] or state['is_sdown'] or state['is_odown']: + return False + # Check if our sentinel doesn't see other nodes + if state['num-other-sentinels'] < self.min_other_sentinels: + return False + return True + + def discover_master(self, service_name): + """ + Asks sentinel servers for the Redis master's address corresponding + to the service labeled ``service_name``. + + Returns a pair (address, port) or raises MasterNotFoundError if no + master is found. + """ + for sentinel_no, sentinel in enumerate(self.sentinels): + try: + masters = sentinel.sentinel_masters() + except (ConnectionError, TimeoutError): + continue + state = masters.get(service_name) + if state and self.check_master_state(state, service_name): + # Put this sentinel at the top of the list + self.sentinels[0], self.sentinels[sentinel_no] = ( + sentinel, self.sentinels[0]) + return state['ip'], state['port'] + raise MasterNotFoundError("No master found for %r" % (service_name,)) + + def filter_slaves(self, slaves): + "Remove slaves that are in an ODOWN or SDOWN state" + slaves_alive = [] + for slave in slaves: + if slave['is_odown'] or slave['is_sdown']: + continue + slaves_alive.append((slave['ip'], slave['port'])) + return slaves_alive + + def discover_slaves(self, service_name): + "Returns a list of alive slaves for service ``service_name``" + for sentinel in self.sentinels: + try: + slaves = sentinel.sentinel_slaves(service_name) + except (ConnectionError, ResponseError, TimeoutError): + continue + slaves = self.filter_slaves(slaves) + if slaves: + return slaves + return [] + + def master_for(self, service_name, redis_class=StrictRedis, + connection_pool_class=SentinelConnectionPool, **kwargs): + """ + Returns a redis client instance for the ``service_name`` master. + + A SentinelConnectionPool class is used to retrive the master's + address before establishing a new connection. + + NOTE: If the master's address has changed, any cached connections to + the old master are closed. + + By default clients will be a redis.StrictRedis instance. Specify a + different class to the ``redis_class`` argument if you desire + something different. + + The ``connection_pool_class`` specifies the connection pool to use. + The SentinelConnectionPool will be used by default. + + All other keyword arguments are merged with any connection_kwargs + passed to this class and passed to the connection pool as keyword + arguments to be used to initialize Redis connections. + """ + kwargs['is_master'] = True + connection_kwargs = dict(self.connection_kwargs) + connection_kwargs.update(kwargs) + return redis_class(connection_pool=connection_pool_class( + service_name, self, **connection_kwargs)) + + def slave_for(self, service_name, redis_class=StrictRedis, + connection_pool_class=SentinelConnectionPool, **kwargs): + """ + Returns redis client instance for the ``service_name`` slave(s). + + A SentinelConnectionPool class is used to retrive the slave's + address before establishing a new connection. + + By default clients will be a redis.StrictRedis instance. Specify a + different class to the ``redis_class`` argument if you desire + something different. + + The ``connection_pool_class`` specifies the connection pool to use. + The SentinelConnectionPool will be used by default. + + All other keyword arguments are merged with any connection_kwargs + passed to this class and passed to the connection pool as keyword + arguments to be used to initialize Redis connections. + """ + kwargs['is_master'] = False + connection_kwargs = dict(self.connection_kwargs) + connection_kwargs.update(kwargs) + return redis_class(connection_pool=connection_pool_class( + service_name, self, **connection_kwargs)) diff --git a/venv/lib/python3.6/site-packages/redis/utils.py b/venv/lib/python3.6/site-packages/redis/utils.py new file mode 100644 index 0000000..0b0067e --- /dev/null +++ b/venv/lib/python3.6/site-packages/redis/utils.py @@ -0,0 +1,33 @@ +from contextlib import contextmanager + + +try: + import hiredis + HIREDIS_AVAILABLE = True +except ImportError: + HIREDIS_AVAILABLE = False + + +def from_url(url, db=None, **kwargs): + """ + Returns an active Redis client generated from the given database URL. + + Will attempt to extract the database id from the path url fragment, if + none is provided. + """ + from redis.client import Redis + return Redis.from_url(url, db, **kwargs) + + +@contextmanager +def pipeline(redis_obj): + p = redis_obj.pipeline() + yield p + p.execute() + + +class dummy(object): + """ + Instances of this class can be used as an attribute container. + """ + pass