From dce73ccdb4adb4188ec99d62e2dfb0a6a3c81be0 Mon Sep 17 00:00:00 2001 From: Julian Porter Date: Wed, 2 Aug 2017 00:42:11 +0100 Subject: [PATCH 1/3] added extension --- redlock/__init__.py | 23 +++++++++++++++++++++++ redlock/cli.py | 22 +++++++++++++++++++++- 2 files changed, 44 insertions(+), 1 deletion(-) diff --git a/redlock/__init__.py b/redlock/__init__.py index 70795f5..5cd7d1f 100644 --- a/redlock/__init__.py +++ b/redlock/__init__.py @@ -46,6 +46,12 @@ class Redlock(object): else return 0 end""" + extend_script = """ + if redis.call("get",KEYS[1]) == ARGV[1] then + return redis.call("expire",KEYS[1],ARGV[2]) + else + return 0 + end""" def __init__(self, connection_list, retry_count=None, retry_delay=None): self.servers = [] @@ -80,6 +86,13 @@ def unlock_instance(self, server, resource, val): server.eval(self.unlock_script, 1, resource, val) except Exception as e: logging.exception("Error unlocking resource %s in server %s", resource, str(server)) + + def extend_instance(self, server, resource, val, ttl): + try: + server.eval(self.extend_script, 1, resource, val, ttl) + except Exception as e: + logging.exception("Error extending lock on resource %s in server %s", resource, str(server)) + def get_unique_id(self): CHARACTERS = string.ascii_letters + string.digits @@ -130,3 +143,13 @@ def unlock(self, lock): redis_errors.append(e) if redis_errors: raise MultipleRedlockException(redis_errors) + + def extend(self, lock, ttl): + redis_errors = [] + for server in self.servers: + try: + self.extend_instance(server, lock.resource, lock.key, ttl) + except RedisError as e: + redis_errors.append(e) + if redis_errors: + raise MultipleRedlockException(redis_errors) diff --git a/redlock/cli.py b/redlock/cli.py index 44e6003..d27ac96 100644 --- a/redlock/cli.py +++ b/redlock/cli.py @@ -54,6 +54,20 @@ def unlock(name, key, redis, **kwargs): log("ok") return 0 +def extend(name, validity, key, redis, **kwargs): + try: + dlm = redlock.Redlock(redis) + lock = redlock.Lock(0, name, key) + dlm.extend(lock, validity) + except Exception as e: + log("Error: %s" % e) + return 3 + + log("ok") + return 0 + + + def main(): parser = argparse.ArgumentParser( @@ -83,7 +97,13 @@ def main(): parser_unlock.set_defaults(func=unlock) parser_unlock.add_argument("name", help="Lock resource name") parser_unlock.add_argument("key", help="Result returned by a prior 'lock' command") - + + parser_extend = subparsers.add_parser('extend', help='Extend a lock') + parser_extend.set_defaults(func=extend) + parser_extend.add_argument("name", help="Lock resource name") + parser_extend.add_argument("key", help="Result returned by a prior 'lock' command") + parser_extend.add_argument("validity", type=int, help="Number of milliseconds the lock's validity will be extended by.") + args = parser.parse_args() log.quiet = args.quiet From 73e2afbe8003c079967cc2b24c9b63ca511c14d5 Mon Sep 17 00:00:00 2001 From: Julian Porter Date: Sun, 6 Aug 2017 04:11:26 +0100 Subject: [PATCH 2/3] added extend and test capabilities --- redlock/__init__.py | 30 ++++++++++++++++++++++++++---- redlock/cli.py | 30 +++++++++++++++++++++++++----- 2 files changed, 51 insertions(+), 9 deletions(-) diff --git a/redlock/__init__.py b/redlock/__init__.py index 5cd7d1f..b53ddef 100644 --- a/redlock/__init__.py +++ b/redlock/__init__.py @@ -48,7 +48,7 @@ class Redlock(object): end""" extend_script = """ if redis.call("get",KEYS[1]) == ARGV[1] then - return redis.call("expire",KEYS[1],ARGV[2]) + return redis.call("pexpire",KEYS[1],ARGV[2]) else return 0 end""" @@ -89,10 +89,15 @@ def unlock_instance(self, server, resource, val): def extend_instance(self, server, resource, val, ttl): try: - server.eval(self.extend_script, 1, resource, val, ttl) + return server.eval(self.extend_script, 1, resource, val, ttl) == 1 except Exception as e: logging.exception("Error extending lock on resource %s in server %s", resource, str(server)) - + + def test_instance(self, server, resource): + try: + return server.get(resource) is not None + except: + logging.exception("Error reading lock on resource %s in server %s", resource, str(server)) def get_unique_id(self): CHARACTERS = string.ascii_letters + string.digits @@ -146,10 +151,27 @@ def unlock(self, lock): def extend(self, lock, ttl): redis_errors = [] + n=0 for server in self.servers: try: - self.extend_instance(server, lock.resource, lock.key, ttl) + if self.extend_instance(server, lock.resource, lock.key, ttl): + n+=1 except RedisError as e: redis_errors.append(e) if redis_errors: raise MultipleRedlockException(redis_errors) + return n>=self.quorum + + def test(self,lock): + redis_errors = [] + n=0 + for server in self.servers: + try: + if self.test_instance(server, lock.resource): + n+=1 + except RedisError as e: + redis_errors.append(e) + if redis_errors: + raise MultipleRedlockException(redis_errors) + return n>=self.quorum + diff --git a/redlock/cli.py b/redlock/cli.py index d27ac96..f5a99e3 100644 --- a/redlock/cli.py +++ b/redlock/cli.py @@ -1,4 +1,4 @@ -from __future__ import print_function +from __future__ import print_function, absolute_import import argparse import sys @@ -58,15 +58,31 @@ def extend(name, validity, key, redis, **kwargs): try: dlm = redlock.Redlock(redis) lock = redlock.Lock(0, name, key) - dlm.extend(lock, validity) + if dlm.extend(lock, validity): + log("ok") + return 0 + else: + log("failed") + return 1 except Exception as e: log("Error: %s" % e) return 3 - log("ok") - return 0 - +def test(name,redis,**kwargs): + try: + dlm = redlock.Redlock(redis) + lock = redlock.Lock(0,name,None) + if dlm.test(lock): + print("Lock {} taken".format(name)) + else: + print("Lock {} available".format(name)) + log("ok") + return 0 + except Exception as e: + log("Error: %s" % e) + return 3 + def main(): @@ -104,6 +120,10 @@ def main(): parser_extend.add_argument("key", help="Result returned by a prior 'lock' command") parser_extend.add_argument("validity", type=int, help="Number of milliseconds the lock's validity will be extended by.") + parser_test = subparsers.add_parser('test', help='Test whether a lock is taken') + parser_test.set_defaults(func=test) + parser_test.add_argument("name", help="Lock resource name") + args = parser.parse_args() log.quiet = args.quiet From cecdf5d9e686ddbf0cb1b90324be6e6707b454d4 Mon Sep 17 00:00:00 2001 From: Julian Porter Date: Sun, 6 Aug 2017 04:22:54 +0100 Subject: [PATCH 3/3] documentation and small tweak --- README.md | 17 ++++++++++++++++- redlock/__init__.py | 3 ++- redlock/cli.py | 3 +-- setup.py | 13 +++++++++++++ 4 files changed, 32 insertions(+), 4 deletions(-) diff --git a/README.md b/README.md index ca49545..d280d82 100644 --- a/README.md +++ b/README.md @@ -30,7 +30,22 @@ It is possible to setup the number of retries (by default 3) and the retry delay (by default 200 milliseconds) used to acquire the lock. -Both `dlm.lock` and `dlm.unlock` raise a exception `MultipleRedlockException` if there are errors when communicating with one or more redis masters. The caller of `dlm` should +To extend your ownership of a lock that you already own: + + dlm.extend(my_lock,ttl) + +where you want to extend the liftime of the lock by `ttl` milliseconds. This returns +`True` if the extension succeeded and `False` if the lock had already expired. + +To test whether a lock is taken: + + dlm.test("lock_name") + +returns `True` if it is taken and `False` if it is free. + + + +`dlm.lock`, `dlm.unlock`, `dlm.extend` and `dlt.test` raise a exception `MultipleRedlockException` if there are errors when communicating with one or more redis masters. The caller of `dlm` should use a try-catch-finally block to handle this exception. A `MultipleRedlockException` object encapsulates multiple `redis-py.exceptions.RedisError` objects. diff --git a/redlock/__init__.py b/redlock/__init__.py index b53ddef..404dbc4 100644 --- a/redlock/__init__.py +++ b/redlock/__init__.py @@ -162,8 +162,9 @@ def extend(self, lock, ttl): raise MultipleRedlockException(redis_errors) return n>=self.quorum - def test(self,lock): + def test(self,name): redis_errors = [] + lock=Lock(0,name,None) n=0 for server in self.servers: try: diff --git a/redlock/cli.py b/redlock/cli.py index f5a99e3..43708c7 100644 --- a/redlock/cli.py +++ b/redlock/cli.py @@ -72,8 +72,7 @@ def extend(name, validity, key, redis, **kwargs): def test(name,redis,**kwargs): try: dlm = redlock.Redlock(redis) - lock = redlock.Lock(0,name,None) - if dlm.test(lock): + if dlm.test(name): print("Lock {} taken".format(name)) else: print("Lock {} available".format(name)) diff --git a/setup.py b/setup.py index 2d585c4..085c6b4 100644 --- a/setup.py +++ b/setup.py @@ -36,6 +36,19 @@ It is possible to setup the number of retries (by default 3) and the retry delay (by default 200 milliseconds) used to acquire the lock. +To extend your ownership of a lock that you already own: + + dlm.extend(my_lock,ttl) + +where you want to extend the liftime of the lock by `ttl` milliseconds. This returns +`True` if the extension succeeded and `False` if the lock had already expired. + +To test whether a lock is taken: + + dlm.test("lock_name") + +returns `True` if it is taken and `False` if it is free. + **Disclaimer**: This code implements an algorithm which is currently a proposal, it was not formally analyzed. Make sure to understand how it works before using it