Version history
---------------
+[2.0.0] - 2020-01-03
+
+* Change find_keys() and find_and_get() API functions to support glob-style
+ regular expression in a key search pattern. API backward incompatible change.
+* Remove 'atomic' parameter of find_and_get() API function. API backward
+ incompatible change.
+
[1.0.2] - 2019-12-18
* Take Hiredis package into use in Redis database backend.
# Finds keys under given namespace that are matching to given key prefix 'my_k'.
_try_func_return(lambda: mysdl.set(MY_NS, {'my_key': b'my_value'}))
-ret_keys = _try_func_return(lambda: mysdl.find_keys(MY_NS, ''))
+ret_keys = _try_func_return(lambda: mysdl.find_keys(MY_NS, 'my_k*'))
assert ret_keys == ['my_key']
-# Finds keys and their values under given namespace that are matching to given key prefix 'my_k'.
+# Finds keys and their values under given namespace that are matching to given key search
+# pattern 'my_k*'.
# Note that the type of returned value is bytes.
-ret_key_values = _try_func_return(lambda: mysdl.find_and_get(MY_NS, '', atomic=True))
+ret_key_values = _try_func_return(lambda: mysdl.find_and_get(MY_NS, 'my_k*'))
assert ret_key_values == {'my_key': b'my_value'}
_try_func_return(lambda: mysdl.remove_all(MY_NS))
)
-__version__ = '1.0.2'
+__version__ = '2.0.0'
__all__ = [
pass
@abstractmethod
- def find_keys(self, ns: str, key_prefix: str) -> List[str]:
+ def find_keys(self, ns: str, key_pattern: str) -> List[str]:
""""Return all the keys matching search pattern under a namespace in database."""
pass
@abstractmethod
- def find_and_get(self, ns: str, key_prefix: str, atomic: bool) -> Dict[str, bytes]:
+ def find_and_get(self, ns: str, key_pattern: str) -> Dict[str, bytes]:
"""
Return all the keys with their values matching search pattern under a namespace in
database.
ret[keys[idx]] = val
return ret
- def find_keys(self, ns: str, key_prefix: str) -> List[str]:
- escaped_key_prefix = self._escape_characters(key_prefix)
- db_escaped_key_prefix = self._add_key_ns_prefix(ns, escaped_key_prefix + '*')
+ def find_keys(self, ns: str, key_pattern: str) -> List[str]:
+ db_key_pattern = self._add_key_ns_prefix(ns, key_pattern)
with _map_to_sdl_exception():
- ret = self.__redis.keys(db_escaped_key_prefix)
+ ret = self.__redis.keys(db_key_pattern)
return self._strip_ns_from_bin_keys(ns, ret)
- def find_and_get(self, ns: str, key_prefix: str, atomic: bool) -> Dict[str, bytes]:
+ def find_and_get(self, ns: str, key_pattern: str) -> Dict[str, bytes]:
# todo: replace below implementation with redis 'NGET' module
ret = dict() # type: Dict[str, bytes]
with _map_to_sdl_exception():
- matched_keys = self.find_keys(ns, key_prefix)
+ matched_keys = self.find_keys(ns, key_pattern)
if matched_keys:
ret = self.get(ns, matched_keys)
return ret
def _strip_ns_from_bin_keys(cls, ns: str, nskeylist: List[bytes]) -> List[str]:
ret_keys = []
for k in nskeylist:
- nskey = k.decode("utf-8").split(',', 1)
+ try:
+ redis_key = k.decode("utf-8")
+ except UnicodeDecodeError as exc:
+ msg = u'Namespace %s key conversion to string failed: %s' % (ns, str(exc))
+ raise RejectedByBackend(msg)
+ nskey = redis_key.split(',', 1)
if len(nskey) != 2:
- msg = u'Illegal namespace %s key:%s' % (ns, nskey)
+ msg = u'Namespace %s key:%s has no namespace prefix' % (ns, redis_key)
raise RejectedByBackend(msg)
ret_keys.append(nskey[1])
return ret_keys
- @classmethod
- def _escape_characters(cls, pattern: str) -> str:
- return pattern.translate(str.maketrans(
- {"(": r"\(",
- ")": r"\)",
- "[": r"\[",
- "]": r"\]",
- "*": r"\*",
- "?": r"\?",
- "\\": r"\\"}))
-
def get_redis_connection(self):
"""Return existing Redis database connection."""
return self.__redis
@func_arg_checker(SdlTypeError, 1, ns=str, keys=(str, builtins.set))
def get(self, ns: str, keys: Union[str, Set[str]]) -> Dict[str, bytes]:
- return self.__dbbackend.get(ns, list(keys))
+ disordered = self.__dbbackend.get(ns, list(keys))
+ return {k: disordered[k] for k in sorted(disordered)}
- @func_arg_checker(SdlTypeError, 1, ns=str, key_prefix=str)
- def find_keys(self, ns: str, key_prefix: str) -> List[str]:
- return self.__dbbackend.find_keys(ns, key_prefix)
+ @func_arg_checker(SdlTypeError, 1, ns=str, key_pattern=str)
+ def find_keys(self, ns: str, key_pattern: str) -> List[str]:
+ return self.__dbbackend.find_keys(ns, key_pattern)
- @func_arg_checker(SdlTypeError, 1, ns=str, key_prefix=str, atomic=bool)
- def find_and_get(self, ns: str, key_prefix: str, atomic: bool) -> Dict[str, bytes]:
- return self.__dbbackend.find_and_get(ns, key_prefix, atomic)
+ @func_arg_checker(SdlTypeError, 1, ns=str, key_pattern=str)
+ def find_and_get(self, ns: str, key_pattern: str) -> Dict[str, bytes]:
+ disordered = self.__dbbackend.find_and_get(ns, key_pattern)
+ return {k: disordered[k] for k in sorted(disordered)}
@func_arg_checker(SdlTypeError, 1, ns=str, keys=(str, builtins.set))
def remove(self, ns: str, keys: Union[str, Set[str]]) -> None:
@func_arg_checker(SdlTypeError, 1, ns=str)
def remove_all(self, ns: str) -> None:
- keys = self.__dbbackend.find_keys(ns, '')
+ keys = self.__dbbackend.find_keys(ns, '*')
if keys:
self.__dbbackend.remove(ns, keys)
Returns:
(dict of str: bytes): A dictionary mapping of a key to the read data from the storage.
+ Dictionary is sorted by key values in alphabetical order.
Raises:
SdlTypeError: If function's argument is of an inappropriate type.
pass
@abstractmethod
- def find_keys(self, ns: str, key_prefix: str) -> List[str]:
- """
+ def find_keys(self, ns: str, key_pattern: str) -> List[str]:
+ r"""
Find all keys matching search pattern under the namespace.
- No prior knowledge about the keys in the given namespace exists, thus operation is not
- guaranteed to be atomic or isolated.
+ Supported glob-style patterns:
+ `?` matches any single character. For example `?at` matches Cat, cat, Bat or bat.
+ `*` matches any number of any characters including none. For example `*Law*` matches
+ Law, GrokLaw, or Lawyer.
+ `[abc]` matches one character given in the bracket. For example `[CB]at` matches Cat or
+ Bat.
+ `[a-z]` matches one character from the range given in the bracket. For example
+ `Letter[0-9]` matches Letter0 up to Letter9.
+ `[^abc]` matches any single character what is not given in the bracket. For example
+ `h[^e]llo` matches hallo, hillo but not hello.
+
+ If searched key itself contains a special character, use a backslash (\) character to
+ escape the special character to match it verbatim.
+
+ NOTE: `find_keys` function is not guaranteed to be atomic or isolated.
+
All the exceptions except SdlTypeError are derived from SdlException base class. Client
can catch only that exception if separate handling for different SDL error situations is
not needed. Exception SdlTypeError is derived from build-in TypeError and it indicates
Args:
ns (str): Namespace under which this operation is targeted.
- key_prefix (str): Only keys starting with given keyPrefix are returned. Passing empty
- string as keyPrefix will return all the keys.
+ key_pattern (str): Key search pattern.
Returns:
(list of str): A list of found keys.
pass
@abstractmethod
- def find_and_get(self, ns: str, key_prefix: str, atomic: bool) -> Dict[str, bytes]:
- """
+ def find_and_get(self, ns: str, key_pattern: str) -> Dict[str, bytes]:
+ r"""
Find keys and get their respective data from SDL storage.
- Only those entries that are matching prefix will be returned.
- NOTE: In atomic action, if the prefix produces huge number of matches, that can have
- a severe impact on system performance, due to DB is blocked for long time.
+ Supported glob-style patterns:
+ `?` matches any single character. For example `?at` matches Cat, cat, Bat or bat.
+ `*` matches any number of any characters including none. For example `*Law*` matches
+ Law, GrokLaw, or Lawyer.
+ `[abc]` matches one character given in the bracket. For example `[CB]at` matches Cat or
+ Bat.
+ `[a-z]` matches one character from the range given in the bracket. For example
+ `Letter[0-9]` matches Letter0 up to Letter9.
+ `[^abc]` matches any single character what is not given in the bracket. For example
+ `h[^e]llo` matches hallo, hillo but not hello.
+
+ If searched key itself contains a special character, use a backslash (\) character to
+ escape the special character to match it verbatim.
+
+ NOTE: `find_and_get` function is not guaranteed to be atomic or isolated.
+
All the exceptions except SdlTypeError are derived from SdlException base class. Client
can catch only that exception if separate handling for different SDL error situations is
not needed. Exception SdlTypeError is derived from build-in TypeError and it indicates
Args:
ns (str): Namespace under which this operation is targeted.
- key_prefix (str): Only keys starting with given keyPrefix are returned. Passing empty
- string as keyPrefix will return all the keys.
- atomic (bool): True to find keys and get their respective data in one atomic operation,
- false to find keys and get their respective data non-atomically.
+ key_pattern (str): Key search pattern.
Returns:
(dict of str: bytes): A dictionary mapping of a key to the read data from the storage.
+ Dictionary is sorted by key values in alphabetical order.
Raises:
SdlTypeError: If function's argument is of an inappropriate type.
request.cls.data = b'123'
request.cls.old_data = b'1'
request.cls.new_data = b'3'
- request.cls.keyprefix = 'x?'
- request.cls.keyprefix_redis = r'{some-ns},x\?*'
- request.cls.matchedkeys = ['x1', 'x2', 'x3', 'x4', 'x5']
- request.cls.matchedkeys_redis = [b'{some-ns},x1', b'{some-ns},x2', b'{some-ns},x3',
- b'{some-ns},x4', b'{some-ns},x5']
- request.cls.matcheddata_dl_redis = [b'10', b'11', b'12', b'13', b'14']
- request.cls.matcheddata_dm = {'x1': b'10', 'x2': b'11', 'x3': b'12',
- 'x4': b'13', 'x5': b'14'}
+ request.cls.keypattern = r'[Aa]bc-\[1\].?-*'
+ request.cls.keypattern_redis = r'{some-ns},[Aa]bc-\[1\].?-*'
+ request.cls.matchedkeys = ['Abc-[1].0-def', 'abc-[1].1-ghi']
+ request.cls.matchedkeys_redis = [b'{some-ns},Abc-[1].0-def',
+ b'{some-ns},abc-[1].1-ghi']
+ request.cls.matcheddata_redis = [b'10', b'11']
+ request.cls.matchedkeydata = {'Abc-[1].0-def': b'10',
+ 'abc-[1].1-ghi': b'11'}
request.cls.group = 'some-group'
request.cls.group_redis = '{some-ns},some-group'
request.cls.groupmembers = set([b'm1', b'm2'])
request.cls.groupmember = b'm1'
- request.cls.is_atomic = True
request.cls.configuration = Mock()
mock_conf_params = _Configuration.Params(db_host=None,
def test_find_keys_function_success(self):
self.mock_redis.keys.return_value = self.matchedkeys_redis
- ret = self.db.find_keys(self.ns, self.keyprefix)
- self.mock_redis.keys.assert_called_once_with(self.keyprefix_redis)
+ ret = self.db.find_keys(self.ns, self.keypattern)
+ self.mock_redis.keys.assert_called_once_with(self.keypattern_redis)
assert ret == self.matchedkeys
def test_find_keys_function_returns_empty_list_when_no_matching_keys_found(self):
self.mock_redis.keys.return_value = []
- ret = self.db.find_keys(self.ns, self.keyprefix)
- self.mock_redis.keys.assert_called_once_with(self.keyprefix_redis)
+ ret = self.db.find_keys(self.ns, self.keypattern)
+ self.mock_redis.keys.assert_called_once_with(self.keypattern_redis)
assert ret == []
def test_find_keys_function_can_map_redis_exception_to_sdl_exception(self):
self.mock_redis.keys.side_effect = redis_exceptions.ResponseError('redis error!')
with pytest.raises(ricsdl.exceptions.RejectedByBackend):
- self.db.find_keys(self.ns, self.keyprefix)
+ self.db.find_keys(self.ns, self.keypattern)
+
+ def test_find_keys_function_can_raise_exception_when_redis_key_convert_to_string_fails(self):
+ # Redis returns an illegal key, which conversion to string fails
+ corrupt_redis_key = b'\x81'
+ self.mock_redis.keys.return_value = [corrupt_redis_key]
+ with pytest.raises(ricsdl.exceptions.RejectedByBackend) as excinfo:
+ self.db.find_keys(self.ns, self.keypattern)
+ assert f"Namespace {self.ns} key:{corrupt_redis_key} "
+ "has no namespace prefix" in str(excinfo.value)
+
+ def test_find_keys_function_can_raise_exception_when_redis_key_is_without_prefix(self):
+ # Redis returns an illegal key, which doesn't have comma separated namespace prefix
+ corrupt_redis_key = 'some-corrupt-key'
+ self.mock_redis.keys.return_value = [f'{corrupt_redis_key}'.encode()]
+ with pytest.raises(ricsdl.exceptions.RejectedByBackend) as excinfo:
+ self.db.find_keys(self.ns, self.keypattern)
+ assert f"Namespace {self.ns} key:{corrupt_redis_key} "
+ "has no namespace prefix" in str(excinfo.value)
def test_find_and_get_function_success(self):
self.mock_redis.keys.return_value = self.matchedkeys_redis
- self.mock_redis.mget.return_value = self.matcheddata_dl_redis
- ret = self.db.find_and_get(self.ns, self.keyprefix, self.is_atomic)
- self.mock_redis.keys.assert_called_once_with(self.keyprefix_redis)
+ self.mock_redis.mget.return_value = self.matcheddata_redis
+ ret = self.db.find_and_get(self.ns, self.keypattern)
+ self.mock_redis.keys.assert_called_once_with(self.keypattern_redis)
self.mock_redis.mget.assert_called_once_with([i.decode() for i in self.matchedkeys_redis])
- assert ret == self.matcheddata_dm
+ assert ret == self.matchedkeydata
def test_find_and_get_function_returns_empty_dict_when_no_matching_keys_exist(self):
self.mock_redis.keys.return_value = list()
- ret = self.db.find_and_get(self.ns, self.keyprefix, self.is_atomic)
- self.mock_redis.keys.assert_called_once_with(self.keyprefix_redis)
+ ret = self.db.find_and_get(self.ns, self.keypattern)
+ self.mock_redis.keys.assert_called_once_with(self.keypattern_redis)
assert not self.mock_redis.mget.called
assert ret == dict()
+ def test_find_and_get_function_can_map_redis_exception_to_sdl_exception(self):
+ self.mock_redis.keys.side_effect = redis_exceptions.ResponseError('redis error!')
+ with pytest.raises(ricsdl.exceptions.RejectedByBackend):
+ self.db.find_and_get(self.ns, self.keypattern)
+
+ def test_find_and_get_function_can_raise_exception_when_redis_key_convert_to_string_fails(self):
+ # Redis returns an illegal key, which conversion to string fails
+ corrupt_redis_key = b'\x81'
+ self.mock_redis.keys.return_value = [corrupt_redis_key]
+ with pytest.raises(ricsdl.exceptions.RejectedByBackend) as excinfo:
+ self.db.find_and_get(self.ns, self.keypattern)
+ assert f"Namespace {self.ns} key:{corrupt_redis_key} "
+ "has no namespace prefix" in str(excinfo.value)
+
+ def test_find_and_get_function_can_raise_exception_when_redis_key_is_without_prefix(self):
+ # Redis returns an illegal key, which doesn't have comma separated namespace prefix
+ corrupt_redis_key = 'some-corrupt-key'
+ self.mock_redis.keys.return_value = [f'{corrupt_redis_key}'.encode()]
+ with pytest.raises(ricsdl.exceptions.RejectedByBackend) as excinfo:
+ self.db.find_and_get(self.ns, self.keypattern)
+ assert f"Namespace {self.ns} key:{corrupt_redis_key} "
+ "has no namespace prefix" in str(excinfo.value)
+
def test_remove_function_success(self):
self.db.remove(self.ns, self.keys)
self.mock_redis.delete.assert_called_once_with(*self.keys_redis)
keys=[self.lockname_redis], args=[123], client=self.mock_redis)
assert ret == 2
+ def test_get_validity_time_function_second_fraction_success(self):
+ self.mock_redis_lock.name = self.lockname_redis
+ self.mock_redis_lock.local.token = 123
+ self.mock_lua_get_validity_time.return_value = 234
+
+ ret = self.lock.get_validity_time()
+ self.mock_lua_get_validity_time.assert_called_once_with(
+ keys=[self.lockname_redis], args=[123], client=self.mock_redis)
+ assert ret == 0.234
+
def test_get_validity_time_function_can_raise_exception_if_lock_is_unlocked(self):
self.mock_redis_lock.name = self.lockname_redis
self.mock_redis_lock.local.token = None
assert f"Getting validity time of a lock {self.lockname} failed with error code: -10" in str(excinfo.value)
def test_redis_backend_lock_object_string_representation(self):
- str_out = str(self.lock)
- assert str_out is not None
+ expected_lock_info = {'lock namespace': 'some-ns',
+ 'lock name': 'some-lock-name',
+ 'lock status': 'locked'}
+ assert str(self.lock) == str(expected_lock_info)
+
+ def test_redis_backend_lock_object_string_representation_can_catch_redis_exception(self):
+ self.mock_redis_lock.owned.side_effect = redis_exceptions.LockError('redis lock error!')
+ expected_lock_info = {'lock namespace': 'some-ns',
+ 'lock name': 'some-lock-name',
+ 'lock status': 'Error: redis lock error!'}
+ assert str(self.lock) == str(expected_lock_info)
def test_redis_response_error_exception_is_mapped_to_rejected_by_backend_sdl_exception():
request.cls.ns = 'some-ns'
request.cls.key = 'a'
request.cls.keys = {'a', 'b'}
- request.cls.dm = {'a': b'1', 'b': b'2'}
+ request.cls.dm = {'b': b'2', 'a': b'1'}
request.cls.old_data = b'1'
request.cls.new_data = b'3'
request.cls.keyprefix = 'x'
request.cls.matchedkeys = ['x1', 'x2', 'x3', 'x4', 'x5']
- request.cls.is_atomic = True
request.cls.group = 'some-group'
request.cls.groupmembers = set([b'm1', b'm2'])
request.cls.groupmember = b'm1'
assert len(call_args[1]) == len(self.keys)
assert all(k in call_args[1] for k in self.keys)
assert ret == self.dm
+ # Validate that SDL returns a dictionary with keys in alphabetical order
+ assert sorted(self.dm)[0] == list(ret.keys())[0]
def test_get_function_can_return_empty_dict_when_no_key_values_exist(self):
self.mock_db_backend.get.return_value = dict()
def test_find_and_get_function_success(self):
self.mock_db_backend.find_and_get.return_value = self.dm
- ret = self.storage.find_and_get(self.ns, self.keyprefix, self.is_atomic)
- self.mock_db_backend.find_and_get.assert_called_once_with(self.ns, self.keyprefix,
- self.is_atomic)
+ ret = self.storage.find_and_get(self.ns, self.keyprefix)
+ self.mock_db_backend.find_and_get.assert_called_once_with(self.ns, self.keyprefix)
assert ret == self.dm
+ # Validate that SDL returns a dictionary with keys in alphabetical order
+ assert sorted(self.dm)[0] == list(ret.keys())[0]
def test_find_and_get_function_can_return_empty_dict_when_no_keys_exist(self):
self.mock_db_backend.find_and_get.return_value = dict()
- ret = self.storage.find_and_get(self.ns, self.keyprefix, self.is_atomic)
- self.mock_db_backend.find_and_get.assert_called_once_with(self.ns, self.keyprefix,
- self.is_atomic)
+ ret = self.storage.find_and_get(self.ns, self.keyprefix)
+ self.mock_db_backend.find_and_get.assert_called_once_with(self.ns, self.keyprefix)
assert ret == dict()
def test_find_and_get_function_can_raise_exception_for_wrong_argument(self):
with pytest.raises(SdlTypeError):
- self.storage.find_and_get(0xbad, self.keyprefix, self.is_atomic)
+ self.storage.find_and_get(0xbad, self.keyprefix)
with pytest.raises(SdlTypeError):
- self.storage.find_and_get(self.ns, 0xbad, self.is_atomic)
- with pytest.raises(SdlTypeError):
- self.storage.find_and_get(self.ns, self.keyprefix, 0xbad)
+ self.storage.find_and_get(self.ns, 0xbad)
def test_remove_function_success(self):
self.storage.remove(self.ns, self.keys)
def test_remove_all_function_success(self):
self.mock_db_backend.find_keys.return_value = ['a1']
self.storage.remove_all(self.ns)
- self.mock_db_backend.find_keys.assert_called_once_with(self.ns, '')
+ self.mock_db_backend.find_keys.assert_called_once_with(self.ns, '*')
self.mock_db_backend.remove.assert_called_once_with(self.ns,
self.mock_db_backend.find_keys.return_value)
Code Review / ric-plt / sdlpy.git / commitdiff