Implement a fake SDL database backend
[ric-plt/sdlpy.git] / ricsdl-package / tests / backend / test_redis.py
1 # Copyright (c) 2019 AT&T Intellectual Property.
2 # Copyright (c) 2018-2019 Nokia.
3 #
4 # Licensed under the Apache License, Version 2.0 (the "License");
5 # you may not use this file except in compliance with the License.
6 # You may obtain a copy of the License at
7 #
8 #     http://www.apache.org/licenses/LICENSE-2.0
9 #
10 # Unless required by applicable law or agreed to in writing, software
11 # distributed under the License is distributed on an "AS IS" BASIS,
12 # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13 # See the License for the specific language governing permissions and
14 # limitations under the License.
15
16 #
17 # This source code is part of the near-RT RIC (RAN Intelligent Controller)
18 # platform project (RICP).
19 #
20
21
22 from unittest.mock import patch, Mock
23 import pytest
24 from redis import exceptions as redis_exceptions
25 import ricsdl.backend
26 from ricsdl.backend.redis import (RedisBackendLock, _map_to_sdl_exception)
27 from ricsdl.configuration import _Configuration
28 from ricsdl.configuration import DbBackendType
29 import ricsdl.exceptions
30
31
32 @pytest.fixture()
33 def redis_backend_fixture(request):
34     request.cls.ns = 'some-ns'
35     request.cls.dl_redis = [b'1', b'2']
36     request.cls.dm = {'a': b'1', 'b': b'2'}
37     request.cls.dm_redis = {'{some-ns},a': b'1', '{some-ns},b': b'2'}
38     request.cls.key = 'a'
39     request.cls.key_redis = '{some-ns},a'
40     request.cls.keys = ['a', 'b']
41     request.cls.keys_redis = ['{some-ns},a', '{some-ns},b']
42     request.cls.data = b'123'
43     request.cls.old_data = b'1'
44     request.cls.new_data = b'3'
45     request.cls.keypattern = r'[Aa]bc-\[1\].?-*'
46     request.cls.keypattern_redis = r'{some-ns},[Aa]bc-\[1\].?-*'
47     request.cls.matchedkeys = ['Abc-[1].0-def', 'abc-[1].1-ghi']
48     request.cls.matchedkeys_redis = [b'{some-ns},Abc-[1].0-def',
49                                      b'{some-ns},abc-[1].1-ghi']
50     request.cls.matcheddata_redis = [b'10', b'11']
51     request.cls.matchedkeydata = {'Abc-[1].0-def': b'10',
52                                   'abc-[1].1-ghi': b'11'}
53     request.cls.group = 'some-group'
54     request.cls.group_redis = '{some-ns},some-group'
55     request.cls.groupmembers = set([b'm1', b'm2'])
56     request.cls.groupmember = b'm1'
57
58     request.cls.configuration = Mock()
59     mock_conf_params = _Configuration.Params(db_host=None,
60                                              db_port=None,
61                                              db_sentinel_port=None,
62                                              db_sentinel_master_name=None,
63                                              db_type=DbBackendType.REDIS)
64     request.cls.configuration.get_params.return_value = mock_conf_params
65     with patch('ricsdl.backend.redis.Redis') as mock_redis:
66         db = ricsdl.backend.get_backend_instance(request.cls.configuration)
67         request.cls.mock_redis = mock_redis.return_value
68     request.cls.db = db
69
70     yield
71
72
73 @pytest.mark.usefixtures('redis_backend_fixture')
74 class TestRedisBackend:
75     def test_set_function_success(self):
76         self.db.set(self.ns, self.dm)
77         self.mock_redis.mset.assert_called_once_with(self.dm_redis)
78
79     def test_set_function_can_map_redis_exception_to_sdl_exception(self):
80         self.mock_redis.mset.side_effect = redis_exceptions.ResponseError('redis error!')
81         with pytest.raises(ricsdl.exceptions.RejectedByBackend):
82             self.db.set(self.ns, self.dm)
83
84     def test_set_if_function_success(self):
85         self.mock_redis.execute_command.return_value = True
86         ret = self.db.set_if(self.ns, self.key, self.old_data, self.new_data)
87         self.mock_redis.execute_command.assert_called_once_with('SETIE', self.key_redis,
88                                                                 self.new_data, self.old_data)
89         assert ret is True
90
91     def test_set_if_function_returns_false_if_existing_key_value_not_expected(self):
92         self.mock_redis.execute_command.return_value = False
93         ret = self.db.set_if(self.ns, self.key, self.old_data, self.new_data)
94         self.mock_redis.execute_command.assert_called_once_with('SETIE', self.key_redis,
95                                                                 self.new_data, self.old_data)
96         assert ret is False
97
98     def test_set_if_function_can_map_redis_exception_to_sdl_exception(self):
99         self.mock_redis.execute_command.side_effect = redis_exceptions.ResponseError('redis error!')
100         with pytest.raises(ricsdl.exceptions.RejectedByBackend):
101             self.db.set_if(self.ns, self.key, self.old_data, self.new_data)
102
103     def test_set_if_not_exists_function_success(self):
104         self.mock_redis.setnx.return_value = True
105         ret = self.db.set_if_not_exists(self.ns, self.key, self.new_data)
106         self.mock_redis.setnx.assert_called_once_with(self.key_redis, self.new_data)
107         assert ret is True
108
109     def test_set_if_not_exists_function_returns_false_if_key_already_exists(self):
110         self.mock_redis.setnx.return_value = False
111         ret = self.db.set_if_not_exists(self.ns, self.key, self.new_data)
112         self.mock_redis.setnx.assert_called_once_with(self.key_redis, self.new_data)
113         assert ret is False
114
115     def test_set_if_not_exists_function_can_map_redis_exception_to_sdl_exception(self):
116         self.mock_redis.setnx.side_effect = redis_exceptions.ResponseError('redis error!')
117         with pytest.raises(ricsdl.exceptions.RejectedByBackend):
118             self.db.set_if_not_exists(self.ns, self.key, self.new_data)
119
120     def test_get_function_success(self):
121         self.mock_redis.mget.return_value = self.dl_redis
122         ret = self.db.get(self.ns, self.keys)
123         self.mock_redis.mget.assert_called_once_with(self.keys_redis)
124         assert ret == self.dm
125
126     def test_get_function_returns_empty_dict_when_no_key_values_exist(self):
127         self.mock_redis.mget.return_value = [None, None]
128         ret = self.db.get(self.ns, self.keys)
129         self.mock_redis.mget.assert_called_once_with(self.keys_redis)
130         assert ret == dict()
131
132     def test_get_function_returns_dict_only_with_found_key_values_when_some_keys_exist(self):
133         self.mock_redis.mget.return_value = [self.data, None]
134         ret = self.db.get(self.ns, self.keys)
135         self.mock_redis.mget.assert_called_once_with(self.keys_redis)
136         assert ret == {self.key: self.data}
137
138     def test_get_function_can_map_redis_exception_to_sdl_exception(self):
139         self.mock_redis.mget.side_effect = redis_exceptions.ResponseError('redis error!')
140         with pytest.raises(ricsdl.exceptions.RejectedByBackend):
141             self.db.get(self.ns, self.keys)
142
143     def test_find_keys_function_success(self):
144         self.mock_redis.keys.return_value = self.matchedkeys_redis
145         ret = self.db.find_keys(self.ns, self.keypattern)
146         self.mock_redis.keys.assert_called_once_with(self.keypattern_redis)
147         assert ret == self.matchedkeys
148
149     def test_find_keys_function_returns_empty_list_when_no_matching_keys_found(self):
150         self.mock_redis.keys.return_value = []
151         ret = self.db.find_keys(self.ns, self.keypattern)
152         self.mock_redis.keys.assert_called_once_with(self.keypattern_redis)
153         assert ret == []
154
155     def test_find_keys_function_can_map_redis_exception_to_sdl_exception(self):
156         self.mock_redis.keys.side_effect = redis_exceptions.ResponseError('redis error!')
157         with pytest.raises(ricsdl.exceptions.RejectedByBackend):
158             self.db.find_keys(self.ns, self.keypattern)
159
160     def test_find_keys_function_can_raise_exception_when_redis_key_convert_to_string_fails(self):
161         # Redis returns an illegal key, which conversion to string fails
162         corrupt_redis_key = b'\x81'
163         self.mock_redis.keys.return_value = [corrupt_redis_key]
164         with pytest.raises(ricsdl.exceptions.RejectedByBackend) as excinfo:
165             self.db.find_keys(self.ns, self.keypattern)
166         assert f"Namespace {self.ns} key:{corrupt_redis_key} "
167         "has no namespace prefix" in str(excinfo.value)
168
169     def test_find_keys_function_can_raise_exception_when_redis_key_is_without_prefix(self):
170         # Redis returns an illegal key, which doesn't have comma separated namespace prefix
171         corrupt_redis_key = 'some-corrupt-key'
172         self.mock_redis.keys.return_value = [f'{corrupt_redis_key}'.encode()]
173         with pytest.raises(ricsdl.exceptions.RejectedByBackend) as excinfo:
174             self.db.find_keys(self.ns, self.keypattern)
175         assert f"Namespace {self.ns} key:{corrupt_redis_key} "
176         "has no namespace prefix" in str(excinfo.value)
177
178     def test_find_and_get_function_success(self):
179         self.mock_redis.keys.return_value = self.matchedkeys_redis
180         self.mock_redis.mget.return_value = self.matcheddata_redis
181         ret = self.db.find_and_get(self.ns, self.keypattern)
182         self.mock_redis.keys.assert_called_once_with(self.keypattern_redis)
183         self.mock_redis.mget.assert_called_once_with([i.decode() for i in self.matchedkeys_redis])
184         assert ret == self.matchedkeydata
185
186     def test_find_and_get_function_returns_empty_dict_when_no_matching_keys_exist(self):
187         self.mock_redis.keys.return_value = list()
188         ret = self.db.find_and_get(self.ns, self.keypattern)
189         self.mock_redis.keys.assert_called_once_with(self.keypattern_redis)
190         assert not self.mock_redis.mget.called
191         assert ret == dict()
192
193     def test_find_and_get_function_can_map_redis_exception_to_sdl_exception(self):
194         self.mock_redis.keys.side_effect = redis_exceptions.ResponseError('redis error!')
195         with pytest.raises(ricsdl.exceptions.RejectedByBackend):
196             self.db.find_and_get(self.ns, self.keypattern)
197
198     def test_find_and_get_function_can_raise_exception_when_redis_key_convert_to_string_fails(self):
199         # Redis returns an illegal key, which conversion to string fails
200         corrupt_redis_key = b'\x81'
201         self.mock_redis.keys.return_value = [corrupt_redis_key]
202         with pytest.raises(ricsdl.exceptions.RejectedByBackend) as excinfo:
203             self.db.find_and_get(self.ns, self.keypattern)
204         assert f"Namespace {self.ns} key:{corrupt_redis_key} "
205         "has no namespace prefix" in str(excinfo.value)
206
207     def test_find_and_get_function_can_raise_exception_when_redis_key_is_without_prefix(self):
208         # Redis returns an illegal key, which doesn't have comma separated namespace prefix
209         corrupt_redis_key = 'some-corrupt-key'
210         self.mock_redis.keys.return_value = [f'{corrupt_redis_key}'.encode()]
211         with pytest.raises(ricsdl.exceptions.RejectedByBackend) as excinfo:
212             self.db.find_and_get(self.ns, self.keypattern)
213         assert f"Namespace {self.ns} key:{corrupt_redis_key} "
214         "has no namespace prefix" in str(excinfo.value)
215
216     def test_remove_function_success(self):
217         self.db.remove(self.ns, self.keys)
218         self.mock_redis.delete.assert_called_once_with(*self.keys_redis)
219
220     def test_remove_function_can_map_redis_exception_to_sdl_exception(self):
221         self.mock_redis.delete.side_effect = redis_exceptions.ResponseError('redis error!')
222         with pytest.raises(ricsdl.exceptions.RejectedByBackend):
223             self.db.remove(self.ns, self.keys)
224
225     def test_remove_if_function_success(self):
226         self.mock_redis.execute_command.return_value = True
227         ret = self.db.remove_if(self.ns, self.key, self.new_data)
228         self.mock_redis.execute_command.assert_called_once_with('DELIE', self.key_redis,
229                                                                 self.new_data)
230         assert ret is True
231
232     def test_remove_if_function_returns_false_if_data_does_not_match(self):
233         self.mock_redis.execute_command.return_value = False
234         ret = self.db.remove_if(self.ns, self.key, self.new_data)
235         self.mock_redis.execute_command.assert_called_once_with('DELIE', self.key_redis,
236                                                                 self.new_data)
237         assert ret is False
238
239     def test_remove_if_function_can_map_redis_exception_to_sdl_exception(self):
240         self.mock_redis.execute_command.side_effect = redis_exceptions.ResponseError('redis error!')
241         with pytest.raises(ricsdl.exceptions.RejectedByBackend):
242             self.db.remove_if(self.ns, self.key, self.new_data)
243
244     def test_add_member_function_success(self):
245         self.db.add_member(self.ns, self.group, self.groupmembers)
246         self.mock_redis.sadd.assert_called_once_with(self.group_redis, *self.groupmembers)
247
248     def test_add_member_function_can_map_redis_exception_to_sdl_exception(self):
249         self.mock_redis.sadd.side_effect = redis_exceptions.ResponseError('redis error!')
250         with pytest.raises(ricsdl.exceptions.RejectedByBackend):
251             self.db.add_member(self.ns, self.group, self.groupmembers)
252
253     def test_remove_member_function_success(self):
254         self.db.remove_member(self.ns, self.group, self.groupmembers)
255         self.mock_redis.srem.assert_called_once_with(self.group_redis, *self.groupmembers)
256
257     def test_remove_member_function_can_map_redis_exception_to_sdl_exception(self):
258         self.mock_redis.srem.side_effect = redis_exceptions.ResponseError('redis error!')
259         with pytest.raises(ricsdl.exceptions.RejectedByBackend):
260             self.db.remove_member(self.ns, self.group, self.groupmembers)
261
262     def test_remove_group_function_success(self):
263         self.db.remove_group(self.ns, self.group)
264         self.mock_redis.delete.assert_called_once_with(self.group_redis)
265
266     def test_remove_group_function_can_map_redis_exception_to_sdl_exception(self):
267         self.mock_redis.delete.side_effect = redis_exceptions.ResponseError('redis error!')
268         with pytest.raises(ricsdl.exceptions.RejectedByBackend):
269             self.db.remove_group(self.ns, self.group)
270
271     def test_get_members_function_success(self):
272         self.mock_redis.smembers.return_value = self.groupmembers
273         ret = self.db.get_members(self.ns, self.group)
274         self.mock_redis.smembers.assert_called_once_with(self.group_redis)
275         assert ret is self.groupmembers
276
277     def test_get_members_function_can_map_redis_exception_to_sdl_exception(self):
278         self.mock_redis.smembers.side_effect = redis_exceptions.ResponseError('redis error!')
279         with pytest.raises(ricsdl.exceptions.RejectedByBackend):
280             self.db.get_members(self.ns, self.group)
281
282     def test_is_member_function_success(self):
283         self.mock_redis.sismember.return_value = True
284         ret = self.db.is_member(self.ns, self.group, self.groupmember)
285         self.mock_redis.sismember.assert_called_once_with(self.group_redis, self.groupmember)
286         assert ret is True
287
288     def test_is_member_function_can_map_redis_exception_to_sdl_exception(self):
289         self.mock_redis.sismember.side_effect = redis_exceptions.ResponseError('redis error!')
290         with pytest.raises(ricsdl.exceptions.RejectedByBackend):
291             self.db.is_member(self.ns, self.group, self.groupmember)
292
293     def test_group_size_function_success(self):
294         self.mock_redis.scard.return_value = 100
295         ret = self.db.group_size(self.ns, self.group)
296         self.mock_redis.scard.assert_called_once_with(self.group_redis)
297         assert ret == 100
298
299     def test_group_size_function_can_map_redis_exception_to_sdl_exception(self):
300         self.mock_redis.scard.side_effect = redis_exceptions.ResponseError('Some redis error!')
301         with pytest.raises(ricsdl.exceptions.RejectedByBackend):
302             self.db.group_size(self.ns, self.group)
303
304     def test_get_redis_connection_function_success(self):
305         ret = self.db.get_redis_connection()
306         assert ret is self.mock_redis
307
308     def test_redis_backend_object_string_representation(self):
309         str_out = str(self.db)
310         assert str_out is not None
311
312
313 class MockRedisLock:
314     def __init__(self, redis, name, timeout=None, sleep=0.1,
315                  blocking=True, blocking_timeout=None, thread_local=True):
316         self.redis = redis
317         self.name = name
318         self.timeout = timeout
319         self.sleep = sleep
320         self.blocking = blocking
321         self.blocking_timeout = blocking_timeout
322         self.thread_local = bool(thread_local)
323
324
325 @pytest.fixture(scope="module")
326 def mock_redis_lock():
327     def _mock_redis_lock(name, timeout=None, sleep=0.1,
328                          blocking=True, blocking_timeout=None, thread_local=True):
329         return MockRedisLock(name, timeout, sleep, blocking, blocking_timeout, thread_local)
330     return _mock_redis_lock
331
332
333 @pytest.fixture()
334 def redis_backend_lock_fixture(request, mock_redis_lock):
335     request.cls.ns = 'some-ns'
336     request.cls.lockname = 'some-lock-name'
337     request.cls.lockname_redis = '{some-ns},some-lock-name'
338     request.cls.expiration = 10
339     request.cls.retry_interval = 0.1
340     request.cls.retry_timeout = 1
341
342     request.cls.mock_lua_get_validity_time = Mock()
343     request.cls.mock_lua_get_validity_time.return_value = 2000
344
345     request.cls.mock_redis = Mock()
346     request.cls.mock_redis.register_script = Mock()
347     request.cls.mock_redis.register_script.return_value = request.cls.mock_lua_get_validity_time
348
349     mocked_dbbackend = Mock()
350     mocked_dbbackend.get_redis_connection.return_value = request.cls.mock_redis
351
352     request.cls.configuration = Mock()
353     mock_conf_params = _Configuration.Params(db_host=None,
354                                              db_port=None,
355                                              db_sentinel_port=None,
356                                              db_sentinel_master_name=None,
357                                              db_type=DbBackendType.REDIS)
358     request.cls.configuration.get_params.return_value = mock_conf_params
359
360     with patch('ricsdl.backend.redis.Lock') as mock_redis_lock:
361         lock = ricsdl.backend.get_backend_lock_instance(request.cls.configuration,
362                                                         request.cls.ns, request.cls.lockname,
363                                                         request.cls.expiration, mocked_dbbackend)
364         request.cls.mock_redis_lock = mock_redis_lock.return_value
365         request.cls.lock = lock
366     yield
367     RedisBackendLock.lua_get_validity_time = None
368
369
370 @pytest.mark.usefixtures('redis_backend_lock_fixture')
371 class TestRedisBackendLock:
372     def test_acquire_function_success(self):
373         self.mock_redis_lock.acquire.return_value = True
374         ret = self.lock.acquire(self.retry_interval, self.retry_timeout)
375         self.mock_redis_lock.acquire.assert_called_once_with(blocking_timeout=self.retry_timeout)
376         assert ret is True
377
378     def test_acquire_function_returns_false_if_lock_is_not_acquired(self):
379         self.mock_redis_lock.acquire.return_value = False
380         ret = self.lock.acquire(self.retry_interval, self.retry_timeout)
381         self.mock_redis_lock.acquire.assert_called_once_with(blocking_timeout=self.retry_timeout)
382         assert ret is False
383
384     def test_acquire_function_can_map_redis_exception_to_sdl_exception(self):
385         self.mock_redis_lock.acquire.side_effect = redis_exceptions.LockError('redis lock error!')
386         with pytest.raises(ricsdl.exceptions.BackendError):
387             self.lock.acquire(self.retry_interval, self.retry_timeout)
388
389     def test_release_function_success(self):
390         self.lock.release()
391         self.mock_redis_lock.release.assert_called_once()
392
393     def test_release_function_can_map_redis_exception_to_sdl_exception(self):
394         self.mock_redis_lock.release.side_effect = redis_exceptions.LockError('redis lock error!')
395         with pytest.raises(ricsdl.exceptions.BackendError):
396             self.lock.release()
397
398     def test_refresh_function_success(self):
399         self.lock.refresh()
400         self.mock_redis_lock.reacquire.assert_called_once()
401
402     def test_refresh_function_can_map_redis_exception_to_sdl_exception(self):
403         self.mock_redis_lock.reacquire.side_effect = redis_exceptions.LockError('redis lock error!')
404         with pytest.raises(ricsdl.exceptions.BackendError):
405             self.lock.refresh()
406
407     def test_get_validity_time_function_success(self):
408         self.mock_redis_lock.name = self.lockname_redis
409         self.mock_redis_lock.local.token = 123
410
411         ret = self.lock.get_validity_time()
412         self.mock_lua_get_validity_time.assert_called_once_with(
413             keys=[self.lockname_redis], args=[123], client=self.mock_redis)
414         assert ret == 2
415
416     def test_get_validity_time_function_second_fraction_success(self):
417         self.mock_redis_lock.name = self.lockname_redis
418         self.mock_redis_lock.local.token = 123
419         self.mock_lua_get_validity_time.return_value = 234
420
421         ret = self.lock.get_validity_time()
422         self.mock_lua_get_validity_time.assert_called_once_with(
423             keys=[self.lockname_redis], args=[123], client=self.mock_redis)
424         assert ret == 0.234
425
426     def test_get_validity_time_function_can_raise_exception_if_lock_is_unlocked(self):
427         self.mock_redis_lock.name = self.lockname_redis
428         self.mock_redis_lock.local.token = None
429
430         with pytest.raises(ricsdl.exceptions.RejectedByBackend) as excinfo:
431             self.lock.get_validity_time()
432         assert f"Cannot get validity time of an unlocked lock {self.lockname}" in str(excinfo.value)
433
434     def test_get_validity_time_function_can_raise_exception_if_lua_script_fails(self):
435         self.mock_redis_lock.name = self.lockname_redis
436         self.mock_redis_lock.local.token = 123
437         self.mock_lua_get_validity_time.return_value = -10
438
439         with pytest.raises(ricsdl.exceptions.RejectedByBackend) as excinfo:
440             self.lock.get_validity_time()
441         assert f"Getting validity time of a lock {self.lockname} failed with error code: -10" in str(excinfo.value)
442
443     def test_redis_backend_lock_object_string_representation(self):
444         expected_lock_info = {'lock DB type': 'Redis',
445                               'lock namespace': 'some-ns',
446                               'lock name': 'some-lock-name',
447                               'lock status': 'locked'}
448         assert str(self.lock) == str(expected_lock_info)
449
450     def test_redis_backend_lock_object_string_representation_can_catch_redis_exception(self):
451         self.mock_redis_lock.owned.side_effect = redis_exceptions.LockError('redis lock error!')
452         expected_lock_info = {'lock DB type': 'Redis',
453                               'lock namespace': 'some-ns',
454                               'lock name': 'some-lock-name',
455                               'lock status': 'Error: redis lock error!'}
456         assert str(self.lock) == str(expected_lock_info)
457
458
459 def test_redis_response_error_exception_is_mapped_to_rejected_by_backend_sdl_exception():
460     with pytest.raises(ricsdl.exceptions.RejectedByBackend) as excinfo:
461         with _map_to_sdl_exception():
462             raise redis_exceptions.ResponseError('Some redis error!')
463     assert "SDL backend rejected the request: Some redis error!" in str(excinfo.value)
464
465
466 def test_redis_connection_error_exception_is_mapped_to_not_connected_sdl_exception():
467     with pytest.raises(ricsdl.exceptions.NotConnected) as excinfo:
468         with _map_to_sdl_exception():
469             raise redis_exceptions.ConnectionError('Some redis error!')
470     assert "SDL not connected to backend: Some redis error!" in str(excinfo.value)
471
472
473 def test_rest_redis_exceptions_are_mapped_to_backend_error_sdl_exception():
474     with pytest.raises(ricsdl.exceptions.BackendError) as excinfo:
475         with _map_to_sdl_exception():
476             raise redis_exceptions.RedisError('Some redis error!')
477     assert "SDL backend failed to process the request: Some redis error!" in str(excinfo.value)
478
479
480 def test_system_error_exceptions_are_not_mapped_to_any_sdl_exception():
481     with pytest.raises(SystemExit):
482         with _map_to_sdl_exception():
483             raise SystemExit('Fatal error')