Add definable timeout for SyncStorage APIs 84/6584/3
authorTimo Tietavainen <timo.tietavainen@nokia.com>
Thu, 5 Aug 2021 08:46:07 +0000 (11:46 +0300)
committerTimo Tietavainen <timo.tietavainen@nokia.com>
Wed, 11 Aug 2021 04:20:48 +0000 (07:20 +0300)
Add a new synchronous API function 'SyncStorage::setOperationTimeout()',
which can be used to set given timeout value for synchronous SDL
instance. If SDL database backend (Redis) is not available, hanging SDL
write, read or delete operation is terminated and an exception is raised
after timeout time has been elapsed. By default if timeout value hasn't
been set, hanging SDL synchronous operation is blocked indefinitely if
Redis is not available.

Following additional improvements:
 - Improved AsyncConnectionImpl UT coverage
 - Fixed member variable order in Hiredis*EpollAdapter-classes.
   Hiredis*System must be first member variable because its functions
   are called via engine event handler. Thus it must be before engine.
 - Redis disconnected debug log is now written also in success status
   (SDL initiated) disconnection.

Issue-ID: RIC-226

Signed-off-by: Timo Tietavainen <timo.tietavainen@nokia.com>
Change-Id: I46e5e9738c25edb1c0d068dff739e765ca11096a

17 files changed:
Makefile.am
README.md
configure.ac
debian/changelog.in
docs/release-notes.rst
include/private/redis/hiredisclusterepolladapter.hpp
include/private/redis/hiredisepolladapter.hpp
include/private/syncstorageimpl.hpp
include/private/tst/syncstorageimplmock.hpp [new file with mode: 0644]
include/sdl/syncstorage.hpp
include/sdl/tst/mockablesyncstorage.hpp
rpm/sdl.spec.in
src/redis/asynchiredisclustercommanddispatcher.cpp
src/redis/hiredisclusterepolladapter.cpp
src/redis/hiredisepolladapter.cpp
src/syncstorageimpl.cpp
tst/syncstorageimpl_test.cpp

index 8471b4f..ad7d57a 100644 (file)
@@ -302,6 +302,7 @@ testrunner_SOURCES = \
     include/private/tst/enginemock.hpp \
     include/private/tst/gettopsrcdir.hpp \
     include/private/tst/namespaceconfigurationsmock.hpp \
+    include/private/tst/syncstorageimplmock.hpp \
     include/private/tst/systemmock.hpp \
     include/private/tst/wellknownerrorcode.hpp \
     tst/abort_test.cpp \
index 72c3422..0b88683 100644 (file)
--- a/README.md
+++ b/README.md
@@ -188,7 +188,7 @@ line options gtest supports, for example:
 Enable unit test gcov code coverage analysis by configuring gcov reporting
 directory:
 
-    configure --with-gcov-report-dir=DIR
+    ./configure --with-gcov-report-dir=DIR
 
 Directory can be an absolute path or a relative path to an SDL source root.
 Unit test build creates directory if it does not exist.
index 1943c3c..9be6d77 100644 (file)
@@ -10,8 +10,8 @@
 # Change the numbers just before release.
 
 m4_define([SDL_MAJOR], [1])
-m4_define([SDL_MINOR], [2])
-m4_define([SDL_MICRO], [1])
+m4_define([SDL_MINOR], [3])
+m4_define([SDL_MICRO], [0])
 
 # SDL ABI version with libtool
 #
@@ -27,8 +27,8 @@ m4_define([SDL_MICRO], [1])
 #
 # Change the numbers just before release.
 
-m4_define([SDL_CURRENT], [2])
-m4_define([SDL_REVISION], [9])
+m4_define([SDL_CURRENT], [3])
+m4_define([SDL_REVISION], [10])
 m4_define([SDL_AGE], [0])
 
 AC_INIT([shareddatalayer], [SDL_MAJOR.SDL_MINOR.SDL_MICRO], [], [], [https://gerrit.o-ran-sc.org/r/admin/repos/ric-plt/sdl])
index 268fce4..45e9059 100644 (file)
@@ -1,3 +1,10 @@
+sdl (1.3.0-1) UNRELEASED; urgency=low
+
+  * Definable timeout for DB backend readiness in synchronous SDL API
+  * Fix SDL configuration file path Valgrind errors
+
+ -- Timo Tietavainen <timo.tietavainen@nokia.com>  Thu, 05 Aug 2021 15:25:59 +0300
+
 sdl (1.2.1-1) UNRELEASED; urgency=low
 
   * Multiple DBAAS Redis standalone groups
index 63d7d29..4e00c74 100644 (file)
@@ -29,6 +29,11 @@ This document provides the release notes of the sdl library.
 Version history
 ---------------
 
+[1.3.0] - 2021-08-05
+
+* Definable timeout for DB backend readiness in synchronous SDL API
+* Fix SDL configuration file path Valgrind errors
+
 [1.2.1] - 2021-05-31
 
 * Multiple DBAAS Redis standalone groups
index 1cab762..b86af06 100644 (file)
@@ -92,8 +92,8 @@ namespace shareddatalayer
             Node& operator = (Node&&) = delete;
 
         private:
-            Engine& engine;
             HiredisClusterSystem& hiredisClusterSystem;
+            Engine& engine;
             redisAsyncContext* ac;
             unsigned int eventState;
             bool reading;
index 414bee1..e4e996b 100644 (file)
@@ -61,8 +61,8 @@ namespace shareddatalayer
             virtual void cleanUp();
 
         private:
-            Engine& engine;
             HiredisSystem& hiredisSystem;
+            Engine& engine;
             redisAsyncContext* ac;
             unsigned int eventState;
             bool reading;
index db762e0..735274f 100644 (file)
 #ifndef SHAREDDATALAYER_SYNCSTORAGEIMPL_HPP_
 #define SHAREDDATALAYER_SYNCSTORAGEIMPL_HPP_
 
+#include <sdl/asyncstorage.hpp>
 #include <sdl/syncstorage.hpp>
+#include <sys/poll.h>
 #include <system_error>
 
 namespace shareddatalayer
 {
-    class AsyncStorage;
-
     class System;
 
     class SyncStorageImpl: public SyncStorage
@@ -39,38 +39,50 @@ namespace shareddatalayer
         SyncStorageImpl(std::unique_ptr<AsyncStorage> asyncStorage,
                         System& system);
 
-        void set(const Namespace& ns, const DataMap& dataMap) override;
+        virtual void set(const Namespace& ns, const DataMap& dataMap) override;
+
+        virtual bool setIf(const Namespace& ns, const Key& key, const Data& oldData, const Data& newData) override;
 
-        bool setIf(const Namespace& ns, const Key& key, const Data& oldData, const Data& newData) override;
+        virtual bool setIfNotExists(const Namespace& ns, const Key& key, const Data& data) override;
 
-        bool setIfNotExists(const Namespace& ns, const Key& key, const Data& data) override;
+        virtual DataMap get(const Namespace& ns, const Keys& keys) override;
 
-        DataMap get(const Namespace& ns, const Keys& keys) override;
+        virtual void remove(const Namespace& ns, const Keys& keys) override;
 
-        void remove(const Namespace& ns, const Keys& keys) override;
+        virtual bool removeIf(const Namespace& ns, const Key& key, const Data& data) override;
 
-        bool removeIf(const Namespace& ns, const Key& key, const Data& data) override;
+        virtual Keys findKeys(const Namespace& ns, const std::string& keyPrefix) override;
 
-        Keys findKeys(const Namespace& ns, const std::string& keyPrefix) override;
+        virtual void removeAll(const Namespace& ns) override;
 
-        void removeAll(const Namespace& ns) override;
+        virtual void setOperationTimeout(const std::chrono::steady_clock::duration& timeout) override;
+
+        static constexpr int NO_TIMEOUT = -1;
 
     private:
         std::unique_ptr<AsyncStorage> asyncStorage;
         System& system;
-        int pFd;
         DataMap localMap;
         Keys localKeys;
         bool localStatus;
         std::error_code localError;
         bool synced;
+        bool isReady;
+        struct pollfd events;
+        std::chrono::steady_clock::duration operationTimeout;
 
         void verifyBackendResponse();
 
-        void waitForCallback();
+        void pollAndHandleEvents(int timeout_ms);
+
+        void waitForReadinessCheckCallback();
+
+        void waitForOperationCallback();
 
         void waitSdlToBeReady(const Namespace& ns);
 
+        void waitReadyAck(const std::error_code& error);
+
         void modifyAck(const std::error_code& error);
 
         void modifyIfAck(const std::error_code& error, bool status);
@@ -78,6 +90,8 @@ namespace shareddatalayer
         void getAck(const std::error_code& error, const DataMap& dataMap);
 
         void findKeysAck(const std::error_code& error, const Keys& keys);
+
+        void handlePendingEvents();
     };
 }
 
diff --git a/include/private/tst/syncstorageimplmock.hpp b/include/private/tst/syncstorageimplmock.hpp
new file mode 100644 (file)
index 0000000..6d1b677
--- /dev/null
@@ -0,0 +1,69 @@
+/*
+   Copyright (c) 2018-2021 Nokia.
+
+   Licensed under the Apache License, Version 2.0 (the "License");
+   you may not use this file except in compliance with the License.
+   You may obtain a copy of the License at
+
+       http://www.apache.org/licenses/LICENSE-2.0
+
+   Unless required by applicable law or agreed to in writing, software
+   distributed under the License is distributed on an "AS IS" BASIS,
+   WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+   See the License for the specific language governing permissions and
+   limitations under the License.
+*/
+
+/*
+ * This source code is part of the near-RT RIC (RAN Intelligent Controller)
+ * platform project (RICP).
+*/
+
+#ifndef SHAREDDATALAYER_TST_SYNCSTORAGEIMPLMOCK_HPP_
+#define SHAREDDATALAYER_TST_SYNCSTORAGEIMPLMOCK_HPP_
+
+#include "config.h"
+#include "private/syncstorageimpl.hpp"
+#include <gmock/gmock.h>
+
+namespace shareddatalayer
+{
+    namespace tst
+    {
+        class SyncStorageImplMock: public SyncStorageImpl
+        {
+        public:
+            SyncStorageImplMock(std::unique_ptr<AsyncStorageImpl> asyncStorageImpl):
+                SyncStorageImpl(std::move(asyncStorageImpl)) {}
+
+            MOCK_METHOD2(set,
+                         void(const Namespace& ns, const DataMap& dataMap));
+
+            MOCK_METHOD4(setIf,
+                         bool(const Namespace& ns, const Key& key, const Data& oldData, const Data& newData));
+
+            MOCK_METHOD4(setIf,
+                         bool(const Namespace& ns, const DataMap& dataToBeChecked, const Keys& keysWhichShouldNotExist, const DataMap& dataToBeWritten));
+
+            MOCK_METHOD3(setIfNotExists,
+                         bool(const Namespace& ns, const Key& key, const Data& data));
+
+            MOCK_METHOD2(get,
+                         DataMap(const Namespace& ns, const Keys& keys));
+
+            MOCK_METHOD2(remove,
+                         void(const Namespace& ns, const Keys& keys));
+
+            MOCK_METHOD3(removeIf,
+                         bool(const Namespace& ns, const Key& key, const Data& data));
+
+            MOCK_METHOD2(findKeys,
+                         Keys(const Namespace& ns, const std::string& keyPrefix));
+
+            MOCK_METHOD1(removeAll,
+                         void(const Namespace& ns));
+        };
+    }
+}
+
+#endif
index 684690b..409207f 100644 (file)
@@ -30,6 +30,7 @@
 #include <string>
 #include <utility>
 #include <vector>
+#include <chrono>
 #include <sdl/exception.hpp>
 #include <sdl/publisherid.hpp>
 
@@ -249,6 +250,21 @@ namespace shareddatalayer
          */
         virtual void removeAll(const Namespace& ns) = 0;
 
+        /**
+         * Set a timeout value for the synchronous SDL read, write and remove operations.
+         * By default synchronous read, write and remove operations do not have any timeout
+         * for the backend data storage readiness, operations are pending interminable to
+         * finish until backend is ready. With this API function default behaviour can be
+         * changed and when a timeout happens, an error exception is risen for the SDL
+         * operation in question. To avoid unnecessary timeout failure due to a temporal
+         * connection issue, it is recommended not to set too short timeout value.
+         * Reasonable timeout value is 5 seconds or bigger value. On a side note, timeout
+         * value 0 means interminable pending time.
+         *
+         * @param timeout Timeout value to set.
+         */
+         virtual void setOperationTimeout(const std::chrono::steady_clock::duration& timeout) = 0;
+
         /**
          * Create a new instance of SyncStorage.
          *
index d3f1d41..cde4adc 100644 (file)
@@ -62,6 +62,8 @@ namespace shareddatalayer
 
             virtual void removeAll(const Namespace&) override { logAndAbort(__PRETTY_FUNCTION__); }
 
+            virtual void setOperationTimeout(const std::chrono::steady_clock::duration&) override { logAndAbort(__PRETTY_FUNCTION__); }
+
         private:
             static void logAndAbort(const char* function) noexcept __attribute__ ((__noreturn__))
             {
index 46197cb..f80ea76 100755 (executable)
@@ -1,5 +1,5 @@
 Name:     sdl
-Version:  1.2.1
+Version:  1.3.0
 Release:  1%{?dist}
 Summary:  C++ API library for Shared Data Layer clients
 
@@ -50,6 +50,10 @@ rm -f %{buildroot}%{_libdir}/lib*.*a
 %{_includedir}/sdl
 
 %changelog
+* Thu Aug 05 2021 Timo Tietavainen <timo.tietavainen@nokia.com> - 1.3.0-1
+- Definable timeout for DB backend readiness in synchronous SDL API
+- Fix SDL configuration file path Valgrind errors
+
 * Mon May 31 2021 Petri Ovaska <petri.ovaska@nokia.com> - 1.2.1-1
 - Multiple DBAAS Redis standalone groups
 
index ac4fc34..86cd95f 100644 (file)
@@ -53,13 +53,11 @@ namespace
 
     void disconnectCb(const redisClusterAsyncContext* acc, const redisAsyncContext* ac, int status)
     {
-        if (status)
-        {
-            std::ostringstream msg;
-            msg << "redis cluster instance disconnected, fd: " << ac->c.fd
-                << ", status: " << ac->err;
-            logDebugOnce(msg.str());
-        }
+        std::ostringstream msg;
+        msg << "redis cluster instance disconnected, status: " << ac->err
+            << ", " << ac->errstr << ", fd: " << ac->c.fd << std::endl;
+        logDebugOnce(msg.str());
+
         auto instance(static_cast<AsyncHiredisClusterCommandDispatcher*>(acc->data));
         instance->handleDisconnect(ac);
     }
index 05e2827..9d2a0df 100644 (file)
@@ -103,8 +103,8 @@ void HiredisClusterEpollAdapter::detach(const redisAsyncContext* ac)
 HiredisClusterEpollAdapter::Node::Node(Engine& engine,
                                        redisAsyncContext* ac,
                                        HiredisClusterSystem& hiredisClusterSystem):
-    engine(engine),
     hiredisClusterSystem(hiredisClusterSystem),
+    engine(engine),
     ac(ac),
     eventState(0),
     reading(false),
index c2529b8..775fad2 100644 (file)
@@ -67,8 +67,8 @@ HiredisEpollAdapter::HiredisEpollAdapter(Engine& engine):
 }
 
 HiredisEpollAdapter::HiredisEpollAdapter(Engine& engine, HiredisSystem& hiredisSystem):
-    engine(engine),
     hiredisSystem(hiredisSystem),
+    engine(engine),
     ac(nullptr),
     eventState(0),
     reading(false),
index 4ddc723..76c1546 100644 (file)
@@ -19,8 +19,8 @@
  * platform project (RICP).
 */
 
+#include <chrono>
 #include <sstream>
-#include <sys/poll.h>
 #include <sdl/asyncstorage.hpp>
 #include <sdl/backenderror.hpp>
 #include <sdl/errorqueries.hpp>
@@ -58,6 +58,10 @@ namespace
     }
 }
 
+/* TODO: This synchronous API implementation could probably be refactored to be boost::asio based
+ * instead of current (bit error prone) poll based implementation.
+ */
+
 SyncStorageImpl::SyncStorageImpl(std::unique_ptr<AsyncStorage> asyncStorage):
     SyncStorageImpl(std::move(asyncStorage), System::getSystem())
 {
@@ -67,10 +71,17 @@ SyncStorageImpl::SyncStorageImpl(std::unique_ptr<AsyncStorage> pAsyncStorage,
                                  System& system):
     asyncStorage(std::move(pAsyncStorage)),
     system(system),
-    pFd(asyncStorage->fd()),
     localStatus(false),
-    synced(false)
+    synced(false),
+    isReady(false),
+    events{ asyncStorage->fd(), POLLIN, 0 },
+    operationTimeout(std::chrono::steady_clock::duration::zero())
+{
+}
+
+void SyncStorageImpl::waitReadyAck(const std::error_code&)
 {
+    isReady = true;
 }
 
 void SyncStorageImpl::modifyAck(const std::error_code& error)
@@ -106,40 +117,62 @@ void SyncStorageImpl::verifyBackendResponse()
         throwExceptionForErrorCode(localError);
 }
 
-void SyncStorageImpl::waitForCallback()
+void SyncStorageImpl::waitForOperationCallback()
 {
-    struct pollfd events { pFd, POLLIN, 0 };
     while(!synced)
-        if (system.poll(&events, 1, -1) > 0 && (events.revents & POLLIN))
-            asyncStorage->handleEvents();
+        pollAndHandleEvents(NO_TIMEOUT);
+}
+
+void SyncStorageImpl::pollAndHandleEvents(int timeout_ms)
+{
+    if (system.poll(&events, 1, timeout_ms) > 0 && (events.revents & POLLIN))
+        asyncStorage->handleEvents();
+}
+
+void SyncStorageImpl::waitForReadinessCheckCallback()
+{
+    if (operationTimeout == std::chrono::steady_clock::duration::zero())
+    {
+        while (!isReady)
+            pollAndHandleEvents(NO_TIMEOUT);
+    }
+    else
+    {
+        int pollTimeout_ms = std::chrono::duration_cast<std::chrono::milliseconds>(operationTimeout).count() / 10;
+        std::chrono::steady_clock::time_point start = std::chrono::steady_clock::now();
+        while(!isReady && (std::chrono::steady_clock::now() - start < operationTimeout))
+            pollAndHandleEvents(pollTimeout_ms);
+    }
 }
 
 void SyncStorageImpl::waitSdlToBeReady(const Namespace& ns)
 {
-    synced = false;
+    isReady = false;
     asyncStorage->waitReadyAsync(ns,
-                                 std::bind(&shareddatalayer::SyncStorageImpl::modifyAck,
+                                 std::bind(&shareddatalayer::SyncStorageImpl::waitReadyAck,
                                            this,
                                            std::error_code()));
-    waitForCallback();
-    verifyBackendResponse();
+    waitForReadinessCheckCallback();
 }
 
 void SyncStorageImpl::set(const Namespace& ns, const DataMap& dataMap)
 {
+    handlePendingEvents();
     waitSdlToBeReady(ns);
     synced = false;
+
     asyncStorage->setAsync(ns,
                            dataMap,
                            std::bind(&shareddatalayer::SyncStorageImpl::modifyAck,
                                      this,
                                      std::placeholders::_1));
-    waitForCallback();
+    waitForOperationCallback();
     verifyBackendResponse();
 }
 
 bool SyncStorageImpl::setIf(const Namespace& ns, const Key& key, const Data& oldData, const Data& newData)
 {
+    handlePendingEvents();
     waitSdlToBeReady(ns);
     synced = false;
     asyncStorage->setIfAsync(ns,
@@ -150,13 +183,14 @@ bool SyncStorageImpl::setIf(const Namespace& ns, const Key& key, const Data& old
                                        this,
                                        std::placeholders::_1,
                                        std::placeholders::_2));
-    waitForCallback();
+    waitForOperationCallback();
     verifyBackendResponse();
     return localStatus;
 }
 
 bool SyncStorageImpl::setIfNotExists(const Namespace& ns, const Key& key, const Data& data)
 {
+    handlePendingEvents();
     waitSdlToBeReady(ns);
     synced = false;
     asyncStorage->setIfNotExistsAsync(ns,
@@ -166,13 +200,14 @@ bool SyncStorageImpl::setIfNotExists(const Namespace& ns, const Key& key, const
                                                 this,
                                                 std::placeholders::_1,
                                                 std::placeholders::_2));
-    waitForCallback();
+    waitForOperationCallback();
     verifyBackendResponse();
     return localStatus;
 }
 
 SyncStorageImpl::DataMap SyncStorageImpl::get(const Namespace& ns, const Keys& keys)
 {
+    handlePendingEvents();
     waitSdlToBeReady(ns);
     synced = false;
     asyncStorage->getAsync(ns,
@@ -181,13 +216,14 @@ SyncStorageImpl::DataMap SyncStorageImpl::get(const Namespace& ns, const Keys& k
                                      this,
                                      std::placeholders::_1,
                                      std::placeholders::_2));
-    waitForCallback();
+    waitForOperationCallback();
     verifyBackendResponse();
     return localMap;
 }
 
 void SyncStorageImpl::remove(const Namespace& ns, const Keys& keys)
 {
+    handlePendingEvents();
     waitSdlToBeReady(ns);
     synced = false;
     asyncStorage->removeAsync(ns,
@@ -195,12 +231,13 @@ void SyncStorageImpl::remove(const Namespace& ns, const Keys& keys)
                               std::bind(&shareddatalayer::SyncStorageImpl::modifyAck,
                                         this,
                                         std::placeholders::_1));
-    waitForCallback();
+    waitForOperationCallback();
     verifyBackendResponse();
 }
 
 bool SyncStorageImpl::removeIf(const Namespace& ns, const Key& key, const Data& data)
 {
+    handlePendingEvents();
     waitSdlToBeReady(ns);
     synced = false;
     asyncStorage->removeIfAsync(ns,
@@ -210,13 +247,14 @@ bool SyncStorageImpl::removeIf(const Namespace& ns, const Key& key, const Data&
                                           this,
                                           std::placeholders::_1,
                                           std::placeholders::_2));
-    waitForCallback();
+    waitForOperationCallback();
     verifyBackendResponse();
     return localStatus;
 }
 
 SyncStorageImpl::Keys SyncStorageImpl::findKeys(const Namespace& ns, const std::string& keyPrefix)
 {
+    handlePendingEvents();
     waitSdlToBeReady(ns);
     synced = false;
     asyncStorage->findKeysAsync(ns,
@@ -225,19 +263,36 @@ SyncStorageImpl::Keys SyncStorageImpl::findKeys(const Namespace& ns, const std::
                                           this,
                                           std::placeholders::_1,
                                           std::placeholders::_2));
-    waitForCallback();
+    waitForOperationCallback();
     verifyBackendResponse();
     return localKeys;
 }
 
 void SyncStorageImpl::removeAll(const Namespace& ns)
 {
+    handlePendingEvents();
     waitSdlToBeReady(ns);
     synced = false;
     asyncStorage->removeAllAsync(ns,
                                  std::bind(&shareddatalayer::SyncStorageImpl::modifyAck,
                                            this,
                                            std::placeholders::_1));
-    waitForCallback();
+    waitForOperationCallback();
     verifyBackendResponse();
 }
+
+void SyncStorageImpl::handlePendingEvents()
+{
+    int pollRetVal = system.poll(&events, 1, 0);
+
+    while (pollRetVal > 0 && events.revents & POLLIN)
+    {
+        asyncStorage->handleEvents();
+        pollRetVal = system.poll(&events, 1, 0);
+    }
+}
+
+void SyncStorageImpl::setOperationTimeout(const std::chrono::steady_clock::duration& timeout)
+{
+    operationTimeout = timeout;
+}
index 59704be..7860334 100644 (file)
@@ -59,18 +59,27 @@ namespace
         SyncStorage::DataMap dataMap;
         SyncStorage::Keys keys;
         const SyncStorage::Namespace ns;
+        std::chrono::steady_clock::duration TEST_OPERATION_WAIT_TIMEOUT;
+        int TEST_OPERATION_POLL_WAIT_TIMEOUT;
         SyncStorageImplTest():
             asyncStorageMockPassedToImplementation(new StrictMock<AsyncStorageMock>()),
             asyncStorageMockRawPtr(asyncStorageMockPassedToImplementation.get()),
             pFd(10),
             dataMap({{ "key1", { 0x0a, 0x0b, 0x0c } }, { "key2", { 0x0d, 0x0e, 0x0f, 0xff } }}),
             keys({ "key1", "key2" }),
-            ns("someKnownNamespace")
+            ns("someKnownNamespace"),
+            TEST_OPERATION_WAIT_TIMEOUT(std::chrono::seconds(1)),
+            TEST_OPERATION_POLL_WAIT_TIMEOUT(std::chrono::duration_cast<std::chrono::milliseconds>(TEST_OPERATION_WAIT_TIMEOUT).count() / 10)
         {
             expectConstructorCalls();
             syncStorage.reset(new SyncStorageImpl(std::move(asyncStorageMockPassedToImplementation), systemMock));
         }
 
+        ~SyncStorageImplTest()
+        {
+            syncStorage->setOperationTimeout(std::chrono::steady_clock::duration::zero());
+        }
+
         void expectConstructorCalls()
         {
             InSequence dummy;
@@ -79,17 +88,28 @@ namespace
                 .WillOnce(Return(pFd));
         }
 
-        void expectSdlReadinessCheck()
+        void expectSdlReadinessCheck(int timeout)
         {
             InSequence dummy;
+            expectPollForPendingEvents_ReturnNoEvents();
             expectWaitReadyAsync();
-            expectPollWait();
-            expectHandleEvents();
+            expectPollWait(timeout);
+            expectHandleEvents_callWaitReadyAck();
         }
 
-        void expectPollWait()
+        void expectPollForPendingEvents_ReturnNoEvents()
         {
-            EXPECT_CALL(systemMock, poll( _, 1, -1))
+            EXPECT_CALL(systemMock, poll( _, 1, 0))
+                .Times(1)
+                .WillOnce(Invoke([](struct pollfd *, nfds_t, int)
+                                 {
+                                     return 0;
+                                 }));
+        }
+
+        void expectPollWait(int timeout)
+        {
+            EXPECT_CALL(systemMock, poll( _, 1, timeout))
                 .Times(1)
                 .WillOnce(Invoke([](struct pollfd *fds, nfds_t, int)
                                  {
@@ -121,6 +141,12 @@ namespace
         }
 
         void expectHandleEvents()
+        {
+            EXPECT_CALL(*asyncStorageMockRawPtr, handleEvents())
+                .Times(1);
+        }
+
+        void expectHandleEvents_callWaitReadyAck()
         {
             EXPECT_CALL(*asyncStorageMockRawPtr, handleEvents())
                 .Times(1)
@@ -130,6 +156,16 @@ namespace
                                  }));
         }
 
+        void expectHandleEvents_callModifyAck()
+        {
+            EXPECT_CALL(*asyncStorageMockRawPtr, handleEvents())
+                .Times(1)
+                .WillOnce(Invoke([this]()
+                                 {
+                                    savedModifyAck(std::error_code());
+                                 }));
+        }
+
         void expectWaitReadyAsync()
         {
             EXPECT_CALL(*asyncStorageMockRawPtr, waitReadyAsync(ns,_))
@@ -272,41 +308,52 @@ TEST_F(SyncStorageImplTest, ImplementssyncStorage)
 TEST_F(SyncStorageImplTest, EventsAreNotHandledWhenPollReturnsError)
 {
     InSequence dummy;
-    expectSdlReadinessCheck();
+    expectSdlReadinessCheck(SyncStorageImpl::NO_TIMEOUT);
     expectSetAsync(dataMap);
     expectPollError();
-    expectPollWait();
-    expectHandleEvents();
+    expectPollWait(SyncStorageImpl::NO_TIMEOUT);
+    expectHandleEvents_callModifyAck();
     syncStorage->set(ns, dataMap);
 }
 
 TEST_F(SyncStorageImplTest, EventsAreNotHandledWhenThereIsAnExceptionalConditionOnTheFd)
 {
     InSequence dummy;
-    expectSdlReadinessCheck();
+    expectSdlReadinessCheck(SyncStorageImpl::NO_TIMEOUT);
     expectSetAsync(dataMap);
     expectPollExceptionalCondition();
-    expectPollWait();
-    expectHandleEvents();
+    expectPollWait(SyncStorageImpl::NO_TIMEOUT);
+    expectHandleEvents_callModifyAck();
     syncStorage->set(ns, dataMap);
 }
 
 TEST_F(SyncStorageImplTest, SetSuccessfully)
 {
     InSequence dummy;
-    expectSdlReadinessCheck();
+    expectSdlReadinessCheck(SyncStorageImpl::NO_TIMEOUT);
+    expectSetAsync(dataMap);
+    expectPollWait(SyncStorageImpl::NO_TIMEOUT);
+    expectHandleEvents_callModifyAck();
+    syncStorage->set(ns, dataMap);
+}
+
+TEST_F(SyncStorageImplTest, SetWithReadinessTimeoutSuccessfully)
+{
+    InSequence dummy;
+    expectSdlReadinessCheck(TEST_OPERATION_POLL_WAIT_TIMEOUT);
     expectSetAsync(dataMap);
-    expectPollWait();
-    expectHandleEvents();
+    expectPollWait(SyncStorageImpl::NO_TIMEOUT);
+    expectHandleEvents_callModifyAck();
+    syncStorage->setOperationTimeout(TEST_OPERATION_WAIT_TIMEOUT);
     syncStorage->set(ns, dataMap);
 }
 
 TEST_F(SyncStorageImplTest, SetCanThrowBackendError)
 {
     InSequence dummy;
-    expectSdlReadinessCheck();
+    expectSdlReadinessCheck(SyncStorageImpl::NO_TIMEOUT);
     expectSetAsync(dataMap);
-    expectPollWait();
+    expectPollWait(SyncStorageImpl::NO_TIMEOUT);
     expectModifyAckWithError();
     EXPECT_THROW(syncStorage->set(ns, dataMap), BackendError);
 }
@@ -314,29 +361,45 @@ TEST_F(SyncStorageImplTest, SetCanThrowBackendError)
 TEST_F(SyncStorageImplTest, SetIfSuccessfully)
 {
     InSequence dummy;
-    expectSdlReadinessCheck();
+    expectSdlReadinessCheck(SyncStorageImpl::NO_TIMEOUT);
     expectSetAsync(dataMap);
-    expectPollWait();
-    expectHandleEvents();
+    expectPollWait(SyncStorageImpl::NO_TIMEOUT);
+    expectHandleEvents_callModifyAck();
     syncStorage->set(ns, dataMap);
-    expectSdlReadinessCheck();
+    expectSdlReadinessCheck(SyncStorageImpl::NO_TIMEOUT);
     expectSetIfAsync("key1", { 0x0a, 0x0b, 0x0c }, { 0x0d, 0x0e, 0x0f });
-    expectPollWait();
-    expectHandleEvents();
+    expectPollWait(SyncStorageImpl::NO_TIMEOUT);
+    expectHandleEvents_callModifyAck();
+    syncStorage->setIf(ns, "key1", { 0x0a, 0x0b, 0x0c }, { 0x0d, 0x0e, 0x0f });
+}
+
+TEST_F(SyncStorageImplTest, SetIfWithReadinessTimeoutSuccessfully)
+{
+    InSequence dummy;
+    expectSdlReadinessCheck(TEST_OPERATION_POLL_WAIT_TIMEOUT);
+    expectSetAsync(dataMap);
+    expectPollWait(SyncStorageImpl::NO_TIMEOUT);
+    expectHandleEvents_callModifyAck();
+    syncStorage->setOperationTimeout(TEST_OPERATION_WAIT_TIMEOUT);
+    syncStorage->set(ns, dataMap);
+    expectSdlReadinessCheck(TEST_OPERATION_POLL_WAIT_TIMEOUT);
+    expectSetIfAsync("key1", { 0x0a, 0x0b, 0x0c }, { 0x0d, 0x0e, 0x0f });
+    expectPollWait(SyncStorageImpl::NO_TIMEOUT);
+    expectHandleEvents_callModifyAck();
     syncStorage->setIf(ns, "key1", { 0x0a, 0x0b, 0x0c }, { 0x0d, 0x0e, 0x0f });
 }
 
 TEST_F(SyncStorageImplTest, SetIfCanThrowBackendError)
 {
     InSequence dummy;
-    expectSdlReadinessCheck();
+    expectSdlReadinessCheck(SyncStorageImpl::NO_TIMEOUT);
     expectSetAsync(dataMap);
-    expectPollWait();
-    expectHandleEvents();
+    expectPollWait(SyncStorageImpl::NO_TIMEOUT);
+    expectHandleEvents_callModifyAck();
     syncStorage->set(ns, dataMap);
-    expectSdlReadinessCheck();
+    expectSdlReadinessCheck(SyncStorageImpl::NO_TIMEOUT);
     expectSetIfAsync("key1", { 0x0a, 0x0b, 0x0c }, { 0x0d, 0x0e, 0x0f });
-    expectPollWait();
+    expectPollWait(SyncStorageImpl::NO_TIMEOUT);
     expectModifyIfAck(AsyncRedisCommandDispatcherErrorCode::OUT_OF_MEMORY, false);
     EXPECT_THROW(syncStorage->setIf(ns, "key1", { 0x0a, 0x0b, 0x0c }, { 0x0d, 0x0e, 0x0f }), BackendError);
 }
@@ -344,19 +407,30 @@ TEST_F(SyncStorageImplTest, SetIfCanThrowBackendError)
 TEST_F(SyncStorageImplTest, SetIfNotExistsSuccessfully)
 {
     InSequence dummy;
-    expectSdlReadinessCheck();
+    expectSdlReadinessCheck(SyncStorageImpl::NO_TIMEOUT);
+    expectSetIfNotExistsAsync("key1", { 0x0a, 0x0b, 0x0c });
+    expectPollWait(SyncStorageImpl::NO_TIMEOUT);
+    expectModifyIfAck(std::error_code(), true);
+    EXPECT_TRUE(syncStorage->setIfNotExists(ns, "key1", { 0x0a, 0x0b, 0x0c }));
+}
+
+TEST_F(SyncStorageImplTest, SetIfNotExistsIfWithReadinessTimeoutSuccessfully)
+{
+    InSequence dummy;
+    expectSdlReadinessCheck(TEST_OPERATION_POLL_WAIT_TIMEOUT);
     expectSetIfNotExistsAsync("key1", { 0x0a, 0x0b, 0x0c });
-    expectPollWait();
+    expectPollWait(SyncStorageImpl::NO_TIMEOUT);
     expectModifyIfAck(std::error_code(), true);
+    syncStorage->setOperationTimeout(TEST_OPERATION_WAIT_TIMEOUT);
     EXPECT_TRUE(syncStorage->setIfNotExists(ns, "key1", { 0x0a, 0x0b, 0x0c }));
 }
 
 TEST_F(SyncStorageImplTest, SetIfNotExistsReturnsFalseIfKeyAlreadyExists)
 {
     InSequence dummy;
-    expectSdlReadinessCheck();
+    expectSdlReadinessCheck(SyncStorageImpl::NO_TIMEOUT);
     expectSetIfNotExistsAsync("key1", { 0x0a, 0x0b, 0x0c });
-    expectPollWait();
+    expectPollWait(SyncStorageImpl::NO_TIMEOUT);
     expectModifyIfAck(std::error_code(), false);
     EXPECT_FALSE(syncStorage->setIfNotExists(ns, "key1", { 0x0a, 0x0b, 0x0c }));
 }
@@ -364,9 +438,9 @@ TEST_F(SyncStorageImplTest, SetIfNotExistsReturnsFalseIfKeyAlreadyExists)
 TEST_F(SyncStorageImplTest, SetIfNotExistsCanThrowBackendError)
 {
     InSequence dummy;
-    expectSdlReadinessCheck();
+    expectSdlReadinessCheck(SyncStorageImpl::NO_TIMEOUT);
     expectSetIfNotExistsAsync("key1", { 0x0a, 0x0b, 0x0c });
-    expectPollWait();
+    expectPollWait(SyncStorageImpl::NO_TIMEOUT);
     expectModifyIfAck(AsyncRedisCommandDispatcherErrorCode::OUT_OF_MEMORY, false);
     EXPECT_THROW(syncStorage->setIfNotExists(ns, "key1", { 0x0a, 0x0b, 0x0c }), BackendError);
 }
@@ -374,10 +448,22 @@ TEST_F(SyncStorageImplTest, SetIfNotExistsCanThrowBackendError)
 TEST_F(SyncStorageImplTest, GetSuccessfully)
 {
     InSequence dummy;
-    expectSdlReadinessCheck();
+    expectSdlReadinessCheck(SyncStorageImpl::NO_TIMEOUT);
+    expectGetAsync(keys);
+    expectPollWait(SyncStorageImpl::NO_TIMEOUT);
+    expectGetAck();
+    auto map(syncStorage->get(ns, keys));
+    EXPECT_EQ(map, dataMap);
+}
+
+TEST_F(SyncStorageImplTest, GetWithReadinessTimeoutSuccessfully)
+{
+    InSequence dummy;
+    expectSdlReadinessCheck(TEST_OPERATION_POLL_WAIT_TIMEOUT);
     expectGetAsync(keys);
-    expectPollWait();
+    expectPollWait(SyncStorageImpl::NO_TIMEOUT);
     expectGetAck();
+    syncStorage->setOperationTimeout(TEST_OPERATION_WAIT_TIMEOUT);
     auto map(syncStorage->get(ns, keys));
     EXPECT_EQ(map, dataMap);
 }
@@ -385,9 +471,9 @@ TEST_F(SyncStorageImplTest, GetSuccessfully)
 TEST_F(SyncStorageImplTest, GetCanThrowBackendError)
 {
     InSequence dummy;
-    expectSdlReadinessCheck();
+    expectSdlReadinessCheck(SyncStorageImpl::NO_TIMEOUT);
     expectGetAsync(keys);
-    expectPollWait();
+    expectPollWait(SyncStorageImpl::NO_TIMEOUT);
     expectGetAckWithError();
     EXPECT_THROW(syncStorage->get(ns, keys), BackendError);
 }
@@ -395,19 +481,30 @@ TEST_F(SyncStorageImplTest, GetCanThrowBackendError)
 TEST_F(SyncStorageImplTest, RemoveSuccessfully)
 {
     InSequence dummy;
-    expectSdlReadinessCheck();
+    expectSdlReadinessCheck(SyncStorageImpl::NO_TIMEOUT);
+    expectRemoveAsync(keys);
+    expectPollWait(SyncStorageImpl::NO_TIMEOUT);
+    expectHandleEvents_callModifyAck();
+    syncStorage->remove(ns, keys);
+}
+
+TEST_F(SyncStorageImplTest, RemoveWithReadinessTimeoutSuccessfully)
+{
+    InSequence dummy;
+    expectSdlReadinessCheck(TEST_OPERATION_POLL_WAIT_TIMEOUT);
     expectRemoveAsync(keys);
-    expectPollWait();
-    expectHandleEvents();
+    expectPollWait(SyncStorageImpl::NO_TIMEOUT);
+    expectHandleEvents_callModifyAck();
+    syncStorage->setOperationTimeout(TEST_OPERATION_WAIT_TIMEOUT);
     syncStorage->remove(ns, keys);
 }
 
 TEST_F(SyncStorageImplTest, RemoveCanThrowBackendError)
 {
     InSequence dummy;
-    expectSdlReadinessCheck();
+    expectSdlReadinessCheck(SyncStorageImpl::NO_TIMEOUT);
     expectRemoveAsync(keys);
-    expectPollWait();
+    expectPollWait(SyncStorageImpl::NO_TIMEOUT);
     expectModifyAckWithError();
     EXPECT_THROW(syncStorage->remove(ns, keys), BackendError);
 }
@@ -415,19 +512,30 @@ TEST_F(SyncStorageImplTest, RemoveCanThrowBackendError)
 TEST_F(SyncStorageImplTest, RemoveIfSuccessfully)
 {
     InSequence dummy;
-    expectSdlReadinessCheck();
+    expectSdlReadinessCheck(SyncStorageImpl::NO_TIMEOUT);
     expectRemoveIfAsync("key1", { 0x0a, 0x0b, 0x0c });
-    expectPollWait();
+    expectPollWait(SyncStorageImpl::NO_TIMEOUT);
     expectModifyIfAck(std::error_code(), true);
     EXPECT_TRUE(syncStorage->removeIf(ns, "key1", { 0x0a, 0x0b, 0x0c }));
 }
 
+TEST_F(SyncStorageImplTest, RemoveIfWithReadinessTimeoutSuccessfully)
+{
+    InSequence dummy;
+    expectSdlReadinessCheck(TEST_OPERATION_POLL_WAIT_TIMEOUT);
+    expectRemoveIfAsync("key1", { 0x0a, 0x0b, 0x0c });
+    expectPollWait(SyncStorageImpl::NO_TIMEOUT);
+    expectModifyIfAck(std::error_code(), true);
+    syncStorage->setOperationTimeout(TEST_OPERATION_WAIT_TIMEOUT);
+    EXPECT_TRUE(syncStorage->removeIf(ns, "key1", { 0x0a, 0x0b, 0x0c }));
+}
+
 TEST_F(SyncStorageImplTest, RemoveIfReturnsFalseIfKeyDoesnotMatch)
 {
     InSequence dummy;
-    expectSdlReadinessCheck();
+    expectSdlReadinessCheck(SyncStorageImpl::NO_TIMEOUT);
     expectRemoveIfAsync("key1", { 0x0a, 0x0b, 0x0c });
-    expectPollWait();
+    expectPollWait(SyncStorageImpl::NO_TIMEOUT);
     expectModifyIfAck(std::error_code(), false);
     EXPECT_FALSE(syncStorage->removeIf(ns, "key1", { 0x0a, 0x0b, 0x0c }));
 }
@@ -435,9 +543,9 @@ TEST_F(SyncStorageImplTest, RemoveIfReturnsFalseIfKeyDoesnotMatch)
 TEST_F(SyncStorageImplTest, RemoveIfCanThrowBackendError)
 {
     InSequence dummy;
-    expectSdlReadinessCheck();
+    expectSdlReadinessCheck(SyncStorageImpl::NO_TIMEOUT);
     expectRemoveIfAsync("key1", { 0x0a, 0x0b, 0x0c });
-    expectPollWait();
+    expectPollWait(SyncStorageImpl::NO_TIMEOUT);
     expectModifyIfAck(AsyncRedisCommandDispatcherErrorCode::OUT_OF_MEMORY, false);
     EXPECT_THROW(syncStorage->removeIf(ns, "key1", { 0x0a, 0x0b, 0x0c }), BackendError);
 }
@@ -445,20 +553,32 @@ TEST_F(SyncStorageImplTest, RemoveIfCanThrowBackendError)
 TEST_F(SyncStorageImplTest, FindKeysSuccessfully)
 {
     InSequence dummy;
-    expectSdlReadinessCheck();
+    expectSdlReadinessCheck(SyncStorageImpl::NO_TIMEOUT);
     expectFindKeysAsync();
-    expectPollWait();
+    expectPollWait(SyncStorageImpl::NO_TIMEOUT);
     expectFindKeysAck();
     auto ids(syncStorage->findKeys(ns, "*"));
     EXPECT_EQ(ids, keys);
 }
 
+TEST_F(SyncStorageImplTest, FindKeysWithReadinessTimeoutSuccessfully)
+{
+    InSequence dummy;
+    expectSdlReadinessCheck(TEST_OPERATION_POLL_WAIT_TIMEOUT);
+    expectFindKeysAsync();
+    expectPollWait(SyncStorageImpl::NO_TIMEOUT);
+    expectFindKeysAck();
+    syncStorage->setOperationTimeout(TEST_OPERATION_WAIT_TIMEOUT);
+    auto ids(syncStorage->findKeys(ns, "*"));
+    EXPECT_EQ(ids, keys);
+}
+
 TEST_F(SyncStorageImplTest, FindKeysAckCanThrowBackendError)
 {
     InSequence dummy;
-    expectSdlReadinessCheck();
+    expectSdlReadinessCheck(SyncStorageImpl::NO_TIMEOUT);
     expectFindKeysAsync();
-    expectPollWait();
+    expectPollWait(SyncStorageImpl::NO_TIMEOUT);
     expectFindKeysAckWithError();
     EXPECT_THROW(syncStorage->findKeys(ns, "*"), BackendError);
 }
@@ -466,19 +586,30 @@ TEST_F(SyncStorageImplTest, FindKeysAckCanThrowBackendError)
 TEST_F(SyncStorageImplTest, RemoveAllSuccessfully)
 {
     InSequence dummy;
-    expectSdlReadinessCheck();
+    expectSdlReadinessCheck(SyncStorageImpl::NO_TIMEOUT);
+    expectRemoveAllAsync();
+    expectPollWait(SyncStorageImpl::NO_TIMEOUT);
+    expectHandleEvents_callModifyAck();
+    syncStorage->removeAll(ns);
+}
+
+TEST_F(SyncStorageImplTest, RemoveAllWithReadinessTimeoutSuccessfully)
+{
+    InSequence dummy;
+    expectSdlReadinessCheck(TEST_OPERATION_POLL_WAIT_TIMEOUT);
     expectRemoveAllAsync();
-    expectPollWait();
-    expectHandleEvents();
+    expectPollWait(SyncStorageImpl::NO_TIMEOUT);
+    expectHandleEvents_callModifyAck();
+    syncStorage->setOperationTimeout(TEST_OPERATION_WAIT_TIMEOUT);
     syncStorage->removeAll(ns);
 }
 
 TEST_F(SyncStorageImplTest, RemoveAllCanThrowBackendError)
 {
     InSequence dummy;
-    expectSdlReadinessCheck();
+    expectSdlReadinessCheck(SyncStorageImpl::NO_TIMEOUT);
     expectRemoveAllAsync();
-    expectPollWait();
+    expectPollWait(SyncStorageImpl::NO_TIMEOUT);
     expectModifyAckWithError();
     EXPECT_THROW(syncStorage->removeAll(ns), BackendError);
 }
@@ -492,9 +623,9 @@ TEST_F(SyncStorageImplTest, AllAsyncRedisStorageErrorCodesThrowCorrectException)
     {
         if (arsec != AsyncRedisStorage::ErrorCode::SUCCESS)
         {
-            expectSdlReadinessCheck();
+            expectSdlReadinessCheck(SyncStorageImpl::NO_TIMEOUT);
             expectSetIfNotExistsAsync("key1", { 0x0a, 0x0b, 0x0c });
-            expectPollWait();
+            expectPollWait(SyncStorageImpl::NO_TIMEOUT);
         }
 
         switch (arsec)
@@ -525,9 +656,9 @@ TEST_F(SyncStorageImplTest, AllDispatcherErrorCodesThrowCorrectException)
     {
         if (aec != AsyncRedisCommandDispatcherErrorCode::SUCCESS)
         {
-            expectSdlReadinessCheck();
+            expectSdlReadinessCheck(SyncStorageImpl::NO_TIMEOUT);
             expectSetIfNotExistsAsync("key1", { 0x0a, 0x0b, 0x0c });
-            expectPollWait();
+            expectPollWait(SyncStorageImpl::NO_TIMEOUT);
         }
 
         switch (aec)
@@ -576,9 +707,9 @@ TEST_F(SyncStorageImplTest, AllDispatcherErrorCodesThrowCorrectException)
 TEST_F(SyncStorageImplTest, CanThrowStdExceptionIfDispatcherErrorCodeCannotBeMappedToSdlException)
 {
     InSequence dummy;
-    expectSdlReadinessCheck();
+    expectSdlReadinessCheck(SyncStorageImpl::NO_TIMEOUT);
     expectSetIfNotExistsAsync("key1", { 0x0a, 0x0b, 0x0c });
-    expectPollWait();
+    expectPollWait(SyncStorageImpl::NO_TIMEOUT);
     expectModifyIfAck(std::error_code(1, std::system_category()), false);
     EXPECT_THROW(syncStorage->setIfNotExists(ns, "key1", { 0x0a, 0x0b, 0x0c }), std::range_error);
 }