From 63869e10ac4d8572238989e1b582c0314da91f9c Mon Sep 17 00:00:00 2001 From: Petri Ovaska Date: Fri, 17 Sep 2021 11:54:21 +0300 Subject: [PATCH] New listKeys() API to support glob-style key search patterns Added new listKeys() API's to list all keys matching search glob-style pattern under the namespace. Deprecated old findKeys() and findKeysAsync() API's. Deprecated API's will be removed later releases. Release version 1.5.0 Issue-Id: RIC-110 Change-Id: Ia411a2e6fa45305580810d88e7434e5460ac3114 Signed-off-by: Petri Ovaska --- Makefile.am | 1 + configure.ac | 6 ++-- debian/changelog.in | 7 +++++ docs/release-notes.rst | 5 ++++ include/private/asyncdummystorage.hpp | 2 ++ include/private/asyncstorageimpl.hpp | 2 ++ include/private/redis/asyncredisstorage.hpp | 6 ++++ include/private/syncstorageimpl.hpp | 2 ++ include/private/tst/asyncstoragemock.hpp | 2 ++ include/sdl/asyncstorage.hpp | 24 ++++++++++++++++ include/sdl/syncstorage.hpp | 28 +++++++++++++++++++ include/sdl/tst/mockableasyncstorage.hpp | 2 ++ include/sdl/tst/mockablesyncstorage.hpp | 2 ++ rpm/sdl.spec.in | 6 +++- src/asyncdummystorage.cpp | 5 ++++ src/asyncstorageimpl.cpp | 7 +++++ src/redis/asyncredisstorage.cpp | 32 ++++++++++++++++++--- src/syncstorageimpl.cpp | 16 +++++++++++ tst/asyncredisstorage_test.cpp | 43 +++++++++++++++++++++++++++++ tst/syncstorageimpl_test.cpp | 18 ++++++++++++ 20 files changed, 208 insertions(+), 8 deletions(-) diff --git a/Makefile.am b/Makefile.am index ad7d57a..bb92ba9 100644 --- a/Makefile.am +++ b/Makefile.am @@ -5,6 +5,7 @@ BASE_CPPFLAGS = \ -Werror \ -Wextra \ -Wno-missing-field-initializers \ + -Wno-error=deprecated-declarations \ -I$(top_srcdir)/include \ -I$(builddir)/include \ -DSDL_CONF_DIR='"@SDL_CONF_DIR@"' \ diff --git a/configure.ac b/configure.ac index 514c403..97965fe 100644 --- a/configure.ac +++ b/configure.ac @@ -10,7 +10,7 @@ # Change the numbers just before release. m4_define([SDL_MAJOR], [1]) -m4_define([SDL_MINOR], [4]) +m4_define([SDL_MINOR], [5]) m4_define([SDL_MICRO], [0]) # SDL ABI version with libtool @@ -27,8 +27,8 @@ m4_define([SDL_MICRO], [0]) # # Change the numbers just before release. -m4_define([SDL_CURRENT], [4]) -m4_define([SDL_REVISION], [11]) +m4_define([SDL_CURRENT], [5]) +m4_define([SDL_REVISION], [12]) m4_define([SDL_AGE], [0]) AC_INIT([shareddatalayer], [SDL_MAJOR.SDL_MINOR.SDL_MICRO], [], [], [https://gerrit.o-ran-sc.org/r/admin/repos/ric-plt/sdl]) diff --git a/debian/changelog.in b/debian/changelog.in index 16fe8b8..78190df 100644 --- a/debian/changelog.in +++ b/debian/changelog.in @@ -1,3 +1,10 @@ +sdl (1.5.0-1) UNRELEASED; urgency=low + + * New listKeys API to support glob-style search pattern + * Deprecated old findKeys and findKeysAsync API + + -- Petri Ovaska Fri, 17 Sep 2021 11:54:21 +0300 + sdl (1.4.0-1) UNRELEASED; urgency=low * Add synchronous readiness check API diff --git a/docs/release-notes.rst b/docs/release-notes.rst index 0f0d870..6654d05 100644 --- a/docs/release-notes.rst +++ b/docs/release-notes.rst @@ -29,6 +29,11 @@ This document provides the release notes of the sdl library. Version history --------------- +[1.5.0] - 2021-09-17 + +* New listKeys API to support glob-style search pattern +* Deprecated old findKeys and findKeysAsync API + [1.4.0] - 2021-08-11 * Add synchronous readiness check API diff --git a/include/private/asyncdummystorage.hpp b/include/private/asyncdummystorage.hpp index f36cdd1..6c94425 100644 --- a/include/private/asyncdummystorage.hpp +++ b/include/private/asyncdummystorage.hpp @@ -61,6 +61,8 @@ namespace shareddatalayer void findKeysAsync(const Namespace& ns, const std::string& keyPrefix, const FindKeysAck& findKeysAck) override; + void listKeys(const Namespace& ns, const std::string& pattern, const FindKeysAck& findKeysAck) override; + void removeAllAsync(const Namespace& ns, const ModifyAck& modifyAck) override; private: diff --git a/include/private/asyncstorageimpl.hpp b/include/private/asyncstorageimpl.hpp index 006e22e..bc072d9 100644 --- a/include/private/asyncstorageimpl.hpp +++ b/include/private/asyncstorageimpl.hpp @@ -85,6 +85,8 @@ namespace shareddatalayer void findKeysAsync(const Namespace& ns, const std::string& keyPrefix, const FindKeysAck& findKeysAck) override; + void listKeys(const Namespace& ns, const std::string& pattern, const FindKeysAck& findKeysAck) override; + void removeAllAsync(const Namespace& ns, const ModifyAck& modifyAck) override; //public for UT diff --git a/include/private/redis/asyncredisstorage.hpp b/include/private/redis/asyncredisstorage.hpp index f095582..ecd7b1c 100644 --- a/include/private/redis/asyncredisstorage.hpp +++ b/include/private/redis/asyncredisstorage.hpp @@ -105,12 +105,16 @@ namespace shareddatalayer void findKeysAsync(const Namespace& ns, const std::string& keyPrefix, const FindKeysAck& findKeysAck) override; + void listKeys(const Namespace& ns, const std::string& pattern, const FindKeysAck& findKeysAck) override; + void removeAllAsync(const Namespace& ns, const ModifyAck& modifyAck) override; redis::DatabaseInfo& getDatabaseInfo(); std::string buildKeyPrefixSearchPattern(const Namespace& ns, const std::string& keyPrefix) const; + std::string buildNamespaceKeySearchPattern(const Namespace& ns, const std::string& pattern) const; + private: std::shared_ptr engine; std::shared_ptr dispatcher; @@ -132,6 +136,8 @@ namespace shareddatalayer void modificationCommandCallback(const std::error_code& error, const redis::Reply&, const ModifyAck&); void conditionalCommandCallback(const std::error_code& error, const redis::Reply&, const ModifyIfAck&); + + void findKeys(const std::string& ns, const std::string& keyPattern, const FindKeysAck& findKeysAck); }; AsyncRedisStorage::ErrorCode& operator++ (AsyncRedisStorage::ErrorCode& ecEnum); diff --git a/include/private/syncstorageimpl.hpp b/include/private/syncstorageimpl.hpp index b0f5fb5..f67009d 100644 --- a/include/private/syncstorageimpl.hpp +++ b/include/private/syncstorageimpl.hpp @@ -55,6 +55,8 @@ namespace shareddatalayer virtual Keys findKeys(const Namespace& ns, const std::string& keyPrefix) override; + virtual Keys listKeys(const Namespace& ns, const std::string& pattern) override; + virtual void removeAll(const Namespace& ns) override; virtual void setOperationTimeout(const std::chrono::steady_clock::duration& timeout) override; diff --git a/include/private/tst/asyncstoragemock.hpp b/include/private/tst/asyncstoragemock.hpp index f764054..b0c3698 100644 --- a/include/private/tst/asyncstoragemock.hpp +++ b/include/private/tst/asyncstoragemock.hpp @@ -46,6 +46,8 @@ namespace shareddatalayer MOCK_METHOD3(findKeysAsync, void(const Namespace& ns, const std::string& keyPrefix, const FindKeysAck& findKeysAck)); + MOCK_METHOD3(listKeys, void(const Namespace& ns, const std::string& pattern, const FindKeysAck& findKeysAck)); + MOCK_METHOD2(removeAllAsync, void(const Namespace& ns, const ModifyAck& modifyAck)); MOCK_METHOD2(waitReadyAsync, void(const Namespace& ns, const ReadyAck& readyAck)); diff --git a/include/sdl/asyncstorage.hpp b/include/sdl/asyncstorage.hpp index 5c3cc6d..ae1f959 100644 --- a/include/sdl/asyncstorage.hpp +++ b/include/sdl/asyncstorage.hpp @@ -301,10 +301,34 @@ namespace shareddatalayer * @param findKeysAck The acknowledgement to be called once the request has been handled. * The given function is called in the context of handleEvents() function. */ + [[deprecated("Use listKeys() instead.")]] virtual void findKeysAsync(const Namespace& ns, const std::string& keyPrefix, const FindKeysAck& findKeysAck) = 0; + /** + * List all keys matching search glob-style pattern under the namespace. + * + * Supported glob-style patterns: + * h?llo matches hello, hallo and hxllo + * h*llo matches hllo and heeeello + * h[ae]llo matches hello and hallo, but not hillo + * h[^e]llo matches hallo, hbllo, ... but not hello + * h[a-b]llo matches hallo and hbllo + * + * The \ escapes character(s) in key search pattern and those will be treated as a normal + * character(s): + * h\[?llo\* matches h[ello* and h[allo* + * + * @param ns Namespace under which this operation is targeted. + * @param pattern Find keys matching a given glob-style pattern. + * @param findKeysAck The acknowledgement to be called once the request has been handled. + * The given function is called in the context of handleEvents() function. + */ + virtual void listKeys(const Namespace& ns, + const std::string& pattern, + const FindKeysAck& findKeysAck) = 0; + /** * Remove all keys under the namespace. Found keys are removed atomically, i.e. * either all succeeds or all fails. diff --git a/include/sdl/syncstorage.hpp b/include/sdl/syncstorage.hpp index 29f1f9a..80f90d0 100644 --- a/include/sdl/syncstorage.hpp +++ b/include/sdl/syncstorage.hpp @@ -267,9 +267,37 @@ namespace shareddatalayer * @throw OperationInterrupted if shareddatalayer does not receive a reply from the backend data storage. * @throw InvalidNamespace if given namespace does not meet the namespace format restrictions. */ + [[deprecated("Use listKeys() instead.")]] virtual Keys findKeys(const Namespace& ns, const std::string& keyPrefix) = 0; + /** + * List all keys matching search glob-style pattern under the namespace. + * + * Supported glob-style patterns: + * h?llo matches hello, hallo and hxllo + * h*llo matches hllo and heeeello + * h[ae]llo matches hello and hallo, but not hillo + * h[^e]llo matches hallo, hbllo, ... but not hello + * h[a-b]llo matches hallo and hbllo + * + * The \ escapes character(s) in key search pattern and those will be treated as a normal + * character(s): + * h\[?llo\* matches h[ello* and h[allo* + * + * @param ns Namespace under which this operation is targeted. + * @param pattern Find keys matching a given glob-style pattern. + * + * @return Found keys. + * + * @throw BackendError if the backend data storage fails to process the request. + * @throw NotConnected if shareddatalayer is not connected to the backend data storage. + * @throw OperationInterrupted if shareddatalayer does not receive a reply from the backend data storage. + * @throw InvalidNamespace if given namespace does not meet the namespace format restrictions. + */ + virtual Keys listKeys(const Namespace& ns, + const std::string& pattern) = 0; + /** * Remove all keys under the namespace. Found keys are removed atomically, i.e. * either all succeeds or all fails. diff --git a/include/sdl/tst/mockableasyncstorage.hpp b/include/sdl/tst/mockableasyncstorage.hpp index 7c0d0ea..4d587f5 100644 --- a/include/sdl/tst/mockableasyncstorage.hpp +++ b/include/sdl/tst/mockableasyncstorage.hpp @@ -66,6 +66,8 @@ namespace shareddatalayer virtual void findKeysAsync(const Namespace&, const std::string&, const FindKeysAck&) override { logAndAbort(__PRETTY_FUNCTION__); } + virtual void listKeys(const Namespace&, const std::string&, const FindKeysAck&) override { logAndAbort(__PRETTY_FUNCTION__); } + virtual void removeAllAsync(const Namespace&, const ModifyAck&) override { logAndAbort(__PRETTY_FUNCTION__); } private: diff --git a/include/sdl/tst/mockablesyncstorage.hpp b/include/sdl/tst/mockablesyncstorage.hpp index 4bb87af..a291f07 100644 --- a/include/sdl/tst/mockablesyncstorage.hpp +++ b/include/sdl/tst/mockablesyncstorage.hpp @@ -62,6 +62,8 @@ namespace shareddatalayer virtual Keys findKeys(const Namespace&, const std::string&) override { logAndAbort(__PRETTY_FUNCTION__); } + virtual Keys listKeys(const Namespace&, const std::string&) override { logAndAbort(__PRETTY_FUNCTION__); } + virtual void removeAll(const Namespace&) override { logAndAbort(__PRETTY_FUNCTION__); } virtual void setOperationTimeout(const std::chrono::steady_clock::duration&) override { logAndAbort(__PRETTY_FUNCTION__); } diff --git a/rpm/sdl.spec.in b/rpm/sdl.spec.in index c63909b..b9c8378 100755 --- a/rpm/sdl.spec.in +++ b/rpm/sdl.spec.in @@ -1,5 +1,5 @@ Name: sdl -Version: 1.4.0 +Version: 1.5.0 Release: 1%{?dist} Summary: C++ API library for Shared Data Layer clients @@ -50,6 +50,10 @@ rm -f %{buildroot}%{_libdir}/lib*.*a %{_includedir}/sdl %changelog +* Fri Sep 17 2021 Petri Ovaska - 1.5.0-1 +- New listKeys API to support glob-style search pattern +- Deprecated old findKeys and findKeysAsync API + * Thu Aug 11 2021 Timo Tietavainen - 1.4.0-1 - Add synchronous readiness check API diff --git a/src/asyncdummystorage.cpp b/src/asyncdummystorage.cpp index c6ea105..db227cd 100644 --- a/src/asyncdummystorage.cpp +++ b/src/asyncdummystorage.cpp @@ -85,6 +85,11 @@ void AsyncDummyStorage::findKeysAsync(const Namespace&, const std::string&, cons postCallback(std::bind(findKeysAck, std::error_code(), Keys())); } +void AsyncDummyStorage::listKeys(const Namespace&, const std::string&, const FindKeysAck& findKeysAck) +{ + postCallback(std::bind(findKeysAck, std::error_code(), Keys())); +} + void AsyncDummyStorage::removeAllAsync(const Namespace&, const ModifyAck& modifyAck) { postCallback(std::bind(modifyAck, std::error_code())); diff --git a/src/asyncstorageimpl.cpp b/src/asyncstorageimpl.cpp index a018c51..0d6d683 100644 --- a/src/asyncstorageimpl.cpp +++ b/src/asyncstorageimpl.cpp @@ -241,6 +241,13 @@ void AsyncStorageImpl::findKeysAsync(const Namespace& ns, getOperationHandler(ns).findKeysAsync(ns, keyPrefix, findKeysAck); } +void AsyncStorageImpl::listKeys(const Namespace& ns, + const std::string& pattern, + const FindKeysAck& findKeysAck) +{ + getOperationHandler(ns).listKeys(ns, pattern, findKeysAck); +} + void AsyncStorageImpl::removeAllAsync(const Namespace& ns, const ModifyAck& modifyAck) { diff --git a/src/redis/asyncredisstorage.cpp b/src/redis/asyncredisstorage.cpp index 482d6d7..6f74020 100644 --- a/src/redis/asyncredisstorage.cpp +++ b/src/redis/asyncredisstorage.cpp @@ -512,9 +512,9 @@ void AsyncRedisStorage::removeAsync(const Namespace& ns, contentsBuilder->build("DEL", ns, keys)); } -void AsyncRedisStorage::findKeysAsync(const Namespace& ns, - const std::string& keyPrefix, - const FindKeysAck& findKeysAck) +void AsyncRedisStorage::findKeys(const Namespace& ns, + const std::string& keyPattern, + const FindKeysAck& findKeysAck) { //TODO: update to more optimal solution than current KEYS-based one. std::error_code ec; @@ -533,7 +533,23 @@ void AsyncRedisStorage::findKeysAsync(const Namespace& ns, findKeysAck(std::error_code(), getKeys(*reply.getArray())); }, ns, - contentsBuilder->build("KEYS", buildKeyPrefixSearchPattern(ns, keyPrefix))); + contentsBuilder->build("KEYS", keyPattern)); +} + +void AsyncRedisStorage::findKeysAsync(const Namespace& ns, + const std::string& keyPrefix, + const FindKeysAck& findKeysAck) +{ + auto keyPattern(buildKeyPrefixSearchPattern(ns, keyPrefix)); + findKeys(ns, keyPattern, findKeysAck); +} + +void AsyncRedisStorage::listKeys(const Namespace& ns, + const std::string& pattern, + const FindKeysAck& findKeysAck) +{ + auto keyPattern(buildNamespaceKeySearchPattern(ns, pattern)); + findKeys(ns, keyPattern, findKeysAck); } void AsyncRedisStorage::removeAllAsync(const Namespace& ns, @@ -574,3 +590,11 @@ std::string AsyncRedisStorage::buildKeyPrefixSearchPattern(const Namespace& ns, oss << '{' << ns << '}' << SEPARATOR << escapedKeyPrefix << "*"; return oss.str(); } + +std::string AsyncRedisStorage::buildNamespaceKeySearchPattern(const Namespace& ns, + const std::string& pattern) const +{ + std::ostringstream oss; + oss << '{' << ns << '}' << SEPARATOR << pattern; + return oss.str(); +} diff --git a/src/syncstorageimpl.cpp b/src/syncstorageimpl.cpp index 3eaeb03..19f0e54 100644 --- a/src/syncstorageimpl.cpp +++ b/src/syncstorageimpl.cpp @@ -283,6 +283,22 @@ SyncStorageImpl::Keys SyncStorageImpl::findKeys(const Namespace& ns, const std:: return localKeys; } +SyncStorageImpl::Keys SyncStorageImpl::listKeys(const Namespace& ns, const std::string& pattern) +{ + handlePendingEvents(); + waitSdlToBeReady(ns); + synced = false; + asyncStorage->listKeys(ns, + pattern, + std::bind(&shareddatalayer::SyncStorageImpl::findKeysAck, + this, + std::placeholders::_1, + std::placeholders::_2)); + waitForOperationCallback(); + verifyBackendResponse(); + return localKeys; +} + void SyncStorageImpl::removeAll(const Namespace& ns) { handlePendingEvents(); diff --git a/tst/asyncredisstorage_test.cpp b/tst/asyncredisstorage_test.cpp index f526b54..f5277a5 100644 --- a/tst/asyncredisstorage_test.cpp +++ b/tst/asyncredisstorage_test.cpp @@ -775,6 +775,29 @@ TEST_F(AsyncRedisStorageTest, FindKeysAsyncSuccessfullyAndErrorIsTranslated) savedCommandCb(getWellKnownErrorCode(), replyMock); } +TEST_F(AsyncRedisStorageTest, ListKeysPatternSuccessfullyAndErrorIsTranslated) +{ + InSequence dummy; + expectContentsBuild("KEYS", "{tag1},key[12]"); + expectDispatchAsync(); + sdlStorage->listKeys(ns, + "key[12]", + std::bind(&AsyncRedisStorageTest::findKeysAck, + this, + std::placeholders::_1, + std::placeholders::_2)); + expectGetArray(); + auto expectedDataItem1(Reply::DataItem { key1, ReplyStringLength(key1.size()) }); + auto expectedDataItem2(Reply::DataItem { key2, ReplyStringLength(key2.size()) }); + expectGetDataString(expectedDataItem1); + expectGetType(Reply::Type::NIL); + expectGetDataString(expectedDataItem2); + expectFindKeysAck(std::error_code(), { key1, key2 }); + savedCommandCb(std::error_code(), replyMock); + expectFindKeysAck(getWellKnownErrorCode(), { }); + savedCommandCb(getWellKnownErrorCode(), replyMock); +} + TEST_F(AsyncRedisStorageTest, RemoveAllAsyncSuccessfully) { InSequence dummy; @@ -941,6 +964,26 @@ TEST_F(AsyncRedisStorageTest, RedisSearchPatternCharactersAreCorrectlyEscapedInK ASSERT_STREQ(expectedKeyPrefixSearchPattern.c_str(), builtKeyPrefixSearchPattern.c_str()); } +TEST_F(AsyncRedisStorageTest, BuildNamespaceKeySearchPatternIsCorrect) +{ + InSequence dummy; + const std::string nsPrefix = '{' + ns + '}' + AsyncStorage::SEPARATOR; + std::string buildPattern; + std::string expectedPattern; + + expectedPattern = nsPrefix; + buildPattern = sdlStorage->buildNamespaceKeySearchPattern(ns, ""); + ASSERT_STREQ(expectedPattern.c_str(), buildPattern.c_str()); + + expectedPattern = nsPrefix + '*'; + buildPattern = sdlStorage->buildNamespaceKeySearchPattern(ns, "*"); + ASSERT_STREQ(expectedPattern.c_str(), buildPattern.c_str()); + + expectedPattern = nsPrefix + "h?llo"; + buildPattern = sdlStorage->buildNamespaceKeySearchPattern(ns, "h?llo"); + ASSERT_STREQ(expectedPattern.c_str(), buildPattern.c_str()); +} + TEST_F(AsyncRedisStorageTestDispatcherNotCreated, ReadyAckNotForwardedIfDispatcherNotYetCreated) { InSequence dummy; diff --git a/tst/syncstorageimpl_test.cpp b/tst/syncstorageimpl_test.cpp index 570b61a..3dd9dd4 100644 --- a/tst/syncstorageimpl_test.cpp +++ b/tst/syncstorageimpl_test.cpp @@ -276,6 +276,13 @@ namespace .WillOnce(SaveArg<2>(&savedFindKeysAck)); } + void expectListKeys() + { + EXPECT_CALL(*asyncStorageMockRawPtr, listKeys(ns, _, _)) + .Times(1) + .WillOnce(SaveArg<2>(&savedFindKeysAck)); + } + void expectRemoveAsync(const SyncStorage::Keys& keys) { EXPECT_CALL(*asyncStorageMockRawPtr, removeAsync(ns, keys, _)) @@ -600,6 +607,17 @@ TEST_F(SyncStorageImplTest, FindKeysSuccessfully) EXPECT_EQ(ids, keys); } +TEST_F(SyncStorageImplTest, ListKeysSuccessfully) +{ + InSequence dummy; + expectSdlReadinessCheck(SyncStorageImpl::NO_TIMEOUT); + expectListKeys(); + expectPollWait(SyncStorageImpl::NO_TIMEOUT); + expectFindKeysAck(); + auto ids(syncStorage->listKeys(ns, "*")); + EXPECT_EQ(ids, keys); +} + TEST_F(SyncStorageImplTest, FindKeysWithReadinessTimeoutSuccessfully) { InSequence dummy; -- 2.16.6