1 /*******************************************************************************
5 *******************************************************************************/
7 /* This is the new utility file for all tests, all new common functionality has to go here.
8 When contributing to the common.hpp please focus on readability and maintainability rather than
10 #ifndef XRANLIB_COMMON_HPP
11 #define XRANLIB_COMMON_HPP
13 /* Disable warnings generated by JSON parser */
14 #pragma warning(disable : 191)
15 #pragma warning(disable : 186)
16 #pragma warning(disable : 192)
24 #include <immintrin.h>
30 #include <rte_config.h>
31 #include <rte_malloc.h>
34 #include "gtest/gtest.h"
36 #include "common_typedef_xran.h"
40 using json = nlohmann::json;
42 #define ASSERT_ARRAY_NEAR(reference, actual, size, precision) \
43 assert_array_near(reference, actual, size, precision)
45 #define ASSERT_ARRAY_EQ(reference, actual, size) \
46 assert_array_eq(reference, actual, size)
48 #define ASSERT_AVG_GREATER_COMPLEX(reference, actual, size, precision) \
49 assert_avg_greater_complex(reference, actual, size, precision)
51 struct BenchmarkParameters
53 static long repetition;
55 static unsigned cpu_id;
58 struct missing_config_file_exception : public std::exception
60 const char * what () const throw () override {
61 return "JSON file cannot be opened!";
65 struct reading_input_file_exception : public std::exception
67 const char * what () const throw () override {
68 return "Input file cannot be read!";
73 \brief Attach current process to the selected core.
74 \param [in] cpu Core number.
75 \return 0 on success, -1 otherwise.
77 int bind_to_cpu(const unsigned cpu);
80 \brief Calculate the mean and variance from the result of the run_benchmark.
81 \param [in] values Vector with result values.
82 \return std::pair where the first element is mean and the second one is standard deviation.
83 \note It's not a general mean/stddev function it only works properly when feed with data from
84 the benchmark function.
86 std::pair<double, double> calculate_statistics(const std::vector<long> values);
89 \brief For a given number return sequence of number from 0 to number - 1.
90 \param [in] number Positive integer value.
91 \return Vector with the sorted integer numbers between 0 and number - 1.
93 std::vector<unsigned> get_sequence(const unsigned number);
96 \brief Read JSON from the given file.
97 \param [in] filename name of the .json file.
98 \return JSON object with data.
99 \throws missing_config_file_exception when file cannot be opened.
101 json read_json_from_file(const std::string &filename);
104 \brief Read binary data from the file.
105 \param [in] filename name of the binary file.
106 \return Pointer to the allocated memory with data from the file.
107 \throws std::runtime_error when memory cannot be allocated.
109 char* read_data_to_aligned_array(const std::string &filename);
112 \brief Measure the TSC on the machine
113 \return Number of ticks per us
115 unsigned long tsc_recovery();
118 \brief Return the current value of the TSC
119 \return Current TSC value
121 unsigned long tsc_tick();
126 Each test class has to inherit from KernelTests class as it provides GTest support and does a lot
127 of setup (including JSON) an provides useful methods to operate on loaded JSON file.
128 Unfortunately GTest is limited in the way that all TEST_P within the class are called for all
129 cases/parameters, but we usually want two different data sets for functional and performance
130 tests (or maybe other types of tests). Because of that to use different data sets we need to
131 create separate classes, hence performance and functional test are in separate classes. it adds
132 an extra overhead, but adds much more flexibility. init_test(...) is used to select data set from
135 Important note on the JSON file structure. Top JSON object can have as many section (JSON
136 objects) as needed, but each have to have a distinct name that is used by init_test. Then
137 each section must contain an array of objects (test cases) where each object has a name,
138 parameters and references. Everything inside parameters and references can be completely custom
139 as it's loaded by get_input/reference_parameter function. JSON values can be either literal
140 values, e.g. 1, 0.001, 5e-05, etc. or filename. Depends on the get type test framework can either
141 read the value or load data from the file - and it happens automatically (*pff* MAGIC!).
143 class KernelTests : public testing::TestWithParam<unsigned>
147 static std::string test_type;
149 static void SetUpTestCase()
155 conf = read_json_from_file("conf.json");
157 catch(missing_config_file_exception &e)
159 std::cout << "[----------] SetUpTestCase failed: " << e.what() << std::endl;
163 tsc = tsc_recovery();
167 std::cout << "[----------] SetUpTestCase failed: TSC recovery failed" << std::endl;
172 static void TearDownTestCase()
174 /* Free resources - nothing to free at the moment */
177 static unsigned get_number_of_cases(const std::string &type)
181 json json_data = read_json_from_file("conf.json");
183 return json_data[type].size();
185 catch(missing_config_file_exception &e)
187 std::cout << "[----------] get_number_of_cases failed: " << e.what() << std::endl;
191 catch(std::domain_error &e)
193 std::cout << "[----------] get_number_of_cases failed: " << e.what() << std::endl;
194 std::cout << "[----------] Use a default value: 0" << std::endl;
201 double division_factor = 1.0;
202 std::string result_units = "None";
203 int parallelization_factor = 1;
206 \brief Set division factor
207 \param [in] factor Division factor that divides mean and standard deviation.
209 void set_division_factor(const double factor)
211 division_factor = factor;
215 \brief Set reults units
216 \param [in] units Units that are displayed in the report.
218 void set_results_units(const std::string &units)
220 result_units = units;
224 \brief Set size of processed data
225 \param [in] size Size of processed data used to calculate module throughput.
227 void set_parallelization_factor(const int factor)
229 parallelization_factor = factor;
233 \brief Run the given function and return the mean run time and stddev.
234 \param [in] function Function to benchmark.
235 \param [in] args Function's arguments.
236 \return std::pair where the first element is mean and the second one is standard deviation.
238 template <typename F, typename ... Args>
239 std::pair<double, double> run_benchmark(F function, Args ... args)
241 std::vector<long> results((unsigned long) BenchmarkParameters::repetition);
243 for(unsigned int outer_loop = 0; outer_loop < BenchmarkParameters::repetition; outer_loop++) {
244 const auto start_time = __rdtsc();
245 for (unsigned int inner_loop = 0; inner_loop < BenchmarkParameters::loop; inner_loop++) {
248 const auto end_time = __rdtsc();
249 results.push_back(end_time - start_time);
252 return calculate_statistics(results);
256 \brief Run performance test case for a given function.
257 \param [in] isa Used Instruction Set.
258 \param [in] module_name name of the tested kernel.
259 \param [in] function function to be tested.
260 \param [in] args function's arguments.
262 template <typename F, typename ... Args>
263 void performance(const std::string &isa, const std::string &module_name, F function,
265 ASSERT_EQ(0, bind_to_cpu(BenchmarkParameters::cpu_id)) << "Failed to bind to cpu!";
267 const auto result = run_benchmark(function, args ...);
268 const auto scaled_mean = result.first / division_factor / tsc;
269 const auto scaled_stddev = result.second / division_factor / tsc;
271 print_and_store_results(isa, get_case_name(), module_name, get_case_name(), result_units,
272 parallelization_factor, scaled_mean, scaled_stddev);
276 \brief Print unique test description to the results xml file
277 \param [in] isa Used Instruction Set.
278 \param [in] module_name name of the tested kernel.
279 \param [in] function function to be tested.
281 void print_test_description(const std::string &isa, const std::string &module_name) {
282 print_and_store_results(isa, get_case_name(), module_name, get_case_name(), result_units,
283 parallelization_factor, 0, 0);
288 \brief Load selected data from a JSON object.
289 get_input_parameter loads data from parameters section of the test case in JSON file and
290 get_reference_parameter does the same thing for references section.
292 Get parameter function uses template type to figure out how to load parameters. If type
293 is NOT a pointer it'll load value directly from the JSON. Otherwise path to the test
294 vector is expected and function will allocate memory, load data from the binary file to
295 this memory location and return pointer to it. For example in here we request to load
296 pointer to float so llrs filed is expected to be
297 a path to the binary file.
299 template <typename T>
300 T get_input_parameter(const std::string ¶meter_name)
304 return get_parameter<T>("parameters", parameter_name);
306 catch (std::domain_error &e)
308 std::cout << "[----------] get_input_parameter (" << parameter_name
309 << ") failed: " << e.what()
310 << ". Did you mispell the parameter name?" << std::endl;
313 catch(reading_input_file_exception &e)
315 std::cout << "[----------] get_input_parameter (" << parameter_name
316 << ") failed: " << e.what() << std::endl;
321 template <typename T>
322 T get_input_parameter(const std::string &subsection_name, const std::string ¶meter_name)
326 return get_parameter<T>("parameters", subsection_name, parameter_name);
328 catch (std::domain_error &e)
330 std::cout << "[----------] get_input_parameter (" << subsection_name << "." << parameter_name
331 << ") failed: " << e.what()
332 << ". Did you mispell the parameter name?" << std::endl;
335 catch(reading_input_file_exception &e)
337 std::cout << "[----------] get_input_parameter (" << subsection_name << "." << parameter_name
338 << ") failed: " << e.what() << std::endl;
343 template <typename T>
344 T get_input_parameter(const std::string &subsection_name, const int index, const std::string ¶meter_name)
348 return get_parameter<T>("parameters", subsection_name, index, parameter_name);
350 catch (std::domain_error &e)
352 std::cout << "[----------] get_input_parameter (" << subsection_name << "[" << index << "]." << parameter_name
353 << ") failed: " << e.what()
354 << ". Did you mispell the parameter name?" << std::endl;
357 catch(reading_input_file_exception &e)
359 std::cout << "[----------] get_input_parameter (" << subsection_name << "[" << index << "]." << parameter_name
360 << ") failed: " << e.what() << std::endl;
364 int get_input_parameter_size(const std::string &subsection_name, const std::string ¶meter_name)
368 auto array_size = conf[test_type][GetParam()]["parameters"][subsection_name][parameter_name].size();
371 catch (std::domain_error &e)
373 std::cout << "[----------] get_input_parameter_size (" << subsection_name << "." << parameter_name
374 << ") failed: " << e.what()
375 << ". Did you mispell the parameter name?" << std::endl;
378 catch(reading_input_file_exception &e)
380 std::cout << "[----------] get_input_parameter_size (" << subsection_name << "." << parameter_name
381 << ") failed: " << e.what() << std::endl;
385 int get_input_subsection_size(const std::string &subsection_name)
389 auto array_size = conf[test_type][GetParam()]["parameters"][subsection_name].size();
392 catch (std::domain_error &e)
394 std::cout << "[----------] get_input_subsection_size (" << subsection_name
395 << ") failed: " << e.what()
396 << ". Did you mispell the subsection name?" << std::endl;
399 catch(reading_input_file_exception &e)
401 std::cout << "[----------] get_input_subsection_size (" << subsection_name
402 << ") failed: " << e.what() << std::endl;
407 template <typename T>
408 T get_reference_parameter(const std::string ¶meter_name)
412 return get_parameter<T>("references", parameter_name);
414 catch (std::domain_error &e)
416 std::cout << "[----------] get_reference_parameter (" << parameter_name
417 << ") failed: " << e.what()
418 << ". Did you mispell the parameter name?" << std::endl;
421 catch(reading_input_file_exception &e)
423 std::cout << "[----------] get_reference_parameter (" << parameter_name
424 << ") failed: " << e.what() << std::endl;
431 \brief Get name of the test case from JSON file.
432 \return Test'ss case name or a default name if name field is missing.
434 const std::string get_case_name()
438 return conf[test_type][GetParam()]["name"];
440 catch (std::domain_error &e)
442 std::cout << "[----------] get_case_name failed: " << e.what()
443 << ". Did you specify a test name in JSON?" << std::endl;
444 std::cout << "[----------] Using a default name instead" << std::endl;
446 return "Default test name";
451 \brief Defines section in the conf.json that is used to load parameters from.
452 \param [in] type Name of the section in the JSON file.
454 void init_test(const std::string &type)
457 const std::string name = get_case_name();
458 std::cout << "[----------] Test case: " << name << std::endl;
462 static unsigned long tsc;
466 static T read_parameter(const int index, const std::string &type,
467 const std::string ¶meter_name)
469 return conf[test_type][index][type][parameter_name];
474 struct data_reader<std::vector<T>> {
475 static std::vector<T> read_parameter(const int index, const std::string &type,
476 const std::string ¶meter_name)
478 auto array_size = conf[test_type][index][type][parameter_name].size();
480 std::vector<T> result(array_size);
482 for(unsigned number = 0; number < array_size; number++)
483 result.at(number) = conf[test_type][index][type][parameter_name][number];
490 struct data_reader<T*> {
491 static T* read_parameter(const int index, const std::string &type,
492 const std::string ¶meter_name)
494 return (T*) read_data_to_aligned_array(conf[test_type][index][type][parameter_name]);
498 template <typename T>
499 T get_parameter(const std::string &type, const std::string ¶meter_name)
501 return data_reader<T>::read_parameter(GetParam(), type, parameter_name);
505 struct data_reader2 {
506 static T read_parameter(const int index, const std::string &type,
507 const std::string &subsection_name,
508 const std::string ¶meter_name)
510 return conf[test_type][index][type][subsection_name][parameter_name];
515 struct data_reader2<std::vector<T>> {
516 static std::vector<T> read_parameter(const int index, const std::string &type,
517 const std::string &subsection_name,
518 const std::string ¶meter_name)
520 auto array_size = conf[test_type][index][type][subsection_name][parameter_name].size();
522 std::vector<T> result(array_size);
524 for(unsigned number = 0; number < array_size; number++)
525 result.at(number) = conf[test_type][index][type][subsection_name][parameter_name][number];
532 struct data_reader2<T*> {
533 static T* read_parameter(const int index, const std::string &type,
534 const std::string &subsection_name,
535 const std::string ¶meter_name)
537 return (T*) read_data_to_aligned_array(conf[test_type][index][type][subsection_name][parameter_name]);
540 template <typename T>
541 T get_parameter(const std::string &type, const std::string &subsection_name, const std::string ¶meter_name)
543 return data_reader2<T>::read_parameter(GetParam(), type, subsection_name, parameter_name);
547 struct data_reader3 {
548 static T read_parameter(const int index, const std::string &type,
549 const std::string &subsection_name,
551 const std::string ¶meter_name)
553 return conf[test_type][index][type][subsection_name][subindex][parameter_name];
558 struct data_reader3<std::vector<T>> {
559 static std::vector<T> read_parameter(const int index, const std::string &type,
560 const std::string &subsection_name,
562 const std::string ¶meter_name)
564 auto array_size = conf[test_type][index][type][subsection_name][subindex][parameter_name].size();
566 std::vector<T> result(array_size);
568 for(unsigned number = 0; number < array_size; number++)
569 result.at(number) = conf[test_type][index][type][subsection_name][subindex][parameter_name][number];
576 struct data_reader3<T*> {
577 static T* read_parameter(const int index, const std::string &type,
578 const std::string &subsection_name,
580 const std::string ¶meter_name)
582 return (T*) read_data_to_aligned_array(conf[test_type][index][type][subsection_name][subindex][parameter_name]);
585 template <typename T>
586 T get_parameter(const std::string &type, const std::string &subsection_name, const int subindex, const std::string ¶meter_name)
588 return data_reader3<T>::read_parameter(GetParam(), type, subsection_name, subindex, parameter_name);
591 void print_and_store_results(const std::string &isa,
592 const std::string ¶meters,
593 const std::string &module_name,
594 const std::string &test_name,
595 const std::string &unit,
596 const int para_factor,
598 const double stddev);
602 \brief Assert elements of two arrays. It calls ASSERT_EQ for each element of the array.
603 \param [in] reference Array with reference values.
604 \param [in] actual Array with the actual output.
605 \param [in] size Size of the array.
607 template <typename T>
608 inline void assert_array_eq(const T* reference, const T* actual, const int size)
610 for(int index = 0; index < size ; index++)
612 ASSERT_EQ(reference[index], actual[index])
613 <<"The wrong number is index: "<< index;
618 \brief Assert elements of two arrays. It calls ASSERT_NEAR for each element of the array.
619 \param [in] reference Array with reference values.
620 \param [in] actual Array with the actual output.
621 \param [in] size Size of the array.
622 \param [in] precision Precision fo the comparision used by ASSERT_NEAR.
624 template <typename T>
625 inline void assert_array_near(const T* reference, const T* actual, const int size, const double precision)
627 for(int index = 0; index < size ; index++)
629 ASSERT_NEAR(reference[index], actual[index], precision)
630 <<"The wrong number is index: "<< index;
635 inline void assert_array_near<complex_float>(const complex_float* reference, const complex_float* actual, const int size, const double precision)
637 for(int index = 0; index < size ; index++)
639 ASSERT_NEAR(reference[index].re, actual[index].re, precision)
640 <<"The wrong number is RE, index: "<< index;
641 ASSERT_NEAR(reference[index].im, actual[index].im, precision)
642 <<"The wrong number is IM, index: "<< index;
647 \brief Assert average diff of two arrays. It calls ASSERT_GT to check the average.
648 \param [in] reference Array with reference values, interleaved IQ inputs.
649 \param [in] actual Array with the actual output, interleaved IQ inputs.
650 \param [in] size Size of the array, based on complex inputs.
651 \param [in] precision Precision for the comparison used by ASSERT_GT.
654 inline void assert_avg_greater_complex(const T* reference, const T* actual, const int size, const double precision)
657 double avgMSEDB = 0.0;
658 for (int index = 0; index < size; index++) {
659 T refReal = reference[2*index];
660 T refImag = reference[(2*index)+1];
661 T resReal = actual[2*index];
662 T resImag = actual[(2*index)+1];
664 T errReal = resReal - refReal;
665 T errIm = resImag - refImag;
667 /* For some unit tests, e.g. PUCCH deomdulation, the expected output is 0. To avoid a
668 divide by zero error, check the reference results to determine if the expected result
669 is 0 and, if so, add a 1 to the division. */
670 if (refReal == 0 && refImag == 0)
671 MSE = (float)(errReal*errReal + errIm*errIm)/(float)(refReal*refReal + refImag*refImag + 1);
673 MSE = (float)(errReal*errReal + errIm*errIm)/(float)(refReal*refReal + refImag*refImag);
676 mseDB = (float)(-100.0);
678 mseDB = (float)(10.0) * (float)log10(MSE);
680 avgMSEDB += (double)mseDB;
685 ASSERT_GT(precision, avgMSEDB);
689 \brief Allocates memory of the given size.
691 aligned_malloc is wrapper to functions that allocate memory:
692 'rte_malloc' from DPDK if hugepages are defined, 'memalign' otherwise.
693 Size is defined as a number of variables of given type e.g. floats, rather than bytes.
694 It hides sizeof(T) multiplication and cast hence makes things cleaner.
696 \param [in] size Size of the memory to allocate.
697 \param [in] alignment Bytes alignment of the allocated memory. If 0, the return is a pointer
698 that is suitably aligned for any kind of variable (in the same manner as malloc()).
699 Otherwise, the return is a pointer that is a multiple of align. In this case,
700 it must be a power of two. (Minimum alignment is the cacheline size, i.e. 64-bytes)
701 \return Pointer to the allocated memory.
703 template <typename T>
704 inline T* aligned_malloc(const int size, const unsigned alignment)
707 return (T*) rte_malloc(NULL, sizeof(T) * size, alignment);
710 return (T*) memalign(alignment, sizeof(T) * size);
712 return (T*)_aligned_malloc(sizeof(T)*size, alignment);
718 \brief Frees memory pointed by the given pointer.
720 aligned_free is a wrapper for functions that free memory allocated by
721 aligned_malloc: 'rte_free' from DPDK if hugepages are defined and 'free' otherwise.
723 \param [in] ptr Pointer to the allocated memory.
725 template <typename T>
726 inline void aligned_free(T* ptr)
729 rte_free((void*)ptr);
735 _aligned_free((void *)ptr);
741 \brief generate random numbers.
743 It allocates memory and populate it with random numbers using C++11 default engine and
744 uniform real / int distribution (where lo_range <= x <up_range). Don't forget to free
747 \param [in] size Size of the memory to be filled with random data.
748 \param [in] alignment Bytes alignment of the memory.
749 \param [in] distribution Distribuiton for random generator.
750 \return Pointer to the allocated memory with random data.
752 template <typename T, typename U>
753 inline T* generate_random_numbers(const long size, const unsigned alignment, U& distribution)
755 auto array = (T*) aligned_malloc<char>(size * sizeof(T), alignment);
757 std::random_device random_device;
758 std::default_random_engine generator(random_device());
760 for(long i = 0; i < size; i++)
761 array[i] = (T)distribution(generator);
767 \brief generate random data.
769 It allocates memory and populate it with random data using C++11 default engine and
770 uniform integer distribution (bytes not floats are uniformly distributed). Don't forget
771 to free allocated memory!
773 \param [in] size Size of the memory to be filled with random data.
774 \param [in] alignment Bytes alignment of the memory.
775 \return Pointer to the allocated memory with random data.
777 template <typename T>
778 inline T* generate_random_data(const long size, const unsigned alignment)
780 std::uniform_int_distribution<> random(0, 255);
782 return (T*)generate_random_numbers<char, std::uniform_int_distribution<>>(size * sizeof(T), alignment, random);
786 \brief generate integer random numbers.
788 It allocates memory and populate it with random numbers using C++11 default engine and
789 uniform integer distribution (where lo_range <= x < up_range). Don't forget
790 to free allocated memory! The result type generated by the generator should be one of
793 \param [in] size Size of the memory to be filled with random data.
794 \param [in] alignment Bytes alignment of the memory.
795 \param [in] lo_range Lower bound of range of values returned by random generator.
796 \param [in] up_range Upper bound of range of values returned by random generator.
797 \return Pointer to the allocated memory with random data.
799 template <typename T>
800 inline T* generate_random_int_numbers(const long size, const unsigned alignment, const T lo_range,
803 std::uniform_int_distribution<T> random(lo_range, up_range);
805 return generate_random_numbers<T, std::uniform_int_distribution<T>>(size, alignment, random);
809 \brief generate real random numbers.
811 It allocates memory and populate it with random numbers using C++11 default engine and
812 uniform real distribution (where lo_range <= x <up_range). Don't forget to free
813 allocated memory! The result type generated by the generator should be one of
814 real types: float, double or long double.
816 \param [in] size Size of the memory to be filled with random data.
817 \param [in] alignment Bytes alignment of the memory.
818 \param [in] lo_range Lower bound of range of values returned by random generator.
819 \param [in] up_range Upper bound of range of values returned by random generator.
820 \return Pointer to the allocated memory with random data.
822 template <typename T>
823 inline T* generate_random_real_numbers(const long size, const unsigned alignment, const T lo_range,
826 std::uniform_real_distribution<T> distribution(lo_range, up_range);
828 return generate_random_numbers<T, std::uniform_real_distribution<T>>(size, alignment, distribution);
831 #endif //XRANLIB_COMMON_HPP