1 /*******************************************************************************
5 *******************************************************************************/
7 /* This is the new utility file for all tests, all new common functionality has to go here.
8 When contributing to the common.hpp please focus on readability and maintainability rather than
10 #ifndef XRANLIB_COMMON_HPP
11 #define XRANLIB_COMMON_HPP
13 /* Disable warnings generated by JSON parser */
14 #pragma warning(disable : 191)
15 #pragma warning(disable : 186)
16 #pragma warning(disable : 192)
24 #include <immintrin.h>
30 #include <rte_config.h>
31 #include <rte_malloc.h>
34 #include "gtest/gtest.h"
36 #include "common_typedef_xran.h"
40 using json = nlohmann::json;
42 #define ASSERT_ARRAY_NEAR(reference, actual, size, precision) \
43 assert_array_near(reference, actual, size, precision)
45 #define ASSERT_ARRAY_EQ(reference, actual, size) \
46 assert_array_eq(reference, actual, size)
48 #define ASSERT_AVG_GREATER_COMPLEX(reference, actual, size, precision) \
49 assert_avg_greater_complex(reference, actual, size, precision)
51 struct BenchmarkParameters
53 static long repetition;
55 static unsigned cpu_id;
58 struct missing_config_file_exception : public std::exception
60 const char * what () const throw () override {
61 return "JSON file cannot be opened!";
65 struct reading_input_file_exception : public std::exception
67 const char * what () const throw () override {
68 return "Input file cannot be read!";
73 \brief Attach current process to the selected core.
74 \param [in] cpu Core number.
75 \return 0 on success, -1 otherwise.
77 int bind_to_cpu(const unsigned cpu);
80 \brief Calculate the mean and variance from the result of the run_benchmark.
81 \param [in] values Vector with result values.
82 \return std::pair where the first element is mean and the second one is standard deviation.
83 \note It's not a general mean/stddev function it only works properly when feed with data from
84 the benchmark function.
86 std::pair<double, double> calculate_statistics(const std::vector<long> values);
89 \brief For a given number return sequence of number from 0 to number - 1.
90 \param [in] number Positive integer value.
91 \return Vector with the sorted integer numbers between 0 and number - 1.
93 std::vector<unsigned> get_sequence(const unsigned number);
96 \brief Read JSON from the given file.
97 \param [in] filename name of the .json file.
98 \return JSON object with data.
99 \throws missing_config_file_exception when file cannot be opened.
101 json read_json_from_file(const std::string &filename);
104 \brief Read binary data from the file.
105 \param [in] filename name of the binary file.
106 \return Pointer to the allocated memory with data from the file.
107 \throws std::runtime_error when memory cannot be allocated.
109 char* read_data_to_aligned_array(const std::string &filename);
112 \brief Measure the TSC on the machine
113 \return Number of ticks per us
115 unsigned long tsc_recovery();
118 \brief Return the current value of the TSC
119 \return Current TSC value
121 unsigned long tsc_tick();
126 Each test class has to inherit from KernelTests class as it provides GTest support and does a lot
127 of setup (including JSON) an provides useful methods to operate on loaded JSON file.
128 Unfortunately GTest is limited in the way that all TEST_P within the class are called for all
129 cases/parameters, but we usually want two different data sets for functional and performance
130 tests (or maybe other types of tests). Because of that to use different data sets we need to
131 create separate classes, hence performance and functional test are in separate classes. it adds
132 an extra overhead, but adds much more flexibility. init_test(...) is used to select data set from
135 Important note on the JSON file structure. Top JSON object can have as many section (JSON
136 objects) as needed, but each have to have a distinct name that is used by init_test. Then
137 each section must contain an array of objects (test cases) where each object has a name,
138 parameters and references. Everything inside parameters and references can be completely custom
139 as it's loaded by get_input/reference_parameter function. JSON values can be either literal
140 values, e.g. 1, 0.001, 5e-05, etc. or filename. Depends on the get type test framework can either
141 read the value or load data from the file - and it happens automatically (*pff* MAGIC!).
143 class KernelTests : public testing::TestWithParam<unsigned>
147 static std::string test_type;
149 static void SetUpTestCase()
155 conf = read_json_from_file("conf.json");
157 catch(missing_config_file_exception &e)
159 std::cout << "[----------] SetUpTestCase failed: " << e.what() << std::endl;
163 tsc = tsc_recovery();
167 std::cout << "[----------] SetUpTestCase failed: TSC recovery failed" << std::endl;
172 static void TearDownTestCase()
174 /* Free resources - nothing to free at the moment */
177 static unsigned get_number_of_cases(const std::string &type)
181 json json_data = read_json_from_file("conf.json");
183 return json_data[type].size();
185 catch(missing_config_file_exception &e)
187 std::cout << "[----------] get_number_of_cases failed: " << e.what() << std::endl;
191 catch(std::domain_error &e)
193 std::cout << "[----------] get_number_of_cases failed: " << e.what() << std::endl;
194 std::cout << "[----------] Use a default value: 0" << std::endl;
201 double division_factor = 1.0;
202 std::string result_units = "None";
203 int parallelization_factor = 1;
206 \brief Set division factor
207 \param [in] factor Division factor that divides mean and standard deviation.
209 void set_division_factor(const double factor)
211 division_factor = factor;
215 \brief Set reults units
216 \param [in] units Units that are displayed in the report.
218 void set_results_units(const std::string &units)
220 result_units = units;
224 \brief Set size of processed data
225 \param [in] size Size of processed data used to calculate module throughput.
227 void set_parallelization_factor(const int factor)
229 parallelization_factor = factor;
233 \brief Run performance test case for a given function.
234 \param [in] isa Used Instruction Set.
235 \param [in] module_name name of the tested kernel.
236 \param [in] function function to be tested.
237 \param [in] args function's arguments.
239 template <typename F, typename ... Args>
240 void performance(const std::string &isa, const std::string &module_name, F function,
242 ASSERT_EQ(0, bind_to_cpu(BenchmarkParameters::cpu_id)) << "Failed to bind to cpu!";
244 const auto result = run_benchmark(function, args ...);
245 const auto scaled_mean = result.first / division_factor / tsc;
246 const auto scaled_stddev = result.second / division_factor / tsc;
248 print_and_store_results(isa, get_case_name(), module_name, get_case_name(), result_units,
249 parallelization_factor, scaled_mean, scaled_stddev);
253 \brief Print unique test description to the results xml file
254 \param [in] isa Used Instruction Set.
255 \param [in] module_name name of the tested kernel.
256 \param [in] function function to be tested.
258 void print_test_description(const std::string &isa, const std::string &module_name) {
259 print_and_store_results(isa, get_case_name(), module_name, get_case_name(), result_units,
260 parallelization_factor, 0, 0);
265 \brief Load selected data from a JSON object.
266 get_input_parameter loads data from parameters section of the test case in JSON file and
267 get_reference_parameter does the same thing for references section.
269 Get parameter function uses template type to figure out how to load parameters. If type
270 is NOT a pointer it'll load value directly from the JSON. Otherwise path to the test
271 vector is expected and function will allocate memory, load data from the binary file to
272 this memory location and return pointer to it. For example in here we request to load
273 pointer to float so llrs filed is expected to be
274 a path to the binary file.
276 template <typename T>
277 T get_input_parameter(const std::string ¶meter_name)
281 return get_parameter<T>("parameters", parameter_name);
283 catch (std::domain_error &e)
285 std::cout << "[----------] get_input_parameter (" << parameter_name
286 << ") failed: " << e.what()
287 << ". Did you mispell the parameter name?" << std::endl;
290 catch(reading_input_file_exception &e)
292 std::cout << "[----------] get_input_parameter (" << parameter_name
293 << ") failed: " << e.what() << std::endl;
298 template <typename T>
299 T get_input_parameter(const std::string &subsection_name, const std::string ¶meter_name)
303 return get_parameter<T>("parameters", subsection_name, parameter_name);
305 catch (std::domain_error &e)
307 std::cout << "[----------] get_input_parameter (" << subsection_name << "." << parameter_name
308 << ") failed: " << e.what()
309 << ". Did you mispell the parameter name?" << std::endl;
312 catch(reading_input_file_exception &e)
314 std::cout << "[----------] get_input_parameter (" << subsection_name << "." << parameter_name
315 << ") failed: " << e.what() << std::endl;
320 template <typename T>
321 T get_input_parameter(const std::string &subsection_name, const int index, const std::string ¶meter_name)
325 return get_parameter<T>("parameters", subsection_name, index, parameter_name);
327 catch (std::domain_error &e)
329 std::cout << "[----------] get_input_parameter (" << subsection_name << "[" << index << "]." << parameter_name
330 << ") failed: " << e.what()
331 << ". Did you mispell the parameter name?" << std::endl;
334 catch(reading_input_file_exception &e)
336 std::cout << "[----------] get_input_parameter (" << subsection_name << "[" << index << "]." << parameter_name
337 << ") failed: " << e.what() << std::endl;
341 int get_input_parameter_size(const std::string &subsection_name, const std::string ¶meter_name)
345 auto array_size = conf[test_type][GetParam()]["parameters"][subsection_name][parameter_name].size();
348 catch (std::domain_error &e)
350 std::cout << "[----------] get_input_parameter_size (" << subsection_name << "." << parameter_name
351 << ") failed: " << e.what()
352 << ". Did you mispell the parameter name?" << std::endl;
355 catch(reading_input_file_exception &e)
357 std::cout << "[----------] get_input_parameter_size (" << subsection_name << "." << parameter_name
358 << ") failed: " << e.what() << std::endl;
362 int get_input_subsection_size(const std::string &subsection_name)
366 auto array_size = conf[test_type][GetParam()]["parameters"][subsection_name].size();
369 catch (std::domain_error &e)
371 std::cout << "[----------] get_input_subsection_size (" << subsection_name
372 << ") failed: " << e.what()
373 << ". Did you mispell the subsection name?" << std::endl;
376 catch(reading_input_file_exception &e)
378 std::cout << "[----------] get_input_subsection_size (" << subsection_name
379 << ") failed: " << e.what() << std::endl;
384 template <typename T>
385 T get_reference_parameter(const std::string ¶meter_name)
389 return get_parameter<T>("references", parameter_name);
391 catch (std::domain_error &e)
393 std::cout << "[----------] get_reference_parameter (" << parameter_name
394 << ") failed: " << e.what()
395 << ". Did you mispell the parameter name?" << std::endl;
398 catch(reading_input_file_exception &e)
400 std::cout << "[----------] get_reference_parameter (" << parameter_name
401 << ") failed: " << e.what() << std::endl;
408 \brief Get name of the test case from JSON file.
409 \return Test'ss case name or a default name if name field is missing.
411 const std::string get_case_name()
415 return conf[test_type][GetParam()]["name"];
417 catch (std::domain_error &e)
419 std::cout << "[----------] get_case_name failed: " << e.what()
420 << ". Did you specify a test name in JSON?" << std::endl;
421 std::cout << "[----------] Using a default name instead" << std::endl;
423 return "Default test name";
428 \brief Defines section in the conf.json that is used to load parameters from.
429 \param [in] type Name of the section in the JSON file.
431 void init_test(const std::string &type)
434 const std::string name = get_case_name();
435 std::cout << "[----------] Test case: " << name << std::endl;
439 static unsigned long tsc;
443 static T read_parameter(const int index, const std::string &type,
444 const std::string ¶meter_name)
446 return conf[test_type][index][type][parameter_name];
451 struct data_reader<std::vector<T>> {
452 static std::vector<T> read_parameter(const int index, const std::string &type,
453 const std::string ¶meter_name)
455 auto array_size = conf[test_type][index][type][parameter_name].size();
457 std::vector<T> result(array_size);
459 for(unsigned number = 0; number < array_size; number++)
460 result.at(number) = conf[test_type][index][type][parameter_name][number];
467 struct data_reader<T*> {
468 static T* read_parameter(const int index, const std::string &type,
469 const std::string ¶meter_name)
471 return (T*) read_data_to_aligned_array(conf[test_type][index][type][parameter_name]);
475 template <typename T>
476 T get_parameter(const std::string &type, const std::string ¶meter_name)
478 return data_reader<T>::read_parameter(GetParam(), type, parameter_name);
482 struct data_reader2 {
483 static T read_parameter(const int index, const std::string &type,
484 const std::string &subsection_name,
485 const std::string ¶meter_name)
487 return conf[test_type][index][type][subsection_name][parameter_name];
492 struct data_reader2<std::vector<T>> {
493 static std::vector<T> read_parameter(const int index, const std::string &type,
494 const std::string &subsection_name,
495 const std::string ¶meter_name)
497 auto array_size = conf[test_type][index][type][subsection_name][parameter_name].size();
499 std::vector<T> result(array_size);
501 for(unsigned number = 0; number < array_size; number++)
502 result.at(number) = conf[test_type][index][type][subsection_name][parameter_name][number];
509 struct data_reader2<T*> {
510 static T* read_parameter(const int index, const std::string &type,
511 const std::string &subsection_name,
512 const std::string ¶meter_name)
514 return (T*) read_data_to_aligned_array(conf[test_type][index][type][subsection_name][parameter_name]);
517 template <typename T>
518 T get_parameter(const std::string &type, const std::string &subsection_name, const std::string ¶meter_name)
520 return data_reader2<T>::read_parameter(GetParam(), type, subsection_name, parameter_name);
524 struct data_reader3 {
525 static T read_parameter(const int index, const std::string &type,
526 const std::string &subsection_name,
528 const std::string ¶meter_name)
530 return conf[test_type][index][type][subsection_name][subindex][parameter_name];
535 struct data_reader3<std::vector<T>> {
536 static std::vector<T> read_parameter(const int index, const std::string &type,
537 const std::string &subsection_name,
539 const std::string ¶meter_name)
541 auto array_size = conf[test_type][index][type][subsection_name][subindex][parameter_name].size();
543 std::vector<T> result(array_size);
545 for(unsigned number = 0; number < array_size; number++)
546 result.at(number) = conf[test_type][index][type][subsection_name][subindex][parameter_name][number];
553 struct data_reader3<T*> {
554 static T* read_parameter(const int index, const std::string &type,
555 const std::string &subsection_name,
557 const std::string ¶meter_name)
559 return (T*) read_data_to_aligned_array(conf[test_type][index][type][subsection_name][subindex][parameter_name]);
562 template <typename T>
563 T get_parameter(const std::string &type, const std::string &subsection_name, const int subindex, const std::string ¶meter_name)
565 return data_reader3<T>::read_parameter(GetParam(), type, subsection_name, subindex, parameter_name);
568 void print_and_store_results(const std::string &isa,
569 const std::string ¶meters,
570 const std::string &module_name,
571 const std::string &test_name,
572 const std::string &unit,
573 const int para_factor,
575 const double stddev);
579 \brief Run the given function and return the mean run time and stddev.
580 \param [in] function Function to benchmark.
581 \param [in] args Function's arguments.
582 \return std::pair where the first element is mean and the second one is standard deviation.
584 template <typename F, typename ... Args>
585 std::pair<double, double> run_benchmark(F function, Args ... args)
587 std::vector<long> results((unsigned long) BenchmarkParameters::repetition);
589 for(unsigned int outer_loop = 0; outer_loop < BenchmarkParameters::repetition; outer_loop++) {
590 const auto start_time = __rdtsc();
591 for (unsigned int inner_loop = 0; inner_loop < BenchmarkParameters::loop; inner_loop++) {
594 const auto end_time = __rdtsc();
595 results.push_back(end_time - start_time);
598 return calculate_statistics(results);
602 \brief Assert elements of two arrays. It calls ASSERT_EQ for each element of the array.
603 \param [in] reference Array with reference values.
604 \param [in] actual Array with the actual output.
605 \param [in] size Size of the array.
607 template <typename T>
608 void assert_array_eq(const T* reference, const T* actual, const int size)
610 for(int index = 0; index < size ; index++)
612 ASSERT_EQ(reference[index], actual[index])
613 <<"The wrong number is index: "<< index;
618 \brief Assert elements of two arrays. It calls ASSERT_NEAR for each element of the array.
619 \param [in] reference Array with reference values.
620 \param [in] actual Array with the actual output.
621 \param [in] size Size of the array.
622 \param [in] precision Precision fo the comparision used by ASSERT_NEAR.
624 template <typename T>
625 void assert_array_near(const T* reference, const T* actual, const int size, const double precision)
627 for(int index = 0; index < size ; index++)
629 ASSERT_NEAR(reference[index], actual[index], precision)
630 <<"The wrong number is index: "<< index;
635 void assert_array_near<complex_float>(const complex_float* reference, const complex_float* actual, const int size, const double precision)
637 for(int index = 0; index < size ; index++)
639 ASSERT_NEAR(reference[index].re, actual[index].re, precision)
640 <<"The wrong number is RE, index: "<< index;
641 ASSERT_NEAR(reference[index].im, actual[index].im, precision)
642 <<"The wrong number is IM, index: "<< index;
647 \brief Assert average diff of two arrays. It calls ASSERT_GT to check the average.
648 \param [in] reference Array with reference values, interleaved IQ inputs.
649 \param [in] actual Array with the actual output, interleaved IQ inputs.
650 \param [in] size Size of the array, based on complex inputs.
651 \param [in] precision Precision for the comparison used by ASSERT_GT.
654 void assert_avg_greater_complex(const T* reference, const T* actual, const int size, const double precision)
657 double avgMSEDB = 0.0;
658 for (int index = 0; index < size; index++) {
659 T refReal = reference[2*index];
660 T refImag = reference[(2*index)+1];
661 T resReal = actual[2*index];
662 T resImag = actual[(2*index)+1];
664 T errReal = resReal - refReal;
665 T errIm = resImag - refImag;
667 /* For some unit tests, e.g. PUCCH deomdulation, the expected output is 0. To avoid a
668 divide by zero error, check the reference results to determine if the expected result
669 is 0 and, if so, add a 1 to the division. */
670 if (refReal == 0 && refImag == 0)
671 MSE = (float)(errReal*errReal + errIm*errIm)/(float)(refReal*refReal + refImag*refImag + 1);
673 MSE = (float)(errReal*errReal + errIm*errIm)/(float)(refReal*refReal + refImag*refImag);
676 mseDB = (float)(-100.0);
678 mseDB = (float)(10.0) * (float)log10(MSE);
680 avgMSEDB += (double)mseDB;
685 ASSERT_GT(precision, avgMSEDB);
689 \brief Allocates memory of the given size.
691 aligned_malloc is wrapper to functions that allocate memory:
692 'rte_malloc' from DPDK if hugepages are defined, 'memalign' otherwise.
693 Size is defined as a number of variables of given type e.g. floats, rather than bytes.
694 It hides sizeof(T) multiplication and cast hence makes things cleaner.
696 \param [in] size Size of the memory to allocate.
697 \param [in] alignment Bytes alignment of the allocated memory. If 0, the return is a pointer
698 that is suitably aligned for any kind of variable (in the same manner as malloc()).
699 Otherwise, the return is a pointer that is a multiple of align. In this case,
700 it must be a power of two. (Minimum alignment is the cacheline size, i.e. 64-bytes)
701 \return Pointer to the allocated memory.
703 template <typename T>
704 T* aligned_malloc(const int size, const unsigned alignment)
707 return (T*) rte_malloc(NULL, sizeof(T) * size, alignment);
710 return (T*) memalign(alignment, sizeof(T) * size);
712 return (T*)_aligned_malloc(sizeof(T)*size, alignment);
718 \brief Frees memory pointed by the given pointer.
720 aligned_free is a wrapper for functions that free memory allocated by
721 aligned_malloc: 'rte_free' from DPDK if hugepages are defined and 'free' otherwise.
723 \param [in] ptr Pointer to the allocated memory.
725 template <typename T>
726 void aligned_free(T* ptr)
729 rte_free((void*)ptr);
735 _aligned_free((void *)ptr);
741 \brief generate random numbers.
743 It allocates memory and populate it with random numbers using C++11 default engine and
744 uniform real / int distribution (where lo_range <= x <up_range). Don't forget to free
747 \param [in] size Size of the memory to be filled with random data.
748 \param [in] alignment Bytes alignment of the memory.
749 \param [in] distribution Distribuiton for random generator.
750 \return Pointer to the allocated memory with random data.
752 template <typename T, typename U>
753 T* generate_random_numbers(const long size, const unsigned alignment, U& distribution)
755 auto array = (T*) aligned_malloc<char>(size * sizeof(T), alignment);
757 std::random_device random_device;
758 std::default_random_engine generator(random_device());
760 for(long i = 0; i < size; i++)
761 array[i] = (T)distribution(generator);
767 \brief generate random data.
769 It allocates memory and populate it with random data using C++11 default engine and
770 uniform integer distribution (bytes not floats are uniformly distributed). Don't forget
771 to free allocated memory!
773 \param [in] size Size of the memory to be filled with random data.
774 \param [in] alignment Bytes alignment of the memory.
775 \return Pointer to the allocated memory with random data.
777 template <typename T>
778 T* generate_random_data(const long size, const unsigned alignment)
780 std::uniform_int_distribution<> random(0, 255);
782 return (T*)generate_random_numbers<char, std::uniform_int_distribution<>>(size * sizeof(T), alignment, random);
786 \brief generate integer random numbers.
788 It allocates memory and populate it with random numbers using C++11 default engine and
789 uniform integer distribution (where lo_range <= x < up_range). Don't forget
790 to free allocated memory! The result type generated by the generator should be one of
793 \param [in] size Size of the memory to be filled with random data.
794 \param [in] alignment Bytes alignment of the memory.
795 \param [in] lo_range Lower bound of range of values returned by random generator.
796 \param [in] up_range Upper bound of range of values returned by random generator.
797 \return Pointer to the allocated memory with random data.
799 template <typename T>
800 T* generate_random_int_numbers(const long size, const unsigned alignment, const T lo_range,
803 std::uniform_int_distribution<T> random(lo_range, up_range);
805 return generate_random_numbers<T, std::uniform_int_distribution<T>>(size, alignment, random);
809 \brief generate real random numbers.
811 It allocates memory and populate it with random numbers using C++11 default engine and
812 uniform real distribution (where lo_range <= x <up_range). Don't forget to free
813 allocated memory! The result type generated by the generator should be one of
814 real types: float, double or long double.
816 \param [in] size Size of the memory to be filled with random data.
817 \param [in] alignment Bytes alignment of the memory.
818 \param [in] lo_range Lower bound of range of values returned by random generator.
819 \param [in] up_range Upper bound of range of values returned by random generator.
820 \return Pointer to the allocated memory with random data.
822 template <typename T>
823 T* generate_random_real_numbers(const long size, const unsigned alignment, const T lo_range,
826 std::uniform_real_distribution<T> distribution(lo_range, up_range);
828 return generate_random_numbers<T, std::uniform_real_distribution<T>>(size, alignment, distribution);
831 #endif //XRANLIB_COMMON_HPP