1 /******************************************************************************
\r
3 * Copyright (c) 2019 Intel.
\r
5 * Licensed under the Apache License, Version 2.0 (the "License");
\r
6 * you may not use this file except in compliance with the License.
\r
7 * You may obtain a copy of the License at
\r
9 * http://www.apache.org/licenses/LICENSE-2.0
\r
11 * Unless required by applicable law or agreed to in writing, software
\r
12 * distributed under the License is distributed on an "AS IS" BASIS,
\r
13 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
\r
14 * See the License for the specific language governing permissions and
\r
15 * limitations under the License.
\r
17 *******************************************************************************/
\r
20 /* This is the new utility file for all tests, all new common functionality has to go here.
\r
21 When contributing to the common.hpp please focus on readability and maintainability rather than
\r
23 #ifndef XRANLIB_COMMON_HPP
\r
24 #define XRANLIB_COMMON_HPP
\r
26 /* Disable warnings generated by JSON parser */
\r
27 #pragma warning(disable : 191)
\r
28 #pragma warning(disable : 186)
\r
29 #pragma warning(disable : 192)
\r
31 #include <exception>
\r
37 #include <immintrin.h>
\r
40 #define _BBLIB_DPDK_
\r
43 #include <rte_config.h>
\r
44 #include <rte_malloc.h>
\r
47 #include "gtest/gtest.h"
\r
49 #include "common_typedef_xran.h"
\r
53 using json = nlohmann::json;
\r
55 #define ASSERT_ARRAY_NEAR(reference, actual, size, precision) \
\r
56 assert_array_near(reference, actual, size, precision)
\r
58 #define ASSERT_ARRAY_EQ(reference, actual, size) \
\r
59 assert_array_eq(reference, actual, size)
\r
61 #define ASSERT_AVG_GREATER_COMPLEX(reference, actual, size, precision) \
\r
62 assert_avg_greater_complex(reference, actual, size, precision)
\r
64 struct BenchmarkParameters
\r
66 static long repetition;
\r
68 static unsigned cpu_id;
\r
71 struct missing_config_file_exception : public std::exception
\r
73 const char * what () const throw () override {
\r
74 return "JSON file cannot be opened!";
\r
78 struct reading_input_file_exception : public std::exception
\r
80 const char * what () const throw () override {
\r
81 return "Input file cannot be read!";
\r
86 \brief Attach current process to the selected core.
\r
87 \param [in] cpu Core number.
\r
88 \return 0 on success, -1 otherwise.
\r
90 int bind_to_cpu(const unsigned cpu);
\r
93 \brief Calculate the mean and variance from the result of the run_benchmark.
\r
94 \param [in] values Vector with result values.
\r
95 \return std::pair where the first element is mean and the second one is standard deviation.
\r
96 \note It's not a general mean/stddev function it only works properly when feed with data from
\r
97 the benchmark function.
\r
99 std::pair<double, double> calculate_statistics(const std::vector<long> values);
\r
102 \brief For a given number return sequence of number from 0 to number - 1.
\r
103 \param [in] number Positive integer value.
\r
104 \return Vector with the sorted integer numbers between 0 and number - 1.
\r
106 std::vector<unsigned> get_sequence(const unsigned number);
\r
109 \brief Read JSON from the given file.
\r
110 \param [in] filename name of the .json file.
\r
111 \return JSON object with data.
\r
112 \throws missing_config_file_exception when file cannot be opened.
\r
114 json read_json_from_file(const std::string &filename);
\r
117 \brief Read binary data from the file.
\r
118 \param [in] filename name of the binary file.
\r
119 \return Pointer to the allocated memory with data from the file.
\r
120 \throws std::runtime_error when memory cannot be allocated.
\r
122 char* read_data_to_aligned_array(const std::string &filename);
\r
125 \brief Measure the TSC on the machine
\r
126 \return Number of ticks per us
\r
128 unsigned long tsc_recovery();
\r
131 \brief Return the current value of the TSC
\r
132 \return Current TSC value
\r
134 unsigned long tsc_tick();
\r
139 Each test class has to inherit from KernelTests class as it provides GTest support and does a lot
\r
140 of setup (including JSON) an provides useful methods to operate on loaded JSON file.
\r
141 Unfortunately GTest is limited in the way that all TEST_P within the class are called for all
\r
142 cases/parameters, but we usually want two different data sets for functional and performance
\r
143 tests (or maybe other types of tests). Because of that to use different data sets we need to
\r
144 create separate classes, hence performance and functional test are in separate classes. it adds
\r
145 an extra overhead, but adds much more flexibility. init_test(...) is used to select data set from
\r
148 Important note on the JSON file structure. Top JSON object can have as many section (JSON
\r
149 objects) as needed, but each have to have a distinct name that is used by init_test. Then
\r
150 each section must contain an array of objects (test cases) where each object has a name,
\r
151 parameters and references. Everything inside parameters and references can be completely custom
\r
152 as it's loaded by get_input/reference_parameter function. JSON values can be either literal
\r
153 values, e.g. 1, 0.001, 5e-05, etc. or filename. Depends on the get type test framework can either
\r
154 read the value or load data from the file - and it happens automatically (*pff* MAGIC!).
\r
156 class KernelTests : public testing::TestWithParam<unsigned>
\r
160 static std::string test_type;
\r
162 static void SetUpTestCase()
\r
164 test_type = "None";
\r
168 conf = read_json_from_file("conf.json");
\r
170 catch(missing_config_file_exception &e)
\r
172 std::cout << "[----------] SetUpTestCase failed: " << e.what() << std::endl;
\r
176 tsc = tsc_recovery();
\r
180 std::cout << "[----------] SetUpTestCase failed: TSC recovery failed" << std::endl;
\r
185 static void TearDownTestCase()
\r
187 /* Free resources - nothing to free at the moment */
\r
190 static unsigned get_number_of_cases(const std::string &type)
\r
194 json json_data = read_json_from_file("conf.json");
\r
196 return json_data[type].size();
\r
198 catch(missing_config_file_exception &e)
\r
200 std::cout << "[----------] get_number_of_cases failed: " << e.what() << std::endl;
\r
204 catch(std::domain_error &e)
\r
206 std::cout << "[----------] get_number_of_cases failed: " << e.what() << std::endl;
\r
207 std::cout << "[----------] Use a default value: 0" << std::endl;
\r
214 double division_factor = 1.0;
\r
215 std::string result_units = "None";
\r
216 int parallelization_factor = 1;
\r
219 \brief Set division factor
\r
220 \param [in] factor Division factor that divides mean and standard deviation.
\r
222 void set_division_factor(const double factor)
\r
224 division_factor = factor;
\r
228 \brief Set reults units
\r
229 \param [in] units Units that are displayed in the report.
\r
231 void set_results_units(const std::string &units)
\r
233 result_units = units;
\r
237 \brief Set size of processed data
\r
238 \param [in] size Size of processed data used to calculate module throughput.
\r
240 void set_parallelization_factor(const int factor)
\r
242 parallelization_factor = factor;
\r
246 \brief Run performance test case for a given function.
\r
247 \param [in] isa Used Instruction Set.
\r
248 \param [in] module_name name of the tested kernel.
\r
249 \param [in] function function to be tested.
\r
250 \param [in] args function's arguments.
\r
252 template <typename F, typename ... Args>
\r
253 void performance(const std::string &isa, const std::string &module_name, F function,
\r
255 ASSERT_EQ(0, bind_to_cpu(BenchmarkParameters::cpu_id)) << "Failed to bind to cpu!";
\r
257 const auto result = run_benchmark(function, args ...);
\r
258 const auto scaled_mean = result.first / division_factor / tsc;
\r
259 const auto scaled_stddev = result.second / division_factor / tsc;
\r
261 print_and_store_results(isa, get_case_name(), module_name, get_case_name(), result_units,
\r
262 parallelization_factor, scaled_mean, scaled_stddev);
\r
266 \brief Print unique test description to the results xml file
\r
267 \param [in] isa Used Instruction Set.
\r
268 \param [in] module_name name of the tested kernel.
\r
269 \param [in] function function to be tested.
\r
271 void print_test_description(const std::string &isa, const std::string &module_name) {
\r
272 print_and_store_results(isa, get_case_name(), module_name, get_case_name(), result_units,
\r
273 parallelization_factor, 0, 0);
\r
278 \brief Load selected data from a JSON object.
\r
279 get_input_parameter loads data from parameters section of the test case in JSON file and
\r
280 get_reference_parameter does the same thing for references section.
\r
282 Get parameter function uses template type to figure out how to load parameters. If type
\r
283 is NOT a pointer it'll load value directly from the JSON. Otherwise path to the test
\r
284 vector is expected and function will allocate memory, load data from the binary file to
\r
285 this memory location and return pointer to it. For example in here we request to load
\r
286 pointer to float so llrs filed is expected to be
\r
287 a path to the binary file.
\r
289 template <typename T>
\r
290 T get_input_parameter(const std::string ¶meter_name)
\r
294 return get_parameter<T>("parameters", parameter_name);
\r
296 catch (std::domain_error &e)
\r
298 std::cout << "[----------] get_input_parameter (" << parameter_name
\r
299 << ") failed: " << e.what()
\r
300 << ". Did you mispell the parameter name?" << std::endl;
\r
303 catch(reading_input_file_exception &e)
\r
305 std::cout << "[----------] get_input_parameter (" << parameter_name
\r
306 << ") failed: " << e.what() << std::endl;
\r
311 template <typename T>
\r
312 T get_reference_parameter(const std::string ¶meter_name)
\r
316 return get_parameter<T>("references", parameter_name);
\r
318 catch (std::domain_error &e)
\r
320 std::cout << "[----------] get_reference_parameter (" << parameter_name
\r
321 << ") failed: " << e.what()
\r
322 << ". Did you mispell the parameter name?" << std::endl;
\r
325 catch(reading_input_file_exception &e)
\r
327 std::cout << "[----------] get_reference_parameter (" << parameter_name
\r
328 << ") failed: " << e.what() << std::endl;
\r
335 \brief Get name of the test case from JSON file.
\r
336 \return Test'ss case name or a default name if name field is missing.
\r
338 const std::string get_case_name()
\r
342 return conf[test_type][GetParam()]["name"];
\r
344 catch (std::domain_error &e)
\r
346 std::cout << "[----------] get_case_name failed: " << e.what()
\r
347 << ". Did you specify a test name in JSON?" << std::endl;
\r
348 std::cout << "[----------] Using a default name instead" << std::endl;
\r
350 return "Default test name";
\r
355 \brief Defines section in the conf.json that is used to load parameters from.
\r
356 \param [in] type Name of the section in the JSON file.
\r
358 void init_test(const std::string &type)
\r
361 const std::string name = get_case_name();
\r
362 std::cout << "[----------] Test case: " << name << std::endl;
\r
366 static unsigned long tsc;
\r
368 template<typename T>
\r
369 struct data_reader {
\r
370 static T read_parameter(const int index, const std::string &type,
\r
371 const std::string ¶meter_name)
\r
373 return conf[test_type][index][type][parameter_name];
\r
377 template<typename T>
\r
378 struct data_reader<std::vector<T>> {
\r
379 static std::vector<T> read_parameter(const int index, const std::string &type,
\r
380 const std::string ¶meter_name)
\r
382 auto array_size = conf[test_type][index][type][parameter_name].size();
\r
384 std::vector<T> result(array_size);
\r
386 for(unsigned number = 0; number < array_size; number++)
\r
387 result.at(number) = conf[test_type][index][type][parameter_name][number];
\r
393 template<typename T>
\r
394 struct data_reader<T*> {
\r
395 static T* read_parameter(const int index, const std::string &type,
\r
396 const std::string ¶meter_name)
\r
398 return (T*) read_data_to_aligned_array(conf[test_type][index][type][parameter_name]);
\r
402 template <typename T>
\r
403 T get_parameter(const std::string &type, const std::string ¶meter_name)
\r
405 return data_reader<T>::read_parameter(GetParam(), type, parameter_name);
\r
408 void print_and_store_results(const std::string &isa,
\r
409 const std::string ¶meters,
\r
410 const std::string &module_name,
\r
411 const std::string &test_name,
\r
412 const std::string &unit,
\r
413 const int para_factor,
\r
415 const double stddev);
\r
419 \brief Run the given function and return the mean run time and stddev.
\r
420 \param [in] function Function to benchmark.
\r
421 \param [in] args Function's arguments.
\r
422 \return std::pair where the first element is mean and the second one is standard deviation.
\r
424 template <typename F, typename ... Args>
\r
425 std::pair<double, double> run_benchmark(F function, Args ... args)
\r
427 std::vector<long> results((unsigned long) BenchmarkParameters::repetition);
\r
429 for(unsigned int outer_loop = 0; outer_loop < BenchmarkParameters::repetition; outer_loop++) {
\r
430 const auto start_time = __rdtsc();
\r
431 for (unsigned int inner_loop = 0; inner_loop < BenchmarkParameters::loop; inner_loop++) {
\r
432 function(args ...);
\r
434 const auto end_time = __rdtsc();
\r
435 results.push_back(end_time - start_time);
\r
438 return calculate_statistics(results);
\r
442 \brief Assert elements of two arrays. It calls ASSERT_EQ for each element of the array.
\r
443 \param [in] reference Array with reference values.
\r
444 \param [in] actual Array with the actual output.
\r
445 \param [in] size Size of the array.
\r
447 template <typename T>
\r
448 void assert_array_eq(const T* reference, const T* actual, const int size)
\r
450 for(int index = 0; index < size ; index++)
\r
452 ASSERT_EQ(reference[index], actual[index])
\r
453 <<"The wrong number is index: "<< index;
\r
458 \brief Assert elements of two arrays. It calls ASSERT_NEAR for each element of the array.
\r
459 \param [in] reference Array with reference values.
\r
460 \param [in] actual Array with the actual output.
\r
461 \param [in] size Size of the array.
\r
462 \param [in] precision Precision fo the comparision used by ASSERT_NEAR.
\r
464 template <typename T>
\r
465 void assert_array_near(const T* reference, const T* actual, const int size, const double precision)
\r
467 for(int index = 0; index < size ; index++)
\r
469 ASSERT_NEAR(reference[index], actual[index], precision)
\r
470 <<"The wrong number is index: "<< index;
\r
475 void assert_array_near<complex_float>(const complex_float* reference, const complex_float* actual, const int size, const double precision)
\r
477 for(int index = 0; index < size ; index++)
\r
479 ASSERT_NEAR(reference[index].re, actual[index].re, precision)
\r
480 <<"The wrong number is RE, index: "<< index;
\r
481 ASSERT_NEAR(reference[index].im, actual[index].im, precision)
\r
482 <<"The wrong number is IM, index: "<< index;
\r
487 \brief Assert average diff of two arrays. It calls ASSERT_GT to check the average.
\r
488 \param [in] reference Array with reference values, interleaved IQ inputs.
\r
489 \param [in] actual Array with the actual output, interleaved IQ inputs.
\r
490 \param [in] size Size of the array, based on complex inputs.
\r
491 \param [in] precision Precision for the comparison used by ASSERT_GT.
\r
493 template<typename T>
\r
494 void assert_avg_greater_complex(const T* reference, const T* actual, const int size, const double precision)
\r
497 double avgMSEDB = 0.0;
\r
498 for (int index = 0; index < size; index++) {
\r
499 T refReal = reference[2*index];
\r
500 T refImag = reference[(2*index)+1];
\r
501 T resReal = actual[2*index];
\r
502 T resImag = actual[(2*index)+1];
\r
504 T errReal = resReal - refReal;
\r
505 T errIm = resImag - refImag;
\r
507 /* For some unit tests, e.g. PUCCH deomdulation, the expected output is 0. To avoid a
\r
508 divide by zero error, check the reference results to determine if the expected result
\r
509 is 0 and, if so, add a 1 to the division. */
\r
510 if (refReal == 0 && refImag == 0)
\r
511 MSE = (float)(errReal*errReal + errIm*errIm)/(float)(refReal*refReal + refImag*refImag + 1);
\r
513 MSE = (float)(errReal*errReal + errIm*errIm)/(float)(refReal*refReal + refImag*refImag);
\r
516 mseDB = (float)(-100.0);
\r
518 mseDB = (float)(10.0) * (float)log10(MSE);
\r
520 avgMSEDB += (double)mseDB;
\r
525 ASSERT_GT(precision, avgMSEDB);
\r
529 \brief Allocates memory of the given size.
\r
531 aligned_malloc is wrapper to functions that allocate memory:
\r
532 'rte_malloc' from DPDK if hugepages are defined, 'memalign' otherwise.
\r
533 Size is defined as a number of variables of given type e.g. floats, rather than bytes.
\r
534 It hides sizeof(T) multiplication and cast hence makes things cleaner.
\r
536 \param [in] size Size of the memory to allocate.
\r
537 \param [in] alignment Bytes alignment of the allocated memory. If 0, the return is a pointer
\r
538 that is suitably aligned for any kind of variable (in the same manner as malloc()).
\r
539 Otherwise, the return is a pointer that is a multiple of align. In this case,
\r
540 it must be a power of two. (Minimum alignment is the cacheline size, i.e. 64-bytes)
\r
541 \return Pointer to the allocated memory.
\r
543 template <typename T>
\r
544 T* aligned_malloc(const int size, const unsigned alignment)
\r
546 #ifdef _BBLIB_DPDK_
\r
547 return (T*) rte_malloc(NULL, sizeof(T) * size, alignment);
\r
550 return (T*) memalign(alignment, sizeof(T) * size);
\r
552 return (T*)_aligned_malloc(sizeof(T)*size, alignment);
\r
558 \brief Frees memory pointed by the given pointer.
\r
560 aligned_free is a wrapper for functions that free memory allocated by
\r
561 aligned_malloc: 'rte_free' from DPDK if hugepages are defined and 'free' otherwise.
\r
563 \param [in] ptr Pointer to the allocated memory.
\r
565 template <typename T>
\r
566 void aligned_free(T* ptr)
\r
568 #ifdef _BBLIB_DPDK_
\r
569 rte_free((void*)ptr);
\r
575 _aligned_free((void *)ptr);
\r
581 \brief generate random numbers.
\r
583 It allocates memory and populate it with random numbers using C++11 default engine and
\r
584 uniform real / int distribution (where lo_range <= x <up_range). Don't forget to free
\r
587 \param [in] size Size of the memory to be filled with random data.
\r
588 \param [in] alignment Bytes alignment of the memory.
\r
589 \param [in] distribution Distribuiton for random generator.
\r
590 \return Pointer to the allocated memory with random data.
\r
592 template <typename T, typename U>
\r
593 T* generate_random_numbers(const long size, const unsigned alignment, U& distribution)
\r
595 auto array = (T*) aligned_malloc<char>(size * sizeof(T), alignment);
\r
597 std::random_device random_device;
\r
598 std::default_random_engine generator(random_device());
\r
600 for(long i = 0; i < size; i++)
\r
601 array[i] = (T)distribution(generator);
\r
607 \brief generate random data.
\r
609 It allocates memory and populate it with random data using C++11 default engine and
\r
610 uniform integer distribution (bytes not floats are uniformly distributed). Don't forget
\r
611 to free allocated memory!
\r
613 \param [in] size Size of the memory to be filled with random data.
\r
614 \param [in] alignment Bytes alignment of the memory.
\r
615 \return Pointer to the allocated memory with random data.
\r
617 template <typename T>
\r
618 T* generate_random_data(const long size, const unsigned alignment)
\r
620 std::uniform_int_distribution<> random(0, 255);
\r
622 return (T*)generate_random_numbers<char, std::uniform_int_distribution<>>(size * sizeof(T), alignment, random);
\r
626 \brief generate integer random numbers.
\r
628 It allocates memory and populate it with random numbers using C++11 default engine and
\r
629 uniform integer distribution (where lo_range <= x < up_range). Don't forget
\r
630 to free allocated memory! The result type generated by the generator should be one of
\r
633 \param [in] size Size of the memory to be filled with random data.
\r
634 \param [in] alignment Bytes alignment of the memory.
\r
635 \param [in] lo_range Lower bound of range of values returned by random generator.
\r
636 \param [in] up_range Upper bound of range of values returned by random generator.
\r
637 \return Pointer to the allocated memory with random data.
\r
639 template <typename T>
\r
640 T* generate_random_int_numbers(const long size, const unsigned alignment, const T lo_range,
\r
643 std::uniform_int_distribution<T> random(lo_range, up_range);
\r
645 return generate_random_numbers<T, std::uniform_int_distribution<T>>(size, alignment, random);
\r
649 \brief generate real random numbers.
\r
651 It allocates memory and populate it with random numbers using C++11 default engine and
\r
652 uniform real distribution (where lo_range <= x <up_range). Don't forget to free
\r
653 allocated memory! The result type generated by the generator should be one of
\r
654 real types: float, double or long double.
\r
656 \param [in] size Size of the memory to be filled with random data.
\r
657 \param [in] alignment Bytes alignment of the memory.
\r
658 \param [in] lo_range Lower bound of range of values returned by random generator.
\r
659 \param [in] up_range Upper bound of range of values returned by random generator.
\r
660 \return Pointer to the allocated memory with random data.
\r
662 template <typename T>
\r
663 T* generate_random_real_numbers(const long size, const unsigned alignment, const T lo_range,
\r
666 std::uniform_real_distribution<T> distribution(lo_range, up_range);
\r
668 return generate_random_numbers<T, std::uniform_real_distribution<T>>(size, alignment, distribution);
\r
671 #endif //XRANLIB_COMMON_HPP
\r