fhi_lib/lib/src/xran_bfp_byte_packing_utils.hpp

   1 /******************************************************************************
   2 *
   3 *   Copyright (c) 2020 Intel.
   4 *
   5 *   Licensed under the Apache License, Version 2.0 (the "License");
   6 *   you may not use this file except in compliance with the License.
   7 *   You may obtain a copy of the License at
   8 *
   9 *       http://www.apache.org/licenses/LICENSE-2.0
  10 *
  11 *   Unless required by applicable law or agreed to in writing, software
  12 *   distributed under the License is distributed on an "AS IS" BASIS,
  13 *   WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
  14 *   See the License for the specific language governing permissions and
  15 *   limitations under the License.
  16 *
  17 *******************************************************************************/
  18
  19 /**
  20  * @brief xRAN BFP byte packing utilities functions
  21  *
  22  * @file xran_bfp_byte_packing_utils.hpp
  23  * @ingroup group_source_xran
  24  * @author Intel Corporation
  25  **/
  26
  27 #pragma once
  28 #include <immintrin.h>
  29
  30 namespace BlockFloatCompander
  31 {
  32   /// Define function signatures for byte packing functions
  33   typedef __m512i(*PackFunction)(const __m512i);
  34   typedef __m512i(*UnpackFunction)(const uint8_t*);
  35   typedef __m256i(*UnpackFunction256)(const uint8_t*);
  36
  37   /// Pack compressed 9 bit data in network byte order
  38   inline __m512i
  39   networkBytePack9b(const __m512i compData)
  40   {
  41     /// Logical shift left to align network order byte parts
  42     const __m512i k_shiftLeft = _mm512_set_epi64(0x0000000100020003, 0x0004000500060007,
  43                                                  0x0000000100020003, 0x0004000500060007,
  44                                                  0x0000000100020003, 0x0004000500060007,
  45                                                  0x0000000100020003, 0x0004000500060007);
  46     const auto compDataPacked = _mm512_sllv_epi16(compData, k_shiftLeft);
  47
  48     /// First epi8 shuffle of even indexed samples
  49     const __m512i k_byteShuffleMask1 = _mm512_set_epi64(0x0000000000000000, 0x0C0D080904050001,
  50                                                         0x0000000000000000, 0x0C0D080904050001,
  51                                                         0x0000000000000000, 0x0C0D080904050001,
  52                                                         0x0000000000000000, 0x0C0D080904050001);
  53     constexpr uint64_t k_byteMask1 = 0x00FF00FF00FF00FF;
  54     const auto compDataShuff1 = _mm512_maskz_shuffle_epi8(k_byteMask1, compDataPacked, k_byteShuffleMask1);
  55
  56     /// Second epi8 shuffle of odd indexed samples
  57     const __m512i k_byteShuffleMask2 = _mm512_set_epi64(0x000000000000000E, 0x0F0A0B0607020300,
  58                                                         0x000000000000000E, 0x0F0A0B0607020300,
  59                                                         0x000000000000000E, 0x0F0A0B0607020300,
  60                                                         0x000000000000000E, 0x0F0A0B0607020300);
  61     constexpr uint64_t k_byteMask2 = 0x01FE01FE01FE01FE;
  62     const auto compDataShuff2 = _mm512_maskz_shuffle_epi8(k_byteMask2, compDataPacked, k_byteShuffleMask2);
  63
  64     /// Ternary blend of the two shuffled results
  65     const __m512i k_ternLogSelect = _mm512_set_epi64(0x00000000000000FF, 0x01FC07F01FC07F00,
  66                                                      0x00000000000000FF, 0x01FC07F01FC07F00,
  67                                                      0x00000000000000FF, 0x01FC07F01FC07F00,
  68                                                      0x00000000000000FF, 0x01FC07F01FC07F00);
  69     return _mm512_ternarylogic_epi64(compDataShuff1, compDataShuff2, k_ternLogSelect, 0xd8);
  70   }
  71
  72
  73   /// Pack compressed 10 bit data in network byte order
  74   inline __m512i
  75   networkBytePack10b(const __m512i compData)
  76   {
  77     /// Logical shift left to align network order byte parts
  78     const __m512i k_shiftLeft = _mm512_set_epi64(0x0000000200040006, 0x0000000200040006,
  79                                                  0x0000000200040006, 0x0000000200040006,
  80                                                  0x0000000200040006, 0x0000000200040006,
  81                                                  0x0000000200040006, 0x0000000200040006);
  82     const auto compDataPacked = _mm512_sllv_epi16(compData, k_shiftLeft);
  83
  84     /// First epi8 shuffle of even indexed samples
  85     const __m512i k_byteShuffleMask1 = _mm512_set_epi64(0x000000000000000C, 0x0D08090004050001,
  86                                                         0x000000000000000C, 0x0D08090004050001,
  87                                                         0x000000000000000C, 0x0D08090004050001,
  88                                                         0x000000000000000C, 0x0D08090004050001);
  89     constexpr uint64_t k_byteMask1 = 0x01EF01EF01EF01EF;
  90     const auto compDataShuff1 = _mm512_maskz_shuffle_epi8(k_byteMask1, compDataPacked, k_byteShuffleMask1);
  91
  92     /// Second epi8 shuffle of odd indexed samples
  93     const __m512i k_byteShuffleMask2 = _mm512_set_epi64(0x0000000000000E0F, 0x0A0B000607020300,
  94                                                         0x0000000000000E0F, 0x0A0B000607020300,
  95                                                         0x0000000000000E0F, 0x0A0B000607020300,
  96                                                         0x0000000000000E0F, 0x0A0B000607020300);
  97     constexpr uint64_t k_byteMask2 = 0x03DE03DE03DE03DE;
  98     const auto compDataShuff2 = _mm512_maskz_shuffle_epi8(k_byteMask2, compDataPacked, k_byteShuffleMask2);
  99
 100     /// Ternary blend of the two shuffled results
 101     const __m512i k_ternLogSelect = _mm512_set_epi64(0x000000000000FF03, 0xF03F00FF03F03F00,
 102                                                      0x000000000000FF03, 0xF03F00FF03F03F00,
 103                                                      0x000000000000FF03, 0xF03F00FF03F03F00,
 104                                                      0x000000000000FF03, 0xF03F00FF03F03F00);
 105     return _mm512_ternarylogic_epi64(compDataShuff1, compDataShuff2, k_ternLogSelect, 0xd8);
 106   }
 107
 108
 109   /// Pack compressed 12 bit data in network byte order
 110   inline __m512i
 111   networkBytePack12b(const __m512i compData)
 112   {
 113     /// Logical shift left to align network order byte parts
 114     const __m512i k_shiftLeft = _mm512_set_epi64(0x0000000400000004, 0x0000000400000004,
 115                                                  0x0000000400000004, 0x0000000400000004,
 116                                                  0x0000000400000004, 0x0000000400000004,
 117                                                  0x0000000400000004, 0x0000000400000004);
 118     const auto compDataPacked = _mm512_sllv_epi16(compData, k_shiftLeft);
 119
 120     /// First epi8 shuffle of even indexed samples
 121     const __m512i k_byteShuffleMask1 = _mm512_set_epi64(0x00000000000C0D00, 0x0809000405000001,
 122                                                         0x00000000000C0D00, 0x0809000405000001,
 123                                                         0x00000000000C0D00, 0x0809000405000001,
 124                                                         0x00000000000C0D00, 0x0809000405000001);
 125     constexpr uint64_t k_byteMask1 = 0x06DB06DB06DB06DB;
 126     const auto compDataShuff1 = _mm512_maskz_shuffle_epi8(k_byteMask1, compDataPacked, k_byteShuffleMask1);
 127
 128     /// Second epi8 shuffle of odd indexed samples
 129     const __m512i k_byteShuffleMask2 = _mm512_set_epi64(0x000000000E0F000A, 0x0B00060700020300,
 130                                                         0x000000000E0F000A, 0x0B00060700020300,
 131                                                         0x000000000E0F000A, 0x0B00060700020300,
 132                                                         0x000000000E0F000A, 0x0B00060700020300);
 133     constexpr uint64_t k_byteMask2 = 0x0DB60DB60DB60DB6;
 134     const auto compDataShuff2 = _mm512_maskz_shuffle_epi8(k_byteMask2, compDataPacked, k_byteShuffleMask2);
 135
 136     /// Ternary blend of the two shuffled results
 137     const __m512i k_ternLogSelect = _mm512_set_epi64(0x00000000FF0F00FF, 0x0F00FF0F00FF0F00,
 138                                                      0x00000000FF0F00FF, 0x0F00FF0F00FF0F00,
 139                                                      0x00000000FF0F00FF, 0x0F00FF0F00FF0F00,
 140                                                      0x00000000FF0F00FF, 0x0F00FF0F00FF0F00);
 141     return _mm512_ternarylogic_epi64(compDataShuff1, compDataShuff2, k_ternLogSelect, 0xd8);
 142   }
 143
 144
 145   /// Unpack compressed 9 bit data in network byte order
 146   inline __m512i
 147   networkByteUnpack9b(const uint8_t* inData)
 148   {
 149     /// Align chunks of compressed bytes into lanes to allow for expansion
 150     const __m512i* rawDataIn = reinterpret_cast<const __m512i*>(inData);
 151     const auto k_expPerm = _mm512_set_epi32(9, 8, 7, 6, 7, 6, 5, 4,
 152                                             5, 4, 3, 2, 3, 2, 1, 0);
 153     const auto inLaneAlign = _mm512_permutexvar_epi32(k_expPerm, *rawDataIn);
 154
 155     /// Byte shuffle to get all bits for each sample into 16b chunks
 156     /// Due to previous permute to get chunks of bytes into each lane, there is
 157     /// a different shuffle offset in each lane
 158     const __m512i k_byteShuffleMask = _mm512_set_epi64(0x0A0B090A08090708, 0x0607050604050304,
 159                                                        0x090A080907080607, 0x0506040503040203,
 160                                                        0x0809070806070506, 0x0405030402030102,
 161                                                        0x0708060705060405, 0x0304020301020001);
 162     const auto inDatContig = _mm512_shuffle_epi8(inLaneAlign, k_byteShuffleMask);
 163
 164     /// Logical shift left to set sign bit
 165     const __m512i k_slBits = _mm512_set_epi64(0x0007000600050004, 0x0003000200010000,
 166                                               0x0007000600050004, 0x0003000200010000,
 167                                               0x0007000600050004, 0x0003000200010000,
 168                                               0x0007000600050004, 0x0003000200010000);
 169     const auto inSetSign = _mm512_sllv_epi16(inDatContig, k_slBits);
 170
 171     /// Mask to zero unwanted bits
 172     const __m512i k_expMask = _mm512_set1_epi16(0xFF80);
 173     return _mm512_and_epi64(inSetSign, k_expMask);
 174   }
 175
 176
 177   /// Unpack compressed 10 bit data in network byte order
 178   inline __m512i
 179   networkByteUnpack10b(const uint8_t* inData)
 180   {
 181     /// Align chunks of compressed bytes into lanes to allow for expansion
 182     const __m512i* rawDataIn = reinterpret_cast<const __m512i*>(inData);
 183     const auto k_expPerm = _mm512_set_epi32(10, 9, 8, 7, 8, 7, 6, 5,
 184                                              5, 4, 3, 2, 3, 2, 1, 0);
 185     const auto inLaneAlign = _mm512_permutexvar_epi32(k_expPerm, *rawDataIn);
 186
 187     /// Byte shuffle to get all bits for each sample into 16b chunks
 188     /// Due to previous permute to get chunks of bytes into each lane, lanes
 189     /// 0 and 2 happen to be aligned, but lane 1 is offset by 2 bytes
 190     const __m512i k_byteShuffleMask = _mm512_set_epi64(0x0A0B090A08090708, 0x0506040503040203,
 191                                                        0x0809070806070506, 0x0304020301020001,
 192                                                        0x0A0B090A08090708, 0x0506040503040203,
 193                                                        0x0809070806070506, 0x0304020301020001);
 194     const auto inDatContig = _mm512_shuffle_epi8(inLaneAlign, k_byteShuffleMask);
 195
 196     /// Logical shift left to set sign bit
 197     const __m512i k_slBits = _mm512_set_epi64(0x0006000400020000, 0x0006000400020000,
 198                                               0x0006000400020000, 0x0006000400020000,
 199                                               0x0006000400020000, 0x0006000400020000,
 200                                               0x0006000400020000, 0x0006000400020000);
 201     const auto inSetSign = _mm512_sllv_epi16(inDatContig, k_slBits);
 202
 203     /// Mask to zero unwanted bits
 204     const __m512i k_expMask = _mm512_set1_epi16(0xFFC0);
 205     return _mm512_and_epi64(inSetSign, k_expMask);
 206   }
 207
 208
 209   /// Unpack compressed 12 bit data in network byte order
 210   inline __m512i
 211   networkByteUnpack12b(const uint8_t* inData)
 212   {
 213     /// Align chunks of compressed bytes into lanes to allow for expansion
 214     const __m512i* rawDataIn = reinterpret_cast<const __m512i*>(inData);
 215     const auto k_expPerm = _mm512_set_epi32(12, 11, 10, 9, 9, 8, 7, 6,
 216                                              6, 5, 4, 3, 3, 2, 1, 0);
 217     const auto inLaneAlign = _mm512_permutexvar_epi32(k_expPerm, *rawDataIn);
 218
 219     /// Byte shuffle to get all bits for each sample into 16b chunks
 220     /// For 12b mantissa all lanes post-permute are aligned and require same shuffle offset
 221     const __m512i k_byteShuffleMask = _mm512_set_epi64(0x0A0B090A07080607, 0x0405030401020001,
 222                                                        0x0A0B090A07080607, 0x0405030401020001,
 223                                                        0x0A0B090A07080607, 0x0405030401020001,
 224                                                        0x0A0B090A07080607, 0x0405030401020001);
 225     const auto inDatContig = _mm512_shuffle_epi8(inLaneAlign, k_byteShuffleMask);
 226
 227     /// Logical shift left to set sign bit
 228     const __m512i k_slBits = _mm512_set_epi64(0x0004000000040000, 0x0004000000040000,
 229                                               0x0004000000040000, 0x0004000000040000,
 230                                               0x0004000000040000, 0x0004000000040000,
 231                                               0x0004000000040000, 0x0004000000040000);
 232     const auto inSetSign = _mm512_sllv_epi16(inDatContig, k_slBits);
 233
 234     /// Mask to zero unwanted bits
 235     const __m512i k_expMask = _mm512_set1_epi16(0xFFF0);
 236     return _mm512_and_epi64(inSetSign, k_expMask);
 237   }
 238
 239
 240   /// Unpack compressed 9 bit data in network byte order
 241   /// This unpacking function is for 256b registers
 242   inline __m256i
 243   networkByteUnpack9b256(const uint8_t* inData)
 244   {
 245     /// Align chunks of compressed bytes into lanes to allow for expansion
 246     const __m256i* rawDataIn = reinterpret_cast<const __m256i*>(inData);
 247     const auto k_expPerm = _mm256_set_epi32(5, 4, 3, 2, 3, 2, 1, 0);
 248     const auto inLaneAlign = _mm256_permutexvar_epi32(k_expPerm, *rawDataIn);
 249
 250     /// Byte shuffle to get all bits for each sample into 16b chunks
 251     /// Due to previous permute to get chunks of bytes into each lane, there is
 252     /// a different shuffle offset in each lane
 253     const __m256i k_byteShuffleMask = _mm256_set_epi64x(0x0809070806070506, 0x0405030402030102,
 254                                                         0x0708060705060405, 0x0304020301020001);
 255     const auto inDatContig = _mm256_shuffle_epi8(inLaneAlign, k_byteShuffleMask);
 256
 257     /// Logical shift left to set sign bit
 258     const __m256i k_slBits = _mm256_set_epi64x(0x0007000600050004, 0x0003000200010000,
 259                                                0x0007000600050004, 0x0003000200010000);
 260     const auto inSetSign = _mm256_sllv_epi16(inDatContig, k_slBits);
 261
 262     /// Mask to zero unwanted bits
 263     const __m256i k_expMask = _mm256_set1_epi16(0xFF80);
 264     return _mm256_and_si256(inSetSign, k_expMask);
 265   }
 266
 267
 268   /// Unpack compressed 10 bit data in network byte order
 269   /// This unpacking function is for 256b registers
 270   inline __m256i
 271   networkByteUnpack10b256(const uint8_t* inData)
 272   {
 273     /// Align chunks of compressed bytes into lanes to allow for expansion
 274     const __m256i* rawDataIn = reinterpret_cast<const __m256i*>(inData);
 275     const auto k_expPerm = _mm256_set_epi32(5, 4, 3, 2, 3, 2, 1, 0);
 276     const auto inLaneAlign = _mm256_permutexvar_epi32(k_expPerm, *rawDataIn);
 277
 278     /// Byte shuffle to get all bits for each sample into 16b chunks
 279     /// Due to previous permute to get chunks of bytes into each lane, lanes
 280     /// 0 and 2 happen to be aligned, but lane 1 is offset by 2 bytes
 281     const __m256i k_byteShuffleMask = _mm256_set_epi64x(0x0A0B090A08090708, 0x0506040503040203,
 282                                                         0x0809070806070506, 0x0304020301020001);
 283     const auto inDatContig = _mm256_shuffle_epi8(inLaneAlign, k_byteShuffleMask);
 284
 285     /// Logical shift left to set sign bit
 286     const __m256i k_slBits = _mm256_set_epi64x(0x0006000400020000, 0x0006000400020000,
 287                                                0x0006000400020000, 0x0006000400020000);
 288     const auto inSetSign = _mm256_sllv_epi16(inDatContig, k_slBits);
 289
 290     /// Mask to zero unwanted bits
 291     const __m256i k_expMask = _mm256_set1_epi16(0xFFC0);
 292     return _mm256_and_si256(inSetSign, k_expMask);
 293   }
 294
 295
 296   /// Unpack compressed 12 bit data in network byte order
 297   /// This unpacking function is for 256b registers
 298   inline __m256i
 299   networkByteUnpack12b256(const uint8_t* inData)
 300   {
 301     /// Align chunks of compressed bytes into lanes to allow for expansion
 302     const __m256i* rawDataIn = reinterpret_cast<const __m256i*>(inData);
 303     const auto k_expPerm = _mm256_set_epi32(6, 5, 4, 3, 3, 2, 1, 0);
 304     const auto inLaneAlign = _mm256_permutexvar_epi32(k_expPerm, *rawDataIn);
 305
 306     /// Byte shuffle to get all bits for each sample into 16b chunks
 307     /// For 12b mantissa all lanes post-permute are aligned and require same shuffle offset
 308     const __m256i k_byteShuffleMask = _mm256_set_epi64x(0x0A0B090A07080607, 0x0405030401020001,
 309                                                         0x0A0B090A07080607, 0x0405030401020001);
 310     const auto inDatContig = _mm256_shuffle_epi8(inLaneAlign, k_byteShuffleMask);
 311
 312     /// Logical shift left to set sign bit
 313     const __m256i k_slBits = _mm256_set_epi64x(0x0004000000040000, 0x0004000000040000,
 314                                                0x0004000000040000, 0x0004000000040000);
 315     const auto inSetSign = _mm256_sllv_epi16(inDatContig, k_slBits);
 316
 317     /// Mask to zero unwanted bits
 318     const __m256i k_expMask = _mm256_set1_epi16(0xFFF0);
 319     return _mm256_and_si256(inSetSign, k_expMask);
 320   }
 321
 322
 323
 324   /// Pack compressed 9 bit data in network byte order
 325   inline __m512i
 326   networkBytePack9bSnc(const __m512i compData)
 327   {
 328     /// Logical shift left to align network order byte parts
 329     const __m512i k_shiftLeft = _mm512_set_epi64(0x0000000100020003, 0x0004000500060007,
 330                                                  0x0000000100020003, 0x0004000500060007,
 331                                                  0x0000000100020003, 0x0004000500060007,
 332                                                  0x0000000100020003, 0x0004000500060007);
 333     const auto compDataPacked = _mm512_sllv_epi16(compData, k_shiftLeft);
 334
 335     /// First epi8 permute of even indexed samples
 336     const __m512i k_byteShuffleMask1 = _mm512_set_epi64(0x0000000000000000, 0x0000000000000000,
 337                                                         0x0000000000000000, 0x00000000003C3D38,
 338                                                         0x3934353031002C2D, 0x282924252021001C,
 339                                                         0x1D18191415101100, 0x0C0D080904050001);
 340     constexpr uint64_t k_byteMask1 = 0x00000007FBFDFEFF;
 341     const auto compDataShuff1 = _mm512_maskz_permutexvar_epi8(k_byteMask1, k_byteShuffleMask1, compDataPacked);
 342
 343     /// Second epi8 permute of odd indexed samples
 344     const __m512i k_byteShuffleMask2 = _mm512_set_epi64(0x0000000000000000, 0x0000000000000000,
 345                                                         0x0000000000000000, 0x000000003E3F3A3B,
 346                                                         0x36373233002E2F2A, 0x2B26272223001E1F,
 347                                                         0x1A1B16171213000E, 0x0F0A0B0607020300);
 348     constexpr uint64_t k_byteMask2 = 0x0000000FF7FBFDFE;
 349     auto compDataShuff2 = _mm512_maskz_permutexvar_epi8(k_byteMask2, k_byteShuffleMask2, compDataPacked);
 350
 351     /// Ternary blend of the two shuffled results
 352     const __m512i k_ternLogSelect = _mm512_set_epi64(0x0000000000000000, 0x0000000000000000,
 353                                                      0x0000000000000000, 0x00000000FF01FC07,
 354                                                      0xF01FC07F00FF01FC, 0x07F01FC07F00FF01,
 355                                                      0xFC07F01FC07F00FF, 0x01FC07F01FC07F00);
 356     return _mm512_ternarylogic_epi64(compDataShuff1, compDataShuff2, k_ternLogSelect, 0xd8);
 357   }
 358
 359
 360   /// Pack compressed 10 bit data in network byte order
 361   inline __m512i
 362   networkBytePack10bSnc(const __m512i compData)
 363   {
 364     /// Logical shift left to align network order byte parts
 365     const __m512i k_shiftLeft = _mm512_set_epi64(0x0000000200040006, 0x0000000200040006,
 366                                                  0x0000000200040006, 0x0000000200040006,
 367                                                  0x0000000200040006, 0x0000000200040006,
 368                                                  0x0000000200040006, 0x0000000200040006);
 369     const auto compDataPacked = _mm512_sllv_epi16(compData, k_shiftLeft);
 370
 371     /// First epi8 shuffle of even indexed samples
 372     const __m512i k_byteShuffleMask1 = _mm512_set_epi64(0x0000000000000000, 0x0000000000000000,
 373                                                         0x0000000000000000, 0x003C3D3839003435,
 374                                                         0x3031002C2D282900, 0x24252021001C1D18,
 375                                                         0x190014151011000C, 0x0D08090004050001);
 376     constexpr uint64_t k_byteMask1 = 0x0000007BDEF7BDEF;
 377     const auto compDataShuff1 = _mm512_maskz_permutexvar_epi8(k_byteMask1, k_byteShuffleMask1, compDataPacked);
 378
 379     /// Second epi8 shuffle of odd indexed samples
 380     const __m512i k_byteShuffleMask2 = _mm512_set_epi64(0x0000000000000000, 0x0000000000000000,
 381                                                         0x0000000000000000, 0x3E3F3A3B00363732,
 382                                                         0x33002E2F2A2B0026, 0x272223001E1F1A1B,
 383                                                         0x0016171213000E0F, 0x0A0B000607020300);
 384     constexpr uint64_t k_byteMask2 = 0x000000F7BDEF7BDE;
 385     auto compDataShuff2 = _mm512_maskz_permutexvar_epi8(k_byteMask2, k_byteShuffleMask2, compDataPacked);
 386
 387     /// Ternary blend of the two shuffled results
 388     const __m512i k_ternLogSelect = _mm512_set_epi64(0x0000000000000000, 0x0000000000000000,
 389                                                      0x0000000000000000, 0xFF03F03F00FF03F0,
 390                                                      0x3F00FF03F03F00FF, 0x03F03F00FF03F03F,
 391                                                      0x00FF03F03F00FF03, 0xF03F00FF03F03F00);
 392     return _mm512_ternarylogic_epi64(compDataShuff1, compDataShuff2, k_ternLogSelect, 0xd8);
 393   }
 394
 395
 396   inline __m512i
 397   networkBytePack12bSnc(const __m512i compData)
 398   {
 399     /// Logical shift left to align network order byte parts
 400     const __m512i k_shiftLeft = _mm512_set_epi64(0x0000000400000004, 0x0000000400000004,
 401                                                  0x0000000400000004, 0x0000000400000004,
 402                                                  0x0000000400000004, 0x0000000400000004,
 403                                                  0x0000000400000004, 0x0000000400000004);
 404     const auto compDataPacked = _mm512_sllv_epi16(compData, k_shiftLeft);
 405
 406     /// First epi8 shuffle of even indexed samples
 407     const __m512i k_byteShuffleMask1 = _mm512_set_epi64(0x0000000000000000, 0x0000000000000000,
 408                                                         0x003C3D0038390034, 0x35003031002C2D00,
 409                                                         0x2829002425002021, 0x001C1D0018190014,
 410                                                         0x15001011000C0D00, 0x0809000405000001);
 411     constexpr uint64_t k_byteMask1 = 0x00006DB6DB6DB6DB;
 412     const auto compDataShuff1 = _mm512_maskz_permutexvar_epi8(k_byteMask1, k_byteShuffleMask1, compDataPacked);
 413
 414     /// Second epi8 shuffle of odd indexed samples
 415     const __m512i k_byteShuffleMask2 = _mm512_set_epi64(0x0000000000000000, 0x0000000000000000,
 416                                                         0x3E3F003A3B003637, 0x003233002E2F002A,
 417                                                         0x2B00262700222300, 0x1E1F001A1B001617,
 418                                                         0x001213000E0F000A, 0x0B00060700020300);
 419     constexpr uint64_t k_byteMask2 = 0x0000DB6DB6DB6DB6;
 420     auto compDataShuff2 = _mm512_maskz_permutexvar_epi8(k_byteMask2, k_byteShuffleMask2, compDataPacked);
 421
 422     /// Ternary blend of the two shuffled results
 423     const __m512i k_ternLogSelect = _mm512_set_epi64(0x0000000000000000, 0x0000000000000000,
 424                                                      0xFF0F00FF0F00FF0F, 0x00FF0F00FF0F00FF,
 425                                                      0x0F00FF0F00FF0F00, 0xFF0F00FF0F00FF0F,
 426                                                      0x00FF0F00FF0F00FF, 0x0F00FF0F00FF0F00);
 427     return _mm512_ternarylogic_epi64(compDataShuff1, compDataShuff2, k_ternLogSelect, 0xd8);
 428   }
 429
 430
 431   /// Pack compressed 9 bit data in network byte order
 432   /// This version is specific to the c-plane 8 antenna case, where 2 compression blocks
 433   /// are handled in one register.
 434   inline __m512i
 435   networkBytePack9bSncB(const __m512i compData)
 436   {
 437     /// Logical shift left to align network order byte parts
 438     const __m512i k_shiftLeft = _mm512_set_epi64(0x0000000100020003, 0x0004000500060007,
 439                                                  0x0000000100020003, 0x0004000500060007,
 440                                                  0x0000000100020003, 0x0004000500060007,
 441                                                  0x0000000100020003, 0x0004000500060007);
 442     const auto compDataPacked = _mm512_sllv_epi16(compData, k_shiftLeft);
 443
 444     /// First epi8 permute of even indexed samples
 445     const __m512i k_byteShuffleMask1 = _mm512_set_epi64(0x0000000000000000, 0x000000000000003C,
 446                                                         0x3D38393435303100, 0x2C2D282924252021,
 447                                                         0x0000000000000000, 0x000000000000001C,
 448                                                         0x1D18191415101100, 0x0C0D080904050001);
 449     constexpr uint64_t k_byteMask1 = 0x0001FEFF0001FEFF;
 450     const auto compDataShuff1 = _mm512_maskz_permutexvar_epi8(k_byteMask1, k_byteShuffleMask1, compDataPacked);
 451
 452     /// Second epi8 permute of odd indexed samples
 453     const __m512i k_byteShuffleMask2 = _mm512_set_epi64(0x0000000000000000, 0x0000000000003E3F,
 454                                                         0x3A3B36373233002E, 0x2F2A2B2627222300,
 455                                                         0x0000000000000000, 0x0000000000001E1F,
 456                                                         0x1A1B16171213000E, 0x0F0A0B0607020300);
 457     constexpr uint64_t k_byteMask2 = 0x0003FDFE0003FDFE;
 458     auto compDataShuff2 = _mm512_maskz_permutexvar_epi8(k_byteMask2, k_byteShuffleMask2, compDataPacked);
 459
 460     /// Ternary blend of the two shuffled results
 461     const __m512i k_ternLogSelect = _mm512_set_epi64(0x0000000000000000, 0x000000000000FF01,
 462                                                      0xFC07F01FC07F00FF, 0x01FC07F01FC07F00,
 463                                                      0x0000000000000000, 0x000000000000FF01,
 464                                                      0xFC07F01FC07F00FF, 0x01FC07F01FC07F00);
 465     return _mm512_ternarylogic_epi64(compDataShuff1, compDataShuff2, k_ternLogSelect, 0xd8);
 466   }
 467
 468
 469   /// Pack compressed 10 bit data in network byte order
 470   /// This version is specific to the c-plane 8 antenna case, where 2 compression blocks
 471   /// are handled in one register.
 472   inline __m512i
 473   networkBytePack10bSncB(const __m512i compData)
 474   {
 475     /// Logical shift left to align network order byte parts
 476     const __m512i k_shiftLeft = _mm512_set_epi64(0x0000000200040006, 0x0000000200040006,
 477                                                  0x0000000200040006, 0x0000000200040006,
 478                                                  0x0000000200040006, 0x0000000200040006,
 479                                                  0x0000000200040006, 0x0000000200040006);
 480     const auto compDataPacked = _mm512_sllv_epi16(compData, k_shiftLeft);
 481
 482     /// First epi8 shuffle of even indexed samples
 483     const __m512i k_byteShuffleMask1 = _mm512_set_epi64(0x0000000000000000, 0x00000000003C3D38,
 484                                                         0x390034353031002C, 0x2D28290024252021,
 485                                                         0x0000000000000000, 0x00000000001C1D18,
 486                                                         0x190014151011000C, 0x0D08090004050001);
 487     constexpr uint64_t k_byteMask1 = 0x0007BDEF0007BDEF;
 488     const auto compDataShuff1 = _mm512_maskz_permutexvar_epi8(k_byteMask1, k_byteShuffleMask1, compDataPacked);
 489
 490     /// Second epi8 shuffle of odd indexed samples
 491     const __m512i k_byteShuffleMask2 = _mm512_set_epi64(0x0000000000000000, 0x000000003E3F3A3B,
 492                                                         0x0036373233002E2F, 0x2A2B002627222300,
 493                                                         0x0000000000000000, 0x000000001E1F1A1B,
 494                                                         0x0016171213000E0F, 0x0A0B000607020300);
 495     constexpr uint64_t k_byteMask2 = 0x000F7BDE000F7BDE;
 496     auto compDataShuff2 = _mm512_maskz_permutexvar_epi8(k_byteMask2, k_byteShuffleMask2, compDataPacked);
 497
 498     /// Ternary blend of the two shuffled results
 499     const __m512i k_ternLogSelect = _mm512_set_epi64(0x0000000000000000, 0x00000000FF03F03F,
 500                                                      0x00FF03F03F00FF03, 0xF03F00FF03F03F00,
 501                                                      0x0000000000000000, 0x00000000FF03F03F,
 502                                                      0x00FF03F03F00FF03, 0xF03F00FF03F03F00);
 503     return _mm512_ternarylogic_epi64(compDataShuff1, compDataShuff2, k_ternLogSelect, 0xd8);
 504   }
 505
 506
 507   /// Pack compressed 12 bit data in network byte order
 508   /// This version is specific to the c-plane 8 antenna case, where 2 compression blocks
 509   /// are handled in one register.
 510   inline __m512i
 511   networkBytePack12bSncB(const __m512i compData)
 512   {
 513     /// Logical shift left to align network order byte parts
 514     const __m512i k_shiftLeft = _mm512_set_epi64(0x0000000400000004, 0x0000000400000004,
 515                                                  0x0000000400000004, 0x0000000400000004,
 516                                                  0x0000000400000004, 0x0000000400000004,
 517                                                  0x0000000400000004, 0x0000000400000004);
 518     const auto compDataPacked = _mm512_sllv_epi16(compData, k_shiftLeft);
 519
 520     /// First epi8 shuffle of even indexed samples
 521     const __m512i k_byteShuffleMask1 = _mm512_set_epi64(0x0000000000000000, 0x003C3D0038390034,
 522                                                         0x35003031002C2D00, 0x2829002425002021,
 523                                                         0x0000000000000000, 0x001C1D0018190014,
 524                                                         0x15001011000C0D00, 0x0809000405000001);
 525     constexpr uint64_t k_byteMask1 = 0x006DB6DB006DB6DB;
 526     const auto compDataShuff1 = _mm512_maskz_permutexvar_epi8(k_byteMask1, k_byteShuffleMask1, compDataPacked);
 527
 528     /// Second epi8 shuffle of odd indexed samples
 529     const __m512i k_byteShuffleMask2 = _mm512_set_epi64(0x0000000000000000, 0x3E3F003A3B003637,
 530                                                         0x003233002E2F002A, 0x2B00262700222300,
 531                                                         0x0000000000000000, 0x1E1F001A1B001617,
 532                                                         0x001213000E0F000A, 0x0B00060700020300);
 533     constexpr uint64_t k_byteMask2 = 0x00DB6DB600DB6DB6;
 534     auto compDataShuff2 = _mm512_maskz_permutexvar_epi8(k_byteMask2, k_byteShuffleMask2, compDataPacked);
 535
 536     /// Ternary blend of the two shuffled results
 537     const __m512i k_ternLogSelect = _mm512_set_epi64(0x0000000000000000, 0xFF0F00FF0F00FF0F,
 538                                                      0x00FF0F00FF0F00FF, 0x0F00FF0F00FF0F00,
 539                                                      0x0000000000000000, 0xFF0F00FF0F00FF0F,
 540                                                      0x00FF0F00FF0F00FF, 0x0F00FF0F00FF0F00);
 541     return _mm512_ternarylogic_epi64(compDataShuff1, compDataShuff2, k_ternLogSelect, 0xd8);
 542   }
 543
 544
 545
 546   /// Unpack compressed 9 bit data in network byte order
 547   inline __m512i
 548   networkByteUnpack9bSnc(const uint8_t* inData)
 549   {
 550     /// Align chunks of compressed bytes into lanes to allow for expansion
 551     const __m512i* rawDataIn = reinterpret_cast<const __m512i*>(inData);
 552
 553     /// Byte shuffle to get all bits for each sample into 16b chunks
 554     /// Due to previous permute to get chunks of bytes into each lane, there is
 555     /// a different shuffle offset in each lane
 556     const __m512i k_byteShuffleMask = _mm512_set_epi64(0x2223212220211F20, 0x1E1F1D1E1C1D1B1C,
 557                                                        0x191A181917181617, 0x1516141513141213,
 558                                                        0x10110F100E0F0D0E, 0x0C0D0B0C0A0B090A,
 559                                                        0x0708060705060405, 0x0304020301020001);
 560     constexpr uint64_t k_byteMask = 0xFFFFFFFFFFFFFFFF;
 561     const auto inDataContig = _mm512_maskz_permutexvar_epi8(k_byteMask, k_byteShuffleMask, *rawDataIn);
 562
 563     /// Logical shift left to set sign bit
 564     const __m512i k_slBits = _mm512_set_epi64(0x0007000600050004, 0x0003000200010000,
 565                                               0x0007000600050004, 0x0003000200010000,
 566                                               0x0007000600050004, 0x0003000200010000,
 567                                               0x0007000600050004, 0x0003000200010000);
 568     const auto inSetSign = _mm512_sllv_epi16(inDataContig, k_slBits);
 569
 570     /// Mask to zero unwanted bits
 571     const __m512i k_expMask = _mm512_set1_epi16(0xFF80);
 572     return _mm512_and_epi64(inSetSign, k_expMask);
 573   }
 574
 575
 576   /// Unpack compressed 10 bit data in network byte order
 577   inline __m512i
 578   networkByteUnpack10bSnc(const uint8_t* inData)
 579   {
 580     /// Align chunks of compressed bytes into lanes to allow for expansion
 581     const __m512i* rawDataIn = reinterpret_cast<const __m512i*>(inData);
 582
 583     /// Byte shuffle to get all bits for each sample into 16b chunks
 584     /// Due to previous permute to get chunks of bytes into each lane, lanes
 585     /// 0 and 2 happen to be aligned, but lane 1 is offset by 2 bytes
 586     const __m512i k_byteShuffleMask = _mm512_set_epi64(0x2627252624252324, 0x212220211F201E1F,
 587                                                        0x1C1D1B1C1A1B191A, 0x1718161715161415,
 588                                                        0x1213111210110F10, 0x0D0E0C0D0B0C0A0B,
 589                                                        0x0809070806070506, 0x0304020301020001);
 590     constexpr uint64_t k_byteMask = 0xFFFFFFFFFFFFFFFF;
 591     const auto inDataContig = _mm512_maskz_permutexvar_epi8(k_byteMask, k_byteShuffleMask, *rawDataIn);
 592
 593     /// Logical shift left to set sign bit
 594     const __m512i k_slBits = _mm512_set_epi64(0x0006000400020000, 0x0006000400020000,
 595                                               0x0006000400020000, 0x0006000400020000,
 596                                               0x0006000400020000, 0x0006000400020000,
 597                                               0x0006000400020000, 0x0006000400020000);
 598     const auto inSetSign = _mm512_sllv_epi16(inDataContig, k_slBits);
 599
 600     /// Mask to zero unwanted bits
 601     const __m512i k_expMask = _mm512_set1_epi16(0xFFC0);
 602     return _mm512_and_epi64(inSetSign, k_expMask);
 603   }
 604
 605
 606   /// Unpack compressed 12 bit data in network byte order
 607   inline __m512i
 608   networkByteUnpack12bSnc(const uint8_t* inData)
 609   {
 610     /// Align chunks of compressed bytes into lanes to allow for expansion
 611     const __m512i* rawDataIn = reinterpret_cast<const __m512i*>(inData);
 612
 613     /// Byte shuffle to get all bits for each sample into 16b chunks
 614     /// For 12b mantissa all lanes post-permute are aligned and require same shuffle offset
 615     const __m512i k_byteShuffleMask = _mm512_set_epi64(0x2E2F2D2E2B2C2A2B, 0x2829272825262425,
 616                                                        0x222321221F201E1F, 0x1C1D1B1C191A1819,
 617                                                        0x1617151613141213, 0x10110F100D0E0C0D,
 618                                                        0x0A0B090A07080607, 0x0405030401020001);
 619     constexpr uint64_t k_byteMask = 0xFFFFFFFFFFFFFFFF;
 620     const auto inDataContig = _mm512_maskz_permutexvar_epi8(k_byteMask, k_byteShuffleMask, *rawDataIn);
 621
 622     /// Logical shift left to set sign bit
 623     const __m512i k_slBits = _mm512_set_epi64(0x0004000000040000, 0x0004000000040000,
 624                                               0x0004000000040000, 0x0004000000040000,
 625                                               0x0004000000040000, 0x0004000000040000,
 626                                               0x0004000000040000, 0x0004000000040000);
 627     const auto inSetSign = _mm512_sllv_epi16(inDataContig, k_slBits);
 628
 629     /// Mask to zero unwanted bits
 630     const __m512i k_expMask = _mm512_set1_epi16(0xFFF0);
 631     return _mm512_and_epi64(inSetSign, k_expMask);
 632   }
 633
 634
 635   /// Unpack compressed 9 bit data in network byte order
 636   /// This unpacking function is for 256b registers
 637   inline __m256i
 638   networkByteUnpack9b256Snc(const uint8_t* inData)
 639   {
 640     /// Align chunks of compressed bytes into lanes to allow for expansion
 641     const __m256i* rawDataIn = reinterpret_cast<const __m256i*>(inData);
 642
 643     /// Byte shuffle to get all bits for each sample into 16b chunks
 644     /// Due to previous permute to get chunks of bytes into each lane, there is
 645     /// a different shuffle offset in each lane
 646     const __m256i k_byteShuffleMask = _mm256_set_epi64x(0x10110F100E0F0D0E, 0x0C0D0B0C0A0B090A,
 647                                                         0x0708060705060405, 0x0304020301020001);
 648     constexpr uint32_t k_byteMask = 0xFFFFFFFF;
 649     const auto inDataContig = _mm256_maskz_permutexvar_epi8(k_byteMask, k_byteShuffleMask, *rawDataIn);
 650
 651     /// Logical shift left to set sign bit
 652     const __m256i k_slBits = _mm256_set_epi64x(0x0007000600050004, 0x0003000200010000,
 653                                                0x0007000600050004, 0x0003000200010000);
 654     const auto inSetSign = _mm256_sllv_epi16(inDataContig, k_slBits);
 655
 656     /// Mask to zero unwanted bits
 657     const __m256i k_expMask = _mm256_set1_epi16(0xFF80);
 658     return _mm256_and_si256(inSetSign, k_expMask);
 659   }
 660
 661
 662   /// Unpack compressed 10 bit data in network byte order
 663   /// This unpacking function is for 256b registers
 664   inline __m256i
 665   networkByteUnpack10b256Snc(const uint8_t* inData)
 666   {
 667     /// Align chunks of compressed bytes into lanes to allow for expansion
 668     const __m256i* rawDataIn = reinterpret_cast<const __m256i*>(inData);
 669
 670     /// Byte shuffle to get all bits for each sample into 16b chunks
 671     /// Due to previous permute to get chunks of bytes into each lane, lanes
 672     /// 0 and 2 happen to be aligned, but lane 1 is offset by 2 bytes
 673     const __m256i k_byteShuffleMask = _mm256_set_epi64x(0x1213111210110F10, 0x0D0E0C0D0B0C0A0B,
 674                                                         0x0809070806070506, 0x0304020301020001);
 675     constexpr uint32_t k_byteMask = 0xFFFFFFFF;
 676     const auto inDataContig = _mm256_maskz_permutexvar_epi8(k_byteMask, k_byteShuffleMask, *rawDataIn);
 677
 678     /// Logical shift left to set sign bit
 679     const __m256i k_slBits = _mm256_set_epi64x(0x0006000400020000, 0x0006000400020000,
 680                                                0x0006000400020000, 0x0006000400020000);
 681     const auto inSetSign = _mm256_sllv_epi16(inDataContig, k_slBits);
 682
 683     /// Mask to zero unwanted bits
 684     const __m256i k_expMask = _mm256_set1_epi16(0xFFC0);
 685     return _mm256_and_si256(inSetSign, k_expMask);
 686   }
 687
 688
 689   /// Unpack compressed 12 bit data in network byte order
 690   /// This unpacking function is for 256b registers
 691   inline __m256i
 692   networkByteUnpack12b256Snc(const uint8_t* inData)
 693   {
 694     /// Align chunks of compressed bytes into lanes to allow for expansion
 695     const __m256i* rawDataIn = reinterpret_cast<const __m256i*>(inData);
 696
 697     /// Byte shuffle to get all bits for each sample into 16b chunks
 698     /// For 12b mantissa all lanes post-permute are aligned and require same shuffle offset
 699     const __m256i k_byteShuffleMask = _mm256_set_epi64x(0x1617151613141213, 0x10110F100D0E0C0D,
 700                                                         0x0A0B090A07080607, 0x0405030401020001);
 701     constexpr uint32_t k_byteMask = 0xFFFFFFFF;
 702     const auto inDataContig = _mm256_maskz_permutexvar_epi8(k_byteMask, k_byteShuffleMask, *rawDataIn);
 703
 704     /// Logical shift left to set sign bit
 705     const __m256i k_slBits = _mm256_set_epi64x(0x0004000000040000, 0x0004000000040000,
 706                                                0x0004000000040000, 0x0004000000040000);
 707     const auto inSetSign = _mm256_sllv_epi16(inDataContig, k_slBits);
 708
 709     /// Mask to zero unwanted bits
 710     const __m256i k_expMask = _mm256_set1_epi16(0xFFF0);
 711     return _mm256_and_si256(inSetSign, k_expMask);
 712   }
 713
 714 }