ccv_nnc_8i_rowwise.c

Bug Summary

File:	nnc/ccv_nnc_8i_rowwise.c
Warning:	line 1924, column 12 Assigned value is garbage or undefined
Annotated Source Code

Press '?' to see keyboard shortcuts
Show analyzer invocation
clang -cc1 -cc1 -triple x86_64-unknown-linux-gnu -analyze -disable-free -clear-ast-before-backend -disable-llvm-verifier -discard-value-names -main-file-name ccv_nnc_8i_rowwise.c -analyzer-checker=core -analyzer-checker=apiModeling -analyzer-checker=unix -analyzer-checker=deadcode -analyzer-checker=security.insecureAPI.UncheckedReturn -analyzer-checker=security.insecureAPI.getpw -analyzer-checker=security.insecureAPI.gets -analyzer-checker=security.insecureAPI.mktemp -analyzer-checker=security.insecureAPI.mkstemp -analyzer-checker=security.insecureAPI.vfork -analyzer-checker=nullability.NullPassedToNonnull -analyzer-checker=nullability.NullReturnedFromNonnull -analyzer-output plist -w -setup-static-analyzer -mrelocation-model pic -pic-level 2 -pic-is-pie -mframe-pointer=none -fmath-errno -ffp-contract=on -fno-rounding-math -mconstructor-aliases -funwind-tables=2 -target-cpu x86-64 -target-feature +sse2 -tune-cpu generic -debugger-tuning=gdb -fdebug-compilation-dir=/home/liu/actions-runner/_work/ccv/ccv/lib/nnc -fcoverage-compilation-dir=/home/liu/actions-runner/_work/ccv/ccv/lib/nnc -resource-dir /usr/local/lib/clang/19 -I ../ -I /usr/local/cuda/include -D HAVE_CBLAS -D HAVE_LIBPNG -D HAVE_LIBJPEG -D HAVE_FFTW3 -D HAVE_PTHREAD -D HAVE_LIBLINEAR -D HAVE_TESSERACT -D HAVE_AVCODEC -D HAVE_AVFORMAT -D HAVE_AVUTIL -D HAVE_SWSCALE -D HAVE_SSE2 -D HAVE_GSL -D HAVE_CUDA -D HAVE_CUDNN -D HAVE_NCCL -D USE_SYSTEM_CUB -I /usr/local/include -internal-isystem /usr/local/lib/clang/19/include -internal-isystem /usr/local/include -internal-isystem /usr/lib/gcc/x86_64-linux-gnu/12/../../../../x86_64-linux-gnu/include -internal-externc-isystem /usr/include/x86_64-linux-gnu -internal-externc-isystem /include -internal-externc-isystem /usr/include -O3 -ferror-limit 19 -fgnuc-version=4.2.1 -fskip-odr-check-in-gmf -vectorize-loops -vectorize-slp -analyzer-output=html -faddrsig -D__GCC_HAVE_DWARF2_CFI_ASM=1 -o /home/liu/actions-runner/_work/ccv/ccv/_analyze/2026-10-21-184038-1771444-1 -x c ccv_nnc_8i_rowwise.c
1#include "ccv_nnc.h"
2#include "ccv_nnc_internal.h"
3#include "ccv_nnc_easy.h"
4#include <float.h>
5#include <limits.h>
6#include "ccv_nnc_8i_rowwise_packed_grids.inc"
7#ifdef USE_DISPATCH
8#include <dispatch/dispatch.h>
9#endif
10#ifdef HAVE_CUDA1
11#include "gpu/ccv_nnc_compat.h"
12#elif defined(HAVE_MPS)
13#include "mps/ccv_nnc_mps.h"
14#endif

16enum {
CCV_NNC_8I_ROWWISE_X_REFINEMENT_STEPS = 8,
CCV_NNC_8I_ROWWISE_IQ2_CANDIDATES = 16,
CCV_NNC_8I_ROWWISE_IQ3_CANDIDATES = 24,
20};

22static size_t _ccv_nnc_8i_rowwise_packed_scale_offset(const int format, const size_t input_length, const size_t row_length)
23{
assert(row_length > 0)((void) sizeof ((row_length > 0) ? 1 : 0), __extension__ (
{ if (row_length > 0) ; else __assert_fail ("row_length > 0"
, "ccv_nnc_8i_rowwise.c", 24, __extension__ __PRETTY_FUNCTION__
); }));
assert(input_length % row_length == 0)((void) sizeof ((input_length % row_length == 0) ? 1 : 0), __extension__
 ({ if (input_length % row_length == 0) ; else __assert_fail (
"input_length % row_length == 0", "ccv_nnc_8i_rowwise.c", 25,
 __extension__ __PRETTY_FUNCTION__); }));
const size_t row_count = input_length / row_length;
const size_t group_size = ccv_nnc_8i_rowwise_x_group_size(format);
const size_t groups_per_row = (row_length + group_size - 1) / group_size;
const size_t group_bits = ccv_nnc_8i_rowwise_x_group_bits(format);
const size_t payload_size = (row_count * groups_per_row * group_bits + 7) / 8;
return (payload_size + 127) & -128;
32}

34CCV_WARN_UNUSED(size_t)size_t __attribute__((warn_unused_result)) ccv_nnc_8i_rowwise_x_data_size(const int format, const int datatype, const size_t input_length, const size_t row_length)
35{
if (row_length == 0 || input_length % row_length != 0 ||
((format & CCV_NNC_QX_8I_ROWWISE_HADAMARD_256) && row_length % 256 != 0))
return 0;
assert(datatype == CCV_16F || datatype == CCV_16BF || datatype == CCV_32F || datatype == CCV_64F)((void) sizeof ((datatype == CCV_16F || datatype == CCV_16BF ||
 datatype == CCV_32F || datatype == CCV_64F) ? 1 : 0), __extension__
 ({ if (datatype == CCV_16F || datatype == CCV_16BF || datatype
 == CCV_32F || datatype == CCV_64F) ; else __assert_fail ("datatype == CCV_16F || datatype == CCV_16BF || datatype == CCV_32F || datatype == CCV_64F"
, "ccv_nnc_8i_rowwise.c", 39, __extension__ __PRETTY_FUNCTION__
); }));
assert(row_length > 0)((void) sizeof ((row_length > 0) ? 1 : 0), __extension__ (
{ if (row_length > 0) ; else __assert_fail ("row_length > 0"
, "ccv_nnc_8i_rowwise.c", 40, __extension__ __PRETTY_FUNCTION__
); }));
assert(input_length % row_length == 0)((void) sizeof ((input_length % row_length == 0) ? 1 : 0), __extension__
 ({ if (input_length % row_length == 0) ; else __assert_fail (
"input_length % row_length == 0", "ccv_nnc_8i_rowwise.c", 41,
 __extension__ __PRETTY_FUNCTION__); }));
const size_t row_count = input_length / row_length;
const size_t scale_offset = _ccv_nnc_8i_rowwise_packed_scale_offset(format, input_length, row_length);
return scale_offset + row_count * CCV_GET_DATA_TYPE_SIZE(datatype)_ccv_get_data_type_size[((datatype) & 0xFF000) >> 12
];
45}

47static void _ccv_nnc_8i_rowwise_packed_write_bits(uint8_t* const data, const size_t bit_offset, const uint32_t value, const int bits)
48{
int i;
for (i = 0; i < bits; i++)
if (value & (1u << i))
	data[(bit_offset + i) >> 3] |= (uint8_t)(1u << ((bit_offset + i) & 7));
53}

55static uint32_t _ccv_nnc_8i_rowwise_packed_read_bits(const uint8_t* const data, const size_t bit_offset, const int bits)
56{
uint32_t value = 0;
int i;
for (i = 0; i < bits; i++)
if (data[(bit_offset + i) >> 3] & (uint8_t)(1u << ((bit_offset + i) & 7)))
	value |= (1u << i);
return value;
63}

65static inline int _ccv_nnc_8i_rowwise_packed_sign_extend(const uint32_t value, const int bits)
66{
const uint32_t sign = 1u << (bits - 1);
return (value & sign) ? (int)value - (int)(1u << bits) : (int)value;
69}

71static inline int _ccv_nnc_8i_rowwise_packed_floor_div(const int numerator, const int denominator)
72{
assert(denominator > 0)((void) sizeof ((denominator > 0) ? 1 : 0), __extension__ (
{ if (denominator > 0) ; else __assert_fail ("denominator > 0"
, "ccv_nnc_8i_rowwise.c", 73, __extension__ __PRETTY_FUNCTION__
); }));
return numerator >= 0 ? numerator / denominator : -((-numerator + denominator - 1) / denominator);
75}

77static inline int _ccv_nnc_8i_rowwise_packed_ceil_div(const int numerator, const int denominator)
78{
assert(denominator > 0)((void) sizeof ((denominator > 0) ? 1 : 0), __extension__ (
{ if (denominator > 0) ; else __assert_fail ("denominator > 0"
, "ccv_nnc_8i_rowwise.c", 79, __extension__ __PRETTY_FUNCTION__
); }));
return numerator >= 0 ? (numerator + denominator - 1) / denominator : -((-numerator) / denominator);
81}

83static double _ccv_nnc_8i_rowwise_packed_stored_scale(const double scale, const int datatype)
84{
if (datatype == CCV_16F)
{
const float scale_f = (float)scale;
uint16_t scale_h;
float stored_scale;
ccv_float_to_half_precision(&scale_f, &scale_h, 1);
ccv_half_precision_to_float(&scale_h, &stored_scale, 1);
return stored_scale;
} else if (datatype == CCV_16BF) {
const float scale_f = (float)scale;
uint16_t scale_bf;
float stored_scale;
ccv_float_to_bfloat(&scale_f, &scale_bf, 1);
ccv_bfloat_to_float(&scale_bf, &stored_scale, 1);
return stored_scale;
} else if (datatype == CCV_32F)
return (float)scale;
return scale;
103}

105static void _ccv_nnc_8i_rowwise_packed_store_scale(uint8_t* const scales, const int datatype, const size_t i, const double scale)
106{
if (datatype == CCV_16F)
{
const float scale_f = (float)scale;
ccv_float_to_half_precision(&scale_f, (uint16_t*)scales + i, 1);
} else if (datatype == CCV_16BF) {
const float scale_f = (float)scale;
ccv_float_to_bfloat(&scale_f, (uint16_t*)scales + i, 1);
} else if (datatype == CCV_32F)
((float*)scales)[i] = (float)scale;
else
((double*)scales)[i] = scale;
118}

120static double _ccv_nnc_8i_rowwise_packed_load_scale(const uint8_t* const scales, const int datatype, const size_t i)
121{
if (datatype == CCV_16F)
{
float scale_f;
ccv_half_precision_to_float((const uint16_t*)scales + i, &scale_f, 1);
return scale_f;
} else if (datatype == CCV_16BF) {
float scale_f;
ccv_bfloat_to_float((const uint16_t*)scales + i, &scale_f, 1);
return scale_f;
} else if (datatype == CCV_32F)
return ((const float*)scales)[i];
return ((const double*)scales)[i];
134}

136static void _ccv_nnc_8i_rowwise_packed_read_row(const void* const input, const int datatype, const size_t row_start, const size_t row_length, const size_t padded_row_length, double* const row)
137{
size_t j;
if (datatype31.1
'datatype' is not equal to CCV_16F
 == CCV_16F)
32
←
Taking false branch→
{
const uint16_t* const f16 = (const uint16_t*)input + row_start;
for (j = 0; j < row_length; j++)
{
	float v;
	ccv_half_precision_to_float(f16 + j, &v, 1);
	row[j] = v;
}
} else if (datatype32.1
'datatype' is not equal to CCV_16BF
 == CCV_16BF) {
33
←
Taking false branch→
const uint16_t* const bf16 = (const uint16_t*)input + row_start;
for (j = 0; j < row_length; j++)
{
	float v;
	ccv_bfloat_to_float(bf16 + j, &v, 1);
	row[j] = v;
}
} else if (datatype33.1
'datatype' is not equal to CCV_32F
 == CCV_32F) {
34
←
Taking false branch→
const float* const f32 = (const float*)input + row_start;
for (j = 0; j < row_length; j++)
	row[j] = f32[j];
} else {
const double* const f64 = (const double*)input + row_start;
for (j = 0; j34.1
'j' is < 'row_length'
 < row_length; j++)
35
←
Loop condition is true.  Entering loop body→
36
←
Assuming 'j' is >= 'row_length'→
37
←
Loop condition is false. Execution continues on line 165→
	row[j] = f64[j];
}
for (; j < padded_row_length; j++)
38
←
Assuming 'j' is >= 'padded_row_length'→
39
←
Loop condition is false. Execution continues on line 165→
row[j] = 0;
167}

169static void _ccv_nnc_8i_rowwise_packed_write_value(void* const output, const int datatype, const size_t j, const double v)
170{
if (datatype == CCV_16F)
{
const float v_f = (float)v;
ccv_float_to_half_precision(&v_f, (uint16_t*)output + j, 1);
} else if (datatype == CCV_16BF) {
const float v_f = (float)v;
ccv_float_to_bfloat(&v_f, (uint16_t*)output + j, 1);
} else if (datatype == CCV_32F)
((float*)output)[j] = (float)v;
else
((double*)output)[j] = v;
182}

184static inline double _ccv_nnc_8i_rowwise_weight(const float* const imatrix, const size_t j)
185{
return imatrix ? ccv_max((double)imatrix[j], 0.)({ typeof ((double)imatrix[j]) _a = ((double)imatrix[j]); typeof
 (0.) _b = (0.); (_a > _b) ? _a : _b; }) : 1.;
187}

189static inline int _ccv_nnc_8i_rowwise_imatrix_is_valid(const float* const imatrix, const size_t imatrix_length, const size_t row_length, const size_t row_count)
190{
if (!imatrix)
return 1;
if (imatrix_length < row_length || imatrix_length % row_length != 0)
return 0;
const size_t imatrix_slices = imatrix_length / row_length;
return imatrix_slices > 0 && row_count % imatrix_slices == 0;
197}

199static inline const float* _ccv_nnc_8i_rowwise_imatrix_for_row(const float* const imatrix, const size_t imatrix_length, const size_t row_length, const size_t row_count, const size_t row_idx)
200{
if (!imatrix)
return 0;
const size_t imatrix_slices = imatrix_length / row_length;
if (imatrix_slices == 1)
return imatrix;
const size_t rows_per_slice = row_count / imatrix_slices;
return imatrix + (row_idx / rows_per_slice) * row_length;
208}

210typedef struct {
int q[32];
int q8[32];
int m;
int b;
int z;
int scale;
int grid[4];
uint32_t signs;
219} ccv_nnc_8i_rowwise_packed_group_t;

221static const int ccv_nnc_8i_rowwise_packed_iq1s_scales[8] = {1, 2, 3, 5, 7, 9, 12, 16};
222static const int ccv_nnc_8i_rowwise_packed_iq1_powers[8] = {1, 3, 9, 27, 81, 243, 729, 2187};

224typedef struct {
int lane;
int trit[3];
double cost[3];
228} ccv_nnc_8i_rowwise_iq1_lane_t;

230typedef struct {
ccv_nnc_8i_rowwise_iq1_lane_t lanes[8];
const int16_t* map;
double best;
int index;
235} ccv_nnc_8i_rowwise_iq1_search_t;

237static void _ccv_nnc_8i_rowwise_packed_iq1_search(ccv_nnc_8i_rowwise_iq1_search_t* const search, const int depth, const int code, const double cost)
238{
if (cost >= search->best)
return;
if (depth == 8)
{
if (search->map[code] >= 0)
{
	search->best = cost;
	search->index = search->map[code];
}
return;
}
const ccv_nnc_8i_rowwise_iq1_lane_t* const lane = search->lanes + depth;
int k;
for (k = 0; k < 3; k++)
{
const double next = cost + lane->cost[k];
if (next >= search->best)
	break;
_ccv_nnc_8i_rowwise_packed_iq1_search(search, depth + 1, code + lane->trit[k] * ccv_nnc_8i_rowwise_packed_iq1_powers[lane->lane], next);
}
259}

261static double _ccv_nnc_8i_rowwise_packed_iq1_nearest(const double* const y, const double* const w, const int m, const int shift, const int16_t* const map, int* const index)
262{
const int levels[3] = {ccv_max((-8 + shift) * m, -127)({ typeof ((-8 + shift) * m) _a = ((-8 + shift) * m); typeof (
-127) _b = (-127); (_a > _b) ? _a : _b; }), shift * m, ccv_min((8 + shift) * m, 127)({ typeof ((8 + shift) * m) _a = ((8 + shift) * m); typeof (127
) _b = (127); (_a < _b) ? _a : _b; })};
ccv_nnc_8i_rowwise_iq1_search_t search;
search.map = map;
double baseline = 0;
int natural_code = 0;
int j, k;
for (j = 0; j < 8; j++)
{
ccv_nnc_8i_rowwise_iq1_lane_t* const lane = search.lanes + j;
lane->lane = j;
for (k = 0; k < 3; k++)
{
	const double d = levels[k] - y[j];
	lane->trit[k] = k;
	lane->cost[k] = w[j] * d * d;
}
for (k = 1; k < 3; k++)
{
	int t;
	for (t = k; t > 0 && lane->cost[t] < lane->cost[t - 1]; t--)
	{
		const double cost = lane->cost[t];
		lane->cost[t] = lane->cost[t - 1];
		lane->cost[t - 1] = cost;
		const int trit = lane->trit[t];
		lane->trit[t] = lane->trit[t - 1];
		lane->trit[t - 1] = trit;
	}
}
natural_code += lane->trit[0] * ccv_nnc_8i_rowwise_packed_iq1_powers[j];
const double minimum = lane->cost[0];
baseline += minimum;
for (k = 0; k < 3; k++)
	lane->cost[k] -= minimum;
}
if (map[natural_code] >= 0)
{
*index = map[natural_code];
return baseline;
}
search.best = DBL_MAX1.7976931348623157e+308;
search.index = -1;
for (j = 0; j < 8; j++)
for (k = 1; k < 3; k++)
{
	const ccv_nnc_8i_rowwise_iq1_lane_t* const lane = search.lanes + j;
	const int code = natural_code + (lane->trit[k] - lane->trit[0]) * ccv_nnc_8i_rowwise_packed_iq1_powers[j];
	if (map[code] >= 0 && lane->cost[k] < search.best)
	{
		search.best = lane->cost[k];
		search.index = map[code];
	}
}
// Independent lane costs bound the exact search; put cheap changes last.
for (j = 1; j < 8; j++)
for (k = j; k > 0 && search.lanes[k].cost[1] > search.lanes[k - 1].cost[1]; k--)
{
	const ccv_nnc_8i_rowwise_iq1_lane_t lane = search.lanes[k];
	search.lanes[k] = search.lanes[k - 1];
	search.lanes[k - 1] = lane;
}
_ccv_nnc_8i_rowwise_packed_iq1_search(&search, 0, 0, 0);
assert(search.index >= 0)((void) sizeof ((search.index >= 0) ? 1 : 0), __extension__
 ({ if (search.index >= 0) ; else __assert_fail ("search.index >= 0"
, "ccv_nnc_8i_rowwise.c", 325, __extension__ __PRETTY_FUNCTION__
); }));
*index = search.index;
return baseline + search.best;
328}

330static void _ccv_nnc_8i_rowwise_packed_quant_iq1_m(const double* const y, const double* const w, const int16_t* const map, ccv_nnc_8i_rowwise_packed_group_t* const group)
331{
double best_sse = DBL_MAX1.7976931348623157e+308;
int m, sg, j;
for (m = 1; m <= 16; m++)
{
double sse = 0;
int grid[2];
uint32_t shifts = 0;
for (sg = 0; sg < 2; sg++)
{
	int index[2];
	const double plus = _ccv_nnc_8i_rowwise_packed_iq1_nearest(y + sg * 8, w + sg * 8, m, 1, map, index);
	const double minus = _ccv_nnc_8i_rowwise_packed_iq1_nearest(y + sg * 8, w + sg * 8, m, -1, map, index + 1);
	const int shift = minus < plus;
	sse += shift ? minus : plus;
	grid[sg] = index[shift];
	shifts |= (uint32_t)shift << sg;
}
if (sse < best_sse)
{
	best_sse = sse;
	group->scale = m;
	group->signs = shifts;
	memcpy(group->grid, grid, sizeof(grid));
}
}
for (sg = 0; sg < 2; sg++)
for (j = 0; j < 8; j++)
{
	const int g = (int)((ccv_nnc_8i_rowwise_packed_iq1_grid[group->grid[sg]] >> (2 * j)) & 3) - 1;
	const int shift = (group->signs & (1u << sg)) ? -1 : 1;
	group->q8[sg * 8 + j] = ccv_clamp(group->scale * (8 * g + shift), -127, 127)({ typeof (-127) _a = (-127); typeof (127) _b = (127); typeof
 (group->scale * (8 * g + shift)) _x = (group->scale * (
* g + shift)); (_x < _a) ? _a : ((_x > _b) ? _b : _x)
; });
}
364}

366static void _ccv_nnc_8i_rowwise_packed_quant_iq1_s(const double* const y, const double* const w, const int16_t* const map, ccv_nnc_8i_rowwise_packed_group_t* const group)
367{
double best_sse = DBL_MAX1.7976931348623157e+308;
int sc, sg, j;
for (sc = 0; sc < 8; sc++)
{
const int m = ccv_nnc_8i_rowwise_packed_iq1s_scales[sc];
double costs[4][2];
int indices[4][2];
for (sg = 0; sg < 4; sg++)
	for (j = 0; j < 2; j++)
		costs[sg][j] = _ccv_nnc_8i_rowwise_packed_iq1_nearest(y + sg * 8, w + sg * 8, m, j ? -1 : 1, map, &indices[sg][j]);
int mask;
for (mask = 0; mask < 16; mask++)
{
	if ((mask ^ (mask >> 1) ^ (mask >> 2) ^ (mask >> 3)) & 1)
		continue;
	double sse = 0;
	for (sg = 0; sg < 4; sg++)
		sse += costs[sg][(mask >> sg) & 1];
	if (sse < best_sse)
	{
		best_sse = sse;
		group->scale = sc;
		group->signs = (uint32_t)mask;
		for (sg = 0; sg < 4; sg++)
			group->grid[sg] = indices[sg][(mask >> sg) & 1];
	}
}
}
const int m = ccv_nnc_8i_rowwise_packed_iq1s_scales[group->scale];
for (sg = 0; sg < 4; sg++)
for (j = 0; j < 8; j++)
{
	const int g = (int)((ccv_nnc_8i_rowwise_packed_iq1_grid[group->grid[sg]] >> (2 * j)) & 3) - 1;
	const int shift = (group->signs & (1u << sg)) ? -1 : 1;
	group->q8[sg * 8 + j] = ccv_clamp(m * (8 * g + shift), -127, 127)({ typeof (-127) _a = (-127); typeof (127) _b = (127); typeof
 (m * (8 * g + shift)) _x = (m * (8 * g + shift)); (_x < _a
) ? _a : ((_x > _b) ? _b : _x); });
}
404}

406static void _ccv_nnc_8i_rowwise_packed_nearest_candidates(const uint8_t* const grid, const int grid_size, const int lanes, const int levels, const int code_count, const int candidate_count, uint16_t* const candidates)
407{
assert(candidate_count <= CCV_NNC_8I_ROWWISE_IQ3_CANDIDATES)((void) sizeof ((candidate_count <= CCV_NNC_8I_ROWWISE_IQ3_CANDIDATES
) ? 1 : 0), __extension__ ({ if (candidate_count <= CCV_NNC_8I_ROWWISE_IQ3_CANDIDATES
) ; else __assert_fail ("candidate_count <= CCV_NNC_8I_ROWWISE_IQ3_CANDIDATES"
, "ccv_nnc_8i_rowwise.c", 408, __extension__ __PRETTY_FUNCTION__
); }));
assert(candidate_count <= grid_size)((void) sizeof ((candidate_count <= grid_size) ? 1 : 0), __extension__
 ({ if (candidate_count <= grid_size) ; else __assert_fail
 ("candidate_count <= grid_size", "ccv_nnc_8i_rowwise.c", 409
, __extension__ __PRETTY_FUNCTION__); }));
int code;
for (code = 0; code < code_count; code++)
{
uint8_t target[8];
int value = code;
int j;
for (j = 0; j < lanes; j++)
{
	target[j] = (uint8_t)(value % levels);
	value /= levels;
}
int best_distance[CCV_NNC_8I_ROWWISE_IQ3_CANDIDATES];
int k;
for (k = 0; k < candidate_count; k++)
{
	best_distance[k] = INT_MAX2147483647;
	candidates[(size_t)code * candidate_count + k] = 0;
}
int index;
for (index = 0; index < grid_size; index++)
{
	int distance = 0;
	for (j = 0; j < lanes; j++)
	{
		const int d = (int)grid[(size_t)index * lanes + j] - target[j];
		distance += d * d;
	}
	if (distance >= best_distance[candidate_count - 1])
		continue;
	k = candidate_count - 1;
	while (k > 0 && distance < best_distance[k - 1])
	{
		best_distance[k] = best_distance[k - 1];
		candidates[(size_t)code * candidate_count + k] = candidates[(size_t)code * candidate_count + k - 1];
		--k;
	}
	best_distance[k] = distance;
	candidates[(size_t)code * candidate_count + k] = (uint16_t)index;
}
}
450}

452static void _ccv_nnc_8i_rowwise_packed_quant_q5(const double* const y, const double* const w, ccv_nnc_8i_rowwise_packed_group_t* const group)
453{
double best_sse = DBL_MAX1.7976931348623157e+308;
int best_q[16] = {0};
int best_q8[16] = {0};
int best_m = 1, best_b = 0;
int m, b, j;
for (m = 1; m <= 8; m++)
for (b = -16; b <= 15; b++)
{
	if (-16 * m + b < -127 || 15 * m + b > 127)
		continue;
	double sse = 0;
	int q[16];
	int q8[16];
	for (j = 0; j < 16; j++)
	{
		q[j] = ccv_clamp((int)lrint((y[j] - b) / m), -16, 15)({ typeof (-16) _a = (-16); typeof (15) _b = (15); typeof ((int
)lrint((y[j] - b) / m)) _x = ((int)lrint((y[j] - b) / m)); (_x
 < _a) ? _a : ((_x > _b) ? _b : _x); });
		q8[j] = q[j] * m + b;
		const double d = q8[j] - y[j];
		sse += w[j] * d * d;
	}
	if (sse < best_sse)
	{
		best_sse = sse;
		best_m = m;
		best_b = b;
		memcpy(best_q, q, sizeof(best_q));
		memcpy(best_q8, q8, sizeof(best_q8));
	}
}
group->m = best_m;
group->b = best_b;
memcpy(group->q, best_q, sizeof(best_q));
memcpy(group->q8, best_q8, sizeof(best_q8));
487}

489static void _ccv_nnc_8i_rowwise_packed_quant_q6(const double* const y, const double* const w, ccv_nnc_8i_rowwise_packed_group_t* const group)
490{
double best_sse = DBL_MAX1.7976931348623157e+308;
int best_q[8] = {0};
int best_q8[8] = {0};
int best_m = 1, best_b = 0;
int m, b, j;
for (m = 1; m <= 4; m++)
for (b = -2; b <= 1; b++)
{
		const int qmin = ccv_max(-32, _ccv_nnc_8i_rowwise_packed_ceil_div(-127 - b, m))({ typeof (-32) _a = (-32); typeof (_ccv_nnc_8i_rowwise_packed_ceil_div
(-127 - b, m)) _b = (_ccv_nnc_8i_rowwise_packed_ceil_div(-127
 - b, m)); (_a > _b) ? _a : _b; });
		const int qmax = ccv_min(31, _ccv_nnc_8i_rowwise_packed_floor_div(127 - b, m))({ typeof (31) _a = (31); typeof (_ccv_nnc_8i_rowwise_packed_floor_div
(127 - b, m)) _b = (_ccv_nnc_8i_rowwise_packed_floor_div(127 -
 b, m)); (_a < _b) ? _a : _b; });
	if (qmin > qmax)
		continue;
	double sse = 0;
	int q[8];
	int q8[8];
	for (j = 0; j < 8; j++)
	{
		q[j] = ccv_clamp((int)lrint((y[j] - b) / m), qmin, qmax)({ typeof (qmin) _a = (qmin); typeof (qmax) _b = (qmax); typeof
 ((int)lrint((y[j] - b) / m)) _x = ((int)lrint((y[j] - b) / m
)); (_x < _a) ? _a : ((_x > _b) ? _b : _x); });
		q8[j] = q[j] * m + b;
		const double d = q8[j] - y[j];
		sse += w[j] * d * d;
	}
	if (sse < best_sse)
	{
		best_sse = sse;
		best_m = m;
		best_b = b;
		memcpy(best_q, q, sizeof(best_q));
		memcpy(best_q8, q8, sizeof(best_q8));
	}
}
group->m = best_m;
group->b = best_b;
memcpy(group->q, best_q, sizeof(best_q));
memcpy(group->q8, best_q8, sizeof(best_q8));
526}

528static void _ccv_nnc_8i_rowwise_packed_quant_q4(const double* const y, const double* const w, ccv_nnc_8i_rowwise_packed_group_t* const group)
529{
double best_sse = DBL_MAX1.7976931348623157e+308;
int best_q[16] = {0};
int best_q8[16] = {0};
int best_m = 1, best_b = 0;
int m, b, j;
for (m = 1; m <= 16; m++)
for (b = -8; b <= 7; b++)
{
	if (-8 * m + b < -127 || 7 * m + b > 127)
		continue;
	double sse = 0;
	int q[16];
	int q8[16];
	for (j = 0; j < 16; j++)
	{
		q[j] = ccv_clamp((int)lrint((y[j] - b) / m), -8, 7)({ typeof (-8) _a = (-8); typeof (7) _b = (7); typeof ((int)lrint
((y[j] - b) / m)) _x = ((int)lrint((y[j] - b) / m)); (_x <
 _a) ? _a : ((_x > _b) ? _b : _x); });
		q8[j] = q[j] * m + b;
		const double d = q8[j] - y[j];
		sse += w[j] * d * d;
	}
	if (sse < best_sse)
	{
		best_sse = sse;
		best_m = m;
		best_b = b;
		memcpy(best_q, q, sizeof(best_q));
		memcpy(best_q8, q8, sizeof(best_q8));
	}
}
group->m = best_m;
group->b = best_b;
memcpy(group->q, best_q, sizeof(best_q));
memcpy(group->q8, best_q8, sizeof(best_q8));
563}

565static void _ccv_nnc_8i_rowwise_packed_quant_q3(const double* const y, const double* const w, ccv_nnc_8i_rowwise_packed_group_t* const group)
566{
double best_sse = DBL_MAX1.7976931348623157e+308;
int best_q[16] = {0};
int best_q8[16] = {0};
int best_m = 1, best_b = 0;
int m, b, j;
for (m = 1; m <= 32; m++)
for (b = -8; b <= 6; b += 2)
{
	if (-4 * m + b < -127 || 3 * m + b > 127)
		continue;
	double sse = 0;
	int q[16];
	int q8[16];
	for (j = 0; j < 16; j++)
	{
		q[j] = ccv_clamp((int)lrint((y[j] - b) / m), -4, 3)({ typeof (-4) _a = (-4); typeof (3) _b = (3); typeof ((int)lrint
((y[j] - b) / m)) _x = ((int)lrint((y[j] - b) / m)); (_x <
 _a) ? _a : ((_x > _b) ? _b : _x); });
		q8[j] = q[j] * m + b;
		const double d = q8[j] - y[j];
		sse += w[j] * d * d;
	}
	if (sse < best_sse)
	{
		best_sse = sse;
		best_m = m;
		best_b = b;
		memcpy(best_q, q, sizeof(best_q));
		memcpy(best_q8, q8, sizeof(best_q8));
	}
}
group->m = best_m;
group->b = best_b;
memcpy(group->q, best_q, sizeof(best_q));
memcpy(group->q8, best_q8, sizeof(best_q8));
600}

602static void _ccv_nnc_8i_rowwise_packed_quant_q2(const double* const y, const double* const w, ccv_nnc_8i_rowwise_packed_group_t* const group)
603{
double best_sse = DBL_MAX1.7976931348623157e+308;
int best_q[16] = {0};
int best_q8[16] = {0};
int best_m = 1, best_z = 0;
int m, z, j;
for (m = 1; m <= 64; m++)
for (z = 0; z <= 120; z += 8)
{
	if (3 * m - z > 127)
		continue;
	double sse = 0;
	int q[16];
	int q8[16];
	for (j = 0; j < 16; j++)
	{
		q[j] = ccv_clamp((int)lrint((y[j] + z) / m), 0, 3)({ typeof (0) _a = (0); typeof (3) _b = (3); typeof ((int)lrint
((y[j] + z) / m)) _x = ((int)lrint((y[j] + z) / m)); (_x <
 _a) ? _a : ((_x > _b) ? _b : _x); });
		q8[j] = q[j] * m - z;
		const double d = q8[j] - y[j];
		sse += w[j] * d * d;
	}
	if (sse < best_sse)
	{
		best_sse = sse;
		best_m = m;
		best_z = z;
		memcpy(best_q, q, sizeof(best_q));
		memcpy(best_q8, q8, sizeof(best_q8));
	}
}
group->m = best_m;
group->z = best_z;
memcpy(group->q, best_q, sizeof(best_q));
memcpy(group->q8, best_q8, sizeof(best_q8));
637}

639static int _ccv_nnc_8i_rowwise_packed_iq2_value(const uint64_t* const grid, const int index, const int lane)
640{
const int v = (int)((grid[index] >> (lane * 8)) & 0xff);
if (v == 8)
return 1;
if (v == 25)
return 3;
assert(v == 43)((void) sizeof ((v == 43) ? 1 : 0), __extension__ ({ if (v ==
 43) ; else __assert_fail ("v == 43", "ccv_nnc_8i_rowwise.c",
 646, __extension__ __PRETTY_FUNCTION__); }));
return 5;
648}

650static int _ccv_nnc_8i_rowwise_packed_iq2xxs_value(const int index, const int lane)
651{
const int v = (int)((ccv_nnc_8i_rowwise_packed_iq2xxs_grid[index] >> (lane * 2)) & 3);
assert(v < 3)((void) sizeof ((v < 3) ? 1 : 0), __extension__ ({ if (v <
 3) ; else __assert_fail ("v < 3", "ccv_nnc_8i_rowwise.c",
 653, __extension__ __PRETTY_FUNCTION__); }));
return 1 + v * 2;
655}

657enum {
CCV_NNC_8I_ROWWISE_PACKED_IQ2XXS_GRID_SIZE = 256,
CCV_NNC_8I_ROWWISE_PACKED_IQ2S_GRID_SIZE = 1024,
CCV_NNC_8I_ROWWISE_PACKED_IQ2_CODE_COUNT = 6561,
661};

663static const int ccv_nnc_8i_rowwise_packed_iq2xxs_scales[16] = {1, 2, 3, 4, 5, 6, 7, 8, 10, 12, 14, 16, 20, 24, 28, 32};
664static int ccv_nnc_8i_rowwise_packed_iq2_scale_initialized = 0;
665static uint8_t ccv_nnc_8i_rowwise_packed_iq2xxs_scale_level[33][3];
666static int ccv_nnc_8i_rowwise_packed_iq2xxs_initialized = 0;
667static int ccv_nnc_8i_rowwise_packed_iq2xxs_candidates_initialized = 0;
668static uint8_t ccv_nnc_8i_rowwise_packed_iq2xxs_level[CCV_NNC_8I_ROWWISE_PACKED_IQ2XXS_GRID_SIZE][8];
669static uint8_t ccv_nnc_8i_rowwise_packed_iq2xxs_scaled_value[33][CCV_NNC_8I_ROWWISE_PACKED_IQ2XXS_GRID_SIZE][8];
670static uint16_t ccv_nnc_8i_rowwise_packed_iq2xxs_candidates[CCV_NNC_8I_ROWWISE_PACKED_IQ2_CODE_COUNT][CCV_NNC_8I_ROWWISE_IQ2_CANDIDATES];

672static void _ccv_nnc_8i_rowwise_packed_iq2_scale_init(void)
673{
if (ccv_nnc_8i_rowwise_packed_iq2_scale_initialized)
return;
int scale, level;
for (scale = 1; scale <= 32; scale++)
for (level = 0; level < 3; level++)
	ccv_nnc_8i_rowwise_packed_iq2xxs_scale_level[scale][level] = (uint8_t)ccv_min((1 + level * 2) * scale, 127)({ typeof ((1 + level * 2) * scale) _a = ((1 + level * 2) * scale
); typeof (127) _b = (127); (_a < _b) ? _a : _b; });
ccv_nnc_8i_rowwise_packed_iq2_scale_initialized = 1;
681}

683static void _ccv_nnc_8i_rowwise_packed_iq2xxs_init(void)
684{
if (ccv_nnc_8i_rowwise_packed_iq2xxs_initialized)
return;
_ccv_nnc_8i_rowwise_packed_iq2_scale_init();
int index, j, scale;
for (index = 0; index < CCV_NNC_8I_ROWWISE_PACKED_IQ2XXS_GRID_SIZE; index++)
for (j = 0; j < 8; j++)
	ccv_nnc_8i_rowwise_packed_iq2xxs_level[index][j] = (uint8_t)((_ccv_nnc_8i_rowwise_packed_iq2xxs_value(index, j) - 1) / 2);
for (scale = 1; scale <= 32; scale++)
{
for (index = 0; index < CCV_NNC_8I_ROWWISE_PACKED_IQ2XXS_GRID_SIZE; index++)
	for (j = 0; j < 8; j++)
		ccv_nnc_8i_rowwise_packed_iq2xxs_scaled_value[scale][index][j] = ccv_nnc_8i_rowwise_packed_iq2xxs_scale_level[scale][ccv_nnc_8i_rowwise_packed_iq2xxs_level[index][j]];
}
ccv_nnc_8i_rowwise_packed_iq2xxs_initialized = 1;
699}

701static void _ccv_nnc_8i_rowwise_packed_iq2xxs_candidates_init(void)
702{
if (ccv_nnc_8i_rowwise_packed_iq2xxs_candidates_initialized)
return;
_ccv_nnc_8i_rowwise_packed_iq2xxs_init();
_ccv_nnc_8i_rowwise_packed_nearest_candidates(&ccv_nnc_8i_rowwise_packed_iq2xxs_level[0][0], CCV_NNC_8I_ROWWISE_PACKED_IQ2XXS_GRID_SIZE, 8, 3, CCV_NNC_8I_ROWWISE_PACKED_IQ2_CODE_COUNT, CCV_NNC_8I_ROWWISE_IQ2_CANDIDATES, &ccv_nnc_8i_rowwise_packed_iq2xxs_candidates[0][0]);
ccv_nnc_8i_rowwise_packed_iq2xxs_candidates_initialized = 1;
708}

710enum {
CCV_NNC_8I_ROWWISE_PACKED_IQ2XS_GRID_SIZE = 512,
712};

714static int ccv_nnc_8i_rowwise_packed_iq2xs_initialized = 0;
715static int ccv_nnc_8i_rowwise_packed_iq2xs_candidates_initialized = 0;
716static uint8_t ccv_nnc_8i_rowwise_packed_iq2xs_level[CCV_NNC_8I_ROWWISE_PACKED_IQ2XS_GRID_SIZE][8];
717static uint8_t ccv_nnc_8i_rowwise_packed_iq2xs_scaled_value[16][CCV_NNC_8I_ROWWISE_PACKED_IQ2XS_GRID_SIZE][8];
718static uint16_t ccv_nnc_8i_rowwise_packed_iq2xs_candidates[CCV_NNC_8I_ROWWISE_PACKED_IQ2_CODE_COUNT][CCV_NNC_8I_ROWWISE_IQ2_CANDIDATES];

720static void _ccv_nnc_8i_rowwise_packed_iq2xs_init(void)
721{
if (ccv_nnc_8i_rowwise_packed_iq2xs_initialized)
return;
_ccv_nnc_8i_rowwise_packed_iq2_scale_init();
int index, j, scale_code;
for (index = 0; index < CCV_NNC_8I_ROWWISE_PACKED_IQ2XS_GRID_SIZE; index++)
for (j = 0; j < 8; j++)
	ccv_nnc_8i_rowwise_packed_iq2xs_level[index][j] = (uint8_t)((_ccv_nnc_8i_rowwise_packed_iq2_value(ccv_nnc_8i_rowwise_packed_iq2xs_grid, index, j) - 1) / 2);
for (scale_code = 0; scale_code < 16; scale_code++)
{
const int scale = ccv_nnc_8i_rowwise_packed_iq2xxs_scales[scale_code];
for (index = 0; index < CCV_NNC_8I_ROWWISE_PACKED_IQ2XS_GRID_SIZE; index++)
	for (j = 0; j < 8; j++)
		ccv_nnc_8i_rowwise_packed_iq2xs_scaled_value[scale_code][index][j] = ccv_nnc_8i_rowwise_packed_iq2xxs_scale_level[scale][ccv_nnc_8i_rowwise_packed_iq2xs_level[index][j]];
}
ccv_nnc_8i_rowwise_packed_iq2xs_initialized = 1;
737}

739static void _ccv_nnc_8i_rowwise_packed_iq2xs_candidates_init(void)
740{
if (ccv_nnc_8i_rowwise_packed_iq2xs_candidates_initialized)
return;
_ccv_nnc_8i_rowwise_packed_iq2xs_init();
_ccv_nnc_8i_rowwise_packed_nearest_candidates(&ccv_nnc_8i_rowwise_packed_iq2xs_level[0][0], CCV_NNC_8I_ROWWISE_PACKED_IQ2XS_GRID_SIZE, 8, 3, CCV_NNC_8I_ROWWISE_PACKED_IQ2_CODE_COUNT, CCV_NNC_8I_ROWWISE_IQ2_CANDIDATES, &ccv_nnc_8i_rowwise_packed_iq2xs_candidates[0][0]);
ccv_nnc_8i_rowwise_packed_iq2xs_candidates_initialized = 1;
746}

748static int ccv_nnc_8i_rowwise_packed_iq2s_initialized = 0;
749static uint8_t ccv_nnc_8i_rowwise_packed_iq2s_level[CCV_NNC_8I_ROWWISE_PACKED_IQ2S_GRID_SIZE][8];
750static uint8_t ccv_nnc_8i_rowwise_packed_iq2s_scale_level[65][3];
751static uint16_t ccv_nnc_8i_rowwise_packed_iq2s_scale_level2[65][3];
752static uint8_t ccv_nnc_8i_rowwise_packed_iq2s_scaled_value[65][CCV_NNC_8I_ROWWISE_PACKED_IQ2S_GRID_SIZE][8];

754static void _ccv_nnc_8i_rowwise_packed_iq2s_init(void)
755{
if (ccv_nnc_8i_rowwise_packed_iq2s_initialized)
return;
int index, j, scale;
for (index = 0; index < CCV_NNC_8I_ROWWISE_PACKED_IQ2S_GRID_SIZE; index++)
for (j = 0; j < 8; j++)
{
	const int v = _ccv_nnc_8i_rowwise_packed_iq2_value(ccv_nnc_8i_rowwise_packed_iq2s_grid, index, j);
	ccv_nnc_8i_rowwise_packed_iq2s_level[index][j] = (uint8_t)((v - 1) / 2);
}
for (scale = 1; scale <= 64; scale++)
{
for (j = 0; j < 3; j++)
{
	const int v = ccv_min((1 + j * 2) * scale, 127)({ typeof ((1 + j * 2) * scale) _a = ((1 + j * 2) * scale); typeof
 (127) _b = (127); (_a < _b) ? _a : _b; });
	ccv_nnc_8i_rowwise_packed_iq2s_scale_level[scale][j] = (uint8_t)v;
	ccv_nnc_8i_rowwise_packed_iq2s_scale_level2[scale][j] = (uint16_t)(v * v);
}
for (index = 0; index < CCV_NNC_8I_ROWWISE_PACKED_IQ2S_GRID_SIZE; index++)
	for (j = 0; j < 8; j++)
		ccv_nnc_8i_rowwise_packed_iq2s_scaled_value[scale][index][j] = ccv_nnc_8i_rowwise_packed_iq2s_scale_level[scale][ccv_nnc_8i_rowwise_packed_iq2s_level[index][j]];
}
ccv_nnc_8i_rowwise_packed_iq2s_initialized = 1;
778}

780static double _ccv_nnc_8i_rowwise_packed_iq2s_sse(const double* const ay, const double* const w, const int lane, const int scale, const int index)
781{
const uint8_t* const mag = ccv_nnc_8i_rowwise_packed_iq2s_scaled_value[scale][index];
double d = (double)mag[0] - ay[lane];
double sse = w[lane] * d * d;
d = (double)mag[1] - ay[lane + 1];
sse += w[lane + 1] * d * d;
d = (double)mag[2] - ay[lane + 2];
sse += w[lane + 2] * d * d;
d = (double)mag[3] - ay[lane + 3];
sse += w[lane + 3] * d * d;
d = (double)mag[4] - ay[lane + 4];
sse += w[lane + 4] * d * d;
d = (double)mag[5] - ay[lane + 5];
sse += w[lane + 5] * d * d;
d = (double)mag[6] - ay[lane + 6];
sse += w[lane + 6] * d * d;
d = (double)mag[7] - ay[lane + 7];
sse += w[lane + 7] * d * d;
return sse;
800}

802static int _ccv_nnc_8i_rowwise_packed_iq3xxs_value(const int index, const int lane)
803{
const int v = (int)((ccv_nnc_8i_rowwise_packed_iq3xxs_grid[index] >> (lane * 8)) & 0xff);
switch (v)
{
case 4: return 1;
case 12: return 3;
case 20: return 5;
case 28: return 7;
case 36: return 9;
case 44: return 11;
case 52: return 13;
default:
	assert(v == 62)((void) sizeof ((v == 62) ? 1 : 0), __extension__ ({ if (v ==
 62) ; else __assert_fail ("v == 62", "ccv_nnc_8i_rowwise.c",
 815, __extension__ __PRETTY_FUNCTION__); }));
	return 15;
}
818}

820static int _ccv_nnc_8i_rowwise_packed_iq3s_value(const int index, const int lane)
821{
return (int)((ccv_nnc_8i_rowwise_packed_iq3s_grid[index] >> (lane * 8)) & 0xff);
823}

825#define CCV_NNC_8I_ROWWISE_PACKED_IQ3S_GRID_SIZE(512) (512)
826#define CCV_NNC_8I_ROWWISE_PACKED_IQ3XXS_GRID_SIZE(256) (256)
827#define CCV_NNC_8I_ROWWISE_PACKED_IQ3_CODE_COUNT(4096) (4096)

829static int ccv_nnc_8i_rowwise_packed_iq3s_initialized = 0;
830static uint8_t ccv_nnc_8i_rowwise_packed_iq3s_scaled_value[17][CCV_NNC_8I_ROWWISE_PACKED_IQ3S_GRID_SIZE(512)][4];

832static int ccv_nnc_8i_rowwise_packed_iq3xxs_initialized = 0;
833static int ccv_nnc_8i_rowwise_packed_iq3xxs_candidates_initialized = 0;
834static uint8_t ccv_nnc_8i_rowwise_packed_iq3xxs_level[CCV_NNC_8I_ROWWISE_PACKED_IQ3XXS_GRID_SIZE(256)][4];
835static uint8_t ccv_nnc_8i_rowwise_packed_iq3xxs_scaled_value[17][CCV_NNC_8I_ROWWISE_PACKED_IQ3XXS_GRID_SIZE(256)][4];
836static uint16_t ccv_nnc_8i_rowwise_packed_iq3xxs_candidates[CCV_NNC_8I_ROWWISE_PACKED_IQ3_CODE_COUNT(4096)][CCV_NNC_8I_ROWWISE_IQ3_CANDIDATES];

838static void _ccv_nnc_8i_rowwise_packed_iq3s_init(void)
839{
if (ccv_nnc_8i_rowwise_packed_iq3s_initialized)
return;
int index, j, scale;
for (scale = 1; scale <= 16; scale++)
for (index = 0; index < CCV_NNC_8I_ROWWISE_PACKED_IQ3S_GRID_SIZE(512); index++)
	for (j = 0; j < 4; j++)
		ccv_nnc_8i_rowwise_packed_iq3s_scaled_value[scale][index][j] = (uint8_t)ccv_min(_ccv_nnc_8i_rowwise_packed_iq3s_value(index, j) * scale, 127)({ typeof (_ccv_nnc_8i_rowwise_packed_iq3s_value(index, j) * scale
) _a = (_ccv_nnc_8i_rowwise_packed_iq3s_value(index, j) * scale
); typeof (127) _b = (127); (_a < _b) ? _a : _b; });
ccv_nnc_8i_rowwise_packed_iq3s_initialized = 1;
848}

850static void _ccv_nnc_8i_rowwise_packed_iq3xxs_init(void)
851{
if (ccv_nnc_8i_rowwise_packed_iq3xxs_initialized)
return;
int index, j, scale;
for (index = 0; index < CCV_NNC_8I_ROWWISE_PACKED_IQ3XXS_GRID_SIZE(256); index++)
for (j = 0; j < 4; j++)
	ccv_nnc_8i_rowwise_packed_iq3xxs_level[index][j] = (uint8_t)((_ccv_nnc_8i_rowwise_packed_iq3xxs_value(index, j) - 1) / 2);
for (scale = 1; scale <= 16; scale++)
for (index = 0; index < CCV_NNC_8I_ROWWISE_PACKED_IQ3XXS_GRID_SIZE(256); index++)
	for (j = 0; j < 4; j++)
		ccv_nnc_8i_rowwise_packed_iq3xxs_scaled_value[scale][index][j] = (uint8_t)ccv_min((1 + ccv_nnc_8i_rowwise_packed_iq3xxs_level[index][j] * 2) * scale, 127)({ typeof ((1 + ccv_nnc_8i_rowwise_packed_iq3xxs_level[index]
[j] * 2) * scale) _a = ((1 + ccv_nnc_8i_rowwise_packed_iq3xxs_level
[index][j] * 2) * scale); typeof (127) _b = (127); (_a < _b
) ? _a : _b; });
ccv_nnc_8i_rowwise_packed_iq3xxs_initialized = 1;
863}

865static void _ccv_nnc_8i_rowwise_packed_iq3xxs_candidates_init(void)
866{
if (ccv_nnc_8i_rowwise_packed_iq3xxs_candidates_initialized)
return;
_ccv_nnc_8i_rowwise_packed_iq3xxs_init();
_ccv_nnc_8i_rowwise_packed_nearest_candidates(&ccv_nnc_8i_rowwise_packed_iq3xxs_level[0][0], CCV_NNC_8I_ROWWISE_PACKED_IQ3XXS_GRID_SIZE(256), 4, 8, CCV_NNC_8I_ROWWISE_PACKED_IQ3_CODE_COUNT(4096), CCV_NNC_8I_ROWWISE_IQ3_CANDIDATES, &ccv_nnc_8i_rowwise_packed_iq3xxs_candidates[0][0]);
ccv_nnc_8i_rowwise_packed_iq3xxs_candidates_initialized = 1;
872}

874static double _ccv_nnc_8i_rowwise_packed_iq3s_sse(const double* const ay, const double* const w, const int lane, const int scale, const int index)
875{
const uint8_t* const mag = ccv_nnc_8i_rowwise_packed_iq3s_scaled_value[scale][index];
double d = (double)mag[0] - ay[lane];
double sse = w[lane] * d * d;
d = (double)mag[1] - ay[lane + 1];
sse += w[lane + 1] * d * d;
d = (double)mag[2] - ay[lane + 2];
sse += w[lane + 2] * d * d;
d = (double)mag[3] - ay[lane + 3];
sse += w[lane + 3] * d * d;
return sse;
886}

888static inline double _ccv_nnc_8i_rowwise_packed_iq2_cost(const double cost[8][3], const uint8_t* const levels)
889{
double sse = 0;
int j;
for (j = 0; j < 8; j++)
sse += cost[j][levels[j]];
return sse;
895}

897static inline double _ccv_nnc_8i_rowwise_packed_safe_lower_bound(const double value)
898{
return nextafter(value - (fabs(value) + 1) * (64 * DBL_EPSILON2.2204460492503131e-16), -INFINITY(__builtin_inff ()));
900}

902static inline double _ccv_nnc_8i_rowwise_packed_iq2xxs_cost(const double cost[8][3], const double flipped_cost[8][3], const double flip_metric[8][3], const uint8_t* const levels, const uint8_t signs, const int flip_sign, int* const sign_index)
903{
int best_flip = -1;
int j;
if (flip_sign)
{
double best_flip_metric = DBL_MAX1.7976931348623157e+308;
for (j = 0; j < 8; j++)
{
	const double metric = flip_metric[j][levels[j]];
	if (metric < best_flip_metric)
	{
		best_flip_metric = metric;
		best_flip = j;
	}
}
}
double sse = 0;
for (j = 0; j < 8; j++)
sse += j == best_flip ? flipped_cost[j][levels[j]] : cost[j][levels[j]];
*sign_index = (signs ^ (best_flip >= 0 ? (uint8_t)(1u << best_flip) : 0)) & 0x7f;
return sse;
924}

926static void _ccv_nnc_8i_rowwise_packed_quant_iq2_xxs(const double* const y, const double* const w, ccv_nnc_8i_rowwise_packed_group_t* const group)
927{
assert(ccv_nnc_8i_rowwise_packed_iq2xxs_initialized)((void) sizeof ((ccv_nnc_8i_rowwise_packed_iq2xxs_initialized
) ? 1 : 0), __extension__ ({ if (ccv_nnc_8i_rowwise_packed_iq2xxs_initialized
) ; else __assert_fail ("ccv_nnc_8i_rowwise_packed_iq2xxs_initialized"
, "ccv_nnc_8i_rowwise.c", 928, __extension__ __PRETTY_FUNCTION__
); }));
double best_sse = DBL_MAX1.7976931348623157e+308;
int best_scale_code = 0;
int best_grid[4] = {0};
int best_sign[4] = {0};
int scale_code;
for (scale_code = 0; scale_code < 16; scale_code++)
{
const int scale = ccv_nnc_8i_rowwise_packed_iq2xxs_scales[scale_code];
double group_sse = 0;
int group_grid[4] = {0};
int group_sign[4] = {0};
int sg;
for (sg = 0; sg < 4; sg++)
{
	double best_sub_sse = DBL_MAX1.7976931348623157e+308;
	int best_sub_grid = 0;
	int best_sub_sign = 0;
	const int lane = sg * 8;
	uint8_t signs = 0;
	int negative_count = 0;
	double cost[8][3];
	double flipped_cost[8][3];
	double flip_metric[8][3];
	int j;
	for (j = 0; j < 8; j++)
	{
		if (y[lane + j] < 0)
		{
			signs |= (uint8_t)(1u << j);
			negative_count++;
		}
		int level;
		for (level = 0; level < 3; level++)
		{
			const int mag = ccv_nnc_8i_rowwise_packed_iq2xxs_scale_level[scale][level];
			const int q8 = (signs & (1u << j)) ? -mag : mag;
			double d = (double)q8 - y[lane + j];
			cost[j][level] = w[lane + j] * d * d;
			d = (double)-q8 - y[lane + j];
			flipped_cost[j][level] = w[lane + j] * d * d;
			flip_metric[j][level] = w[lane + j] * (double)mag * fabs(y[lane + j]);
		}
	}
	int target_code = 0;
	int code_multiplier = 1;
	uint8_t target_levels[8];
	double lower_bound_base = 0;
	double lower_bound_ratio = DBL_MAX1.7976931348623157e+308;
	double lower_bound_flip = DBL_MAX1.7976931348623157e+308;
	for (j = 0; j < 8; j++)
	{
		int target_level = 0;
		if (cost[j][1] < cost[j][target_level])
			target_level = 1;
		if (cost[j][2] < cost[j][target_level])
			target_level = 2;
		target_levels[j] = (uint8_t)target_level;
		target_code += target_level * code_multiplier;
		code_multiplier *= 3;
		lower_bound_base += cost[j][target_level];
		int level;
		for (level = 0; level < 3; level++)
		{
			if (level != target_level)
			{
				const int d = level - target_level;
				lower_bound_ratio = ccv_min(lower_bound_ratio, (cost[j][level] - cost[j][target_level]) / (double)(d * d))({ typeof (lower_bound_ratio) _a = (lower_bound_ratio); typeof
 ((cost[j][level] - cost[j][target_level]) / (double)(d * d))
 _b = ((cost[j][level] - cost[j][target_level]) / (double)(d *
 d)); (_a < _b) ? _a : _b; });
			}
			lower_bound_flip = ccv_min(lower_bound_flip, flipped_cost[j][level] - cost[j][level])({ typeof (lower_bound_flip) _a = (lower_bound_flip); typeof (
flipped_cost[j][level] - cost[j][level]) _b = (flipped_cost[j
][level] - cost[j][level]); (_a < _b) ? _a : _b; });
		}
	}
	int candidate;
	for (candidate = 0; candidate < CCV_NNC_8I_ROWWISE_IQ2_CANDIDATES; candidate++)
	{
		const int index = ccv_nnc_8i_rowwise_packed_iq2xxs_candidates[target_code][candidate];
		const uint8_t* const levels = ccv_nnc_8i_rowwise_packed_iq2xxs_level[index];
		int sign_index;
		const double sse = _ccv_nnc_8i_rowwise_packed_iq2xxs_cost(cost, flipped_cost, flip_metric, levels, signs, negative_count & 1, &sign_index);
		if (sse < best_sub_sse || (sse == best_sub_sse && index < best_sub_grid))
		{
			best_sub_sse = sse;
			best_sub_grid = index;
			best_sub_sign = sign_index;
		}
	}
	const uint8_t* const cutoff_levels = ccv_nnc_8i_rowwise_packed_iq2xxs_level[ccv_nnc_8i_rowwise_packed_iq2xxs_candidates[target_code][CCV_NNC_8I_ROWWISE_IQ2_CANDIDATES - 1]];
	int cutoff_distance = 0;
	for (j = 0; j < 8; j++)
	{
		const int d = (int)cutoff_levels[j] - target_levels[j];
		cutoff_distance += d * d;
	}
	const double safe_ratio = nextafter(lower_bound_ratio, 0);
	const double lower_bound = _ccv_nnc_8i_rowwise_packed_safe_lower_bound(lower_bound_base + safe_ratio * cutoff_distance + ((negative_count & 1) ? nextafter(lower_bound_flip, -INFINITY(__builtin_inff ())) : 0));
	if (!(best_sub_sse < lower_bound))
	{
		int index;
		for (index = 0; index < CCV_NNC_8I_ROWWISE_PACKED_IQ2XXS_GRID_SIZE; index++)
		{
			int sign_index;
			const double sse = _ccv_nnc_8i_rowwise_packed_iq2xxs_cost(cost, flipped_cost, flip_metric, ccv_nnc_8i_rowwise_packed_iq2xxs_level[index], signs, negative_count & 1, &sign_index);
			if (sse < best_sub_sse || (sse == best_sub_sse && index < best_sub_grid))
			{
				best_sub_sse = sse;
				best_sub_grid = index;
				best_sub_sign = sign_index;
			}
		}
	}
	group_sse += best_sub_sse;
	group_grid[sg] = best_sub_grid;
	group_sign[sg] = best_sub_sign;
}
if (group_sse < best_sse)
{
	best_sse = group_sse;
	best_scale_code = scale_code;
	memcpy(best_grid, group_grid, sizeof(best_grid));
	memcpy(best_sign, group_sign, sizeof(best_sign));
}
}
group->scale = best_scale_code;
group->signs = 0;
memcpy(group->grid, best_grid, sizeof(best_grid));
int j;
for (j = 0; j < 4; j++)
group->signs |= (uint32_t)best_sign[j] << (j * 7);
for (j = 0; j < 32; j++)
{
const int sg = j >> 3;
const int lane = j & 7;
const uint8_t signs = ccv_nnc_8i_rowwise_packed_iq2xxs_ksigns[best_sign[sg]];
const int mag = ccv_nnc_8i_rowwise_packed_iq2xxs_scaled_value[ccv_nnc_8i_rowwise_packed_iq2xxs_scales[best_scale_code]][best_grid[sg]][lane];
group->q8[j] = (signs & (1u << lane)) ? -mag : mag;
}
1064}

1066static void _ccv_nnc_8i_rowwise_packed_quant_iq2_s(const double* const y, const double* const w, ccv_nnc_8i_rowwise_packed_group_t* const group)
1067{
assert(ccv_nnc_8i_rowwise_packed_iq2s_initialized)((void) sizeof ((ccv_nnc_8i_rowwise_packed_iq2s_initialized) ?
: 0), __extension__ ({ if (ccv_nnc_8i_rowwise_packed_iq2s_initialized
) ; else __assert_fail ("ccv_nnc_8i_rowwise_packed_iq2s_initialized"
, "ccv_nnc_8i_rowwise.c", 1068, __extension__ __PRETTY_FUNCTION__
); }));
double best_sse = DBL_MAX1.7976931348623157e+308;
int best_scale = 1;
int best_grid[2] = {0};
double ay[16];
double wy[16];
uint32_t signs = 0;
int j;
for (j = 0; j < 16; j++)
{
ay[j] = fabs(y[j]);
wy[j] = w[j] * ay[j];
if (y[j] < 0)
	signs |= (1u << j);
}
double sub_sse[2][65];
int sub_grid[2][65];
int sg, scale;
for (sg = 0; sg < 2; sg++)
for (scale = 1; scale <= 64; scale++)
{
	sub_sse[sg][scale] = DBL_MAX1.7976931348623157e+308;
	sub_grid[sg][scale] = 0;
}
for (sg = 0; sg < 2; sg++)
{
const int lane = sg * 8;
double sum_y2 = 0;
for (j = 0; j < 8; j++)
	sum_y2 += w[lane + j] * ay[lane + j] * ay[lane + j];
int index;
for (index = 0; index < CCV_NNC_8I_ROWWISE_PACKED_IQ2S_GRID_SIZE; index++)
{
	double sw[3] = {0};
	double swy[3] = {0};
	for (j = 0; j < 8; j++)
	{
		const int level = ccv_nnc_8i_rowwise_packed_iq2s_level[index][j];
		sw[level] += w[lane + j];
		swy[level] += wy[lane + j];
	}
	for (scale = 1; scale <= 64; scale++)
	{
		const double sse = sum_y2 +
			sw[0] * (double)ccv_nnc_8i_rowwise_packed_iq2s_scale_level2[scale][0] - 2 * swy[0] * (double)ccv_nnc_8i_rowwise_packed_iq2s_scale_level[scale][0] +
			sw[1] * (double)ccv_nnc_8i_rowwise_packed_iq2s_scale_level2[scale][1] - 2 * swy[1] * (double)ccv_nnc_8i_rowwise_packed_iq2s_scale_level[scale][1] +
			sw[2] * (double)ccv_nnc_8i_rowwise_packed_iq2s_scale_level2[scale][2] - 2 * swy[2] * (double)ccv_nnc_8i_rowwise_packed_iq2s_scale_level[scale][2];
		if (sub_sse[sg][scale] == DBL_MAX1.7976931348623157e+308 || sse <= sub_sse[sg][scale] + ccv_max(1., fabs(sub_sse[sg][scale]))({ typeof (1.) _a = (1.); typeof (fabs(sub_sse[sg][scale])) _b
 = (fabs(sub_sse[sg][scale])); (_a > _b) ? _a : _b; }) * 1e-9)
		{
			const double exact_sse = _ccv_nnc_8i_rowwise_packed_iq2s_sse(ay, w, lane, scale, index);
			if (exact_sse < sub_sse[sg][scale])
			{
				sub_sse[sg][scale] = exact_sse;
				sub_grid[sg][scale] = index;
			}
		}
	}
}
}
for (scale = 1; scale <= 64; scale++)
{
const double group_sse = sub_sse[0][scale] + sub_sse[1][scale];
if (group_sse < best_sse)
{
	best_sse = group_sse;
	best_scale = scale;
	best_grid[0] = sub_grid[0][scale];
	best_grid[1] = sub_grid[1][scale];
}
}
group->scale = best_scale;
group->signs = signs;
memcpy(group->grid, best_grid, sizeof(best_grid));
for (j = 0; j < 16; j++)
{
const int sg = j >> 3;
const int lane = j & 7;
const int mag = ccv_nnc_8i_rowwise_packed_iq2s_scaled_value[best_scale][best_grid[sg]][lane];
group->q8[j] = (signs & (1u << j)) ? -mag : mag;
}
1148}

1150static void _ccv_nnc_8i_rowwise_packed_quant_iq2_xs(const double* const y, const double* const w, ccv_nnc_8i_rowwise_packed_group_t* const group)
1151{
assert(ccv_nnc_8i_rowwise_packed_iq2xs_initialized)((void) sizeof ((ccv_nnc_8i_rowwise_packed_iq2xs_initialized)
 ? 1 : 0), __extension__ ({ if (ccv_nnc_8i_rowwise_packed_iq2xs_initialized
) ; else __assert_fail ("ccv_nnc_8i_rowwise_packed_iq2xs_initialized"
, "ccv_nnc_8i_rowwise.c", 1152, __extension__ __PRETTY_FUNCTION__
); }));
double best_sse = DBL_MAX1.7976931348623157e+308;
int best_scale_code = 0;
int best_grid = 0;
uint32_t signs = 0;
int j;
for (j = 0; j < 8; j++)
if (y[j] < 0)
	signs |= (1u << j);
int scale_code;
for (scale_code = 0; scale_code < 16; scale_code++)
{
double cost[8][3];
for (j = 0; j < 8; j++)
{
	int level;
	for (level = 0; level < 3; level++)
	{
		const int mag = ccv_nnc_8i_rowwise_packed_iq2xxs_scale_level[ccv_nnc_8i_rowwise_packed_iq2xxs_scales[scale_code]][level];
		const int q8 = (signs & (1u << j)) ? -mag : mag;
		const double d = (double)q8 - y[j];
		cost[j][level] = w[j] * d * d;
	}
}
int target_code = 0;
int code_multiplier = 1;
uint8_t target_levels[8];
double lower_bound_base = 0;
double lower_bound_ratio = DBL_MAX1.7976931348623157e+308;
for (j = 0; j < 8; j++)
{
	int target_level = 0;
	if (cost[j][1] < cost[j][target_level])
		target_level = 1;
	if (cost[j][2] < cost[j][target_level])
		target_level = 2;
	target_levels[j] = (uint8_t)target_level;
	target_code += target_level * code_multiplier;
	code_multiplier *= 3;
	lower_bound_base += cost[j][target_level];
	int level;
	for (level = 0; level < 3; level++)
		if (level != target_level)
		{
			const int d = level - target_level;
			lower_bound_ratio = ccv_min(lower_bound_ratio, (cost[j][level] - cost[j][target_level]) / (double)(d * d))({ typeof (lower_bound_ratio) _a = (lower_bound_ratio); typeof
 ((cost[j][level] - cost[j][target_level]) / (double)(d * d))
 _b = ((cost[j][level] - cost[j][target_level]) / (double)(d *
 d)); (_a < _b) ? _a : _b; });
		}
}
int candidate;
for (candidate = 0; candidate < CCV_NNC_8I_ROWWISE_IQ2_CANDIDATES; candidate++)
{
	const int index = ccv_nnc_8i_rowwise_packed_iq2xs_candidates[target_code][candidate];
	const uint8_t* const levels = ccv_nnc_8i_rowwise_packed_iq2xs_level[index];
	const double sse = _ccv_nnc_8i_rowwise_packed_iq2_cost(cost, levels);
	if (sse < best_sse || (sse == best_sse && scale_code == best_scale_code && index < best_grid))
	{
		best_sse = sse;
		best_scale_code = scale_code;
		best_grid = index;
	}
}
const uint8_t* const cutoff_levels = ccv_nnc_8i_rowwise_packed_iq2xs_level[ccv_nnc_8i_rowwise_packed_iq2xs_candidates[target_code][CCV_NNC_8I_ROWWISE_IQ2_CANDIDATES - 1]];
int cutoff_distance = 0;
for (j = 0; j < 8; j++)
{
	const int d = (int)cutoff_levels[j] - target_levels[j];
	cutoff_distance += d * d;
}
const double lower_bound = _ccv_nnc_8i_rowwise_packed_safe_lower_bound(lower_bound_base + nextafter(lower_bound_ratio, 0) * cutoff_distance);
if (!(best_sse < lower_bound))
{
	int index;
	for (index = 0; index < CCV_NNC_8I_ROWWISE_PACKED_IQ2XS_GRID_SIZE; index++)
	{
		const double sse = _ccv_nnc_8i_rowwise_packed_iq2_cost(cost, ccv_nnc_8i_rowwise_packed_iq2xs_level[index]);
		if (sse < best_sse || (sse == best_sse && scale_code == best_scale_code && index < best_grid))
		{
			best_sse = sse;
			best_scale_code = scale_code;
			best_grid = index;
		}
	}
}
}
group->scale = best_scale_code;
group->grid[0] = best_grid;
group->signs = signs;
memset(group->q8, 0, sizeof(group->q8));
for (j = 0; j < 8; j++)
{
const int mag = ccv_nnc_8i_rowwise_packed_iq2xs_scaled_value[best_scale_code][best_grid][j];
group->q8[j] = (signs & (1u << j)) ? -mag : mag;
}
1245}

1247static void _ccv_nnc_8i_rowwise_packed_quant_iq3_s(const double* const y, const double* const w, ccv_nnc_8i_rowwise_packed_group_t* const group)
1248{
assert(ccv_nnc_8i_rowwise_packed_iq3s_initialized)((void) sizeof ((ccv_nnc_8i_rowwise_packed_iq3s_initialized) ?
: 0), __extension__ ({ if (ccv_nnc_8i_rowwise_packed_iq3s_initialized
) ; else __assert_fail ("ccv_nnc_8i_rowwise_packed_iq3s_initialized"
, "ccv_nnc_8i_rowwise.c", 1249, __extension__ __PRETTY_FUNCTION__
); }));
double best_sse = DBL_MAX1.7976931348623157e+308;
int best_scale = 1;
int best_grid[4] = {0};
double ay[16];
uint32_t signs = 0;
int j;
for (j = 0; j < 16; j++)
{
ay[j] = fabs(y[j]);
if (y[j] < 0)
	signs |= (1u << j);
}
int scale;
for (scale = 1; scale <= 16; scale++)
{
double group_sse = 0;
int group_grid[4] = {0};
int sg;
for (sg = 0; sg < 4; sg++)
{
	double best_sub_sse = DBL_MAX1.7976931348623157e+308;
	int best_sub_grid = 0;
	const int lane = sg * 4;
	int index;
	for (index = 0; index < CCV_NNC_8I_ROWWISE_PACKED_IQ3S_GRID_SIZE(512); index++)
	{
		const double sse = _ccv_nnc_8i_rowwise_packed_iq3s_sse(ay, w, lane, scale, index);
		if (sse < best_sub_sse)
		{
			best_sub_sse = sse;
			best_sub_grid = index;
		}
	}
	group_sse += best_sub_sse;
	group_grid[sg] = best_sub_grid;
}
if (group_sse < best_sse)
{
	best_sse = group_sse;
	best_scale = scale;
	memcpy(best_grid, group_grid, sizeof(best_grid));
}
}
group->scale = best_scale;
group->signs = signs;
memcpy(group->grid, best_grid, sizeof(best_grid));
for (j = 0; j < 16; j++)
{
const int sg = j >> 2;
const int lane = j & 3;
const int mag = ccv_nnc_8i_rowwise_packed_iq3s_scaled_value[best_scale][best_grid[sg]][lane];
group->q8[j] = (signs & (1u << j)) ? -mag : mag;
}
1303}

1305static void _ccv_nnc_8i_rowwise_packed_quant_iq3_xxs(const double* const y, const double* const w, ccv_nnc_8i_rowwise_packed_group_t* const group)
1306{
assert(ccv_nnc_8i_rowwise_packed_iq3xxs_initialized)((void) sizeof ((ccv_nnc_8i_rowwise_packed_iq3xxs_initialized
) ? 1 : 0), __extension__ ({ if (ccv_nnc_8i_rowwise_packed_iq3xxs_initialized
) ; else __assert_fail ("ccv_nnc_8i_rowwise_packed_iq3xxs_initialized"
, "ccv_nnc_8i_rowwise.c", 1307, __extension__ __PRETTY_FUNCTION__
); }));
double best_sse = DBL_MAX1.7976931348623157e+308;
int best_scale = 1;
int best_grid[2] = {0};
double ay[8];
uint32_t signs = 0;
int j;
for (j = 0; j < 8; j++)
{
ay[j] = fabs(y[j]);
if (y[j] < 0)
	signs |= (1u << j);
}
int scale;
for (scale = 1; scale <= 16; scale++)
{
double group_sse = 0;
int group_grid[2] = {0};
int sg;
for (sg = 0; sg < 2; sg++)
{
	double best_sub_sse = DBL_MAX1.7976931348623157e+308;
	int best_sub_grid = 0;
	const int lane = sg * 4;
	double cost[4][8];
	for (j = 0; j < 4; j++)
	{
		int level;
		for (level = 0; level < 8; level++)
		{
			const int mag = ccv_min((1 + level * 2) * scale, 127)({ typeof ((1 + level * 2) * scale) _a = ((1 + level * 2) * scale
); typeof (127) _b = (127); (_a < _b) ? _a : _b; });
			const double d = (double)mag - ay[lane + j];
			cost[j][level] = w[lane + j] * d * d;
		}
	}
	int target_code = 0;
	uint8_t target_levels[4];
	double lower_bound_base = 0;
	double lower_bound_ratio = DBL_MAX1.7976931348623157e+308;
	for (j = 0; j < 4; j++)
	{
		int target_level = 0;
		int level;
		for (level = 1; level < 8; level++)
			if (cost[j][level] < cost[j][target_level])
				target_level = level;
		target_levels[j] = (uint8_t)target_level;
		target_code |= target_level << (j * 3);
		lower_bound_base += cost[j][target_level];
		for (level = 0; level < 8; level++)
			if (level != target_level)
			{
				const int d = level - target_level;
				lower_bound_ratio = ccv_min(lower_bound_ratio, (cost[j][level] - cost[j][target_level]) / (double)(d * d))({ typeof (lower_bound_ratio) _a = (lower_bound_ratio); typeof
 ((cost[j][level] - cost[j][target_level]) / (double)(d * d))
 _b = ((cost[j][level] - cost[j][target_level]) / (double)(d *
 d)); (_a < _b) ? _a : _b; });
			}
	}
	int candidate;
	for (candidate = 0; candidate < CCV_NNC_8I_ROWWISE_IQ3_CANDIDATES; candidate++)
	{
		const int index = ccv_nnc_8i_rowwise_packed_iq3xxs_candidates[target_code][candidate];
		const uint8_t* const levels = ccv_nnc_8i_rowwise_packed_iq3xxs_level[index];
		const double sse = cost[0][levels[0]] + cost[1][levels[1]] + cost[2][levels[2]] + cost[3][levels[3]];
		if (sse < best_sub_sse || (sse == best_sub_sse && index < best_sub_grid))
		{
			best_sub_sse = sse;
			best_sub_grid = index;
		}
	}
	const uint8_t* const cutoff_levels = ccv_nnc_8i_rowwise_packed_iq3xxs_level[ccv_nnc_8i_rowwise_packed_iq3xxs_candidates[target_code][CCV_NNC_8I_ROWWISE_IQ3_CANDIDATES - 1]];
	int cutoff_distance = 0;
	for (j = 0; j < 4; j++)
	{
		const int d = (int)cutoff_levels[j] - target_levels[j];
		cutoff_distance += d * d;
	}
	const double lower_bound = _ccv_nnc_8i_rowwise_packed_safe_lower_bound(lower_bound_base + nextafter(lower_bound_ratio, 0) * cutoff_distance);
	if (!(best_sub_sse < lower_bound))
	{
		int index;
		for (index = 0; index < CCV_NNC_8I_ROWWISE_PACKED_IQ3XXS_GRID_SIZE(256); index++)
		{
			const uint8_t* const levels = ccv_nnc_8i_rowwise_packed_iq3xxs_level[index];
			const double sse = cost[0][levels[0]] + cost[1][levels[1]] + cost[2][levels[2]] + cost[3][levels[3]];
			if (sse < best_sub_sse || (sse == best_sub_sse && index < best_sub_grid))
			{
				best_sub_sse = sse;
				best_sub_grid = index;
			}
		}
	}
	group_sse += best_sub_sse;
	group_grid[sg] = best_sub_grid;
}
if (group_sse < best_sse)
{
	best_sse = group_sse;
	best_scale = scale;
	memcpy(best_grid, group_grid, sizeof(best_grid));
}
}
group->scale = best_scale;
group->signs = signs;
memcpy(group->grid, best_grid, sizeof(best_grid));
memset(group->q8, 0, sizeof(group->q8));
for (j = 0; j < 8; j++)
{
const int sg = j >> 2;
const int lane = j & 3;
const int mag = ccv_nnc_8i_rowwise_packed_iq3xxs_scaled_value[best_scale][best_grid[sg]][lane];
group->q8[j] = (signs & (1u << j)) ? -mag : mag;
}
1418}

1420static void _ccv_nnc_8i_rowwise_packed_quant_group(const int format, const double* const y, const double* const w, const int16_t* const iq1_map, ccv_nnc_8i_rowwise_packed_group_t* const group)
1421{
switch (format)
{
case CCV_NNC_QX_8I_ROWWISE_IQ1_S:
	_ccv_nnc_8i_rowwise_packed_quant_iq1_s(y, w, iq1_map, group);
	break;
case CCV_NNC_QX_8I_ROWWISE_IQ1_M:
	_ccv_nnc_8i_rowwise_packed_quant_iq1_m(y, w, iq1_map, group);
	break;
case CCV_NNC_QX_8I_ROWWISE_Q5_K:
	_ccv_nnc_8i_rowwise_packed_quant_q5(y, w, group);
	break;
case CCV_NNC_QX_8I_ROWWISE_Q6_K:
	_ccv_nnc_8i_rowwise_packed_quant_q6(y, w, group);
	break;
case CCV_NNC_QX_8I_ROWWISE_Q4_K:
	_ccv_nnc_8i_rowwise_packed_quant_q4(y, w, group);
	break;
case CCV_NNC_QX_8I_ROWWISE_Q3_K:
	_ccv_nnc_8i_rowwise_packed_quant_q3(y, w, group);
	break;
case CCV_NNC_QX_8I_ROWWISE_Q2_K:
	_ccv_nnc_8i_rowwise_packed_quant_q2(y, w, group);
	break;
case CCV_NNC_QX_8I_ROWWISE_IQ2_XXS:
	_ccv_nnc_8i_rowwise_packed_quant_iq2_xxs(y, w, group);
	break;
case CCV_NNC_QX_8I_ROWWISE_IQ2_S:
	_ccv_nnc_8i_rowwise_packed_quant_iq2_s(y, w, group);
	break;
case CCV_NNC_QX_8I_ROWWISE_IQ2_XS:
	_ccv_nnc_8i_rowwise_packed_quant_iq2_xs(y, w, group);
	break;
case CCV_NNC_QX_8I_ROWWISE_IQ3_S:
	_ccv_nnc_8i_rowwise_packed_quant_iq3_s(y, w, group);
	break;
case CCV_NNC_QX_8I_ROWWISE_IQ3_XXS:
	_ccv_nnc_8i_rowwise_packed_quant_iq3_xxs(y, w, group);
	break;
default:
	assert(0)((void) sizeof ((0) ? 1 : 0), __extension__ ({ if (0) ; else __assert_fail
 ("0", "ccv_nnc_8i_rowwise.c", 1461, __extension__ __PRETTY_FUNCTION__
); }));
}
1463}

1465static void _ccv_nnc_8i_rowwise_packed_pack_group(uint8_t* const output, const size_t group_index, const int format, const ccv_nnc_8i_rowwise_packed_group_t* const group)
1466{
const size_t bit_offset = group_index * ccv_nnc_8i_rowwise_x_group_bits(format);
size_t bit = bit_offset;
int j;
switch (format)
{
case CCV_NNC_QX_8I_ROWWISE_IQ1_S:
	for (j = 0; j < 4; j++)
		_ccv_nnc_8i_rowwise_packed_write_bits(output, bit + j * 11, (uint32_t)group->grid[j], 11);
	_ccv_nnc_8i_rowwise_packed_write_bits(output, bit + 44, group->signs & 7, 3);
	_ccv_nnc_8i_rowwise_packed_write_bits(output, bit + 47, (uint32_t)group->scale, 3);
	break;
case CCV_NNC_QX_8I_ROWWISE_IQ1_M:
	for (j = 0; j < 2; j++)
		_ccv_nnc_8i_rowwise_packed_write_bits(output, bit + j * 12, (uint32_t)group->grid[j] | (((group->signs >> j) & 1) << 11), 12);
	_ccv_nnc_8i_rowwise_packed_write_bits(output, bit + 24, (uint32_t)(group->scale - 1), 4);
	break;
case CCV_NNC_QX_8I_ROWWISE_Q5_K:
	for (j = 0; j < 16; j++, bit += 5)
		_ccv_nnc_8i_rowwise_packed_write_bits(output, bit, (uint32_t)(group->q[j] + 16), 5);
	_ccv_nnc_8i_rowwise_packed_write_bits(output, bit, (uint32_t)(group->m - 1), 3);
	_ccv_nnc_8i_rowwise_packed_write_bits(output, bit + 3, (uint32_t)(group->b + 16), 5);
	break;
case CCV_NNC_QX_8I_ROWWISE_Q6_K:
	for (j = 0; j < 8; j++, bit += 6)
		_ccv_nnc_8i_rowwise_packed_write_bits(output, bit, (uint32_t)(group->q[j] & 0x3f), 6);
	_ccv_nnc_8i_rowwise_packed_write_bits(output, bit, (uint32_t)(group->m - 1), 2);
	_ccv_nnc_8i_rowwise_packed_write_bits(output, bit + 2, (uint32_t)(group->b & 3), 2);
	break;
case CCV_NNC_QX_8I_ROWWISE_Q4_K:
	for (j = 0; j < 16; j++, bit += 4)
		_ccv_nnc_8i_rowwise_packed_write_bits(output, bit, (uint32_t)(group->q[j] + 8), 4);
	_ccv_nnc_8i_rowwise_packed_write_bits(output, bit, (uint32_t)(group->m - 1), 4);
	_ccv_nnc_8i_rowwise_packed_write_bits(output, bit + 4, (uint32_t)(group->b + 8), 4);
	break;
case CCV_NNC_QX_8I_ROWWISE_Q3_K:
	for (j = 0; j < 16; j++, bit += 3)
		_ccv_nnc_8i_rowwise_packed_write_bits(output, bit, (uint32_t)(group->q[j] + 4), 3);
	_ccv_nnc_8i_rowwise_packed_write_bits(output, bit, (uint32_t)(group->m - 1), 5);
	_ccv_nnc_8i_rowwise_packed_write_bits(output, bit + 5, (uint32_t)(group->b / 2 + 4), 3);
	break;
case CCV_NNC_QX_8I_ROWWISE_Q2_K:
	for (j = 0; j < 16; j++, bit += 2)
		_ccv_nnc_8i_rowwise_packed_write_bits(output, bit, (uint32_t)group->q[j], 2);
	_ccv_nnc_8i_rowwise_packed_write_bits(output, bit, (uint32_t)(group->m - 1), 6);
	_ccv_nnc_8i_rowwise_packed_write_bits(output, bit + 6, (uint32_t)(group->z >> 3), 4);
	break;
case CCV_NNC_QX_8I_ROWWISE_IQ2_S:
	_ccv_nnc_8i_rowwise_packed_write_bits(output, bit, (uint32_t)group->grid[0], 10);
	_ccv_nnc_8i_rowwise_packed_write_bits(output, bit + 10, (uint32_t)group->grid[1], 10);
	_ccv_nnc_8i_rowwise_packed_write_bits(output, bit + 20, group->signs, 16);
	_ccv_nnc_8i_rowwise_packed_write_bits(output, bit + 36, (uint32_t)(group->scale - 1), 6);
	break;
case CCV_NNC_QX_8I_ROWWISE_IQ2_XS:
	_ccv_nnc_8i_rowwise_packed_write_bits(output, bit, (uint32_t)group->grid[0], 9);
	_ccv_nnc_8i_rowwise_packed_write_bits(output, bit + 9, group->signs, 8);
	_ccv_nnc_8i_rowwise_packed_write_bits(output, bit + 17, (uint32_t)group->scale, 4);
	break;
case CCV_NNC_QX_8I_ROWWISE_IQ2_XXS:
	for (j = 0; j < 4; j++)
		_ccv_nnc_8i_rowwise_packed_write_bits(output, bit + j * 8, (uint32_t)group->grid[j], 8);
	_ccv_nnc_8i_rowwise_packed_write_bits(output, bit + 32, group->signs, 28);
	_ccv_nnc_8i_rowwise_packed_write_bits(output, bit + 60, (uint32_t)group->scale, 4);
	break;
case CCV_NNC_QX_8I_ROWWISE_IQ3_S:
	for (j = 0; j < 4; j++)
		_ccv_nnc_8i_rowwise_packed_write_bits(output, bit + j * 9, (uint32_t)group->grid[j], 9);
	_ccv_nnc_8i_rowwise_packed_write_bits(output, bit + 36, group->signs, 16);
	_ccv_nnc_8i_rowwise_packed_write_bits(output, bit + 52, (uint32_t)(group->scale - 1), 4);
	break;
case CCV_NNC_QX_8I_ROWWISE_IQ3_XXS:
	_ccv_nnc_8i_rowwise_packed_write_bits(output, bit, (uint32_t)group->grid[0], 8);
	_ccv_nnc_8i_rowwise_packed_write_bits(output, bit + 8, (uint32_t)group->grid[1], 8);
	_ccv_nnc_8i_rowwise_packed_write_bits(output, bit + 16, group->signs, 8);
	_ccv_nnc_8i_rowwise_packed_write_bits(output, bit + 24, (uint32_t)(group->scale - 1), 4);
	break;
default:
	assert(0)((void) sizeof ((0) ? 1 : 0), __extension__ ({ if (0) ; else __assert_fail
 ("0", "ccv_nnc_8i_rowwise.c", 1543, __extension__ __PRETTY_FUNCTION__
); }));
}
1545}

1547static void _ccv_nnc_8i_rowwise_packed_decode_group(const uint8_t* const input, const size_t group_index, const int format, int* const q8)
1548{
static const int q2_xs_scales[16] = {1, 2, 3, 4, 5, 6, 7, 8, 10, 12, 14, 16, 20, 24, 28, 32};
const size_t bit_offset = group_index * ccv_nnc_8i_rowwise_x_group_bits(format);
size_t bit = bit_offset;
int j;
switch (format)
{
case CCV_NNC_QX_8I_ROWWISE_IQ1_S: {
	uint32_t shifts = _ccv_nnc_8i_rowwise_packed_read_bits(input, bit + 44, 3);
	shifts |= ((shifts ^ (shifts >> 1) ^ (shifts >> 2)) & 1) << 3;
	const int m = ccv_nnc_8i_rowwise_packed_iq1s_scales[_ccv_nnc_8i_rowwise_packed_read_bits(input, bit + 47, 3)];
	int sg;
	for (sg = 0; sg < 4; sg++)
	{
		const int index = (int)_ccv_nnc_8i_rowwise_packed_read_bits(input, bit + sg * 11, 11);
		const int shift = (shifts & (1u << sg)) ? -1 : 1;
		for (j = 0; j < 8; j++)
		{
			const int g = (int)((ccv_nnc_8i_rowwise_packed_iq1_grid[index] >> (2 * j)) & 3) - 1;
			q8[sg * 8 + j] = ccv_clamp(m * (8 * g + shift), -127, 127)({ typeof (-127) _a = (-127); typeof (127) _b = (127); typeof
 (m * (8 * g + shift)) _x = (m * (8 * g + shift)); (_x < _a
) ? _a : ((_x > _b) ? _b : _x); });
		}
	}
	break;
}
case CCV_NNC_QX_8I_ROWWISE_IQ1_M: {
	const int m = (int)_ccv_nnc_8i_rowwise_packed_read_bits(input, bit + 24, 4) + 1;
	int sg;
	for (sg = 0; sg < 2; sg++)
	{
		const uint32_t code = _ccv_nnc_8i_rowwise_packed_read_bits(input, bit + sg * 12, 12);
		const int shift = (code & 2048) ? -1 : 1;
		for (j = 0; j < 8; j++)
		{
			const int g = (int)((ccv_nnc_8i_rowwise_packed_iq1_grid[code & 2047] >> (2 * j)) & 3) - 1;
			q8[sg * 8 + j] = ccv_clamp(m * (8 * g + shift), -127, 127)({ typeof (-127) _a = (-127); typeof (127) _b = (127); typeof
 (m * (8 * g + shift)) _x = (m * (8 * g + shift)); (_x < _a
) ? _a : ((_x > _b) ? _b : _x); });
		}
	}
	break;
}
case CCV_NNC_QX_8I_ROWWISE_Q5_K: {
	int q[16];
	for (j = 0; j < 16; j++, bit += 5)
		q[j] = (int)_ccv_nnc_8i_rowwise_packed_read_bits(input, bit, 5) - 16;
	const int m = (int)_ccv_nnc_8i_rowwise_packed_read_bits(input, bit, 3) + 1;
	const int b = (int)_ccv_nnc_8i_rowwise_packed_read_bits(input, bit + 3, 5) - 16;
	for (j = 0; j < 16; j++)
		q8[j] = q[j] * m + b;
	break;
}
case CCV_NNC_QX_8I_ROWWISE_Q6_K: {
	int q[8];
	for (j = 0; j < 8; j++, bit += 6)
		q[j] = _ccv_nnc_8i_rowwise_packed_sign_extend(_ccv_nnc_8i_rowwise_packed_read_bits(input, bit, 6), 6);
	const int m = (int)_ccv_nnc_8i_rowwise_packed_read_bits(input, bit, 2) + 1;
	const int b = _ccv_nnc_8i_rowwise_packed_sign_extend(_ccv_nnc_8i_rowwise_packed_read_bits(input, bit + 2, 2), 2);
	for (j = 0; j < 8; j++)
		q8[j] = q[j] * m + b;
	break;
}
case CCV_NNC_QX_8I_ROWWISE_Q4_K: {
	int q[16];
	for (j = 0; j < 16; j++, bit += 4)
		q[j] = (int)_ccv_nnc_8i_rowwise_packed_read_bits(input, bit, 4) - 8;
	const int m = (int)_ccv_nnc_8i_rowwise_packed_read_bits(input, bit, 4) + 1;
	const int b = (int)_ccv_nnc_8i_rowwise_packed_read_bits(input, bit + 4, 4) - 8;
	for (j = 0; j < 16; j++)
		q8[j] = q[j] * m + b;
	break;
}
case CCV_NNC_QX_8I_ROWWISE_Q3_K: {
	int q[16];
	for (j = 0; j < 16; j++, bit += 3)
		q[j] = (int)_ccv_nnc_8i_rowwise_packed_read_bits(input, bit, 3) - 4;
	const int m = (int)_ccv_nnc_8i_rowwise_packed_read_bits(input, bit, 5) + 1;
	const int b = ((int)_ccv_nnc_8i_rowwise_packed_read_bits(input, bit + 5, 3) - 4) << 1;
	for (j = 0; j < 16; j++)
		q8[j] = q[j] * m + b;
	break;
}
case CCV_NNC_QX_8I_ROWWISE_Q2_K: {
	int q[16];
	for (j = 0; j < 16; j++, bit += 2)
		q[j] = (int)_ccv_nnc_8i_rowwise_packed_read_bits(input, bit, 2);
	const int m = (int)_ccv_nnc_8i_rowwise_packed_read_bits(input, bit, 6) + 1;
	const int z = (int)_ccv_nnc_8i_rowwise_packed_read_bits(input, bit + 6, 4) << 3;
	for (j = 0; j < 16; j++)
		q8[j] = q[j] * m - z;
	break;
}
case CCV_NNC_QX_8I_ROWWISE_IQ2_S: {
	const int grid0 = (int)_ccv_nnc_8i_rowwise_packed_read_bits(input, bit, 10);
	const int grid1 = (int)_ccv_nnc_8i_rowwise_packed_read_bits(input, bit + 10, 10);
	const uint32_t signs = _ccv_nnc_8i_rowwise_packed_read_bits(input, bit + 20, 16);
	const int scale = (int)_ccv_nnc_8i_rowwise_packed_read_bits(input, bit + 36, 6) + 1;
	for (j = 0; j < 8; j++)
	{
		const int mag0 = ccv_min(_ccv_nnc_8i_rowwise_packed_iq2_value(ccv_nnc_8i_rowwise_packed_iq2s_grid, grid0, j) * scale, 127)({ typeof (_ccv_nnc_8i_rowwise_packed_iq2_value(ccv_nnc_8i_rowwise_packed_iq2s_grid
, grid0, j) * scale) _a = (_ccv_nnc_8i_rowwise_packed_iq2_value
(ccv_nnc_8i_rowwise_packed_iq2s_grid, grid0, j) * scale); typeof
 (127) _b = (127); (_a < _b) ? _a : _b; });
		const int mag1 = ccv_min(_ccv_nnc_8i_rowwise_packed_iq2_value(ccv_nnc_8i_rowwise_packed_iq2s_grid, grid1, j) * scale, 127)({ typeof (_ccv_nnc_8i_rowwise_packed_iq2_value(ccv_nnc_8i_rowwise_packed_iq2s_grid
, grid1, j) * scale) _a = (_ccv_nnc_8i_rowwise_packed_iq2_value
(ccv_nnc_8i_rowwise_packed_iq2s_grid, grid1, j) * scale); typeof
 (127) _b = (127); (_a < _b) ? _a : _b; });
		q8[j] = (signs & (1u << j)) ? -mag0 : mag0;
		q8[8 + j] = (signs & (1u << (8 + j))) ? -mag1 : mag1;
	}
	break;
}
case CCV_NNC_QX_8I_ROWWISE_IQ2_XS: {
	const int grid0 = (int)_ccv_nnc_8i_rowwise_packed_read_bits(input, bit, 9);
	const uint32_t signs = _ccv_nnc_8i_rowwise_packed_read_bits(input, bit + 9, 8);
	const int scale = q2_xs_scales[_ccv_nnc_8i_rowwise_packed_read_bits(input, bit + 17, 4)];
	for (j = 0; j < 8; j++)
	{
		const int mag = ccv_min(_ccv_nnc_8i_rowwise_packed_iq2_value(ccv_nnc_8i_rowwise_packed_iq2xs_grid, grid0, j) * scale, 127)({ typeof (_ccv_nnc_8i_rowwise_packed_iq2_value(ccv_nnc_8i_rowwise_packed_iq2xs_grid
, grid0, j) * scale) _a = (_ccv_nnc_8i_rowwise_packed_iq2_value
(ccv_nnc_8i_rowwise_packed_iq2xs_grid, grid0, j) * scale); typeof
 (127) _b = (127); (_a < _b) ? _a : _b; });
		q8[j] = (signs & (1u << j)) ? -mag : mag;
	}
	break;
}
case CCV_NNC_QX_8I_ROWWISE_IQ2_XXS: {
	int grid[4];
	for (j = 0; j < 4; j++)
		grid[j] = (int)_ccv_nnc_8i_rowwise_packed_read_bits(input, bit + j * 8, 8);
	const uint32_t sign_codes = _ccv_nnc_8i_rowwise_packed_read_bits(input, bit + 32, 28);
	const int scale = ccv_nnc_8i_rowwise_packed_iq2xxs_scales[_ccv_nnc_8i_rowwise_packed_read_bits(input, bit + 60, 4)];
	int sg;
	for (sg = 0; sg < 4; sg++)
	{
		const uint8_t signs = ccv_nnc_8i_rowwise_packed_iq2xxs_ksigns[(sign_codes >> (sg * 7)) & 0x7f];
		for (j = 0; j < 8; j++)
		{
			const int lane = sg * 8 + j;
			const int mag = ccv_min(_ccv_nnc_8i_rowwise_packed_iq2xxs_value(grid[sg], j) * scale, 127)({ typeof (_ccv_nnc_8i_rowwise_packed_iq2xxs_value(grid[sg], j
) * scale) _a = (_ccv_nnc_8i_rowwise_packed_iq2xxs_value(grid
[sg], j) * scale); typeof (127) _b = (127); (_a < _b) ? _a
 : _b; });
			q8[lane] = (signs & (1u << j)) ? -mag : mag;
		}
	}
	break;
}
case CCV_NNC_QX_8I_ROWWISE_IQ3_S: {
	int grid[4];
	for (j = 0; j < 4; j++)
		grid[j] = (int)_ccv_nnc_8i_rowwise_packed_read_bits(input, bit + j * 9, 9);
	const uint32_t signs = _ccv_nnc_8i_rowwise_packed_read_bits(input, bit + 36, 16);
	const int scale = (int)_ccv_nnc_8i_rowwise_packed_read_bits(input, bit + 52, 4) + 1;
	int sg;
	for (sg = 0; sg < 4; sg++)
		for (j = 0; j < 4; j++)
		{
			const int lane = sg * 4 + j;
			const int mag = ccv_min(_ccv_nnc_8i_rowwise_packed_iq3s_value(grid[sg], j) * scale, 127)({ typeof (_ccv_nnc_8i_rowwise_packed_iq3s_value(grid[sg], j)
 * scale) _a = (_ccv_nnc_8i_rowwise_packed_iq3s_value(grid[sg
], j) * scale); typeof (127) _b = (127); (_a < _b) ? _a : _b
; });
			q8[lane] = (signs & (1u << lane)) ? -mag : mag;
		}
	break;
}
case CCV_NNC_QX_8I_ROWWISE_IQ3_XXS: {
	const int grid0 = (int)_ccv_nnc_8i_rowwise_packed_read_bits(input, bit, 8);
	const int grid1 = (int)_ccv_nnc_8i_rowwise_packed_read_bits(input, bit + 8, 8);
	const uint32_t signs = _ccv_nnc_8i_rowwise_packed_read_bits(input, bit + 16, 8);
	const int scale = (int)_ccv_nnc_8i_rowwise_packed_read_bits(input, bit + 24, 4) + 1;
	for (j = 0; j < 4; j++)
	{
		const int mag0 = ccv_min(_ccv_nnc_8i_rowwise_packed_iq3xxs_value(grid0, j) * scale, 127)({ typeof (_ccv_nnc_8i_rowwise_packed_iq3xxs_value(grid0, j) *
 scale) _a = (_ccv_nnc_8i_rowwise_packed_iq3xxs_value(grid0, j
) * scale); typeof (127) _b = (127); (_a < _b) ? _a : _b; }
);
		const int mag1 = ccv_min(_ccv_nnc_8i_rowwise_packed_iq3xxs_value(grid1, j) * scale, 127)({ typeof (_ccv_nnc_8i_rowwise_packed_iq3xxs_value(grid1, j) *
 scale) _a = (_ccv_nnc_8i_rowwise_packed_iq3xxs_value(grid1, j
) * scale); typeof (127) _b = (127); (_a < _b) ? _a : _b; }
);
		q8[j] = (signs & (1u << j)) ? -mag0 : mag0;
		q8[4 + j] = (signs & (1u << (4 + j))) ? -mag1 : mag1;
	}
	break;
}
default:
	assert(0)((void) sizeof ((0) ? 1 : 0), __extension__ ({ if (0) ; else __assert_fail
 ("0", "ccv_nnc_8i_rowwise.c", 1712, __extension__ __PRETTY_FUNCTION__
); }));
}
1714}

1716// Regular normalized H256, matching NAInt8MatMul activation staging. This is
1717// not the Sylvester WHT. Transform the existing floating row before fitting scales.
1718static void _ccv_nnc_8i_rowwise_hadamard_256(double* const row, const size_t row_length)
1719{
size_t base;
for (base = 0; base < row_length; base += 256)
{
int stride;
for (stride = 1; stride < 256; stride *= 4)
{
	int i, j;
	for (i = 0; i < 256; i += stride * 4)
		for (j = 0; j < stride; j++)
		{
			double* const v = row + base + i + j;
			const double a = v[0], b = v[stride], c = v[stride * 2], d = v[stride * 3];
			v[0] = a + b + c - d;
			v[stride] = a + b - c + d;
			v[stride * 2] = a - b + c + d;
			v[stride * 3] = -a + b + c + d;
		}
}
int i;
for (i = 0; i < 256; i++)
	row[base + i] *= 1. / 16.;
}
1742}

1744CCV_WARN_UNUSED(size_t)size_t __attribute__((warn_unused_result)) ccv_nnc_quantize_8i_rowwise_x(const void* input, const int datatype, const int memory_type, const size_t input_length, const size_t row_length, const int format, const float* const imatrix, const size_t imatrix_length, void* output, const size_t output_length)
1745{
if (row_length == 0 || input_length % row_length != 0 ||
1
Assuming 'row_length' is not equal to 0→
2
←
Assuming the condition is false→
((format & CCV_NNC_QX_8I_ROWWISE_HADAMARD_256) && row_length % 256 != 0))
3
←
Assuming the condition is false→
return 0;
assert(datatype == CCV_16F || datatype == CCV_16BF || datatype == CCV_32F || datatype == CCV_64F)((void) sizeof ((datatype == CCV_16F || datatype == CCV_16BF ||
 datatype == CCV_32F || datatype == CCV_64F) ? 1 : 0), __extension__
 ({ if (datatype == CCV_16F || datatype == CCV_16BF || datatype
 == CCV_32F || datatype == CCV_64F) ; else __assert_fail ("datatype == CCV_16F || datatype == CCV_16BF || datatype == CCV_32F || datatype == CCV_64F"
, "ccv_nnc_8i_rowwise.c", 1749, __extension__ __PRETTY_FUNCTION__
); }));
4
←
Assuming 'datatype' is not equal to CCV_16F→
5
←
Assuming 'datatype' is not equal to CCV_16BF→
6
←
Assuming 'datatype' is not equal to CCV_32F→
7
←
Assuming 'datatype' is equal to CCV_64F→
8
←
Taking true branch→
assert(memory_type == CCV_TENSOR_CPU_MEMORY)((void) sizeof ((memory_type == CCV_TENSOR_CPU_MEMORY) ? 1 : 0
), __extension__ ({ if (memory_type == CCV_TENSOR_CPU_MEMORY)
 ; else __assert_fail ("memory_type == CCV_TENSOR_CPU_MEMORY"
, "ccv_nnc_8i_rowwise.c", 1750, __extension__ __PRETTY_FUNCTION__
); }));
9
←
Assuming 'memory_type' is equal to CCV_TENSOR_CPU_MEMORY→
10
←
Taking true branch→
assert(row_length > 0)((void) sizeof ((row_length > 0) ? 1 : 0), __extension__ (
{ if (row_length > 0) ; else __assert_fail ("row_length > 0"
, "ccv_nnc_8i_rowwise.c", 1751, __extension__ __PRETTY_FUNCTION__
); }));
11
←
Taking true branch→
assert(input_length % row_length == 0)((void) sizeof ((input_length % row_length == 0) ? 1 : 0), __extension__
 ({ if (input_length % row_length == 0) ; else __assert_fail (
"input_length % row_length == 0", "ccv_nnc_8i_rowwise.c", 1752
, __extension__ __PRETTY_FUNCTION__); }));
12
←
Taking true branch→
const int codec = format & CCV_NNC_QX_8I_ROWWISE_FORMAT_MASK;
const size_t row_count = input_length / row_length;
if (!_ccv_nnc_8i_rowwise_imatrix_is_valid(imatrix, imatrix_length, row_length, row_count))
13
←
Taking false branch→
return 0;
const size_t group_size = ccv_nnc_8i_rowwise_x_group_size(format);
const int group_bits = ccv_nnc_8i_rowwise_x_group_bits(format);
const size_t groups_per_row = (row_length + group_size - 1) / group_size;
const size_t padded_row_length = groups_per_row * group_size;
const size_t scale_offset = _ccv_nnc_8i_rowwise_packed_scale_offset(format, input_length, row_length);
const size_t output_size = scale_offset + row_count * CCV_GET_DATA_TYPE_SIZE(datatype)_ccv_get_data_type_size[((datatype) & 0xFF000) >> 12
];
if (output_length < output_size)
14
←
Assuming 'output_length' is >= 'output_size'→
15
←
Taking false branch→
return 0;
int16_t* iq1_map = 0;
const int starts = (codec == CCV_NNC_QX_8I_ROWWISE_IQ1_S || codec == CCV_NNC_QX_8I_ROWWISE_IQ1_M) ? 3 : 1;
16
←
Assuming 'codec' is not equal to CCV_NNC_QX_8I_ROWWISE_IQ1_S→
17
←
Assuming 'codec' is not equal to CCV_NNC_QX_8I_ROWWISE_IQ1_M→
18
←
'?' condition is false→
if (starts18.1
'starts' is not equal to 3
 == 3)
19
←
Taking false branch→
{
iq1_map = (int16_t*)ccmallocmalloc(sizeof(int16_t) * 6561);
int i, j;
for (i = 0; i < 6561; i++)
	iq1_map[i] = -1;
for (i = 0; i < 2048; i++)
{
	int code = 0;
	for (j = 0; j < 8; j++)
		code += ((ccv_nnc_8i_rowwise_packed_iq1_grid[i] >> (2 * j)) & 3) * ccv_nnc_8i_rowwise_packed_iq1_powers[j];
	iq1_map[code] = (int16_t)i;
}
}
switch (codec)
20
←
'Default' branch taken. Execution continues on line 1799→
{
case CCV_NNC_QX_8I_ROWWISE_IQ2_XXS:
	_ccv_nnc_8i_rowwise_packed_iq2xxs_candidates_init();
	break;
case CCV_NNC_QX_8I_ROWWISE_IQ2_XS:
	_ccv_nnc_8i_rowwise_packed_iq2xs_candidates_init();
	break;
case CCV_NNC_QX_8I_ROWWISE_IQ2_S:
	_ccv_nnc_8i_rowwise_packed_iq2s_init();
	break;
case CCV_NNC_QX_8I_ROWWISE_IQ3_S:
	_ccv_nnc_8i_rowwise_packed_iq3s_init();
	break;
case CCV_NNC_QX_8I_ROWWISE_IQ3_XXS:
	_ccv_nnc_8i_rowwise_packed_iq3xxs_candidates_init();
	break;
}
uint8_t* const u8 = (uint8_t*)output;
uint8_t* const scales = u8 + scale_offset;
memset(u8, 0, scale_offset);
const size_t row_bits = groups_per_row * group_bits;
size_t rows_per_chunk;
switch (row_bits & 7)
21
←
Control jumps to the 'default' case at line 1816→
{
case 0:
	rows_per_chunk = 1;
	break;
case 4:
	rows_per_chunk = 2;
	break;
case 2:
case 6:
	rows_per_chunk = 4;
	break;
default:
	rows_per_chunk = 8;
	break;
22
←
 Execution continues on line 1820→
}
rows_per_chunk = ccv_max(rows_per_chunk, (size_t)8)({ typeof (rows_per_chunk) _a = (rows_per_chunk); typeof ((size_t
)8) _b = ((size_t)8); (_a > _b) ? _a : _b; });
23
←
'?' condition is false→
const size_t row_chunks = (row_count + rows_per_chunk - 1) / rows_per_chunk;
1822#ifdef USE_DISPATCH
dispatch_apply(row_chunks, dispatch_get_global_queue(DISPATCH_QUEUE_PRIORITY_DEFAULT, 0), ^(size_t chunk_idx) {
1824#else
parallel_for(chunk_idx, (int)row_chunks){ int chunk_idx; for ((chunk_idx) = 0; (chunk_idx) < ((int
)row_chunks); (chunk_idx)++) { {
24
←
Assuming 'chunk_idx' is < 'row_chunks'→
25
←
Loop condition is true.  Entering loop body→
1826#endif
const size_t chunk_begin = (size_t)chunk_idx * rows_per_chunk;
const size_t chunk_end = ccv_min(chunk_begin + rows_per_chunk, row_count)({ typeof (chunk_begin + rows_per_chunk) _a = (chunk_begin + rows_per_chunk
); typeof (row_count) _b = (row_count); (_a < _b) ? _a : _b
; });
26
←
Assuming '_a' is >= '_b'→
27
←
'?' condition is false→
double* const row = (double*)ccmallocmalloc(sizeof(double) * padded_row_length);
28
←
Storing uninitialized value→
double* const weights = (double*)ccmallocmalloc(sizeof(double) * padded_row_length);
ccv_nnc_8i_rowwise_packed_group_t* const groups = (ccv_nnc_8i_rowwise_packed_group_t*)ccmallocmalloc(sizeof(ccv_nnc_8i_rowwise_packed_group_t) * groups_per_row);
ccv_nnc_8i_rowwise_packed_group_t* const best_groups = (ccv_nnc_8i_rowwise_packed_group_t*)ccmallocmalloc(sizeof(ccv_nnc_8i_rowwise_packed_group_t) * groups_per_row);
size_t i;
for (i = chunk_begin; i < chunk_end; i++)
29
←
Assuming 'i' is < 'chunk_end'→
30
←
Loop condition is true.  Entering loop body→
{
	const size_t row_start = i * row_length;
	const float* const row_imatrix = _ccv_nnc_8i_rowwise_imatrix_for_row(imatrix, imatrix_length, row_length, row_count, i);
	_ccv_nnc_8i_rowwise_packed_read_row(input, datatype, row_start, row_length, padded_row_length, row);
31
←
Calling '_ccv_nnc_8i_rowwise_packed_read_row'→
40
←
Returning from '_ccv_nnc_8i_rowwise_packed_read_row'→
	if (format & CCV_NNC_QX_8I_ROWWISE_HADAMARD_256)
41
←
Taking false branch→
		_ccv_nnc_8i_rowwise_hadamard_256(row, row_length);
	double max_abs = 0;
	size_t j;
	for (j = 0; j < row_length; j++)
42
←
Loop condition is true.  Entering loop body→
45
←
Loop condition is false. Execution continues on line 1848→
	{
		max_abs = ccv_max(max_abs, fabs(row[j]))({ typeof (max_abs) _a = (max_abs); typeof (fabs(row[j])) _b =
 (fabs(row[j])); (_a > _b) ? _a : _b; });
43
←
Assuming '_a' is <= '_b'→
44
←
'?' condition is false→
		weights[j] = _ccv_nnc_8i_rowwise_weight(row_imatrix, j);
	}
	for (; j45.1
'j' is >= 'padded_row_length'
 < padded_row_length; j++)
46
←
Loop condition is false. Execution continues on line 1850→
		weights[j] = 0;
	double best_scale = 0;
	double best_sse = DBL_MAX1.7976931348623157e+308;
	int has_best = 0;
	int start;
	for (start = 0; start < starts; start++)
47
←
Loop condition is true.  Entering loop body→
53
←
Loop condition is false. Execution continues on line 1911→
	{
		double scale = max_abs / 127.;
		if (starts47.1
'starts' is not equal to 3
 == 3)
48
←
Taking false branch→
		{
			scale *= start == 0 ? 0.75 : start == 1 ? 1. : 1.25;
			if (datatype == CCV_16F && max_abs > 0 && _ccv_nnc_8i_rowwise_packed_stored_scale(scale, datatype) == 0)
				scale = 0x1p-24;
		}
		int k;
		for (k = 0; k < CCV_NNC_8I_ROWWISE_X_REFINEMENT_STEPS; k++)
49
←
Loop condition is true.  Entering loop body→
		{
			const double stored_scale = _ccv_nnc_8i_rowwise_packed_stored_scale(scale, datatype);
			if (!(stored_scale > 0))
50
←
Assuming 'stored_scale' is <= 0→
51
←
Taking true branch→
				break;
52
←
 Execution continues on line 1854→
			double sse = 0;
			double sum_qx = 0;
			double sum_qq = 0;
			size_t g;
			for (g = 0; g < groups_per_row; g++)
			{
				double y[32] = {0};
				double w[32] = {0};
				for (j = 0; j < group_size; j++)
				{
					y[j] = row[g * group_size + j] / stored_scale;
					w[j] = weights[g * group_size + j];
				}
				ccv_nnc_8i_rowwise_packed_group_t group;
				_ccv_nnc_8i_rowwise_packed_quant_group(codec, y, w, iq1_map, &group);
				groups[g] = group;
				const size_t group_start = g * group_size;
				const size_t group_end = ccv_min(group_start + group_size, row_length)({ typeof (group_start + group_size) _a = (group_start + group_size
); typeof (row_length) _b = (row_length); (_a < _b) ? _a :
 _b; });
				for (j = group_start; j < group_end; j++)
				{
					const int q8 = group.q8[j - group_start];
					const double d = row[j] - stored_scale * q8;
					sse += weights[j] * d * d;
					sum_qx += weights[j] * q8 * row[j];
					sum_qq += weights[j] * q8 * q8;
				}
			}
			if (sse < best_sse)
			{
				best_sse = sse;
				best_scale = stored_scale;
				has_best = 1;
				memcpy(best_groups, groups, sizeof(ccv_nnc_8i_rowwise_packed_group_t) * groups_per_row);
			}
			if (!(sum_qq > 0) || !(sum_qx > 0))
				break;
			const double next_scale = sum_qx / sum_qq;
			if (_ccv_nnc_8i_rowwise_packed_stored_scale(next_scale, datatype) == stored_scale)
				break;
			scale = next_scale;
		}
	}
	_ccv_nnc_8i_rowwise_packed_store_scale(scales, datatype, i, best_scale);
	size_t g;
	if (has_best53.1
'has_best' is 0
)
54
←
Taking false branch→
	{
		for (g = 0; g < groups_per_row; g++)
			_ccv_nnc_8i_rowwise_packed_pack_group(u8, i * groups_per_row + g, codec, best_groups + g);
	} else {
		for (g = 0; g < groups_per_row; g++)
55
←
Assuming 'g' is < 'groups_per_row'→
56
←
Loop condition is true.  Entering loop body→
		{
			double y[32] = {0};
			double w[32] = {0};
			for (j = 0; j < group_size; j++)
57
←
Assuming 'j' is < 'group_size'→
58
←
Loop condition is true.  Entering loop body→
59
←
Assuming 'j' is < 'group_size'→
60
←
Loop condition is true.  Entering loop body→
			{
				y[j] = row[g * group_size + j];
61
←
Assigned value is garbage or undefined
				w[j] = weights[g * group_size + j];
			}
			ccv_nnc_8i_rowwise_packed_group_t group;
			_ccv_nnc_8i_rowwise_packed_quant_group(codec, y, w, iq1_map, &group);
			_ccv_nnc_8i_rowwise_packed_pack_group(u8, i * groups_per_row + g, codec, &group);
		}
	}
}
ccfreefree(best_groups);
ccfreefree(groups);
ccfreefree(weights);
ccfreefree(row);
1937#ifdef USE_DISPATCH
});
1939#else
} parallel_endfor} }
1941#endif
ccfreefree(iq1_map);
return output_size;
1944}

1946void ccv_nnc_dequantize_8i_rowwise_x(const void* input, const int datatype, const int memory_type, const size_t input_length, const size_t row_length, const int format, void* output, const size_t output_length)
1947{
assert(datatype == CCV_16F || datatype == CCV_16BF || datatype == CCV_32F || datatype == CCV_64F)((void) sizeof ((datatype == CCV_16F || datatype == CCV_16BF ||
 datatype == CCV_32F || datatype == CCV_64F) ? 1 : 0), __extension__
 ({ if (datatype == CCV_16F || datatype == CCV_16BF || datatype
 == CCV_32F || datatype == CCV_64F) ; else __assert_fail ("datatype == CCV_16F || datatype == CCV_16BF || datatype == CCV_32F || datatype == CCV_64F"
, "ccv_nnc_8i_rowwise.c", 1948, __extension__ __PRETTY_FUNCTION__
); }));
assert(memory_type == CCV_TENSOR_CPU_MEMORY || memory_type == CCV_TENSOR_GPU_MEMORY)((void) sizeof ((memory_type == CCV_TENSOR_CPU_MEMORY || memory_type
 == CCV_TENSOR_GPU_MEMORY) ? 1 : 0), __extension__ ({ if (memory_type
 == CCV_TENSOR_CPU_MEMORY || memory_type == CCV_TENSOR_GPU_MEMORY
) ; else __assert_fail ("memory_type == CCV_TENSOR_CPU_MEMORY || memory_type == CCV_TENSOR_GPU_MEMORY"
, "ccv_nnc_8i_rowwise.c", 1949, __extension__ __PRETTY_FUNCTION__
); }));
assert(row_length > 0)((void) sizeof ((row_length > 0) ? 1 : 0), __extension__ (
{ if (row_length > 0) ; else __assert_fail ("row_length > 0"
, "ccv_nnc_8i_rowwise.c", 1950, __extension__ __PRETTY_FUNCTION__
); }));
assert(output_length % row_length == 0)((void) sizeof ((output_length % row_length == 0) ? 1 : 0), __extension__
 ({ if (output_length % row_length == 0) ; else __assert_fail
 ("output_length % row_length == 0", "ccv_nnc_8i_rowwise.c", 1951
, __extension__ __PRETTY_FUNCTION__); }));
assert(!(format & CCV_NNC_QX_8I_ROWWISE_HADAMARD_256) || row_length % 256 == 0)((void) sizeof ((!(format & CCV_NNC_QX_8I_ROWWISE_HADAMARD_256
) || row_length % 256 == 0) ? 1 : 0), __extension__ ({ if (!(
format & CCV_NNC_QX_8I_ROWWISE_HADAMARD_256) || row_length
 % 256 == 0) ; else __assert_fail ("!(format & CCV_NNC_QX_8I_ROWWISE_HADAMARD_256) || row_length % 256 == 0"
, "ccv_nnc_8i_rowwise.c", 1952, __extension__ __PRETTY_FUNCTION__
); }));
if (memory_type != CCV_TENSOR_CPU_MEMORY)
{
1955#ifdef HAVE_CUDA1
ccv_nnc_compat_dequantize_8i_rowwise_x_fp(input, datatype, input_length, row_length, format, output, output_length, 0);
1957#elif defined(HAVE_MPS)
assert(datatype != CCV_64F)((void) sizeof ((datatype != CCV_64F) ? 1 : 0), __extension__
 ({ if (datatype != CCV_64F) ; else __assert_fail ("datatype != CCV_64F"
, "ccv_nnc_8i_rowwise.c", 1958, __extension__ __PRETTY_FUNCTION__
); }));
ccv_nnc_mps_dequantize_8i_rowwise_x(input, datatype, input_length, row_length, format, output, output_length, 0);
1960#else
assert(memory_type == CCV_TENSOR_CPU_MEMORY)((void) sizeof ((memory_type == CCV_TENSOR_CPU_MEMORY) ? 1 : 0
), __extension__ ({ if (memory_type == CCV_TENSOR_CPU_MEMORY)
 ; else __assert_fail ("memory_type == CCV_TENSOR_CPU_MEMORY"
, "ccv_nnc_8i_rowwise.c", 1961, __extension__ __PRETTY_FUNCTION__
); }));
1962#endif
return;
}
const size_t row_count = output_length / row_length;
const size_t group_size = ccv_nnc_8i_rowwise_x_group_size(format);
const size_t groups_per_row = (row_length + group_size - 1) / group_size;
const size_t scale_offset = _ccv_nnc_8i_rowwise_packed_scale_offset(format, output_length, row_length);
assert(input_length >= scale_offset + row_count * CCV_GET_DATA_TYPE_SIZE(datatype))((void) sizeof ((input_length >= scale_offset + row_count *
 _ccv_get_data_type_size[((datatype) & 0xFF000) >> 12
]) ? 1 : 0), __extension__ ({ if (input_length >= scale_offset
 + row_count * _ccv_get_data_type_size[((datatype) & 0xFF000
) >> 12]) ; else __assert_fail ("input_length >= scale_offset + row_count * CCV_GET_DATA_TYPE_SIZE(datatype)"
, "ccv_nnc_8i_rowwise.c", 1969, __extension__ __PRETTY_FUNCTION__
); }));
const uint8_t* const u8 = (const uint8_t*)input;
const uint8_t* const scales = u8 + scale_offset;
parallel_for(i, (int)row_count){ int i; for ((i) = 0; (i) < ((int)row_count); (i)++) { {
const double scale = _ccv_nnc_8i_rowwise_packed_load_scale(scales, datatype, i);
if (format & CCV_NNC_QX_8I_ROWWISE_HADAMARD_256)
{
	// Transform decoded integers before scaling / rounding to the destination
	// precision. H256 is its own inverse and acts on independent blocks.
	double block[256];
	size_t base;
	for (base = 0; base < row_length; base += 256)
	{
		size_t g;
		for (g = 0; g < 256 / group_size; g++)
		{
			int q8[32] = {0};
			_ccv_nnc_8i_rowwise_packed_decode_group(u8, (size_t)i * groups_per_row + base / group_size + g, format & CCV_NNC_QX_8I_ROWWISE_FORMAT_MASK, q8);
			size_t j;
			for (j = 0; j < group_size; j++)
				block[g * group_size + j] = q8[j];
		}
		_ccv_nnc_8i_rowwise_hadamard_256(block, 256);
		size_t j;
		for (j = 0; j < 256; j++)
			_ccv_nnc_8i_rowwise_packed_write_value(output, datatype, (size_t)i * row_length + base + j, scale * block[j]);
	}
} else {
	size_t g;
	for (g = 0; g < groups_per_row; g++)
	{
		int q8[32] = {0};
		_ccv_nnc_8i_rowwise_packed_decode_group(u8, (size_t)i * groups_per_row + g, format, q8);
		size_t j;
		for (j = 0; j < group_size; j++)
		{
			const size_t col = g * group_size + j;
			if (col < row_length)
				_ccv_nnc_8i_rowwise_packed_write_value(output, datatype, (size_t)i * row_length + col, scale * q8[j]);
		}
	}
}
} parallel_endfor} }
2012}

2014static inline int _ccv_nnc_8i_rowwise_quantize(const double v, const double inv_scale)
2015{
const int q = (int)lrint(v * inv_scale);
return ccv_clamp(q, -127, 127)({ typeof (-127) _a = (-127); typeof (127) _b = (127); typeof
 (q) _x = (q); (_x < _a) ? _a : ((_x > _b) ? _b : _x); }
);
2018}

2020static float _ccv_nnc_quantize_8i_rowwise_16f(const uint16_t* const row, const size_t row_length, const float* const imatrix, int8_t* const q)
2021{
size_t j;
double max_abs = 0;
for (j = 0; j < row_length; j++)
{
float v;
ccv_half_precision_to_float(row + j, &v, 1);
max_abs = ccv_max(max_abs, fabs(v))({ typeof (max_abs) _a = (max_abs); typeof (fabs(v)) _b = (fabs
(v)); (_a > _b) ? _a : _b; });
}
if (max_abs == 0)
{
memset(q, 0, row_length);
return 0;
}
double scale = max_abs / 127.;
float best_scale = 0;
double best_sse = DBL_MAX1.7976931348623157e+308;
int k;
for (k = 0; k < 8; k++)
{
// Round with the scale that will actually be stored, then refit scale by least squares.
const float scale_f = (float)scale;
uint16_t scale_h;
float stored_scale;
ccv_float_to_half_precision(&scale_f, &scale_h, 1);
ccv_half_precision_to_float(&scale_h, &stored_scale, 1);
if (!(stored_scale > 0))
	break;
const double inv_scale = 1. / stored_scale;
double sum_qx = 0;
double sum_qq = 0;
double sse = 0;
for (j = 0; j < row_length; j++)
{
	const double w = _ccv_nnc_8i_rowwise_weight(imatrix, j);
	float v_f;
	ccv_half_precision_to_float(row + j, &v_f, 1);
	const double v = v_f;
	const int qj = _ccv_nnc_8i_rowwise_quantize(v, inv_scale);
	const double d = v - stored_scale * qj;
	sse += w * d * d;
	sum_qx += w * qj * v;
	sum_qq += w * qj * qj;
}
if (sse < best_sse)
{
	best_sse = sse;
	best_scale = stored_scale;
}
if (!(sum_qq > 0) || !(sum_qx > 0))
	break;
const double next_scale = sum_qx / sum_qq;
const float next_scale_f = (float)next_scale;
uint16_t next_scale_h;
float next_stored_scale;
ccv_float_to_half_precision(&next_scale_f, &next_scale_h, 1);
ccv_half_precision_to_float(&next_scale_h, &next_stored_scale, 1);
if (next_stored_scale == stored_scale)
	break;
scale = next_scale;
}
if (!(best_scale > 0))
{
memset(q, 0, row_length);
return 0;
}
const double inv_scale = 1. / best_scale;
for (j = 0; j < row_length; j++)
{
float v;
ccv_half_precision_to_float(row + j, &v, 1);
q[j] = (int8_t)_ccv_nnc_8i_rowwise_quantize(v, inv_scale);
}
return best_scale;
2095}

2097static float _ccv_nnc_quantize_8i_rowwise_16bf(const uint16_t* const row, const size_t row_length, const float* const imatrix, int8_t* const q)
2098{
size_t j;
double max_abs = 0;
for (j = 0; j < row_length; j++)
{
float v;
ccv_bfloat_to_float(row + j, &v, 1);
max_abs = ccv_max(max_abs, fabs(v))({ typeof (max_abs) _a = (max_abs); typeof (fabs(v)) _b = (fabs
(v)); (_a > _b) ? _a : _b; });
}
if (max_abs == 0)
{
memset(q, 0, row_length);
return 0;
}
double scale = max_abs / 127.;
float best_scale = 0;
double best_sse = DBL_MAX1.7976931348623157e+308;
int k;
for (k = 0; k < 8; k++)
{
const float scale_f = (float)scale;
uint16_t scale_bf;
float stored_scale;
ccv_float_to_bfloat(&scale_f, &scale_bf, 1);
ccv_bfloat_to_float(&scale_bf, &stored_scale, 1);
if (!(stored_scale > 0))
	break;
const double inv_scale = 1. / stored_scale;
double sum_qx = 0;
double sum_qq = 0;
double sse = 0;
for (j = 0; j < row_length; j++)
{
	const double w = _ccv_nnc_8i_rowwise_weight(imatrix, j);
	float v_f;
	ccv_bfloat_to_float(row + j, &v_f, 1);
	const double v = v_f;
	const int qj = _ccv_nnc_8i_rowwise_quantize(v, inv_scale);
	const double d = v - stored_scale * qj;
	sse += w * d * d;
	sum_qx += w * qj * v;
	sum_qq += w * qj * qj;
}
if (sse < best_sse)
{
	best_sse = sse;
	best_scale = stored_scale;
}
if (!(sum_qq > 0) || !(sum_qx > 0))
	break;
const double next_scale = sum_qx / sum_qq;
const float next_scale_f = (float)next_scale;
uint16_t next_scale_bf;
float next_stored_scale;
ccv_float_to_bfloat(&next_scale_f, &next_scale_bf, 1);
ccv_bfloat_to_float(&next_scale_bf, &next_stored_scale, 1);
if (next_stored_scale == stored_scale)
	break;
scale = next_scale;
}
if (!(best_scale > 0))
{
memset(q, 0, row_length);
return 0;
}
const double inv_scale = 1. / best_scale;
for (j = 0; j < row_length; j++)
{
float v;
ccv_bfloat_to_float(row + j, &v, 1);
q[j] = (int8_t)_ccv_nnc_8i_rowwise_quantize(v, inv_scale);
}
return best_scale;
2171}

2173static float _ccv_nnc_quantize_8i_rowwise_32f(const float* const row, const size_t row_length, const float* const imatrix, int8_t* const q)
2174{
size_t j;
double max_abs = 0;
for (j = 0; j < row_length; j++)
max_abs = ccv_max(max_abs, fabs(row[j]))({ typeof (max_abs) _a = (max_abs); typeof (fabs(row[j])) _b =
 (fabs(row[j])); (_a > _b) ? _a : _b; });
if (max_abs == 0)
{
memset(q, 0, row_length);
return 0;
}
double scale = max_abs / 127.;
float best_scale = 0;
double best_sse = DBL_MAX1.7976931348623157e+308;
int k;
for (k = 0; k < 8; k++)
{
const float stored_scale = (float)scale;
if (!(stored_scale > 0))
	break;
const double inv_scale = 1. / stored_scale;
double sum_qx = 0;
double sum_qq = 0;
double sse = 0;
for (j = 0; j < row_length; j++)
{
	const double w = _ccv_nnc_8i_rowwise_weight(imatrix, j);
	const double v = row[j];
	const int qj = _ccv_nnc_8i_rowwise_quantize(v, inv_scale);
	const double d = v - stored_scale * qj;
	sse += w * d * d;
	sum_qx += w * qj * v;
	sum_qq += w * qj * qj;
}
if (sse < best_sse)
{
	best_sse = sse;
	best_scale = stored_scale;
}
if (!(sum_qq > 0) || !(sum_qx > 0))
	break;
const double next_scale = sum_qx / sum_qq;
if ((float)next_scale == stored_scale)
	break;
scale = next_scale;
}
if (!(best_scale > 0))
{
memset(q, 0, row_length);
return 0;
}
const double inv_scale = 1. / best_scale;
for (j = 0; j < row_length; j++)
q[j] = (int8_t)_ccv_nnc_8i_rowwise_quantize(row[j], inv_scale);
return best_scale;
2228}

2230static double _ccv_nnc_quantize_8i_rowwise_64f(const double* const row, const size_t row_length, const float* const imatrix, int8_t* const q)
2231{
size_t j;
double max_abs = 0;
for (j = 0; j < row_length; j++)
max_abs = ccv_max(max_abs, fabs(row[j]))({ typeof (max_abs) _a = (max_abs); typeof (fabs(row[j])) _b =
 (fabs(row[j])); (_a > _b) ? _a : _b; });
if (max_abs == 0)
{
memset(q, 0, row_length);
return 0;
}
double scale = max_abs / 127.;
double best_scale = 0;
double best_sse = DBL_MAX1.7976931348623157e+308;
int k;
for (k = 0; k < 8; k++)
{
const double stored_scale = scale;
if (!(stored_scale > 0))
	break;
const double inv_scale = 1. / stored_scale;
double sum_qx = 0;
double sum_qq = 0;
double sse = 0;
for (j = 0; j < row_length; j++)
{
	const double w = _ccv_nnc_8i_rowwise_weight(imatrix, j);
	const double v = row[j];
	const int qj = _ccv_nnc_8i_rowwise_quantize(v, inv_scale);
	const double d = v - stored_scale * qj;
	sse += w * d * d;
	sum_qx += w * qj * v;
	sum_qq += w * qj * qj;
}
if (sse < best_sse)
{
	best_sse = sse;
	best_scale = stored_scale;
}
if (!(sum_qq > 0) || !(sum_qx > 0))
	break;
const double next_scale = sum_qx / sum_qq;
if (next_scale == stored_scale)
	break;
scale = next_scale;
}
if (!(best_scale > 0))
{
memset(q, 0, row_length);
return 0;
}
const double inv_scale = 1. / best_scale;
for (j = 0; j < row_length; j++)
q[j] = (int8_t)_ccv_nnc_8i_rowwise_quantize(row[j], inv_scale);
return best_scale;
2285}

2287CCV_WARN_UNUSED(size_t)size_t __attribute__((warn_unused_result)) ccv_nnc_quantize_8i_rowwise(const void* input, const int datatype, const int memory_type, const size_t input_length, const size_t row_length, const float* const imatrix, const size_t imatrix_length, void* output, const size_t output_length)
2288{
assert(datatype == CCV_16F || datatype == CCV_16BF || datatype == CCV_32F || datatype == CCV_64F)((void) sizeof ((datatype == CCV_16F || datatype == CCV_16BF ||
 datatype == CCV_32F || datatype == CCV_64F) ? 1 : 0), __extension__
 ({ if (datatype == CCV_16F || datatype == CCV_16BF || datatype
 == CCV_32F || datatype == CCV_64F) ; else __assert_fail ("datatype == CCV_16F || datatype == CCV_16BF || datatype == CCV_32F || datatype == CCV_64F"
, "ccv_nnc_8i_rowwise.c", 2289, __extension__ __PRETTY_FUNCTION__
); }));
assert(memory_type == CCV_TENSOR_CPU_MEMORY)((void) sizeof ((memory_type == CCV_TENSOR_CPU_MEMORY) ? 1 : 0
), __extension__ ({ if (memory_type == CCV_TENSOR_CPU_MEMORY)
 ; else __assert_fail ("memory_type == CCV_TENSOR_CPU_MEMORY"
, "ccv_nnc_8i_rowwise.c", 2290, __extension__ __PRETTY_FUNCTION__
); }));
assert(row_length > 0)((void) sizeof ((row_length > 0) ? 1 : 0), __extension__ (
{ if (row_length > 0) ; else __assert_fail ("row_length > 0"
, "ccv_nnc_8i_rowwise.c", 2291, __extension__ __PRETTY_FUNCTION__
); }));
assert(input_length % row_length == 0)((void) sizeof ((input_length % row_length == 0) ? 1 : 0), __extension__
 ({ if (input_length % row_length == 0) ; else __assert_fail (
"input_length % row_length == 0", "ccv_nnc_8i_rowwise.c", 2292
, __extension__ __PRETTY_FUNCTION__); }));
const size_t row_count = input_length / row_length;
if (!_ccv_nnc_8i_rowwise_imatrix_is_valid(imatrix, imatrix_length, row_length, row_count))
return 0;
const size_t scale_offset = (input_length + 127) & -128;
const size_t scale_size = row_count * CCV_GET_DATA_TYPE_SIZE(datatype)_ccv_get_data_type_size[((datatype) & 0xFF000) >> 12
];
assert(output_length >= scale_offset + scale_size)((void) sizeof ((output_length >= scale_offset + scale_size
) ? 1 : 0), __extension__ ({ if (output_length >= scale_offset
 + scale_size) ; else __assert_fail ("output_length >= scale_offset + scale_size"
, "ccv_nnc_8i_rowwise.c", 2298, __extension__ __PRETTY_FUNCTION__
); }));
int8_t* const q = (int8_t*)output;
uint8_t* const u8 = (uint8_t*)output;
if (datatype == CCV_16F)
{
const uint16_t* const f16 = (const uint16_t*)input;
uint16_t* const scales = (uint16_t*)(u8 + scale_offset);
parallel_for(i, (int)row_count){ int i; for ((i) = 0; (i) < ((int)row_count); (i)++) { {
	const size_t row_start = (size_t)i * row_length;
	const float* const row_imatrix = _ccv_nnc_8i_rowwise_imatrix_for_row(imatrix, imatrix_length, row_length, row_count, (size_t)i);
	const float scale_f = _ccv_nnc_quantize_8i_rowwise_16f(f16 + row_start, row_length, row_imatrix, q + row_start);
	ccv_float_to_half_precision(&scale_f, scales + i, 1);
} parallel_endfor} }
} else if (datatype == CCV_16BF) {
const uint16_t* const bf16 = (const uint16_t*)input;
uint16_t* const scales = (uint16_t*)(u8 + scale_offset);
parallel_for(i, (int)row_count){ int i; for ((i) = 0; (i) < ((int)row_count); (i)++) { {
	const size_t row_start = (size_t)i * row_length;
	const float* const row_imatrix = _ccv_nnc_8i_rowwise_imatrix_for_row(imatrix, imatrix_length, row_length, row_count, (size_t)i);
	const float scale_f = _ccv_nnc_quantize_8i_rowwise_16bf(bf16 + row_start, row_length, row_imatrix, q + row_start);
	ccv_float_to_bfloat(&scale_f, scales + i, 1);
} parallel_endfor} }
} else if (datatype == CCV_32F) {
const float* const f32 = (const float*)input;
float* const scales = (float*)(u8 + scale_offset);
parallel_for(i, (int)row_count){ int i; for ((i) = 0; (i) < ((int)row_count); (i)++) { {
	const size_t row_start = (size_t)i * row_length;
	const float* const row_imatrix = _ccv_nnc_8i_rowwise_imatrix_for_row(imatrix, imatrix_length, row_length, row_count, (size_t)i);
	scales[i] = _ccv_nnc_quantize_8i_rowwise_32f(f32 + row_start, row_length, row_imatrix, q + row_start);
} parallel_endfor} }
} else {
assert(datatype == CCV_64F)((void) sizeof ((datatype == CCV_64F) ? 1 : 0), __extension__
 ({ if (datatype == CCV_64F) ; else __assert_fail ("datatype == CCV_64F"
, "ccv_nnc_8i_rowwise.c", 2329, __extension__ __PRETTY_FUNCTION__
); }));
const double* const f64 = (const double*)input;
double* const scales = (double*)(u8 + scale_offset);
parallel_for(i, (int)row_count){ int i; for ((i) = 0; (i) < ((int)row_count); (i)++) { {
	const size_t row_start = (size_t)i * row_length;
	const float* const row_imatrix = _ccv_nnc_8i_rowwise_imatrix_for_row(imatrix, imatrix_length, row_length, row_count, (size_t)i);
	scales[i] = _ccv_nnc_quantize_8i_rowwise_64f(f64 + row_start, row_length, row_imatrix, q + row_start);
} parallel_endfor} }
}
return scale_offset + scale_size;
2339}

2341void ccv_nnc_dequantize_8i_rowwise(const void* input, const int datatype, const int memory_type, const size_t input_length, const size_t row_length, void* output, const size_t output_length)
2342{
assert(datatype == CCV_16F || datatype == CCV_16BF || datatype == CCV_32F || datatype == CCV_64F)((void) sizeof ((datatype == CCV_16F || datatype == CCV_16BF ||
 datatype == CCV_32F || datatype == CCV_64F) ? 1 : 0), __extension__
 ({ if (datatype == CCV_16F || datatype == CCV_16BF || datatype
 == CCV_32F || datatype == CCV_64F) ; else __assert_fail ("datatype == CCV_16F || datatype == CCV_16BF || datatype == CCV_32F || datatype == CCV_64F"
, "ccv_nnc_8i_rowwise.c", 2343, __extension__ __PRETTY_FUNCTION__
); }));
assert(memory_type == CCV_TENSOR_CPU_MEMORY || memory_type == CCV_TENSOR_GPU_MEMORY)((void) sizeof ((memory_type == CCV_TENSOR_CPU_MEMORY || memory_type
 == CCV_TENSOR_GPU_MEMORY) ? 1 : 0), __extension__ ({ if (memory_type
 == CCV_TENSOR_CPU_MEMORY || memory_type == CCV_TENSOR_GPU_MEMORY
) ; else __assert_fail ("memory_type == CCV_TENSOR_CPU_MEMORY || memory_type == CCV_TENSOR_GPU_MEMORY"
, "ccv_nnc_8i_rowwise.c", 2344, __extension__ __PRETTY_FUNCTION__
); }));
assert(row_length > 0)((void) sizeof ((row_length > 0) ? 1 : 0), __extension__ (
{ if (row_length > 0) ; else __assert_fail ("row_length > 0"
, "ccv_nnc_8i_rowwise.c", 2345, __extension__ __PRETTY_FUNCTION__
); }));
assert(output_length % row_length == 0)((void) sizeof ((output_length % row_length == 0) ? 1 : 0), __extension__
 ({ if (output_length % row_length == 0) ; else __assert_fail
 ("output_length % row_length == 0", "ccv_nnc_8i_rowwise.c", 2346
, __extension__ __PRETTY_FUNCTION__); }));
if (memory_type != CCV_TENSOR_CPU_MEMORY)
{
2349#ifdef HAVE_CUDA1
ccv_nnc_compat_dequantize_8i_rowwise(input, datatype, input_length, row_length, output, output_length, 0);
2351#elif defined(HAVE_MPS)
assert(datatype != CCV_64F)((void) sizeof ((datatype != CCV_64F) ? 1 : 0), __extension__
 ({ if (datatype != CCV_64F) ; else __assert_fail ("datatype != CCV_64F"
, "ccv_nnc_8i_rowwise.c", 2352, __extension__ __PRETTY_FUNCTION__
); }));
ccv_nnc_mps_dequantize_8i_rowwise(input, datatype, input_length, row_length, output, output_length, 0);
2354#else
assert(memory_type == CCV_TENSOR_CPU_MEMORY)((void) sizeof ((memory_type == CCV_TENSOR_CPU_MEMORY) ? 1 : 0
), __extension__ ({ if (memory_type == CCV_TENSOR_CPU_MEMORY)
 ; else __assert_fail ("memory_type == CCV_TENSOR_CPU_MEMORY"
, "ccv_nnc_8i_rowwise.c", 2355, __extension__ __PRETTY_FUNCTION__
); }));
2356#endif
return;
}
const size_t row_count = output_length / row_length;
const size_t scale_offset = (output_length + 127) & -128;
assert(input_length >= scale_offset + row_count * CCV_GET_DATA_TYPE_SIZE(datatype))((void) sizeof ((input_length >= scale_offset + row_count *
 _ccv_get_data_type_size[((datatype) & 0xFF000) >> 12
]) ? 1 : 0), __extension__ ({ if (input_length >= scale_offset
 + row_count * _ccv_get_data_type_size[((datatype) & 0xFF000
) >> 12]) ; else __assert_fail ("input_length >= scale_offset + row_count * CCV_GET_DATA_TYPE_SIZE(datatype)"
, "ccv_nnc_8i_rowwise.c", 2361, __extension__ __PRETTY_FUNCTION__
); }));
const int8_t* const q = (const int8_t*)input;
const uint8_t* const u8 = (const uint8_t*)input;
if (datatype == CCV_16F)
{
uint16_t* const f16 = (uint16_t*)output;
const uint16_t* const scales = (const uint16_t*)(u8 + scale_offset);
parallel_for(i, (int)row_count){ int i; for ((i) = 0; (i) < ((int)row_count); (i)++) { {
	const size_t row_start = (size_t)i * row_length;
	float scale_f;
	ccv_half_precision_to_float(scales + i, &scale_f, 1);
	size_t j;
	for (j = 0; j < row_length; j++)
	{
		const float v = q[row_start + j] * scale_f;
		ccv_float_to_half_precision(&v, f16 + row_start + j, 1);
	}
} parallel_endfor} }
} else if (datatype == CCV_16BF) {
uint16_t* const bf16 = (uint16_t*)output;
const uint16_t* const scales = (const uint16_t*)(u8 + scale_offset);
parallel_for(i, (int)row_count){ int i; for ((i) = 0; (i) < ((int)row_count); (i)++) { {
	const size_t row_start = (size_t)i * row_length;
	float scale_f;
	ccv_bfloat_to_float(scales + i, &scale_f, 1);
	size_t j;
	for (j = 0; j < row_length; j++)
	{
		const float v = q[row_start + j] * scale_f;
		ccv_float_to_bfloat(&v, bf16 + row_start + j, 1);
	}
} parallel_endfor} }
} else if (datatype == CCV_32F) {
float* const f32 = (float*)output;
const float* const scales = (const float*)(u8 + scale_offset);
parallel_for(i, (int)row_count){ int i; for ((i) = 0; (i) < ((int)row_count); (i)++) { {
	const size_t row_start = (size_t)i * row_length;
	const float scale = scales[i];
	size_t j;
	for (j = 0; j < row_length; j++)
		f32[row_start + j] = q[row_start + j] * scale;
} parallel_endfor} }
} else {
assert(datatype == CCV_64F)((void) sizeof ((datatype == CCV_64F) ? 1 : 0), __extension__
 ({ if (datatype == CCV_64F) ; else __assert_fail ("datatype == CCV_64F"
, "ccv_nnc_8i_rowwise.c", 2404, __extension__ __PRETTY_FUNCTION__
); }));
double* const f64 = (double*)output;
const double* const scales = (const double*)(u8 + scale_offset);
parallel_for(i, (int)row_count){ int i; for ((i) = 0; (i) < ((int)row_count); (i)++) { {
	const size_t row_start = (size_t)i * row_length;
	const double scale = scales[i];
	size_t j;
	for (j = 0; j < row_length; j++)
		f64[row_start + j] = q[row_start + j] * scale;
} parallel_endfor} }
}
2415}