Coverage Report

Created: 2025-02-24 17:43

next uncovered line (L), next uncovered region (R), next uncovered branch (B)
/home/liu/actions-runner/_work/ccv/ccv/lib/ccv_memory.c
Line
Count
Source
1
#include "ccv.h"
2
#include "ccv_internal.h"
3
#include "3rdparty/siphash/siphash24.h"
4
5
static __thread ccv_cache_t ccv_cache;
6
7
/**
8
 * For new typed cache object:
9
 * ccv_dense_matrix_t: type 0
10
 * ccv_array_t: type 1
11
 **/
12
13
/* option to enable/disable cache */
14
static __thread int ccv_cache_opt = 0;
15
16
ccv_dense_matrix_t* ccv_dense_matrix_new(int rows, int cols, int type, void* data, uint64_t sig)
17
5.14M
{
18
5.14M
  ccv_dense_matrix_t* mat;
19
5.14M
  if (ccv_cache_opt && 
sig != 04.65M
&&
!data2.25M
&&
!(type & CCV_NO_DATA_ALLOC)2.25M
)
20
2.25M
  {
21
2.25M
    uint8_t type;
22
2.25M
    mat = (ccv_dense_matrix_t*)ccv_cache_out(&ccv_cache, sig, &type);
23
2.25M
    if (mat)
24
1.80M
    {
25
1.80M
      assert(type == 0);
26
1.80M
      mat->type |= CCV_GARBAGE; // set the flag so the upper level function knows this is from recycle-bin
27
1.80M
      mat->refcount = 1;
28
1.80M
      return mat;
29
1.80M
    }
30
2.25M
  }
31
3.34M
  if (type & CCV_NO_DATA_ALLOC)
32
1
  {
33
1
    mat = (ccv_dense_matrix_t*)ccmalloc(sizeof(ccv_dense_matrix_t));
34
1
    mat->type = (CCV_GET_CHANNEL(type) | CCV_GET_DATA_TYPE(type) | CCV_MATRIX_DENSE | CCV_NO_DATA_ALLOC) & ~CCV_GARBAGE;
35
1
    mat->data.u8 = data;
36
3.34M
  } else {
37
3.34M
    const size_t hdr_size = (sizeof(ccv_dense_matrix_t) + 63) & -64;
38
3.34M
    mat = (ccv_dense_matrix_t*)(data ? 
data0
: ccmalloc(ccv_compute_dense_matrix_size(rows, cols, type)));
39
3.34M
    mat->type = (CCV_GET_CHANNEL(type) | CCV_GET_DATA_TYPE(type) | CCV_MATRIX_DENSE) & ~CCV_GARBAGE;
40
3.34M
    mat->type |= data ? 
CCV_UNMANAGED0
: CCV_REUSABLE; // it still could be reusable because the signature could be derived one.
41
3.34M
    mat->data.u8 = (unsigned char*)mat + hdr_size;
42
3.34M
  }
43
3.34M
  mat->sig = sig;
44
3.34M
#if CCV_NNC_TENSOR_TFB
45
3.34M
  mat->reserved0 = 0;
46
3.34M
  mat->reserved1 = 0;
47
3.34M
  mat->reserved2 = 0;
48
3.34M
  mat->resides = CCV_TENSOR_CPU_MEMORY;
49
3.34M
  mat->format = CCV_TENSOR_FORMAT_NHWC;
50
3.34M
  mat->datatype = CCV_GET_DATA_TYPE(type);
51
3.34M
  mat->channels = CCV_GET_CHANNEL(type);
52
3.34M
  mat->reserved4 = 0;
53
3.34M
#endif
54
3.34M
  mat->rows = rows;
55
3.34M
  mat->cols = cols;
56
3.34M
  mat->step = CCV_GET_STEP(cols, type);
57
3.34M
  mat->refcount = 1;
58
3.34M
  return mat;
59
5.14M
}
60
61
ccv_dense_matrix_t* ccv_dense_matrix_renew(ccv_dense_matrix_t* x, int rows, int cols, int types, int prefer_type, uint64_t sig)
62
391k
{
63
391k
  if (x != 0)
64
83.7k
  {
65
83.7k
    assert(x->rows == rows && x->cols == cols && (CCV_GET_DATA_TYPE(x->type) & types) && (CCV_GET_CHANNEL(x->type) == CCV_GET_CHANNEL(types)));
66
83.7k
    prefer_type = CCV_GET_DATA_TYPE(x->type) | CCV_GET_CHANNEL(x->type);
67
83.7k
  }
68
391k
  if (sig != 0)
69
57
    sig = ccv_cache_generate_signature((const char*)&prefer_type, sizeof(int), sig, CCV_EOF_SIGN);
70
391k
  if (x == 0)
71
307k
  {
72
307k
    x = ccv_dense_matrix_new(rows, cols, prefer_type, 0, sig);
73
307k
  } else {
74
83.7k
    x->sig = sig;
75
83.7k
  }
76
391k
  return x;
77
391k
}
78
79
void ccv_make_matrix_mutable(ccv_matrix_t* mat)
80
0
{
81
0
  int type = *(int*)mat;
82
0
  if (type & CCV_MATRIX_DENSE)
83
0
  {
84
0
    ccv_dense_matrix_t* dmt = (ccv_dense_matrix_t*)mat;
85
0
    dmt->sig = 0;
86
0
    dmt->type &= ~CCV_REUSABLE;
87
0
  }
88
0
}
89
90
void ccv_make_matrix_immutable(ccv_matrix_t* mat)
91
108
{
92
108
  int type = *(int*)mat;
93
108
  if (type & CCV_MATRIX_DENSE)
94
108
  {
95
108
    ccv_dense_matrix_t* dmt = (ccv_dense_matrix_t*)mat;
96
108
    assert(dmt->sig == 0); // you cannot make matrix with derived signature immutable (it is immutable already)
97
    /* immutable matrix made this way is not reusable (collected), because its signature
98
     * only depends on the content, not the operation to generate it */
99
108
    dmt->type &= ~CCV_REUSABLE;
100
108
    dmt->sig = ccv_cache_generate_signature((char*)dmt->data.u8, dmt->rows * dmt->step, (uint64_t)dmt->type, CCV_EOF_SIGN);
101
108
  }
102
108
}
103
104
ccv_dense_matrix_t ccv_dense_matrix(int rows, int cols, int type, void* data, uint64_t sig)
105
3.99k
{
106
3.99k
  ccv_dense_matrix_t mat;
107
3.99k
  mat.reserved0 = 0;
108
3.99k
  mat.sig = sig;
109
3.99k
  mat.type = (CCV_GET_CHANNEL(type) | CCV_GET_DATA_TYPE(type) | CCV_MATRIX_DENSE | CCV_NO_DATA_ALLOC | CCV_UNMANAGED) & ~CCV_GARBAGE;
110
3.99k
  mat.rows = rows;
111
3.99k
  mat.cols = cols;
112
3.99k
  mat.step = CCV_GET_STEP(cols, type);
113
3.99k
  mat.refcount = 1;
114
3.99k
#if CCV_NNC_TENSOR_TFB
115
3.99k
  mat.reserved0 = 0;
116
3.99k
  mat.reserved1 = 0;
117
3.99k
  mat.reserved2 = 0;
118
3.99k
  mat.resides = CCV_TENSOR_CPU_MEMORY;
119
3.99k
  mat.format = CCV_TENSOR_FORMAT_NHWC;
120
3.99k
  mat.datatype = CCV_GET_DATA_TYPE(type);
121
3.99k
  mat.channels = CCV_GET_CHANNEL(type);
122
3.99k
  mat.reserved4 = 0;
123
3.99k
#endif
124
3.99k
  mat.data.u8 = (unsigned char*)data;
125
3.99k
  return mat;
126
3.99k
}
127
128
ccv_sparse_matrix_t* ccv_sparse_matrix_new(int rows, int cols, int type, int major, uint64_t sig)
129
25.3k
{
130
25.3k
  ccv_sparse_matrix_t* mat;
131
25.3k
  mat = (ccv_sparse_matrix_t*)ccmalloc(sizeof(ccv_sparse_matrix_t));
132
25.3k
  mat->rows = rows;
133
25.3k
  mat->cols = cols;
134
25.3k
  mat->type = type | CCV_MATRIX_SPARSE | ((type & CCV_DENSE_VECTOR) ? 
CCV_DENSE_VECTOR2
:
CCV_SPARSE_VECTOR25.3k
);
135
25.3k
  mat->major = major;
136
25.3k
  mat->prime_index = 1; // See ccv_util.c to know why this is 1 and why size is 2.
137
25.3k
  mat->size = 2;
138
25.3k
  mat->rnum = 0;
139
25.3k
  mat->refcount = 1;
140
25.3k
  mat->index = (ccv_sparse_matrix_index_t*)cccalloc(sizeof(ccv_sparse_matrix_index_t), mat->size);
141
25.3k
  mat->vector = (ccv_sparse_matrix_vector_t*)ccmalloc(sizeof(ccv_sparse_matrix_vector_t) * mat->size);
142
25.3k
  return mat;
143
25.3k
}
144
145
void ccv_matrix_free_immediately(ccv_matrix_t* mat)
146
2.85M
{
147
2.85M
  int type = *(int*)mat;
148
2.85M
  assert(!(type & CCV_UNMANAGED));
149
2.85M
  if (type & CCV_MATRIX_DENSE)
150
2.85M
  {
151
2.85M
    ccv_dense_matrix_t* dmt = (ccv_dense_matrix_t*)mat;
152
2.85M
    dmt->refcount = 0;
153
2.85M
    ccfree(dmt);
154
2.85M
  } else 
if (0
type & CCV_MATRIX_SPARSE0
) {
155
0
    ccv_sparse_matrix_t* smt = (ccv_sparse_matrix_t*)mat;
156
0
    int i;
157
0
    for (i = 0; i < smt->size; i++)
158
0
      if (smt->index[i].ifbit)
159
0
        ccfree(smt->vector[i].data.u8);
160
0
    ccfree(smt->vector);
161
0
    ccfree(smt);
162
0
  } else if ((type & CCV_MATRIX_CSR) || (type & CCV_MATRIX_CSC)) {
163
0
    ccv_compressed_sparse_matrix_t* csm = (ccv_compressed_sparse_matrix_t*)mat;
164
0
    csm->refcount = 0;
165
0
    ccfree(csm);
166
0
  }
167
2.85M
}
168
169
void ccv_matrix_free(ccv_matrix_t* mat)
170
2.73M
{
171
2.73M
  int type = *(int*)mat;
172
2.73M
  assert(!(type & CCV_UNMANAGED));
173
2.73M
  if (type & CCV_MATRIX_DENSE)
174
2.70M
  {
175
2.70M
    ccv_dense_matrix_t* dmt = (ccv_dense_matrix_t*)mat;
176
2.70M
    dmt->refcount = 0;
177
2.70M
    if (!ccv_cache_opt || // e don't enable cache
178
2.70M
      
!(dmt->type & CCV_REUSABLE)2.40M
|| // or this is not a reusable piece
179
2.70M
      
dmt->sig == 02.40M
|| // or this doesn't have valid signature
180
2.70M
      
(dmt->type & CCV_NO_DATA_ALLOC)2.40M
) // or this matrix is allocated as header-only, therefore we cannot cache it
181
307k
      ccfree(dmt);
182
2.40M
    else {
183
2.40M
      assert(CCV_GET_DATA_TYPE(dmt->type) == CCV_8U ||
184
2.40M
           CCV_GET_DATA_TYPE(dmt->type) == CCV_32S ||
185
2.40M
           CCV_GET_DATA_TYPE(dmt->type) == CCV_32F ||
186
2.40M
           CCV_GET_DATA_TYPE(dmt->type) == CCV_64S ||
187
2.40M
           CCV_GET_DATA_TYPE(dmt->type) == CCV_64F);
188
2.40M
      size_t size = ccv_compute_dense_matrix_size(dmt->rows, dmt->cols, dmt->type);
189
2.40M
      ccv_cache_put(&ccv_cache, dmt->sig, dmt, size, 0 /* type 0 */);
190
2.40M
    }
191
2.70M
  } else 
if (25.3k
type & CCV_MATRIX_SPARSE25.3k
) {
192
25.3k
    ccv_sparse_matrix_t* smt = (ccv_sparse_matrix_t*)mat;
193
25.3k
    int i;
194
206k
    for (i = 0; i < smt->size; 
i++181k
)
195
181k
    {
196
181k
      if (smt->index[i].ifbit > 1)
197
93.4k
        ccfree(smt->vector[i].index); // It is a union of index / data, can just free them.
198
181k
    }
199
25.3k
    ccfree(smt->index);
200
25.3k
    ccfree(smt->vector);
201
25.3k
    ccfree(smt);
202
25.3k
  } else 
if (1
(type & CCV_MATRIX_CSR)1
||
(type & CCV_MATRIX_CSC)0
) {
203
1
    ccv_compressed_sparse_matrix_t* csm = (ccv_compressed_sparse_matrix_t*)mat;
204
1
    csm->refcount = 0;
205
1
    ccfree(csm);
206
1
  }
207
2.73M
}
208
209
ccv_array_t* ccv_array_new(int rsize, int rnum, uint64_t sig)
210
2.06M
{
211
2.06M
  ccv_array_t* array;
212
2.06M
  if (ccv_cache_opt && 
sig != 01.55M
)
213
751k
  {
214
751k
    uint8_t type;
215
751k
    array = (ccv_array_t*)ccv_cache_out(&ccv_cache, sig, &type);
216
751k
    if (array)
217
720k
    {
218
720k
      assert(type == 1);
219
720k
      array->type |= CCV_GARBAGE;
220
720k
      array->refcount = 1;
221
720k
      return array;
222
720k
    }
223
751k
  }
224
1.34M
  array = (ccv_array_t*)ccmalloc(sizeof(ccv_array_t));
225
1.34M
  array->sig = sig;
226
1.34M
  array->type = CCV_REUSABLE & ~CCV_GARBAGE;
227
1.34M
  array->rnum = 0;
228
1.34M
  array->rsize = rsize;
229
1.34M
  array->size = ccv_max(rnum, 2 /* allocate memory for at least 2 items */);
230
1.34M
  array->data = ccmalloc((size_t)array->size * (size_t)rsize);
231
1.34M
  return array;
232
2.06M
}
233
234
void ccv_make_array_mutable(ccv_array_t* array)
235
0
{
236
0
  array->sig = 0;
237
0
  array->type &= ~CCV_REUSABLE;
238
0
}
239
240
void ccv_make_array_immutable(ccv_array_t* array)
241
0
{
242
0
  assert(array->sig == 0);
243
0
  array->type &= ~CCV_REUSABLE;
244
  /* TODO: trim the array */
245
0
  array->sig = ccv_cache_generate_signature(array->data, array->size * array->rsize, (uint64_t)array->rsize, CCV_EOF_SIGN);
246
0
}
247
248
void ccv_array_free_immediately(ccv_array_t* array)
249
831k
{
250
831k
  array->refcount = 0;
251
831k
  ccfree(array->data);
252
831k
  ccfree(array);
253
831k
}
254
255
void ccv_array_free(ccv_array_t* array)
256
1.31M
{
257
1.31M
  if (!ccv_cache_opt || 
!(array->type & CCV_REUSABLE)800k
||
array->sig == 0800k
)
258
512k
  {
259
512k
    array->refcount = 0;
260
512k
    ccfree(array->data);
261
512k
    ccfree(array);
262
800k
  } else {
263
800k
    size_t size = sizeof(ccv_array_t) + array->size * array->rsize;
264
800k
    ccv_cache_put(&ccv_cache, array->sig, array, size, 1 /* type 1 */);
265
800k
  }
266
1.31M
}
267
268
void ccv_drain_cache(void)
269
0
{
270
0
  if (ccv_cache.rnum > 0)
271
0
    ccv_cache_cleanup(&ccv_cache);
272
0
}
273
274
void ccv_disable_cache(void)
275
3
{
276
3
  ccv_cache_opt = 0;
277
3
  ccv_cache_close(&ccv_cache);
278
3
}
279
280
void ccv_enable_cache(size_t size)
281
3
{
282
3
  ccv_cache_opt = 1;
283
3
  ccv_cache_init(&ccv_cache, size, 2, ccv_matrix_free_immediately, ccv_array_free_immediately);
284
3
}
285
286
void ccv_enable_default_cache(void)
287
0
{
288
0
  ccv_enable_cache(CCV_DEFAULT_CACHE_SIZE);
289
0
}
290
291
static uint8_t key_siphash[16] = "libccvky4siphash";
292
293
uint64_t ccv_cache_generate_signature(const char* msg, int len, uint64_t sig_start, ...)
294
6.20M
{
295
6.20M
  uint64_t sig_out, sig_in[2]; // 1 is in, 0 is out
296
6.20M
  siphash((uint8_t*)&sig_out, (const uint8_t*)msg, len, key_siphash);
297
6.20M
  va_list arguments;
298
6.20M
  va_start(arguments, sig_start);
299
6.20M
  sig_in[0] = sig_out;
300
6.20M
  sig_in[1] = sig_start;
301
6.20M
  while (sig_in[1] != 0)
302
227
  {
303
227
    siphash((uint8_t*)&sig_out, (const uint8_t*)sig_in, sizeof(uint64_t) * 2, key_siphash);
304
227
    sig_in[0] = sig_out;
305
227
    sig_in[1] = va_arg(arguments, uint64_t);
306
227
  }
307
  va_end(arguments);
308
6.20M
  return sig_out;
309
6.20M
}