Coverage Report

Created: 2021-04-12 03:25

/home/liu/buildslave/linux-x64-runtests/build/lib/ccv_memory.c
Line
Count
Source (jump to first uncovered line)
1
#include "ccv.h"
2
#include "ccv_internal.h"
3
#include "3rdparty/siphash/siphash24.h"
4
5
static __thread ccv_cache_t ccv_cache;
6
7
/**
8
 * For new typed cache object:
9
 * ccv_dense_matrix_t: type 0
10
 * ccv_array_t: type 1
11
 **/
12
13
/* option to enable/disable cache */
14
static __thread int ccv_cache_opt = 0;
15
16
ccv_dense_matrix_t* ccv_dense_matrix_new(int rows, int cols, int type, void* data, uint64_t sig)
17
5.14M
{
18
5.14M
  ccv_dense_matrix_t* mat;
19
5.14M
  if (ccv_cache_opt && 
sig != 04.65M
&&
!data2.25M
&&
!(type & CCV_NO_DATA_ALLOC)2.25M
)
20
2.25M
  {
21
2.25M
    uint8_t type;
22
2.25M
    mat = (ccv_dense_matrix_t*)ccv_cache_out(&ccv_cache, sig, &type);
23
2.25M
    if (mat)
24
1.80M
    {
25
1.80M
      assert(type == 0);
26
1.80M
      mat->type |= CCV_GARBAGE; // set the flag so the upper level function knows this is from recycle-bin
27
1.80M
      mat->refcount = 1;
28
1.80M
      return mat;
29
3.34M
    }
30
2.25M
  }
31
3.34M
  if (type & CCV_NO_DATA_ALLOC)
32
1
  {
33
1
    mat = (ccv_dense_matrix_t*)ccmalloc(sizeof(ccv_dense_matrix_t));
34
1
    mat->type = (CCV_GET_CHANNEL(type) | CCV_GET_DATA_TYPE(type) | CCV_MATRIX_DENSE | CCV_NO_DATA_ALLOC) & ~CCV_GARBAGE;
35
1
    mat->data.u8 = data;
36
3.34M
  } else {
37
3.34M
    const size_t hdr_size = (sizeof(ccv_dense_matrix_t) + 15) & -16;
38
3.34M
    mat = (ccv_dense_matrix_t*)(data ? 
data0
: ccmalloc(ccv_compute_dense_matrix_size(rows, cols, type)));
39
3.34M
    mat->type = (CCV_GET_CHANNEL(type) | CCV_GET_DATA_TYPE(type) | CCV_MATRIX_DENSE) & ~CCV_GARBAGE;
40
3.34M
    mat->type |= data ? 
CCV_UNMANAGED0
: CCV_REUSABLE; // it still could be reusable because the signature could be derived one.
41
3.34M
    mat->data.u8 = (unsigned char*)mat + hdr_size;
42
3.34M
  }
43
3.34M
  mat->sig = sig;
44
3.34M
#if CCV_NNC_TENSOR_TFB
45
3.34M
  mat->reserved0 = 0;
46
3.34M
  mat->resides = CCV_TENSOR_CPU_MEMORY;
47
3.34M
  mat->format = CCV_TENSOR_FORMAT_NHWC;
48
3.34M
  mat->datatype = CCV_GET_DATA_TYPE(type);
49
3.34M
  mat->channels = CCV_GET_CHANNEL(type);
50
3.34M
  mat->reserved1 = 0;
51
3.34M
  mat->reserved2 = 0;
52
3.34M
#endif
53
3.34M
  mat->rows = rows;
54
3.34M
  mat->cols = cols;
55
3.34M
  mat->step = CCV_GET_STEP(cols, type);
56
3.34M
  mat->refcount = 1;
57
3.34M
  return mat;
58
3.34M
}
59
60
ccv_dense_matrix_t* ccv_dense_matrix_renew(ccv_dense_matrix_t* x, int rows, int cols, int types, int prefer_type, uint64_t sig)
61
388k
{
62
388k
  if (x != 0)
63
83.4k
  {
64
83.4k
    assert(x->rows == rows && x->cols == cols && (CCV_GET_DATA_TYPE(x->type) & types) && (CCV_GET_CHANNEL(x->type) == CCV_GET_CHANNEL(types)));
65
83.4k
    
prefer_type = 83.3k
CCV_GET_DATA_TYPE83.3k
(x->type) |
CCV_GET_CHANNEL83.3k
(x->type);
66
83.3k
  }
67
388k
  
if (388k
sig != 0388k
)
68
53
    sig = ccv_cache_generate_signature((const char*)&prefer_type, sizeof(int), sig, CCV_EOF_SIGN);
69
388k
  if (x == 0)
70
305k
  {
71
305k
    x = ccv_dense_matrix_new(rows, cols, prefer_type, 0, sig);
72
305k
  } else {
73
83.4k
    x->sig = sig;
74
83.4k
  }
75
388k
  return x;
76
388k
}
77
78
void ccv_make_matrix_mutable(ccv_matrix_t* mat)
79
0
{
80
0
  int type = *(int*)mat;
81
0
  if (type & CCV_MATRIX_DENSE)
82
0
  {
83
0
    ccv_dense_matrix_t* dmt = (ccv_dense_matrix_t*)mat;
84
0
    dmt->sig = 0;
85
0
    dmt->type &= ~CCV_REUSABLE;
86
0
  }
87
0
}
88
89
void ccv_make_matrix_immutable(ccv_matrix_t* mat)
90
90
{
91
90
  int type = *(int*)mat;
92
90
  if (type & CCV_MATRIX_DENSE)
93
90
  {
94
90
    ccv_dense_matrix_t* dmt = (ccv_dense_matrix_t*)mat;
95
90
    assert(dmt->sig == 0); // you cannot make matrix with derived signature immutable (it is immutable already)
96
90
    /* immutable matrix made this way is not reusable (collected), because its signature
97
90
     * only depends on the content, not the operation to generate it */
98
90
    dmt->type &= ~CCV_REUSABLE;
99
90
    dmt->sig = ccv_cache_generate_signature((char*)dmt->data.u8, dmt->rows * dmt->step, (uint64_t)dmt->type, CCV_EOF_SIGN);
100
90
  }
101
90
}
102
103
ccv_dense_matrix_t ccv_dense_matrix(int rows, int cols, int type, void* data, uint64_t sig)
104
3.24k
{
105
3.24k
  ccv_dense_matrix_t mat;
106
3.24k
  mat.reserved0 = 0;
107
3.24k
  mat.sig = sig;
108
3.24k
  mat.type = (CCV_GET_CHANNEL(type) | CCV_GET_DATA_TYPE(type) | CCV_MATRIX_DENSE | CCV_NO_DATA_ALLOC | CCV_UNMANAGED) & ~CCV_GARBAGE;
109
3.24k
  mat.rows = rows;
110
3.24k
  mat.cols = cols;
111
3.24k
  mat.step = CCV_GET_STEP(cols, type);
112
3.24k
  mat.refcount = 1;
113
3.24k
#if CCV_NNC_TENSOR_TFB
114
3.24k
  mat.reserved0 = 0;
115
3.24k
  mat.resides = CCV_TENSOR_CPU_MEMORY;
116
3.24k
  mat.format = CCV_TENSOR_FORMAT_NHWC;
117
3.24k
  mat.datatype = CCV_GET_DATA_TYPE(type);
118
3.24k
  mat.channels = CCV_GET_CHANNEL(type);
119
3.24k
  mat.reserved1 = 0;
120
3.24k
  mat.reserved2 = 0;
121
3.24k
#endif
122
3.24k
  mat.data.u8 = (unsigned char*)data;
123
3.24k
  return mat;
124
3.24k
}
125
126
ccv_sparse_matrix_t* ccv_sparse_matrix_new(int rows, int cols, int type, int major, uint64_t sig)
127
24.7k
{
128
24.7k
  ccv_sparse_matrix_t* mat;
129
24.7k
  mat = (ccv_sparse_matrix_t*)ccmalloc(sizeof(ccv_sparse_matrix_t));
130
24.7k
  mat->rows = rows;
131
24.7k
  mat->cols = cols;
132
24.7k
  mat->type = type | CCV_MATRIX_SPARSE | ((type & CCV_DENSE_VECTOR) ? 
CCV_DENSE_VECTOR2
:
CCV_SPARSE_VECTOR24.7k
);
133
24.7k
  mat->major = major;
134
24.7k
  mat->prime_index = 1; // See ccv_util.c to know why this is 1 and why size is 2.
135
24.7k
  mat->size = 2;
136
24.7k
  mat->rnum = 0;
137
24.7k
  mat->refcount = 1;
138
24.7k
  mat->index = (ccv_sparse_matrix_index_t*)cccalloc(sizeof(ccv_sparse_matrix_index_t), mat->size);
139
24.7k
  mat->vector = (ccv_sparse_matrix_vector_t*)ccmalloc(sizeof(ccv_sparse_matrix_vector_t) * mat->size);
140
24.7k
  return mat;
141
24.7k
}
142
143
void ccv_matrix_free_immediately(ccv_matrix_t* mat)
144
2.85M
{
145
2.85M
  int type = *(int*)mat;
146
2.85M
  assert(!(type & CCV_UNMANAGED));
147
2.85M
  if (type & CCV_MATRIX_DENSE)
148
2.85M
  {
149
2.85M
    ccv_dense_matrix_t* dmt = (ccv_dense_matrix_t*)mat;
150
2.85M
    dmt->refcount = 0;
151
2.85M
    ccfree(dmt);
152
2.85M
  } else 
if (0
type & CCV_MATRIX_SPARSE0
) {
153
0
    ccv_sparse_matrix_t* smt = (ccv_sparse_matrix_t*)mat;
154
0
    int i;
155
0
    for (i = 0; i < smt->size; i++)
156
0
      if (smt->index[i].ifbit)
157
0
        ccfree(smt->vector[i].data.u8);
158
0
    ccfree(smt->vector);
159
0
    ccfree(smt);
160
0
  } else if ((type & CCV_MATRIX_CSR) || (type & CCV_MATRIX_CSC)) {
161
0
    ccv_compressed_sparse_matrix_t* csm = (ccv_compressed_sparse_matrix_t*)mat;
162
0
    csm->refcount = 0;
163
0
    ccfree(csm);
164
0
  }
165
2.85M
}
166
167
void ccv_matrix_free(ccv_matrix_t* mat)
168
2.71M
{
169
2.71M
  int type = *(int*)mat;
170
2.71M
  assert(!(type & CCV_UNMANAGED));
171
2.71M
  if (type & CCV_MATRIX_DENSE)
172
2.69M
  {
173
2.69M
    ccv_dense_matrix_t* dmt = (ccv_dense_matrix_t*)mat;
174
2.69M
    dmt->refcount = 0;
175
2.69M
    if (!ccv_cache_opt || // e don't enable cache
176
2.69M
      
!(dmt->type & CCV_REUSABLE)2.40M
|| // or this is not a reusable piece
177
2.69M
      
dmt->sig == 02.40M
|| // or this doesn't have valid signature
178
2.69M
      
(dmt->type & CCV_NO_DATA_ALLOC)2.40M
) // or this matrix is allocated as header-only, therefore we cannot cache it
179
2.69M
      
ccfree288k
(dmt)288k
;
180
2.40M
    else {
181
2.40M
      assert(CCV_GET_DATA_TYPE(dmt->type) == CCV_8U ||
182
2.40M
           CCV_GET_DATA_TYPE(dmt->type) == CCV_32S ||
183
2.40M
           CCV_GET_DATA_TYPE(dmt->type) == CCV_32F ||
184
2.40M
           CCV_GET_DATA_TYPE(dmt->type) == CCV_64S ||
185
2.40M
           CCV_GET_DATA_TYPE(dmt->type) == CCV_64F);
186
2.40M
      
size_t size = 2.40M
ccv_compute_dense_matrix_size2.40M
(dmt->rows, dmt->cols, dmt->type);
187
2.40M
      ccv_cache_put(&ccv_cache, dmt->sig, dmt, size, 0 /* type 0 */);
188
2.40M
    }
189
2.69M
  } else 
if (24.7k
type & CCV_MATRIX_SPARSE24.7k
) {
190
24.7k
    ccv_sparse_matrix_t* smt = (ccv_sparse_matrix_t*)mat;
191
24.7k
    int i;
192
214k
    for (i = 0; i < smt->size; 
i++189k
)
193
189k
    {
194
189k
      if (smt->index[i].ifbit > 1)
195
189k
        
ccfree98.4k
(smt->vector[i].index)98.4k
; // It is a union of index / data, can just free them.
196
189k
    }
197
24.7k
    ccfree(smt->index);
198
24.7k
    ccfree(smt->vector);
199
24.7k
    ccfree(smt);
200
18.4E
  } else if ((type & CCV_MATRIX_CSR) || 
(type & CCV_MATRIX_CSC)0
) {
201
1
    ccv_compressed_sparse_matrix_t* csm = (ccv_compressed_sparse_matrix_t*)mat;
202
1
    csm->refcount = 0;
203
1
    ccfree(csm);
204
1
  }
205
2.71M
}
206
207
ccv_array_t* ccv_array_new(int rsize, int rnum, uint64_t sig)
208
2.06M
{
209
2.06M
  ccv_array_t* array;
210
2.06M
  if (ccv_cache_opt && 
sig != 01.55M
)
211
751k
  {
212
751k
    uint8_t type;
213
751k
    array = (ccv_array_t*)ccv_cache_out(&ccv_cache, sig, &type);
214
751k
    if (array)
215
720k
    {
216
720k
      assert(type == 1);
217
720k
      array->type |= CCV_GARBAGE;
218
720k
      array->refcount = 1;
219
720k
      return array;
220
1.34M
    }
221
751k
  }
222
1.34M
  array = (ccv_array_t*)ccmalloc(sizeof(ccv_array_t));
223
1.34M
  array->sig = sig;
224
1.34M
  array->type = CCV_REUSABLE & ~CCV_GARBAGE;
225
1.34M
  array->rnum = 0;
226
1.34M
  array->rsize = rsize;
227
1.34M
  array->size = ccv_max(rnum, 2 /* allocate memory for at least 2 items */);
228
1.34M
  array->data = ccmalloc((size_t)array->size * (size_t)rsize);
229
1.34M
  return array;
230
1.34M
}
231
232
void ccv_make_array_mutable(ccv_array_t* array)
233
0
{
234
0
  array->sig = 0;
235
0
  array->type &= ~CCV_REUSABLE;
236
0
}
237
238
void ccv_make_array_immutable(ccv_array_t* array)
239
0
{
240
0
  assert(array->sig == 0);
241
0
  array->type &= ~CCV_REUSABLE;
242
0
  /* TODO: trim the array */
243
0
  array->sig = ccv_cache_generate_signature(array->data, array->size * array->rsize, (uint64_t)array->rsize, CCV_EOF_SIGN);
244
0
}
245
246
void ccv_array_free_immediately(ccv_array_t* array)
247
831k
{
248
831k
  array->refcount = 0;
249
831k
  ccfree(array->data);
250
831k
  ccfree(array);
251
831k
}
252
253
void ccv_array_free(ccv_array_t* array)
254
1.31M
{
255
1.31M
  if (!ccv_cache_opt || 
!(array->type & CCV_REUSABLE)800k
||
array->sig == 0800k
)
256
510k
  {
257
510k
    array->refcount = 0;
258
510k
    ccfree(array->data);
259
510k
    ccfree(array);
260
800k
  } else {
261
800k
    size_t size = sizeof(ccv_array_t) + array->size * array->rsize;
262
800k
    ccv_cache_put(&ccv_cache, array->sig, array, size, 1 /* type 1 */);
263
800k
  }
264
1.31M
}
265
266
void ccv_drain_cache(void)
267
0
{
268
0
  if (ccv_cache.rnum > 0)
269
0
    ccv_cache_cleanup(&ccv_cache);
270
0
}
271
272
void ccv_disable_cache(void)
273
3
{
274
3
  ccv_cache_opt = 0;
275
3
  ccv_cache_close(&ccv_cache);
276
3
}
277
278
void ccv_enable_cache(size_t size)
279
3
{
280
3
  ccv_cache_opt = 1;
281
3
  ccv_cache_init(&ccv_cache, size, 2, ccv_matrix_free_immediately, ccv_array_free_immediately);
282
3
}
283
284
void ccv_enable_default_cache(void)
285
0
{
286
0
  ccv_enable_cache(CCV_DEFAULT_CACHE_SIZE);
287
0
}
288
289
static uint8_t key_siphash[16] = "libccvky4siphash";
290
291
uint64_t ccv_cache_generate_signature(const char* msg, int len, uint64_t sig_start, ...)
292
6.20M
{
293
6.20M
  uint64_t sig_out, sig_in[2]; // 1 is in, 0 is out
294
6.20M
  siphash((uint8_t*)&sig_out, (const uint8_t*)msg, len, key_siphash);
295
6.20M
  va_list arguments;
296
6.20M
  va_start(arguments, sig_start);
297
6.20M
  sig_in[0] = sig_out;
298
6.20M
  sig_in[1] = sig_start;
299
6.20M
  while (sig_in[1] != 0)
300
201
  {
301
201
    siphash((uint8_t*)&sig_out, (const uint8_t*)sig_in, sizeof(uint64_t) * 2, key_siphash);
302
201
    sig_in[0] = sig_out;
303
201
    sig_in[1] = va_arg(arguments, uint64_t);
304
201
  }
305
6.20M
  va_end(arguments);
306
6.20M
  return sig_out;
307
6.20M
}