/home/liu/actions-runner/_work/ccv/ccv/lib/ccv_memory.c
Line | Count | Source |
1 | | #include "ccv.h" |
2 | | #include "ccv_internal.h" |
3 | | #include "3rdparty/siphash/siphash24.h" |
4 | | |
5 | | static __thread ccv_cache_t ccv_cache; |
6 | | |
7 | | /** |
8 | | * For new typed cache object: |
9 | | * ccv_dense_matrix_t: type 0 |
10 | | * ccv_array_t: type 1 |
11 | | **/ |
12 | | |
13 | | /* option to enable/disable cache */ |
14 | | static __thread int ccv_cache_opt = 0; |
15 | | |
16 | | ccv_dense_matrix_t* ccv_dense_matrix_new(int rows, int cols, int type, void* data, uint64_t sig) |
17 | 5.14M | { |
18 | 5.14M | ccv_dense_matrix_t* mat; |
19 | 5.14M | if (ccv_cache_opt && sig != 04.65M && !data2.25M && !(type & CCV_NO_DATA_ALLOC)2.25M ) |
20 | 2.25M | { |
21 | 2.25M | uint8_t type; |
22 | 2.25M | mat = (ccv_dense_matrix_t*)ccv_cache_out(&ccv_cache, sig, &type); |
23 | 2.25M | if (mat) |
24 | 1.80M | { |
25 | 1.80M | assert(type == 0); |
26 | 1.80M | mat->type |= CCV_GARBAGE; // set the flag so the upper level function knows this is from recycle-bin |
27 | 1.80M | mat->refcount = 1; |
28 | 1.80M | return mat; |
29 | 1.80M | } |
30 | 2.25M | } |
31 | 3.34M | if (type & CCV_NO_DATA_ALLOC) |
32 | 1 | { |
33 | 1 | mat = (ccv_dense_matrix_t*)ccmalloc(sizeof(ccv_dense_matrix_t)); |
34 | 1 | mat->type = (CCV_GET_CHANNEL(type) | CCV_GET_DATA_TYPE(type) | CCV_MATRIX_DENSE | CCV_NO_DATA_ALLOC) & ~CCV_GARBAGE; |
35 | 1 | mat->data.u8 = data; |
36 | 3.34M | } else { |
37 | 3.34M | const size_t hdr_size = (sizeof(ccv_dense_matrix_t) + 63) & -64; |
38 | 3.34M | mat = (ccv_dense_matrix_t*)(data ? data0 : ccmalloc(ccv_compute_dense_matrix_size(rows, cols, type))); |
39 | 3.34M | mat->type = (CCV_GET_CHANNEL(type) | CCV_GET_DATA_TYPE(type) | CCV_MATRIX_DENSE) & ~CCV_GARBAGE; |
40 | 3.34M | mat->type |= data ? CCV_UNMANAGED0 : CCV_REUSABLE; // it still could be reusable because the signature could be derived one. |
41 | 3.34M | mat->data.u8 = (unsigned char*)mat + hdr_size; |
42 | 3.34M | } |
43 | 3.34M | mat->sig = sig; |
44 | 3.34M | #if CCV_NNC_TENSOR_TFB |
45 | 3.34M | mat->reserved0 = 0; |
46 | 3.34M | mat->reserved1 = 0; |
47 | 3.34M | mat->reserved2 = 0; |
48 | 3.34M | mat->resides = CCV_TENSOR_CPU_MEMORY; |
49 | 3.34M | mat->format = CCV_TENSOR_FORMAT_NHWC; |
50 | 3.34M | mat->datatype = CCV_GET_DATA_TYPE(type); |
51 | 3.34M | mat->channels = CCV_GET_CHANNEL(type); |
52 | 3.34M | mat->reserved4 = 0; |
53 | 3.34M | #endif |
54 | 3.34M | mat->rows = rows; |
55 | 3.34M | mat->cols = cols; |
56 | 3.34M | mat->step = CCV_GET_STEP(cols, type); |
57 | 3.34M | mat->refcount = 1; |
58 | 3.34M | return mat; |
59 | 5.14M | } |
60 | | |
61 | | ccv_dense_matrix_t* ccv_dense_matrix_renew(ccv_dense_matrix_t* x, int rows, int cols, int types, int prefer_type, uint64_t sig) |
62 | 391k | { |
63 | 391k | if (x != 0) |
64 | 83.7k | { |
65 | 83.7k | assert(x->rows == rows && x->cols == cols && (CCV_GET_DATA_TYPE(x->type) & types) && (CCV_GET_CHANNEL(x->type) == CCV_GET_CHANNEL(types))); |
66 | 83.7k | prefer_type = CCV_GET_DATA_TYPE(x->type) | CCV_GET_CHANNEL(x->type); |
67 | 83.7k | } |
68 | 391k | if (sig != 0) |
69 | 57 | sig = ccv_cache_generate_signature((const char*)&prefer_type, sizeof(int), sig, CCV_EOF_SIGN); |
70 | 391k | if (x == 0) |
71 | 307k | { |
72 | 307k | x = ccv_dense_matrix_new(rows, cols, prefer_type, 0, sig); |
73 | 307k | } else { |
74 | 83.7k | x->sig = sig; |
75 | 83.7k | } |
76 | 391k | return x; |
77 | 391k | } |
78 | | |
79 | | void ccv_make_matrix_mutable(ccv_matrix_t* mat) |
80 | 0 | { |
81 | 0 | int type = *(int*)mat; |
82 | 0 | if (type & CCV_MATRIX_DENSE) |
83 | 0 | { |
84 | 0 | ccv_dense_matrix_t* dmt = (ccv_dense_matrix_t*)mat; |
85 | 0 | dmt->sig = 0; |
86 | 0 | dmt->type &= ~CCV_REUSABLE; |
87 | 0 | } |
88 | 0 | } |
89 | | |
90 | | void ccv_make_matrix_immutable(ccv_matrix_t* mat) |
91 | 108 | { |
92 | 108 | int type = *(int*)mat; |
93 | 108 | if (type & CCV_MATRIX_DENSE) |
94 | 108 | { |
95 | 108 | ccv_dense_matrix_t* dmt = (ccv_dense_matrix_t*)mat; |
96 | 108 | assert(dmt->sig == 0); // you cannot make matrix with derived signature immutable (it is immutable already) |
97 | | /* immutable matrix made this way is not reusable (collected), because its signature |
98 | | * only depends on the content, not the operation to generate it */ |
99 | 108 | dmt->type &= ~CCV_REUSABLE; |
100 | 108 | dmt->sig = ccv_cache_generate_signature((char*)dmt->data.u8, dmt->rows * dmt->step, (uint64_t)dmt->type, CCV_EOF_SIGN); |
101 | 108 | } |
102 | 108 | } |
103 | | |
104 | | ccv_dense_matrix_t ccv_dense_matrix(int rows, int cols, int type, void* data, uint64_t sig) |
105 | 3.99k | { |
106 | 3.99k | ccv_dense_matrix_t mat; |
107 | 3.99k | mat.reserved0 = 0; |
108 | 3.99k | mat.sig = sig; |
109 | 3.99k | mat.type = (CCV_GET_CHANNEL(type) | CCV_GET_DATA_TYPE(type) | CCV_MATRIX_DENSE | CCV_NO_DATA_ALLOC | CCV_UNMANAGED) & ~CCV_GARBAGE; |
110 | 3.99k | mat.rows = rows; |
111 | 3.99k | mat.cols = cols; |
112 | 3.99k | mat.step = CCV_GET_STEP(cols, type); |
113 | 3.99k | mat.refcount = 1; |
114 | 3.99k | #if CCV_NNC_TENSOR_TFB |
115 | 3.99k | mat.reserved0 = 0; |
116 | 3.99k | mat.reserved1 = 0; |
117 | 3.99k | mat.reserved2 = 0; |
118 | 3.99k | mat.resides = CCV_TENSOR_CPU_MEMORY; |
119 | 3.99k | mat.format = CCV_TENSOR_FORMAT_NHWC; |
120 | 3.99k | mat.datatype = CCV_GET_DATA_TYPE(type); |
121 | 3.99k | mat.channels = CCV_GET_CHANNEL(type); |
122 | 3.99k | mat.reserved4 = 0; |
123 | 3.99k | #endif |
124 | 3.99k | mat.data.u8 = (unsigned char*)data; |
125 | 3.99k | return mat; |
126 | 3.99k | } |
127 | | |
128 | | ccv_sparse_matrix_t* ccv_sparse_matrix_new(int rows, int cols, int type, int major, uint64_t sig) |
129 | 25.3k | { |
130 | 25.3k | ccv_sparse_matrix_t* mat; |
131 | 25.3k | mat = (ccv_sparse_matrix_t*)ccmalloc(sizeof(ccv_sparse_matrix_t)); |
132 | 25.3k | mat->rows = rows; |
133 | 25.3k | mat->cols = cols; |
134 | 25.3k | mat->type = type | CCV_MATRIX_SPARSE | ((type & CCV_DENSE_VECTOR) ? CCV_DENSE_VECTOR2 : CCV_SPARSE_VECTOR25.3k ); |
135 | 25.3k | mat->major = major; |
136 | 25.3k | mat->prime_index = 1; // See ccv_util.c to know why this is 1 and why size is 2. |
137 | 25.3k | mat->size = 2; |
138 | 25.3k | mat->rnum = 0; |
139 | 25.3k | mat->refcount = 1; |
140 | 25.3k | mat->index = (ccv_sparse_matrix_index_t*)cccalloc(sizeof(ccv_sparse_matrix_index_t), mat->size); |
141 | 25.3k | mat->vector = (ccv_sparse_matrix_vector_t*)ccmalloc(sizeof(ccv_sparse_matrix_vector_t) * mat->size); |
142 | 25.3k | return mat; |
143 | 25.3k | } |
144 | | |
145 | | void ccv_matrix_free_immediately(ccv_matrix_t* mat) |
146 | 2.85M | { |
147 | 2.85M | int type = *(int*)mat; |
148 | 2.85M | assert(!(type & CCV_UNMANAGED)); |
149 | 2.85M | if (type & CCV_MATRIX_DENSE) |
150 | 2.85M | { |
151 | 2.85M | ccv_dense_matrix_t* dmt = (ccv_dense_matrix_t*)mat; |
152 | 2.85M | dmt->refcount = 0; |
153 | 2.85M | ccfree(dmt); |
154 | 2.85M | } else if (0 type & CCV_MATRIX_SPARSE0 ) { |
155 | 0 | ccv_sparse_matrix_t* smt = (ccv_sparse_matrix_t*)mat; |
156 | 0 | int i; |
157 | 0 | for (i = 0; i < smt->size; i++) |
158 | 0 | if (smt->index[i].ifbit) |
159 | 0 | ccfree(smt->vector[i].data.u8); |
160 | 0 | ccfree(smt->vector); |
161 | 0 | ccfree(smt); |
162 | 0 | } else if ((type & CCV_MATRIX_CSR) || (type & CCV_MATRIX_CSC)) { |
163 | 0 | ccv_compressed_sparse_matrix_t* csm = (ccv_compressed_sparse_matrix_t*)mat; |
164 | 0 | csm->refcount = 0; |
165 | 0 | ccfree(csm); |
166 | 0 | } |
167 | 2.85M | } |
168 | | |
169 | | void ccv_matrix_free(ccv_matrix_t* mat) |
170 | 2.73M | { |
171 | 2.73M | int type = *(int*)mat; |
172 | 2.73M | assert(!(type & CCV_UNMANAGED)); |
173 | 2.73M | if (type & CCV_MATRIX_DENSE) |
174 | 2.70M | { |
175 | 2.70M | ccv_dense_matrix_t* dmt = (ccv_dense_matrix_t*)mat; |
176 | 2.70M | dmt->refcount = 0; |
177 | 2.70M | if (!ccv_cache_opt || // e don't enable cache |
178 | 2.70M | !(dmt->type & CCV_REUSABLE)2.40M || // or this is not a reusable piece |
179 | 2.70M | dmt->sig == 02.40M || // or this doesn't have valid signature |
180 | 2.70M | (dmt->type & CCV_NO_DATA_ALLOC)2.40M ) // or this matrix is allocated as header-only, therefore we cannot cache it |
181 | 307k | ccfree(dmt); |
182 | 2.40M | else { |
183 | 2.40M | assert(CCV_GET_DATA_TYPE(dmt->type) == CCV_8U || |
184 | 2.40M | CCV_GET_DATA_TYPE(dmt->type) == CCV_32S || |
185 | 2.40M | CCV_GET_DATA_TYPE(dmt->type) == CCV_32F || |
186 | 2.40M | CCV_GET_DATA_TYPE(dmt->type) == CCV_64S || |
187 | 2.40M | CCV_GET_DATA_TYPE(dmt->type) == CCV_64F); |
188 | 2.40M | size_t size = ccv_compute_dense_matrix_size(dmt->rows, dmt->cols, dmt->type); |
189 | 2.40M | ccv_cache_put(&ccv_cache, dmt->sig, dmt, size, 0 /* type 0 */); |
190 | 2.40M | } |
191 | 2.70M | } else if (25.3k type & CCV_MATRIX_SPARSE25.3k ) { |
192 | 25.3k | ccv_sparse_matrix_t* smt = (ccv_sparse_matrix_t*)mat; |
193 | 25.3k | int i; |
194 | 206k | for (i = 0; i < smt->size; i++181k ) |
195 | 181k | { |
196 | 181k | if (smt->index[i].ifbit > 1) |
197 | 93.4k | ccfree(smt->vector[i].index); // It is a union of index / data, can just free them. |
198 | 181k | } |
199 | 25.3k | ccfree(smt->index); |
200 | 25.3k | ccfree(smt->vector); |
201 | 25.3k | ccfree(smt); |
202 | 25.3k | } else if (1 (type & CCV_MATRIX_CSR)1 || (type & CCV_MATRIX_CSC)0 ) { |
203 | 1 | ccv_compressed_sparse_matrix_t* csm = (ccv_compressed_sparse_matrix_t*)mat; |
204 | 1 | csm->refcount = 0; |
205 | 1 | ccfree(csm); |
206 | 1 | } |
207 | 2.73M | } |
208 | | |
209 | | ccv_array_t* ccv_array_new(int rsize, int rnum, uint64_t sig) |
210 | 2.06M | { |
211 | 2.06M | ccv_array_t* array; |
212 | 2.06M | if (ccv_cache_opt && sig != 01.55M ) |
213 | 751k | { |
214 | 751k | uint8_t type; |
215 | 751k | array = (ccv_array_t*)ccv_cache_out(&ccv_cache, sig, &type); |
216 | 751k | if (array) |
217 | 720k | { |
218 | 720k | assert(type == 1); |
219 | 720k | array->type |= CCV_GARBAGE; |
220 | 720k | array->refcount = 1; |
221 | 720k | return array; |
222 | 720k | } |
223 | 751k | } |
224 | 1.34M | array = (ccv_array_t*)ccmalloc(sizeof(ccv_array_t)); |
225 | 1.34M | array->sig = sig; |
226 | 1.34M | array->type = CCV_REUSABLE & ~CCV_GARBAGE; |
227 | 1.34M | array->rnum = 0; |
228 | 1.34M | array->rsize = rsize; |
229 | 1.34M | array->size = ccv_max(rnum, 2 /* allocate memory for at least 2 items */); |
230 | 1.34M | array->data = ccmalloc((size_t)array->size * (size_t)rsize); |
231 | 1.34M | return array; |
232 | 2.06M | } |
233 | | |
234 | | void ccv_make_array_mutable(ccv_array_t* array) |
235 | 0 | { |
236 | 0 | array->sig = 0; |
237 | 0 | array->type &= ~CCV_REUSABLE; |
238 | 0 | } |
239 | | |
240 | | void ccv_make_array_immutable(ccv_array_t* array) |
241 | 0 | { |
242 | 0 | assert(array->sig == 0); |
243 | 0 | array->type &= ~CCV_REUSABLE; |
244 | | /* TODO: trim the array */ |
245 | 0 | array->sig = ccv_cache_generate_signature(array->data, array->size * array->rsize, (uint64_t)array->rsize, CCV_EOF_SIGN); |
246 | 0 | } |
247 | | |
248 | | void ccv_array_free_immediately(ccv_array_t* array) |
249 | 831k | { |
250 | 831k | array->refcount = 0; |
251 | 831k | ccfree(array->data); |
252 | 831k | ccfree(array); |
253 | 831k | } |
254 | | |
255 | | void ccv_array_free(ccv_array_t* array) |
256 | 1.31M | { |
257 | 1.31M | if (!ccv_cache_opt || !(array->type & CCV_REUSABLE)800k || array->sig == 0800k ) |
258 | 512k | { |
259 | 512k | array->refcount = 0; |
260 | 512k | ccfree(array->data); |
261 | 512k | ccfree(array); |
262 | 800k | } else { |
263 | 800k | size_t size = sizeof(ccv_array_t) + array->size * array->rsize; |
264 | 800k | ccv_cache_put(&ccv_cache, array->sig, array, size, 1 /* type 1 */); |
265 | 800k | } |
266 | 1.31M | } |
267 | | |
268 | | void ccv_drain_cache(void) |
269 | 0 | { |
270 | 0 | if (ccv_cache.rnum > 0) |
271 | 0 | ccv_cache_cleanup(&ccv_cache); |
272 | 0 | } |
273 | | |
274 | | void ccv_disable_cache(void) |
275 | 3 | { |
276 | 3 | ccv_cache_opt = 0; |
277 | 3 | ccv_cache_close(&ccv_cache); |
278 | 3 | } |
279 | | |
280 | | void ccv_enable_cache(size_t size) |
281 | 3 | { |
282 | 3 | ccv_cache_opt = 1; |
283 | 3 | ccv_cache_init(&ccv_cache, size, 2, ccv_matrix_free_immediately, ccv_array_free_immediately); |
284 | 3 | } |
285 | | |
286 | | void ccv_enable_default_cache(void) |
287 | 0 | { |
288 | 0 | ccv_enable_cache(CCV_DEFAULT_CACHE_SIZE); |
289 | 0 | } |
290 | | |
291 | | static uint8_t key_siphash[16] = "libccvky4siphash"; |
292 | | |
293 | | uint64_t ccv_cache_generate_signature(const char* msg, int len, uint64_t sig_start, ...) |
294 | 6.20M | { |
295 | 6.20M | uint64_t sig_out, sig_in[2]; // 1 is in, 0 is out |
296 | 6.20M | siphash((uint8_t*)&sig_out, (const uint8_t*)msg, len, key_siphash); |
297 | 6.20M | va_list arguments; |
298 | 6.20M | va_start(arguments, sig_start); |
299 | 6.20M | sig_in[0] = sig_out; |
300 | 6.20M | sig_in[1] = sig_start; |
301 | 6.20M | while (sig_in[1] != 0) |
302 | 227 | { |
303 | 227 | siphash((uint8_t*)&sig_out, (const uint8_t*)sig_in, sizeof(uint64_t) * 2, key_siphash); |
304 | 227 | sig_in[0] = sig_out; |
305 | 227 | sig_in[1] = va_arg(arguments, uint64_t); |
306 | 227 | } |
307 | | va_end(arguments); |
308 | 6.20M | return sig_out; |
309 | 6.20M | } |