Bug Summary

File:nnc/ccv_cnnp_model.c
Warning:line 2877, column 25
Array access (via field 'vals') results in a null pointer dereference

Annotated Source Code

Press '?' to see keyboard shortcuts

clang -cc1 -cc1 -triple x86_64-unknown-linux-gnu -analyze -disable-free -clear-ast-before-backend -disable-llvm-verifier -discard-value-names -main-file-name ccv_cnnp_model.c -analyzer-checker=core -analyzer-checker=apiModeling -analyzer-checker=unix -analyzer-checker=deadcode -analyzer-checker=security.insecureAPI.UncheckedReturn -analyzer-checker=security.insecureAPI.getpw -analyzer-checker=security.insecureAPI.gets -analyzer-checker=security.insecureAPI.mktemp -analyzer-checker=security.insecureAPI.mkstemp -analyzer-checker=security.insecureAPI.vfork -analyzer-checker=nullability.NullPassedToNonnull -analyzer-checker=nullability.NullReturnedFromNonnull -analyzer-output plist -w -setup-static-analyzer -mrelocation-model pic -pic-level 2 -pic-is-pie -mframe-pointer=none -fmath-errno -ffp-contract=on -fno-rounding-math -mconstructor-aliases -funwind-tables=2 -target-cpu x86-64 -target-feature +sse2 -tune-cpu generic -debugger-tuning=gdb -fdebug-compilation-dir=/home/liu/actions-runner/_work/ccv/ccv/lib/nnc -fcoverage-compilation-dir=/home/liu/actions-runner/_work/ccv/ccv/lib/nnc -resource-dir /usr/local/lib/clang/19 -I ../ -I /usr/local/cuda/include -D HAVE_CBLAS -D HAVE_LIBPNG -D HAVE_LIBJPEG -D HAVE_FFTW3 -D HAVE_PTHREAD -D HAVE_LIBLINEAR -D HAVE_TESSERACT -D HAVE_AVCODEC -D HAVE_AVFORMAT -D HAVE_AVUTIL -D HAVE_SWSCALE -D HAVE_SSE2 -D HAVE_GSL -D HAVE_CUDA -D HAVE_CUDNN -D HAVE_NCCL -D USE_SYSTEM_CUB -I /usr/local/include -internal-isystem /usr/local/lib/clang/19/include -internal-isystem /usr/local/include -internal-isystem /usr/lib/gcc/x86_64-linux-gnu/12/../../../../x86_64-linux-gnu/include -internal-externc-isystem /usr/include/x86_64-linux-gnu -internal-externc-isystem /include -internal-externc-isystem /usr/include -O3 -ferror-limit 19 -fgnuc-version=4.2.1 -fskip-odr-check-in-gmf -vectorize-loops -vectorize-slp -analyzer-output=html -faddrsig -D__GCC_HAVE_DWARF2_CFI_ASM=1 -o /home/liu/actions-runner/_work/ccv/ccv/_analyze/2026-09-09-174646-2504664-1 -x c ccv_cnnp_model.c
1#include "ccv_nnc.h"
2#include "ccv_nnc_easy.h"
3#include "ccv_nnc_internal.h"
4#include "ccv_internal.h"
5#include "_ccv_cnnp_model.h"
6#include "_ccv_nnc_graph.h"
7#include "_ccv_nnc_symbolic_graph.h"
8#ifdef HAVE_CUDA1
9#include "gpu/ccv_nnc_compat.h"
10#elif defined(HAVE_MPS)
11#include "mps/ccv_nnc_mps.h"
12#endif
13
14// MARK - Level-5 API
15
16ccv_cnnp_model_io_t ccv_cnnp_model_apply(ccv_cnnp_model_t* const model, const ccv_cnnp_model_io_t* const inputs, const int input_size)
17{
18 if (!model->io)
19 model->io = ccv_array_new(sizeof(ccv_cnnp_model_io_t), 1, 0);
20 ccv_cnnp_model_io_t model_io = ccmallocmalloc(sizeof(struct ccv_cnnp_model_io_s) + sizeof(ccv_nnc_tensor_symbol_t) * model->output_size);
21 model_io->param_ref = 0;
22 model_io->param_sel = 0;
23 model_io->visit = 0;
24 model_io->model = model;
25 model_io->dependencies = 0;
26 model_io->dependents = 0;
27 model_io->outgoings = 0;
28 model_io->outputs = (ccv_nnc_tensor_symbol_t*)(model_io + 1);
29 ccv_array_push(model->io, &model_io);
30 if (input_size > 0)
31 {
32 model_io->incomings = ccv_array_new(sizeof(ccv_cnnp_model_io_t), input_size, 0);
33 ccv_array_resize(model_io->incomings, input_size);
34 int i;
35 memcpy(ccv_array_get(model_io->incomings, 0)((void*)(((char*)((model_io->incomings)->data)) + (size_t
)(model_io->incomings)->rsize * (size_t)(0))), inputs, sizeof(ccv_cnnp_model_io_t) * input_size);
36 for (i = 0; i < input_size; i++)
37 {
38 if (!inputs[i]->outgoings)
39 inputs[i]->outgoings = ccv_array_new(sizeof(ccv_cnnp_model_io_t), 1, 0);
40 ccv_array_push(inputs[i]->outgoings, &model_io);
41 }
42 } else {
43 model_io->incomings = 0;
44 }
45 return model_io;
46}
47
48void ccv_cnnp_model_add_dependencies(ccv_cnnp_model_io_t model_io, const ccv_cnnp_model_io_t* const dependencies, const int dependency_size)
49{
50 assert(dependency_size > 0)((void) sizeof ((dependency_size > 0) ? 1 : 0), __extension__
({ if (dependency_size > 0) ; else __assert_fail ("dependency_size > 0"
, "ccv_cnnp_model.c", 50, __extension__ __PRETTY_FUNCTION__);
}));
51 if (!model_io->dependencies)
52 model_io->dependencies = ccv_array_new(sizeof(ccv_cnnp_model_io_t), dependency_size, 0);
53 int i, j;
54 for (i = 0; i < dependency_size; i++)
55 {
56 int flag = 0;
57 // Check if it is already exist or not.
58 for (j = 0; !flag && j < model_io->dependencies->rnum; j++)
59 if (*(ccv_cnnp_model_io_t*)ccv_array_get(model_io->dependencies, j)((void*)(((char*)((model_io->dependencies)->data)) + (size_t
)(model_io->dependencies)->rsize * (size_t)(j))) == dependencies[i])
60 flag = 1;
61 if (flag)
62 continue;
63 ccv_array_push(model_io->dependencies, dependencies + i);
64 ++dependencies[i]->dependents;
65 }
66}
67
68int ccv_cnnp_model_output_size(const ccv_cnnp_model_t* const model)
69{
70 return model->output_size;
71}
72
73int ccv_cnnp_model_is_trainable(const ccv_cnnp_model_t* const model)
74{
75 // If the model is compiled, it is default to 1 unless it is not.
76 if (model->compiled_data)
77 return model->is_trainable >= 0 ? model->is_trainable : 1;
78 return model->is_trainable;
79}
80
81ccv_cnnp_model_io_t ccv_cnnp_model_parameters(ccv_cnnp_model_t* const model, const int selector, const int index)
82{
83 if (!model->io)
84 model->io = ccv_array_new(sizeof(ccv_cnnp_model_io_t), 1, 0);
85 ccv_cnnp_model_io_t model_io = ccmallocmalloc(sizeof(struct ccv_cnnp_model_io_s));
86 model_io->param_ref = index >= 0 ? index + 1 : ALL_PARAMETERS-1;
87 model_io->param_sel = selector >= 0 ? selector + 1 : ALL_PARAMETERS-1;
88 model_io->visit = 0;
89 model_io->model = model;
90 model_io->outputs = 0;
91 model_io->dependencies = 0;
92 model_io->dependents = 0;
93 model_io->incomings = 0;
94 model_io->outgoings = 0;
95 ccv_array_push(model->io, &model_io);
96 return model_io;
97}
98
99void ccv_cnnp_model_notify_hook(ccv_cnnp_model_t* const model, ccv_cnnp_model_notify_f func, void* const context)
100{
101 model->notify_hook.func = func;
102 model->notify_hook.context = context;
103}
104
105void ccv_cnnp_model_notify(const ccv_cnnp_model_t* const model, const int tag, void* const payload)
106{
107 if (model->notify_hook.func)
108 model->notify_hook.func(model, tag, payload, model->notify_hook.context);
109 if (model->isa->notify)
110 model->isa->notify(model, tag, payload);
111}
112
113static int _ccv_nnc_array_dedup_graph_exec_symbols(ccv_nnc_graph_exec_symbol_t* const graph_exec_symbols, int graph_exec_symbol_size)
114{
115 int i, j;
116 for (i = 0; i < graph_exec_symbol_size; i++)
117 {
118 ccv_nnc_graph_exec_symbol_t* const graph_exec_symbol = graph_exec_symbols + i;
119 // Check whether this tensor symbol has any duplicate.
120 for (j = i + 1; j < graph_exec_symbol_size;)
121 {
122 ccv_nnc_graph_exec_symbol_t* const other_symbol = graph_exec_symbols + j;
123 // If there is a same tensor symbol, remove it.
124 if (other_symbol->d == graph_exec_symbol->d && other_symbol->graph == graph_exec_symbol->graph)
125 {
126 if (j + 1 < graph_exec_symbol_size)
127 *other_symbol = graph_exec_symbols[graph_exec_symbol_size - 1];
128 --graph_exec_symbol_size;
129 continue;
130 }
131 ++j;
132 }
133 }
134 return graph_exec_symbol_size;
135}
136
137void ccv_cnnp_model_add_to_array(void* const context, const ccv_nnc_tensor_symbol_t symbol, const int is_trainable)
138{
139 ccv_cnnp_model_add_to_array_context_t* const add_to_array_context = (ccv_cnnp_model_add_to_array_context_t*)context;
140 ccv_cnnp_model_t* const model = add_to_array_context->sequence->model;
141 int i;
142 if (add_to_array_context->add_parameter_indices && !model->parameter_indices)
143 model->parameter_indices = ccv_array_new(sizeof(int), 0, 0);
144 for (i = 0; i < add_to_array_context->symbols->rnum; i++)
145 {
146 const ccv_nnc_tensor_symbol_t other_symbol = *(ccv_nnc_tensor_symbol_t*)ccv_array_get(add_to_array_context->symbols, i)((void*)(((char*)((add_to_array_context->symbols)->data
)) + (size_t)(add_to_array_context->symbols)->rsize * (
size_t)(i)));
147 if (other_symbol.d == symbol.d && other_symbol.graph == symbol.graph)
148 {
149 // Only add to parameter_indices if it is trainable.
150 if (add_to_array_context->add_parameter_indices)
151 ccv_array_add_unique_int(model->parameter_indices, i);
152 // Found it, return, don't add it.
153 return;
154 }
155 }
156 // Only add to parameter_indices if it is trainable.
157 if (add_to_array_context->add_parameter_indices)
158 ccv_array_push(model->parameter_indices, &add_to_array_context->symbols->rnum);
159 // This is a new one, no need to add_unique_int, it is unique.
160 ccv_array_push(add_to_array_context->symbols, &symbol);
161 if (add_to_array_context->trainables)
162 ccv_array_push(add_to_array_context->trainables, &is_trainable);
163 char id[2048];
164 id[0] = add_to_array_context->prefix;
165 id[1] = '-';
166 int total_len = 2;
167 for (i = 0; i < add_to_array_context->sequence->sequences->rnum; i++)
168 {
169 const ccv_cnnp_model_name_t* const name = (ccv_cnnp_model_name_t*)ccv_array_get(add_to_array_context->sequence->sequences, i)((void*)(((char*)((add_to_array_context->sequence->sequences
)->data)) + (size_t)(add_to_array_context->sequence->
sequences)->rsize * (size_t)(i)));
170 int len;
171 if (name->name && name->name[0] != '\0')
172 len = snprintf(id + total_len, 2048 - total_len, "%s-%d-", name->name, name->sequence);
173 else
174 len = snprintf(id + total_len, 2048 - total_len, "%d-", name->sequence);
175 total_len += len;
176 if (total_len >= 2047)
177 break;
178 }
179 if (total_len < 2047)
180 total_len += snprintf(id + total_len, 2048 - total_len, "%d", add_to_array_context->sequence->it);
181 assert(total_len < 2048)((void) sizeof ((total_len < 2048) ? 1 : 0), __extension__
({ if (total_len < 2048) ; else __assert_fail ("total_len < 2048"
, "ccv_cnnp_model.c", 181, __extension__ __PRETTY_FUNCTION__)
; }));
182 char *heap_id = (char*)ccmallocmalloc(total_len + 1);
183 memcpy(heap_id, id, total_len + 1);
184 ccv_array_push(add_to_array_context->ids, &heap_id);
185 ++add_to_array_context->sequence->it;
186}
187
188static void _ccv_cnnp_compiled_data_init(ccv_cnnp_compiled_data_t* const compiled_data, const int output_size, ccv_array_t* const gradient_checkpoints)
189{
190 compiled_data->f = compiled_data->fits + output_size;
191 compiled_data->xpu_alloc.mp_hdr = -1;
192 compiled_data->xpu_alloc.freed = kh_init(dy_str)kh_init_dy_str();
193 compiled_data->xpu_alloc.allocd = kh_init(dy_alloc)kh_init_dy_alloc();
194 compiled_data->gradient_checkpoints = gradient_checkpoints;
195}
196
197static int _ccv_cnnp_model_root_parallel_count(const ccv_cnnp_model_t* const model)
198{
199 return ccv_max(model->parallel_count, 1)({ typeof (model->parallel_count) _a = (model->parallel_count
); typeof (1) _b = (1); (_a > _b) ? _a : _b; });
200}
201
202static int _ccv_cnnp_model_effective_parallel_count(const ccv_cnnp_model_t* const model)
203{
204 int parallel_count = _ccv_cnnp_model_root_parallel_count(model);
205 if (model->graph && model->graph->data_parallel.count > parallel_count)
206 parallel_count = model->graph->data_parallel.count;
207 return parallel_count;
208}
209
210static int _ccv_cnnp_compiled_data_parallel_count(const ccv_cnnp_model_t* const model, const ccv_cnnp_compiled_data_t* const compiled_data)
211{
212 return compiled_data->parallel_count > 0 ? compiled_data->parallel_count : _ccv_cnnp_model_effective_parallel_count(model);
213}
214
215ccv_nnc_tensor_symbol_t ccv_cnnp_model_get_symbol(ccv_cnnp_model_t* const self, const ccv_nnc_tensor_symbol_t symbol)
216{
217 assert(self->data)((void) sizeof ((self->data) ? 1 : 0), __extension__ ({ if
(self->data) ; else __assert_fail ("self->data", "ccv_cnnp_model.c"
, 217, __extension__ __PRETTY_FUNCTION__); }));
218 ccv_cnnp_model_build_data_t* const build_data = (ccv_cnnp_model_build_data_t*)self->data;
219 if (build_data->parallel_count <= 1 || build_data->parallel_rank == 0)
220 return symbol;
221 const int rank = build_data->parallel_rank;
222 assert(rank > 0)((void) sizeof ((rank > 0) ? 1 : 0), __extension__ ({ if (
rank > 0) ; else __assert_fail ("rank > 0", "ccv_cnnp_model.c"
, 222, __extension__ __PRETTY_FUNCTION__); }));
223 assert(rank < build_data->parallel_count)((void) sizeof ((rank < build_data->parallel_count) ? 1
: 0), __extension__ ({ if (rank < build_data->parallel_count
) ; else __assert_fail ("rank < build_data->parallel_count"
, "ccv_cnnp_model.c", 223, __extension__ __PRETTY_FUNCTION__)
; }));
224 ccv_nnc_symbolic_graph_t* const graph = (ccv_nnc_symbolic_graph_t*)symbol.graph;
225 ccv_nnc_tensor_symbol_t copy = ccv_nnc_tensor_symbol_copy(graph, symbol, rank);
226 if (copy.d != CCV_NNC_NO_TENSOR_SYMBOL)
227 return copy;
228 ccv_nnc_tensor_param_t params = ccv_nnc_tensor_symbol_params(graph, symbol);
229 if (CCV_TENSOR_GET_MEMORY(params.type)((params.type) & 0x3) == CCV_TENSOR_GPU_MEMORY)
230 CCV_TENSOR_SET_DEVICE_ID(params.type, rank)(params.type) = (((params.type) & ~0xfff00) | (((rank) &
0xfff) << 8));
231 copy = ccv_nnc_tensor_symbol_new(graph, params, 0);
232 ccv_nnc_tensor_symbol_set_copy(graph, symbol, rank, copy);
233 return copy;
234}
235
236typedef struct {
237 void* old_graph_exec_symbol_new_hook_context;
238 ccv_nnc_graph_exec_symbol_new_hook_f old_graph_exec_symbol_new_hook;
239 ccv_nnc_symbolic_graph_t* graph;
240 ccv_cnnp_model_build_data_t* build_data;
241} ccv_cnnp_model_set_exec_flags_context_t;
242
243static void _ccv_cnnp_model_set_exec_flags(void* context, const ccv_nnc_graph_exec_symbol_t symbol, const ccv_nnc_cmd_t cmd, const ccv_nnc_tensor_symbol_t* const inputs, const int input_size, const ccv_nnc_tensor_symbol_t* const outputs, const int output_size, const char* const name)
244{
245 ccv_cnnp_model_set_exec_flags_context_t* flags_context = (ccv_cnnp_model_set_exec_flags_context_t*)context;
246 if (flags_context->build_data->exec_flags)
247 ccv_nnc_graph_exec_symbol_set_flags(flags_context->graph, symbol, flags_context->build_data->exec_flags);
248 if (flags_context->old_graph_exec_symbol_new_hook)
249 flags_context->old_graph_exec_symbol_new_hook(flags_context->old_graph_exec_symbol_new_hook_context, symbol, cmd, inputs, input_size, outputs, output_size, name);
250}
251
252static void _ccv_cnnp_model_compile(ccv_cnnp_model_t* const model, const ccv_nnc_tensor_param_t* const inputs, const int input_size, const ccv_nnc_cmd_t loss)
253{
254 assert(model->graph)((void) sizeof ((model->graph) ? 1 : 0), __extension__ ({ if
(model->graph) ; else __assert_fail ("model->graph", "ccv_cnnp_model.c"
, 254, __extension__ __PRETTY_FUNCTION__); }));
255 model->inputs = ccmallocmalloc(sizeof(ccv_nnc_tensor_symbol_t) * input_size);
256 int i;
257 for (i = 0; i < input_size; i++)
258 model->inputs[i] = ccv_nnc_tensor_symbol_new(model->graph, inputs[i], 0);
259 ccv_array_t* const parameters = ccv_array_new(sizeof(ccv_nnc_tensor_symbol_t), 0, 0);
260 ccv_array_t* const parameter_ids = ccv_array_new(sizeof(char*), 0, 0);
261 ccv_array_t* const parameter_trainables = ccv_array_new(sizeof(int), 0, 0);
262 ccv_cnnp_model_sequence_t model_sequence = {
263 .bank = kh_init(ccv_cnnp_model_name_bank)kh_init_ccv_cnnp_model_name_bank()
264 };
265 ccv_cnnp_model_add_to_array_context_t add_to_parameter_context = {
266 .add_parameter_indices = 1,
267 .prefix = 't',
268 .sequence = &model_sequence,
269 .symbols = parameters,
270 .ids = parameter_ids,
271 .trainables = parameter_trainables,
272 };
273 ccv_array_t* const internals = ccv_array_new(sizeof(ccv_nnc_tensor_symbol_t), 0, 0);
274 ccv_array_t* const internal_ids = ccv_array_new(sizeof(char*), 0, 0);
275 ccv_cnnp_model_add_to_array_context_t add_to_output_context = {
276 .add_parameter_indices = 0,
277 .prefix = 'r',
278 .sequence = &model_sequence,
279 .symbols = internals,
280 .ids = internal_ids,
281 .trainables = 0,
282 };
283 ccv_cnnp_model_build_data_t build_data = {
284 .exec_flags = 0,
285 .is_trainable = model->is_trainable >= 0 ? model->is_trainable : 1,
286 .parallel_count = 1,
287 .parallel_rank = 0,
288 .model_sequence = &model_sequence,
289 .add_to_array = ccv_cnnp_model_add_to_array,
290 .parameters = parameters,
291 .context = {
292 .add_to_parameter = &add_to_parameter_context,
293 .add_to_output = &add_to_output_context,
294 },
295 .gradient_checkpoints = 0,
296 };
297 model->data = &build_data;
298 ccv_cnnp_model_set_exec_flags_context_t flags_context = {
299 .graph = model->graph,
300 .build_data = &build_data,
301 .old_graph_exec_symbol_new_hook = 0,
302 .old_graph_exec_symbol_new_hook_context = 0
303 };
304 flags_context.old_graph_exec_symbol_new_hook_context = ccv_nnc_graph_exec_symbol_new_hook(model->graph, _ccv_cnnp_model_set_exec_flags, &flags_context, &flags_context.old_graph_exec_symbol_new_hook);
305 ccv_cnnp_model_build(model, model->graph, model->inputs, input_size, 0, 0);
306 // Reset back to previous hook.
307 ccv_nnc_graph_exec_symbol_new_hook(model->graph, flags_context.old_graph_exec_symbol_new_hook, flags_context.old_graph_exec_symbol_new_hook_context, 0);
308 for (i = 0; i < model->output_size; i++)
309 {
310 const ccv_nnc_tensor_symbol_t output = model->outputs[i];
311 const ccv_nnc_tensor_symbol_t alias_to = ccv_nnc_tensor_symbol_alias_to(model->graph, output);
312 if (alias_to.d == CCV_NNC_NO_TENSOR_SYMBOL)
313 continue;
314 // If output is an alias, insert data transform regardless for result correctness (we cannot bind an alias). You can check ccv_nnc_tensor_bind_symbol method
315 // to see that we can correctly bind a tensor which from it, has aliases, but we cannot bind an alias tensor correctly (this is expected, sort of, to be
316 // honest, because we cannot handle cases of alias is part of the original tensor but bind differently).
317 const ccv_nnc_tensor_param_t output_params = ccv_nnc_tensor_symbol_params(model->graph, output);
318 model->outputs[i] = ccv_nnc_tensor_symbol_new(model->graph, output_params, 0);
319 ccv_nnc_graph_exec_symbol_t make_contiguous = ccv_nnc_graph_exec_symbol_new(model->graph, CMD_FORMAT_TRANSFORM_FORWARD()ccv_nnc_cmd(CCV_NNC_FORMAT_TRANSFORM_FORWARD, 0, ccv_nnc_cmd_auto
, 0), &output, 1, model->outputs + i, 1, "contiguous");
320 ccv_nnc_graph_exec_symbol_set_flags(model->graph, make_contiguous, CCV_NNC_GRAPH_EXEC_DISABLE_OPT);
321 }
322 model->data = 0;
323 kh_destroy(ccv_cnnp_model_name_bank, model_sequence.bank)kh_destroy_ccv_cnnp_model_name_bank(model_sequence.bank);
324 if (model_sequence.sequences)
325 ccv_array_free(model_sequence.sequences);
326 // Check if there are parameters that are not trainables. If there are, we will allocate uint64 bitmap to record that.
327 int not_trainables = 0;
328 // Assert no parameter is alias.
329 for (i = 0; i < parameters->rnum; i++)
330 {
331 const ccv_nnc_tensor_symbol_t parameter = *(ccv_nnc_tensor_symbol_t*)ccv_array_get(parameters, i)((void*)(((char*)((parameters)->data)) + (size_t)(parameters
)->rsize * (size_t)(i)));
332 const ccv_nnc_tensor_symbol_t alias_to = ccv_nnc_tensor_symbol_alias_to(parameter.graph, parameter);
333 assert(alias_to.graph == 0)((void) sizeof ((alias_to.graph == 0) ? 1 : 0), __extension__
({ if (alias_to.graph == 0) ; else __assert_fail ("alias_to.graph == 0"
, "ccv_cnnp_model.c", 333, __extension__ __PRETTY_FUNCTION__)
; })); // Cannot find the one alias to.
334 if (*(int*)ccv_array_get(parameter_trainables, i)((void*)(((char*)((parameter_trainables)->data)) + (size_t
)(parameter_trainables)->rsize * (size_t)(i))) == 0)
335 not_trainables = 1;
336 }
337 assert(parameters->rnum == parameter_trainables->rnum)((void) sizeof ((parameters->rnum == parameter_trainables->
rnum) ? 1 : 0), __extension__ ({ if (parameters->rnum == parameter_trainables
->rnum) ; else __assert_fail ("parameters->rnum == parameter_trainables->rnum"
, "ccv_cnnp_model.c", 337, __extension__ __PRETTY_FUNCTION__)
; }));
338 uint64_t* parameter_flags = 0;
339 if (not_trainables)
340 {
341 parameter_flags = (uint64_t*)cccalloccalloc(((parameters->rnum + 63) >> 6), sizeof(uint64_t));
342 for (i = 0; i < parameter_trainables->rnum; i++)
343 if (*(int*)ccv_array_get(parameter_trainables, i)((void*)(((char*)((parameter_trainables)->data)) + (size_t
)(parameter_trainables)->rsize * (size_t)(i))))
344 parameter_flags[i >> 6] |= ((uint64_t)1 << (i & 63));
345 }
346 ccv_array_free(parameter_trainables);
347 // Assert no internal is alias.
348 for (i = 0; i < internals->rnum; i++)
349 {
350 const ccv_nnc_tensor_symbol_t internal = *(ccv_nnc_tensor_symbol_t*)ccv_array_get(internals, i)((void*)(((char*)((internals)->data)) + (size_t)(internals
)->rsize * (size_t)(i)));
351 const ccv_nnc_tensor_symbol_t alias_to = ccv_nnc_tensor_symbol_alias_to(internal.graph, internal);
352 assert(alias_to.graph == 0)((void) sizeof ((alias_to.graph == 0) ? 1 : 0), __extension__
({ if (alias_to.graph == 0) ; else __assert_fail ("alias_to.graph == 0"
, "ccv_cnnp_model.c", 352, __extension__ __PRETTY_FUNCTION__)
; })); // Cannot find the one alias to.
353 }
354 const int output_size = model->output_size;
355 ccv_nnc_graph_exec_symbol_autogen(model->graph, 0, 0, CCV_NNC_AUTOGEN_ALL_EXECS | CCV_NNC_AUTOGEN_SOURCES_AND_DESTINATIONS);
356 const int parameters_rnum = parameters->rnum;
357 if (input_size > 0)
358 {
359 ccv_array_resize(parameters, parameters_rnum + input_size);
360 memcpy(ccv_array_get(parameters, parameters_rnum)((void*)(((char*)((parameters)->data)) + (size_t)(parameters
)->rsize * (size_t)(parameters_rnum))), model->inputs, input_size * sizeof(ccv_nnc_tensor_symbol_t));
361 }
362 ccv_nnc_symbolic_graph_simplify(model->graph,
363 SYMBOLIC_GRAPH_PASSES(CCV_NNC_SIMPLIFY_COMMON_SUBEXPRESSION_ELIMINATION,(const int []){CCV_NNC_SIMPLIFY_COMMON_SUBEXPRESSION_ELIMINATION
, CCV_NNC_SIMPLIFY_DATA_TRANSFER_OPT, CCV_NNC_SIMPLIFY_OPS_FUSION
, CCV_NNC_SIMPLIFY_GRAPH_PRUNING}, (1 +1 +1 +1 +1 +0 +0 +0 +0
+0 +0 +0 +0 +0 +0 +0 +0 +0 +0 +0 +0 +0 +0 +0 +0 +0 +0 +0 +0 +
0 +0 +0 +0 +0 +0 +0 +0 +0 +0 +0 +0 +0 +0 +0 +0 +0 +0 +0 +0 +0
+0 +0 +0 +0 +0 +0 +0 +0 +0 +0 +0 +0 +0 +0 -1)
364 CCV_NNC_SIMPLIFY_DATA_TRANSFER_OPT,(const int []){CCV_NNC_SIMPLIFY_COMMON_SUBEXPRESSION_ELIMINATION
, CCV_NNC_SIMPLIFY_DATA_TRANSFER_OPT, CCV_NNC_SIMPLIFY_OPS_FUSION
, CCV_NNC_SIMPLIFY_GRAPH_PRUNING}, (1 +1 +1 +1 +1 +0 +0 +0 +0
+0 +0 +0 +0 +0 +0 +0 +0 +0 +0 +0 +0 +0 +0 +0 +0 +0 +0 +0 +0 +
0 +0 +0 +0 +0 +0 +0 +0 +0 +0 +0 +0 +0 +0 +0 +0 +0 +0 +0 +0 +0
+0 +0 +0 +0 +0 +0 +0 +0 +0 +0 +0 +0 +0 +0 -1)
365 CCV_NNC_SIMPLIFY_OPS_FUSION,(const int []){CCV_NNC_SIMPLIFY_COMMON_SUBEXPRESSION_ELIMINATION
, CCV_NNC_SIMPLIFY_DATA_TRANSFER_OPT, CCV_NNC_SIMPLIFY_OPS_FUSION
, CCV_NNC_SIMPLIFY_GRAPH_PRUNING}, (1 +1 +1 +1 +1 +0 +0 +0 +0
+0 +0 +0 +0 +0 +0 +0 +0 +0 +0 +0 +0 +0 +0 +0 +0 +0 +0 +0 +0 +
0 +0 +0 +0 +0 +0 +0 +0 +0 +0 +0 +0 +0 +0 +0 +0 +0 +0 +0 +0 +0
+0 +0 +0 +0 +0 +0 +0 +0 +0 +0 +0 +0 +0 +0 -1)
366 CCV_NNC_SIMPLIFY_GRAPH_PRUNING)(const int []){CCV_NNC_SIMPLIFY_COMMON_SUBEXPRESSION_ELIMINATION
, CCV_NNC_SIMPLIFY_DATA_TRANSFER_OPT, CCV_NNC_SIMPLIFY_OPS_FUSION
, CCV_NNC_SIMPLIFY_GRAPH_PRUNING}, (1 +1 +1 +1 +1 +0 +0 +0 +0
+0 +0 +0 +0 +0 +0 +0 +0 +0 +0 +0 +0 +0 +0 +0 +0 +0 +0 +0 +0 +
0 +0 +0 +0 +0 +0 +0 +0 +0 +0 +0 +0 +0 +0 +0 +0 +0 +0 +0 +0 +0
+0 +0 +0 +0 +0 +0 +0 +0 +0 +0 +0 +0 +0 +0 -1),
367 ccv_array_get(parameters, 0)((void*)(((char*)((parameters)->data)) + (size_t)(parameters
)->rsize * (size_t)(0))), parameters_rnum + input_size,
368 model->outputs, output_size,
369 SYMBOLIC_GRAPH_SOURCES(model->graph)ccv_nnc_symbolic_graph_sources(model->graph), ccv_nnc_symbolic_graph_source_size
(model->graph), SYMBOLIC_GRAPH_DESTINATIONS(model->graph)ccv_nnc_symbolic_graph_destinations(model->graph), ccv_nnc_symbolic_graph_destination_size
(model->graph));
370 ccv_nnc_graph_exec_symbol_autogen(model->graph, 0, 0, CCV_NNC_AUTOGEN_SOURCES_AND_DESTINATIONS);
371 // Size it down.
372 parameters->rnum = parameters_rnum;
373 ccv_cnnp_compiled_data_t* compiled_data = model->compiled_data = cccalloccalloc(1, sizeof(ccv_cnnp_compiled_data_t) + sizeof(ccv_nnc_tensor_symbol_t) * (output_size * 2 - 1));
374 _ccv_cnnp_compiled_data_init(compiled_data, output_size, build_data.gradient_checkpoints);
375 const int evaluate_to_size = compiled_data->evaluate.to_size = ccv_nnc_symbolic_graph_destination_size(model->graph);
376 assert(evaluate_to_size > 0)((void) sizeof ((evaluate_to_size > 0) ? 1 : 0), __extension__
({ if (evaluate_to_size > 0) ; else __assert_fail ("evaluate_to_size > 0"
, "ccv_cnnp_model.c", 376, __extension__ __PRETTY_FUNCTION__)
; }));
377 compiled_data->evaluate.tos = ccmallocmalloc(sizeof(ccv_nnc_graph_exec_symbol_t) * evaluate_to_size);
378 memcpy(compiled_data->evaluate.tos, ccv_nnc_symbolic_graph_destinations(model->graph), sizeof(ccv_nnc_graph_exec_symbol_t) * evaluate_to_size);
379 compiled_data->loss = loss;
380 if (loss.cmd == CCV_NNC_NOOP)
381 {
382 // If no loss function provided, there is no fits.
383 for (i = 0; i < output_size; i++)
384 {
385 compiled_data->fits[i] = NO_TENSOR_SYMBOL(const ccv_nnc_tensor_symbol_t){.d = CCV_NNC_NO_TENSOR_SYMBOL
};
386 const ccv_nnc_tensor_symbol_t alias_to = ccv_nnc_tensor_symbol_alias_to(model->graph, model->outputs[i]);
387 if (alias_to.d < 0)
388 compiled_data->f[i] = model->outputs[i];
389 else { // We cannot differentiate against an alias, therefore, we have to verify this output is full, and we can diff against the original.
390 int ofs[CCV_NNC_MAX_DIM_ALLOC(12)];
391 int inc[CCV_NNC_MAX_DIM_ALLOC(12)];
392 ccv_nnc_tensor_symbol_alias_params(model->graph, model->outputs[i], ofs, inc);
393 int j;
394 for (j = 0; j < CCV_NNC_MAX_DIM_ALLOC(12); j++)
395 { assert(ofs[j] == 0)((void) sizeof ((ofs[j] == 0) ? 1 : 0), __extension__ ({ if (
ofs[j] == 0) ; else __assert_fail ("ofs[j] == 0", "ccv_cnnp_model.c"
, 395, __extension__ __PRETTY_FUNCTION__); })); } // There is no ofs.
396 compiled_data->f[i] = alias_to; // Unfortunately, I cannot assert the size yet.
397 }
398 }
399 } else {
400 for (i = 0; i < output_size; i++)
401 {
402 const ccv_nnc_tensor_param_t info = ccv_nnc_tensor_symbol_params(model->graph, model->outputs[i]);
403 const ccv_nnc_tensor_symbol_t fit = compiled_data->fits[i] = ccv_nnc_tensor_symbol_new(model->graph, info, 0);
404 compiled_data->f[i] = ccv_nnc_tensor_symbol_new(model->graph, ccv_nnc_tensor_auto, 0);
405 ccv_nnc_graph_exec_symbol_new(model->graph, loss, TENSOR_SYMBOL_LIST(model->outputs[i], fit)(const ccv_nnc_tensor_symbol_t []){model->outputs[i], fit}
, (1 +1 +1 +0 +0 +0 +0 +0 +0 +0 +0 +0 +0 +0 +0 +0 +0 +0 +0 +0
+0 +0 +0 +0 +0 +0 +0 +0 +0 +0 +0 +0 +0 +0 +0 +0 +0 +0 +0 +0 +
0 +0 +0 +0 +0 +0 +0 +0 +0 +0 +0 +0 +0 +0 +0 +0 +0 +0 +0 +0 +0
+0 +0 +0 -1), TENSOR_SYMBOL_LIST(compiled_data->f[i])(const ccv_nnc_tensor_symbol_t []){compiled_data->f[i]}, (
1 +1 +0 +0 +0 +0 +0 +0 +0 +0 +0 +0 +0 +0 +0 +0 +0 +0 +0 +0 +0
+0 +0 +0 +0 +0 +0 +0 +0 +0 +0 +0 +0 +0 +0 +0 +0 +0 +0 +0 +0 +
0 +0 +0 +0 +0 +0 +0 +0 +0 +0 +0 +0 +0 +0 +0 +0 +0 +0 +0 +0 +0
+0 +0 -1), 0);
406 }
407 }
408 if (loss.cmd != CCV_NNC_NOOP)
409 {
410 ccv_nnc_graph_exec_symbol_autogen(model->graph, 0, 0, CCV_NNC_AUTOGEN_ALL_EXECS | CCV_NNC_AUTOGEN_SOURCES_AND_DESTINATIONS);
411 ccv_nnc_symbolic_graph_simplify(model->graph,
412 SYMBOLIC_GRAPH_PASSES(CCV_NNC_SIMPLIFY_OPS_FUSION)(const int []){CCV_NNC_SIMPLIFY_OPS_FUSION}, (1 +1 +0 +0 +0 +
0 +0 +0 +0 +0 +0 +0 +0 +0 +0 +0 +0 +0 +0 +0 +0 +0 +0 +0 +0 +0
+0 +0 +0 +0 +0 +0 +0 +0 +0 +0 +0 +0 +0 +0 +0 +0 +0 +0 +0 +0 +
0 +0 +0 +0 +0 +0 +0 +0 +0 +0 +0 +0 +0 +0 +0 +0 +0 +0 -1), // Only do Ops fusion, in this way, we can fuse the loss function.
413 0, 0, // No need to provide binds at this point.
414 compiled_data->f, model->output_size,
415 SYMBOLIC_GRAPH_SOURCES(model->graph)ccv_nnc_symbolic_graph_sources(model->graph), ccv_nnc_symbolic_graph_source_size
(model->graph), SYMBOLIC_GRAPH_DESTINATIONS(model->graph)ccv_nnc_symbolic_graph_destinations(model->graph), ccv_nnc_symbolic_graph_destination_size
(model->graph));
416 ccv_nnc_graph_exec_symbol_autogen(model->graph, 0, 0, CCV_NNC_AUTOGEN_SOURCES_AND_DESTINATIONS);
417 }
418 // If inputs are from GPU, stream type is GPU.
419 compiled_data->parameters = parameters;
420 compiled_data->parameter_flags = parameter_flags;
421 compiled_data->internals = internals;
422 compiled_data->ids.parameters = parameter_ids;
423 compiled_data->ids.internals = internal_ids;
424 ccv_cnnp_model_gradient_checkpoints_cleanup_after_build(compiled_data, model->graph);
425}
426
427static void _ccv_cnnp_graph_push_graph_exec_symbol(void* context, const ccv_nnc_graph_exec_symbol_t symbol, const ccv_nnc_cmd_t cmd, const ccv_nnc_tensor_symbol_t* const inputs, const int input_size, const ccv_nnc_tensor_symbol_t* const outputs, const int output_size, const char* const name)
428{
429 ccv_array_t* const stack = (ccv_array_t*)context;
430 ccv_array_push(stack, &symbol.d);
431}
432
433static void _ccv_nnc_tensor_symbol_reinit(const ccv_nnc_symbolic_graph_t* const src_graph, ccv_nnc_symbolic_graph_t* const dest_graph, const int src_index, const int dest_index)
434{
435 int src_alias_ref = src_index;
436 const ccv_nnc_tensor_symbol_info_t* src_info = (ccv_nnc_tensor_symbol_info_t*)ccv_array_get(src_graph->tensor_symbol_info, src_alias_ref)((void*)(((char*)((src_graph->tensor_symbol_info)->data
)) + (size_t)(src_graph->tensor_symbol_info)->rsize * (
size_t)(src_alias_ref)));
437 while (src_info->alias_ref)
438 {
439 src_alias_ref = src_info->alias_ref - 1;
440 src_info = (ccv_nnc_tensor_symbol_info_t*)ccv_array_get(src_graph->tensor_symbol_info, src_alias_ref)((void*)(((char*)((src_graph->tensor_symbol_info)->data
)) + (size_t)(src_graph->tensor_symbol_info)->rsize * (
size_t)(src_alias_ref)));
441 }
442 int dest_alias_ref = dest_index;
443 const ccv_nnc_tensor_symbol_info_t* dest_info = (ccv_nnc_tensor_symbol_info_t*)ccv_array_get(dest_graph->tensor_symbol_info, dest_alias_ref)((void*)(((char*)((dest_graph->tensor_symbol_info)->data
)) + (size_t)(dest_graph->tensor_symbol_info)->rsize * (
size_t)(dest_alias_ref)));
444 while (dest_info->alias_ref)
445 {
446 dest_alias_ref = dest_info->alias_ref - 1;
447 dest_info = (ccv_nnc_tensor_symbol_info_t*)ccv_array_get(dest_graph->tensor_symbol_info, dest_alias_ref)((void*)(((char*)((dest_graph->tensor_symbol_info)->data
)) + (size_t)(dest_graph->tensor_symbol_info)->rsize * (
size_t)(dest_alias_ref)));
448 }
449 assert((src_alias_ref != src_index) == (dest_alias_ref != dest_index))((void) sizeof (((src_alias_ref != src_index) == (dest_alias_ref
!= dest_index)) ? 1 : 0), __extension__ ({ if ((src_alias_ref
!= src_index) == (dest_alias_ref != dest_index)) ; else __assert_fail
("(src_alias_ref != src_index) == (dest_alias_ref != dest_index)"
, "ccv_cnnp_model.c", 449, __extension__ __PRETTY_FUNCTION__)
; }));
450 if (src_alias_ref != src_index)
451 {
452 const ccv_nnc_tensor_symbol_t src_alias_symbol = {
453 .d = src_alias_ref,
454 .graph = src_graph
455 };
456 const ccv_nnc_tensor_symbol_t dest_alias_symbol = {
457 .d = dest_alias_ref,
458 .graph = dest_graph
459 };
460 const ccv_nnc_tensor_param_t params = ccv_nnc_tensor_symbol_params(src_graph, src_alias_symbol);
461 ccv_nnc_tensor_symbol_set(dest_graph, dest_alias_symbol, params);
462 }
463 const ccv_nnc_tensor_symbol_t src_symbol = {
464 .d = src_index,
465 .graph = src_graph
466 };
467 const ccv_nnc_tensor_symbol_t dest_symbol = {
468 .d = dest_index,
469 .graph = dest_graph
470 };
471 const ccv_nnc_tensor_param_t params = ccv_nnc_tensor_symbol_params(src_graph, src_symbol);
472 ccv_nnc_tensor_symbol_set(dest_graph, dest_symbol, params);
473 int ofs[CCV_NNC_MAX_DIM_ALLOC(12)];
474 int inc[CCV_NNC_MAX_DIM_ALLOC(12)];
475 if (0 == ccv_nnc_tensor_symbol_alias_params(src_graph, src_symbol, ofs, inc))
476 ccv_nnc_tensor_symbol_alias_set(dest_graph, dest_symbol, ofs, inc);
477}
478
479static int _ccv_nnc_tensor_symbol_check_dim(const ccv_nnc_symbolic_graph_t* const src_graph, ccv_nnc_symbolic_graph_t* const dest_graph, const int src_index, const int dest_index)
480{
481 const ccv_nnc_tensor_symbol_t src_symbol = {
482 .d = src_index,
483 .graph = src_graph
484 };
485 const ccv_nnc_tensor_param_t src_params = ccv_nnc_tensor_symbol_params(src_graph, src_symbol);
486 const ccv_nnc_tensor_symbol_t dest_symbol = {
487 .d = dest_index,
488 .graph = dest_graph
489 };
490 const ccv_nnc_tensor_param_t dest_params = ccv_nnc_tensor_symbol_params(dest_graph, dest_symbol);
491 if (src_params.dim[0] == 0 || dest_params.dim[0] == 0)
492 return 1;
493 return memcmp(src_params.dim, dest_params.dim, sizeof(src_params.dim)) == 0;
494}
495
496static void _ccv_cnnp_model_gradient_init(ccv_cnnp_model_t* const model, const int gradient_mode, const uint64_t disable_outgrad, ccv_nnc_tensor_t* const* const fits, const int fit_size);
497static void _ccv_cnnp_compiled_data_graph_free(ccv_cnnp_compiled_data_t* const compiled_data);
498
499typedef struct {
500 int parallel_count;
501 ccv_nnc_symbolic_graph_t* graph;
502 ccv_nnc_graph_exec_arena_t* graph_exec_arena;
503} ccv_nnc_graph_exec_update_t;
504
505static void _ccv_cnnp_cmd_update_for_execs(void* const context, const ccv_nnc_graph_exec_symbol_t symbol, const ccv_nnc_cmd_t cmd, const ccv_nnc_hint_t hint)
506{
507 ccv_nnc_graph_exec_update_t* const graph_exec_update = (ccv_nnc_graph_exec_update_t*)context;
508 ccv_nnc_graph_exec_arena_t* const graph_exec_arena = graph_exec_update->graph_exec_arena;
509 ccv_nnc_graph_exec_t graph_exec = ccv_nnc_graph_exec_from_symbol(graph_exec_arena, symbol);
510 ccv_nnc_graph_exec_set(graph_exec.graph, graph_exec, cmd);
511 ccv_nnc_graph_exec_set_hint(graph_exec.graph, graph_exec, hint);
512 const ccv_nnc_symbolic_graph_t* const graph = graph_exec_update->graph;
513 const int parallel_count = graph_exec_update->parallel_count;
514 int i;
515 for (i = 1; i < parallel_count; i++)
516 {
517 const ccv_nnc_graph_exec_t copy = ccv_nnc_graph_exec_from_symbol(graph_exec_arena, ccv_nnc_graph_exec_symbol_copy(graph, symbol, i));
518 if (!CCV_NO_GRAPH_EXEC(copy)((copy).graph == 0))
519 {
520 ccv_nnc_graph_exec_set(copy.graph, copy, cmd);
521 ccv_nnc_graph_exec_set_hint(copy.graph, copy, hint);
522 }
523 }
524}
525
526void ccv_cnnp_model_absorb(ccv_cnnp_model_t* const model, ccv_cnnp_model_t* const init, const ccv_nnc_tensor_param_t* const inputs, const int input_size)
527{
528 assert(model->graph)((void) sizeof ((model->graph) ? 1 : 0), __extension__ ({ if
(model->graph) ; else __assert_fail ("model->graph", "ccv_cnnp_model.c"
, 528, __extension__ __PRETTY_FUNCTION__); }));
529 assert(model->compiled_data)((void) sizeof ((model->compiled_data) ? 1 : 0), __extension__
({ if (model->compiled_data) ; else __assert_fail ("model->compiled_data"
, "ccv_cnnp_model.c", 529, __extension__ __PRETTY_FUNCTION__)
; }));
530 assert(!init->graph)((void) sizeof ((!init->graph) ? 1 : 0), __extension__ ({ if
(!init->graph) ; else __assert_fail ("!init->graph", "ccv_cnnp_model.c"
, 530, __extension__ __PRETTY_FUNCTION__); }));
531 ccv_cnnp_compiled_data_t* const compiled_data = model->compiled_data;
532 init->graph = ccv_nnc_symbolic_graph_new();
533 ccv_array_t* const stack = ccv_array_new(sizeof(int), 0, 0);
534 ccv_nnc_graph_exec_symbol_new_hook(init->graph, _ccv_cnnp_graph_push_graph_exec_symbol, stack, 0);
535 _ccv_cnnp_model_compile(init, inputs, input_size, compiled_data->loss);
536 init->parallel_count = model->parallel_count;
537 init->memory_compression = model->memory_compression;
538 init->memory_reduction = model->memory_reduction;
539 init->gradient_checkpointing = model->gradient_checkpointing;
540 init->compiled_data->stream_type = model->compiled_data->stream_type;
541 init->compiled_data->minimize.minimizer = model->compiled_data->minimize.minimizer;
542 init->compiled_data->minimize.max_saved_aux_size = model->compiled_data->minimize.max_saved_aux_size;
543 if (model->compiled_data->gradient_mode != CCV_CNNP_COMPILED_DATA_GRADIENT_NONE)
544 _ccv_cnnp_model_gradient_init(init, model->compiled_data->gradient_mode, model->compiled_data->disable_outgrad, 0, 0);
545 ccv_nnc_graph_exec_symbol_new_hook(init->graph, 0, 0, 0);
546 ccv_nnc_symbolic_graph_tensor_auto(init->graph, TRAVERSE_FULL0,0,0,0);
547 int i, j;
548 // Verify parameters, internals and saved_aux in both graph has the same dimensionality.
549 for (i = 0; i < compiled_data->parameters->rnum; i++)
550 {
551 const int d = ((ccv_nnc_tensor_symbol_t*)ccv_array_get(compiled_data->parameters, i)((void*)(((char*)((compiled_data->parameters)->data)) +
(size_t)(compiled_data->parameters)->rsize * (size_t)(
i))))->d;
552 assert(_ccv_nnc_tensor_symbol_check_dim(model->graph, init->graph, d, d))((void) sizeof ((_ccv_nnc_tensor_symbol_check_dim(model->graph
, init->graph, d, d)) ? 1 : 0), __extension__ ({ if (_ccv_nnc_tensor_symbol_check_dim
(model->graph, init->graph, d, d)) ; else __assert_fail
("_ccv_nnc_tensor_symbol_check_dim(model->graph, init->graph, d, d)"
, "ccv_cnnp_model.c", 552, __extension__ __PRETTY_FUNCTION__)
; }));
553 }
554 for (i = 0; i < compiled_data->internals->rnum; i++)
555 {
556 const int d = ((ccv_nnc_tensor_symbol_t*)ccv_array_get(compiled_data->internals, i)((void*)(((char*)((compiled_data->internals)->data)) + (
size_t)(compiled_data->internals)->rsize * (size_t)(i))
))->d;
557 assert(_ccv_nnc_tensor_symbol_check_dim(model->graph, init->graph, d, d))((void) sizeof ((_ccv_nnc_tensor_symbol_check_dim(model->graph
, init->graph, d, d)) ? 1 : 0), __extension__ ({ if (_ccv_nnc_tensor_symbol_check_dim
(model->graph, init->graph, d, d)) ; else __assert_fail
("_ccv_nnc_tensor_symbol_check_dim(model->graph, init->graph, d, d)"
, "ccv_cnnp_model.c", 557, __extension__ __PRETTY_FUNCTION__)
; }));
558 }
559 // Update inputs.
560 assert(model->input_size == init->input_size)((void) sizeof ((model->input_size == init->input_size)
? 1 : 0), __extension__ ({ if (model->input_size == init->
input_size) ; else __assert_fail ("model->input_size == init->input_size"
, "ccv_cnnp_model.c", 560, __extension__ __PRETTY_FUNCTION__)
; }));
561 for (i = 0; i < model->input_size; i++)
562 if (model->inputs[i].d >= 0)
563 {
564 assert(init->inputs[i].d >= 0)((void) sizeof ((init->inputs[i].d >= 0) ? 1 : 0), __extension__
({ if (init->inputs[i].d >= 0) ; else __assert_fail ("init->inputs[i].d >= 0"
, "ccv_cnnp_model.c", 564, __extension__ __PRETTY_FUNCTION__)
; }));
565 _ccv_nnc_tensor_symbol_reinit(init->graph, model->graph, init->inputs[i].d, model->inputs[i].d);
566 }
567 // Update outputs.
568 assert(model->output_size == init->output_size)((void) sizeof ((model->output_size == init->output_size
) ? 1 : 0), __extension__ ({ if (model->output_size == init
->output_size) ; else __assert_fail ("model->output_size == init->output_size"
, "ccv_cnnp_model.c", 568, __extension__ __PRETTY_FUNCTION__)
; }));
569 for (i = 0; i < model->output_size; i++)
570 {
571 if (model->outputs[i].d >= 0)
572 {
573 assert(init->outputs[i].d >= 0)((void) sizeof ((init->outputs[i].d >= 0) ? 1 : 0), __extension__
({ if (init->outputs[i].d >= 0) ; else __assert_fail (
"init->outputs[i].d >= 0", "ccv_cnnp_model.c", 573, __extension__
__PRETTY_FUNCTION__); }));
574 _ccv_nnc_tensor_symbol_reinit(init->graph, model->graph, init->outputs[i].d, model->outputs[i].d);
575 }
576 if (model->outputs[i].d != model->compiled_data->f[i].d)
577 {
578 assert(init->outputs[i].d != init->compiled_data->f[i].d)((void) sizeof ((init->outputs[i].d != init->compiled_data
->f[i].d) ? 1 : 0), __extension__ ({ if (init->outputs[
i].d != init->compiled_data->f[i].d) ; else __assert_fail
("init->outputs[i].d != init->compiled_data->f[i].d"
, "ccv_cnnp_model.c", 578, __extension__ __PRETTY_FUNCTION__)
; }));
579 if (model->compiled_data->f[i].d >= 0)
580 {
581 assert(init->compiled_data->f[i].d >= 0)((void) sizeof ((init->compiled_data->f[i].d >= 0) ?
1 : 0), __extension__ ({ if (init->compiled_data->f[i]
.d >= 0) ; else __assert_fail ("init->compiled_data->f[i].d >= 0"
, "ccv_cnnp_model.c", 581, __extension__ __PRETTY_FUNCTION__)
; }));
582 _ccv_nnc_tensor_symbol_reinit(init->graph, model->graph, init->compiled_data->f[i].d, model->compiled_data->f[i].d);
583 }
584 }
585 }
586 // Go through the graph to set tensor on matching symbols
587 for (i = 0; i < stack->rnum; i++)
588 {
589 const int d = *(int*)ccv_array_get(stack, i)((void*)(((char*)((stack)->data)) + (size_t)(stack)->rsize
* (size_t)(i)));
590 // If exceed range, skip.
591 if (d >= ccv_nnc_graph_exec_symbol_count(init->graph) ||
592 d >= ccv_nnc_graph_exec_symbol_count(model->graph))
593 continue;
594 const ccv_nnc_graph_exec_symbol_t src_symbol = {
595 .d = d,
596 .graph = init->graph
597 };
598 const ccv_nnc_graph_exec_symbol_t dest_symbol = {
599 .d = d,
600 .graph = model->graph
601 };
602 const ccv_nnc_cmd_t src_cmd = ccv_nnc_graph_exec_symbol_cmd(init->graph, src_symbol);
603 const ccv_nnc_cmd_t dest_cmd = ccv_nnc_graph_exec_symbol_cmd(model->graph, dest_symbol);
604 // If the name doesn't match, skip.
605 if (dest_cmd.cmd != src_cmd.cmd && src_cmd.cmd != CCV_NNC_NOOP)
606 continue;
607 // Now get all the inputs and outputs, if matches, set them.
608 const int* src_inputs;
609 int src_input_size;
610 const int* src_outputs;
611 int src_output_size;
612 ccv_nnc_graph_exec_symbol_io(init->graph, src_symbol, &src_inputs, &src_input_size, &src_outputs, &src_output_size);
613 const int* dest_inputs;
614 int dest_input_size;
615 const int* dest_outputs;
616 int dest_output_size;
617 ccv_nnc_graph_exec_symbol_io(model->graph, dest_symbol, &dest_inputs, &dest_input_size, &dest_outputs, &dest_output_size);
618 // We may have unmatched input / output size because this is the minimizer and it has
619 // different saved_aux (for example, when we shrunk with CMD_NOOP).
620 if (src_input_size != dest_input_size)
621 continue;
622 if (src_output_size != dest_output_size)
623 continue;
624 ccv_nnc_graph_exec_symbol_set(model->graph, dest_symbol, src_cmd);
625 // There may be mismatches of the source tensor symbols and destination tensor symbols. The reason is because
626 // we may later passed-in the minimizer, therefore, we may allocate tensors for minimizer later in the original
627 // graph whereas in the newly created graph, it is streamlined (the minimizer exists from the beginning). That
628 // will make the order of tensor symbols creation different, therefore, exact which tensor is which wrong as
629 // well. However, set a new minimizer won't change the exec symbol ordering, because we never create new exec
630 // symbols after gradient init step. Changing a new minimizer just updated that exec symbols setting, it is not
631 // a new exec symbol.
632 for (j = 0; j < src_input_size; j++)
633 if (src_inputs[j] >= 0)
634 _ccv_nnc_tensor_symbol_reinit(init->graph, model->graph, src_inputs[j], dest_inputs[j]);
635 for (j = 0; j < src_output_size; j++)
636 if (src_outputs[j] >= 0)
637 _ccv_nnc_tensor_symbol_reinit(init->graph, model->graph, src_outputs[j], dest_outputs[j]);
638 }
639 ccv_array_free(stack);
640 // After this, we get all tensors in the model graph resolved through tensor_auto.
641 ccv_nnc_symbolic_graph_tensor_auto(model->graph, TRAVERSE_FULL0,0,0,0);
642 // Verify symbols we get matches.
643 const int parameter_size = compiled_data->parameters->rnum;
644 for (i = 0; i < parameter_size; i++)
645 { assert(((ccv_nnc_tensor_symbol_t*)ccv_array_get(compiled_data->parameters, i))->d == ((ccv_nnc_tensor_symbol_t*)ccv_array_get(init->compiled_data->parameters, i))->d)((void) sizeof ((((ccv_nnc_tensor_symbol_t*)((void*)(((char*)
((compiled_data->parameters)->data)) + (size_t)(compiled_data
->parameters)->rsize * (size_t)(i))))->d == ((ccv_nnc_tensor_symbol_t
*)((void*)(((char*)((init->compiled_data->parameters)->
data)) + (size_t)(init->compiled_data->parameters)->
rsize * (size_t)(i))))->d) ? 1 : 0), __extension__ ({ if (
((ccv_nnc_tensor_symbol_t*)((void*)(((char*)((compiled_data->
parameters)->data)) + (size_t)(compiled_data->parameters
)->rsize * (size_t)(i))))->d == ((ccv_nnc_tensor_symbol_t
*)((void*)(((char*)((init->compiled_data->parameters)->
data)) + (size_t)(init->compiled_data->parameters)->
rsize * (size_t)(i))))->d) ; else __assert_fail ("((ccv_nnc_tensor_symbol_t*)ccv_array_get(compiled_data->parameters, i))->d == ((ccv_nnc_tensor_symbol_t*)ccv_array_get(init->compiled_data->parameters, i))->d"
, "ccv_cnnp_model.c", 645, __extension__ __PRETTY_FUNCTION__)
; })); }
646 const int internal_size = compiled_data->internals->rnum;
647 for (i = 0; i < internal_size; i++)
648 { assert(((ccv_nnc_tensor_symbol_t*)ccv_array_get(compiled_data->internals, i))->d == ((ccv_nnc_tensor_symbol_t*)ccv_array_get(init->compiled_data->internals, i))->d)((void) sizeof ((((ccv_nnc_tensor_symbol_t*)((void*)(((char*)
((compiled_data->internals)->data)) + (size_t)(compiled_data
->internals)->rsize * (size_t)(i))))->d == ((ccv_nnc_tensor_symbol_t
*)((void*)(((char*)((init->compiled_data->internals)->
data)) + (size_t)(init->compiled_data->internals)->rsize
* (size_t)(i))))->d) ? 1 : 0), __extension__ ({ if (((ccv_nnc_tensor_symbol_t
*)((void*)(((char*)((compiled_data->internals)->data)) +
(size_t)(compiled_data->internals)->rsize * (size_t)(i
))))->d == ((ccv_nnc_tensor_symbol_t*)((void*)(((char*)((init
->compiled_data->internals)->data)) + (size_t)(init->
compiled_data->internals)->rsize * (size_t)(i))))->d
) ; else __assert_fail ("((ccv_nnc_tensor_symbol_t*)ccv_array_get(compiled_data->internals, i))->d == ((ccv_nnc_tensor_symbol_t*)ccv_array_get(init->compiled_data->internals, i))->d"
, "ccv_cnnp_model.c", 648, __extension__ __PRETTY_FUNCTION__)
; })); }
649 // Go through compiled data.
650 if (compiled_data->tensor_arena)
651 {
652 const int flag = ccv_nnc_tensor_arena_reinit(compiled_data->tensor_arena, model->graph);
653 if (flag == 0 && compiled_data->graph_exec_arena)
654 {
655 ccv_nnc_graph_exec_reinit(compiled_data->graph_exec_arena, compiled_data->graph, model->graph);
656 // Since we will reinit, if we previously set is_test, we need to set it again.
657 if (compiled_data->is_test)
658 {
659 const int parallel_count = ccv_max(model->parallel_count, 1)({ typeof (model->parallel_count) _a = (model->parallel_count
); typeof (1) _b = (1); (_a > _b) ? _a : _b; });
660 ccv_nnc_graph_exec_update_t update = {
661 .parallel_count = parallel_count,
662 .graph = model->graph,
663 .graph_exec_arena = compiled_data->graph_exec_arena,
664 };
665 ccv_cnnp_model_set_is_test(model, 1, _ccv_cnnp_cmd_update_for_execs, &update);
666 }
667 } else
668 // Free-up tensor arena & graph exec arena.
669 _ccv_cnnp_compiled_data_graph_free(compiled_data);
670 }
671 // There are other compiled graphs, for accum and apply gradients.
672 // However, the main conclusion is, these absorb operations shouldn't impact parameters.
673 // Thus, it won't impact the shape of gradients (only outgrad). Since for outgrad, we
674 // don't allocate ourselves, it is not a concern. For normal gradients, the shape cannot
675 // be changed otherwise parameters' shape will be meaningless. The same goes to internals.
676 // That is why we don't update these compiled graphs at all this point.
677 // Free the model, we've already "absorbed" it.
678 ccv_cnnp_model_free(init);
679}
680
681void ccv_cnnp_model_compile(ccv_cnnp_model_t* const model, const ccv_nnc_tensor_param_t* const inputs, const int input_size, const ccv_nnc_cmd_t minimizer, const ccv_nnc_cmd_t loss)
682{
683 assert(input_size == model->input_size || model->input_size == 0)((void) sizeof ((input_size == model->input_size || model->
input_size == 0) ? 1 : 0), __extension__ ({ if (input_size ==
model->input_size || model->input_size == 0) ; else __assert_fail
("input_size == model->input_size || model->input_size == 0"
, "ccv_cnnp_model.c", 683, __extension__ __PRETTY_FUNCTION__)
; }));
684 if (model->input_size == 0)
685 model->input_size = input_size;
686 if (!model->graph) // The graph is not compiled yet.
687 {
688 model->graph = ccv_nnc_symbolic_graph_new();
689 _ccv_cnnp_model_compile(model, inputs, input_size, loss);
690 assert(model->compiled_data)((void) sizeof ((model->compiled_data) ? 1 : 0), __extension__
({ if (model->compiled_data) ; else __assert_fail ("model->compiled_data"
, "ccv_cnnp_model.c", 690, __extension__ __PRETTY_FUNCTION__)
; }));
691 int i, flag = 0;
692 for (i = 0; !flag && i < input_size; i++)
693 flag = (CCV_TENSOR_GET_MEMORY(inputs[i].type)((inputs[i].type) & 0x3) == CCV_TENSOR_GPU_MEMORY);
694 // If inputs are from GPU, stream type is GPU.
695 model->compiled_data->stream_type = flag ? CCV_STREAM_CONTEXT_GPU : CCV_STREAM_CONTEXT_CPU;
696 model->compiled_data->minimize.minimizer = minimizer;
697 model->compiled_data->minimize.max_saved_aux_size = ccv_nnc_minimizer_saved_aux_size(minimizer);
698 } else {
699 // Now, finally fill in this part. If the graph is already compiled, we make a copy of the model.
700 // And then absorb the "new model" to the old one.
701 ccv_cnnp_model_t* const init = ccv_cnnp_model_copy(model, model->is_trainable);
702 ccv_cnnp_model_absorb(model, init, inputs, input_size);
703 // Reset minimizer.
704 ccv_cnnp_model_set_minimizer(model, minimizer, 1, 0, 0);
705 }
706}
707
708ccv_cnnp_model_t* ccv_cnnp_model_copy(const ccv_cnnp_model_t* const model, const int is_trainable)
709{
710 ccv_cnnp_model_t* const new_model = _ccv_cnnp_model_copy(model, 0);
711 new_model->is_trainable = is_trainable;
712 return new_model;
713}
714
715void ccv_cnnp_model_tensor_auto(ccv_cnnp_model_t* const model, ccv_nnc_tensor_param_t* const outputs, const int output_size)
716{
717 assert(model->graph)((void) sizeof ((model->graph) ? 1 : 0), __extension__ ({ if
(model->graph) ; else __assert_fail ("model->graph", "ccv_cnnp_model.c"
, 717, __extension__ __PRETTY_FUNCTION__); }));
718 assert(output_size == model->output_size)((void) sizeof ((output_size == model->output_size) ? 1 : 0
), __extension__ ({ if (output_size == model->output_size)
; else __assert_fail ("output_size == model->output_size"
, "ccv_cnnp_model.c", 718, __extension__ __PRETTY_FUNCTION__)
; }));
719 ccv_nnc_symbolic_graph_t* const graph = model->graph;
720 ccv_nnc_symbolic_graph_tensor_auto(graph, TRAVERSE_FULL0,0,0,0);
721 int i;
722 for (i = 0; i < output_size; i++)
723 {
724 assert(model->outputs[i].d != CCV_NNC_NO_TENSOR_SYMBOL)((void) sizeof ((model->outputs[i].d != CCV_NNC_NO_TENSOR_SYMBOL
) ? 1 : 0), __extension__ ({ if (model->outputs[i].d != CCV_NNC_NO_TENSOR_SYMBOL
) ; else __assert_fail ("model->outputs[i].d != CCV_NNC_NO_TENSOR_SYMBOL"
, "ccv_cnnp_model.c", 724, __extension__ __PRETTY_FUNCTION__)
; }));
725 outputs[i] = ccv_nnc_tensor_symbol_params(graph, model->outputs[i]);
726 }
727}
728
729void ccv_cnnp_model_set_workspace_size(ccv_cnnp_model_t* const model, size_t workspace_size)
730{
731 if (workspace_size == model->workspace_size)
732 return;
733 model->workspace_size = workspace_size;
734 ccv_cnnp_compiled_data_t* const compiled_data = model->compiled_data;
735 if (compiled_data && compiled_data->graph)
736 ccv_nnc_graph_autotune(compiled_data->graph, workspace_size, 0, TRAVERSE_FULL0,0,0,0);
737}
738
739size_t ccv_cnnp_model_workspace_size(ccv_cnnp_model_t* const model)
740{
741 return model->workspace_size;
742}
743
744void ccv_cnnp_model_set_data_parallel(ccv_cnnp_model_t* const model, const int parallel)
745{
746 if (parallel == 0)
747 model->parallel_count = ccv_nnc_device_count(CCV_STREAM_CONTEXT_GPU);
748 else
749 model->parallel_count = parallel;
750 ccv_cnnp_compiled_data_t* const compiled_data = model->compiled_data;
751 if (compiled_data)
752 { assert(!compiled_data->graph)((void) sizeof ((!compiled_data->graph) ? 1 : 0), __extension__
({ if (!compiled_data->graph) ; else __assert_fail ("!compiled_data->graph"
, "ccv_cnnp_model.c", 752, __extension__ __PRETTY_FUNCTION__)
; })); }
753}
754
755void ccv_cnnp_model_set_max_concurrency(ccv_cnnp_model_t* const model, const int max_stream_count)
756{
757 model->max_stream_count = max_stream_count;
758 ccv_cnnp_compiled_data_t* const compiled_data = model->compiled_data;
759 if (compiled_data)
760 { assert(!compiled_data->graph)((void) sizeof ((!compiled_data->graph) ? 1 : 0), __extension__
({ if (!compiled_data->graph) ; else __assert_fail ("!compiled_data->graph"
, "ccv_cnnp_model.c", 760, __extension__ __PRETTY_FUNCTION__)
; })); }
761}
762
763void ccv_cnnp_model_set_memory_compression(ccv_cnnp_model_t* const model, const int memory_compression)
764{
765 model->memory_compression = memory_compression;
766 ccv_cnnp_compiled_data_t* const compiled_data = model->compiled_data;
767 if (compiled_data)
768 { assert(!compiled_data->graph)((void) sizeof ((!compiled_data->graph) ? 1 : 0), __extension__
({ if (!compiled_data->graph) ; else __assert_fail ("!compiled_data->graph"
, "ccv_cnnp_model.c", 768, __extension__ __PRETTY_FUNCTION__)
; })); }
769}
770
771void ccv_cnnp_model_set_memory_reduction(ccv_cnnp_model_t* const model, const int memory_reduction)
772{
773 model->memory_reduction = memory_reduction;
774 ccv_cnnp_compiled_data_t* const compiled_data = model->compiled_data;
775 if (compiled_data)
776 { assert(!compiled_data->graph)((void) sizeof ((!compiled_data->graph) ? 1 : 0), __extension__
({ if (!compiled_data->graph) ; else __assert_fail ("!compiled_data->graph"
, "ccv_cnnp_model.c", 776, __extension__ __PRETTY_FUNCTION__)
; })); }
777}
778
779void ccv_cnnp_model_set_gradient_checkpointing(ccv_cnnp_model_t* const model, const int gradient_checkpointing)
780{
781 model->gradient_checkpointing = gradient_checkpointing;
782}
783
784int ccv_cnnp_model_gradient_checkpointing(ccv_cnnp_model_t* const model)
785{
786 return model->gradient_checkpointing;
787}
788
789typedef struct {
790 int parallel_count;
791 ccv_nnc_symbolic_graph_t* graph;
792 ccv_cnnp_compiled_data_t* compiled_data;
793 ccv_nnc_tensor_arena_t* tensor_arena;
794} ccv_nnc_tensor_init_states_t;
795
796static int _ccv_cnnp_any_to_init(const ccv_cnnp_compiled_data_t* const compiled_data)
797{
798 int i;
799 const uint32_t* const init_v = CCV_NNC_INIT_V(compiled_data->tensors_init.v)((uint32_t*)((uintptr_t)(compiled_data->tensors_init.v) &
~(uintptr_t)1));
800 for (i = 0; i < compiled_data->parameters->rnum; i++)
801 {
802 const int d = ((ccv_nnc_tensor_symbol_t*)ccv_array_get(compiled_data->parameters, i)((void*)(((char*)((compiled_data->parameters)->data)) +
(size_t)(compiled_data->parameters)->rsize * (size_t)(
i))))->d;
803 if (!(init_v[d >> 5] & (1u << (d & 0x1f))))
804 return 1;
805 }
806 for (i = 0; i < compiled_data->internals->rnum; i++)
807 {
808 const int d = ((ccv_nnc_tensor_symbol_t*)ccv_array_get(compiled_data->internals, i)((void*)(((char*)((compiled_data->internals)->data)) + (
size_t)(compiled_data->internals)->rsize * (size_t)(i))
))->d;
809 if (!(init_v[d >> 5] & (1u << (d & 0x1f))))
810 return 1;
811 }
812 return 0;
813}
814
815static void _ccv_cnnp_init_states_for_tensors(void* const context, const ccv_nnc_cmd_t cmd, const ccv_nnc_hint_t hint, const int flags, ccv_nnc_tensor_t* const input, const ccv_nnc_tensor_symbol_t output_symbol)
816{
817 ccv_nnc_tensor_init_states_t* const tensor_init_states = (ccv_nnc_tensor_init_states_t*)context;
818 ccv_nnc_tensor_arena_t* const tensor_arena = tensor_init_states->tensor_arena;
819 ccv_nnc_tensor_t* const output_tensor = ccv_nnc_tensor_from_symbol(tensor_arena, output_symbol);
820 if (!output_tensor)
821 return;
822 const int d = output_symbol.d;
823 assert(d < tensor_init_states->compiled_data->tensors_init.size)((void) sizeof ((d < tensor_init_states->compiled_data->
tensors_init.size) ? 1 : 0), __extension__ ({ if (d < tensor_init_states
->compiled_data->tensors_init.size) ; else __assert_fail
("d < tensor_init_states->compiled_data->tensors_init.size"
, "ccv_cnnp_model.c", 823, __extension__ __PRETTY_FUNCTION__)
; }));
824 uint32_t* const init_v = CCV_NNC_INIT_V(tensor_init_states->compiled_data->tensors_init.v)((uint32_t*)((uintptr_t)(tensor_init_states->compiled_data
->tensors_init.v) & ~(uintptr_t)1));
825 if (init_v[d >> 5] & (1u << (d & 0x1f)))
826 return;
827 init_v[d >> 5] |= (1u << (d & 0x1f));
828 ccv_nnc_cmd_exec(cmd, hint, flags, &input, input ? 1 : 0, &output_tensor, 1, 0);
829 const ccv_nnc_symbolic_graph_t* const graph = tensor_init_states->graph;
830 const int parallel_count = tensor_init_states->parallel_count;
831 int i;
832 for (i = 1; i < parallel_count; i++)
833 {
834 ccv_nnc_tensor_t* const copy = ccv_nnc_tensor_from_symbol(tensor_arena, ccv_nnc_tensor_symbol_copy(graph, output_symbol, i));
835 if (copy)
836 ccv_nnc_cmd_exec(CMD_DATA_TRANSFER_FORWARD()ccv_nnc_cmd(CCV_NNC_DATA_TRANSFER_FORWARD, 0, ccv_nnc_cmd_auto
, 0), ccv_nnc_no_hint, 0, &output_tensor, 1, &copy, 1, 0);
837 }
838}
839
840// This method can only handle cases we added new tensors and exec, never delete. This invariant is true because
841// we setup everything (including calling simplify method) in ccv_cnnp_model_compile method, before this rewind setup.
842static void _ccv_cnnp_model_rewind_graph(ccv_cnnp_model_t* const model)
843{
844 assert(model->graph)((void) sizeof ((model->graph) ? 1 : 0), __extension__ ({ if
(model->graph) ; else __assert_fail ("model->graph", "ccv_cnnp_model.c"
, 844, __extension__ __PRETTY_FUNCTION__); }));
845 assert(model->compiled_data)((void) sizeof ((model->compiled_data) ? 1 : 0), __extension__
({ if (model->compiled_data) ; else __assert_fail ("model->compiled_data"
, "ccv_cnnp_model.c", 845, __extension__ __PRETTY_FUNCTION__)
; }));
846 ccv_cnnp_compiled_data_t* const compiled_data = model->compiled_data;
847 assert(compiled_data->rewindables)((void) sizeof ((compiled_data->rewindables) ? 1 : 0), __extension__
({ if (compiled_data->rewindables) ; else __assert_fail (
"compiled_data->rewindables", "ccv_cnnp_model.c", 847, __extension__
__PRETTY_FUNCTION__); }));
848 int i;
849 for (i = 0; i < compiled_data->rewindables->rnum; i++)
850 {
851 const ccv_cnnp_rewind_symbol_t* const rewind_symbol = (ccv_cnnp_rewind_symbol_t*)ccv_array_get(compiled_data->rewindables, i)((void*)(((char*)((compiled_data->rewindables)->data)) +
(size_t)(compiled_data->rewindables)->rsize * (size_t)
(i)));
852 if (rewind_symbol->type == CCV_CNNP_REWIND_GRAPH_EXEC)
853 ccv_nnc_graph_exec_symbol_free(model->graph, rewind_symbol->graph_exec);
854 else if (rewind_symbol->type == CCV_CNNP_REWIND_TENSOR)
855 ccv_nnc_tensor_symbol_free(model->graph, rewind_symbol->tensor);
856 }
857 ccv_array_clear(compiled_data->rewindables);
858 ccv_nnc_graph_exec_symbol_autogen(model->graph, 0, 0, CCV_NNC_AUTOGEN_SOURCES_AND_DESTINATIONS);
859}
860
861static void _ccv_cnnp_model_tensor_symbol_new_hook(void* context, const ccv_nnc_tensor_symbol_t symbol, const ccv_nnc_tensor_param_t info, const char* const name)
862{
863 const ccv_cnnp_rewind_symbol_t rewind_symbol = {
864 .type = CCV_CNNP_REWIND_TENSOR,
865 .tensor = symbol
866 };
867 ccv_array_t* const rewind_symbols = (ccv_array_t*)context;
868 ccv_array_push(rewind_symbols, &rewind_symbol);
869}
870
871static void _ccv_cnnp_model_tensor_symbol_alias_new_hook(void* context, const ccv_nnc_tensor_symbol_t symbol, const ccv_nnc_tensor_symbol_t from_symbol, const int ofs[CCV_NNC_MAX_DIM_ALLOC(12)], const int inc[CCV_NNC_MAX_DIM_ALLOC(12)], const ccv_nnc_tensor_param_t info, const char* const name)
872{
873 const ccv_cnnp_rewind_symbol_t rewind_symbol = {
874 .type = CCV_CNNP_REWIND_TENSOR,
875 .tensor = symbol
876 };
877 ccv_array_t* const rewind_symbols = (ccv_array_t*)context;
878 ccv_array_push(rewind_symbols, &rewind_symbol);
879}
880
881static void _ccv_cnnp_model_graph_exec_symbol_new_hook(void* context, const ccv_nnc_graph_exec_symbol_t symbol, const ccv_nnc_cmd_t cmd, const ccv_nnc_tensor_symbol_t* const inputs, const int input_size, const ccv_nnc_tensor_symbol_t* const outputs, const int output_size, const char* const name)
882{
883 const ccv_cnnp_rewind_symbol_t rewind_symbol = {
884 .type = CCV_CNNP_REWIND_GRAPH_EXEC,
885 .graph_exec = symbol
886 };
887 ccv_array_t* const rewind_symbols = (ccv_array_t*)context;
888 ccv_array_push(rewind_symbols, &rewind_symbol);
889}
890
891static void _ccv_cnnp_model_graph_symbol_exec_set_for_graph_exec_arena(const ccv_nnc_graph_exec_arena_t* const graph_exec_arena, const int parallel_count, const ccv_nnc_graph_exec_symbol_t exec_symbol, const ccv_nnc_cmd_t cmd, ccv_nnc_symbolic_graph_t* const symbolic_graph)
892{
893 ccv_nnc_graph_exec_t const update_exec = ccv_nnc_graph_exec_from_symbol(graph_exec_arena, exec_symbol);
894 if (!CCV_NO_GRAPH_EXEC(update_exec)((update_exec).graph == 0))
895 ccv_nnc_graph_exec_set(update_exec.graph, update_exec, cmd);
896 int i;
897 for (i = 1; i < parallel_count; i++)
898 {
899 ccv_nnc_graph_exec_symbol_t copy_symbol = ccv_nnc_graph_exec_symbol_copy(symbolic_graph, exec_symbol, i);
900 const ccv_nnc_graph_exec_t copy = ccv_nnc_graph_exec_from_symbol(graph_exec_arena, copy_symbol);
901 if (!CCV_NO_GRAPH_EXEC(copy)((copy).graph == 0))
902 ccv_nnc_graph_exec_set(copy.graph, copy, cmd);
903 }
904}
905
906static void _ccv_cnnp_model_graph_exec_symbol_set(ccv_nnc_symbolic_graph_t* const symbolic_graph, ccv_cnnp_compiled_data_t* const compiled_data, const int parallel_count, const ccv_nnc_graph_exec_symbol_t exec_symbol, const ccv_nnc_cmd_t cmd)
907{
908 assert(compiled_data)((void) sizeof ((compiled_data) ? 1 : 0), __extension__ ({ if
(compiled_data) ; else __assert_fail ("compiled_data", "ccv_cnnp_model.c"
, 908, __extension__ __PRETTY_FUNCTION__); }));
909 assert(symbolic_graph)((void) sizeof ((symbolic_graph) ? 1 : 0), __extension__ ({ if
(symbolic_graph) ; else __assert_fail ("symbolic_graph", "ccv_cnnp_model.c"
, 909, __extension__ __PRETTY_FUNCTION__); }));
910 ccv_nnc_graph_exec_symbol_set(symbolic_graph, exec_symbol, cmd);
911 int i;
912 for (i = 1; i < parallel_count; i++)
913 {
914 ccv_nnc_graph_exec_symbol_t copy_symbol = ccv_nnc_graph_exec_symbol_copy(symbolic_graph, exec_symbol, i);
915 if (copy_symbol.graph)
916 ccv_nnc_graph_exec_symbol_set(symbolic_graph, copy_symbol, cmd);
917 }
918 ccv_nnc_graph_exec_arena_t* const graph_exec_arena = compiled_data->graph_exec_arena;
919 if (graph_exec_arena)
920 _ccv_cnnp_model_graph_symbol_exec_set_for_graph_exec_arena(graph_exec_arena, parallel_count, exec_symbol, cmd, symbolic_graph);
921 // Skip backward graph exec arena because it is for a specific accum symbolic graph, not the main graph (model->graph)
922 ccv_nnc_graph_exec_arena_t* const gradient_graph_exec_arena = compiled_data->apply_gradients.graph_exec_arena;
923 if (gradient_graph_exec_arena)
924 _ccv_cnnp_model_graph_symbol_exec_set_for_graph_exec_arena(gradient_graph_exec_arena, parallel_count, exec_symbol, cmd, symbolic_graph);
925}
926
927static int _ccv_cnnp_set_minimizer_for_parameter(ccv_nnc_symbolic_graph_t* const graph, ccv_cnnp_compiled_data_t* const compiled_data, ccv_nnc_graph_exec_symbol_t* const update_nodes, ccv_nnc_tensor_symbol_t* const updated_parameters, ccv_nnc_tensor_symbol_map_t* const saved_aux, const int parallel_count, const ccv_nnc_cmd_t minimizer, const int saved_aux_size, const int max_saved_aux_size, const int parameter_indice)
928{
929 int this_parameter_flag = 0;
930 if (update_nodes[parameter_indice].d == CCV_NNC_NO_TENSOR_SYMBOL)
931 return this_parameter_flag;
932 const ccv_nnc_cmd_t old_minimizer = ccv_nnc_graph_exec_symbol_cmd(graph, update_nodes[parameter_indice]);
933 int j, k;
934 // For no-op, we can preserve previous saved_aux_size.
935 if (old_minimizer.cmd != minimizer.cmd && minimizer.cmd != CCV_NNC_NOOP)
936 {
937 // If the old minimizer is a noop, then the old_saved_aux_size should be whatever its previous
938 // saved_aux_size is, otherwise we will reinit the saved_aux repeatedly if you switch between
939 // noop and a minimizer. We don't want that because we do that in high-level frameworks to
940 // make sure some model parameters don't update if we don't want them to.
941 int old_saved_aux_size;
942 if (old_minimizer.cmd == CCV_NNC_NOOP)
943 {
944 int input_size;
945 ccv_nnc_graph_exec_symbol_io(graph, update_nodes[parameter_indice], 0, &input_size, 0, 0);
946 if (input_size < 2) // This is not legit.
947 old_saved_aux_size = ccv_nnc_minimizer_saved_aux_size(old_minimizer);
948 else // See ccv_nnc_minimizer_saved_aux_size, the saved_aux is inputs excluding gradients and parameters.
949 old_saved_aux_size = input_size - 2;
950 } else
951 old_saved_aux_size = ccv_nnc_minimizer_saved_aux_size(old_minimizer);
952 if (old_saved_aux_size != saved_aux_size)
953 {
954 this_parameter_flag = 1;
955 if (saved_aux_size > old_saved_aux_size)
956 {
957 // Allocate new tensor symbols.
958 const ccv_nnc_tensor_param_t info = ccv_nnc_tensor_symbol_params(graph, updated_parameters[parameter_indice]);
959 for (j = old_saved_aux_size; j < saved_aux_size; j++)
960 {
961 saved_aux[parameter_indice * max_saved_aux_size + j].source = ccv_nnc_tensor_symbol_new(graph, info, 0);
962 saved_aux[parameter_indice * max_saved_aux_size + j].destination = ccv_nnc_tensor_symbol_new(graph, info, 0);
963 const int device_id = CCV_TENSOR_GET_DEVICE_ID(info.type)(((info.type) & 0xfff00) >> 8);
964 for (k = 1; k < parallel_count; k++)
965 {
966 ccv_nnc_tensor_param_t dev_info = info;
967 if (k != device_id)
968 CCV_TENSOR_SET_DEVICE_ID(dev_info.type, k)(dev_info.type) = (((dev_info.type) & ~0xfff00) | (((k) &
0xfff) << 8));
969 else
970 CCV_TENSOR_SET_DEVICE_ID(dev_info.type, 0)(dev_info.type) = (((dev_info.type) & ~0xfff00) | (((0) &
0xfff) << 8));
971 const ccv_nnc_tensor_symbol_t src_copy = ccv_nnc_tensor_symbol_new(graph, dev_info, 0);
972 const ccv_nnc_tensor_symbol_t dest_copy = ccv_nnc_tensor_symbol_new(graph, dev_info, 0);
973 ccv_nnc_tensor_symbol_set_copy(graph, saved_aux[parameter_indice * max_saved_aux_size + j].source, k, src_copy);
974 ccv_nnc_tensor_symbol_set_copy(graph, saved_aux[parameter_indice * max_saved_aux_size + j].destination, k, dest_copy);
975 }
976 }
977 } else {
978 for (j = saved_aux_size; j < old_saved_aux_size; j++)
979 {
980 for (k = 1; k < parallel_count; k++)
981 {
982 const ccv_nnc_tensor_symbol_t src_copy = ccv_nnc_tensor_symbol_copy(graph, saved_aux[parameter_indice * max_saved_aux_size + j].source, k);
983 if (src_copy.d >= 0)
984 {
985 ccv_nnc_tensor_symbol_free(graph, src_copy);
986 ccv_nnc_tensor_symbol_set_copy(graph, saved_aux[parameter_indice * max_saved_aux_size + j].source, k, NO_TENSOR_SYMBOL(const ccv_nnc_tensor_symbol_t){.d = CCV_NNC_NO_TENSOR_SYMBOL
});
987 }
988 const ccv_nnc_tensor_symbol_t dest_copy = ccv_nnc_tensor_symbol_copy(graph, saved_aux[parameter_indice * max_saved_aux_size + j].destination, k);
989 if (dest_copy.d >= 0)
990 {
991 ccv_nnc_tensor_symbol_free(graph, dest_copy);
992 ccv_nnc_tensor_symbol_set_copy(graph, saved_aux[parameter_indice * max_saved_aux_size + j].destination, k, NO_TENSOR_SYMBOL(const ccv_nnc_tensor_symbol_t){.d = CCV_NNC_NO_TENSOR_SYMBOL
});
993 }
994 }
995 ccv_nnc_tensor_symbol_free(graph, saved_aux[parameter_indice * max_saved_aux_size + j].source);
996 ccv_nnc_tensor_symbol_free(graph, saved_aux[parameter_indice * max_saved_aux_size + j].destination);
997 saved_aux[parameter_indice * max_saved_aux_size + j].source = saved_aux[parameter_indice * max_saved_aux_size + j].destination = NO_TENSOR_SYMBOL(const ccv_nnc_tensor_symbol_t){.d = CCV_NNC_NO_TENSOR_SYMBOL
};
998 }
999 }
1000 }
1001 }
1002 _ccv_cnnp_model_graph_exec_symbol_set(graph, compiled_data, parallel_count, update_nodes[parameter_indice], minimizer);
1003 if (this_parameter_flag)
1004 {
1005 ccv_nnc_tensor_symbol_t update_inputs[saved_aux_size + 2];
1006 ccv_nnc_tensor_symbol_t update_outputs[saved_aux_size + 1];
1007 const int* inputs = 0;
1008 int input_size = 0;
1009 ccv_nnc_graph_exec_symbol_io(graph, update_nodes[parameter_indice], &inputs, &input_size, 0, 0);
1010 assert(input_size >= 1)((void) sizeof ((input_size >= 1) ? 1 : 0), __extension__ (
{ if (input_size >= 1) ; else __assert_fail ("input_size >= 1"
, "ccv_cnnp_model.c", 1010, __extension__ __PRETTY_FUNCTION__
); }));
1011 update_inputs[0].d = inputs[0];
1012 update_inputs[0].graph = graph;
1013 update_inputs[1].d = inputs[1];
1014 update_inputs[1].graph = graph;
1015 update_outputs[0] = updated_parameters[parameter_indice];
1016 for (j = 0; j < saved_aux_size; j++)
1017 {
1018 update_inputs[j + 2] = saved_aux[parameter_indice * max_saved_aux_size + j].source;
1019 update_outputs[j + 1] = saved_aux[parameter_indice * max_saved_aux_size + j].destination;
1020 }
1021 ccv_nnc_graph_exec_symbol_set_io(graph, update_nodes[parameter_indice], update_inputs, saved_aux_size + 2, update_outputs, saved_aux_size + 1);
1022 for (k = 1; k < parallel_count; k++)
1023 {
1024 const ccv_nnc_graph_exec_symbol_t copy = ccv_nnc_graph_exec_symbol_copy(graph, update_nodes[parameter_indice], k);
1025 assert(copy.d >= 0)((void) sizeof ((copy.d >= 0) ? 1 : 0), __extension__ ({ if
(copy.d >= 0) ; else __assert_fail ("copy.d >= 0", "ccv_cnnp_model.c"
, 1025, __extension__ __PRETTY_FUNCTION__); }));
1026 ccv_nnc_graph_exec_symbol_io(graph, copy, &inputs, &input_size, 0, 0);
1027 assert(input_size >= 1)((void) sizeof ((input_size >= 1) ? 1 : 0), __extension__ (
{ if (input_size >= 1) ; else __assert_fail ("input_size >= 1"
, "ccv_cnnp_model.c", 1027, __extension__ __PRETTY_FUNCTION__
); }));
1028 update_inputs[0].d = inputs[0];
1029 update_inputs[0].graph = graph;
1030 update_inputs[1].d = inputs[1];
1031 update_inputs[1].graph = graph;
1032 update_outputs[0] = ccv_nnc_tensor_symbol_copy(graph, updated_parameters[parameter_indice], k);
1033 for (j = 0; j < saved_aux_size; j++)
1034 {
1035 update_inputs[j + 2] = ccv_nnc_tensor_symbol_copy(graph, saved_aux[parameter_indice * max_saved_aux_size + j].source, k);
1036 update_outputs[j + 1] = ccv_nnc_tensor_symbol_copy(graph, saved_aux[parameter_indice * max_saved_aux_size + j].destination, k);
1037 }
1038 ccv_nnc_graph_exec_symbol_set_io(graph, copy, update_inputs, saved_aux_size + 2, update_outputs, saved_aux_size + 1);
1039 }
1040 }
1041 return this_parameter_flag;
1042}
1043
1044typedef struct {
1045 int parameter_size;
1046 ccv_nnc_cmd_t minimizer;
1047 ccv_cnnp_model_io_t parameters[1];
1048} ccv_cnnp_set_minimizer_for_parameter_t;
1049
1050static int _ccv_cnnp_apply_parameters_with_minimizer(ccv_cnnp_model_t* const model)
1051{
1052 ccv_cnnp_compiled_data_t* const compiled_data = model->compiled_data;
1053 assert(compiled_data)((void) sizeof ((compiled_data) ? 1 : 0), __extension__ ({ if
(compiled_data) ; else __assert_fail ("compiled_data", "ccv_cnnp_model.c"
, 1053, __extension__ __PRETTY_FUNCTION__); }));
1054 const int max_saved_aux_size = compiled_data->minimize.max_saved_aux_size;
1055 // We update all parameters, at this point, we have one minimizer.
1056 const int parameter_size = compiled_data->parameters->rnum;
1057 ccv_nnc_graph_exec_symbol_t* const update_nodes = compiled_data->update_nodes;
1058 ccv_nnc_symbolic_graph_t* const symbolic_graph = model->graph;
1059 assert(symbolic_graph)((void) sizeof ((symbolic_graph) ? 1 : 0), __extension__ ({ if
(symbolic_graph) ; else __assert_fail ("symbolic_graph", "ccv_cnnp_model.c"
, 1059, __extension__ __PRETTY_FUNCTION__); }));
1060 const int parallel_count = _ccv_cnnp_model_root_parallel_count(model);
1061 assert(_ccv_cnnp_model_effective_parallel_count(model) == parallel_count && "local replicated stateful models only support forward / no-grad evaluation for now")((void) sizeof ((_ccv_cnnp_model_effective_parallel_count(model
) == parallel_count && "local replicated stateful models only support forward / no-grad evaluation for now"
) ? 1 : 0), __extension__ ({ if (_ccv_cnnp_model_effective_parallel_count
(model) == parallel_count && "local replicated stateful models only support forward / no-grad evaluation for now"
) ; else __assert_fail ("_ccv_cnnp_model_effective_parallel_count(model) == parallel_count && \"local replicated stateful models only support forward / no-grad evaluation for now\""
, "ccv_cnnp_model.c", 1061, __extension__ __PRETTY_FUNCTION__
); }));
1062 ccv_array_t* const parameters = compiled_data->minimize.parameters;
1063 ccv_array_t* const parameter_indices = ccv_array_new(sizeof(int), 0, 0);
1064 int i, j, flag = 0;
1065 for (i = 0; i < parameters->rnum; i++)
1066 {
1067 ccv_cnnp_set_minimizer_for_parameter_t* const set_minimizer_for_parameter = *(ccv_cnnp_set_minimizer_for_parameter_t**)ccv_array_get(parameters, i)((void*)(((char*)((parameters)->data)) + (size_t)(parameters
)->rsize * (size_t)(i)));
1068 for (j = 0; j < set_minimizer_for_parameter->parameter_size; j++)
1069 {
1070 const int param_sel = set_minimizer_for_parameter->parameters[j]->param_sel > 0 ? set_minimizer_for_parameter->parameters[j]->param_sel - 1 : set_minimizer_for_parameter->parameters[j]->param_sel;
1071 assert(set_minimizer_for_parameter->parameters[j]->param_sel != 0)((void) sizeof ((set_minimizer_for_parameter->parameters[j
]->param_sel != 0) ? 1 : 0), __extension__ ({ if (set_minimizer_for_parameter
->parameters[j]->param_sel != 0) ; else __assert_fail (
"set_minimizer_for_parameter->parameters[j]->param_sel != 0"
, "ccv_cnnp_model.c", 1071, __extension__ __PRETTY_FUNCTION__
); }));
1072 const int old_rnum = parameter_indices->rnum;
1073 ccv_cnnp_model_add_to_parameter_indices(set_minimizer_for_parameter->parameters[j]->model, param_sel, parameter_indices);
1074 const int param_ref = set_minimizer_for_parameter->parameters[j]->param_ref > 0 ? set_minimizer_for_parameter->parameters[j]->param_ref - 1 : set_minimizer_for_parameter->parameters[j]->param_ref;
1075 assert(set_minimizer_for_parameter->parameters[j]->param_ref != 0)((void) sizeof ((set_minimizer_for_parameter->parameters[j
]->param_ref != 0) ? 1 : 0), __extension__ ({ if (set_minimizer_for_parameter
->parameters[j]->param_ref != 0) ; else __assert_fail (
"set_minimizer_for_parameter->parameters[j]->param_ref != 0"
, "ccv_cnnp_model.c", 1075, __extension__ __PRETTY_FUNCTION__
); }));
1076 if (param_ref >= 0)
1077 {
1078 assert(param_ref + old_rnum < parameter_indices->rnum)((void) sizeof ((param_ref + old_rnum < parameter_indices->
rnum) ? 1 : 0), __extension__ ({ if (param_ref + old_rnum <
parameter_indices->rnum) ; else __assert_fail ("param_ref + old_rnum < parameter_indices->rnum"
, "ccv_cnnp_model.c", 1078, __extension__ __PRETTY_FUNCTION__
); }));
1079 *(int*)ccv_array_get(parameter_indices, old_rnum)((void*)(((char*)((parameter_indices)->data)) + (size_t)(parameter_indices
)->rsize * (size_t)(old_rnum))) = *(int*)ccv_array_get(parameter_indices, param_ref + old_rnum)((void*)(((char*)((parameter_indices)->data)) + (size_t)(parameter_indices
)->rsize * (size_t)(param_ref + old_rnum)));
1080 parameter_indices->rnum = old_rnum + 1;
1081 }
1082 }
1083 const int saved_aux_size = ccv_nnc_minimizer_saved_aux_size(set_minimizer_for_parameter->minimizer);
1084 // We may have duplicated indices, but that is OK, we will set it twice.
1085 for (j = 0; j < parameter_indices->rnum; j++)
1086 {
1087 const int d = *(int*)ccv_array_get(parameter_indices, j)((void*)(((char*)((parameter_indices)->data)) + (size_t)(parameter_indices
)->rsize * (size_t)(j)));
1088 assert(d <= parameter_size)((void) sizeof ((d <= parameter_size) ? 1 : 0), __extension__
({ if (d <= parameter_size) ; else __assert_fail ("d <= parameter_size"
, "ccv_cnnp_model.c", 1088, __extension__ __PRETTY_FUNCTION__
); }));
1089 if (_ccv_cnnp_set_minimizer_for_parameter(symbolic_graph, compiled_data, update_nodes, compiled_data->updated_parameters, compiled_data->saved_aux, parallel_count, set_minimizer_for_parameter->minimizer, saved_aux_size, max_saved_aux_size, d))
1090 flag = 1;
1091 }
1092 ccv_array_clear(parameter_indices);
1093 }
1094 ccv_array_free(parameter_indices);
1095 return flag;
1096}
1097
1098static void _ccv_cnnp_scatter_saved_aux(ccv_nnc_tensor_symbol_map_t* const saved_aux, const int parameter_size, const int old_saved_aux_size, const int new_saved_aux_size)
1099{
1100 if (new_saved_aux_size == old_saved_aux_size)
1101 return;
1102 assert(new_saved_aux_size > old_saved_aux_size)((void) sizeof ((new_saved_aux_size > old_saved_aux_size) ?
1 : 0), __extension__ ({ if (new_saved_aux_size > old_saved_aux_size
) ; else __assert_fail ("new_saved_aux_size > old_saved_aux_size"
, "ccv_cnnp_model.c", 1102, __extension__ __PRETTY_FUNCTION__
); }));
1103 int i, j;
1104 for (i = parameter_size - 1; i >= 0; i--)
1105 {
1106 for (j = new_saved_aux_size - 1; j >= old_saved_aux_size; j--)
1107 saved_aux[i * new_saved_aux_size + j].source = saved_aux[i * new_saved_aux_size + j].destination = NO_TENSOR_SYMBOL(const ccv_nnc_tensor_symbol_t){.d = CCV_NNC_NO_TENSOR_SYMBOL
};
1108 for (j = old_saved_aux_size - 1; j >= 0; j--)
1109 saved_aux[i * new_saved_aux_size + j] = saved_aux[i * old_saved_aux_size + j];
1110 }
1111}
1112
1113static void _ccv_cnnp_model_set_rewindables(ccv_cnnp_model_t* const model)
1114{
1115 ccv_cnnp_compiled_data_t* const compiled_data = model->compiled_data;
1116 assert(compiled_data)((void) sizeof ((compiled_data) ? 1 : 0), __extension__ ({ if
(compiled_data) ; else __assert_fail ("compiled_data", "ccv_cnnp_model.c"
, 1116, __extension__ __PRETTY_FUNCTION__); }));
1117 if (!compiled_data->rewindables)
1118 compiled_data->rewindables = ccv_array_new(sizeof(ccv_cnnp_rewind_symbol_t), 0, 0);
1119 ccv_nnc_tensor_symbol_new_hook(model->graph, _ccv_cnnp_model_tensor_symbol_new_hook, compiled_data->rewindables, 0);
1120 ccv_nnc_tensor_symbol_alias_new_hook(model->graph, _ccv_cnnp_model_tensor_symbol_alias_new_hook, compiled_data->rewindables, 0);
1121 ccv_nnc_graph_exec_symbol_new_hook(model->graph, _ccv_cnnp_model_graph_exec_symbol_new_hook, compiled_data->rewindables, 0);
1122}
1123
1124static void _ccv_cnnp_model_gradient_init(ccv_cnnp_model_t* const model, const int gradient_mode, const uint64_t disable_outgrad, ccv_nnc_tensor_t* const* const fits, const int fit_size)
1125{
1126 ccv_cnnp_compiled_data_t* const compiled_data = model->compiled_data;
1127 assert(compiled_data->gradient_mode == CCV_CNNP_COMPILED_DATA_GRADIENT_NONE)((void) sizeof ((compiled_data->gradient_mode == CCV_CNNP_COMPILED_DATA_GRADIENT_NONE
) ? 1 : 0), __extension__ ({ if (compiled_data->gradient_mode
== CCV_CNNP_COMPILED_DATA_GRADIENT_NONE) ; else __assert_fail
("compiled_data->gradient_mode == CCV_CNNP_COMPILED_DATA_GRADIENT_NONE"
, "ccv_cnnp_model.c", 1127, __extension__ __PRETTY_FUNCTION__
); }));
1128 assert(gradient_mode != CCV_CNNP_COMPILED_DATA_GRADIENT_NONE)((void) sizeof ((gradient_mode != CCV_CNNP_COMPILED_DATA_GRADIENT_NONE
) ? 1 : 0), __extension__ ({ if (gradient_mode != CCV_CNNP_COMPILED_DATA_GRADIENT_NONE
) ; else __assert_fail ("gradient_mode != CCV_CNNP_COMPILED_DATA_GRADIENT_NONE"
, "ccv_cnnp_model.c", 1128, __extension__ __PRETTY_FUNCTION__
); }));
1129 const int evaluate_to_size = compiled_data->evaluate.to_size;
1130 assert(evaluate_to_size > 0)((void) sizeof ((evaluate_to_size > 0) ? 1 : 0), __extension__
({ if (evaluate_to_size > 0) ; else __assert_fail ("evaluate_to_size > 0"
, "ccv_cnnp_model.c", 1130, __extension__ __PRETTY_FUNCTION__
); }));
1131 const int parallel_count = _ccv_cnnp_model_root_parallel_count(model);
1132 assert(_ccv_cnnp_model_effective_parallel_count(model) == parallel_count && "local replicated stateful models only support forward / no-grad evaluation for now")((void) sizeof ((_ccv_cnnp_model_effective_parallel_count(model
) == parallel_count && "local replicated stateful models only support forward / no-grad evaluation for now"
) ? 1 : 0), __extension__ ({ if (_ccv_cnnp_model_effective_parallel_count
(model) == parallel_count && "local replicated stateful models only support forward / no-grad evaluation for now"
) ; else __assert_fail ("_ccv_cnnp_model_effective_parallel_count(model) == parallel_count && \"local replicated stateful models only support forward / no-grad evaluation for now\""
, "ccv_cnnp_model.c", 1132, __extension__ __PRETTY_FUNCTION__
); }));
1133 compiled_data->evaluate.tos = ccreallocrealloc(compiled_data->evaluate.tos, sizeof(ccv_nnc_graph_exec_symbol_t) * evaluate_to_size * parallel_count + sizeof(ccv_nnc_graph_exec_t) * evaluate_to_size * parallel_count);
1134 compiled_data->evaluate.to_ops = (ccv_nnc_graph_exec_t*)(compiled_data->evaluate.tos + evaluate_to_size * parallel_count);
1135 int i, j;
1136 const int output_size = model->output_size;
1137 assert(!fits || fit_size == output_size * parallel_count)((void) sizeof ((!fits || fit_size == output_size * parallel_count
) ? 1 : 0), __extension__ ({ if (!fits || fit_size == output_size
* parallel_count) ; else __assert_fail ("!fits || fit_size == output_size * parallel_count"
, "ccv_cnnp_model.c", 1137, __extension__ __PRETTY_FUNCTION__
); }));
1138 if (fits)
1139 for (i = 0; i < output_size; i++)
1140 ccv_nnc_tensor_symbol_set(model->graph, compiled_data->fits[i], fits[i]->info);
1141 const int max_saved_aux_size = compiled_data->minimize.max_saved_aux_size;
1142 const int parameter_size = compiled_data->parameters->rnum;
1143 compiled_data->updated_parameters = (ccv_nnc_tensor_symbol_t*)ccmallocmalloc(sizeof(ccv_nnc_tensor_symbol_t) * parameter_size + sizeof(ccv_nnc_graph_exec_symbol_t) * parameter_size + sizeof(ccv_nnc_tensor_symbol_map_t) * max_saved_aux_size * parameter_size);
1144 compiled_data->update_nodes = (ccv_nnc_graph_exec_symbol_t*)(compiled_data->updated_parameters + parameter_size);
1145 compiled_data->saved_aux = (ccv_nnc_tensor_symbol_map_t*)(compiled_data->update_nodes + parameter_size);
1146 int parameter_size_maybe_more = parameter_size;
1147 compiled_data->disable_outgrad = disable_outgrad;
1148 int outgrad_size;
1149 if (gradient_mode == CCV_CNNP_COMPILED_DATA_GRADIENT_TRAINABLES || model->input_size == 0)
1150 outgrad_size = 0;
1151 else if (disable_outgrad == CCV_CNNP_DISABLE_OUTGRAD_NONE) // Compute minimize with gradients including inputs.
1152 outgrad_size = model->input_size;
1153 else {
1154 assert(disable_outgrad != CCV_CNNP_DISABLE_OUTGRAD_ALL)((void) sizeof ((disable_outgrad != CCV_CNNP_DISABLE_OUTGRAD_ALL
) ? 1 : 0), __extension__ ({ if (disable_outgrad != CCV_CNNP_DISABLE_OUTGRAD_ALL
) ; else __assert_fail ("disable_outgrad != CCV_CNNP_DISABLE_OUTGRAD_ALL"
, "ccv_cnnp_model.c", 1154, __extension__ __PRETTY_FUNCTION__
); })); // If it is disable all, gradient mode won't be this.
1155 outgrad_size = 0;
1156 for (i = 0; i < model->input_size; i++)
1157 if (!(disable_outgrad & ((uint64_t)1 << i)))
1158 ++outgrad_size;
1159 }
1160 compiled_data->outgrad_size = outgrad_size;
1161 parameter_size_maybe_more += outgrad_size;
1162 compiled_data->gradients = (ccv_nnc_tensor_symbol_t*)ccmallocmalloc(sizeof(ccv_nnc_tensor_symbol_t) * parameter_size_maybe_more + sizeof(ccv_nnc_graph_exec_symbol_t) * parameter_size_maybe_more * parallel_count);
1163 compiled_data->outgrads = parameter_size_maybe_more > parameter_size ? compiled_data->gradients + parameter_size : 0;
1164 compiled_data->backward.tos = (ccv_nnc_graph_exec_symbol_t*)(compiled_data->gradients + parameter_size_maybe_more);
1165 compiled_data->backward.to_size = parameter_size_maybe_more;
1166 ccv_nnc_tensor_symbol_t* parameters = (ccv_nnc_tensor_symbol_t*)ccv_array_get(compiled_data->parameters, 0)((void*)(((char*)((compiled_data->parameters)->data)) +
(size_t)(compiled_data->parameters)->rsize * (size_t)(
0)));
1167 if (compiled_data->parameter_flags)
1168 {
1169 parameters = (ccv_nnc_tensor_symbol_t*)ccmallocmalloc(sizeof(ccv_nnc_tensor_symbol_t) * parameter_size);
1170 for (i = 0; i < parameter_size; i++)
1171 if (compiled_data->parameter_flags[i >> 6] & ((uint64_t)1 << (i & 63)))
1172 parameters[i] = *(ccv_nnc_tensor_symbol_t*)ccv_array_get(compiled_data->parameters, i)((void*)(((char*)((compiled_data->parameters)->data)) +
(size_t)(compiled_data->parameters)->rsize * (size_t)(
i)));
1173 else
1174 parameters[i] = NO_TENSOR_SYMBOL(const ccv_nnc_tensor_symbol_t){.d = CCV_NNC_NO_TENSOR_SYMBOL
};
1175 }
1176 if (gradient_mode == CCV_CNNP_COMPILED_DATA_GRADIENT_TRAINABLES || model->input_size == 0)
1177 ccv_nnc_symbolic_graph_minimize(model->graph, compiled_data->minimize.minimizer, compiled_data->f, output_size, parameters, parameter_size, 0, 0, SYMBOLIC_GRAPH_SOURCES(model->graph)ccv_nnc_symbolic_graph_sources(model->graph), ccv_nnc_symbolic_graph_source_size
(model->graph), SYMBOLIC_GRAPH_DESTINATIONS(model->graph)ccv_nnc_symbolic_graph_destinations(model->graph), ccv_nnc_symbolic_graph_destination_size
(model->graph), compiled_data->gradients, compiled_data->updated_parameters, compiled_data->saved_aux, compiled_data->update_nodes);
1178 else if (disable_outgrad == CCV_CNNP_DISABLE_OUTGRAD_NONE) // Compute minimize with gradients including inputs.
1179 ccv_nnc_symbolic_graph_minimize(model->graph, compiled_data->minimize.minimizer, compiled_data->f, output_size, parameters, parameter_size, model->inputs, model->input_size, SYMBOLIC_GRAPH_SOURCES(model->graph)ccv_nnc_symbolic_graph_sources(model->graph), ccv_nnc_symbolic_graph_source_size
(model->graph), SYMBOLIC_GRAPH_DESTINATIONS(model->graph)ccv_nnc_symbolic_graph_destinations(model->graph), ccv_nnc_symbolic_graph_destination_size
(model->graph), compiled_data->gradients, compiled_data->updated_parameters, compiled_data->saved_aux, compiled_data->update_nodes);
1180 else { // Compute minimize with gradients including selected inputs.
1181 assert(model->input_size > 0)((void) sizeof ((model->input_size > 0) ? 1 : 0), __extension__
({ if (model->input_size > 0) ; else __assert_fail ("model->input_size > 0"
, "ccv_cnnp_model.c", 1181, __extension__ __PRETTY_FUNCTION__
); }));
1182 assert(disable_outgrad != CCV_CNNP_DISABLE_OUTGRAD_ALL)((void) sizeof ((disable_outgrad != CCV_CNNP_DISABLE_OUTGRAD_ALL
) ? 1 : 0), __extension__ ({ if (disable_outgrad != CCV_CNNP_DISABLE_OUTGRAD_ALL
) ; else __assert_fail ("disable_outgrad != CCV_CNNP_DISABLE_OUTGRAD_ALL"
, "ccv_cnnp_model.c", 1182, __extension__ __PRETTY_FUNCTION__
); })); // If it is disable all, gradient mode won't be this.
1183 assert(outgrad_size > 0)((void) sizeof ((outgrad_size > 0) ? 1 : 0), __extension__
({ if (outgrad_size > 0) ; else __assert_fail ("outgrad_size > 0"
, "ccv_cnnp_model.c", 1183, __extension__ __PRETTY_FUNCTION__
); }));
1184 ccv_nnc_tensor_symbol_t outgrads[outgrad_size];
1185 j = 0;
1186 for (i = 0; i < model->input_size; i++)
1187 if (!(disable_outgrad & ((uint64_t)1 << i)))
1188 outgrads[j++] = model->inputs[i];
1189 ccv_nnc_symbolic_graph_minimize(model->graph, compiled_data->minimize.minimizer, compiled_data->f, output_size, parameters, parameter_size, outgrads, outgrad_size, SYMBOLIC_GRAPH_SOURCES(model->graph)ccv_nnc_symbolic_graph_sources(model->graph), ccv_nnc_symbolic_graph_source_size
(model->graph), SYMBOLIC_GRAPH_DESTINATIONS(model->graph)ccv_nnc_symbolic_graph_destinations(model->graph), ccv_nnc_symbolic_graph_destination_size
(model->graph), compiled_data->gradients, compiled_data->updated_parameters, compiled_data->saved_aux, compiled_data->update_nodes);
1190 }
1191 if (compiled_data->parameter_flags)
1192 ccfreefree(parameters);
1193 _ccv_cnnp_scatter_saved_aux(compiled_data->saved_aux, parameter_size, ccv_nnc_minimizer_saved_aux_size(compiled_data->minimize.minimizer), compiled_data->minimize.max_saved_aux_size);
1194 if (compiled_data->minimize.parameters)
1195 _ccv_cnnp_apply_parameters_with_minimizer(model);
1196 // Go through gradient checkpoints to generate tensor inputs for backward pass just before executing the backward pass.
1197 ccv_cnnp_model_apply_gradient_checkpoints(compiled_data, model->graph);
1198 for (i = 0; i < output_size; i++)
1199 {
1200 const ccv_nnc_tensor_symbol_t df = ccv_nnc_tensor_symbol_for_backward(model->graph, compiled_data->f[i]);
1201 // Init this to 1 so we can backprop.
1202 ccv_nnc_tensor_symbol_set_flags(model->graph, df, CCV_NNC_TENSOR_SYMBOL_INIT_ONES);
1203 }
1204 compiled_data->backward.to_size = 0;
1205 for (i = 0; i < parameter_size_maybe_more; i++)
1206 if (compiled_data->gradients[i].d != CCV_NNC_NO_TENSOR_SYMBOL)
1207 compiled_data->backward.tos[compiled_data->backward.to_size++] = ccv_nnc_graph_exec_symbol_for_backward(model->graph, compiled_data->gradients[i]);
1208 ccv_nnc_graph_exec_symbol_autogen(model->graph, 0, 0, CCV_NNC_AUTOGEN_ALL_EXECS);
1209 ccv_nnc_symbolic_graph_set_destinations(model->graph, compiled_data->update_nodes, parameter_size);
1210 for (i = 0; i < parameter_size_maybe_more - parameter_size; i++)
1211 {
1212 if (compiled_data->outgrads[i].d < 0) // When we go through input, we might find zero-length inputs, and for these, we cannot have any outgrads.
1213 continue;
1214 const ccv_nnc_graph_exec_symbol_t outgrad = ccv_nnc_graph_exec_symbol_for_backward(model->graph, compiled_data->outgrads[i]);
1215 const int* tos;
1216 int to_size;
1217 ccv_nnc_graph_exec_symbol_to(model->graph, outgrad, &tos, &to_size);
1218 if (to_size == 0) // If this is the end (no minimizers afterwards). We need to attach this as a destination. Otherwise this is covered in update_nodes.
1219 {
1220 const ccv_nnc_graph_exec_symbol_t* destinations = ccv_nnc_symbolic_graph_destinations(model->graph);
1221 const int destination_count = ccv_nnc_symbolic_graph_destination_size(model->graph);
1222 int flag = 0;
1223 const int outgrad_destination_start = ccv_max(0, destination_count - i)({ typeof (0) _a = (0); typeof (destination_count - i) _b = (
destination_count - i); (_a > _b) ? _a : _b; });
1224 for (j = i - 1; !flag && j >= 0; j--)
1225 if (j + outgrad_destination_start < destination_count)
1226 flag = (destinations[j + outgrad_destination_start].d == outgrad.d);
1227 if (!flag) // Only if we cannot find it, we add it.
1228 ccv_nnc_symbolic_graph_add_destination(model->graph, outgrad);
1229 }
1230 }
1231 if (parallel_count > 1)
1232 {
1233 ccv_nnc_symbolic_graph_data_parallel(model->graph, parallel_count,
1234 0, 0,
1235 compiled_data->gradients, parameter_size /* No need to deal with outgrads, we don't allreduce outgrads */,
1236 compiled_data->gradients /* We only care about gradients before allreduce, thus, update our current pointers */,
1237 0, 0, 0,
1238 CCV_NNC_PARALLEL_REDUCE_OP_SUM,
1239 SYMBOLIC_GRAPH_SOURCES(model->graph)ccv_nnc_symbolic_graph_sources(model->graph), ccv_nnc_symbolic_graph_source_size
(model->graph), SYMBOLIC_GRAPH_DESTINATIONS(model->graph)ccv_nnc_symbolic_graph_destinations(model->graph), ccv_nnc_symbolic_graph_destination_size
(model->graph));
1240 ccv_nnc_graph_exec_symbol_autogen(model->graph, 0, 0, CCV_NNC_AUTOGEN_SOURCES_AND_DESTINATIONS);
1241 for (i = 0; i < evaluate_to_size; i++)
1242 for (j = 1; j < parallel_count; j++)
1243 {
1244 const ccv_nnc_graph_exec_symbol_t copy = ccv_nnc_graph_exec_symbol_copy(model->graph, compiled_data->evaluate.tos[i], j);
1245 if (copy.d != CCV_NNC_NO_GRAPH_EXEC_SYMBOL)
1246 compiled_data->evaluate.tos[compiled_data->evaluate.to_size++] = copy;
1247 }
1248 const int backward_to_size = compiled_data->backward.to_size;
1249 for (i = 0; i < backward_to_size; i++)
1250 for (j = 1; j < parallel_count; j++)
1251 {
1252 const ccv_nnc_graph_exec_symbol_t copy = ccv_nnc_graph_exec_symbol_copy(model->graph, compiled_data->backward.tos[i], j);
1253 if (copy.d != CCV_NNC_NO_GRAPH_EXEC_SYMBOL)
1254 compiled_data->backward.tos[compiled_data->backward.to_size++] = copy;
1255 }
1256 }
1257 // Only use memory compression if we are in gradient parameter mode.
1258 if (gradient_mode == CCV_CNNP_COMPILED_DATA_GRADIENT_TRAINABLES || gradient_mode == CCV_CNNP_COMPILED_DATA_GRADIENT_TRAINABLES_AND_INPUTS)
1259 {
1260 if (model->memory_compression)
1261 ccv_nnc_symbolic_graph_memory_compression(model->graph, SYMBOLIC_GRAPH_SOURCES(model->graph)ccv_nnc_symbolic_graph_sources(model->graph), ccv_nnc_symbolic_graph_source_size
(model->graph), SYMBOLIC_GRAPH_DESTINATIONS(model->graph)ccv_nnc_symbolic_graph_destinations(model->graph), ccv_nnc_symbolic_graph_destination_size
(model->graph));
1262 if (model->memory_reduction)
1263 ccv_nnc_symbolic_graph_memory_reduction(model->graph, SYMBOLIC_GRAPH_SOURCES(model->graph)ccv_nnc_symbolic_graph_sources(model->graph), ccv_nnc_symbolic_graph_source_size
(model->graph), SYMBOLIC_GRAPH_DESTINATIONS(model->graph)ccv_nnc_symbolic_graph_destinations(model->graph), ccv_nnc_symbolic_graph_destination_size
(model->graph));
1264 }
1265 compiled_data->backward.to_size = _ccv_nnc_array_dedup_graph_exec_symbols(compiled_data->backward.tos, compiled_data->backward.to_size);
1266 compiled_data->gradient_mode = gradient_mode;
1267}
1268
1269void ccv_cnnp_model_tensors_init_0(const ccv_cnnp_model_t* const model, ccv_cnnp_compiled_data_t* const compiled_data)
1270{
1271 assert(!compiled_data->tensors.parameters)((void) sizeof ((!compiled_data->tensors.parameters) ? 1 :
0), __extension__ ({ if (!compiled_data->tensors.parameters
) ; else __assert_fail ("!compiled_data->tensors.parameters"
, "ccv_cnnp_model.c", 1271, __extension__ __PRETTY_FUNCTION__
); }));
1272 const int parameter_size = compiled_data->parameters->rnum;
1273 const int parallel_count = _ccv_cnnp_model_effective_parallel_count(model);
1274 compiled_data->parallel_count = parallel_count;
1275 const int internal_size = compiled_data->internals->rnum;
1276 compiled_data->tensors_init.size = ccv_nnc_tensor_symbol_count(model->graph);
1277 compiled_data->tensors_init.v = cccalloccalloc(((compiled_data->tensors_init.size + 31) >> 5), sizeof(uint32_t));
1278 compiled_data->tensors.parameters = (ccv_nnc_tensor_t**)cccalloccalloc((parameter_size + internal_size) * parallel_count, sizeof(ccv_nnc_tensor_t*));
1279 compiled_data->tensors.internals = compiled_data->tensors.parameters + parameter_size * parallel_count;
1280}
1281
1282int ccv_cnnp_model_tensors_any_to_alloc(const ccv_cnnp_model_t* const model, ccv_cnnp_compiled_data_t* const compiled_data)
1283{
1284 int i, j;
1285 const int parameter_size = compiled_data->parameters->rnum;
1286 const int parallel_count = _ccv_cnnp_compiled_data_parallel_count(model, compiled_data);
1287 const int internal_size = compiled_data->internals->rnum;
1288 for (i = 0; i < parameter_size; i++)
1289 {
1290 // parameters has to be allocated all together.
1291 if (compiled_data->tensors.parameters[i])
1292 {
1293 for (j = 1; j < parallel_count; j++)
1294 { assert(compiled_data->tensors.parameters[i + j * parameter_size])((void) sizeof ((compiled_data->tensors.parameters[i + j *
parameter_size]) ? 1 : 0), __extension__ ({ if (compiled_data
->tensors.parameters[i + j * parameter_size]) ; else __assert_fail
("compiled_data->tensors.parameters[i + j * parameter_size]"
, "ccv_cnnp_model.c", 1294, __extension__ __PRETTY_FUNCTION__
); })); }
1295 continue;
1296 }
1297 return 1;
1298 }
1299 for (i = 0; i < internal_size; i++)
1300 {
1301 if (!compiled_data->tensors.internals[i])
1302 return 1;
1303 for (j = 1; j < parallel_count; j++)
1304 if (!compiled_data->tensors.internals[i + j * internal_size])
1305 return 1;
1306 }
1307 return 0;
1308}
1309
1310void ccv_cnnp_model_tensors_init_1(const ccv_cnnp_model_t* const model, ccv_cnnp_compiled_data_t* const compiled_data)
1311{
1312 int i, j;
1313 const int parameter_size = compiled_data->parameters->rnum;
1314 const int parallel_count = _ccv_cnnp_compiled_data_parallel_count(model, compiled_data);
1315 compiled_data->parallel_count = parallel_count;
1316 const int internal_size = compiled_data->internals->rnum;
1317 for (i = 0; i < parameter_size; i++)
1318 {
1319 // parameters has to be allocated all together.
1320 if (compiled_data->tensors.parameters[i])
1321 {
1322 for (j = 1; j < parallel_count; j++)
1323 { assert(compiled_data->tensors.parameters[i + j * parameter_size])((void) sizeof ((compiled_data->tensors.parameters[i + j *
parameter_size]) ? 1 : 0), __extension__ ({ if (compiled_data
->tensors.parameters[i + j * parameter_size]) ; else __assert_fail
("compiled_data->tensors.parameters[i + j * parameter_size]"
, "ccv_cnnp_model.c", 1323, __extension__ __PRETTY_FUNCTION__
); })); }
1324 continue;
1325 }
1326 const ccv_nnc_tensor_symbol_t parameter = *(ccv_nnc_tensor_symbol_t*)ccv_array_get(compiled_data->parameters, i)((void*)(((char*)((compiled_data->parameters)->data)) +
(size_t)(compiled_data->parameters)->rsize * (size_t)(
i)));
1327 ccv_nnc_tensor_param_t info = ccv_nnc_tensor_symbol_params(parameter.graph, parameter);
1328 if (CCV_TENSOR_GET_DEVICE(info.type)((info.type) & 0xfff00) == CCV_COMPUTE_DEVICE_ANY)
1329 CCV_TENSOR_SET_DEVICE_ID(info.type, 0)(info.type) = (((info.type) & ~0xfff00) | (((0) & 0xfff
) << 8));
1330 const int device_id = CCV_TENSOR_GET_DEVICE_ID(info.type)(((info.type) & 0xfff00) >> 8);
1331 compiled_data->tensors.parameters[i] = ccv_nnc_tensor_new(0, info, 0);
1332 for (j = 1; j < parallel_count; j++)
1333 {
1334 if (j != device_id)
1335 CCV_TENSOR_SET_DEVICE_ID(info.type, j)(info.type) = (((info.type) & ~0xfff00) | (((j) & 0xfff
) << 8));
1336 else
1337 CCV_TENSOR_SET_DEVICE_ID(info.type, 0)(info.type) = (((info.type) & ~0xfff00) | (((0) & 0xfff
) << 8));
1338 compiled_data->tensors.parameters[i + j * parameter_size] = ccv_nnc_tensor_new(0, info, 0);
1339 }
1340 }
1341 const uint32_t* const init_v = CCV_NNC_INIT_V(compiled_data->tensors_init.v)((uint32_t*)((uintptr_t)(compiled_data->tensors_init.v) &
~(uintptr_t)1));
1342 for (i = 0; i < internal_size; i++)
1343 {
1344 const ccv_nnc_tensor_symbol_t retained = *(ccv_nnc_tensor_symbol_t*)ccv_array_get(compiled_data->internals, i)((void*)(((char*)((compiled_data->internals)->data)) + (
size_t)(compiled_data->internals)->rsize * (size_t)(i))
);
1345 const int d = retained.d;
1346 if (init_v[d >> 5] & (1u << (d & 0x1f)))
1347 continue;
1348 ccv_nnc_tensor_param_t info = ccv_nnc_tensor_symbol_params(retained.graph, retained);
1349 if (CCV_TENSOR_GET_DEVICE(info.type)((info.type) & 0xfff00) == CCV_COMPUTE_DEVICE_ANY)
1350 CCV_TENSOR_SET_DEVICE_ID(info.type, 0)(info.type) = (((info.type) & ~0xfff00) | (((0) & 0xfff
) << 8));
1351 const int device_id = CCV_TENSOR_GET_DEVICE_ID(info.type)(((info.type) & 0xfff00) >> 8);
1352 if (!compiled_data->tensors.internals[i])
1353 compiled_data->tensors.internals[i] = ccv_nnc_tensor_new(0, info, 0);
1354 for (j = 1; j < parallel_count; j++)
1355 {
1356 if (j != device_id)
1357 CCV_TENSOR_SET_DEVICE_ID(info.type, j)(info.type) = (((info.type) & ~0xfff00) | (((j) & 0xfff
) << 8));
1358 else
1359 CCV_TENSOR_SET_DEVICE_ID(info.type, 0)(info.type) = (((info.type) & ~0xfff00) | (((0) & 0xfff
) << 8));
1360 if (!compiled_data->tensors.internals[i + j * internal_size])
1361 compiled_data->tensors.internals[i + j * internal_size] = ccv_nnc_tensor_new(0, info, 0);
1362 }
1363 }
1364 compiled_data->tensors_init.v = CCV_NNC_INIT_V(compiled_data->tensors_init.v)((uint32_t*)((uintptr_t)(compiled_data->tensors_init.v) &
~(uintptr_t)1)); // Remove 1 if any.
1365}
1366
1367static void _ccv_cnnp_model_tensors_init(const ccv_cnnp_model_t* const model, ccv_cnnp_compiled_data_t* const compiled_data)
1368{
1369 ccv_cnnp_model_tensors_init_0(model, compiled_data);
1370 ccv_cnnp_model_tensors_init_1(model, compiled_data);
1371}
1372
1373static void _ccv_cnnp_model_copy_tensors(const uint32_t* const tensors_init, const ccv_nnc_tensor_symbol_t* const tensor_symbols, ccv_nnc_tensor_t* const* const tensors, const int tensor_size, const int parallel_count)
1374{
1375 assert(parallel_count > 0)((void) sizeof ((parallel_count > 0) ? 1 : 0), __extension__
({ if (parallel_count > 0) ; else __assert_fail ("parallel_count > 0"
, "ccv_cnnp_model.c", 1375, __extension__ __PRETTY_FUNCTION__
); }));
1376 int i, j;
1377 for (i = 0; i < tensor_size; i++)
1378 {
1379 if (!tensors[i])
1380 continue;
1381 const int d = tensor_symbols[i].d;
1382 if (!(tensors_init[d >> 5] & (1u << (d & 0x1f))))
1383 continue;
1384 for (j = 1; j < parallel_count; j++)
1385 if (tensors[i + j * tensor_size])
1386 {
1387 ccv_nnc_tensor_t* const input = CCV_NNC_TENSOR(tensors[i])((ccv_nnc_tensor_t*)((uintptr_t)(tensors[i]) & ~(uintptr_t
)1));
1388 ccv_nnc_tensor_t* const output = CCV_NNC_TENSOR(tensors[i + j * tensor_size])((ccv_nnc_tensor_t*)((uintptr_t)(tensors[i + j * tensor_size]
) & ~(uintptr_t)1));
1389 ccv_nnc_cmd_exec(CMD_DATA_TRANSFER_FORWARD()ccv_nnc_cmd(CCV_NNC_DATA_TRANSFER_FORWARD, 0, ccv_nnc_cmd_auto
, 0), ccv_nnc_no_hint, 0, &input, 1, &output, 1, 0);
1390 }
1391 }
1392}
1393
1394static void _ccv_cnnp_model_remove_nocopies(const ccv_nnc_symbolic_graph_t* const graph, const ccv_nnc_tensor_symbol_t* const tensor_symbols, ccv_nnc_tensor_t** const tensors, const int tensor_size, const int parallel_count)
1395{
1396 assert(parallel_count > 0)((void) sizeof ((parallel_count > 0) ? 1 : 0), __extension__
({ if (parallel_count > 0) ; else __assert_fail ("parallel_count > 0"
, "ccv_cnnp_model.c", 1396, __extension__ __PRETTY_FUNCTION__
); }));
1397 int i, j;
1398 for (i = 0; i < tensor_size; i++)
1399 {
1400 const ccv_nnc_tensor_symbol_t tensor_symbol = tensor_symbols[i];
1401 for (j = 1; j < parallel_count; j++)
1402 {
1403 const ccv_nnc_tensor_symbol_t copy = ccv_nnc_tensor_symbol_copy(graph, tensor_symbol, j);
1404 ccv_nnc_tensor_t* copy_tensor = tensors[i + j * tensor_size];
1405 if (copy_tensor && copy.d == CCV_NNC_NO_TENSOR_SYMBOL)
1406 { // We shouldn't allocate this, free it up.
1407 ccv_nnc_tensor_free(tensors[i + j * tensor_size]);
1408 tensors[i + j * tensor_size] = 0;
1409 }
1410 }
1411 }
1412}
1413
1414static void _ccv_cnnp_model_bind_tensors(const ccv_nnc_symbolic_graph_t* const graph, const ccv_nnc_tensor_symbol_t* const tensor_symbols, ccv_nnc_tensor_t* const* const tensors, const int tensor_size, const int parallel_count, ccv_array_t* const tensor_binds)
1415{
1416 assert(parallel_count > 0)((void) sizeof ((parallel_count > 0) ? 1 : 0), __extension__
({ if (parallel_count > 0) ; else __assert_fail ("parallel_count > 0"
, "ccv_cnnp_model.c", 1416, __extension__ __PRETTY_FUNCTION__
); }));
1417 int i, j;
1418 for (i = 0; i < tensor_size; i++)
1419 {
1420 ccv_nnc_tensor_symbol_t tensor_symbol = tensor_symbols[i];
1421 if (tensor_symbol.d == CCV_NNC_NO_TENSOR_SYMBOL)
1422 continue;
1423 if (graph)
1424 {
1425 const ccv_nnc_tensor_symbol_t alias_to = ccv_nnc_tensor_symbol_alias_to(graph, tensor_symbol);
1426 if (alias_to.d != CCV_NNC_NO_TENSOR_SYMBOL)
1427 tensor_symbol = alias_to;
1428 }
1429 ccv_nnc_tensor_t* const tensor = CCV_NNC_TENSOR(tensors[i])((ccv_nnc_tensor_t*)((uintptr_t)(tensors[i]) & ~(uintptr_t
)1));
1430 if (tensor && tensor_symbol.d != CCV_NNC_NO_TENSOR_SYMBOL)
1431 {
1432 const ccv_nnc_tensor_bind_t retained_bind = {
1433 .symbol = tensor_symbol,
1434 .tensor = tensor
1435 };
1436 ccv_array_push(tensor_binds, &retained_bind);
1437 }
1438 for (j = 1; j < parallel_count; j++)
1439 {
1440 const ccv_nnc_tensor_symbol_t copy = ccv_nnc_tensor_symbol_copy(graph, tensor_symbol, j);
1441 ccv_nnc_tensor_t* copy_tensor = tensors[i + j * tensor_size];
1442 if (copy_tensor && copy.d != CCV_NNC_NO_TENSOR_SYMBOL)
1443 {
1444 const ccv_nnc_tensor_bind_t bind = {
1445 .symbol = copy,
1446 .tensor = tensors[i + j * tensor_size]
1447 };
1448 ccv_array_push(tensor_binds, &bind);
1449 }
1450 }
1451 }
1452}
1453
1454static void _ccv_cnnp_compiled_data_graph_free(ccv_cnnp_compiled_data_t* const compiled_data)
1455{
1456 if (compiled_data->graph)
1457 ccv_nnc_graph_free(compiled_data->graph);
1458 compiled_data->graph = 0;
1459 compiled_data->is_test = 0;
1460 if (compiled_data->tensor_arena)
1461 ccv_nnc_tensor_arena_free(compiled_data->tensor_arena);
1462 compiled_data->tensor_arena = 0;
1463 if (compiled_data->graph_exec_arena)
1464 ccv_nnc_graph_exec_arena_free(compiled_data->graph_exec_arena);
1465 compiled_data->graph_exec_arena = 0;
1466 if (compiled_data->backward.from_ops)
1467 ccfreefree(compiled_data->backward.from_ops);
1468 compiled_data->backward.from_ops = 0;
1469 if (compiled_data->evaluate.schedule)
1470 ccv_nnc_graph_static_schedule_free(compiled_data->evaluate.schedule);
1471 compiled_data->evaluate.schedule = 0;
1472 if (compiled_data->backward.schedule)
1473 ccv_nnc_graph_static_schedule_free(compiled_data->backward.schedule);
1474 compiled_data->backward.schedule = 0;
1475}
1476
1477static void _ccv_cnnp_compiled_data_gradient_free(ccv_cnnp_compiled_data_t* const compiled_data)
1478{
1479 if (compiled_data->gradients)
1480 ccfreefree(compiled_data->gradients);
1481 compiled_data->gradients = 0;
1482 if (compiled_data->updated_parameters)
1483 ccfreefree(compiled_data->updated_parameters);
1484 compiled_data->updated_parameters = 0;
1485 compiled_data->update_nodes = 0;
1486 compiled_data->saved_aux = 0;
1487}
1488
1489static void _ccv_cnnp_compiled_data_backward_free(ccv_cnnp_compiled_data_t* const compiled_data)
1490{
1491 if (compiled_data->backward.gradients)
1492 ccfreefree(compiled_data->backward.gradients);
1493 compiled_data->backward.gradients = 0;
1494 if (compiled_data->backward.accum)
1495 ccv_nnc_graph_free(compiled_data->backward.accum);
1496 compiled_data->backward.accum = 0;
1497 if (compiled_data->backward.tensor_arena)
1498 ccv_nnc_tensor_arena_free(compiled_data->backward.tensor_arena);
1499 compiled_data->backward.tensor_arena = 0;
1500 if (compiled_data->backward.graph_exec_arena)
1501 ccv_nnc_graph_exec_arena_free(compiled_data->backward.graph_exec_arena);
1502 compiled_data->backward.graph_exec_arena = 0;
1503}
1504
1505static void _ccv_cnnp_compiled_data_apply_gradients_free(ccv_cnnp_compiled_data_t* const compiled_data)
1506{
1507 if (compiled_data->apply_gradients.graph)
1508 ccv_nnc_graph_free(compiled_data->apply_gradients.graph);
1509 compiled_data->apply_gradients.graph = 0;
1510 if (compiled_data->apply_gradients.tensor_arena)
1511 ccv_nnc_tensor_arena_free(compiled_data->apply_gradients.tensor_arena);
1512 compiled_data->apply_gradients.tensor_arena = 0;
1513 if (compiled_data->apply_gradients.graph_exec_arena)
1514 ccv_nnc_graph_exec_arena_free(compiled_data->apply_gradients.graph_exec_arena);
1515 compiled_data->apply_gradients.graph_exec_arena = 0;
1516}
1517
1518// Compile the graph to run ccv_cnnp_model_fit
1519static void _ccv_cnnp_model_fit_jit(ccv_cnnp_model_t* const model, ccv_nnc_tensor_t* const* const inputs, const int input_size, ccv_nnc_tensor_t* const* const fits, const int fit_size, ccv_nnc_tensor_t* const* const outputs, const int output_size)
1520{
1521 int i, j;
1522 ccv_cnnp_compiled_data_t* const compiled_data = model->compiled_data;
1523 assert(!compiled_data->graph || compiled_data->graph_mode != CCV_CNNP_MODEL_GRAPH_FIT_MODE)((void) sizeof ((!compiled_data->graph || compiled_data->
graph_mode != CCV_CNNP_MODEL_GRAPH_FIT_MODE) ? 1 : 0), __extension__
({ if (!compiled_data->graph || compiled_data->graph_mode
!= CCV_CNNP_MODEL_GRAPH_FIT_MODE) ; else __assert_fail ("!compiled_data->graph || compiled_data->graph_mode != CCV_CNNP_MODEL_GRAPH_FIT_MODE"
, "ccv_cnnp_model.c", 1523, __extension__ __PRETTY_FUNCTION__
); }));
1524 compiled_data->graph_mode = CCV_CNNP_MODEL_GRAPH_FIT_MODE;
1525 const int parallel_count = _ccv_cnnp_model_root_parallel_count(model);
1526 assert(output_size == model->output_size * parallel_count)((void) sizeof ((output_size == model->output_size * parallel_count
) ? 1 : 0), __extension__ ({ if (output_size == model->output_size
* parallel_count) ; else __assert_fail ("output_size == model->output_size * parallel_count"
, "ccv_cnnp_model.c", 1526, __extension__ __PRETTY_FUNCTION__
); }));
1527 assert(!fits || output_size == fit_size)((void) sizeof ((!fits || output_size == fit_size) ? 1 : 0), __extension__
({ if (!fits || output_size == fit_size) ; else __assert_fail
("!fits || output_size == fit_size", "ccv_cnnp_model.c", 1527
, __extension__ __PRETTY_FUNCTION__); }));
1528 assert(output_size > 0)((void) sizeof ((output_size > 0) ? 1 : 0), __extension__ (
{ if (output_size > 0) ; else __assert_fail ("output_size > 0"
, "ccv_cnnp_model.c", 1528, __extension__ __PRETTY_FUNCTION__
); }));
1529 if (compiled_data->gradient_mode == CCV_CNNP_COMPILED_DATA_GRADIENT_NONE)
1530 {
1531 _ccv_cnnp_model_set_rewindables(model);
1532 _ccv_cnnp_model_gradient_init(model, CCV_CNNP_COMPILED_DATA_GRADIENT_TRAINABLES, CCV_CNNP_DISABLE_OUTGRAD_ALL, fits, fit_size);
1533 } else if (compiled_data->gradient_mode != CCV_CNNP_COMPILED_DATA_GRADIENT_TRAINABLES) {
1534 _ccv_cnnp_model_rewind_graph(model);
1535 _ccv_cnnp_compiled_data_gradient_free(compiled_data);
1536 compiled_data->gradient_mode = CCV_CNNP_COMPILED_DATA_GRADIENT_NONE;
1537 _ccv_cnnp_model_gradient_init(model, CCV_CNNP_COMPILED_DATA_GRADIENT_TRAINABLES, CCV_CNNP_DISABLE_OUTGRAD_ALL, fits, fit_size);
1538 }
1539 const int tensors_init = !!compiled_data->tensors_init.v;
1540 if (!tensors_init)
1541 _ccv_cnnp_model_tensors_init(model, compiled_data);
1542 else if ((uintptr_t)compiled_data->tensors_init.v & (uintptr_t)1)
1543 // Check if it is not fully allocated, if it is not, init_1.
1544 ccv_cnnp_model_tensors_init_1(model, compiled_data);
1545 ccv_array_t* const tensor_binds = ccv_array_new(sizeof(ccv_nnc_tensor_bind_t), 0, 0);
1546 assert((input_size % parallel_count) == 0)((void) sizeof (((input_size % parallel_count) == 0) ? 1 : 0)
, __extension__ ({ if ((input_size % parallel_count) == 0) ; else
__assert_fail ("(input_size % parallel_count) == 0", "ccv_cnnp_model.c"
, 1546, __extension__ __PRETTY_FUNCTION__); }));
1547 assert((output_size % parallel_count) == 0)((void) sizeof (((output_size % parallel_count) == 0) ? 1 : 0
), __extension__ ({ if ((output_size % parallel_count) == 0) ;
else __assert_fail ("(output_size % parallel_count) == 0", "ccv_cnnp_model.c"
, 1547, __extension__ __PRETTY_FUNCTION__); }));
1548 assert((fit_size % parallel_count) == 0)((void) sizeof (((fit_size % parallel_count) == 0) ? 1 : 0), __extension__
({ if ((fit_size % parallel_count) == 0) ; else __assert_fail
("(fit_size % parallel_count) == 0", "ccv_cnnp_model.c", 1548
, __extension__ __PRETTY_FUNCTION__); }));
1549 const int input_size_per_p = input_size / parallel_count;
1550 _ccv_cnnp_model_bind_tensors(model->graph, model->inputs, inputs, input_size_per_p, parallel_count, tensor_binds);
1551 const int output_size_per_p = output_size / parallel_count;
1552 _ccv_cnnp_model_bind_tensors(model->graph, model->outputs, outputs, output_size_per_p, parallel_count, tensor_binds);
1553 const int fit_size_per_p = fit_size / parallel_count;
1554 _ccv_cnnp_model_bind_tensors(model->graph, compiled_data->fits, fits, fit_size_per_p, parallel_count, tensor_binds);
1555 const int parameter_size = compiled_data->parameters->rnum;
1556 _ccv_cnnp_model_bind_tensors(model->graph, (ccv_nnc_tensor_symbol_t*)ccv_array_get(compiled_data->parameters, 0)((void*)(((char*)((compiled_data->parameters)->data)) +
(size_t)(compiled_data->parameters)->rsize * (size_t)(
0))), compiled_data->tensors.parameters, parameter_size, parallel_count, tensor_binds);
1557 _ccv_cnnp_model_bind_tensors(model->graph, compiled_data->updated_parameters, compiled_data->tensors.parameters, parameter_size, parallel_count, tensor_binds);
1558 const int internal_size = compiled_data->internals->rnum;
1559 _ccv_cnnp_model_remove_nocopies(model->graph, (ccv_nnc_tensor_symbol_t*)ccv_array_get(compiled_data->internals, 0)((void*)(((char*)((compiled_data->internals)->data)) + (
size_t)(compiled_data->internals)->rsize * (size_t)(0))
), compiled_data->tensors.internals, internal_size, parallel_count);
1560 _ccv_cnnp_model_bind_tensors(model->graph, (ccv_nnc_tensor_symbol_t*)ccv_array_get(compiled_data->internals, 0)((void*)(((char*)((compiled_data->internals)->data)) + (
size_t)(compiled_data->internals)->rsize * (size_t)(0))
), compiled_data->tensors.internals, internal_size, parallel_count, tensor_binds);
1561 ccv_nnc_symbolic_graph_compile(model->graph, compiled_data->compile_params, (ccv_nnc_tensor_bind_t*)ccv_array_get(tensor_binds, 0)((void*)(((char*)((tensor_binds)->data)) + (size_t)(tensor_binds
)->rsize * (size_t)(0))), tensor_binds->rnum, 0, 0, SYMBOLIC_GRAPH_SOURCES(model->graph)ccv_nnc_symbolic_graph_sources(model->graph), ccv_nnc_symbolic_graph_source_size
(model->graph), SYMBOLIC_GRAPH_DESTINATIONS(model->graph)ccv_nnc_symbolic_graph_destinations(model->graph), ccv_nnc_symbolic_graph_destination_size
(model->graph), &compiled_data->graph, &compiled_data->tensor_arena, &compiled_data->graph_exec_arena);
1562 ccv_array_free(tensor_binds);
1563 const uint32_t* const init_v = CCV_NNC_INIT_V(compiled_data->tensors_init.v)((uint32_t*)((uintptr_t)(compiled_data->tensors_init.v) &
~(uintptr_t)1));
1564 if (tensors_init && parallel_count > 1)
1565 _ccv_cnnp_model_copy_tensors(init_v, (ccv_nnc_tensor_symbol_t*)ccv_array_get(compiled_data->parameters, 0)((void*)(((char*)((compiled_data->parameters)->data)) +
(size_t)(compiled_data->parameters)->rsize * (size_t)(
0))), compiled_data->tensors.parameters, compiled_data->parameters->rnum, parallel_count);
1566 // If tensor is not init'ed, we need to init states first.
1567 if (_ccv_cnnp_any_to_init(compiled_data))
1568 {
1569 ccv_nnc_tensor_init_states_t tensor_init_states = {
1570 .parallel_count = parallel_count,
1571 .graph = model->graph,
1572 .compiled_data = compiled_data,
1573 .tensor_arena = compiled_data->tensor_arena
1574 };
1575 ccv_cnnp_model_init_states(model, model->graph, _ccv_cnnp_init_states_for_tensors, &tensor_init_states);
1576 }
1577 compiled_data->is_test = 0;
1578 const int saved_aux_size = ccv_nnc_minimizer_saved_aux_size(compiled_data->minimize.minimizer);
1579 // No need to set because it is default to training mode.
1580 // ccv_cnnp_model_set_is_test(model, 0, _ccv_cnnp_cmd_update_for_execs, &update);
1581 for (i = 0; i < saved_aux_size * parameter_size; i++)
1582 {
1583 if (compiled_data->saved_aux[i].source.d == CCV_NNC_NO_TENSOR_SYMBOL)
1584 continue;
1585 ccv_nnc_tensor_t* const tensor = ccv_nnc_tensor_from_symbol(compiled_data->tensor_arena, compiled_data->saved_aux[i].source);
1586 ccv_nnc_cmd_exec(CMD_SET_FORWARD(0)ccv_nnc_cmd(CCV_NNC_SET_FORWARD, 0, (ccv_nnc_cmd_param_t){.size
={.dim={1,1,1}},.blas={.a={0,}}}, 0), ccv_nnc_no_hint, 0, 0, 0, &tensor, 1, 0);
1587 for (j = 1; j < parallel_count; j++)
1588 {
1589 ccv_nnc_tensor_t* const copy = ccv_nnc_tensor_from_symbol(compiled_data->tensor_arena, ccv_nnc_tensor_symbol_copy(model->graph, compiled_data->saved_aux[i].source, j));
1590 if (copy)
1591 ccv_nnc_cmd_exec(CMD_SET_FORWARD(0)ccv_nnc_cmd(CCV_NNC_SET_FORWARD, 0, (ccv_nnc_cmd_param_t){.size
={.dim={1,1,1}},.blas={.a={0,}}}, 0), ccv_nnc_no_hint, 0, 0, 0, &copy, 1, 0);
1592 }
1593 }
1594 const int evaluate_to_size = compiled_data->evaluate.to_size;
1595 compiled_data->evaluate.to_op_size = 0;
1596 for (i = 0; i < evaluate_to_size; i++)
1597 {
1598 ccv_nnc_graph_exec_t const to = ccv_nnc_graph_exec_from_symbol(compiled_data->graph_exec_arena, compiled_data->evaluate.tos[i]);
1599 if (to.graph)
1600 compiled_data->evaluate.to_ops[compiled_data->evaluate.to_op_size++] = to;
1601 }
1602 ccv_nnc_graph_set_default_static_schedule(compiled_data->graph, compiled_data->stream_type, model->max_stream_count);
1603 ccv_nnc_graph_autotune(compiled_data->graph, model->workspace_size, 0, TRAVERSE_FULL0,0,0,0);
1604}
1605
1606ccv_nnc_stream_context_t* ccv_cnnp_model_default_stream(const ccv_cnnp_model_t* const model)
1607{
1608 const ccv_cnnp_compiled_data_t* const compiled_data = model->compiled_data;
1609 if (!compiled_data || !compiled_data->graph)
1610 return 0;
1611 return ccv_nnc_graph_default_stream(compiled_data->graph);
1612}
1613
1614uint64_t ccv_cnnp_model_memory_size(const ccv_cnnp_model_t* const model)
1615{
1616 const ccv_cnnp_compiled_data_t* const compiled_data = model->compiled_data;
1617 if (!compiled_data || !compiled_data->tensor_arena)
1618 return 0;
1619 return ccv_nnc_tensor_arena_size(compiled_data->tensor_arena);
1620}
1621
1622int ccv_cnnp_model_pin_memory(ccv_cnnp_model_t* const model, const int pin_memory)
1623{
1624#ifdef HAVE_MPS
1625 ccv_cnnp_compiled_data_t* const compiled_data = model->compiled_data;
1626 if (!compiled_data)
1627 return 0;
1628 int i, status = 0;
1629 if (!pin_memory)
1630 {
1631 if (!compiled_data->pinned_refs)
1632 return 0;
1633 for (i = 0; i < compiled_data->pinned_refs->rnum; i++)
1634 if (mpunpinmemory(*(void**)ccv_array_get(compiled_data->pinned_refs, i)((void*)(((char*)((compiled_data->pinned_refs)->data)) +
(size_t)(compiled_data->pinned_refs)->rsize * (size_t)
(i)))) != 0)
1635 status = -1;
1636 ccv_array_free(compiled_data->pinned_refs);
1637 compiled_data->pinned_refs = 0;
1638 return status;
1639 }
1640 if (!compiled_data->tensors.parameters || compiled_data->pinned_refs)
1641 return 0;
1642 compiled_data->pinned_refs = ccv_array_new(sizeof(void*), 0, 0);
1643 const int parallel_count = compiled_data->parallel_count > 0 ? compiled_data->parallel_count : _ccv_cnnp_model_root_parallel_count(model);
1644 for (i = 0; i < compiled_data->parameters->rnum * parallel_count; i++)
1645 {
1646 ccv_nnc_tensor_t* const tensor = CCV_NNC_TENSOR(compiled_data->tensors.parameters[i])((ccv_nnc_tensor_t*)((uintptr_t)(compiled_data->tensors.parameters
[i]) & ~(uintptr_t)1));
1647 if (!tensor || CCV_TENSOR_GET_MEMORY(tensor->info.type)((tensor->info.type) & 0x3) != CCV_TENSOR_GPU_MEMORY || !tensor->data.u8)
1648 continue;
1649 int result;
1650 void* const mapping = mppinmemory(tensor->data.u8, &result);
1651 if (mapping)
1652 ccv_array_push(compiled_data->pinned_refs, &mapping);
1653 if (result != 0)
1654 status = -1; // Best effort: keep going after a failed mapping.
1655 }
1656 return status;
1657#else
1658 return 0;
1659#endif
1660}
1661
1662static void _ccv_cnnp_bind_tensors_to_arena(ccv_nnc_tensor_arena_t* const tensor_arena, const ccv_nnc_symbolic_graph_t* const graph, const ccv_nnc_tensor_symbol_t* const tensor_symbols, ccv_nnc_tensor_t* const* const tensors, const int tensor_size, const int parallel_count)
1663{
1664 int i, j;
1665 for (i = 0; i < tensor_size; i++)
1666 {
1667 ccv_nnc_tensor_symbol_t tensor_symbol = tensor_symbols[i];
1668 if (tensor_symbol.d == CCV_NNC_NO_TENSOR_SYMBOL)
1669 continue;
1670 if (graph)
1671 {
1672 const ccv_nnc_tensor_symbol_t alias_to = ccv_nnc_tensor_symbol_alias_to(graph, tensor_symbol);
1673 if (alias_to.d != CCV_NNC_NO_TENSOR_SYMBOL)
1674 tensor_symbol = alias_to;
1675 }
1676 ccv_nnc_tensor_bind_symbol(tensor_arena, tensor_symbol, tensors[i]);
1677 for (j = 1; j < parallel_count; j++)
1678 {
1679 const ccv_nnc_tensor_symbol_t copy = ccv_nnc_tensor_symbol_copy(graph, tensor_symbol, j);
1680 if (copy.d != CCV_NNC_NO_TENSOR_SYMBOL)
1681 ccv_nnc_tensor_bind_symbol(tensor_arena, copy, tensors[i + tensor_size * j]);
1682 }
1683 }
1684}
1685
1686void ccv_cnnp_model_fit(ccv_cnnp_model_t* const model, ccv_nnc_tensor_t* const* const inputs, const int input_size, ccv_nnc_tensor_t* const* const fits, const int fit_size, ccv_nnc_tensor_t* const* const outputs, const int output_size, ccv_nnc_tensor_tape_t* const tensor_tape, ccv_nnc_stream_context_t* const stream_context)
1687{
1688 ccv_cnnp_compiled_data_t* const compiled_data = model->compiled_data;
1689 assert(compiled_data)((void) sizeof ((compiled_data) ? 1 : 0), __extension__ ({ if
(compiled_data) ; else __assert_fail ("compiled_data", "ccv_cnnp_model.c"
, 1689, __extension__ __PRETTY_FUNCTION__); }));
1690 const int parallel_count = ccv_max(model->parallel_count, 1)({ typeof (model->parallel_count) _a = (model->parallel_count
); typeof (1) _b = (1); (_a > _b) ? _a : _b; });
1691 assert(output_size == model->output_size * parallel_count)((void) sizeof ((output_size == model->output_size * parallel_count
) ? 1 : 0), __extension__ ({ if (output_size == model->output_size
* parallel_count) ; else __assert_fail ("output_size == model->output_size * parallel_count"
, "ccv_cnnp_model.c", 1691, __extension__ __PRETTY_FUNCTION__
); }));
1692 assert(input_size == model->input_size * parallel_count)((void) sizeof ((input_size == model->input_size * parallel_count
) ? 1 : 0), __extension__ ({ if (input_size == model->input_size
* parallel_count) ; else __assert_fail ("input_size == model->input_size * parallel_count"
, "ccv_cnnp_model.c", 1692, __extension__ __PRETTY_FUNCTION__
); }));
1693 assert(!fits || fit_size == output_size)((void) sizeof ((!fits || fit_size == output_size) ? 1 : 0), __extension__
({ if (!fits || fit_size == output_size) ; else __assert_fail
("!fits || fit_size == output_size", "ccv_cnnp_model.c", 1693
, __extension__ __PRETTY_FUNCTION__); }));
1694 assert(model->graph)((void) sizeof ((model->graph) ? 1 : 0), __extension__ ({ if
(model->graph) ; else __assert_fail ("model->graph", "ccv_cnnp_model.c"
, 1694, __extension__ __PRETTY_FUNCTION__); }));
1695 if (!compiled_data->graph || compiled_data->graph_mode != CCV_CNNP_MODEL_GRAPH_FIT_MODE)
1696 {
1697 _ccv_cnnp_compiled_data_graph_free(compiled_data);
1698 _ccv_cnnp_compiled_data_backward_free(compiled_data);
1699 _ccv_cnnp_compiled_data_apply_gradients_free(compiled_data);
1700 // Compile the symbolic graph down only when needed.
1701 _ccv_cnnp_model_fit_jit(model, inputs, input_size, fits, fit_size, outputs, output_size);
1702 } else {
1703 assert((input_size % parallel_count) == 0)((void) sizeof (((input_size % parallel_count) == 0) ? 1 : 0)
, __extension__ ({ if ((input_size % parallel_count) == 0) ; else
__assert_fail ("(input_size % parallel_count) == 0", "ccv_cnnp_model.c"
, 1703, __extension__ __PRETTY_FUNCTION__); }));
1704 assert((output_size % parallel_count) == 0)((void) sizeof (((output_size % parallel_count) == 0) ? 1 : 0
), __extension__ ({ if ((output_size % parallel_count) == 0) ;
else __assert_fail ("(output_size % parallel_count) == 0", "ccv_cnnp_model.c"
, 1704, __extension__ __PRETTY_FUNCTION__); }));
1705 assert((fit_size % parallel_count) == 0)((void) sizeof (((fit_size % parallel_count) == 0) ? 1 : 0), __extension__
({ if ((fit_size % parallel_count) == 0) ; else __assert_fail
("(fit_size % parallel_count) == 0", "ccv_cnnp_model.c", 1705
, __extension__ __PRETTY_FUNCTION__); }));
1706 const int input_size_per_p = input_size / parallel_count;
1707 _ccv_cnnp_bind_tensors_to_arena(compiled_data->tensor_arena, model->graph, model->inputs, inputs, input_size_per_p, parallel_count);
1708 const int output_size_per_p = output_size / parallel_count;
1709 _ccv_cnnp_bind_tensors_to_arena(compiled_data->tensor_arena, model->graph, model->outputs, outputs, output_size_per_p, parallel_count);
1710 const int fit_size_per_p = fit_size / parallel_count;
1711 _ccv_cnnp_bind_tensors_to_arena(compiled_data->tensor_arena, model->graph, compiled_data->fits, fits, fit_size_per_p, parallel_count);
1712 }
1713 if (compiled_data->is_test)
1714 {
1715 compiled_data->is_test = 0;
1716 ccv_nnc_graph_exec_update_t update = {
1717 .parallel_count = parallel_count,
1718 .graph = model->graph,
1719 .graph_exec_arena = compiled_data->graph_exec_arena,
1720 };
1721 ccv_cnnp_model_set_is_test(model, 0, _ccv_cnnp_cmd_update_for_execs, &update);
1722 }
1723 ccv_nnc_graph_run_with_schedule(compiled_data->graph, 0, 0, tensor_tape, stream_context);
1724}
1725
1726// Compile the graph to run ccv_cnnp_model_evaluate with require_grad = false (MULTISTAGE_MODE_NO_GRAD).
1727static void _ccv_cnnp_model_multistage_no_grad_jit(ccv_cnnp_model_t* const model, ccv_nnc_tensor_t* const* const inputs, const int input_size, ccv_nnc_tensor_t* const* const outputs, const int output_size)
1728{
1729 ccv_cnnp_compiled_data_t* const compiled_data = model->compiled_data;
1730 compiled_data->graph_mode = CCV_CNNP_MODEL_GRAPH_MULTISTAGE_MODE_NO_GRAD;
1731 const int parallel_count = ccv_max(model->parallel_count, 1)({ typeof (model->parallel_count) _a = (model->parallel_count
); typeof (1) _b = (1); (_a > _b) ? _a : _b; });
1732 assert(output_size == model->output_size * parallel_count)((void) sizeof ((output_size == model->output_size * parallel_count
) ? 1 : 0), __extension__ ({ if (output_size == model->output_size
* parallel_count) ; else __assert_fail ("output_size == model->output_size * parallel_count"
, "ccv_cnnp_model.c", 1732, __extension__ __PRETTY_FUNCTION__
); }));
1733 assert(output_size > 0)((void) sizeof ((output_size > 0) ? 1 : 0), __extension__ (
{ if (output_size > 0) ; else __assert_fail ("output_size > 0"
, "ccv_cnnp_model.c", 1733, __extension__ __PRETTY_FUNCTION__
); }));
1734 // If the gradient is not initialized, continue to setup parallel process. We don't init gradient here, but rather,
1735 // we setup proper rewindables so the graph can be rewinded to previous state before we run data parallel.
1736 if (parallel_count > 1 && compiled_data->gradient_mode == CCV_CNNP_COMPILED_DATA_GRADIENT_NONE)
1737 {
1738 const int evaluate_to_size = compiled_data->evaluate.to_size;
1739 compiled_data->evaluate.tos = ccreallocrealloc(compiled_data->evaluate.tos, sizeof(ccv_nnc_graph_exec_symbol_t) * evaluate_to_size * parallel_count + sizeof(ccv_nnc_graph_exec_t) * evaluate_to_size * parallel_count);
1740 _ccv_cnnp_model_set_rewindables(model);
1741 ccv_nnc_symbolic_graph_data_parallel(model->graph, parallel_count,
1742 0, 0,
1743 0, 0, 0,
1744 0, 0, 0,
1745 CCV_NNC_PARALLEL_REDUCE_OP_SUM,
1746 SYMBOLIC_GRAPH_SOURCES(model->graph)ccv_nnc_symbolic_graph_sources(model->graph), ccv_nnc_symbolic_graph_source_size
(model->graph), SYMBOLIC_GRAPH_DESTINATIONS(model->graph)ccv_nnc_symbolic_graph_destinations(model->graph), ccv_nnc_symbolic_graph_destination_size
(model->graph));
1747 ccv_nnc_graph_exec_symbol_autogen(model->graph, 0, 0, CCV_NNC_AUTOGEN_SOURCES_AND_DESTINATIONS);
1748 int i, j;
1749 for (i = 0; i < evaluate_to_size; i++)
1750 for (j = 1; j < parallel_count; j++)
1751 {
1752 const ccv_nnc_graph_exec_symbol_t copy = ccv_nnc_graph_exec_symbol_copy(model->graph, compiled_data->evaluate.tos[i], j);
1753 if (copy.d != CCV_NNC_NO_GRAPH_EXEC_SYMBOL)
1754 compiled_data->evaluate.tos[compiled_data->evaluate.to_size++] = copy;
1755 }
1756 }
1757 const int tensors_init = !!compiled_data->tensors_init.v;
1758 if (!tensors_init)
1759 _ccv_cnnp_model_tensors_init(model, compiled_data);
1760 else if ((uintptr_t)compiled_data->tensors_init.v & (uintptr_t)1)
1761 // Check if it is not fully allocated, if it is not, init_1.
1762 ccv_cnnp_model_tensors_init_1(model, compiled_data);
1763 const int tensor_parallel_count = _ccv_cnnp_compiled_data_parallel_count(model, compiled_data);
1764 ccv_array_t* const tensor_binds = ccv_array_new(sizeof(ccv_nnc_tensor_bind_t), 0, 0);
1765 assert((input_size % parallel_count) == 0)((void) sizeof (((input_size % parallel_count) == 0) ? 1 : 0)
, __extension__ ({ if ((input_size % parallel_count) == 0) ; else
__assert_fail ("(input_size % parallel_count) == 0", "ccv_cnnp_model.c"
, 1765, __extension__ __PRETTY_FUNCTION__); }));
1766 assert((output_size % parallel_count) == 0)((void) sizeof (((output_size % parallel_count) == 0) ? 1 : 0
), __extension__ ({ if ((output_size % parallel_count) == 0) ;
else __assert_fail ("(output_size % parallel_count) == 0", "ccv_cnnp_model.c"
, 1766, __extension__ __PRETTY_FUNCTION__); }));
1767 const int input_size_per_p = input_size / parallel_count;
1768 _ccv_cnnp_model_bind_tensors(model->graph, model->inputs, inputs, input_size_per_p, parallel_count, tensor_binds);
1769 const int output_size_per_p = output_size / parallel_count;
1770 _ccv_cnnp_model_bind_tensors(model->graph, model->outputs, outputs, output_size_per_p, parallel_count, tensor_binds);
1771 const int parameter_size = compiled_data->parameters->rnum;
1772 _ccv_cnnp_model_bind_tensors(model->graph, (ccv_nnc_tensor_symbol_t*)ccv_array_get(compiled_data->parameters, 0)((void*)(((char*)((compiled_data->parameters)->data)) +
(size_t)(compiled_data->parameters)->rsize * (size_t)(
0))), compiled_data->tensors.parameters, parameter_size, tensor_parallel_count, tensor_binds);
1773 const int internal_size = compiled_data->internals->rnum;
1774 _ccv_cnnp_model_remove_nocopies(model->graph, (ccv_nnc_tensor_symbol_t*)ccv_array_get(compiled_data->internals, 0)((void*)(((char*)((compiled_data->internals)->data)) + (
size_t)(compiled_data->internals)->rsize * (size_t)(0))
), compiled_data->tensors.internals, internal_size, tensor_parallel_count);
1775 _ccv_cnnp_model_bind_tensors(model->graph, (ccv_nnc_tensor_symbol_t*)ccv_array_get(compiled_data->internals, 0)((void*)(((char*)((compiled_data->internals)->data)) + (
size_t)(compiled_data->internals)->rsize * (size_t)(0))
), compiled_data->tensors.internals, internal_size, tensor_parallel_count, tensor_binds);
1776 // If we generated gradient for the graph, only compile part of the graph because the rest is irrelevant for evaluation.
1777 ccv_nnc_symbolic_graph_compile(model->graph, compiled_data->compile_params, (ccv_nnc_tensor_bind_t*)ccv_array_get(tensor_binds, 0)((void*)(((char*)((tensor_binds)->data)) + (size_t)(tensor_binds
)->rsize * (size_t)(0))), tensor_binds->rnum, 0, 0, SYMBOLIC_GRAPH_SOURCES(model->graph)ccv_nnc_symbolic_graph_sources(model->graph), ccv_nnc_symbolic_graph_source_size
(model->graph), compiled_data->evaluate.tos, compiled_data->evaluate.to_size, &compiled_data->graph, &compiled_data->tensor_arena, &compiled_data->graph_exec_arena);
1778 ccv_array_free(tensor_binds);
1779 const uint32_t* const init_v = CCV_NNC_INIT_V(compiled_data->tensors_init.v)((uint32_t*)((uintptr_t)(compiled_data->tensors_init.v) &
~(uintptr_t)1));
1780 // If tensor is not init'ed, we need to init states first.
1781 if (tensors_init && tensor_parallel_count > 1)
1782 _ccv_cnnp_model_copy_tensors(init_v, (ccv_nnc_tensor_symbol_t*)ccv_array_get(compiled_data->parameters, 0)((void*)(((char*)((compiled_data->parameters)->data)) +
(size_t)(compiled_data->parameters)->rsize * (size_t)(
0))), compiled_data->tensors.parameters, compiled_data->parameters->rnum, tensor_parallel_count);
1783 if (_ccv_cnnp_any_to_init(compiled_data))
1784 {
1785 ccv_nnc_tensor_init_states_t tensor_init_states = {
1786 .parallel_count = tensor_parallel_count,
1787 .graph = model->graph,
1788 .compiled_data = compiled_data,
1789 .tensor_arena = compiled_data->tensor_arena
1790 };
1791 ccv_cnnp_model_init_states(model, model->graph, _ccv_cnnp_init_states_for_tensors, &tensor_init_states);
1792 }
1793 compiled_data->is_test = 1;
1794 ccv_nnc_graph_exec_update_t update = {
1795 .parallel_count = parallel_count,
1796 .graph = model->graph,
1797 .graph_exec_arena = compiled_data->graph_exec_arena,
1798 };
1799 ccv_cnnp_model_set_is_test(model, 1, _ccv_cnnp_cmd_update_for_execs, &update);
1800 ccv_nnc_graph_set_default_static_schedule(compiled_data->graph, compiled_data->stream_type, model->max_stream_count);
1801 ccv_nnc_graph_autotune(compiled_data->graph, model->workspace_size, 0, TRAVERSE_FULL0,0,0,0);
1802}
1803
1804static void _ccv_cnnp_model_gradient_tensors_init(const ccv_cnnp_model_t* const model, ccv_cnnp_compiled_data_t* const compiled_data)
1805{
1806 assert(!compiled_data->tensors.gradients)((void) sizeof ((!compiled_data->tensors.gradients) ? 1 : 0
), __extension__ ({ if (!compiled_data->tensors.gradients)
; else __assert_fail ("!compiled_data->tensors.gradients"
, "ccv_cnnp_model.c", 1806, __extension__ __PRETTY_FUNCTION__
); }));
1807 const int parameter_size = compiled_data->parameters->rnum;
1808 const int parallel_count = ccv_max(model->parallel_count, 1)({ typeof (model->parallel_count) _a = (model->parallel_count
); typeof (1) _b = (1); (_a > _b) ? _a : _b; });
1809 compiled_data->tensors.gradients = (ccv_nnc_tensor_t**)ccmallocmalloc(sizeof(ccv_nnc_tensor_t*) * parameter_size * 2 * parallel_count);
1810 compiled_data->tensors.accum_gradients = compiled_data->tensors.gradients + parameter_size * parallel_count;
1811 int i, j;
1812 for (i = 0; i < parameter_size; i++)
1813 {
1814 if (compiled_data->parameter_flags && !(compiled_data->parameter_flags[i >> 6] & ((uint64_t)1 << (i & 63))))
1815 {
1816 compiled_data->tensors.gradients[i] = 0;
1817 compiled_data->tensors.accum_gradients[i] = 0;
1818 for (j = 1; j < parallel_count; j++)
1819 {
1820 compiled_data->tensors.gradients[i + j * parameter_size] = 0;
1821 compiled_data->tensors.accum_gradients[i + j * parameter_size] = 0;
1822 }
1823 continue;
1824 }
1825 const ccv_nnc_tensor_symbol_t parameter = *(ccv_nnc_tensor_symbol_t*)ccv_array_get(compiled_data->parameters, i)((void*)(((char*)((compiled_data->parameters)->data)) +
(size_t)(compiled_data->parameters)->rsize * (size_t)(
i)));
1826 ccv_nnc_tensor_param_t info = ccv_nnc_tensor_symbol_params(parameter.graph, parameter);
1827 if (CCV_TENSOR_GET_DEVICE(info.type)((info.type) & 0xfff00) == CCV_COMPUTE_DEVICE_ANY)
1828 CCV_TENSOR_SET_DEVICE_ID(info.type, 0)(info.type) = (((info.type) & ~0xfff00) | (((0) & 0xfff
) << 8));
1829 const int device_id = CCV_TENSOR_GET_DEVICE_ID(info.type)(((info.type) & 0xfff00) >> 8);
1830 compiled_data->tensors.gradients[i] = ccv_nnc_tensor_new(0, info, 0);
1831 compiled_data->tensors.accum_gradients[i] = 0; // delay the accumulated gradient allocation until when we need it.
1832 for (j = 1; j < parallel_count; j++)
1833 {
1834 if (j != device_id)
1835 CCV_TENSOR_SET_DEVICE_ID(info.type, j)(info.type) = (((info.type) & ~0xfff00) | (((j) & 0xfff
) << 8));
1836 else
1837 CCV_TENSOR_SET_DEVICE_ID(info.type, 0)(info.type) = (((info.type) & ~0xfff00) | (((0) & 0xfff
) << 8));
1838 compiled_data->tensors.gradients[i + j * parameter_size] = ccv_nnc_tensor_new(0, info, 0);
1839 compiled_data->tensors.accum_gradients[i + j * parameter_size] = 0;
1840 }
1841 }
1842}
1843
1844static int _ccv_cnnp_is_disable_outgrad_all(const uint64_t disable_outgrad, const int input_size)
1845{
1846 if (disable_outgrad == CCV_CNNP_DISABLE_OUTGRAD_ALL)
1847 return 1;
1848 if (disable_outgrad == CCV_CNNP_DISABLE_OUTGRAD_NONE)
1849 return 0;
1850 int i;
1851 for (i = 0; i < input_size; i++)
1852 if (!(disable_outgrad & ((uint64_t)1 << i)))
1853 return 0;
1854 return 1;
1855}
1856
1857// Compile the graph to run ccv_cnnp_model_evaluate with requires_grad = true (MULTISTAGE_MODE).
1858// Particularly, this method compiles the evaluation and backprop graph (the main graph).
1859static void _ccv_cnnp_model_multistage_jit_0(ccv_cnnp_model_t* const model, const uint64_t disable_outgrad, const int is_test, ccv_nnc_tensor_t* const* const inputs, const int input_size, ccv_nnc_tensor_t* const* const outputs, const int output_size)
1860{
1861 int i, j;
1862 ccv_cnnp_compiled_data_t* const compiled_data = model->compiled_data;
1863 const int target_gradient_mode = _ccv_cnnp_is_disable_outgrad_all(disable_outgrad, model->input_size) ? CCV_CNNP_COMPILED_DATA_GRADIENT_TRAINABLES : CCV_CNNP_COMPILED_DATA_GRADIENT_TRAINABLES_AND_INPUTS;
1864 assert(!compiled_data->graph || compiled_data->graph_mode != CCV_CNNP_MODEL_GRAPH_MULTISTAGE_MODE || compiled_data->gradient_mode != target_gradient_mode)((void) sizeof ((!compiled_data->graph || compiled_data->
graph_mode != CCV_CNNP_MODEL_GRAPH_MULTISTAGE_MODE || compiled_data
->gradient_mode != target_gradient_mode) ? 1 : 0), __extension__
({ if (!compiled_data->graph || compiled_data->graph_mode
!= CCV_CNNP_MODEL_GRAPH_MULTISTAGE_MODE || compiled_data->
gradient_mode != target_gradient_mode) ; else __assert_fail (
"!compiled_data->graph || compiled_data->graph_mode != CCV_CNNP_MODEL_GRAPH_MULTISTAGE_MODE || compiled_data->gradient_mode != target_gradient_mode"
, "ccv_cnnp_model.c", 1864, __extension__ __PRETTY_FUNCTION__
); }));
1865 compiled_data->graph_mode = CCV_CNNP_MODEL_GRAPH_MULTISTAGE_MODE;
1866 const int parallel_count = ccv_max(model->parallel_count, 1)({ typeof (model->parallel_count) _a = (model->parallel_count
); typeof (1) _b = (1); (_a > _b) ? _a : _b; });
1867 assert(output_size == model->output_size * parallel_count)((void) sizeof ((output_size == model->output_size * parallel_count
) ? 1 : 0), __extension__ ({ if (output_size == model->output_size
* parallel_count) ; else __assert_fail ("output_size == model->output_size * parallel_count"
, "ccv_cnnp_model.c", 1867, __extension__ __PRETTY_FUNCTION__
); }));
1868 assert(output_size > 0)((void) sizeof ((output_size > 0) ? 1 : 0), __extension__ (
{ if (output_size > 0) ; else __assert_fail ("output_size > 0"
, "ccv_cnnp_model.c", 1868, __extension__ __PRETTY_FUNCTION__
); }));
1869 // There shouldn't be a loss function if we evaluate with multistage jit.
1870 assert(compiled_data->loss.cmd == CCV_NNC_NOOP)((void) sizeof ((compiled_data->loss.cmd == CCV_NNC_NOOP) ?
1 : 0), __extension__ ({ if (compiled_data->loss.cmd == CCV_NNC_NOOP
) ; else __assert_fail ("compiled_data->loss.cmd == CCV_NNC_NOOP"
, "ccv_cnnp_model.c", 1870, __extension__ __PRETTY_FUNCTION__
); }));
1871 if (compiled_data->gradient_mode == CCV_CNNP_COMPILED_DATA_GRADIENT_NONE)
1872 {
1873 _ccv_cnnp_model_set_rewindables(model);
1874 _ccv_cnnp_model_gradient_init(model, target_gradient_mode, disable_outgrad, 0, 0); // The type of outputs and fits should be the same. We only use type here.
1875 } else if (compiled_data->gradient_mode != target_gradient_mode) {
1876 _ccv_cnnp_model_rewind_graph(model);
1877 _ccv_cnnp_compiled_data_gradient_free(compiled_data);
1878 compiled_data->gradient_mode = CCV_CNNP_COMPILED_DATA_GRADIENT_NONE;
1879 _ccv_cnnp_model_gradient_init(model, target_gradient_mode, disable_outgrad, 0, 0); // The type of outputs and fits should be the same. We only use type here.
1880 }
1881 const int tensors_init = !!compiled_data->tensors_init.v;
1882 if (!tensors_init)
1883 _ccv_cnnp_model_tensors_init(model, compiled_data);
1884 else if ((uintptr_t)compiled_data->tensors_init.v & (uintptr_t)1)
1885 // Check if it is not fully allocated, if it is not, init_1.
1886 ccv_cnnp_model_tensors_init_1(model, compiled_data);
1887 ccv_array_t* const tensor_binds = ccv_array_new(sizeof(ccv_nnc_tensor_bind_t), 0, 0);
1888 assert((input_size % parallel_count) == 0)((void) sizeof (((input_size % parallel_count) == 0) ? 1 : 0)
, __extension__ ({ if ((input_size % parallel_count) == 0) ; else
__assert_fail ("(input_size % parallel_count) == 0", "ccv_cnnp_model.c"
, 1888, __extension__ __PRETTY_FUNCTION__); }));
1889 assert((output_size % parallel_count) == 0)((void) sizeof (((output_size % parallel_count) == 0) ? 1 : 0
), __extension__ ({ if ((output_size % parallel_count) == 0) ;
else __assert_fail ("(output_size % parallel_count) == 0", "ccv_cnnp_model.c"
, 1889, __extension__ __PRETTY_FUNCTION__); }));
1890 const int input_size_per_p = input_size / parallel_count;
1891 _ccv_cnnp_model_bind_tensors(model->graph, model->inputs, inputs, input_size_per_p, parallel_count, tensor_binds);
1892 const int output_size_per_p = output_size / parallel_count;
1893 _ccv_cnnp_model_bind_tensors(model->graph, model->outputs, outputs, output_size_per_p, parallel_count, tensor_binds);
1894 const int parameter_size = compiled_data->parameters->rnum;
1895 _ccv_cnnp_model_bind_tensors(model->graph, (ccv_nnc_tensor_symbol_t*)ccv_array_get(compiled_data->parameters, 0)((void*)(((char*)((compiled_data->parameters)->data)) +
(size_t)(compiled_data->parameters)->rsize * (size_t)(
0))), compiled_data->tensors.parameters, parameter_size, parallel_count, tensor_binds);
1896 const int internal_size = compiled_data->internals->rnum;
1897 _ccv_cnnp_model_remove_nocopies(model->graph, (ccv_nnc_tensor_symbol_t*)ccv_array_get(compiled_data->internals, 0)((void*)(((char*)((compiled_data->internals)->data)) + (
size_t)(compiled_data->internals)->rsize * (size_t)(0))
), compiled_data->tensors.internals, internal_size, parallel_count);
1898 _ccv_cnnp_model_bind_tensors(model->graph, (ccv_nnc_tensor_symbol_t*)ccv_array_get(compiled_data->internals, 0)((void*)(((char*)((compiled_data->internals)->data)) + (
size_t)(compiled_data->internals)->rsize * (size_t)(0))
), compiled_data->tensors.internals, internal_size, parallel_count, tensor_binds);
1899 if (!compiled_data->tensors.gradients)
1900 _ccv_cnnp_model_gradient_tensors_init(model, compiled_data);
1901 _ccv_cnnp_model_bind_tensors(model->graph, compiled_data->gradients, compiled_data->tensors.gradients, parameter_size, parallel_count, tensor_binds);
1902 if (compiled_data->backward.to_size > 0)
1903 ccv_nnc_symbolic_graph_compile(model->graph, compiled_data->compile_params, (ccv_nnc_tensor_bind_t*)ccv_array_get(tensor_binds, 0)((void*)(((char*)((tensor_binds)->data)) + (size_t)(tensor_binds
)->rsize * (size_t)(0))), tensor_binds->rnum, 0, 0, SYMBOLIC_GRAPH_SOURCES(model->graph)ccv_nnc_symbolic_graph_sources(model->graph), ccv_nnc_symbolic_graph_source_size
(model->graph), compiled_data->backward.tos, compiled_data->backward.to_size, &compiled_data->graph, &compiled_data->tensor_arena, &compiled_data->graph_exec_arena);
1904 else
1905 ccv_nnc_symbolic_graph_compile(model->graph, compiled_data->compile_params, (ccv_nnc_tensor_bind_t*)ccv_array_get(tensor_binds, 0)((void*)(((char*)((tensor_binds)->data)) + (size_t)(tensor_binds
)->rsize * (size_t)(0))), tensor_binds->rnum, 0, 0, SYMBOLIC_GRAPH_SOURCES(model->graph)ccv_nnc_symbolic_graph_sources(model->graph), ccv_nnc_symbolic_graph_source_size
(model->graph), compiled_data->evaluate.tos, compiled_data->evaluate.to_size, &compiled_data->graph, &compiled_data->tensor_arena, &compiled_data->graph_exec_arena);
1906 ccv_array_free(tensor_binds);
1907 const uint32_t* const init_v = CCV_NNC_INIT_V(compiled_data->tensors_init.v)((uint32_t*)((uintptr_t)(compiled_data->tensors_init.v) &
~(uintptr_t)1));
1908 if (tensors_init && parallel_count > 1)
1909 _ccv_cnnp_model_copy_tensors(init_v, (ccv_nnc_tensor_symbol_t*)ccv_array_get(compiled_data->parameters, 0)((void*)(((char*)((compiled_data->parameters)->data)) +
(size_t)(compiled_data->parameters)->rsize * (size_t)(
0))), compiled_data->tensors.parameters, compiled_data->parameters->rnum, parallel_count);
1910 // If tensor is not init'ed, we need to init states first.
1911 if (_ccv_cnnp_any_to_init(compiled_data))
1912 {
1913 ccv_nnc_tensor_init_states_t tensor_init_states = {
1914 .parallel_count = parallel_count,
1915 .graph = model->graph,
1916 .compiled_data = compiled_data,
1917 .tensor_arena = compiled_data->tensor_arena
1918 };
1919 ccv_cnnp_model_init_states(model, model->graph, _ccv_cnnp_init_states_for_tensors, &tensor_init_states);
1920 }
1921 compiled_data->is_test = is_test;
1922 ccv_nnc_graph_exec_update_t update = {
1923 .parallel_count = parallel_count,
1924 .graph = model->graph,
1925 .graph_exec_arena = compiled_data->graph_exec_arena,
1926 };
1927 ccv_cnnp_model_set_is_test(model, is_test, _ccv_cnnp_cmd_update_for_execs, &update);
1928 const int evaluate_to_size = compiled_data->evaluate.to_size;
1929 compiled_data->evaluate.to_op_size = 0;
1930 ccv_array_t* const backward_from = ccv_array_new(sizeof(int), 0, 0);
1931 for (i = 0; i < evaluate_to_size; i++)
1932 {
1933 ccv_nnc_graph_exec_t const to_op = ccv_nnc_graph_exec_from_symbol(compiled_data->graph_exec_arena, compiled_data->evaluate.tos[i]);
1934 if (to_op.graph)
1935 compiled_data->evaluate.to_ops[compiled_data->evaluate.to_op_size++] = to_op;
1936 const int* tos;
1937 int to_size;
1938 ccv_nnc_graph_exec_symbol_to(model->graph, compiled_data->evaluate.tos[i], &tos, &to_size);
1939 for (j = 0; j < to_size; j++)
1940 {
1941 ccv_nnc_graph_exec_t const to_op = ccv_nnc_graph_exec_from_symbol(compiled_data->graph_exec_arena, (ccv_nnc_graph_exec_symbol_t){
1942 .d = tos[j],
1943 .graph = model->graph
1944 });
1945 if (to_op.graph)
1946 ccv_array_add_unique_int(backward_from, to_op.d);
1947 }
1948 }
1949 assert(backward_from->rnum > 0)((void) sizeof ((backward_from->rnum > 0) ? 1 : 0), __extension__
({ if (backward_from->rnum > 0) ; else __assert_fail (
"backward_from->rnum > 0", "ccv_cnnp_model.c", 1949, __extension__
__PRETTY_FUNCTION__); }));
1950 compiled_data->backward.from_op_size = backward_from->rnum;
1951 compiled_data->backward.from_ops = (ccv_nnc_graph_exec_t*)ccmallocmalloc(sizeof(ccv_nnc_graph_exec_t) * backward_from->rnum);
1952 for (i = 0; i < backward_from->rnum; i++)
1953 compiled_data->backward.from_ops[i] = (ccv_nnc_graph_exec_t){
1954 .d = *(int*)ccv_array_get(backward_from, i)((void*)(((char*)((backward_from)->data)) + (size_t)(backward_from
)->rsize * (size_t)(i))),
1955 .graph = compiled_data->graph,
1956 };
1957 // If there are any set node (to set some tensors to 0) inserted through backward pass, these won't be executed if we just do sources -> evaluate.to_ops, backward.from_ops -> destinations. We need this logic to find out these nodes and explicitly adding them to backward.from_ops.
1958 ccv_nnc_graph_exec_info_t* const exec_info = (ccv_nnc_graph_exec_info_t*)ccv_array_get(compiled_data->graph->exec_info, 0)((void*)(((char*)((compiled_data->graph->exec_info)->
data)) + (size_t)(compiled_data->graph->exec_info)->
rsize * (size_t)(0)));
1959 const int exec_info_size = compiled_data->graph->exec_info->rnum;
1960 uint32_t* const visited = cccalloccalloc((exec_info_size + 31) >> 5, sizeof(uint32_t));
1961 const ccv_nnc_graph_exec_t* const sources = (ccv_nnc_graph_exec_t*)ccv_array_get(compiled_data->graph->sources, 0)((void*)(((char*)((compiled_data->graph->sources)->data
)) + (size_t)(compiled_data->graph->sources)->rsize *
(size_t)(0)));
1962 const int source_size = compiled_data->graph->sources->rnum;
1963 ccv_nnc_graph_visit_t* visit = ccv_nnc_graph_visit_new(compiled_data->graph, exec_info, exec_info_size, sources, source_size, compiled_data->evaluate.to_ops, compiled_data->evaluate.to_op_size, 0)({ ccv_nnc_graph_visit_t* _visit_ = (ccv_nnc_graph_visit_t*)malloc
(sizeof(ccv_nnc_graph_visit_t) + sizeof(_visit_->node[0]) *
((exec_info_size) - 1)); _visit_->size = 0; do { typedef struct
{ int8_t d; int8_t r; uint16_t c; int32_t edges; } ccv_nnc_incoming_t
; int _i_, _j_; int _incoming_edges_ = 0; for (_i_ = 0; _i_ <
(exec_info_size); _i_++) _incoming_edges_ += ((exec_info)[_i_
].outgoings) ? (exec_info)[_i_].outgoings->rnum : 0; const
int _heap_mem_ = ((exec_info_size) + _incoming_edges_ > 1024
); ccv_nnc_incoming_t* _incomings_; if (_heap_mem_) _incomings_
= (ccv_nnc_incoming_t*)malloc(sizeof(ccv_nnc_incoming_t) * (
exec_info_size) + sizeof(int32_t) * ((exec_info_size) * 2 + _incoming_edges_
)); else _incomings_ = (ccv_nnc_incoming_t*)__builtin_alloca (
sizeof(ccv_nnc_incoming_t) * (exec_info_size) + sizeof(int32_t
) * ((exec_info_size) * 2 + _incoming_edges_)); memset(_incomings_
, 0, sizeof(ccv_nnc_incoming_t) * (exec_info_size)); int32_t*
_exists_[2] = { (int32_t*)(_incomings_ + (exec_info_size)), (
int32_t*)(_incomings_ + (exec_info_size)) + (exec_info_size),
}; int32_t* const _edges_ = _exists_[1] + (exec_info_size); for
(_i_ = 0; _i_ < (source_size); _i_++) { ((void) sizeof ((
(sources)[_i_].graph == compiled_data->graph) ? 1 : 0), __extension__
({ if ((sources)[_i_].graph == compiled_data->graph) ; else
__assert_fail ("(sources)[_i_].graph == compiled_data->graph"
, "ccv_cnnp_model.c", 1963, __extension__ __PRETTY_FUNCTION__
); })); _incomings_[(sources)[_i_].d].r = 1; _exists_[0][_i_]
= (sources)[_i_].d; } int _exist_size_[2] = { (source_size),
0, }; int _p_ = 0, _q_ = 1; while (_exist_size_[_p_] > 0)
{ _exist_size_[_q_] = 0; for (_i_ = 0; _i_ < _exist_size_
[_p_]; _i_++) { const int32_t _idx_ = _exists_[_p_][_i_]; if (
_incomings_[_idx_].r != 1) continue; _incomings_[_idx_].r = 2
; if ((exec_info)[_idx_].outgoings) for (_j_ = 0; _j_ < (exec_info
)[_idx_].outgoings->rnum; _j_++) { const int d = *(int*)((
void*)(((char*)(((exec_info)[_idx_].outgoings)->data)) + (
size_t)((exec_info)[_idx_].outgoings)->rsize * (size_t)(_j_
))); ++_incomings_[d].c; if (_incomings_[d].r != 0) continue;
_incomings_[d].r = 1; ((void) sizeof ((_exist_size_[_q_] <
(exec_info_size)) ? 1 : 0), __extension__ ({ if (_exist_size_
[_q_] < (exec_info_size)) ; else __assert_fail ("_exist_size_[_q_] < (exec_info_size)"
, "ccv_cnnp_model.c", 1963, __extension__ __PRETTY_FUNCTION__
); })); _exists_[_q_][_exist_size_[_q_]] = d; ++_exist_size_[
_q_]; } } ((_i_) = (_p_), (_p_) = (_q_), (_q_) = (_i_)); } for
(_i_ = 0; _i_ < (source_size); _i_++) { ((void) sizeof ((
(sources)[_i_].graph == compiled_data->graph) ? 1 : 0), __extension__
({ if ((sources)[_i_].graph == compiled_data->graph) ; else
__assert_fail ("(sources)[_i_].graph == compiled_data->graph"
, "ccv_cnnp_model.c", 1963, __extension__ __PRETTY_FUNCTION__
); })); _incomings_[(sources)[_i_].d].r = 3; _exists_[0][_i_]
= (sources)[_i_].d; } _exist_size_[0] = (source_size); _exist_size_
[1] = 0; _p_ = 0, _q_ = 1; int _bump_ = 1; while (_exist_size_
[_p_] > 0) { _exist_size_[_q_] = 0; for (_i_ = 0; _i_ <
_exist_size_[_p_]; _i_++) { const int32_t _idx_ = _exists_[_p_
][_i_]; if (_incomings_[_idx_].r != 3) continue; _incomings_[
_idx_].r = 4; if ((exec_info)[_idx_].outgoings) for (_j_ = 0;
_j_ < (exec_info)[_idx_].outgoings->rnum; _j_++) { const
int d = *(int*)((void*)(((char*)(((exec_info)[_idx_].outgoings
)->data)) + (size_t)((exec_info)[_idx_].outgoings)->rsize
* (size_t)(_j_))); if (_incomings_[d].edges == 0) { _incomings_
[d].edges = _bump_; _bump_ += _incomings_[d].c; _incomings_[d
].c = 0; } _edges_[_incomings_[d].edges - 1 + _incomings_[d].
c] = _idx_; ++_incomings_[d].c; if (_incomings_[d].r != 2) continue
; _incomings_[d].r = 3; ((void) sizeof ((_exist_size_[_q_] <
(exec_info_size)) ? 1 : 0), __extension__ ({ if (_exist_size_
[_q_] < (exec_info_size)) ; else __assert_fail ("_exist_size_[_q_] < (exec_info_size)"
, "ccv_cnnp_model.c", 1963, __extension__ __PRETTY_FUNCTION__
); })); _exists_[_q_][_exist_size_[_q_]] = d; ++_exist_size_[
_q_]; } } ((_i_) = (_p_), (_p_) = (_q_), (_q_) = (_i_)); } for
(_i_ = 0; _i_ < (compiled_data->evaluate.to_op_size); _i_
++) { ((void) sizeof (((compiled_data->evaluate.to_ops)[_i_
].graph == compiled_data->graph) ? 1 : 0), __extension__ (
{ if ((compiled_data->evaluate.to_ops)[_i_].graph == compiled_data
->graph) ; else __assert_fail ("(compiled_data->evaluate.to_ops)[_i_].graph == compiled_data->graph"
, "ccv_cnnp_model.c", 1963, __extension__ __PRETTY_FUNCTION__
); })); _incomings_[(compiled_data->evaluate.to_ops)[_i_].
d].r = 5; _exists_[0][_i_] = (compiled_data->evaluate.to_ops
)[_i_].d; } _exist_size_[0] = (compiled_data->evaluate.to_op_size
); _exist_size_[1] = 0; _p_ = 0, _q_ = 1; while (_exist_size_
[_p_] > 0) { _exist_size_[_q_] = 0; for (_i_ = 0; _i_ <
_exist_size_[_p_]; _i_++) { const int32_t _idx_ = _exists_[_p_
][_i_]; if (_incomings_[_idx_].r != 5) continue; _incomings_[
_idx_].r = 6; if (_incomings_[_idx_].edges > 0) for (_j_ =
0; _j_ < _incomings_[_idx_].c; _j_++) { const int d = _edges_
[_incomings_[_idx_].edges - 1 + _j_]; if (_incomings_[d].r !=
4) continue; _incomings_[d].r = 5; ((void) sizeof ((_exist_size_
[_q_] < (exec_info_size)) ? 1 : 0), __extension__ ({ if (_exist_size_
[_q_] < (exec_info_size)) ; else __assert_fail ("_exist_size_[_q_] < (exec_info_size)"
, "ccv_cnnp_model.c", 1963, __extension__ __PRETTY_FUNCTION__
); })); _exists_[_q_][_exist_size_[_q_]] = d; ++_exist_size_[
_q_]; } } ((_i_) = (_p_), (_p_) = (_q_), (_q_) = (_i_)); } for
(_i_ = 0; _i_ < (compiled_data->evaluate.to_op_size); _i_
++) { ((void) sizeof (((compiled_data->evaluate.to_ops)[_i_
].graph == compiled_data->graph) ? 1 : 0), __extension__ (
{ if ((compiled_data->evaluate.to_ops)[_i_].graph == compiled_data
->graph) ; else __assert_fail ("(compiled_data->evaluate.to_ops)[_i_].graph == compiled_data->graph"
, "ccv_cnnp_model.c", 1963, __extension__ __PRETTY_FUNCTION__
); })); _incomings_[(compiled_data->evaluate.to_ops)[_i_].
d].d = 1; } for (_i_ = 0; _i_ < (source_size); _i_++) { ((
void) sizeof (((sources)[_i_].graph == compiled_data->graph
) ? 1 : 0), __extension__ ({ if ((sources)[_i_].graph == compiled_data
->graph) ; else __assert_fail ("(sources)[_i_].graph == compiled_data->graph"
, "ccv_cnnp_model.c", 1963, __extension__ __PRETTY_FUNCTION__
); })); _exists_[0][_i_] = (sources)[_i_].d; } _p_ = 0; _q_ =
1; _exist_size_[0] = (source_size); _exist_size_[1] = 0; int
_d_ = 0; while (_exist_size_[_p_] > 0) { _exist_size_[_q_
] = 0; for (_i_ = 0; _i_ < _exist_size_[_p_];) { const int32_t
_idx_ = _exists_[_p_][_i_]; _visit_->node[_visit_->size
].index = ((_idx_)); _visit_->node[_visit_->size].term =
((_incomings_[_idx_].d)); ++_visit_->size;; if (_incomings_
[_idx_].d) { ++_d_; _incomings_[_idx_].r = 7; } if ((exec_info
)[_idx_].outgoings) { if ((exec_info)[_idx_].outgoings->rnum
== 1) { const int d = *(int*)((void*)(((char*)(((exec_info)[
_idx_].outgoings)->data)) + (size_t)((exec_info)[_idx_].outgoings
)->rsize * (size_t)(0))); --_incomings_[d].c; if (_incomings_
[d].c == 0 && _incomings_[d].r == 6 && _d_ <
(compiled_data->evaluate.to_op_size)) { _exists_[_p_][_i_
] = d; continue; } } else for (_j_ = 0; _j_ < (exec_info)[
_idx_].outgoings->rnum; _j_++) { const int d = *(int*)((void
*)(((char*)(((exec_info)[_idx_].outgoings)->data)) + (size_t
)((exec_info)[_idx_].outgoings)->rsize * (size_t)(_j_))); --
_incomings_[d].c; if (_incomings_[d].c == 0 && _incomings_
[d].r == 6 && _d_ < (compiled_data->evaluate.to_op_size
)) { ((void) sizeof ((_exist_size_[_q_] < (exec_info_size)
) ? 1 : 0), __extension__ ({ if (_exist_size_[_q_] < (exec_info_size
)) ; else __assert_fail ("_exist_size_[_q_] < (exec_info_size)"
, "ccv_cnnp_model.c", 1963, __extension__ __PRETTY_FUNCTION__
); })); _exists_[_q_][_exist_size_[_q_]] = d; ++_exist_size_[
_q_]; } } } ++_i_; } ((_i_) = (_p_), (_p_) = (_q_), (_q_) = (
_i_)); } for (_i_ = 0; _i_ < (compiled_data->evaluate.to_op_size
); _i_++) { ((void) sizeof (((compiled_data->evaluate.to_ops
)[_i_].graph == compiled_data->graph) ? 1 : 0), __extension__
({ if ((compiled_data->evaluate.to_ops)[_i_].graph == compiled_data
->graph) ; else __assert_fail ("(compiled_data->evaluate.to_ops)[_i_].graph == compiled_data->graph"
, "ccv_cnnp_model.c", 1963, __extension__ __PRETTY_FUNCTION__
); })); if (_incomings_[(compiled_data->evaluate.to_ops)[_i_
].d].r == 7) continue; if (!(0)) { ((void) sizeof ((_incomings_
[(compiled_data->evaluate.to_ops)[_i_].d].c == 0) ? 1 : 0)
, __extension__ ({ if (_incomings_[(compiled_data->evaluate
.to_ops)[_i_].d].c == 0) ; else __assert_fail ("_incomings_[(compiled_data->evaluate.to_ops)[_i_].d].c == 0"
, "ccv_cnnp_model.c", 1963, __extension__ __PRETTY_FUNCTION__
); })); } else if (_incomings_[(compiled_data->evaluate.to_ops
)[_i_].d].c > 0) continue; _visit_->node[_visit_->size
].index = (((compiled_data->evaluate.to_ops)[_i_].d)); _visit_
->node[_visit_->size].term = ((_incomings_[(compiled_data
->evaluate.to_ops)[_i_].d].d)); ++_visit_->size;; } if (
_heap_mem_) free(_incomings_); } while (0);; ((void) sizeof (
(_visit_->size <= (exec_info_size)) ? 1 : 0), __extension__
({ if (_visit_->size <= (exec_info_size)) ; else __assert_fail
("_visit_->size <= (exec_info_size)", "ccv_cnnp_model.c"
, 1963, __extension__ __PRETTY_FUNCTION__); })); _visit_; });
1964 ccv_nnc_graph_visit_for(visit, exec_info, node, idx){ int _i_; for (_i_ = 0; _i_ < (visit)->size; _i_++) { const
int idx __attribute__((unused)) = (visit)->node[_i_].index
; const int _node_unused_ __attribute__((unused)) = (visit)->
node[_i_].term; typeof ((exec_info)) const node __attribute__
((unused)) = (exec_info) + idx; {
1965 visited[(idx >> 5)] |= (1u << (idx & 31));
1966 } ccv_nnc_graph_visit_endfor} }
1967 ccv_nnc_graph_visit_free(visit);
1968 const ccv_nnc_graph_exec_t* const destinations = (ccv_nnc_graph_exec_t*)ccv_array_get(compiled_data->graph->destinations, 0)((void*)(((char*)((compiled_data->graph->destinations)->
data)) + (size_t)(compiled_data->graph->destinations)->
rsize * (size_t)(0)));
1969 const int destination_size = compiled_data->graph->destinations->rnum;
1970 visit = ccv_nnc_graph_visit_new(compiled_data->graph, exec_info, exec_info_size, compiled_data->backward.from_ops, compiled_data->backward.from_op_size, destinations, destination_size, 0)({ ccv_nnc_graph_visit_t* _visit_ = (ccv_nnc_graph_visit_t*)malloc
(sizeof(ccv_nnc_graph_visit_t) + sizeof(_visit_->node[0]) *
((exec_info_size) - 1)); _visit_->size = 0; do { typedef struct
{ int8_t d; int8_t r; uint16_t c; int32_t edges; } ccv_nnc_incoming_t
; int _i_, _j_; int _incoming_edges_ = 0; for (_i_ = 0; _i_ <
(exec_info_size); _i_++) _incoming_edges_ += ((exec_info)[_i_
].outgoings) ? (exec_info)[_i_].outgoings->rnum : 0; const
int _heap_mem_ = ((exec_info_size) + _incoming_edges_ > 1024
); ccv_nnc_incoming_t* _incomings_; if (_heap_mem_) _incomings_
= (ccv_nnc_incoming_t*)malloc(sizeof(ccv_nnc_incoming_t) * (
exec_info_size) + sizeof(int32_t) * ((exec_info_size) * 2 + _incoming_edges_
)); else _incomings_ = (ccv_nnc_incoming_t*)__builtin_alloca (
sizeof(ccv_nnc_incoming_t) * (exec_info_size) + sizeof(int32_t
) * ((exec_info_size) * 2 + _incoming_edges_)); memset(_incomings_
, 0, sizeof(ccv_nnc_incoming_t) * (exec_info_size)); int32_t*
_exists_[2] = { (int32_t*)(_incomings_ + (exec_info_size)), (
int32_t*)(_incomings_ + (exec_info_size)) + (exec_info_size),
}; int32_t* const _edges_ = _exists_[1] + (exec_info_size); for
(_i_ = 0; _i_ < (compiled_data->backward.from_op_size)
; _i_++) { ((void) sizeof (((compiled_data->backward.from_ops
)[_i_].graph == compiled_data->graph) ? 1 : 0), __extension__
({ if ((compiled_data->backward.from_ops)[_i_].graph == compiled_data
->graph) ; else __assert_fail ("(compiled_data->backward.from_ops)[_i_].graph == compiled_data->graph"
, "ccv_cnnp_model.c", 1970, __extension__ __PRETTY_FUNCTION__
); })); _incomings_[(compiled_data->backward.from_ops)[_i_
].d].r = 1; _exists_[0][_i_] = (compiled_data->backward.from_ops
)[_i_].d; } int _exist_size_[2] = { (compiled_data->backward
.from_op_size), 0, }; int _p_ = 0, _q_ = 1; while (_exist_size_
[_p_] > 0) { _exist_size_[_q_] = 0; for (_i_ = 0; _i_ <
_exist_size_[_p_]; _i_++) { const int32_t _idx_ = _exists_[_p_
][_i_]; if (_incomings_[_idx_].r != 1) continue; _incomings_[
_idx_].r = 2; if ((exec_info)[_idx_].outgoings) for (_j_ = 0;
_j_ < (exec_info)[_idx_].outgoings->rnum; _j_++) { const
int d = *(int*)((void*)(((char*)(((exec_info)[_idx_].outgoings
)->data)) + (size_t)((exec_info)[_idx_].outgoings)->rsize
* (size_t)(_j_))); ++_incomings_[d].c; if (_incomings_[d].r !=
0) continue; _incomings_[d].r = 1; ((void) sizeof ((_exist_size_
[_q_] < (exec_info_size)) ? 1 : 0), __extension__ ({ if (_exist_size_
[_q_] < (exec_info_size)) ; else __assert_fail ("_exist_size_[_q_] < (exec_info_size)"
, "ccv_cnnp_model.c", 1970, __extension__ __PRETTY_FUNCTION__
); })); _exists_[_q_][_exist_size_[_q_]] = d; ++_exist_size_[
_q_]; } } ((_i_) = (_p_), (_p_) = (_q_), (_q_) = (_i_)); } for
(_i_ = 0; _i_ < (compiled_data->backward.from_op_size)
; _i_++) { ((void) sizeof (((compiled_data->backward.from_ops
)[_i_].graph == compiled_data->graph) ? 1 : 0), __extension__
({ if ((compiled_data->backward.from_ops)[_i_].graph == compiled_data
->graph) ; else __assert_fail ("(compiled_data->backward.from_ops)[_i_].graph == compiled_data->graph"
, "ccv_cnnp_model.c", 1970, __extension__ __PRETTY_FUNCTION__
); })); _incomings_[(compiled_data->backward.from_ops)[_i_
].d].r = 3; _exists_[0][_i_] = (compiled_data->backward.from_ops
)[_i_].d; } _exist_size_[0] = (compiled_data->backward.from_op_size
); _exist_size_[1] = 0; _p_ = 0, _q_ = 1; int _bump_ = 1; while
(_exist_size_[_p_] > 0) { _exist_size_[_q_] = 0; for (_i_
= 0; _i_ < _exist_size_[_p_]; _i_++) { const int32_t _idx_
= _exists_[_p_][_i_]; if (_incomings_[_idx_].r != 3) continue
; _incomings_[_idx_].r = 4; if ((exec_info)[_idx_].outgoings)
for (_j_ = 0; _j_ < (exec_info)[_idx_].outgoings->rnum
; _j_++) { const int d = *(int*)((void*)(((char*)(((exec_info
)[_idx_].outgoings)->data)) + (size_t)((exec_info)[_idx_].
outgoings)->rsize * (size_t)(_j_))); if (_incomings_[d].edges
== 0) { _incomings_[d].edges = _bump_; _bump_ += _incomings_
[d].c; _incomings_[d].c = 0; } _edges_[_incomings_[d].edges -
1 + _incomings_[d].c] = _idx_; ++_incomings_[d].c; if (_incomings_
[d].r != 2) continue; _incomings_[d].r = 3; ((void) sizeof ((
_exist_size_[_q_] < (exec_info_size)) ? 1 : 0), __extension__
({ if (_exist_size_[_q_] < (exec_info_size)) ; else __assert_fail
("_exist_size_[_q_] < (exec_info_size)", "ccv_cnnp_model.c"
, 1970, __extension__ __PRETTY_FUNCTION__); })); _exists_[_q_
][_exist_size_[_q_]] = d; ++_exist_size_[_q_]; } } ((_i_) = (
_p_), (_p_) = (_q_), (_q_) = (_i_)); } for (_i_ = 0; _i_ <
(destination_size); _i_++) { ((void) sizeof (((destinations)
[_i_].graph == compiled_data->graph) ? 1 : 0), __extension__
({ if ((destinations)[_i_].graph == compiled_data->graph)
; else __assert_fail ("(destinations)[_i_].graph == compiled_data->graph"
, "ccv_cnnp_model.c", 1970, __extension__ __PRETTY_FUNCTION__
); })); _incomings_[(destinations)[_i_].d].r = 5; _exists_[0]
[_i_] = (destinations)[_i_].d; } _exist_size_[0] = (destination_size
); _exist_size_[1] = 0; _p_ = 0, _q_ = 1; while (_exist_size_
[_p_] > 0) { _exist_size_[_q_] = 0; for (_i_ = 0; _i_ <
_exist_size_[_p_]; _i_++) { const int32_t _idx_ = _exists_[_p_
][_i_]; if (_incomings_[_idx_].r != 5) continue; _incomings_[
_idx_].r = 6; if (_incomings_[_idx_].edges > 0) for (_j_ =
0; _j_ < _incomings_[_idx_].c; _j_++) { const int d = _edges_
[_incomings_[_idx_].edges - 1 + _j_]; if (_incomings_[d].r !=
4) continue; _incomings_[d].r = 5; ((void) sizeof ((_exist_size_
[_q_] < (exec_info_size)) ? 1 : 0), __extension__ ({ if (_exist_size_
[_q_] < (exec_info_size)) ; else __assert_fail ("_exist_size_[_q_] < (exec_info_size)"
, "ccv_cnnp_model.c", 1970, __extension__ __PRETTY_FUNCTION__
); })); _exists_[_q_][_exist_size_[_q_]] = d; ++_exist_size_[
_q_]; } } ((_i_) = (_p_), (_p_) = (_q_), (_q_) = (_i_)); } for
(_i_ = 0; _i_ < (destination_size); _i_++) { ((void) sizeof
(((destinations)[_i_].graph == compiled_data->graph) ? 1 :
0), __extension__ ({ if ((destinations)[_i_].graph == compiled_data
->graph) ; else __assert_fail ("(destinations)[_i_].graph == compiled_data->graph"
, "ccv_cnnp_model.c", 1970, __extension__ __PRETTY_FUNCTION__
); })); _incomings_[(destinations)[_i_].d].d = 1; } for (_i_ =
0; _i_ < (compiled_data->backward.from_op_size); _i_++
) { ((void) sizeof (((compiled_data->backward.from_ops)[_i_
].graph == compiled_data->graph) ? 1 : 0), __extension__ (
{ if ((compiled_data->backward.from_ops)[_i_].graph == compiled_data
->graph) ; else __assert_fail ("(compiled_data->backward.from_ops)[_i_].graph == compiled_data->graph"
, "ccv_cnnp_model.c", 1970, __extension__ __PRETTY_FUNCTION__
); })); _exists_[0][_i_] = (compiled_data->backward.from_ops
)[_i_].d; } _p_ = 0; _q_ = 1; _exist_size_[0] = (compiled_data
->backward.from_op_size); _exist_size_[1] = 0; int _d_ = 0
; while (_exist_size_[_p_] > 0) { _exist_size_[_q_] = 0; for
(_i_ = 0; _i_ < _exist_size_[_p_];) { const int32_t _idx_
= _exists_[_p_][_i_]; _visit_->node[_visit_->size].index
= ((_idx_)); _visit_->node[_visit_->size].term = ((_incomings_
[_idx_].d)); ++_visit_->size;; if (_incomings_[_idx_].d) {
++_d_; _incomings_[_idx_].r = 7; } if ((exec_info)[_idx_].outgoings
) { if ((exec_info)[_idx_].outgoings->rnum == 1) { const int
d = *(int*)((void*)(((char*)(((exec_info)[_idx_].outgoings)->
data)) + (size_t)((exec_info)[_idx_].outgoings)->rsize * (
size_t)(0))); --_incomings_[d].c; if (_incomings_[d].c == 0 &&
_incomings_[d].r == 6 && _d_ < (destination_size)
) { _exists_[_p_][_i_] = d; continue; } } else for (_j_ = 0; _j_
< (exec_info)[_idx_].outgoings->rnum; _j_++) { const int
d = *(int*)((void*)(((char*)(((exec_info)[_idx_].outgoings)->
data)) + (size_t)((exec_info)[_idx_].outgoings)->rsize * (
size_t)(_j_))); --_incomings_[d].c; if (_incomings_[d].c == 0
&& _incomings_[d].r == 6 && _d_ < (destination_size
)) { ((void) sizeof ((_exist_size_[_q_] < (exec_info_size)
) ? 1 : 0), __extension__ ({ if (_exist_size_[_q_] < (exec_info_size
)) ; else __assert_fail ("_exist_size_[_q_] < (exec_info_size)"
, "ccv_cnnp_model.c", 1970, __extension__ __PRETTY_FUNCTION__
); })); _exists_[_q_][_exist_size_[_q_]] = d; ++_exist_size_[
_q_]; } } } ++_i_; } ((_i_) = (_p_), (_p_) = (_q_), (_q_) = (
_i_)); } for (_i_ = 0; _i_ < (destination_size); _i_++) { (
(void) sizeof (((destinations)[_i_].graph == compiled_data->
graph) ? 1 : 0), __extension__ ({ if ((destinations)[_i_].graph
== compiled_data->graph) ; else __assert_fail ("(destinations)[_i_].graph == compiled_data->graph"
, "ccv_cnnp_model.c", 1970, __extension__ __PRETTY_FUNCTION__
); })); if (_incomings_[(destinations)[_i_].d].r == 7) continue
; if (!(0)) { ((void) sizeof ((_incomings_[(destinations)[_i_
].d].c == 0) ? 1 : 0), __extension__ ({ if (_incomings_[(destinations
)[_i_].d].c == 0) ; else __assert_fail ("_incomings_[(destinations)[_i_].d].c == 0"
, "ccv_cnnp_model.c", 1970, __extension__ __PRETTY_FUNCTION__
); })); } else if (_incomings_[(destinations)[_i_].d].c > 0
) continue; _visit_->node[_visit_->size].index = (((destinations
)[_i_].d)); _visit_->node[_visit_->size].term = ((_incomings_
[(destinations)[_i_].d].d)); ++_visit_->size;; } if (_heap_mem_
) free(_incomings_); } while (0);; ((void) sizeof ((_visit_->
size <= (exec_info_size)) ? 1 : 0), __extension__ ({ if (_visit_
->size <= (exec_info_size)) ; else __assert_fail ("_visit_->size <= (exec_info_size)"
, "ccv_cnnp_model.c", 1970, __extension__ __PRETTY_FUNCTION__
); })); _visit_; });
1971 ccv_nnc_graph_visit_for(visit, exec_info, node, idx){ int _i_; for (_i_ = 0; _i_ < (visit)->size; _i_++) { const
int idx __attribute__((unused)) = (visit)->node[_i_].index
; const int _node_unused_ __attribute__((unused)) = (visit)->
node[_i_].term; typeof ((exec_info)) const node __attribute__
((unused)) = (exec_info) + idx; {
1972 visited[(idx >> 5)] |= (1u << (idx & 31));
1973 } ccv_nnc_graph_visit_endfor} }
1974 ccv_nnc_graph_visit_free(visit);
1975 visit = ccv_nnc_graph_visit_new(compiled_data->graph, exec_info, exec_info_size, sources, source_size, destinations, destination_size, 0)({ ccv_nnc_graph_visit_t* _visit_ = (ccv_nnc_graph_visit_t*)malloc
(sizeof(ccv_nnc_graph_visit_t) + sizeof(_visit_->node[0]) *
((exec_info_size) - 1)); _visit_->size = 0; do { typedef struct
{ int8_t d; int8_t r; uint16_t c; int32_t edges; } ccv_nnc_incoming_t
; int _i_, _j_; int _incoming_edges_ = 0; for (_i_ = 0; _i_ <
(exec_info_size); _i_++) _incoming_edges_ += ((exec_info)[_i_
].outgoings) ? (exec_info)[_i_].outgoings->rnum : 0; const
int _heap_mem_ = ((exec_info_size) + _incoming_edges_ > 1024
); ccv_nnc_incoming_t* _incomings_; if (_heap_mem_) _incomings_
= (ccv_nnc_incoming_t*)malloc(sizeof(ccv_nnc_incoming_t) * (
exec_info_size) + sizeof(int32_t) * ((exec_info_size) * 2 + _incoming_edges_
)); else _incomings_ = (ccv_nnc_incoming_t*)__builtin_alloca (
sizeof(ccv_nnc_incoming_t) * (exec_info_size) + sizeof(int32_t
) * ((exec_info_size) * 2 + _incoming_edges_)); memset(_incomings_
, 0, sizeof(ccv_nnc_incoming_t) * (exec_info_size)); int32_t*
_exists_[2] = { (int32_t*)(_incomings_ + (exec_info_size)), (
int32_t*)(_incomings_ + (exec_info_size)) + (exec_info_size),
}; int32_t* const _edges_ = _exists_[1] + (exec_info_size); for
(_i_ = 0; _i_ < (source_size); _i_++) { ((void) sizeof ((
(sources)[_i_].graph == compiled_data->graph) ? 1 : 0), __extension__
({ if ((sources)[_i_].graph == compiled_data->graph) ; else
__assert_fail ("(sources)[_i_].graph == compiled_data->graph"
, "ccv_cnnp_model.c", 1975, __extension__ __PRETTY_FUNCTION__
); })); _incomings_[(sources)[_i_].d].r = 1; _exists_[0][_i_]
= (sources)[_i_].d; } int _exist_size_[2] = { (source_size),
0, }; int _p_ = 0, _q_ = 1; while (_exist_size_[_p_] > 0)
{ _exist_size_[_q_] = 0; for (_i_ = 0; _i_ < _exist_size_
[_p_]; _i_++) { const int32_t _idx_ = _exists_[_p_][_i_]; if (
_incomings_[_idx_].r != 1) continue; _incomings_[_idx_].r = 2
; if ((exec_info)[_idx_].outgoings) for (_j_ = 0; _j_ < (exec_info
)[_idx_].outgoings->rnum; _j_++) { const int d = *(int*)((
void*)(((char*)(((exec_info)[_idx_].outgoings)->data)) + (
size_t)((exec_info)[_idx_].outgoings)->rsize * (size_t)(_j_
))); ++_incomings_[d].c; if (_incomings_[d].r != 0) continue;
_incomings_[d].r = 1; ((void) sizeof ((_exist_size_[_q_] <
(exec_info_size)) ? 1 : 0), __extension__ ({ if (_exist_size_
[_q_] < (exec_info_size)) ; else __assert_fail ("_exist_size_[_q_] < (exec_info_size)"
, "ccv_cnnp_model.c", 1975, __extension__ __PRETTY_FUNCTION__
); })); _exists_[_q_][_exist_size_[_q_]] = d; ++_exist_size_[
_q_]; } } ((_i_) = (_p_), (_p_) = (_q_), (_q_) = (_i_)); } for
(_i_ = 0; _i_ < (source_size); _i_++) { ((void) sizeof ((
(sources)[_i_].graph == compiled_data->graph) ? 1 : 0), __extension__
({ if ((sources)[_i_].graph == compiled_data->graph) ; else
__assert_fail ("(sources)[_i_].graph == compiled_data->graph"
, "ccv_cnnp_model.c", 1975, __extension__ __PRETTY_FUNCTION__
); })); _incomings_[(sources)[_i_].d].r = 3; _exists_[0][_i_]
= (sources)[_i_].d; } _exist_size_[0] = (source_size); _exist_size_
[1] = 0; _p_ = 0, _q_ = 1; int _bump_ = 1; while (_exist_size_
[_p_] > 0) { _exist_size_[_q_] = 0; for (_i_ = 0; _i_ <
_exist_size_[_p_]; _i_++) { const int32_t _idx_ = _exists_[_p_
][_i_]; if (_incomings_[_idx_].r != 3) continue; _incomings_[
_idx_].r = 4; if ((exec_info)[_idx_].outgoings) for (_j_ = 0;
_j_ < (exec_info)[_idx_].outgoings->rnum; _j_++) { const
int d = *(int*)((void*)(((char*)(((exec_info)[_idx_].outgoings
)->data)) + (size_t)((exec_info)[_idx_].outgoings)->rsize
* (size_t)(_j_))); if (_incomings_[d].edges == 0) { _incomings_
[d].edges = _bump_; _bump_ += _incomings_[d].c; _incomings_[d
].c = 0; } _edges_[_incomings_[d].edges - 1 + _incomings_[d].
c] = _idx_; ++_incomings_[d].c; if (_incomings_[d].r != 2) continue
; _incomings_[d].r = 3; ((void) sizeof ((_exist_size_[_q_] <
(exec_info_size)) ? 1 : 0), __extension__ ({ if (_exist_size_
[_q_] < (exec_info_size)) ; else __assert_fail ("_exist_size_[_q_] < (exec_info_size)"
, "ccv_cnnp_model.c", 1975, __extension__ __PRETTY_FUNCTION__
); })); _exists_[_q_][_exist_size_[_q_]] = d; ++_exist_size_[
_q_]; } } ((_i_) = (_p_), (_p_) = (_q_), (_q_) = (_i_)); } for
(_i_ = 0; _i_ < (destination_size); _i_++) { ((void) sizeof
(((destinations)[_i_].graph == compiled_data->graph) ? 1 :
0), __extension__ ({ if ((destinations)[_i_].graph == compiled_data
->graph) ; else __assert_fail ("(destinations)[_i_].graph == compiled_data->graph"
, "ccv_cnnp_model.c", 1975, __extension__ __PRETTY_FUNCTION__
); })); _incomings_[(destinations)[_i_].d].r = 5; _exists_[0]
[_i_] = (destinations)[_i_].d; } _exist_size_[0] = (destination_size
); _exist_size_[1] = 0; _p_ = 0, _q_ = 1; while (_exist_size_
[_p_] > 0) { _exist_size_[_q_] = 0; for (_i_ = 0; _i_ <
_exist_size_[_p_]; _i_++) { const int32_t _idx_ = _exists_[_p_
][_i_]; if (_incomings_[_idx_].r != 5) continue; _incomings_[
_idx_].r = 6; if (_incomings_[_idx_].edges > 0) for (_j_ =
0; _j_ < _incomings_[_idx_].c; _j_++) { const int d = _edges_
[_incomings_[_idx_].edges - 1 + _j_]; if (_incomings_[d].r !=
4) continue; _incomings_[d].r = 5; ((void) sizeof ((_exist_size_
[_q_] < (exec_info_size)) ? 1 : 0), __extension__ ({ if (_exist_size_
[_q_] < (exec_info_size)) ; else __assert_fail ("_exist_size_[_q_] < (exec_info_size)"
, "ccv_cnnp_model.c", 1975, __extension__ __PRETTY_FUNCTION__
); })); _exists_[_q_][_exist_size_[_q_]] = d; ++_exist_size_[
_q_]; } } ((_i_) = (_p_), (_p_) = (_q_), (_q_) = (_i_)); } for
(_i_ = 0; _i_ < (destination_size); _i_++) { ((void) sizeof
(((destinations)[_i_].graph == compiled_data->graph) ? 1 :
0), __extension__ ({ if ((destinations)[_i_].graph == compiled_data
->graph) ; else __assert_fail ("(destinations)[_i_].graph == compiled_data->graph"
, "ccv_cnnp_model.c", 1975, __extension__ __PRETTY_FUNCTION__
); })); _incomings_[(destinations)[_i_].d].d = 1; } for (_i_ =
0; _i_ < (source_size); _i_++) { ((void) sizeof (((sources
)[_i_].graph == compiled_data->graph) ? 1 : 0), __extension__
({ if ((sources)[_i_].graph == compiled_data->graph) ; else
__assert_fail ("(sources)[_i_].graph == compiled_data->graph"
, "ccv_cnnp_model.c", 1975, __extension__ __PRETTY_FUNCTION__
); })); _exists_[0][_i_] = (sources)[_i_].d; } _p_ = 0; _q_ =
1; _exist_size_[0] = (source_size); _exist_size_[1] = 0; int
_d_ = 0; while (_exist_size_[_p_] > 0) { _exist_size_[_q_
] = 0; for (_i_ = 0; _i_ < _exist_size_[_p_];) { const int32_t
_idx_ = _exists_[_p_][_i_]; _visit_->node[_visit_->size
].index = ((_idx_)); _visit_->node[_visit_->size].term =
((_incomings_[_idx_].d)); ++_visit_->size;; if (_incomings_
[_idx_].d) { ++_d_; _incomings_[_idx_].r = 7; } if ((exec_info
)[_idx_].outgoings) { if ((exec_info)[_idx_].outgoings->rnum
== 1) { const int d = *(int*)((void*)(((char*)(((exec_info)[
_idx_].outgoings)->data)) + (size_t)((exec_info)[_idx_].outgoings
)->rsize * (size_t)(0))); --_incomings_[d].c; if (_incomings_
[d].c == 0 && _incomings_[d].r == 6 && _d_ <
(destination_size)) { _exists_[_p_][_i_] = d; continue; } } else
for (_j_ = 0; _j_ < (exec_info)[_idx_].outgoings->rnum
; _j_++) { const int d = *(int*)((void*)(((char*)(((exec_info
)[_idx_].outgoings)->data)) + (size_t)((exec_info)[_idx_].
outgoings)->rsize * (size_t)(_j_))); --_incomings_[d].c; if
(_incomings_[d].c == 0 && _incomings_[d].r == 6 &&
_d_ < (destination_size)) { ((void) sizeof ((_exist_size_
[_q_] < (exec_info_size)) ? 1 : 0), __extension__ ({ if (_exist_size_
[_q_] < (exec_info_size)) ; else __assert_fail ("_exist_size_[_q_] < (exec_info_size)"
, "ccv_cnnp_model.c", 1975, __extension__ __PRETTY_FUNCTION__
); })); _exists_[_q_][_exist_size_[_q_]] = d; ++_exist_size_[
_q_]; } } } ++_i_; } ((_i_) = (_p_), (_p_) = (_q_), (_q_) = (
_i_)); } for (_i_ = 0; _i_ < (destination_size); _i_++) { (
(void) sizeof (((destinations)[_i_].graph == compiled_data->
graph) ? 1 : 0), __extension__ ({ if ((destinations)[_i_].graph
== compiled_data->graph) ; else __assert_fail ("(destinations)[_i_].graph == compiled_data->graph"
, "ccv_cnnp_model.c", 1975, __extension__ __PRETTY_FUNCTION__
); })); if (_incomings_[(destinations)[_i_].d].r == 7) continue
; if (!(0)) { ((void) sizeof ((_incomings_[(destinations)[_i_
].d].c == 0) ? 1 : 0), __extension__ ({ if (_incomings_[(destinations
)[_i_].d].c == 0) ; else __assert_fail ("_incomings_[(destinations)[_i_].d].c == 0"
, "ccv_cnnp_model.c", 1975, __extension__ __PRETTY_FUNCTION__
); })); } else if (_incomings_[(destinations)[_i_].d].c > 0
) continue; _visit_->node[_visit_->size].index = (((destinations
)[_i_].d)); _visit_->node[_visit_->size].term = ((_incomings_
[(destinations)[_i_].d].d)); ++_visit_->size;; } if (_heap_mem_
) free(_incomings_); } while (0);; ((void) sizeof ((_visit_->
size <= (exec_info_size)) ? 1 : 0), __extension__ ({ if (_visit_
->size <= (exec_info_size)) ; else __assert_fail ("_visit_->size <= (exec_info_size)"
, "ccv_cnnp_model.c", 1975, __extension__ __PRETTY_FUNCTION__
); })); _visit_; });
1976 // Find any missing nodes to be added as source. Right now, these are only set nodes.
1977 ccv_nnc_graph_visit_for(visit, exec_info, node, idx){ int _i_; for (_i_ = 0; _i_ < (visit)->size; _i_++) { const
int idx __attribute__((unused)) = (visit)->node[_i_].index
; const int _node_unused_ __attribute__((unused)) = (visit)->
node[_i_].term; typeof ((exec_info)) const node __attribute__
((unused)) = (exec_info) + idx; {
1978 if (!(visited[(idx >> 5)] & (1u << (idx & 31))))
1979 {
1980 assert(exec_info[idx].cmd.cmd == CCV_NNC_SET_FORWARD)((void) sizeof ((exec_info[idx].cmd.cmd == CCV_NNC_SET_FORWARD
) ? 1 : 0), __extension__ ({ if (exec_info[idx].cmd.cmd == CCV_NNC_SET_FORWARD
) ; else __assert_fail ("exec_info[idx].cmd.cmd == CCV_NNC_SET_FORWARD"
, "ccv_cnnp_model.c", 1980, __extension__ __PRETTY_FUNCTION__
); }));
1981 if (exec_info[idx].cmd.info.blas.a[0] == 0) // Special-casing for empty out the tensor set function, not for the set grad to 1 one.
1982 ccv_array_add_unique_int(backward_from, idx);
1983 }
1984 } ccv_nnc_graph_visit_endfor} }
1985 ccv_nnc_graph_visit_free(visit);
1986 ccfreefree(visited);
1987 if (backward_from->rnum != compiled_data->backward.from_op_size) // If it doesn't match, need to redo this.
1988 {
1989 compiled_data->backward.from_op_size = backward_from->rnum;
1990 compiled_data->backward.from_ops = (ccv_nnc_graph_exec_t*)ccreallocrealloc(compiled_data->backward.from_ops, sizeof(ccv_nnc_graph_exec_t) * backward_from->rnum);
1991 for (i = 0; i < backward_from->rnum; i++)
1992 compiled_data->backward.from_ops[i] = (ccv_nnc_graph_exec_t){
1993 .d = *(int*)ccv_array_get(backward_from, i)((void*)(((char*)((backward_from)->data)) + (size_t)(backward_from
)->rsize * (size_t)(i))),
1994 .graph = compiled_data->graph,
1995 };
1996 }
1997 ccv_array_free(backward_from);
1998 ccv_nnc_graph_set_default_static_schedule(compiled_data->graph, compiled_data->stream_type, model->max_stream_count);
1999 ccv_nnc_graph_autotune(compiled_data->graph, model->workspace_size, 0, TRAVERSE_FULL0,0,0,0);
2000}
2001
2002void ccv_cnnp_model_dry_run(ccv_cnnp_model_t* const model, const ccv_cnnp_evaluate_param_t params, ccv_nnc_tensor_t* const* const inputs, const int input_size, ccv_nnc_tensor_t* const* const outputs, const int output_size)
2003{
2004 ccv_cnnp_compiled_data_t* const compiled_data = model->compiled_data;
2005 assert(compiled_data)((void) sizeof ((compiled_data) ? 1 : 0), __extension__ ({ if
(compiled_data) ; else __assert_fail ("compiled_data", "ccv_cnnp_model.c"
, 2005, __extension__ __PRETTY_FUNCTION__); }));
2006 const int parallel_count = ccv_max(model->parallel_count, 1)({ typeof (model->parallel_count) _a = (model->parallel_count
); typeof (1) _b = (1); (_a > _b) ? _a : _b; });
2007 assert(output_size == model->output_size * parallel_count)((void) sizeof ((output_size == model->output_size * parallel_count
) ? 1 : 0), __extension__ ({ if (output_size == model->output_size
* parallel_count) ; else __assert_fail ("output_size == model->output_size * parallel_count"
, "ccv_cnnp_model.c", 2007, __extension__ __PRETTY_FUNCTION__
); }));
2008 assert(input_size == model->input_size * parallel_count)((void) sizeof ((input_size == model->input_size * parallel_count
) ? 1 : 0), __extension__ ({ if (input_size == model->input_size
* parallel_count) ; else __assert_fail ("input_size == model->input_size * parallel_count"
, "ccv_cnnp_model.c", 2008, __extension__ __PRETTY_FUNCTION__
); }));
2009 assert(model->graph)((void) sizeof ((model->graph) ? 1 : 0), __extension__ ({ if
(model->graph) ; else __assert_fail ("model->graph", "ccv_cnnp_model.c"
, 2009, __extension__ __PRETTY_FUNCTION__); }));
2010 const int target_gradient_mode = _ccv_cnnp_is_disable_outgrad_all(params.disable_outgrad, model->input_size) ? CCV_CNNP_COMPILED_DATA_GRADIENT_TRAINABLES : CCV_CNNP_COMPILED_DATA_GRADIENT_TRAINABLES_AND_INPUTS;
2011 const int mode_mismatch = (params.requires_grad && (compiled_data->graph_mode != CCV_CNNP_MODEL_GRAPH_MULTISTAGE_MODE || compiled_data->gradient_mode != target_gradient_mode || compiled_data->disable_outgrad != params.disable_outgrad));
2012 if (!compiled_data->graph || mode_mismatch)
2013 {
2014 _ccv_cnnp_compiled_data_graph_free(compiled_data);
2015 if (mode_mismatch) // If mode mismatch, we need to redo the backward as well (no need to redo apply_gradients, it doesn't require target_gradient_mode or disable_outgrad.
2016 _ccv_cnnp_compiled_data_backward_free(compiled_data);
2017 if (params.requires_grad)
2018 _ccv_cnnp_model_multistage_jit_0(model, params.disable_outgrad, params.is_test, inputs, input_size, outputs, output_size);
2019 else
2020 _ccv_cnnp_model_multistage_no_grad_jit(model, inputs, input_size, outputs, output_size);
2021 } else {
2022 ccv_nnc_tensor_arena_clear_bindings(compiled_data->tensor_arena);
2023 assert((input_size % parallel_count) == 0)((void) sizeof (((input_size % parallel_count) == 0) ? 1 : 0)
, __extension__ ({ if ((input_size % parallel_count) == 0) ; else
__assert_fail ("(input_size % parallel_count) == 0", "ccv_cnnp_model.c"
, 2023, __extension__ __PRETTY_FUNCTION__); }));
2024 const int input_size_per_p = input_size / parallel_count;
2025 _ccv_cnnp_bind_tensors_to_arena(compiled_data->tensor_arena, model->graph, model->inputs, inputs, input_size_per_p, parallel_count);
2026 assert((output_size % parallel_count) == 0)((void) sizeof (((output_size % parallel_count) == 0) ? 1 : 0
), __extension__ ({ if ((output_size % parallel_count) == 0) ;
else __assert_fail ("(output_size % parallel_count) == 0", "ccv_cnnp_model.c"
, 2026, __extension__ __PRETTY_FUNCTION__); }));
2027 const int output_size_per_p = output_size / parallel_count;
2028 _ccv_cnnp_bind_tensors_to_arena(compiled_data->tensor_arena, model->graph, model->outputs, outputs, output_size_per_p, parallel_count);
2029 }
2030 if (compiled_data->is_test != params.is_test)
2031 {
2032 compiled_data->is_test = params.is_test;
2033 ccv_nnc_graph_exec_update_t update = {
2034 .parallel_count = parallel_count,
2035 .graph = model->graph,
2036 .graph_exec_arena = compiled_data->graph_exec_arena,
2037 };
2038 ccv_cnnp_model_set_is_test(model, params.is_test, _ccv_cnnp_cmd_update_for_execs, &update);
2039 }
2040}
2041
2042void ccv_cnnp_model_evaluate(ccv_cnnp_model_t* const model, const ccv_cnnp_evaluate_param_t params, ccv_nnc_tensor_t* const* const inputs, const int input_size, ccv_nnc_tensor_t* const* const outputs, const int output_size, ccv_nnc_tensor_tape_t* const tensor_tape, ccv_nnc_stream_context_t* const stream_context)
2043{
2044 ccv_cnnp_compiled_data_t* const compiled_data = model->compiled_data;
2045 assert(compiled_data)((void) sizeof ((compiled_data) ? 1 : 0), __extension__ ({ if
(compiled_data) ; else __assert_fail ("compiled_data", "ccv_cnnp_model.c"
, 2045, __extension__ __PRETTY_FUNCTION__); }));
2046 ccv_cnnp_model_dry_run(model, params, inputs, input_size, outputs, output_size);
2047 if (compiled_data->graph_mode == CCV_CNNP_MODEL_GRAPH_MULTISTAGE_MODE_NO_GRAD)
2048 ccv_nnc_graph_run_with_schedule(compiled_data->graph, 0, 0, tensor_tape, stream_context);
2049 else {
2050 if (!compiled_data->evaluate.schedule)
2051 compiled_data->evaluate.schedule = ccv_nnc_graph_static_schedule_new(compiled_data->graph, compiled_data->stream_type, model->max_stream_count, 0, 0, compiled_data->evaluate.to_ops, compiled_data->evaluate.to_op_size);
2052 ccv_nnc_graph_run_with_schedule(compiled_data->graph, 0, compiled_data->evaluate.schedule, tensor_tape, stream_context);
2053 }
2054}
2055
2056// Compile the graph to run ccv_cnnp_model_backward after ccv_cnnp_model_evaluate with requires_grad = true (MULTISTAGE_MODE).
2057// Particularly, this method compiles the accumulator graph.
2058static void _ccv_cnnp_model_multistage_jit_1(ccv_cnnp_model_t* const model)
2059{
2060 ccv_cnnp_compiled_data_t* const compiled_data = model->compiled_data;
2061 assert(compiled_data)((void) sizeof ((compiled_data) ? 1 : 0), __extension__ ({ if
(compiled_data) ; else __assert_fail ("compiled_data", "ccv_cnnp_model.c"
, 2061, __extension__ __PRETTY_FUNCTION__); }));
2062 assert(compiled_data->graph_mode == CCV_CNNP_MODEL_GRAPH_MULTISTAGE_MODE)((void) sizeof ((compiled_data->graph_mode == CCV_CNNP_MODEL_GRAPH_MULTISTAGE_MODE
) ? 1 : 0), __extension__ ({ if (compiled_data->graph_mode
== CCV_CNNP_MODEL_GRAPH_MULTISTAGE_MODE) ; else __assert_fail
("compiled_data->graph_mode == CCV_CNNP_MODEL_GRAPH_MULTISTAGE_MODE"
, "ccv_cnnp_model.c", 2062, __extension__ __PRETTY_FUNCTION__
); }));
2063 ccv_nnc_symbolic_graph_t* accum = ccv_nnc_symbolic_graph_new();
2064 const int parallel_count = ccv_max(model->parallel_count, 1)({ typeof (model->parallel_count) _a = (model->parallel_count
); typeof (1) _b = (1); (_a > _b) ? _a : _b; });
2065 const int parameter_size = compiled_data->parameters->rnum;
2066 int i, j;
2067 compiled_data->backward.gradients = (ccv_nnc_tensor_symbol_t*)ccmallocmalloc(sizeof(ccv_nnc_tensor_symbol_t) * parameter_size * parallel_count * 3);
2068 compiled_data->backward.accum_gradients = compiled_data->backward.gradients + parameter_size * parallel_count;
2069 compiled_data->backward.updated_accum_gradients = compiled_data->backward.accum_gradients + parameter_size * parallel_count;
2070 for (i = 0; i < parameter_size; i++)
2071 for (j = 0; j < parallel_count; j++)
2072 if (compiled_data->tensors.gradients[i + j * parameter_size])
2073 {
2074 const ccv_nnc_tensor_param_t info = compiled_data->tensors.gradients[i + j * parameter_size]->info;
2075 // Now, the old gradient is the accumulated gradient, getting new gradient tensor setup so we can collect them.
2076 compiled_data->tensors.accum_gradients[i + j * parameter_size] = compiled_data->tensors.gradients[i + j * parameter_size];
2077 compiled_data->tensors.gradients[i + j * parameter_size] = ccv_nnc_tensor_new(0, info, 0);
2078 ccv_nnc_tensor_symbol_t inputs[2];
2079 inputs[0] = compiled_data->backward.accum_gradients[i + j * parameter_size] = ccv_nnc_tensor_symbol_new(accum, info, 0);
2080 inputs[1] = compiled_data->backward.gradients[i + j * parameter_size] = ccv_nnc_tensor_symbol_new(accum, info, 0);
2081 ccv_nnc_tensor_symbol_t output = compiled_data->backward.updated_accum_gradients[i + j * parameter_size] = ccv_nnc_tensor_symbol_new(accum, info, 0);
2082 ccv_nnc_graph_exec_symbol_new(accum, CMD_EWSUM_FORWARD()ccv_nnc_cmd(CCV_NNC_EWSUM_FORWARD, 0, ccv_nnc_cmd_auto, 0), inputs, 2, &output, 1, 0);
2083 } else {
2084 compiled_data->backward.accum_gradients[i + j * parameter_size] = NO_TENSOR_SYMBOL(const ccv_nnc_tensor_symbol_t){.d = CCV_NNC_NO_TENSOR_SYMBOL
};
2085 compiled_data->backward.gradients[i + j * parameter_size] = NO_TENSOR_SYMBOL(const ccv_nnc_tensor_symbol_t){.d = CCV_NNC_NO_TENSOR_SYMBOL
};
2086 compiled_data->backward.updated_accum_gradients[i + j * parameter_size] = NO_TENSOR_SYMBOL(const ccv_nnc_tensor_symbol_t){.d = CCV_NNC_NO_TENSOR_SYMBOL
};
2087 }
2088 ccv_nnc_graph_exec_symbol_autogen(accum, 0, 0, CCV_NNC_AUTOGEN_ALL_EXECS | CCV_NNC_AUTOGEN_SOURCES_AND_DESTINATIONS);
2089 if (ccv_nnc_symbolic_graph_source_size(accum) == 0)
2090 {
2091 ccv_nnc_symbolic_graph_free(accum);
2092 // Create empty graph.
2093 compiled_data->backward.accum = ccv_nnc_graph_new();
2094 ccv_nnc_graph_topsort(compiled_data->backward.accum, 0, 0);
2095 return;
2096 }
2097 ccv_array_t* const tensor_binds = ccv_array_new(sizeof(ccv_nnc_tensor_bind_t), 0, 0);
2098 _ccv_cnnp_model_bind_tensors(accum, compiled_data->backward.accum_gradients, compiled_data->tensors.accum_gradients, parameter_size * parallel_count, 1, tensor_binds);
2099 _ccv_cnnp_model_bind_tensors(accum, compiled_data->backward.gradients, compiled_data->tensors.gradients, parameter_size * parallel_count, 1, tensor_binds);
2100 _ccv_cnnp_model_bind_tensors(accum, compiled_data->backward.updated_accum_gradients, compiled_data->tensors.accum_gradients, parameter_size * parallel_count, 1, tensor_binds);
2101 ccv_nnc_symbolic_graph_compile(accum, compiled_data->compile_params, (ccv_nnc_tensor_bind_t*)ccv_array_get(tensor_binds, 0)((void*)(((char*)((tensor_binds)->data)) + (size_t)(tensor_binds
)->rsize * (size_t)(0))), tensor_binds->rnum, 0, 0, SYMBOLIC_GRAPH_SOURCES(accum)ccv_nnc_symbolic_graph_sources(accum), ccv_nnc_symbolic_graph_source_size
(accum), SYMBOLIC_GRAPH_DESTINATIONS(accum)ccv_nnc_symbolic_graph_destinations(accum), ccv_nnc_symbolic_graph_destination_size
(accum), &compiled_data->backward.accum, &compiled_data->backward.tensor_arena, &compiled_data->backward.graph_exec_arena);
2102 ccv_nnc_symbolic_graph_free(accum);
2103 ccv_array_free(tensor_binds);
2104 ccv_nnc_graph_set_default_static_schedule(compiled_data->backward.accum, compiled_data->stream_type, model->max_stream_count);
2105}
2106
2107void ccv_cnnp_model_backward(ccv_cnnp_model_t* const model, ccv_nnc_tensor_t* const* const ingrads, const int ingrad_size, ccv_nnc_tensor_t* const* const outgrads, const int outgrad_size, ccv_nnc_tensor_tape_t* const tensor_tape, ccv_nnc_stream_context_t* const stream_context)
2108{
2109 ccv_cnnp_compiled_data_t* const compiled_data = model->compiled_data;
2110 assert(compiled_data)((void) sizeof ((compiled_data) ? 1 : 0), __extension__ ({ if
(compiled_data) ; else __assert_fail ("compiled_data", "ccv_cnnp_model.c"
, 2110, __extension__ __PRETTY_FUNCTION__); }));
2111 assert(compiled_data->graph_mode == CCV_CNNP_MODEL_GRAPH_MULTISTAGE_MODE)((void) sizeof ((compiled_data->graph_mode == CCV_CNNP_MODEL_GRAPH_MULTISTAGE_MODE
) ? 1 : 0), __extension__ ({ if (compiled_data->graph_mode
== CCV_CNNP_MODEL_GRAPH_MULTISTAGE_MODE) ; else __assert_fail
("compiled_data->graph_mode == CCV_CNNP_MODEL_GRAPH_MULTISTAGE_MODE"
, "ccv_cnnp_model.c", 2111, __extension__ __PRETTY_FUNCTION__
); }));
2112 const int parallel_count = ccv_max(model->parallel_count, 1)({ typeof (model->parallel_count) _a = (model->parallel_count
); typeof (1) _b = (1); (_a > _b) ? _a : _b; });
2113 assert(ingrad_size == 0 || ingrad_size == model->output_size * parallel_count)((void) sizeof ((ingrad_size == 0 || ingrad_size == model->
output_size * parallel_count) ? 1 : 0), __extension__ ({ if (
ingrad_size == 0 || ingrad_size == model->output_size * parallel_count
) ; else __assert_fail ("ingrad_size == 0 || ingrad_size == model->output_size * parallel_count"
, "ccv_cnnp_model.c", 2113, __extension__ __PRETTY_FUNCTION__
); }));
2114 if (outgrad_size > 0)
2115 { assert(outgrad_size == compiled_data->outgrad_size * parallel_count)((void) sizeof ((outgrad_size == compiled_data->outgrad_size
* parallel_count) ? 1 : 0), __extension__ ({ if (outgrad_size
== compiled_data->outgrad_size * parallel_count) ; else __assert_fail
("outgrad_size == compiled_data->outgrad_size * parallel_count"
, "ccv_cnnp_model.c", 2115, __extension__ __PRETTY_FUNCTION__
); })); }
2116 assert(model->graph)((void) sizeof ((model->graph) ? 1 : 0), __extension__ ({ if
(model->graph) ; else __assert_fail ("model->graph", "ccv_cnnp_model.c"
, 2116, __extension__ __PRETTY_FUNCTION__); }));
2117 assert(compiled_data->graph)((void) sizeof ((compiled_data->graph) ? 1 : 0), __extension__
({ if (compiled_data->graph) ; else __assert_fail ("compiled_data->graph"
, "ccv_cnnp_model.c", 2117, __extension__ __PRETTY_FUNCTION__
); }));
2118 const int parameter_size = compiled_data->parameters->rnum;
2119 // If we need to accumulate the gradients now, do jit on accumulator.
2120 if (compiled_data->backward.count > 0)
2121 {
2122 if (!compiled_data->backward.accum)
2123 _ccv_cnnp_model_multistage_jit_1(model);
2124 else if (compiled_data->backward.count == 1) {
2125 // On this round, we need to switch accumulated gradients with gradients (so we can do accumulation properly).
2126 int i;
2127 for (i = 0; i < parameter_size * parallel_count; i++)
2128 {
2129 ccv_nnc_tensor_t* tensor;
2130 CCV_SWAP(compiled_data->tensors.accum_gradients[i], compiled_data->tensors.gradients[i], tensor)((tensor) = (compiled_data->tensors.accum_gradients[i]), (
compiled_data->tensors.accum_gradients[i]) = (compiled_data
->tensors.gradients[i]), (compiled_data->tensors.gradients
[i]) = (tensor));
2131 }
2132 if (compiled_data->backward.tensor_arena)
2133 {
2134 ccv_nnc_tensor_arena_clear_bindings(compiled_data->backward.tensor_arena);
2135 // Do rebind in case we messed up the binding (we switch accum_gradients and gradients).
2136 _ccv_cnnp_bind_tensors_to_arena(compiled_data->backward.tensor_arena, 0, compiled_data->backward.gradients, compiled_data->tensors.gradients, parameter_size * parallel_count, 1);
2137 _ccv_cnnp_bind_tensors_to_arena(compiled_data->backward.tensor_arena, 0, compiled_data->backward.accum_gradients, compiled_data->tensors.accum_gradients, parameter_size * parallel_count, 1);
2138 _ccv_cnnp_bind_tensors_to_arena(compiled_data->backward.tensor_arena, 0, compiled_data->backward.updated_accum_gradients, compiled_data->tensors.accum_gradients, parameter_size * parallel_count, 1);
2139 }
2140 }
2141 }
2142 const int ingrad_size_per_p = model->output_size;
2143 const int outgrad_size_per_p = compiled_data->outgrad_size;
2144 int i, j;
2145 for (i = 0; i < ingrad_size_per_p; i++)
2146 {
2147 const ccv_nnc_tensor_symbol_t ingrad = ccv_nnc_tensor_symbol_for_backward(model->graph, compiled_data->f[i]);
2148 if (!ingrad_size || !ingrads || ingrads[i] == 0)
2149 {
2150 // Set it to 1 if it is not specified.
2151 ccv_nnc_tensor_t* const ingrad_tensor = ccv_nnc_tensor_from_symbol(compiled_data->tensor_arena, ingrad);
2152 if (ingrad_tensor)
2153 ccv_nnc_cmd_exec(CMD_SET_FORWARD(1)ccv_nnc_cmd(CCV_NNC_SET_FORWARD, 0, (ccv_nnc_cmd_param_t){.size
={.dim={1,1,1}},.blas={.a={1,}}}, 0), ccv_nnc_no_hint, 0, 0, 0, TENSOR_LIST(ingrad_tensor)(ccv_nnc_tensor_t* []){ingrad_tensor}, (1 +1 +0 +0 +0 +0 +0 +
0 +0 +0 +0 +0 +0 +0 +0 +0 +0 +0 +0 +0 +0 +0 +0 +0 +0 +0 +0 +0
+0 +0 +0 +0 +0 +0 +0 +0 +0 +0 +0 +0 +0 +0 +0 +0 +0 +0 +0 +0 +
0 +0 +0 +0 +0 +0 +0 +0 +0 +0 +0 +0 +0 +0 +0 +0 -1), stream_context);
2154 for (j = 1; j < parallel_count; j++)
2155 {
2156 ccv_nnc_tensor_t* const ingrad_tensor = ccv_nnc_tensor_from_symbol(compiled_data->tensor_arena, ccv_nnc_tensor_symbol_copy(model->graph, ingrad, j));
2157 if (ingrad_tensor)
2158 ccv_nnc_cmd_exec(CMD_SET_FORWARD(1)ccv_nnc_cmd(CCV_NNC_SET_FORWARD, 0, (ccv_nnc_cmd_param_t){.size
={.dim={1,1,1}},.blas={.a={1,}}}, 0), ccv_nnc_no_hint, 0, 0, 0, TENSOR_LIST(ingrad_tensor)(ccv_nnc_tensor_t* []){ingrad_tensor}, (1 +1 +0 +0 +0 +0 +0 +
0 +0 +0 +0 +0 +0 +0 +0 +0 +0 +0 +0 +0 +0 +0 +0 +0 +0 +0 +0 +0
+0 +0 +0 +0 +0 +0 +0 +0 +0 +0 +0 +0 +0 +0 +0 +0 +0 +0 +0 +0 +
0 +0 +0 +0 +0 +0 +0 +0 +0 +0 +0 +0 +0 +0 +0 +0 -1), stream_context);
2159 }
2160 } else {
2161 // Make sure the length matches, in case it is an alias.
2162 assert(ccv_nnc_tensor_count(ingrads[i]->info) == ccv_nnc_tensor_count(ccv_nnc_tensor_symbol_params(model->graph, ingrad)))((void) sizeof ((ccv_nnc_tensor_count(ingrads[i]->info) ==
ccv_nnc_tensor_count(ccv_nnc_tensor_symbol_params(model->
graph, ingrad))) ? 1 : 0), __extension__ ({ if (ccv_nnc_tensor_count
(ingrads[i]->info) == ccv_nnc_tensor_count(ccv_nnc_tensor_symbol_params
(model->graph, ingrad))) ; else __assert_fail ("ccv_nnc_tensor_count(ingrads[i]->info) == ccv_nnc_tensor_count(ccv_nnc_tensor_symbol_params(model->graph, ingrad))"
, "ccv_cnnp_model.c", 2162, __extension__ __PRETTY_FUNCTION__
); }));
2163 ccv_nnc_tensor_bind_symbol(compiled_data->tensor_arena, ingrad, ingrads[i]);
2164 for (j = 1; j < parallel_count; j++)
2165 ccv_nnc_tensor_bind_symbol(compiled_data->tensor_arena, ccv_nnc_tensor_symbol_copy(model->graph, ingrad, j), ingrads[i + ingrad_size_per_p * j]);
2166 }
2167 }
2168 if (outgrad_size > 0)
2169 {
2170 assert(compiled_data->gradient_mode == CCV_CNNP_COMPILED_DATA_GRADIENT_TRAINABLES_AND_INPUTS && "shouldn't pass disable_outgrad to ccv_cnnp_model_evaluate before if you plan to compute outgrad")((void) sizeof ((compiled_data->gradient_mode == CCV_CNNP_COMPILED_DATA_GRADIENT_TRAINABLES_AND_INPUTS
&& "shouldn't pass disable_outgrad to ccv_cnnp_model_evaluate before if you plan to compute outgrad"
) ? 1 : 0), __extension__ ({ if (compiled_data->gradient_mode
== CCV_CNNP_COMPILED_DATA_GRADIENT_TRAINABLES_AND_INPUTS &&
"shouldn't pass disable_outgrad to ccv_cnnp_model_evaluate before if you plan to compute outgrad"
) ; else __assert_fail ("compiled_data->gradient_mode == CCV_CNNP_COMPILED_DATA_GRADIENT_TRAINABLES_AND_INPUTS && \"shouldn't pass disable_outgrad to ccv_cnnp_model_evaluate before if you plan to compute outgrad\""
, "ccv_cnnp_model.c", 2170, __extension__ __PRETTY_FUNCTION__
); }));
2171 for (i = 0; i < outgrad_size_per_p; i++)
2172 if (outgrads[i])
2173 {
2174 const ccv_nnc_tensor_symbol_t outgrad = compiled_data->outgrads[i];
2175 ccv_nnc_tensor_bind_symbol(compiled_data->tensor_arena, outgrad, outgrads[i]);
2176 for (j = 1; j < parallel_count; j++)
2177 ccv_nnc_tensor_bind_symbol(compiled_data->tensor_arena, ccv_nnc_tensor_symbol_copy(model->graph, outgrad, j), outgrads[i + outgrad_size_per_p * j]);
2178 }
2179 } else {
2180 assert(compiled_data->gradient_mode == CCV_CNNP_COMPILED_DATA_GRADIENT_TRAINABLES ||((void) sizeof ((compiled_data->gradient_mode == CCV_CNNP_COMPILED_DATA_GRADIENT_TRAINABLES
|| compiled_data->gradient_mode == CCV_CNNP_COMPILED_DATA_GRADIENT_TRAINABLES_AND_INPUTS
) ? 1 : 0), __extension__ ({ if (compiled_data->gradient_mode
== CCV_CNNP_COMPILED_DATA_GRADIENT_TRAINABLES || compiled_data
->gradient_mode == CCV_CNNP_COMPILED_DATA_GRADIENT_TRAINABLES_AND_INPUTS
) ; else __assert_fail ("compiled_data->gradient_mode == CCV_CNNP_COMPILED_DATA_GRADIENT_TRAINABLES || compiled_data->gradient_mode == CCV_CNNP_COMPILED_DATA_GRADIENT_TRAINABLES_AND_INPUTS"
, "ccv_cnnp_model.c", 2181, __extension__ __PRETTY_FUNCTION__
); }))
2181 compiled_data->gradient_mode == CCV_CNNP_COMPILED_DATA_GRADIENT_TRAINABLES_AND_INPUTS)((void) sizeof ((compiled_data->gradient_mode == CCV_CNNP_COMPILED_DATA_GRADIENT_TRAINABLES
|| compiled_data->gradient_mode == CCV_CNNP_COMPILED_DATA_GRADIENT_TRAINABLES_AND_INPUTS
) ? 1 : 0), __extension__ ({ if (compiled_data->gradient_mode
== CCV_CNNP_COMPILED_DATA_GRADIENT_TRAINABLES || compiled_data
->gradient_mode == CCV_CNNP_COMPILED_DATA_GRADIENT_TRAINABLES_AND_INPUTS
) ; else __assert_fail ("compiled_data->gradient_mode == CCV_CNNP_COMPILED_DATA_GRADIENT_TRAINABLES || compiled_data->gradient_mode == CCV_CNNP_COMPILED_DATA_GRADIENT_TRAINABLES_AND_INPUTS"
, "ccv_cnnp_model.c", 2181, __extension__ __PRETTY_FUNCTION__
); }));
2182 }
2183 // We need to rebind here because in ccv_cnnp_evaluate, we clear bindings, that will reset all bindings for the gradients.
2184 // For parameters and internals these are fine because when we clear bindings, it restores to original bindings, which are these
2185 // parameters and internals. The same cannot be said for gradients due to the accum_gradients switching.
2186 _ccv_cnnp_bind_tensors_to_arena(compiled_data->tensor_arena, model->graph, compiled_data->gradients, compiled_data->tensors.gradients, parameter_size, parallel_count);
2187 if (!compiled_data->backward.schedule)
2188 compiled_data->backward.schedule = ccv_nnc_graph_static_schedule_new(compiled_data->graph, compiled_data->stream_type, model->max_stream_count, compiled_data->backward.from_ops, compiled_data->backward.from_op_size, 0, 0);
2189 // Run the backward pass.
2190 ccv_nnc_graph_run_with_schedule(compiled_data->graph, 0, compiled_data->backward.schedule, tensor_tape, stream_context);
2191 // If we need to run accumulation round, do that now.
2192 if (compiled_data->backward.count > 0)
2193 ccv_nnc_graph_run_with_schedule(compiled_data->backward.accum, 0, 0, 0, stream_context);
2194 // Update the count, this determines whether we need to accumulate or not.
2195 ++compiled_data->backward.count;
2196}
2197
2198// Compile the graph to run ccv_cnnp_model_apply_gradients after ccv_cnnp_model_backward (MULTISTAGE_MODE).
2199// Particularly, this method compiles the parameter update graph.
2200static void _ccv_cnnp_model_multistage_jit_2(ccv_cnnp_model_t* const model)
2201{
2202 ccv_cnnp_compiled_data_t* const compiled_data = model->compiled_data;
2203 assert(compiled_data->graph_mode == CCV_CNNP_MODEL_GRAPH_MULTISTAGE_MODE)((void) sizeof ((compiled_data->graph_mode == CCV_CNNP_MODEL_GRAPH_MULTISTAGE_MODE
) ? 1 : 0), __extension__ ({ if (compiled_data->graph_mode
== CCV_CNNP_MODEL_GRAPH_MULTISTAGE_MODE) ; else __assert_fail
("compiled_data->graph_mode == CCV_CNNP_MODEL_GRAPH_MULTISTAGE_MODE"
, "ccv_cnnp_model.c", 2203, __extension__ __PRETTY_FUNCTION__
); }));
2204 const int parallel_count = ccv_max(model->parallel_count, 1)({ typeof (model->parallel_count) _a = (model->parallel_count
); typeof (1) _b = (1); (_a > _b) ? _a : _b; });
2205 const int parameter_size = compiled_data->parameters->rnum;
2206 ccv_array_t* const tensor_binds = ccv_array_new(sizeof(ccv_nnc_tensor_bind_t), 0, 0);
2207 _ccv_cnnp_model_bind_tensors(model->graph, (ccv_nnc_tensor_symbol_t*)ccv_array_get(compiled_data->parameters, 0)((void*)(((char*)((compiled_data->parameters)->data)) +
(size_t)(compiled_data->parameters)->rsize * (size_t)(
0))), compiled_data->tensors.parameters, parameter_size, parallel_count, tensor_binds);
2208 _ccv_cnnp_model_bind_tensors(model->graph, compiled_data->updated_parameters, compiled_data->tensors.parameters, parameter_size, parallel_count, tensor_binds);
2209 // Bind accumulated gradients.
2210 if (compiled_data->backward.count > 1)
2211 _ccv_cnnp_model_bind_tensors(model->graph, compiled_data->gradients, compiled_data->tensors.accum_gradients, parameter_size, parallel_count, tensor_binds);
2212 else
2213 _ccv_cnnp_model_bind_tensors(model->graph, compiled_data->gradients, compiled_data->tensors.gradients, parameter_size, parallel_count, tensor_binds);
2214 ccv_array_t* const apply_gradients_from = ccv_array_new(sizeof(int), 0, 0);
2215 int i, j;
2216 for (i = 0; i < compiled_data->backward.to_size; i++)
2217 {
2218 const int* tos;
2219 int to_size;
2220 ccv_nnc_graph_exec_symbol_to(model->graph, compiled_data->backward.tos[i], &tos, &to_size);
2221 for (j = 0; j < to_size; j++)
2222 {
2223 // Check if this is already show up in the backward graph, if that is the case, it won't be in the apply
2224 // gradients graph.
2225 const ccv_nnc_graph_exec_t exec = ccv_nnc_graph_exec_from_symbol(compiled_data->graph_exec_arena, (ccv_nnc_graph_exec_symbol_t){
2226 .d = tos[j],
2227 .graph = model->graph,
2228 });
2229 if (!exec.graph)
2230 ccv_array_add_unique_int(apply_gradients_from, tos[j]);
2231 }
2232 }
2233 const int from_size = apply_gradients_from->rnum;
2234 if (from_size == 0)
2235 {
2236 ccv_array_free(apply_gradients_from);
2237 ccv_array_free(tensor_binds);
2238 return;
2239 }
2240 ccv_nnc_graph_exec_symbol_t* const froms = (ccv_nnc_graph_exec_symbol_t*)ccmallocmalloc(sizeof(ccv_nnc_graph_exec_symbol_t) * from_size);
2241 for (i = 0; i < from_size; i++)
2242 froms[i] = (ccv_nnc_graph_exec_symbol_t){
2243 .d = *(int*)ccv_array_get(apply_gradients_from, i)((void*)(((char*)((apply_gradients_from)->data)) + (size_t
)(apply_gradients_from)->rsize * (size_t)(i))),
2244 .graph = model->graph
2245 };
2246 ccv_array_free(apply_gradients_from);
2247 // It can only ends with updates on the parameters.
2248 ccv_array_t* const tos = ccv_array_new(sizeof(ccv_nnc_graph_exec_symbol_t), parameter_size * parallel_count, 0);
2249 for (i = 0; i < parameter_size; i++)
2250 {
2251 if (compiled_data->update_nodes[i].d == CCV_NNC_NO_TENSOR_SYMBOL)
2252 continue;
2253 ccv_array_push(tos, &compiled_data->update_nodes[i]);
2254 for (j = 1; j < parallel_count; j++)
2255 {
2256 const ccv_nnc_graph_exec_symbol_t copy = ccv_nnc_graph_exec_symbol_copy(model->graph, compiled_data->update_nodes[i], j);
2257 ccv_array_push(tos, &copy);
2258 }
2259 }
2260 ccv_nnc_symbolic_graph_compile(model->graph, compiled_data->compile_params, (ccv_nnc_tensor_bind_t*)ccv_array_get(tensor_binds, 0)((void*)(((char*)((tensor_binds)->data)) + (size_t)(tensor_binds
)->rsize * (size_t)(0))), tensor_binds->rnum, 0, 0, froms, from_size, (ccv_nnc_graph_exec_symbol_t*)ccv_array_get(tos, 0)((void*)(((char*)((tos)->data)) + (size_t)(tos)->rsize *
(size_t)(0))), tos->rnum, &compiled_data->apply_gradients.graph, &compiled_data->apply_gradients.tensor_arena, &compiled_data->apply_gradients.graph_exec_arena);
2261 ccv_array_free(tos);
2262 ccv_array_free(tensor_binds);
2263 ccfreefree(froms);
2264 const int max_saved_aux_size = compiled_data->minimize.max_saved_aux_size;
2265 for (i = 0; i < max_saved_aux_size * parameter_size; i++)
2266 {
2267 // Skip on no tensor.
2268 if (compiled_data->saved_aux[i].source.d == CCV_NNC_NO_TENSOR_SYMBOL)
2269 continue;
2270 ccv_nnc_tensor_t* const tensor = ccv_nnc_tensor_from_symbol(compiled_data->apply_gradients.tensor_arena, compiled_data->saved_aux[i].source);
2271 ccv_nnc_cmd_exec(CMD_SET_FORWARD(0)ccv_nnc_cmd(CCV_NNC_SET_FORWARD, 0, (ccv_nnc_cmd_param_t){.size
={.dim={1,1,1}},.blas={.a={0,}}}, 0), ccv_nnc_no_hint, 0, 0, 0, &tensor, 1, 0);
2272 for (j = 1; j < parallel_count; j++)
2273 {
2274 ccv_nnc_tensor_t* const copy = ccv_nnc_tensor_from_symbol(compiled_data->apply_gradients.tensor_arena, ccv_nnc_tensor_symbol_copy(model->graph, compiled_data->saved_aux[i].source, j));
2275 if (copy)
2276 ccv_nnc_cmd_exec(CMD_SET_FORWARD(0)ccv_nnc_cmd(CCV_NNC_SET_FORWARD, 0, (ccv_nnc_cmd_param_t){.size
={.dim={1,1,1}},.blas={.a={0,}}}, 0), ccv_nnc_no_hint, 0, 0, 0, &copy, 1, 0);
2277 }
2278 }
2279 ccv_nnc_graph_set_default_static_schedule(compiled_data->apply_gradients.graph, compiled_data->stream_type, model->max_stream_count);
2280}
2281
2282void ccv_cnnp_model_apply_gradients(ccv_cnnp_model_t* const model, ccv_nnc_stream_context_t* const stream_context)
2283{
2284 ccv_cnnp_compiled_data_t* const compiled_data = model->compiled_data;
2285 assert(compiled_data)((void) sizeof ((compiled_data) ? 1 : 0), __extension__ ({ if
(compiled_data) ; else __assert_fail ("compiled_data", "ccv_cnnp_model.c"
, 2285, __extension__ __PRETTY_FUNCTION__); }));
2286 assert(compiled_data->graph_mode == CCV_CNNP_MODEL_GRAPH_MULTISTAGE_MODE)((void) sizeof ((compiled_data->graph_mode == CCV_CNNP_MODEL_GRAPH_MULTISTAGE_MODE
) ? 1 : 0), __extension__ ({ if (compiled_data->graph_mode
== CCV_CNNP_MODEL_GRAPH_MULTISTAGE_MODE) ; else __assert_fail
("compiled_data->graph_mode == CCV_CNNP_MODEL_GRAPH_MULTISTAGE_MODE"
, "ccv_cnnp_model.c", 2286, __extension__ __PRETTY_FUNCTION__
); }));
2287 const int parallel_count = ccv_max(model->parallel_count, 1)({ typeof (model->parallel_count) _a = (model->parallel_count
); typeof (1) _b = (1); (_a > _b) ? _a : _b; });
2288 assert(model->graph)((void) sizeof ((model->graph) ? 1 : 0), __extension__ ({ if
(model->graph) ; else __assert_fail ("model->graph", "ccv_cnnp_model.c"
, 2288, __extension__ __PRETTY_FUNCTION__); }));
2289 assert(compiled_data->graph)((void) sizeof ((compiled_data->graph) ? 1 : 0), __extension__
({ if (compiled_data->graph) ; else __assert_fail ("compiled_data->graph"
, "ccv_cnnp_model.c", 2289, __extension__ __PRETTY_FUNCTION__
); }));
2290 // Skip if there is no backward pass.
2291 if (compiled_data->backward.count <= 0)
2292 return;
2293 // Skip if there is no parameters.
2294 if (compiled_data->parameters->rnum == 0)
2295 {
2296 compiled_data->backward.count = 0;
2297 return;
2298 }
2299 if (!compiled_data->apply_gradients.graph)
2300 _ccv_cnnp_model_multistage_jit_2(model);
2301 else {
2302 const int parameter_size = compiled_data->parameters->rnum;
2303 ccv_nnc_tensor_arena_clear_bindings(compiled_data->apply_gradients.tensor_arena);
2304 // Change to bind accum_gradients if we do gradient accumulation (run backward more than once).
2305 if (compiled_data->backward.count > 1)
2306 _ccv_cnnp_bind_tensors_to_arena(compiled_data->apply_gradients.tensor_arena, model->graph, compiled_data->gradients, compiled_data->tensors.accum_gradients, parameter_size, parallel_count);
2307 else
2308 _ccv_cnnp_bind_tensors_to_arena(compiled_data->apply_gradients.tensor_arena, model->graph, compiled_data->gradients, compiled_data->tensors.gradients, parameter_size, parallel_count);
2309 }
2310 if (compiled_data->apply_gradients.graph)
2311 ccv_nnc_graph_run_with_schedule(compiled_data->apply_gradients.graph, 0, 0, 0, stream_context);
2312 // Reset backward count to 0.
2313 compiled_data->backward.count = 0;
2314}
2315
2316void ccv_cnnp_model_set_parameter(ccv_cnnp_model_t* const model, const ccv_cnnp_model_io_t parameter, const ccv_nnc_tensor_t* const tensor)
2317{
2318 ccv_cnnp_compiled_data_t* const compiled_data = model->compiled_data;
2319 const int param_sel = parameter->param_sel > 0 ? parameter->param_sel - 1 : parameter->param_sel;
2320 assert(parameter->param_sel != 0)((void) sizeof ((parameter->param_sel != 0) ? 1 : 0), __extension__
({ if (parameter->param_sel != 0) ; else __assert_fail ("parameter->param_sel != 0"
, "ccv_cnnp_model.c", 2320, __extension__ __PRETTY_FUNCTION__
); }));
2321 const int tensors_init = !!compiled_data->tensors_init.v;
2322 int this_tensor_init = tensors_init;
2323 if (!tensors_init)
2324 ccv_cnnp_model_tensors_init_0(model, compiled_data);
2325 else if ((uintptr_t)compiled_data->tensors_init.v & (uintptr_t)1)
2326 // Check if it is not fully allocated, if it is not, init_1.
2327 this_tensor_init = 0;
2328 ccv_array_t* const parameter_indices = ccv_array_new(sizeof(int), 0, 0);
2329 ccv_cnnp_model_add_to_parameter_indices(parameter->model, param_sel, parameter_indices);
2330 const int param_ref = parameter->param_ref > 0 ? parameter->param_ref - 1 : parameter->param_ref;
2331 if (param_ref < 0)
2332 { assert(parameter_indices->rnum == 1)((void) sizeof ((parameter_indices->rnum == 1) ? 1 : 0), __extension__
({ if (parameter_indices->rnum == 1) ; else __assert_fail
("parameter_indices->rnum == 1", "ccv_cnnp_model.c", 2332
, __extension__ __PRETTY_FUNCTION__); })); }
2333 else
2334 { assert(param_ref < parameter_indices->rnum)((void) sizeof ((param_ref < parameter_indices->rnum) ?
1 : 0), __extension__ ({ if (param_ref < parameter_indices
->rnum) ; else __assert_fail ("param_ref < parameter_indices->rnum"
, "ccv_cnnp_model.c", 2334, __extension__ __PRETTY_FUNCTION__
); })); }
2335 const int d = *(int*)ccv_array_get(parameter_indices, param_ref >= 0 ? param_ref : 0)((void*)(((char*)((parameter_indices)->data)) + (size_t)(parameter_indices
)->rsize * (size_t)(param_ref >= 0 ? param_ref : 0)));
2336 ccv_array_free(parameter_indices);
2337 const int parameter_size = compiled_data->parameters->rnum;
2338 assert(d >= 0)((void) sizeof ((d >= 0) ? 1 : 0), __extension__ ({ if (d >=
0) ; else __assert_fail ("d >= 0", "ccv_cnnp_model.c", 2338
, __extension__ __PRETTY_FUNCTION__); }));
2339 assert(d < parameter_size)((void) sizeof ((d < parameter_size) ? 1 : 0), __extension__
({ if (d < parameter_size) ; else __assert_fail ("d < parameter_size"
, "ccv_cnnp_model.c", 2339, __extension__ __PRETTY_FUNCTION__
); }));
2340 const int parallel_count = _ccv_cnnp_compiled_data_parallel_count(model, compiled_data);
2341 int i;
2342 if (!this_tensor_init)
2343 {
2344 if (compiled_data->tensors.parameters[d])
2345 {
2346 for (i = 1; i < parallel_count; i++)
2347 { assert(compiled_data->tensors.parameters[d + i * parameter_size])((void) sizeof ((compiled_data->tensors.parameters[d + i *
parameter_size]) ? 1 : 0), __extension__ ({ if (compiled_data
->tensors.parameters[d + i * parameter_size]) ; else __assert_fail
("compiled_data->tensors.parameters[d + i * parameter_size]"
, "ccv_cnnp_model.c", 2347, __extension__ __PRETTY_FUNCTION__
); })); }
2348 this_tensor_init = 1;
2349 } else {
2350 const ccv_nnc_tensor_symbol_t parameter = *(ccv_nnc_tensor_symbol_t*)ccv_array_get(compiled_data->parameters, d)((void*)(((char*)((compiled_data->parameters)->data)) +
(size_t)(compiled_data->parameters)->rsize * (size_t)(
d)));
2351 ccv_nnc_tensor_param_t info = ccv_nnc_tensor_symbol_params(parameter.graph, parameter);
2352 if (CCV_TENSOR_GET_DEVICE(info.type)((info.type) & 0xfff00) == CCV_COMPUTE_DEVICE_ANY)
2353 CCV_TENSOR_SET_DEVICE_ID(info.type, 0)(info.type) = (((info.type) & ~0xfff00) | (((0) & 0xfff
) << 8));
2354 const int device_id = CCV_TENSOR_GET_DEVICE_ID(info.type)(((info.type) & 0xfff00) >> 8);
2355 compiled_data->tensors.parameters[d] = ccv_nnc_tensor_new(0, info, 0);
2356 for (i = 1; i < parallel_count; i++)
2357 {
2358 if (i != device_id)
2359 CCV_TENSOR_SET_DEVICE_ID(info.type, i)(info.type) = (((info.type) & ~0xfff00) | (((i) & 0xfff
) << 8));
2360 else
2361 CCV_TENSOR_SET_DEVICE_ID(info.type, 0)(info.type) = (((info.type) & ~0xfff00) | (((0) & 0xfff
) << 8));
2362 compiled_data->tensors.parameters[d + i * parameter_size] = ccv_nnc_tensor_new(0, info, 0);
2363 }
2364 }
2365 }
2366 ccv_nnc_tensor_t* const dest = CCV_NNC_TENSOR(compiled_data->tensors.parameters[d])((ccv_nnc_tensor_t*)((uintptr_t)(compiled_data->tensors.parameters
[d]) & ~(uintptr_t)1));
2367 assert(dest)((void) sizeof ((dest) ? 1 : 0), __extension__ ({ if (dest) ;
else __assert_fail ("dest", "ccv_cnnp_model.c", 2367, __extension__
__PRETTY_FUNCTION__); }));
2368 ccv_nnc_cmd_exec(CMD_DATA_TRANSFER_FORWARD()ccv_nnc_cmd(CCV_NNC_DATA_TRANSFER_FORWARD, 0, ccv_nnc_cmd_auto
, 0), ccv_nnc_no_hint, 0, TENSOR_LIST((ccv_nnc_tensor_t*)tensor)(ccv_nnc_tensor_t* []){(ccv_nnc_tensor_t*)tensor}, (1 +1 +0 +
0 +0 +0 +0 +0 +0 +0 +0 +0 +0 +0 +0 +0 +0 +0 +0 +0 +0 +0 +0 +0
+0 +0 +0 +0 +0 +0 +0 +0 +0 +0 +0 +0 +0 +0 +0 +0 +0 +0 +0 +0 +
0 +0 +0 +0 +0 +0 +0 +0 +0 +0 +0 +0 +0 +0 +0 +0 +0 +0 +0 +0 -1
), TENSOR_LIST(dest)(ccv_nnc_tensor_t* []){dest}, (1 +1 +0 +0 +0 +0 +0 +0 +0 +0 +
0 +0 +0 +0 +0 +0 +0 +0 +0 +0 +0 +0 +0 +0 +0 +0 +0 +0 +0 +0 +0
+0 +0 +0 +0 +0 +0 +0 +0 +0 +0 +0 +0 +0 +0 +0 +0 +0 +0 +0 +0 +
0 +0 +0 +0 +0 +0 +0 +0 +0 +0 +0 +0 +0 -1), 0);
2369 for (i = 1; i < parallel_count; i++)
2370 {
2371 ccv_nnc_tensor_t* const copy_tensor = CCV_NNC_TENSOR(compiled_data->tensors.parameters[d + i * parameter_size])((ccv_nnc_tensor_t*)((uintptr_t)(compiled_data->tensors.parameters
[d + i * parameter_size]) & ~(uintptr_t)1));
2372 if (copy_tensor)
2373 ccv_nnc_cmd_exec(CMD_DATA_TRANSFER_FORWARD()ccv_nnc_cmd(CCV_NNC_DATA_TRANSFER_FORWARD, 0, ccv_nnc_cmd_auto
, 0), ccv_nnc_no_hint, 0, TENSOR_LIST(dest)(ccv_nnc_tensor_t* []){dest}, (1 +1 +0 +0 +0 +0 +0 +0 +0 +0 +
0 +0 +0 +0 +0 +0 +0 +0 +0 +0 +0 +0 +0 +0 +0 +0 +0 +0 +0 +0 +0
+0 +0 +0 +0 +0 +0 +0 +0 +0 +0 +0 +0 +0 +0 +0 +0 +0 +0 +0 +0 +
0 +0 +0 +0 +0 +0 +0 +0 +0 +0 +0 +0 +0 -1), TENSOR_LIST(copy_tensor)(ccv_nnc_tensor_t* []){copy_tensor}, (1 +1 +0 +0 +0 +0 +0 +0 +
0 +0 +0 +0 +0 +0 +0 +0 +0 +0 +0 +0 +0 +0 +0 +0 +0 +0 +0 +0 +0
+0 +0 +0 +0 +0 +0 +0 +0 +0 +0 +0 +0 +0 +0 +0 +0 +0 +0 +0 +0 +
0 +0 +0 +0 +0 +0 +0 +0 +0 +0 +0 +0 +0 +0 +0 -1), 0);
2374 }
2375 // Mark this symbol as init'ed.
2376 const int s = ((ccv_nnc_tensor_symbol_t*)ccv_array_get(compiled_data->parameters, d)((void*)(((char*)((compiled_data->parameters)->data)) +
(size_t)(compiled_data->parameters)->rsize * (size_t)(
d))))->d;
2377 uint32_t* const init_v = CCV_NNC_INIT_V(compiled_data->tensors_init.v)((uint32_t*)((uintptr_t)(compiled_data->tensors_init.v) &
~(uintptr_t)1));
2378 init_v[s >> 5] |= (1u << (s & 0x1f));
2379 // If we just allocated this tensor, now it is time to check if we need to mark it as fully allocated.
2380 if (!this_tensor_init)
2381 {
2382 if (ccv_cnnp_model_tensors_any_to_alloc(model, compiled_data))
2383 compiled_data->tensors_init.v = (uint32_t*)((uintptr_t)compiled_data->tensors_init.v | (uintptr_t)1);
2384 else // Remove the flag.
2385 compiled_data->tensors_init.v = CCV_NNC_INIT_V(compiled_data->tensors_init.v)((uint32_t*)((uintptr_t)(compiled_data->tensors_init.v) &
~(uintptr_t)1));
2386 }
2387}
2388
2389void ccv_cnnp_model_parameter_copy(ccv_cnnp_model_t* const model, const ccv_cnnp_model_io_t parameter, ccv_nnc_tensor_t* const tensor)
2390{
2391 ccv_cnnp_compiled_data_t* const compiled_data = model->compiled_data;
2392 const int param_sel = parameter->param_sel > 0 ? parameter->param_sel - 1 : parameter->param_sel;
2393 assert(parameter->param_sel != 0)((void) sizeof ((parameter->param_sel != 0) ? 1 : 0), __extension__
({ if (parameter->param_sel != 0) ; else __assert_fail ("parameter->param_sel != 0"
, "ccv_cnnp_model.c", 2393, __extension__ __PRETTY_FUNCTION__
); }));
2394 assert(compiled_data->tensors.parameters)((void) sizeof ((compiled_data->tensors.parameters) ? 1 : 0
), __extension__ ({ if (compiled_data->tensors.parameters)
; else __assert_fail ("compiled_data->tensors.parameters"
, "ccv_cnnp_model.c", 2394, __extension__ __PRETTY_FUNCTION__
); }));
2395 ccv_array_t* const parameter_indices = ccv_array_new(sizeof(int), 0, 0);
2396 ccv_cnnp_model_add_to_parameter_indices(parameter->model, param_sel, parameter_indices);
2397 const int param_ref = parameter->param_ref > 0 ? parameter->param_ref - 1 : parameter->param_ref;
2398 if (param_ref < 0)
2399 { assert(parameter_indices->rnum == 1)((void) sizeof ((parameter_indices->rnum == 1) ? 1 : 0), __extension__
({ if (parameter_indices->rnum == 1) ; else __assert_fail
("parameter_indices->rnum == 1", "ccv_cnnp_model.c", 2399
, __extension__ __PRETTY_FUNCTION__); })); }
2400 else
2401 { assert(param_ref < parameter_indices->rnum)((void) sizeof ((param_ref < parameter_indices->rnum) ?
1 : 0), __extension__ ({ if (param_ref < parameter_indices
->rnum) ; else __assert_fail ("param_ref < parameter_indices->rnum"
, "ccv_cnnp_model.c", 2401, __extension__ __PRETTY_FUNCTION__
); })); }
2402 const int d = *(int*)ccv_array_get(parameter_indices, param_ref >= 0 ? param_ref : 0)((void*)(((char*)((parameter_indices)->data)) + (size_t)(parameter_indices
)->rsize * (size_t)(param_ref >= 0 ? param_ref : 0)));
2403 ccv_array_free(parameter_indices);
2404 const int parameter_size = compiled_data->parameters->rnum;
2405 assert(d >= 0)((void) sizeof ((d >= 0) ? 1 : 0), __extension__ ({ if (d >=
0) ; else __assert_fail ("d >= 0", "ccv_cnnp_model.c", 2405
, __extension__ __PRETTY_FUNCTION__); }));
2406 assert(d < parameter_size)((void) sizeof ((d < parameter_size) ? 1 : 0), __extension__
({ if (d < parameter_size) ; else __assert_fail ("d < parameter_size"
, "ccv_cnnp_model.c", 2406, __extension__ __PRETTY_FUNCTION__
); }));
2407 // We don't need to consider parallel_count, every parameter on each device is identical.
2408 ccv_nnc_tensor_t* const src = CCV_NNC_TENSOR(compiled_data->tensors.parameters[d])((ccv_nnc_tensor_t*)((uintptr_t)(compiled_data->tensors.parameters
[d]) & ~(uintptr_t)1));
2409 assert(src)((void) sizeof ((src) ? 1 : 0), __extension__ ({ if (src) ; else
__assert_fail ("src", "ccv_cnnp_model.c", 2409, __extension__
__PRETTY_FUNCTION__); }));
2410 ccv_nnc_cmd_exec(CMD_DATA_TRANSFER_FORWARD()ccv_nnc_cmd(CCV_NNC_DATA_TRANSFER_FORWARD, 0, ccv_nnc_cmd_auto
, 0), ccv_nnc_no_hint, 0, TENSOR_LIST(src)(ccv_nnc_tensor_t* []){src}, (1 +1 +0 +0 +0 +0 +0 +0 +0 +0 +0
+0 +0 +0 +0 +0 +0 +0 +0 +0 +0 +0 +0 +0 +0 +0 +0 +0 +0 +0 +0 +
0 +0 +0 +0 +0 +0 +0 +0 +0 +0 +0 +0 +0 +0 +0 +0 +0 +0 +0 +0 +0
+0 +0 +0 +0 +0 +0 +0 +0 +0 +0 +0 +0 -1), TENSOR_LIST(tensor)(ccv_nnc_tensor_t* []){tensor}, (1 +1 +0 +0 +0 +0 +0 +0 +0 +0
+0 +0 +0 +0 +0 +0 +0 +0 +0 +0 +0 +0 +0 +0 +0 +0 +0 +0 +0 +0 +
0 +0 +0 +0 +0 +0 +0 +0 +0 +0 +0 +0 +0 +0 +0 +0 +0 +0 +0 +0 +0
+0 +0 +0 +0 +0 +0 +0 +0 +0 +0 +0 +0 +0 -1), 0);
2411}
2412
2413ccv_nnc_tensor_param_t ccv_cnnp_model_parameter_tensor_params(ccv_cnnp_model_t* const model, const ccv_cnnp_model_io_t parameter)
2414{
2415 ccv_cnnp_compiled_data_t* const compiled_data = model->compiled_data;
2416 const int param_sel = parameter->param_sel > 0 ? parameter->param_sel - 1 : parameter->param_sel;
2417 assert(parameter->param_sel != 0)((void) sizeof ((parameter->param_sel != 0) ? 1 : 0), __extension__
({ if (parameter->param_sel != 0) ; else __assert_fail ("parameter->param_sel != 0"
, "ccv_cnnp_model.c", 2417, __extension__ __PRETTY_FUNCTION__
); }));
2418 assert(compiled_data->tensors.parameters)((void) sizeof ((compiled_data->tensors.parameters) ? 1 : 0
), __extension__ ({ if (compiled_data->tensors.parameters)
; else __assert_fail ("compiled_data->tensors.parameters"
, "ccv_cnnp_model.c", 2418, __extension__ __PRETTY_FUNCTION__
); }));
2419 ccv_array_t* const parameter_indices = ccv_array_new(sizeof(int), 0, 0);
2420 ccv_cnnp_model_add_to_parameter_indices(parameter->model, param_sel, parameter_indices);
2421 const int param_ref = parameter->param_ref > 0 ? parameter->param_ref - 1 : parameter->param_ref;
2422 if (param_ref < 0)
2423 { assert(parameter_indices->rnum == 1)((void) sizeof ((parameter_indices->rnum == 1) ? 1 : 0), __extension__
({ if (parameter_indices->rnum == 1) ; else __assert_fail
("parameter_indices->rnum == 1", "ccv_cnnp_model.c", 2423
, __extension__ __PRETTY_FUNCTION__); })); }
2424 else
2425 { assert(param_ref < parameter_indices->rnum)((void) sizeof ((param_ref < parameter_indices->rnum) ?
1 : 0), __extension__ ({ if (param_ref < parameter_indices
->rnum) ; else __assert_fail ("param_ref < parameter_indices->rnum"
, "ccv_cnnp_model.c", 2425, __extension__ __PRETTY_FUNCTION__
); })); }
2426 const int d = *(int*)ccv_array_get(parameter_indices, param_ref >= 0 ? param_ref : 0)((void*)(((char*)((parameter_indices)->data)) + (size_t)(parameter_indices
)->rsize * (size_t)(param_ref >= 0 ? param_ref : 0)));
2427 ccv_array_free(parameter_indices);
2428 const int parameter_size = compiled_data->parameters->rnum;
2429 assert(d >= 0)((void) sizeof ((d >= 0) ? 1 : 0), __extension__ ({ if (d >=
0) ; else __assert_fail ("d >= 0", "ccv_cnnp_model.c", 2429
, __extension__ __PRETTY_FUNCTION__); }));
2430 assert(d < parameter_size)((void) sizeof ((d < parameter_size) ? 1 : 0), __extension__
({ if (d < parameter_size) ; else __assert_fail ("d < parameter_size"
, "ccv_cnnp_model.c", 2430, __extension__ __PRETTY_FUNCTION__
); }));
2431 // We don't need to consider parallel_count, every parameter on each device is identical.
2432 ccv_nnc_tensor_t* const tensor = CCV_NNC_TENSOR(compiled_data->tensors.parameters[d])((ccv_nnc_tensor_t*)((uintptr_t)(compiled_data->tensors.parameters
[d]) & ~(uintptr_t)1));
2433 assert(tensor)((void) sizeof ((tensor) ? 1 : 0), __extension__ ({ if (tensor
) ; else __assert_fail ("tensor", "ccv_cnnp_model.c", 2433, __extension__
__PRETTY_FUNCTION__); }));
2434 return tensor->info;
2435}
2436
2437const char* ccv_cnnp_model_parameter_name(ccv_cnnp_model_t* const model, const ccv_cnnp_model_io_t parameter)
2438{
2439 ccv_cnnp_compiled_data_t* const compiled_data = model->compiled_data;
2440 const int param_sel = parameter->param_sel > 0 ? parameter->param_sel - 1 : parameter->param_sel;
2441 assert(parameter->param_sel != 0)((void) sizeof ((parameter->param_sel != 0) ? 1 : 0), __extension__
({ if (parameter->param_sel != 0) ; else __assert_fail ("parameter->param_sel != 0"
, "ccv_cnnp_model.c", 2441, __extension__ __PRETTY_FUNCTION__
); }));
2442 ccv_array_t* const parameter_indices = ccv_array_new(sizeof(int), 0, 0);
2443 ccv_cnnp_model_add_to_parameter_indices(parameter->model, param_sel, parameter_indices);
2444 const int param_ref = parameter->param_ref > 0 ? parameter->param_ref - 1 : parameter->param_ref;
2445 if (param_ref < 0)
2446 { assert(parameter_indices->rnum == 1)((void) sizeof ((parameter_indices->rnum == 1) ? 1 : 0), __extension__
({ if (parameter_indices->rnum == 1) ; else __assert_fail
("parameter_indices->rnum == 1", "ccv_cnnp_model.c", 2446
, __extension__ __PRETTY_FUNCTION__); })); }
2447 else
2448 { assert(param_ref < parameter_indices->rnum)((void) sizeof ((param_ref < parameter_indices->rnum) ?
1 : 0), __extension__ ({ if (param_ref < parameter_indices
->rnum) ; else __assert_fail ("param_ref < parameter_indices->rnum"
, "ccv_cnnp_model.c", 2448, __extension__ __PRETTY_FUNCTION__
); })); }
2449 const int d = *(int*)ccv_array_get(parameter_indices, param_ref >= 0 ? param_ref : 0)((void*)(((char*)((parameter_indices)->data)) + (size_t)(parameter_indices
)->rsize * (size_t)(param_ref >= 0 ? param_ref : 0)));
2450 ccv_array_free(parameter_indices);
2451 const int parameter_size = compiled_data->parameters->rnum;
2452 assert(d >= 0)((void) sizeof ((d >= 0) ? 1 : 0), __extension__ ({ if (d >=
0) ; else __assert_fail ("d >= 0", "ccv_cnnp_model.c", 2452
, __extension__ __PRETTY_FUNCTION__); }));
2453 assert(d < parameter_size)((void) sizeof ((d < parameter_size) ? 1 : 0), __extension__
({ if (d < parameter_size) ; else __assert_fail ("d < parameter_size"
, "ccv_cnnp_model.c", 2453, __extension__ __PRETTY_FUNCTION__
); }));
2454 return *(char**)ccv_array_get(compiled_data->ids.parameters, d)((void*)(((char*)((compiled_data->ids.parameters)->data
)) + (size_t)(compiled_data->ids.parameters)->rsize * (
size_t)(d)));
2455}
2456
2457int ccv_cnnp_model_parameter_count(ccv_cnnp_model_t* const model)
2458{
2459 assert(model->compiled_data)((void) sizeof ((model->compiled_data) ? 1 : 0), __extension__
({ if (model->compiled_data) ; else __assert_fail ("model->compiled_data"
, "ccv_cnnp_model.c", 2459, __extension__ __PRETTY_FUNCTION__
); }));
2460 ccv_cnnp_compiled_data_t* const compiled_data = model->compiled_data;
2461 return compiled_data->parameters->rnum;
2462}
2463
2464uint64_t ccv_cnnp_model_parameters_size(ccv_cnnp_model_t* const model)
2465{
2466 assert(model->compiled_data)((void) sizeof ((model->compiled_data) ? 1 : 0), __extension__
({ if (model->compiled_data) ; else __assert_fail ("model->compiled_data"
, "ccv_cnnp_model.c", 2466, __extension__ __PRETTY_FUNCTION__
); }));
2467 ccv_cnnp_compiled_data_t* const compiled_data = model->compiled_data;
2468 const int parameter_size = compiled_data->parameters->rnum;
2469 int i;
2470 const ccv_nnc_symbolic_graph_t* const graph = model->graph;
2471 uint64_t size = 0;
2472 const int tensors_init = !!compiled_data->tensors_init.v;
2473 uint32_t* const init_v = tensors_init ? CCV_NNC_INIT_V(compiled_data->tensors_init.v)((uint32_t*)((uintptr_t)(compiled_data->tensors_init.v) &
~(uintptr_t)1)) : 0;
2474 for (i = 0; i < parameter_size; i++)
2475 {
2476 const int d = ((ccv_nnc_tensor_symbol_t*)ccv_array_get(compiled_data->parameters, i)((void*)(((char*)((compiled_data->parameters)->data)) +
(size_t)(compiled_data->parameters)->rsize * (size_t)(
i))))->d;
2477 if (tensors_init && compiled_data->tensors.parameters && (init_v[d >> 5] | (1u << (d & 0x1f))) && compiled_data->tensors.parameters[i])
2478 {
2479 ccv_nnc_tensor_param_t params = compiled_data->tensors.parameters[i]->info;
2480 size += ccv_nnc_tensor_data_size(params);
2481 continue;
2482 }
2483 ccv_nnc_tensor_param_t params = ccv_nnc_tensor_symbol_params(graph, (ccv_nnc_tensor_symbol_t){
2484 .graph = graph,
2485 .d = d
2486 });
2487 size += ccv_nnc_tensor_data_size(params);
2488 }
2489 return size;
2490}
2491
2492int ccv_cnnp_model_parameters_move(ccv_cnnp_model_t* const model, char** const names, ccv_nnc_tensor_t** const tensors, const int count, int type)
2493{
2494 assert(model->compiled_data)((void) sizeof ((model->compiled_data) ? 1 : 0), __extension__
({ if (model->compiled_data) ; else __assert_fail ("model->compiled_data"
, "ccv_cnnp_model.c", 2494, __extension__ __PRETTY_FUNCTION__
); }));
2495 ccv_cnnp_compiled_data_t* const compiled_data = model->compiled_data;
2496 if (count != compiled_data->parameters->rnum)
2497 return 0;
2498 if (CCV_TENSOR_GET_DEVICE(type)((type) & 0xfff00) == CCV_COMPUTE_DEVICE_ANY)
2499 CCV_TENSOR_SET_DEVICE_ID(type, 0)(type) = (((type) & ~0xfff00) | (((0) & 0xfff) <<
8));
2500 int i;
2501 // We don't need to consider parallel_count, every parameter on each device is identical.
2502 for (i = 0; i < count; i++)
2503 {
2504 ccv_nnc_tensor_t* tensor = compiled_data->tensors.parameters[i];
2505 if ((uintptr_t)tensor & (uintptr_t)1) // If it is not owned. We don't do anything.
2506 {
2507 tensors[i] = 0;
2508 continue;
2509 }
2510 tensor = CCV_NNC_TENSOR(tensor)((ccv_nnc_tensor_t*)((uintptr_t)(tensor) & ~(uintptr_t)1)
);
2511 if (tensor->info.type == type)
2512 tensors[i] = tensor;
2513 else {
2514 ccv_nnc_tensor_param_t info = tensor->info;
2515 info.type = type;
2516 tensors[i] = ccv_nnc_tensor_new(0, info, 0); // Create this tensor, don't initiate copy yet.
2517 }
2518 }
2519 for (i = 0; i < count; i++)
2520 {
2521 ccv_nnc_tensor_t* tensor = compiled_data->tensors.parameters[i];
2522 if ((uintptr_t)tensor & (uintptr_t)1) // If it is not owned. We don't do anything.
2523 continue;
2524 tensor = CCV_NNC_TENSOR(tensor)((ccv_nnc_tensor_t*)((uintptr_t)(tensor) & ~(uintptr_t)1)
);
2525 // Now initiate transfer. We should do this one on a stream.
2526 if (tensor->info.type != type)
2527 ccv_nnc_cmd_exec(CMD_DATA_TRANSFER_FORWARD()ccv_nnc_cmd(CCV_NNC_DATA_TRANSFER_FORWARD, 0, ccv_nnc_cmd_auto
, 0), ccv_nnc_no_hint, 0, TENSOR_LIST(tensor)(ccv_nnc_tensor_t* []){tensor}, (1 +1 +0 +0 +0 +0 +0 +0 +0 +0
+0 +0 +0 +0 +0 +0 +0 +0 +0 +0 +0 +0 +0 +0 +0 +0 +0 +0 +0 +0 +
0 +0 +0 +0 +0 +0 +0 +0 +0 +0 +0 +0 +0 +0 +0 +0 +0 +0 +0 +0 +0
+0 +0 +0 +0 +0 +0 +0 +0 +0 +0 +0 +0 +0 -1), TENSOR_LIST(tensors[i])(ccv_nnc_tensor_t* []){tensors[i]}, (1 +1 +0 +0 +0 +0 +0 +0 +
0 +0 +0 +0 +0 +0 +0 +0 +0 +0 +0 +0 +0 +0 +0 +0 +0 +0 +0 +0 +0
+0 +0 +0 +0 +0 +0 +0 +0 +0 +0 +0 +0 +0 +0 +0 +0 +0 +0 +0 +0 +
0 +0 +0 +0 +0 +0 +0 +0 +0 +0 +0 +0 +0 +0 +0 -1), 0);
2528 }
2529 // Copy names and remove parameters.
2530 for (i = 0; i < count; i++)
2531 {
2532 ccv_nnc_tensor_t* const tensor = compiled_data->tensors.parameters[i];
2533 if ((uintptr_t)tensor & (uintptr_t)1) // If it is not owned. We don't do anything.
2534 {
2535 names[i] = 0;
2536 continue;
2537 }
2538 const char* const name = *(char**)ccv_array_get(compiled_data->ids.parameters, i)((void*)(((char*)((compiled_data->ids.parameters)->data
)) + (size_t)(compiled_data->ids.parameters)->rsize * (
size_t)(i)));
2539 const size_t name_len = ccv_min(strnlen(name, 1023), 1023)({ typeof (strnlen(name, 1023)) _a = (strnlen(name, 1023)); typeof
(1023) _b = (1023); (_a < _b) ? _a : _b; });
2540 names[i] = ccmallocmalloc(name_len + 1);
2541 names[i][name_len] = 0;
2542 memcpy(names[i], name, name_len);
2543 if (tensor->info.type == type)
2544 compiled_data->tensors.parameters[i] = 0; // Only move when it is moved.
2545 }
2546 return 1;
2547}
2548
2549KHASH_MAP_INIT_STR(ccv_cnnp_parameter_id, int)typedef struct kh_ccv_cnnp_parameter_id_s { khint_t n_buckets
, size, n_occupied, upper_bound; khint32_t *flags; kh_cstr_t *
keys; int *vals; } kh_ccv_cnnp_parameter_id_t; static inline __attribute__
((__unused__)) kh_ccv_cnnp_parameter_id_t *kh_init_ccv_cnnp_parameter_id
(void) { return (kh_ccv_cnnp_parameter_id_t*)calloc(1,sizeof(
kh_ccv_cnnp_parameter_id_t)); } static inline __attribute__ (
(__unused__)) void kh_destroy_ccv_cnnp_parameter_id(kh_ccv_cnnp_parameter_id_t
*h) { if (h) { free((void *)h->keys); free(h->flags); free
((void *)h->vals); free(h); } } static inline __attribute__
((__unused__)) void kh_clear_ccv_cnnp_parameter_id(kh_ccv_cnnp_parameter_id_t
*h) { if (h && h->flags) { memset(h->flags, 0xaa
, ((h->n_buckets) < 16? 1 : (h->n_buckets)>>4)
* sizeof(khint32_t)); h->size = h->n_occupied = 0; } }
static inline __attribute__ ((__unused__)) khint_t kh_get_ccv_cnnp_parameter_id
(const kh_ccv_cnnp_parameter_id_t *h, kh_cstr_t key) { if (h->
n_buckets) { khint_t k, i, last, mask, step = 0; mask = h->
n_buckets - 1; k = __ac_X31_hash_string(key); i = k & mask
; last = i; while (!((h->flags[i>>4]>>((i&
0xfU)<<1))&2) && (((h->flags[i>>4]
>>((i&0xfU)<<1))&1) || !(strcmp(h->keys
[i], key) == 0))) { i = (i + (++step)) & mask; if (i == last
) return h->n_buckets; } return ((h->flags[i>>4]>>
((i&0xfU)<<1))&3)? h->n_buckets : i; } else return
0; } static inline __attribute__ ((__unused__)) int kh_resize_ccv_cnnp_parameter_id
(kh_ccv_cnnp_parameter_id_t *h, khint_t new_n_buckets) { khint32_t
*new_flags = 0; khint_t j = 1; { (--(new_n_buckets), (new_n_buckets
)|=(new_n_buckets)>>1, (new_n_buckets)|=(new_n_buckets)
>>2, (new_n_buckets)|=(new_n_buckets)>>4, (new_n_buckets
)|=(new_n_buckets)>>8, (new_n_buckets)|=(new_n_buckets)
>>16, ++(new_n_buckets)); if (new_n_buckets < 4) new_n_buckets
= 4; if (h->size >= (khint_t)(new_n_buckets * __ac_HASH_UPPER
+ 0.5)) j = 0; else { new_flags = (khint32_t*)malloc(((new_n_buckets
) < 16? 1 : (new_n_buckets)>>4) * sizeof(khint32_t))
; if (!new_flags) return -1; memset(new_flags, 0xaa, ((new_n_buckets
) < 16? 1 : (new_n_buckets)>>4) * sizeof(khint32_t))
; if (h->n_buckets < new_n_buckets) { kh_cstr_t *new_keys
= (kh_cstr_t*)realloc((void *)h->keys,new_n_buckets * sizeof
(kh_cstr_t)); if (!new_keys) { free(new_flags); return -1; } h
->keys = new_keys; if (1) { int *new_vals = (int*)realloc(
(void *)h->vals,new_n_buckets * sizeof(int)); if (!new_vals
) { free(new_flags); return -1; } h->vals = new_vals; } } }
} if (j) { for (j = 0; j != h->n_buckets; ++j) { if (((h->
flags[j>>4]>>((j&0xfU)<<1))&3) == 0
) { kh_cstr_t key = h->keys[j]; int val; khint_t new_mask;
new_mask = new_n_buckets - 1; if (1) val = h->vals[j]; (h
->flags[j>>4]|=1ul<<((j&0xfU)<<1)); while
(1) { khint_t k, i, step = 0; k = __ac_X31_hash_string(key);
i = k & new_mask; while (!((new_flags[i>>4]>>
((i&0xfU)<<1))&2)) i = (i + (++step)) & new_mask
; (new_flags[i>>4]&=~(2ul<<((i&0xfU)<<
1))); if (i < h->n_buckets && ((h->flags[i>>
4]>>((i&0xfU)<<1))&3) == 0) { { kh_cstr_t
tmp = h->keys[i]; h->keys[i] = key; key = tmp; } if (1
) { int tmp = h->vals[i]; h->vals[i] = val; val = tmp; }
(h->flags[i>>4]|=1ul<<((i&0xfU)<<1)
); } else { h->keys[i] = key; if (1) h->vals[i] = val; break
; } } } } if (h->n_buckets > new_n_buckets) { h->keys
= (kh_cstr_t*)realloc((void *)h->keys,new_n_buckets * sizeof
(kh_cstr_t)); if (1) h->vals = (int*)realloc((void *)h->
vals,new_n_buckets * sizeof(int)); } free(h->flags); h->
flags = new_flags; h->n_buckets = new_n_buckets; h->n_occupied
= h->size; h->upper_bound = (khint_t)(h->n_buckets *
__ac_HASH_UPPER + 0.5); } return 0; } static inline __attribute__
((__unused__)) khint_t kh_put_ccv_cnnp_parameter_id(kh_ccv_cnnp_parameter_id_t
*h, kh_cstr_t key, int *ret) { khint_t x; if (h->n_occupied
>= h->upper_bound) { if (h->n_buckets > (h->size
<<1)) { if (kh_resize_ccv_cnnp_parameter_id(h, h->n_buckets
- 1) < 0) { *ret = -1; return h->n_buckets; } } else if
(kh_resize_ccv_cnnp_parameter_id(h, h->n_buckets + 1) <
0) { *ret = -1; return h->n_buckets; } } { khint_t k, i, site
, last, mask = h->n_buckets - 1, step = 0; x = site = h->
n_buckets; k = __ac_X31_hash_string(key); i = k & mask; if
(((h->flags[i>>4]>>((i&0xfU)<<1))&
2)) x = i; else { last = i; while (!((h->flags[i>>4]
>>((i&0xfU)<<1))&2) && (((h->flags
[i>>4]>>((i&0xfU)<<1))&1) || !(strcmp
(h->keys[i], key) == 0))) { if (((h->flags[i>>4]>>
((i&0xfU)<<1))&1)) site = i; i = (i + (++step))
& mask; if (i == last) { x = site; break; } } if (x == h
->n_buckets) { if (((h->flags[i>>4]>>((i&
0xfU)<<1))&2) && site != h->n_buckets) x
= site; else x = i; } } } if (((h->flags[x>>4]>>
((x&0xfU)<<1))&2)) { h->keys[x] = key; (h->
flags[x>>4]&=~(3ul<<((x&0xfU)<<1)))
; ++h->size; ++h->n_occupied; *ret = 1; } else if (((h->
flags[x>>4]>>((x&0xfU)<<1))&1)) { h
->keys[x] = key; (h->flags[x>>4]&=~(3ul<<
((x&0xfU)<<1))); ++h->size; *ret = 2; } else *ret
= 0; return x; } static inline __attribute__ ((__unused__)) void
kh_del_ccv_cnnp_parameter_id(kh_ccv_cnnp_parameter_id_t *h, khint_t
x) { if (x != h->n_buckets && !((h->flags[x>>
4]>>((x&0xfU)<<1))&3)) { (h->flags[x>>
4]|=1ul<<((x&0xfU)<<1)); --h->size; } }
25
←
Null pointer value stored to field 'vals'
→
30
←
Taking true branch
→
31
←
Taking false branch
→
32
←
Calling 'kh_resize_ccv_cnnp_parameter_id'
→
33
←
Taking true branch
→
34
←
Assuming the condition is false
→
35
←
Taking false branch
→
36
←
'?' condition is true
→
37
←
Assuming 'new_flags' is null
→
38
←
Taking true branch
→
39
←
Returning without writing to 'h->vals'
→
40
←
Returning from 'kh_resize_ccv_cnnp_parameter_id'
→
41
←
Taking true branch
→
42
←
Returning without writing to 'h->vals'
→
2550
2551void ccv_cnnp_model_set_parameters_from_key_values(ccv_cnnp_model_t* const model, char* const* const names, ccv_nnc_tensor_t** const tensors, const int count, const int invalidates)
2552{
2553 assert(model->compiled_data)((void) sizeof ((model->compiled_data) ? 1 : 0), __extension__
({ if (model->compiled_data) ; else __assert_fail ("model->compiled_data"
, "ccv_cnnp_model.c", 2553, __extension__ __PRETTY_FUNCTION__
); }));
2554 ccv_cnnp_compiled_data_t* const compiled_data = model->compiled_data;
2555 int i;
2556 khash_t(ccv_cnnp_parameter_id)kh_ccv_cnnp_parameter_id_t* id_map = 0;
2557 if (count != compiled_data->parameters->rnum)
2558 {
2559 id_map = kh_init(ccv_cnnp_parameter_id)kh_init_ccv_cnnp_parameter_id();
2560 // Build the map between name and the index.
2561 for (i = 0; i < count; i++)
2562 {
2563 int ret;
2564 const khiter_t k = kh_put(ccv_cnnp_parameter_id, id_map, names[i], &ret)kh_put_ccv_cnnp_parameter_id(id_map, names[i], &ret);
2565 assert(ret != 0)((void) sizeof ((ret != 0) ? 1 : 0), __extension__ ({ if (ret
!= 0) ; else __assert_fail ("ret != 0", "ccv_cnnp_model.c", 2565
, __extension__ __PRETTY_FUNCTION__); }));
2566 kh_val(id_map, k)((id_map)->vals[k]) = i;
2567 }
2568 }
2569 const int parameter_size = compiled_data->parameters->rnum;
2570 int* copy_back = 0;
2571 const int tensors_init = !!compiled_data->tensors_init.v;
2572 if (!tensors_init)
2573 ccv_cnnp_model_tensors_init_0(model, compiled_data);
2574 const int parallel_count = ccv_max(model->parallel_count, 1)({ typeof (model->parallel_count) _a = (model->parallel_count
); typeof (1) _b = (1); (_a > _b) ? _a : _b; });
2575 uint32_t* const init_v = CCV_NNC_INIT_V(compiled_data->tensors_init.v)((uint32_t*)((uintptr_t)(compiled_data->tensors_init.v) &
~(uintptr_t)1));
2576 for (i = 0; i < parameter_size; i++)
2577 {
2578 int j = i;
2579 const char* const name = *(char**)ccv_array_get(compiled_data->ids.parameters, i)((void*)(((char*)((compiled_data->ids.parameters)->data
)) + (size_t)(compiled_data->ids.parameters)->rsize * (
size_t)(i)));
2580 if (i >= 0 || strncmp(name, names[i], 1023) != 0)
2581 {
2582 // Build the map.
2583 if (id_map == 0)
2584 {
2585 id_map = kh_init(ccv_cnnp_parameter_id)kh_init_ccv_cnnp_parameter_id();
2586 for (j = 0; j < count; j++)
2587 {
2588 int ret;
2589 const khiter_t k = kh_put(ccv_cnnp_parameter_id, id_map, names[j], &ret)kh_put_ccv_cnnp_parameter_id(id_map, names[j], &ret);
2590 assert(ret != 0)((void) sizeof ((ret != 0) ? 1 : 0), __extension__ ({ if (ret
!= 0) ; else __assert_fail ("ret != 0", "ccv_cnnp_model.c", 2590
, __extension__ __PRETTY_FUNCTION__); }));
2591 kh_val(id_map, k)((id_map)->vals[k]) = j;
2592 }
2593 }
2594 const khiter_t k = kh_get(ccv_cnnp_parameter_id, id_map, name)kh_get_ccv_cnnp_parameter_id(id_map, name);
2595 if (k == kh_end(id_map)((id_map)->n_buckets)) // Cannot find the name, skip.
2596 continue;
2597 j = kh_val(id_map, k)((id_map)->vals[k]);
2598 }
2599 if (compiled_data->tensors.parameters[i]) // Cannot be a shared parameter to read.
2600 { assert(!((uintptr_t)compiled_data->tensors.parameters[i] & (uintptr_t)1))((void) sizeof ((!((uintptr_t)compiled_data->tensors.parameters
[i] & (uintptr_t)1)) ? 1 : 0), __extension__ ({ if (!((uintptr_t
)compiled_data->tensors.parameters[i] & (uintptr_t)1))
; else __assert_fail ("!((uintptr_t)compiled_data->tensors.parameters[i] & (uintptr_t)1)"
, "ccv_cnnp_model.c", 2600, __extension__ __PRETTY_FUNCTION__
); })); }
2601 const ccv_nnc_tensor_symbol_t parameter = *(ccv_nnc_tensor_symbol_t*)ccv_array_get(compiled_data->parameters, i)((void*)(((char*)((compiled_data->parameters)->data)) +
(size_t)(compiled_data->parameters)->rsize * (size_t)(
i)));
2602 ccv_nnc_tensor_param_t info = ccv_nnc_tensor_symbol_params(parameter.graph, parameter);
2603 if (CCV_TENSOR_GET_DEVICE(info.type)((info.type) & 0xfff00) == CCV_COMPUTE_DEVICE_ANY)
2604 CCV_TENSOR_SET_DEVICE_ID(info.type, 0)(info.type) = (((info.type) & ~0xfff00) | (((0) & 0xfff
) << 8));
2605 const int d = parameter.d;
2606 if (info.type == tensors[j]->info.type && invalidates) // Can move.
2607 {
2608 // Deallocate it if needed.
2609 if (!((uintptr_t)compiled_data->tensors.parameters[i] & (uintptr_t)1))
2610 if (compiled_data->tensors.parameters[i])
2611 ccv_nnc_tensor_free(compiled_data->tensors.parameters[i]);
2612 compiled_data->tensors.parameters[i] = tensors[j];
2613 tensors[j] = 0;
2614 } else {
2615 if (!compiled_data->tensors.parameters[i])
2616 { // Not allocated, to allocate first.
2617 // Create new one, make sure we create this by having the right parameters.
2618 const int type = info.type;
2619 info = tensors[j]->info;
2620 info.type = type; // Revert back the type.
2621 compiled_data->tensors.parameters[i] = ccv_nnc_tensor_new(0, info, 0);
2622 }
2623 if (!copy_back)
2624 copy_back = (int*)cccalloccalloc(parameter_size, sizeof(int));
2625 copy_back[i] = j + 1;
2626 }
2627 init_v[d >> 5] |= (1u << (d & 0x1f));
2628 // Create this tensor for other data parallel allocations.
2629 info = compiled_data->tensors.parameters[i]->info; // In case we loaded a different info.
2630 const int device_id = CCV_TENSOR_GET_DEVICE_ID(info.type)(((info.type) & 0xfff00) >> 8);
2631 for (j = 1; j < parallel_count; j++)
2632 if (!compiled_data->tensors.parameters[i + j * parameter_size])
2633 {
2634 if (j != device_id)
2635 CCV_TENSOR_SET_DEVICE_ID(info.type, j)(info.type) = (((info.type) & ~0xfff00) | (((j) & 0xfff
) << 8));
2636 else
2637 CCV_TENSOR_SET_DEVICE_ID(info.type, 0)(info.type) = (((info.type) & ~0xfff00) | (((0) & 0xfff
) << 8));
2638 compiled_data->tensors.parameters[i + j * parameter_size] = ccv_nnc_tensor_new(0, info, 0);
2639 }
2640 // No need to copy over, this is done in ccv_cnnp_model.c's copy_tensors method.
2641 }
2642 if (id_map)
2643 kh_destroy(ccv_cnnp_parameter_id, id_map)kh_destroy_ccv_cnnp_parameter_id(id_map);
2644 // Now do the transfer.
2645 if (copy_back)
2646 {
2647 for (i = 0; i < parameter_size; i++)
2648 {
2649 ccv_nnc_tensor_t* const tensor = CCV_NNC_TENSOR(compiled_data->tensors.parameters[i])((ccv_nnc_tensor_t*)((uintptr_t)(compiled_data->tensors.parameters
[i]) & ~(uintptr_t)1));
2650 if (copy_back[i] == 0)
2651 continue;
2652 const int j = copy_back[i] - 1;
2653 ccv_nnc_cmd_exec(CMD_DATA_TRANSFER_FORWARD()ccv_nnc_cmd(CCV_NNC_DATA_TRANSFER_FORWARD, 0, ccv_nnc_cmd_auto
, 0), ccv_nnc_no_hint, 0, TENSOR_LIST(tensors[j])(ccv_nnc_tensor_t* []){tensors[j]}, (1 +1 +0 +0 +0 +0 +0 +0 +
0 +0 +0 +0 +0 +0 +0 +0 +0 +0 +0 +0 +0 +0 +0 +0 +0 +0 +0 +0 +0
+0 +0 +0 +0 +0 +0 +0 +0 +0 +0 +0 +0 +0 +0 +0 +0 +0 +0 +0 +0 +
0 +0 +0 +0 +0 +0 +0 +0 +0 +0 +0 +0 +0 +0 +0 -1), TENSOR_LIST(tensor)(ccv_nnc_tensor_t* []){tensor}, (1 +1 +0 +0 +0 +0 +0 +0 +0 +0
+0 +0 +0 +0 +0 +0 +0 +0 +0 +0 +0 +0 +0 +0 +0 +0 +0 +0 +0 +0 +
0 +0 +0 +0 +0 +0 +0 +0 +0 +0 +0 +0 +0 +0 +0 +0 +0 +0 +0 +0 +0
+0 +0 +0 +0 +0 +0 +0 +0 +0 +0 +0 +0 +0 -1), 0);
2654 }
2655 ccfreefree(copy_back);
2656 }
2657}
2658
2659ccv_cnnp_model_io_t ccv_cnnp_model_parameter_first(ccv_cnnp_model_t* const model, ccv_cnnp_model_parameters_filter_f first, void* const context)
2660{
2661 ccv_cnnp_compiled_data_t* const compiled_data = model->compiled_data;
2662 assert(compiled_data)((void) sizeof ((compiled_data) ? 1 : 0), __extension__ ({ if
(compiled_data) ; else __assert_fail ("compiled_data", "ccv_cnnp_model.c"
, 2662, __extension__ __PRETTY_FUNCTION__); }));
2663 const int parameter_size = compiled_data->parameters->rnum;
2664 int i;
2665 for (i = 0; i < parameter_size; i++)
2666 {
2667 const char* const name = *(char**)ccv_array_get(compiled_data->ids.parameters, i)((void*)(((char*)((compiled_data->ids.parameters)->data
)) + (size_t)(compiled_data->ids.parameters)->rsize * (
size_t)(i)));
2668 if (first(model, name, context))
2669 return ccv_cnnp_model_parameters(model, -1, i);
2670 }
2671 return 0;
2672}
2673
2674ccv_array_t* ccv_cnnp_model_parameters_filter(ccv_cnnp_model_t* const model, ccv_cnnp_model_parameters_filter_f filter, void* const context)
2675{
2676 ccv_cnnp_compiled_data_t* const compiled_data = model->compiled_data;
2677 assert(compiled_data)((void) sizeof ((compiled_data) ? 1 : 0), __extension__ ({ if
(compiled_data) ; else __assert_fail ("compiled_data", "ccv_cnnp_model.c"
, 2677, __extension__ __PRETTY_FUNCTION__); }));
2678 ccv_array_t* const parameters = ccv_array_new(sizeof(ccv_cnnp_model_io_t), 0, 0);
2679 const int parameter_size = compiled_data->parameters->rnum;
2680 int i;
2681 for (i = 0; i < parameter_size; i++)
2682 {
2683 const char* const name = *(char**)ccv_array_get(compiled_data->ids.parameters, i)((void*)(((char*)((compiled_data->ids.parameters)->data
)) + (size_t)(compiled_data->ids.parameters)->rsize * (
size_t)(i)));
2684 if (filter(model, name, context))
2685 {
2686 ccv_cnnp_model_io_t parameter = ccv_cnnp_model_parameters(model, -1, i);
2687 ccv_array_push(parameters, &parameter);
2688 }
2689 }
2690 return parameters;
2691
2692}
2693
2694CCV_WARN_UNUSED(ccv_cnnp_model_io_t)ccv_cnnp_model_io_t __attribute__((warn_unused_result)) ccv_cnnp_model_parameter_first_uninit(ccv_cnnp_model_t* const model)
2695{
2696 ccv_cnnp_compiled_data_t* const compiled_data = model->compiled_data;
2697 assert(compiled_data)((void) sizeof ((compiled_data) ? 1 : 0), __extension__ ({ if
(compiled_data) ; else __assert_fail ("compiled_data", "ccv_cnnp_model.c"
, 2697, __extension__ __PRETTY_FUNCTION__); }));
2698 const int tensors_init = !!compiled_data->tensors_init.v;
2699 if (!tensors_init) // If nothing initialized, we return parameter 0.
2700 return ccv_cnnp_model_parameters(model, -1, 0);
2701 const int parameter_size = compiled_data->parameters->rnum;
2702 int i;
2703 const uint32_t* const init_v = CCV_NNC_INIT_V(compiled_data->tensors_init.v)((uint32_t*)((uintptr_t)(compiled_data->tensors_init.v) &
~(uintptr_t)1));
2704 for (i = 0; i < parameter_size; i++)
2705 {
2706 const int d = ((ccv_nnc_tensor_symbol_t*)ccv_array_get(compiled_data->parameters, i)((void*)(((char*)((compiled_data->parameters)->data)) +
(size_t)(compiled_data->parameters)->rsize * (size_t)(
i))))->d;
2707 if (!(init_v[d >> 5] & (1u << (d & 0x1f))))
2708 return ccv_cnnp_model_parameters(model, -1, i);
2709 }
2710 return 0;
2711}
2712
2713static ccv_array_t* _ccv_cnnp_model_parameter_indices(const ccv_cnnp_model_t* const model, const ccv_cnnp_model_io_t parameters, int* const param_ref)
2714{
2715 const int to_param_sel = parameters->param_sel > 0 ? parameters->param_sel - 1 : parameters->param_sel;
2716 assert(parameters->param_sel != 0)((void) sizeof ((parameters->param_sel != 0) ? 1 : 0), __extension__
({ if (parameters->param_sel != 0) ; else __assert_fail (
"parameters->param_sel != 0", "ccv_cnnp_model.c", 2716, __extension__
__PRETTY_FUNCTION__); }));
2717 ccv_array_t* const to_parameter_indices = ccv_array_new(sizeof(int), 0, 0);
2718 ccv_cnnp_model_add_to_parameter_indices(parameters->model, to_param_sel, to_parameter_indices);
2719 *param_ref = parameters->param_ref > 0 ? parameters->param_ref - 1 : parameters->param_ref;
2720 return to_parameter_indices;
2721}
2722
2723static void _ccv_cnnp_model_to_parameter_indices_and_from_parameter_indices(ccv_cnnp_model_t* const model, const ccv_cnnp_model_io_t parameters, const ccv_cnnp_model_t* const from_model, const ccv_cnnp_model_io_t from_parameters, ccv_array_t** const parameter_indices, int* const param_ref, ccv_array_t** const from_parameter_indices, int* const from_param_ref, const int only_init_0)
2724{
2725 // If the model is not compiled yet. Compile them now.
2726 if (!model->graph)
2727 {
2728 model->graph = ccv_nnc_symbolic_graph_new();
2729 assert(from_model->compiled_data)((void) sizeof ((from_model->compiled_data) ? 1 : 0), __extension__
({ if (from_model->compiled_data) ; else __assert_fail ("from_model->compiled_data"
, "ccv_cnnp_model.c", 2729, __extension__ __PRETTY_FUNCTION__
); }));
2730 const int input_size = from_model->input_size;
2731 ccv_nnc_tensor_param_t input_params[input_size];
2732 int i;
2733 for (i = 0; i < input_size; i++)
2734 input_params[i] = ccv_nnc_tensor_symbol_params(from_model->graph, from_model->inputs[i]);
2735 _ccv_cnnp_model_compile(model, input_params, input_size, from_model->compiled_data->loss);
2736 model->parallel_count = from_model->parallel_count;
2737 model->memory_compression = from_model->memory_compression;
2738 model->memory_reduction = from_model->memory_reduction;
2739 model->gradient_checkpointing = from_model->gradient_checkpointing;
2740 model->compiled_data->stream_type = from_model->compiled_data->stream_type;
2741 model->compiled_data->minimize.minimizer = from_model->compiled_data->minimize.minimizer;
2742 model->compiled_data->minimize.max_saved_aux_size = from_model->compiled_data->minimize.max_saved_aux_size;
2743 }
2744 ccv_cnnp_compiled_data_t* const to_compiled_data = model->compiled_data;
2745 assert(to_compiled_data)((void) sizeof ((to_compiled_data) ? 1 : 0), __extension__ ({
if (to_compiled_data) ; else __assert_fail ("to_compiled_data"
, "ccv_cnnp_model.c", 2745, __extension__ __PRETTY_FUNCTION__
); }));
2746 const int to_tensors_init = !!to_compiled_data->tensors_init.v;
2747 if (!to_tensors_init)
2748 {
2749 if (only_init_0)
2750 ccv_cnnp_model_tensors_init_0(model, to_compiled_data);
2751 else
2752 _ccv_cnnp_model_tensors_init(model, to_compiled_data);
2753 } else if (!only_init_0 && (uintptr_t)to_compiled_data->tensors_init.v & (uintptr_t)1)
2754 // Check if it is not fully allocated, if it is not, init_1.
2755 ccv_cnnp_model_tensors_init_1(model, to_compiled_data);
2756 assert(to_compiled_data->tensors.parameters)((void) sizeof ((to_compiled_data->tensors.parameters) ? 1
: 0), __extension__ ({ if (to_compiled_data->tensors.parameters
) ; else __assert_fail ("to_compiled_data->tensors.parameters"
, "ccv_cnnp_model.c", 2756, __extension__ __PRETTY_FUNCTION__
); }));
2757 *parameter_indices = _ccv_cnnp_model_parameter_indices(model, parameters, param_ref);
2758 *from_parameter_indices = _ccv_cnnp_model_parameter_indices(from_model, from_parameters, from_param_ref);
2759 if (*from_param_ref < 0 && *param_ref >= 0)
2760 { assert((*from_parameter_indices)->rnum == 1)((void) sizeof (((*from_parameter_indices)->rnum == 1) ? 1
: 0), __extension__ ({ if ((*from_parameter_indices)->rnum
== 1) ; else __assert_fail ("(*from_parameter_indices)->rnum == 1"
, "ccv_cnnp_model.c", 2760, __extension__ __PRETTY_FUNCTION__
); })); }
2761 else if (*from_param_ref >= 0)
2762 { assert(*from_param_ref < (*from_parameter_indices)->rnum)((void) sizeof ((*from_param_ref < (*from_parameter_indices
)->rnum) ? 1 : 0), __extension__ ({ if (*from_param_ref <
(*from_parameter_indices)->rnum) ; else __assert_fail ("*from_param_ref < (*from_parameter_indices)->rnum"
, "ccv_cnnp_model.c", 2762, __extension__ __PRETTY_FUNCTION__
); })); }
2763 if (*param_ref < 0 && *from_param_ref >= 0)
2764 { assert((*parameter_indices)->rnum == 1)((void) sizeof (((*parameter_indices)->rnum == 1) ? 1 : 0)
, __extension__ ({ if ((*parameter_indices)->rnum == 1) ; else
__assert_fail ("(*parameter_indices)->rnum == 1", "ccv_cnnp_model.c"
, 2764, __extension__ __PRETTY_FUNCTION__); })); }
2765 else if (*param_ref >= 0)
2766 { assert(*param_ref < (*parameter_indices)->rnum)((void) sizeof ((*param_ref < (*parameter_indices)->rnum
) ? 1 : 0), __extension__ ({ if (*param_ref < (*parameter_indices
)->rnum) ; else __assert_fail ("*param_ref < (*parameter_indices)->rnum"
, "ccv_cnnp_model.c", 2766, __extension__ __PRETTY_FUNCTION__
); })); }
2767}
2768
2769void ccv_cnnp_model_set_parameters(ccv_cnnp_model_t* const model, const ccv_cnnp_model_io_t parameters, const ccv_cnnp_model_t* const from_model, const ccv_cnnp_model_io_t from_parameters)
2770{
2771 ccv_array_t* to_parameter_indices;
2772 int to_param_ref;
2773 ccv_array_t* from_parameter_indices;
2774 int from_param_ref;
2775 _ccv_cnnp_model_to_parameter_indices_and_from_parameter_indices(model, parameters, from_model, from_parameters, &to_parameter_indices, &to_param_ref, &from_parameter_indices, &from_param_ref, 0);
2776 // Should be exactly the same tensor.
2777 if (to_param_ref < 0 && from_param_ref < 0)
2778 { assert(from_parameter_indices->rnum == to_parameter_indices->rnum)((void) sizeof ((from_parameter_indices->rnum == to_parameter_indices
->rnum) ? 1 : 0), __extension__ ({ if (from_parameter_indices
->rnum == to_parameter_indices->rnum) ; else __assert_fail
("from_parameter_indices->rnum == to_parameter_indices->rnum"
, "ccv_cnnp_model.c", 2778, __extension__ __PRETTY_FUNCTION__
); })); }
2779 // To models.
2780 ccv_cnnp_compiled_data_t* const to_compiled_data = model->compiled_data;
2781 assert(to_compiled_data)((void) sizeof ((to_compiled_data) ? 1 : 0), __extension__ ({
if (to_compiled_data) ; else __assert_fail ("to_compiled_data"
, "ccv_cnnp_model.c", 2781, __extension__ __PRETTY_FUNCTION__
); }));
2782 // From models.
2783 const ccv_cnnp_compiled_data_t* const from_compiled_data = from_model->compiled_data;
2784 const int parallel_count = ccv_max(model->parallel_count, 1)({ typeof (model->parallel_count) _a = (model->parallel_count
); typeof (1) _b = (1); (_a > _b) ? _a : _b; });
2785 const int to_parameter_size = to_compiled_data->parameters->rnum;
2786 const int rnum = (to_param_ref < 0 && from_param_ref < 0) ? from_parameter_indices->rnum : 1;
2787 int i, j;
2788 const uint32_t* const from_init_v = CCV_NNC_INIT_V(from_compiled_data->tensors_init.v)((uint32_t*)((uintptr_t)(from_compiled_data->tensors_init.
v) & ~(uintptr_t)1));
2789 uint32_t* const to_init_v = CCV_NNC_INIT_V(to_compiled_data->tensors_init.v)((uint32_t*)((uintptr_t)(to_compiled_data->tensors_init.v)
& ~(uintptr_t)1));
2790 for (i = 0; i < rnum; i++)
2791 {
2792 const int src_d = *(int*)ccv_array_get(from_parameter_indices,from_param_ref >= 0 ? from_param_ref : i)((void*)(((char*)((from_parameter_indices)->data)) + (size_t
)(from_parameter_indices)->rsize * (size_t)(from_param_ref
>= 0 ? from_param_ref : i)));
2793 assert(src_d >= 0)((void) sizeof ((src_d >= 0) ? 1 : 0), __extension__ ({ if
(src_d >= 0) ; else __assert_fail ("src_d >= 0", "ccv_cnnp_model.c"
, 2793, __extension__ __PRETTY_FUNCTION__); }));
2794 assert(src_d < from_compiled_data->parameters->rnum)((void) sizeof ((src_d < from_compiled_data->parameters
->rnum) ? 1 : 0), __extension__ ({ if (src_d < from_compiled_data
->parameters->rnum) ; else __assert_fail ("src_d < from_compiled_data->parameters->rnum"
, "ccv_cnnp_model.c", 2794, __extension__ __PRETTY_FUNCTION__
); }));
2795 const int s = ((ccv_nnc_tensor_symbol_t*)ccv_array_get(from_compiled_data->parameters, src_d)((void*)(((char*)((from_compiled_data->parameters)->data
)) + (size_t)(from_compiled_data->parameters)->rsize * (
size_t)(src_d))))->d;
2796 // If the original is not init'ed. We cannot copy from.
2797 if (!(from_init_v[s >> 5] & (1u << (s & 0x1f))))
2798 continue;
2799 const int dest_d = *(int*)ccv_array_get(to_parameter_indices, to_param_ref >= 0 ? to_param_ref : i)((void*)(((char*)((to_parameter_indices)->data)) + (size_t
)(to_parameter_indices)->rsize * (size_t)(to_param_ref >=
0 ? to_param_ref : i)));
2800 assert(dest_d >= 0)((void) sizeof ((dest_d >= 0) ? 1 : 0), __extension__ ({ if
(dest_d >= 0) ; else __assert_fail ("dest_d >= 0", "ccv_cnnp_model.c"
, 2800, __extension__ __PRETTY_FUNCTION__); }));
2801 assert(dest_d < to_compiled_data->parameters->rnum)((void) sizeof ((dest_d < to_compiled_data->parameters->
rnum) ? 1 : 0), __extension__ ({ if (dest_d < to_compiled_data
->parameters->rnum) ; else __assert_fail ("dest_d < to_compiled_data->parameters->rnum"
, "ccv_cnnp_model.c", 2801, __extension__ __PRETTY_FUNCTION__
); }));
2802 ccv_nnc_tensor_t* const src = CCV_NNC_TENSOR(from_compiled_data->tensors.parameters[src_d])((ccv_nnc_tensor_t*)((uintptr_t)(from_compiled_data->tensors
.parameters[src_d]) & ~(uintptr_t)1));
2803 assert(src)((void) sizeof ((src) ? 1 : 0), __extension__ ({ if (src) ; else
__assert_fail ("src", "ccv_cnnp_model.c", 2803, __extension__
__PRETTY_FUNCTION__); }));
2804 ccv_nnc_tensor_t* const dest = CCV_NNC_TENSOR(to_compiled_data->tensors.parameters[dest_d])((ccv_nnc_tensor_t*)((uintptr_t)(to_compiled_data->tensors
.parameters[dest_d]) & ~(uintptr_t)1));
2805 assert(dest)((void) sizeof ((dest) ? 1 : 0), __extension__ ({ if (dest) ;
else __assert_fail ("dest", "ccv_cnnp_model.c", 2805, __extension__
__PRETTY_FUNCTION__); }));
2806 ccv_nnc_cmd_exec(CMD_DATA_TRANSFER_FORWARD()ccv_nnc_cmd(CCV_NNC_DATA_TRANSFER_FORWARD, 0, ccv_nnc_cmd_auto
, 0), ccv_nnc_no_hint, 0, TENSOR_LIST(src)(ccv_nnc_tensor_t* []){src}, (1 +1 +0 +0 +0 +0 +0 +0 +0 +0 +0
+0 +0 +0 +0 +0 +0 +0 +0 +0 +0 +0 +0 +0 +0 +0 +0 +0 +0 +0 +0 +
0 +0 +0 +0 +0 +0 +0 +0 +0 +0 +0 +0 +0 +0 +0 +0 +0 +0 +0 +0 +0
+0 +0 +0 +0 +0 +0 +0 +0 +0 +0 +0 +0 -1), TENSOR_LIST(dest)(ccv_nnc_tensor_t* []){dest}, (1 +1 +0 +0 +0 +0 +0 +0 +0 +0 +
0 +0 +0 +0 +0 +0 +0 +0 +0 +0 +0 +0 +0 +0 +0 +0 +0 +0 +0 +0 +0
+0 +0 +0 +0 +0 +0 +0 +0 +0 +0 +0 +0 +0 +0 +0 +0 +0 +0 +0 +0 +
0 +0 +0 +0 +0 +0 +0 +0 +0 +0 +0 +0 +0 -1), 0);
2807 for (j = 1; j < parallel_count; j++)
2808 {
2809 ccv_nnc_tensor_t* const copy_tensor = CCV_NNC_TENSOR(to_compiled_data->tensors.parameters[dest_d + j * to_parameter_size])((ccv_nnc_tensor_t*)((uintptr_t)(to_compiled_data->tensors
.parameters[dest_d + j * to_parameter_size]) & ~(uintptr_t
)1));
2810 if (copy_tensor)
2811 ccv_nnc_cmd_exec(CMD_DATA_TRANSFER_FORWARD()ccv_nnc_cmd(CCV_NNC_DATA_TRANSFER_FORWARD, 0, ccv_nnc_cmd_auto
, 0), ccv_nnc_no_hint, 0, TENSOR_LIST(dest)(ccv_nnc_tensor_t* []){dest}, (1 +1 +0 +0 +0 +0 +0 +0 +0 +0 +
0 +0 +0 +0 +0 +0 +0 +0 +0 +0 +0 +0 +0 +0 +0 +0 +0 +0 +0 +0 +0
+0 +0 +0 +0 +0 +0 +0 +0 +0 +0 +0 +0 +0 +0 +0 +0 +0 +0 +0 +0 +
0 +0 +0 +0 +0 +0 +0 +0 +0 +0 +0 +0 +0 -1), TENSOR_LIST(copy_tensor)(ccv_nnc_tensor_t* []){copy_tensor}, (1 +1 +0 +0 +0 +0 +0 +0 +
0 +0 +0 +0 +0 +0 +0 +0 +0 +0 +0 +0 +0 +0 +0 +0 +0 +0 +0 +0 +0
+0 +0 +0 +0 +0 +0 +0 +0 +0 +0 +0 +0 +0 +0 +0 +0 +0 +0 +0 +0 +
0 +0 +0 +0 +0 +0 +0 +0 +0 +0 +0 +0 +0 +0 +0 -1), 0);
2812 }
2813 // Mark this symbol as init'ed.
2814 const int d = ((ccv_nnc_tensor_symbol_t*)ccv_array_get(to_compiled_data->parameters, dest_d)((void*)(((char*)((to_compiled_data->parameters)->data)
) + (size_t)(to_compiled_data->parameters)->rsize * (size_t
)(dest_d))))->d;
2815 to_init_v[d >> 5] |= (1u << (d & 0x1f));
2816 }
2817 ccv_array_free(to_parameter_indices);
2818 ccv_array_free(from_parameter_indices);
2819}
2820
2821void ccv_cnnp_model_share_parameters(ccv_cnnp_model_t* const model, const ccv_cnnp_model_io_t parameters, const ccv_cnnp_model_t* const from_model, const ccv_cnnp_model_io_t from_parameters, ccv_cnnp_model_parameters_renamer_f renamer, void* const context)
2822{
2823 ccv_array_t* to_parameter_indices;
2824 int to_param_ref;
2825 ccv_array_t* from_parameter_indices;
2826 int from_param_ref;
2827 _ccv_cnnp_model_to_parameter_indices_and_from_parameter_indices(model, parameters, from_model, from_parameters, &to_parameter_indices, &to_param_ref, &from_parameter_indices, &from_param_ref, 1);
2828 // Should be exactly the same tensor.
2829 if (renamer == 0 && to_param_ref < 0 && from_param_ref < 0)
1
Assuming 'renamer' is not equal to null
→
2830 { assert(from_parameter_indices->rnum == to_parameter_indices->rnum)((void) sizeof ((from_parameter_indices->rnum == to_parameter_indices
->rnum) ? 1 : 0), __extension__ ({ if (from_parameter_indices
->rnum == to_parameter_indices->rnum) ; else __assert_fail
("from_parameter_indices->rnum == to_parameter_indices->rnum"
, "ccv_cnnp_model.c", 2830, __extension__ __PRETTY_FUNCTION__
); })); }
2831 // To models.
2832 ccv_cnnp_compiled_data_t* const to_compiled_data = model->compiled_data;
2833 assert(to_compiled_data)((void) sizeof ((to_compiled_data) ? 1 : 0), __extension__ ({
if (to_compiled_data) ; else __assert_fail ("to_compiled_data"
, "ccv_cnnp_model.c", 2833, __extension__ __PRETTY_FUNCTION__
); }));
2
←
Assuming 'to_compiled_data' is non-null
→
3
←
Taking true branch
→
2834 // From models.
2835 const ccv_cnnp_compiled_data_t* const from_compiled_data = from_model->compiled_data;
2836 const int parallel_count = ccv_max(model->parallel_count, 1)({ typeof (model->parallel_count) _a = (model->parallel_count
); typeof (1) _b = (1); (_a > _b) ? _a : _b; });
4
←
Assuming '_a' is <= '_b'
→
5
←
'?' condition is false
→
2837 assert(parallel_count == ccv_max(from_model->parallel_count, 1))((void) sizeof ((parallel_count == ({ typeof (from_model->
parallel_count) _a = (from_model->parallel_count); typeof (
1) _b = (1); (_a > _b) ? _a : _b; })) ? 1 : 0), __extension__
({ if (parallel_count == ({ typeof (from_model->parallel_count
) _a = (from_model->parallel_count); typeof (1) _b = (1); (
_a > _b) ? _a : _b; })) ; else __assert_fail ("parallel_count == ccv_max(from_model->parallel_count, 1)"
, "ccv_cnnp_model.c", 2837, __extension__ __PRETTY_FUNCTION__
); })); // Should have the same parallel count can share parameters.
6
←
Assuming '_a' is <= '_b'
→
7
←
'?' condition is false
→
8
←
Taking true branch
→
2838 const int from_parameter_size = from_compiled_data->parameters->rnum;
2839 const int to_parameter_size = to_compiled_data->parameters->rnum;
2840 const int rnum = (to_param_ref < 0 && from_param_ref < 0) ? to_parameter_indices->rnum : 1;
9
←
Assuming 'to_param_ref' is >= 0
→
2841 int i, j;
2842 khash_t(ccv_cnnp_parameter_id)kh_ccv_cnnp_parameter_id_t* id_map = 0;
2843 char* updated_name = 0;
2844 const uint32_t* const from_init_v = CCV_NNC_INIT_V(from_compiled_data->tensors_init.v)((uint32_t*)((uintptr_t)(from_compiled_data->tensors_init.
v) & ~(uintptr_t)1));
2845 uint32_t* const to_init_v = CCV_NNC_INIT_V(to_compiled_data->tensors_init.v)((uint32_t*)((uintptr_t)(to_compiled_data->tensors_init.v)
& ~(uintptr_t)1));
2846 for (i = 0; i < rnum; i++)
2847 {
2848 int src_d = (from_param_ref >= 0 ? from_param_ref : i) < from_parameter_indices->rnum ? *(int*)ccv_array_get(from_parameter_indices,from_param_ref >= 0 ? from_param_ref : i)((void*)(((char*)((from_parameter_indices)->data)) + (size_t
)(from_parameter_indices)->rsize * (size_t)(from_param_ref
>= 0 ? from_param_ref : i))) : from_parameter_size;
10
←
Assuming 'from_param_ref' is < 0
→
11
←
'?' condition is false
→
12
←
Assuming the condition is false
→
13
←
'?' condition is false
→
2849 // Need to figure out how to use the renamer here.
2850 const int dest_d = *(int*)ccv_array_get(to_parameter_indices, to_param_ref >= 0 ? to_param_ref : i)((void*)(((char*)((to_parameter_indices)->data)) + (size_t
)(to_parameter_indices)->rsize * (size_t)(to_param_ref >=
0 ? to_param_ref : i)));
14
←
'?' condition is true
→
2851 assert(dest_d >= 0)((void) sizeof ((dest_d >= 0) ? 1 : 0), __extension__ ({ if
(dest_d >= 0) ; else __assert_fail ("dest_d >= 0", "ccv_cnnp_model.c"
, 2851, __extension__ __PRETTY_FUNCTION__); }));
15
←
Assuming 'dest_d' is >= 0
→
16
←
Taking true branch
→
2852 assert(dest_d < to_parameter_size)((void) sizeof ((dest_d < to_parameter_size) ? 1 : 0), __extension__
({ if (dest_d < to_parameter_size) ; else __assert_fail (
"dest_d < to_parameter_size", "ccv_cnnp_model.c", 2852, __extension__
__PRETTY_FUNCTION__); }));
17
←
Assuming 'dest_d' is < 'to_parameter_size'
→
18
←
Taking true branch
→
2853 if (renamer
18.1
'renamer' is non-null
)
2854 {
2855 const char* const src_name = (src_d
18.2
'src_d' is >= 'from_parameter_size'
 < from_parameter_size && src_d >= 0) ? *(char**)ccv_array_get(from_compiled_data->ids.parameters, src_d)((void*)(((char*)((from_compiled_data->ids.parameters)->
data)) + (size_t)(from_compiled_data->ids.parameters)->
rsize * (size_t)(src_d))) : 0;
2856 const char* const dest_name = *(char**)ccv_array_get(to_compiled_data->ids.parameters, dest_d)((void*)(((char*)((to_compiled_data->ids.parameters)->data
)) + (size_t)(to_compiled_data->ids.parameters)->rsize *
(size_t)(dest_d)));
2857 if (!updated_name
18.3
'updated_name' is null
)
19
←
Taking true branch
→
2858 updated_name = (char*)ccmallocmalloc(1024);
2859 const size_t src_name_len = src_name
19.1
'src_name' is equal to null
 == 0 ? 0 : ccv_min(strnlen(src_name, 1023), 1023)({ typeof (strnlen(src_name, 1023)) _a = (strnlen(src_name, 1023
)); typeof (1023) _b = (1023); (_a < _b) ? _a : _b; });
20
←
'?' condition is true
→
2860 if (src_name_len
20.1
'src_name_len' is <= 0
 > 0)
21
←
Taking false branch
→
2861 memcpy(updated_name, src_name, src_name_len);
2862 updated_name[src_name_len] = 0;
2863 if (renamer(context, dest_name, updated_name, 1024) != 0)
22
←
Assuming the condition is false
→
2864 continue; // Skip this.
2865 if (src_name
22.1
'src_name' is equal to null
 != 0 && memcmp(updated_name, src_name, src_name_len) == 0 && strnlen(updated_name, 1023) == src_name_len)
2866 {
2867 // Nothing changed.
2868 } else {
2869 if (!id_map
22.2
'id_map' is null
)
23
←
Taking true branch
→
2870 {
2871 id_map = kh_init(ccv_cnnp_parameter_id)kh_init_ccv_cnnp_parameter_id();
24
←
Calling 'kh_init_ccv_cnnp_parameter_id'
→
26
←
Returning from 'kh_init_ccv_cnnp_parameter_id'
→
2872 for (j = 0; j < from_parameter_size; j++)
27
←
Assuming 'j' is < 'from_parameter_size'
→
28
←
Loop condition is true. Entering loop body
→
2873 {
2874 int ret;
2875 const khiter_t k = kh_put(ccv_cnnp_parameter_id, id_map, *(char**)ccv_array_get(from_compiled_data->ids.parameters, j), &ret)kh_put_ccv_cnnp_parameter_id(id_map, *(char**)((void*)(((char
*)((from_compiled_data->ids.parameters)->data)) + (size_t
)(from_compiled_data->ids.parameters)->rsize * (size_t)
(j))), &ret);
29
←
Calling 'kh_put_ccv_cnnp_parameter_id'
→
43
←
Returning from 'kh_put_ccv_cnnp_parameter_id'
→
2876 assert(ret != 0)((void) sizeof ((ret != 0) ? 1 : 0), __extension__ ({ if (ret
!= 0) ; else __assert_fail ("ret != 0", "ccv_cnnp_model.c", 2876
, __extension__ __PRETTY_FUNCTION__); }));
44
←
Taking true branch
→
2877 kh_val(id_map, k)((id_map)->vals[k]) = j;
45
←
Array access (via field 'vals') results in a null pointer dereference
2878 }
2879 }
2880 const khiter_t k = kh_get(ccv_cnnp_parameter_id, id_map, updated_name)kh_get_ccv_cnnp_parameter_id(id_map, updated_name);
2881 if (k == kh_end(id_map)((id_map)->n_buckets)) // Cannot find the name, skip.
2882 continue;
2883 src_d = kh_val(id_map, k)((id_map)->vals[k]);
2884 assert(src_d >= 0)((void) sizeof ((src_d >= 0) ? 1 : 0), __extension__ ({ if
(src_d >= 0) ; else __assert_fail ("src_d >= 0", "ccv_cnnp_model.c"
, 2884, __extension__ __PRETTY_FUNCTION__); }));
2885 assert(src_d < from_parameter_size)((void) sizeof ((src_d < from_parameter_size) ? 1 : 0), __extension__
({ if (src_d < from_parameter_size) ; else __assert_fail (
"src_d < from_parameter_size", "ccv_cnnp_model.c", 2885, __extension__
__PRETTY_FUNCTION__); }));
2886 }
2887 }
2888 assert(src_d >= 0)((void) sizeof ((src_d >= 0) ? 1 : 0), __extension__ ({ if
(src_d >= 0) ; else __assert_fail ("src_d >= 0", "ccv_cnnp_model.c"
, 2888, __extension__ __PRETTY_FUNCTION__); }));
2889 assert(src_d < from_parameter_size)((void) sizeof ((src_d < from_parameter_size) ? 1 : 0), __extension__
({ if (src_d < from_parameter_size) ; else __assert_fail (
"src_d < from_parameter_size", "ccv_cnnp_model.c", 2889, __extension__
__PRETTY_FUNCTION__); }));
2890 const int s = ((ccv_nnc_tensor_symbol_t*)ccv_array_get(from_compiled_data->parameters, src_d)((void*)(((char*)((from_compiled_data->parameters)->data
)) + (size_t)(from_compiled_data->parameters)->rsize * (
size_t)(src_d))))->d;
2891 // If the original is not init'ed. We cannot share from.
2892 if (!(from_init_v[s >> 5] & (1u << (s & 0x1f))))
2893 continue;
2894 for (j = 0; j < parallel_count; j++)
2895 {
2896 ccv_nnc_tensor_t* const src = CCV_NNC_TENSOR(from_compiled_data->tensors.parameters[src_d + j * from_parameter_size])((ccv_nnc_tensor_t*)((uintptr_t)(from_compiled_data->tensors
.parameters[src_d + j * from_parameter_size]) & ~(uintptr_t
)1));
2897 assert(src)((void) sizeof ((src) ? 1 : 0), __extension__ ({ if (src) ; else
__assert_fail ("src", "ccv_cnnp_model.c", 2897, __extension__
__PRETTY_FUNCTION__); }));
2898 ccv_nnc_tensor_t* const dest = to_compiled_data->tensors.parameters[dest_d + j * to_parameter_size];
2899 if (dest && !((uintptr_t)dest & (uintptr_t)1))
2900 ccv_nnc_tensor_free(dest);
2901 to_compiled_data->tensors.parameters[dest_d + j * to_parameter_size] = (ccv_nnc_tensor_t*)((uintptr_t)src | (uintptr_t)1);
2902 }
2903 // Mark this symbol as init'ed.
2904 const int d = ((ccv_nnc_tensor_symbol_t*)ccv_array_get(to_compiled_data->parameters, dest_d)((void*)(((char*)((to_compiled_data->parameters)->data)
) + (size_t)(to_compiled_data->parameters)->rsize * (size_t
)(dest_d))))->d;
2905 to_init_v[d >> 5] |= (1u << (d & 0x1f));
2906 }
2907 ccv_array_free(to_parameter_indices);
2908 ccv_array_free(from_parameter_indices);
2909 if (id_map)
2910 kh_destroy(ccv_cnnp_parameter_id, id_map)kh_destroy_ccv_cnnp_parameter_id(id_map);
2911 if (updated_name)
2912 ccfreefree(updated_name);
2913 // Mark it as incomplete so we will call init_1.
2914 if (ccv_cnnp_model_tensors_any_to_alloc(model, to_compiled_data))
2915 to_compiled_data->tensors_init.v = (uint32_t*)((uintptr_t)to_compiled_data->tensors_init.v | (uintptr_t)1);
2916 else // Remove the flag.
2917 to_compiled_data->tensors_init.v = CCV_NNC_INIT_V(to_compiled_data->tensors_init.v)((uint32_t*)((uintptr_t)(to_compiled_data->tensors_init.v)
& ~(uintptr_t)1));
2918}
2919
2920ccv_nnc_stream_context_t* ccv_cnnp_compiled_data_get_stream(ccv_cnnp_compiled_data_t* const compiled_data, const int type)
2921{
2922 if (!compiled_data->stream_map)
2923 compiled_data->stream_map = kh_init(stream_map)kh_init_stream_map();
2924 int ret = 0;
2925 khiter_t k = kh_put(stream_map, compiled_data->stream_map, type, &ret)kh_put_stream_map(compiled_data->stream_map, type, &ret
);
2926 assert(ret >= 0)((void) sizeof ((ret >= 0) ? 1 : 0), __extension__ ({ if (
ret >= 0) ; else __assert_fail ("ret >= 0", "ccv_cnnp_model.c"
, 2926, __extension__ __PRETTY_FUNCTION__); }));
2927 ccv_nnc_stream_context_t* stream = kh_val(compiled_data->stream_map, k)((compiled_data->stream_map)->vals[k]);
2928 // If ret == 0, the key already exist, we can return directly, otherwise, create and return.
2929 if (ret != 0)
2930 {
2931 stream = ccv_nnc_stream_context_new(type);
2932 kh_val(compiled_data->stream_map, k)((compiled_data->stream_map)->vals[k]) = stream;
2933 }
2934 return stream;
2935}
2936
2937void ccv_cnnp_model_parameters_zip_map(ccv_cnnp_model_t* const model, const ccv_cnnp_model_io_t parameters, const ccv_nnc_cmd_t cmd, const ccv_nnc_hint_t hint, const int flags, ccv_nnc_tensor_t* const* const aux_ins, const int aux_in_size, ccv_nnc_tensor_t* const* const aux_outs, const int aux_out_size, ccv_nnc_stream_context_t* const stream_context, const ccv_cnnp_model_t* const from_model, const ccv_cnnp_model_io_t from_parameters)
2938{
2939 ccv_array_t* to_parameter_indices;
2940 int to_param_ref;
2941 ccv_array_t* from_parameter_indices;
2942 int from_param_ref;
2943 _ccv_cnnp_model_to_parameter_indices_and_from_parameter_indices(model, parameters, from_model, from_parameters, &to_parameter_indices, &to_param_ref, &from_parameter_indices, &from_param_ref, 0);
2944 // Should be exactly the same tensor.
2945 if (to_param_ref < 0 && from_param_ref < 0)
2946 { assert(from_parameter_indices->rnum == to_parameter_indices->rnum)((void) sizeof ((from_parameter_indices->rnum == to_parameter_indices
->rnum) ? 1 : 0), __extension__ ({ if (from_parameter_indices
->rnum == to_parameter_indices->rnum) ; else __assert_fail
("from_parameter_indices->rnum == to_parameter_indices->rnum"
, "ccv_cnnp_model.c", 2946, __extension__ __PRETTY_FUNCTION__
); })); }
2947 // To models.
2948 ccv_cnnp_compiled_data_t* const to_compiled_data = model->compiled_data;
2949 assert(to_compiled_data)((void) sizeof ((to_compiled_data) ? 1 : 0), __extension__ ({
if (to_compiled_data) ; else __assert_fail ("to_compiled_data"
, "ccv_cnnp_model.c", 2949, __extension__ __PRETTY_FUNCTION__
); }));
2950 // From models.
2951 const ccv_cnnp_compiled_data_t* const from_compiled_data = from_model->compiled_data;
2952 const int parallel_count = ccv_max(model->parallel_count, 1)({ typeof (model->parallel_count) _a = (model->parallel_count
); typeof (1) _b = (1); (_a > _b) ? _a : _b; });
2953 const int to_parameter_size = to_compiled_data->parameters->rnum;
2954 const int rnum = (to_param_ref < 0 && from_param_ref < 0) ? from_parameter_indices->rnum : 1;
2955 assert(aux_in_size >= 0)((void) sizeof ((aux_in_size >= 0) ? 1 : 0), __extension__
({ if (aux_in_size >= 0) ; else __assert_fail ("aux_in_size >= 0"
, "ccv_cnnp_model.c", 2955, __extension__ __PRETTY_FUNCTION__
); }));
2956 assert(aux_out_size >= 0)((void) sizeof ((aux_out_size >= 0) ? 1 : 0), __extension__
({ if (aux_out_size >= 0) ; else __assert_fail ("aux_out_size >= 0"
, "ccv_cnnp_model.c", 2956, __extension__ __PRETTY_FUNCTION__
); }));
2957 int i, j;
2958 ccv_nnc_tensor_t* inputs[aux_in_size + 2];
2959 ccv_nnc_tensor_t* outputs[aux_out_size + 1];
2960 for (i = 0; i < aux_in_size; i++)
2961 inputs[i + 2] = aux_ins[i];
2962 for (i = 0; i < aux_out_size; i++)
2963 outputs[i + 1] = aux_outs[i];
2964 const uint32_t* const from_init_v = CCV_NNC_INIT_V(from_compiled_data->tensors_init.v)((uint32_t*)((uintptr_t)(from_compiled_data->tensors_init.
v) & ~(uintptr_t)1));
2965 uint32_t* const to_init_v = CCV_NNC_INIT_V(to_compiled_data->tensors_init.v)((uint32_t*)((uintptr_t)(to_compiled_data->tensors_init.v)
& ~(uintptr_t)1));
2966 for (i = 0; i < rnum; i++)
2967 {
2968 const int src_d = *(int*)ccv_array_get(from_parameter_indices,from_param_ref >= 0 ? from_param_ref : i)((void*)(((char*)((from_parameter_indices)->data)) + (size_t
)(from_parameter_indices)->rsize * (size_t)(from_param_ref
>= 0 ? from_param_ref : i)));
2969 assert(src_d >= 0)((void) sizeof ((src_d >= 0) ? 1 : 0), __extension__ ({ if
(src_d >= 0) ; else __assert_fail ("src_d >= 0", "ccv_cnnp_model.c"
, 2969, __extension__ __PRETTY_FUNCTION__); }));
2970 assert(src_d < from_compiled_data->parameters->rnum)((void) sizeof ((src_d < from_compiled_data->parameters
->rnum) ? 1 : 0), __extension__ ({ if (src_d < from_compiled_data
->parameters->rnum) ; else __assert_fail ("src_d < from_compiled_data->parameters->rnum"
, "ccv_cnnp_model.c", 2970, __extension__ __PRETTY_FUNCTION__
); }));
2971 const int s = ((ccv_nnc_tensor_symbol_t*)ccv_array_get(from_compiled_data->parameters, src_d)((void*)(((char*)((from_compiled_data->parameters)->data
)) + (size_t)(from_compiled_data->parameters)->rsize * (
size_t)(src_d))))->d;
2972 // If the original is not init'ed. We cannot copy from.
2973 if (!(from_init_v[s >> 5] & (1u << (s & 0x1f))))
2974 continue;
2975 const int dest_d = *(int*)ccv_array_get(to_parameter_indices, to_param_ref >= 0 ? to_param_ref : i)((void*)(((char*)((to_parameter_indices)->data)) + (size_t
)(to_parameter_indices)->rsize * (size_t)(to_param_ref >=
0 ? to_param_ref : i)));
2976 assert(dest_d >= 0)((void) sizeof ((dest_d >= 0) ? 1 : 0), __extension__ ({ if
(dest_d >= 0) ; else __assert_fail ("dest_d >= 0", "ccv_cnnp_model.c"
, 2976, __extension__ __PRETTY_FUNCTION__); }));
2977 assert(dest_d < to_compiled_data->parameters->rnum)((void) sizeof ((dest_d < to_compiled_data->parameters->
rnum) ? 1 : 0), __extension__ ({ if (dest_d < to_compiled_data
->parameters->rnum) ; else __assert_fail ("dest_d < to_compiled_data->parameters->rnum"
, "ccv_cnnp_model.c", 2977, __extension__ __PRETTY_FUNCTION__
); }));
2978 if (parallel_count > 1)
2979 {
2980 ccv_nnc_stream_context_t* streams[parallel_count];
2981 ccv_nnc_stream_signal_t* signal;
2982 if (stream_context)
2983 signal = ccv_nnc_stream_context_emit_signal_new(stream_context);
2984 for (j = 0; j < parallel_count; j++)
2985 {
2986 ccv_nnc_tensor_t* const src = CCV_NNC_TENSOR(from_compiled_data->tensors.parameters[src_d + j * to_parameter_size])((ccv_nnc_tensor_t*)((uintptr_t)(from_compiled_data->tensors
.parameters[src_d + j * to_parameter_size]) & ~(uintptr_t
)1));
2987 ccv_nnc_tensor_t* const dest = CCV_NNC_TENSOR(to_compiled_data->tensors.parameters[dest_d + j * to_parameter_size])((ccv_nnc_tensor_t*)((uintptr_t)(to_compiled_data->tensors
.parameters[dest_d + j * to_parameter_size]) & ~(uintptr_t
)1));
2988 if (!dest || !src)
2989 {
2990 streams[j] = 0;
2991 continue;
2992 }
2993 // At the moment, can only handle them on the same device.
2994 assert(CCV_TENSOR_GET_MEMORY(src->info.type) == CCV_TENSOR_GET_MEMORY(dest->info.type))((void) sizeof ((((src->info.type) & 0x3) == ((dest->
info.type) & 0x3)) ? 1 : 0), __extension__ ({ if (((src->
info.type) & 0x3) == ((dest->info.type) & 0x3)) ; else
__assert_fail ("CCV_TENSOR_GET_MEMORY(src->info.type) == CCV_TENSOR_GET_MEMORY(dest->info.type)"
, "ccv_cnnp_model.c", 2994, __extension__ __PRETTY_FUNCTION__
); }));
2995 assert(CCV_TENSOR_GET_DEVICE_ID(src->info.type) == CCV_TENSOR_GET_DEVICE_ID(dest->info.type))((void) sizeof (((((src->info.type) & 0xfff00) >>
8) == (((dest->info.type) & 0xfff00) >> 8)) ? 1
: 0), __extension__ ({ if ((((src->info.type) & 0xfff00
) >> 8) == (((dest->info.type) & 0xfff00) >>
8)) ; else __assert_fail ("CCV_TENSOR_GET_DEVICE_ID(src->info.type) == CCV_TENSOR_GET_DEVICE_ID(dest->info.type)"
, "ccv_cnnp_model.c", 2995, __extension__ __PRETTY_FUNCTION__
); }));
2996 const int stream_type = CCV_TENSOR_GET_MEMORY(src->info.type)((src->info.type) & 0x3) == CCV_TENSOR_GPU_MEMORY ? CCV_STREAM_CONTEXT_GPU : CCV_STREAM_CONTEXT_CPU;
2997 const int device_id = CCV_TENSOR_GET_DEVICE_ID(src->info.type)(((src->info.type) & 0xfff00) >> 8);
2998 int type = stream_type;
2999 CCV_STREAM_SET_DEVICE_ID(type, device_id)(type) = (((type) & ~0xfff00) | (((device_id) & 0xfff
) << 8));
3000 ccv_nnc_stream_context_t* const stream_0 = ccv_cnnp_compiled_data_get_stream(to_compiled_data, type);
3001 // Wait signal to finish.
3002 if (stream_context)
3003 ccv_nnc_stream_context_wait_signal(stream_0, signal);
3004 inputs[0] = outputs[0] = dest;
3005 inputs[1] = src;
3006 ccv_nnc_cmd_exec(cmd, hint, flags, inputs, aux_in_size + 2, outputs, aux_out_size + 1, stream_0);
3007 if (stream_context)
3008 {
3009 ccv_nnc_stream_signal_t* const signal = ccv_nnc_stream_context_emit_signal_new(stream_0);
3010 ccv_nnc_stream_context_wait_signal(stream_context, signal);
3011 }
3012 streams[j] = stream_0;
3013 }
3014 // If this should be blocking, blocking it.
3015 if (!stream_context)
3016 for (j = 0; j < parallel_count; j++)
3017 if (streams[j])
3018 ccv_nnc_stream_context_wait(streams[j]);
3019 } else {
3020 ccv_nnc_tensor_t* const src = CCV_NNC_TENSOR(from_compiled_data->tensors.parameters[src_d])((ccv_nnc_tensor_t*)((uintptr_t)(from_compiled_data->tensors
.parameters[src_d]) & ~(uintptr_t)1));
3021 assert(src)((void) sizeof ((src) ? 1 : 0), __extension__ ({ if (src) ; else
__assert_fail ("src", "ccv_cnnp_model.c", 3021, __extension__
__PRETTY_FUNCTION__); }));
3022 ccv_nnc_tensor_t* const dest = CCV_NNC_TENSOR(to_compiled_data->tensors.parameters[dest_d])((ccv_nnc_tensor_t*)((uintptr_t)(to_compiled_data->tensors
.parameters[dest_d]) & ~(uintptr_t)1));
3023 assert(dest)((void) sizeof ((dest) ? 1 : 0), __extension__ ({ if (dest) ;
else __assert_fail ("dest", "ccv_cnnp_model.c", 3023, __extension__
__PRETTY_FUNCTION__); }));
3024 inputs[0] = outputs[0] = dest;
3025 inputs[1] = src;
3026 ccv_nnc_cmd_exec(cmd, hint, flags, inputs, aux_in_size + 2, outputs, aux_out_size + 1, stream_context);
3027 }
3028 // Mark this symbol as init'ed.
3029 const int d = ((ccv_nnc_tensor_symbol_t*)ccv_array_get(to_compiled_data->parameters, dest_d)((void*)(((char*)((to_compiled_data->parameters)->data)
) + (size_t)(to_compiled_data->parameters)->rsize * (size_t
)(dest_d))))->d;
3030 to_init_v[d >> 5] |= (1u << (d & 0x1f));
3031 }
3032 ccv_array_free(to_parameter_indices);
3033 ccv_array_free(from_parameter_indices);
3034}
3035
3036void ccv_cnnp_model_parameters_map(ccv_cnnp_model_t* const model, const ccv_cnnp_model_io_t parameters, const ccv_nnc_cmd_t cmd, const ccv_nnc_hint_t hint, const int flags, ccv_nnc_tensor_t* const* const aux_ins, const int aux_in_size, ccv_nnc_tensor_t* const* const aux_outs, const int aux_out_size, ccv_nnc_stream_context_t* const stream_context)
3037{
3038 int to_param_ref;
3039 ccv_array_t* const to_parameter_indices = _ccv_cnnp_model_parameter_indices(model, parameters, &to_param_ref);
3040 // To models.
3041 ccv_cnnp_compiled_data_t* const to_compiled_data = model->compiled_data;
3042 assert(to_compiled_data)((void) sizeof ((to_compiled_data) ? 1 : 0), __extension__ ({
if (to_compiled_data) ; else __assert_fail ("to_compiled_data"
, "ccv_cnnp_model.c", 3042, __extension__ __PRETTY_FUNCTION__
); }));
3043 // Tensor has to be inited already.
3044 assert(!!to_compiled_data->tensors_init.v)((void) sizeof ((!!to_compiled_data->tensors_init.v) ? 1 :
0), __extension__ ({ if (!!to_compiled_data->tensors_init
.v) ; else __assert_fail ("!!to_compiled_data->tensors_init.v"
, "ccv_cnnp_model.c", 3044, __extension__ __PRETTY_FUNCTION__
); }));
3045 assert(to_compiled_data->tensors.parameters)((void) sizeof ((to_compiled_data->tensors.parameters) ? 1
: 0), __extension__ ({ if (to_compiled_data->tensors.parameters
) ; else __assert_fail ("to_compiled_data->tensors.parameters"
, "ccv_cnnp_model.c", 3045, __extension__ __PRETTY_FUNCTION__
); }));
3046 // From models.
3047 const int parallel_count = ccv_max(model->parallel_count, 1)({ typeof (model->parallel_count) _a = (model->parallel_count
); typeof (1) _b = (1); (_a > _b) ? _a : _b; });
3048 const int to_parameter_size = to_compiled_data->parameters->rnum;
3049 const int rnum = (to_param_ref < 0) ? to_parameter_indices->rnum : 1;
3050 assert(aux_in_size >= 0)((void) sizeof ((aux_in_size >= 0) ? 1 : 0), __extension__
({ if (aux_in_size >= 0) ; else __assert_fail ("aux_in_size >= 0"
, "ccv_cnnp_model.c", 3050, __extension__ __PRETTY_FUNCTION__
); }));
3051 assert(aux_out_size >= 0)((void) sizeof ((aux_out_size >= 0) ? 1 : 0), __extension__
({ if (aux_out_size >= 0) ; else __assert_fail ("aux_out_size >= 0"
, "ccv_cnnp_model.c", 3051, __extension__ __PRETTY_FUNCTION__
); }));
3052 int i, j;
3053 ccv_nnc_tensor_t* inputs[aux_in_size + 1];
3054 ccv_nnc_tensor_t* outputs[aux_out_size + 1];
3055 for (i = 0; i < aux_in_size; i++)
3056 inputs[i + 1] = aux_ins[i];
3057 for (i = 0; i < aux_out_size; i++)
3058 outputs[i + 1] = aux_outs[i];
3059 for (i = 0; i < rnum; i++)
3060 {
3061 const int dest_d = *(int*)ccv_array_get(to_parameter_indices, to_param_ref >= 0 ? to_param_ref : i)((void*)(((char*)((to_parameter_indices)->data)) + (size_t
)(to_parameter_indices)->rsize * (size_t)(to_param_ref >=
0 ? to_param_ref : i)));
3062 assert(dest_d >= 0)((void) sizeof ((dest_d >= 0) ? 1 : 0), __extension__ ({ if
(dest_d >= 0) ; else __assert_fail ("dest_d >= 0", "ccv_cnnp_model.c"
, 3062, __extension__ __PRETTY_FUNCTION__); }));
3063 assert(dest_d < to_compiled_data->parameters->rnum)((void) sizeof ((dest_d < to_compiled_data->parameters->
rnum) ? 1 : 0), __extension__ ({ if (dest_d < to_compiled_data
->parameters->rnum) ; else __assert_fail ("dest_d < to_compiled_data->parameters->rnum"
, "ccv_cnnp_model.c", 3063, __extension__ __PRETTY_FUNCTION__
); }));
3064 if (parallel_count > 1)
3065 {
3066 ccv_nnc_stream_context_t* streams[parallel_count];
3067 ccv_nnc_stream_signal_t* signal;
3068 if (stream_context)
3069 signal = ccv_nnc_stream_context_emit_signal_new(stream_context);
3070 for (j = 0; j < parallel_count; j++)
3071 {
3072 ccv_nnc_tensor_t* const dest = CCV_NNC_TENSOR(to_compiled_data->tensors.parameters[dest_d + j * to_parameter_size])((ccv_nnc_tensor_t*)((uintptr_t)(to_compiled_data->tensors
.parameters[dest_d + j * to_parameter_size]) & ~(uintptr_t
)1));
3073 if (!dest)
3074 {
3075 streams[j] = 0;
3076 continue;
3077 }
3078 const int stream_type = CCV_TENSOR_GET_MEMORY(dest->info.type)((dest->info.type) & 0x3) == CCV_TENSOR_GPU_MEMORY ? CCV_STREAM_CONTEXT_GPU : CCV_STREAM_CONTEXT_CPU;
3079 const int device_id = CCV_TENSOR_GET_DEVICE_ID(dest->info.type)(((dest->info.type) & 0xfff00) >> 8);
3080 int type = stream_type;
3081 CCV_STREAM_SET_DEVICE_ID(type, device_id)(type) = (((type) & ~0xfff00) | (((device_id) & 0xfff
) << 8));
3082 ccv_nnc_stream_context_t* const stream_0 = ccv_cnnp_compiled_data_get_stream(to_compiled_data, type);
3083 // Wait signal to finish.
3084 if (stream_context)
3085 ccv_nnc_stream_context_wait_signal(stream_0, signal);
3086 inputs[0] = outputs[0] = dest;
3087 ccv_nnc_cmd_exec(cmd, hint, flags, inputs, aux_in_size + 1, outputs, aux_out_size + 1, stream_0);
3088 if (stream_context)
3089 {
3090 ccv_nnc_stream_signal_t* const signal = ccv_nnc_stream_context_emit_signal_new(stream_0);
3091 ccv_nnc_stream_context_wait_signal(stream_context, signal);
3092 }
3093 streams[j] = stream_0;
3094 }
3095 // If this should be blocking, blocking it.
3096 if (!stream_context)
3097 for (j = 0; j < parallel_count; j++)
3098 if (streams[j])
3099 ccv_nnc_stream_context_wait(streams[j]);
3100 } else {
3101 ccv_nnc_tensor_t* const dest = CCV_NNC_TENSOR(to_compiled_data->tensors.parameters[dest_d])((ccv_nnc_tensor_t*)((uintptr_t)(to_compiled_data->tensors
.parameters[dest_d]) & ~(uintptr_t)1));
3102 assert(dest)((void) sizeof ((dest) ? 1 : 0), __extension__ ({ if (dest) ;
else __assert_fail ("dest", "ccv_cnnp_model.c", 3102, __extension__
__PRETTY_FUNCTION__); }));
3103 inputs[0] = outputs[0] = dest;
3104 ccv_nnc_cmd_exec(cmd, hint, flags, inputs, aux_in_size + 1, outputs, aux_out_size + 1, stream_context);
3105 }
3106 // No need to mark this symbol as init'ed, it is already.
3107 }
3108 ccv_array_free(to_parameter_indices);
3109}
3110
3111void ccv_cnnp_model_parameter_gradients_map(ccv_cnnp_model_t* const model, const ccv_cnnp_model_io_t parameters, const ccv_nnc_cmd_t cmd, const ccv_nnc_hint_t hint, const int flags, ccv_nnc_tensor_t* const* const aux_ins, const int aux_in_size, ccv_nnc_tensor_t* const* const aux_outs, const int aux_out_size, ccv_nnc_stream_context_t* const stream_context)
3112{
3113 int to_param_ref;
3114 ccv_array_t* const to_parameter_indices = _ccv_cnnp_model_parameter_indices(model, parameters, &to_param_ref);
3115 // To models.
3116 ccv_cnnp_compiled_data_t* const to_compiled_data = model->compiled_data;
3117 assert(to_compiled_data)((void) sizeof ((to_compiled_data) ? 1 : 0), __extension__ ({
if (to_compiled_data) ; else __assert_fail ("to_compiled_data"
, "ccv_cnnp_model.c", 3117, __extension__ __PRETTY_FUNCTION__
); }));
3118 // Tensor has to be inited already.
3119 assert(!!to_compiled_data->tensors_init.v)((void) sizeof ((!!to_compiled_data->tensors_init.v) ? 1 :
0), __extension__ ({ if (!!to_compiled_data->tensors_init
.v) ; else __assert_fail ("!!to_compiled_data->tensors_init.v"
, "ccv_cnnp_model.c", 3119, __extension__ __PRETTY_FUNCTION__
); }));
3120 ccv_nnc_tensor_t** tensor_gradients;
3121 if (to_compiled_data->backward.count > 1)
3122 tensor_gradients = to_compiled_data->tensors.accum_gradients;
3123 else
3124 tensor_gradients = to_compiled_data->tensors.gradients;
3125 assert(tensor_gradients)((void) sizeof ((tensor_gradients) ? 1 : 0), __extension__ ({
if (tensor_gradients) ; else __assert_fail ("tensor_gradients"
, "ccv_cnnp_model.c", 3125, __extension__ __PRETTY_FUNCTION__
); }));
3126 // From models.
3127 const int parallel_count = ccv_max(model->parallel_count, 1)({ typeof (model->parallel_count) _a = (model->parallel_count
); typeof (1) _b = (1); (_a > _b) ? _a : _b; });
3128 const int to_parameter_size = to_compiled_data->parameters->rnum;
3129 const int rnum = (to_param_ref < 0) ? to_parameter_indices->rnum : 1;
3130 assert(aux_in_size >= 0)((void) sizeof ((aux_in_size >= 0) ? 1 : 0), __extension__
({ if (aux_in_size >= 0) ; else __assert_fail ("aux_in_size >= 0"
, "ccv_cnnp_model.c", 3130, __extension__ __PRETTY_FUNCTION__
); }));
3131 assert(aux_out_size >= 0)((void) sizeof ((aux_out_size >= 0) ? 1 : 0), __extension__
({ if (aux_out_size >= 0) ; else __assert_fail ("aux_out_size >= 0"
, "ccv_cnnp_model.c", 3131, __extension__ __PRETTY_FUNCTION__
); }));
3132 int i, j;
3133 ccv_nnc_tensor_t* inputs[aux_in_size + 1];
3134 ccv_nnc_tensor_t* outputs[aux_out_size + 1];
3135 for (i = 0; i < aux_in_size; i++)
3136 inputs[i + 1] = aux_ins[i];
3137 for (i = 0; i < aux_out_size; i++)
3138 outputs[i + 1] = aux_outs[i];
3139 for (i = 0; i < rnum; i++)
3140 {
3141 const int dest_d = *(int*)ccv_array_get(to_parameter_indices, to_param_ref >= 0 ? to_param_ref : i)((void*)(((char*)((to_parameter_indices)->data)) + (size_t
)(to_parameter_indices)->rsize * (size_t)(to_param_ref >=
0 ? to_param_ref : i)));
3142 assert(dest_d >= 0)((void) sizeof ((dest_d >= 0) ? 1 : 0), __extension__ ({ if
(dest_d >= 0) ; else __assert_fail ("dest_d >= 0", "ccv_cnnp_model.c"
, 3142, __extension__ __PRETTY_FUNCTION__); }));
3143 assert(dest_d < to_compiled_data->parameters->rnum)((void) sizeof ((dest_d < to_compiled_data->parameters->
rnum) ? 1 : 0), __extension__ ({ if (dest_d < to_compiled_data
->parameters->rnum) ; else __assert_fail ("dest_d < to_compiled_data->parameters->rnum"
, "ccv_cnnp_model.c", 3143, __extension__ __PRETTY_FUNCTION__
); }));
3144 if (parallel_count > 1)
3145 {
3146 ccv_nnc_stream_context_t* streams[parallel_count];
3147 ccv_nnc_stream_signal_t* signal;
3148 if (stream_context)
3149 signal = ccv_nnc_stream_context_emit_signal_new(stream_context);
3150 for (j = 0; j < parallel_count; j++)
3151 {
3152 ccv_nnc_tensor_t* const dest = tensor_gradients[dest_d + j * to_parameter_size];
3153 if (!dest)
3154 {
3155 streams[j] = 0;
3156 continue;
3157 }
3158 const int stream_type = CCV_TENSOR_GET_MEMORY(dest->info.type)((dest->info.type) & 0x3) == CCV_TENSOR_GPU_MEMORY ? CCV_STREAM_CONTEXT_GPU : CCV_STREAM_CONTEXT_CPU;
3159 const int device_id = CCV_TENSOR_GET_DEVICE_ID(dest->info.type)(((dest->info.type) & 0xfff00) >> 8);
3160 int type = stream_type;
3161 CCV_STREAM_SET_DEVICE_ID(type, device_id)(type) = (((type) & ~0xfff00) | (((device_id) & 0xfff
) << 8));
3162 ccv_nnc_stream_context_t* const stream_0 = ccv_cnnp_compiled_data_get_stream(to_compiled_data, type);
3163 // Wait signal to finish.
3164 if (stream_context)
3165 ccv_nnc_stream_context_wait_signal(stream_0, signal);
3166 inputs[0] = outputs[0] = dest;
3167 ccv_nnc_cmd_exec(cmd, hint, flags, inputs, aux_in_size + 1, outputs, aux_out_size + 1, stream_0);
3168 if (stream_context)
3169 {
3170 ccv_nnc_stream_signal_t* const signal = ccv_nnc_stream_context_emit_signal_new(stream_0);
3171 ccv_nnc_stream_context_wait_signal(stream_context, signal);
3172 }
3173 streams[j] = stream_0;
3174 }
3175 // If this should be blocking, blocking it.
3176 if (!stream_context)
3177 for (j = 0; j < parallel_count; j++)
3178 if (streams[j])
3179 ccv_nnc_stream_context_wait(streams[j]);
3180 } else {
3181 ccv_nnc_tensor_t* const dest = tensor_gradients[dest_d];
3182 if (!dest)
3183 continue;
3184 assert(dest)((void) sizeof ((dest) ? 1 : 0), __extension__ ({ if (dest) ;
else __assert_fail ("dest", "ccv_cnnp_model.c", 3184, __extension__
__PRETTY_FUNCTION__); }));
3185 inputs[0] = outputs[0] = dest;
3186 ccv_nnc_cmd_exec(cmd, hint, flags, inputs, aux_in_size + 1, outputs, aux_out_size + 1, stream_context);
3187 }
3188 // No need to mark this symbol as init'ed, it is already.
3189 }
3190 ccv_array_free(to_parameter_indices);
3191}
3192
3193void ccv_cnnp_model_parameters_to_unified_memory(ccv_cnnp_model_t* const model, const ccv_cnnp_model_io_t parameters, ccv_nnc_stream_context_t* const stream_context)
3194{
3195 // Only CUDA backend has this feature.
3196#ifdef HAVE_CUDA1
3197 int to_param_ref;
3198 ccv_array_t* const to_parameter_indices = _ccv_cnnp_model_parameter_indices(model, parameters, &to_param_ref);
3199 // To models.
3200 ccv_cnnp_compiled_data_t* const compiled_data = model->compiled_data;
3201 assert(compiled_data)((void) sizeof ((compiled_data) ? 1 : 0), __extension__ ({ if
(compiled_data) ; else __assert_fail ("compiled_data", "ccv_cnnp_model.c"
, 3201, __extension__ __PRETTY_FUNCTION__); }));
3202 // Tensor has to be inited already.
3203 assert(!!compiled_data->tensors_init.v)((void) sizeof ((!!compiled_data->tensors_init.v) ? 1 : 0)
, __extension__ ({ if (!!compiled_data->tensors_init.v) ; else
__assert_fail ("!!compiled_data->tensors_init.v", "ccv_cnnp_model.c"
, 3203, __extension__ __PRETTY_FUNCTION__); }));
3204 assert(compiled_data->tensors.parameters)((void) sizeof ((compiled_data->tensors.parameters) ? 1 : 0
), __extension__ ({ if (compiled_data->tensors.parameters)
; else __assert_fail ("compiled_data->tensors.parameters"
, "ccv_cnnp_model.c", 3204, __extension__ __PRETTY_FUNCTION__
); }));
3205 // From models.
3206 const int parallel_count = ccv_max(model->parallel_count, 1)({ typeof (model->parallel_count) _a = (model->parallel_count
); typeof (1) _b = (1); (_a > _b) ? _a : _b; });
3207 const int rnum = (to_param_ref < 0) ? to_parameter_indices->rnum : 1;
3208 int i;
3209 for (i = 0; i < rnum; i++)
3210 {
3211 const int dest_d = *(int*)ccv_array_get(to_parameter_indices, to_param_ref >= 0 ? to_param_ref : i)((void*)(((char*)((to_parameter_indices)->data)) + (size_t
)(to_parameter_indices)->rsize * (size_t)(to_param_ref >=
0 ? to_param_ref : i)));
3212 assert(dest_d >= 0)((void) sizeof ((dest_d >= 0) ? 1 : 0), __extension__ ({ if
(dest_d >= 0) ; else __assert_fail ("dest_d >= 0", "ccv_cnnp_model.c"
, 3212, __extension__ __PRETTY_FUNCTION__); }));
3213 assert(dest_d < compiled_data->parameters->rnum)((void) sizeof ((dest_d < compiled_data->parameters->
rnum) ? 1 : 0), __extension__ ({ if (dest_d < compiled_data
->parameters->rnum) ; else __assert_fail ("dest_d < compiled_data->parameters->rnum"
, "ccv_cnnp_model.c", 3213, __extension__ __PRETTY_FUNCTION__
); }));
3214 if (parallel_count > 1)
3215 {
3216 assert(0 && "Cannot support this when data parallel is in effect.")((void) sizeof ((0 && "Cannot support this when data parallel is in effect."
) ? 1 : 0), __extension__ ({ if (0 && "Cannot support this when data parallel is in effect."
) ; else __assert_fail ("0 && \"Cannot support this when data parallel is in effect.\""
, "ccv_cnnp_model.c", 3216, __extension__ __PRETTY_FUNCTION__
); }));
3217 } else {
3218 ccv_nnc_tensor_t* const src = CCV_NNC_TENSOR(compiled_data->tensors.parameters[dest_d])((ccv_nnc_tensor_t*)((uintptr_t)(compiled_data->tensors.parameters
[dest_d]) & ~(uintptr_t)1));
3219 assert(src)((void) sizeof ((src) ? 1 : 0), __extension__ ({ if (src) ; else
__assert_fail ("src", "ccv_cnnp_model.c", 3219, __extension__
__PRETTY_FUNCTION__); }));
3220 ccv_nnc_tensor_param_t params = src->info;
3221 if (CCV_TENSOR_GET_MEMORY(params.type)((params.type) & 0x3) != CCV_TENSOR_GPU_MEMORY)
3222 continue;
3223 const size_t size = ccv_nnc_tensor_data_size(params);
3224 if (size <= 0)
3225 continue;
3226 const int should_free = !((uintptr_t)compiled_data->tensors.parameters[dest_d] & (uintptr_t)1);
3227 const int tfb = (CCV_TENSOR_GET_MEMORY(params.type)((params.type) & 0x3) == CCV_TENSOR_CPU_MEMORY && params.format == CCV_TENSOR_FORMAT_NHWC && params.dim[2] > 0 && params.dim[2] <= CCV_MAX_CHANNEL(0xFFF) && params.dim[0] > 0 && params.dim[1] > 0 && params.dim[3] == 0);
3228 ccv_nnc_tensor_t* const tensor = (ccv_nnc_tensor_t*)ccmallocmalloc(sizeof(ccv_nnc_tensor_t));
3229 tensor->dataof = 0;
3230 tensor->alias_ref = 0;
3231 tensor->sig = 0;
3232 tensor->refcount = 1;
3233 tensor->info = params;
3234 if (tfb)
3235 {
3236 tensor->type = CCV_NO_DATA_ALLOC | CCV_MATRIX_DENSE | CCV_GET_DATA_TYPE(params.datatype)((params.datatype) & 0xFF000) | params.dim[2];
3237 // This corresponding to mat->step
3238 tensor->info.dim[4] = CCV_GET_STEP(params.dim[1], (CCV_GET_DATA_TYPE(params.datatype) | params.dim[2]))(((params.dim[1]) * _ccv_get_data_type_size[(((((params.datatype
) & 0xFF000) | params.dim[2])) & 0xFF000) >> 12
] * (((((params.datatype) & 0xFF000) | params.dim[2])) &
0xFFF) + 3) & -4);
3239 } else // This won't be recognized by ccv_dense_matrix_t
3240 tensor->type = CCV_NO_DATA_ALLOC | CCV_MATRIX_DENSE | CCV_GET_DATA_TYPE(params.datatype)((params.datatype) & 0xFF000);
3241 // Remove this flag so it can be deallocated as usual.
3242 tensor->type &= ~CCV_NO_DATA_ALLOC;
3243 assert(CCV_TENSOR_GET_DEVICE(params.type) != CCV_COMPUTE_DEVICE_ANY)((void) sizeof ((((params.type) & 0xfff00) != CCV_COMPUTE_DEVICE_ANY
) ? 1 : 0), __extension__ ({ if (((params.type) & 0xfff00
) != CCV_COMPUTE_DEVICE_ANY) ; else __assert_fail ("CCV_TENSOR_GET_DEVICE(params.type) != CCV_COMPUTE_DEVICE_ANY"
, "ccv_cnnp_model.c", 3243, __extension__ __PRETTY_FUNCTION__
); }));
3244 void* ptr = cumallocmanaged(CCV_TENSOR_GET_DEVICE_ID(params.type)(((params.type) & 0xfff00) >> 8), size);
3245 if (ptr) // If allocated successfully. Otherwise we go through the fallback path.
3246 {
3247 tensor->data.u8 = (uint8_t*)ptr;
3248 tensor->type |= CCV_MAPPED_MEM; // This denotes the tensor is mapped to CPU, and would prefer a explicit prefetch call.
3249 } else {
3250 // Allocation failed.
3251 ccfreefree(tensor);
3252 continue;
3253 }
3254 // TODO: Cannot run this on the stream context yet, due to allocation and deallocations.
3255 ccv_nnc_cmd_exec(CMD_DATA_TRANSFER_FORWARD()ccv_nnc_cmd(CCV_NNC_DATA_TRANSFER_FORWARD, 0, ccv_nnc_cmd_auto
, 0), ccv_nnc_no_hint, 0, &src, 1, &tensor, 1, 0);
3256 cumemadvisereadmostly(CCV_TENSOR_GET_DEVICE_ID(params.type)(((params.type) & 0xfff00) >> 8), tensor->data.u8, size);
3257 compiled_data->tensors.parameters[dest_d] = tensor;
3258 // Can free out the old one.
3259 if (should_free)
3260 ccv_nnc_tensor_free(src);
3261 }
3262 // No need to mark this symbol as init'ed, it is already.
3263 }
3264 ccv_array_free(to_parameter_indices);
3265#endif
3266}
3267
3268ccv_nnc_cmd_t ccv_cnnp_model_minimizer(ccv_cnnp_model_t* const model)
3269{
3270 ccv_cnnp_compiled_data_t* const compiled_data = model->compiled_data;
3271 assert(compiled_data)((void) sizeof ((compiled_data) ? 1 : 0), __extension__ ({ if
(compiled_data) ; else __assert_fail ("compiled_data", "ccv_cnnp_model.c"
, 3271, __extension__ __PRETTY_FUNCTION__); }));
3272 return compiled_data->minimize.minimizer;
3273}
3274
3275void ccv_cnnp_model_set_minimizer(ccv_cnnp_model_t* const model, const ccv_nnc_cmd_t minimizer, const int reset, const ccv_cnnp_model_io_t* const set_parameters, const int set_parameter_size)
3276{
3277 ccv_cnnp_compiled_data_t* const compiled_data = model->compiled_data;
3278 assert(compiled_data)((void) sizeof ((compiled_data) ? 1 : 0), __extension__ ({ if
(compiled_data) ; else __assert_fail ("compiled_data", "ccv_cnnp_model.c"
, 3278, __extension__ __PRETTY_FUNCTION__); }));
3279 const int parameter_size = compiled_data->parameters->rnum;
3280 if (parameter_size == 0)
3281 return;
3282 if (reset)
3283 { assert(set_parameters == 0 && set_parameter_size == 0)((void) sizeof ((set_parameters == 0 && set_parameter_size
== 0) ? 1 : 0), __extension__ ({ if (set_parameters == 0 &&
set_parameter_size == 0) ; else __assert_fail ("set_parameters == 0 && set_parameter_size == 0"
, "ccv_cnnp_model.c", 3283, __extension__ __PRETTY_FUNCTION__
); })); }
3284 const int old_max_saved_aux_size = compiled_data->minimize.max_saved_aux_size;
3285 const int saved_aux_size = ccv_nnc_minimizer_saved_aux_size(minimizer);
3286 if (saved_aux_size > compiled_data->minimize.max_saved_aux_size)
3287 compiled_data->minimize.max_saved_aux_size = saved_aux_size;
3288 const int max_saved_aux_size = compiled_data->minimize.max_saved_aux_size;
3289 // We update all parameters, at this point, we have one minimizer.
3290 if (set_parameters == 0 || set_parameter_size == 0)
3291 compiled_data->minimize.minimizer = minimizer;
3292 int i;
3293 if (set_parameters && set_parameter_size)
3294 {
3295 // I need to save what's the minimizer along with this.
3296 if (!compiled_data->minimize.parameters)
3297 compiled_data->minimize.parameters = ccv_array_new(sizeof(ccv_cnnp_set_minimizer_for_parameter_t*), 1, 0);
3298 ccv_cnnp_set_minimizer_for_parameter_t* const set_minimizer_for_parameter = ccmallocmalloc(sizeof(ccv_cnnp_set_minimizer_for_parameter_t) + (set_parameter_size - 1) * sizeof(ccv_cnnp_model_io_t));
3299 set_minimizer_for_parameter->minimizer = minimizer;
3300 set_minimizer_for_parameter->parameter_size = set_parameter_size;
3301 memcpy(set_minimizer_for_parameter->parameters, set_parameters, sizeof(ccv_cnnp_model_io_t) * set_parameter_size);
3302 ccv_array_push(compiled_data->minimize.parameters, &set_minimizer_for_parameter);
3303 }
3304 // If reset is true, clear the parameters array.
3305 if (reset && compiled_data->minimize.parameters)
3306 {
3307 for (i = 0; i < compiled_data->minimize.parameters->rnum; i++)
3308 ccfreefree(*(ccv_cnnp_set_minimizer_for_parameter_t**)ccv_array_get(compiled_data->minimize.parameters, i)((void*)(((char*)((compiled_data->minimize.parameters)->
data)) + (size_t)(compiled_data->minimize.parameters)->
rsize * (size_t)(i))));
3309 ccv_array_clear(compiled_data->minimize.parameters);
3310 }
3311 if (!compiled_data->update_nodes)
3312 return;
3313 ccv_nnc_symbolic_graph_t* const symbolic_graph = model->graph;
3314 assert(symbolic_graph)((void) sizeof ((symbolic_graph) ? 1 : 0), __extension__ ({ if
(symbolic_graph) ; else __assert_fail ("symbolic_graph", "ccv_cnnp_model.c"
, 3314, __extension__ __PRETTY_FUNCTION__); }));
3315 if (saved_aux_size > old_max_saved_aux_size)
3316 {
3317 assert(compiled_data->updated_parameters)((void) sizeof ((compiled_data->updated_parameters) ? 1 : 0
), __extension__ ({ if (compiled_data->updated_parameters)
; else __assert_fail ("compiled_data->updated_parameters"
, "ccv_cnnp_model.c", 3317, __extension__ __PRETTY_FUNCTION__
); }));
3318 // Reallocate first, move them around later.
3319 compiled_data->updated_parameters = (ccv_nnc_tensor_symbol_t*)ccreallocrealloc(compiled_data->updated_parameters, sizeof(ccv_nnc_tensor_symbol_t) * parameter_size + sizeof(ccv_nnc_graph_exec_symbol_t) * parameter_size + sizeof(ccv_nnc_tensor_symbol_map_t) * saved_aux_size * parameter_size);
3320 compiled_data->update_nodes = (ccv_nnc_graph_exec_symbol_t*)(compiled_data->updated_parameters + parameter_size);
3321 compiled_data->saved_aux = (ccv_nnc_tensor_symbol_map_t*)(compiled_data->update_nodes + parameter_size);
3322 // We need to do this from back to front because saved_aux_size > old_saved_aux_size, it could overlap.
3323 _ccv_cnnp_scatter_saved_aux(compiled_data->saved_aux, parameter_size, old_max_saved_aux_size, saved_aux_size);
3324 }
3325 int flag = 0;
3326 const int parallel_count = ccv_max(model->parallel_count, 1)({ typeof (model->parallel_count) _a = (model->parallel_count
); typeof (1) _b = (1); (_a > _b) ? _a : _b; });
3327 if (set_parameters && set_parameter_size)
3328 {
3329 ccv_array_t* const parameter_indices = ccv_array_new(sizeof(int), 0, 0);
3330 for (i = 0; i < set_parameter_size; i++)
3331 {
3332 const int param_sel = set_parameters[i]->param_sel > 0 ? set_parameters[i]->param_sel - 1 : set_parameters[i]->param_sel;
3333 assert(set_parameters[i]->param_sel != 0)((void) sizeof ((set_parameters[i]->param_sel != 0) ? 1 : 0
), __extension__ ({ if (set_parameters[i]->param_sel != 0)
; else __assert_fail ("set_parameters[i]->param_sel != 0"
, "ccv_cnnp_model.c", 3333, __extension__ __PRETTY_FUNCTION__
); }));
3334 const int old_rnum = parameter_indices->rnum;
3335 ccv_cnnp_model_add_to_parameter_indices(set_parameters[i]->model, param_sel, parameter_indices);
3336 const int param_ref = set_parameters[i]->param_ref > 0 ? set_parameters[i]->param_ref - 1 : set_parameters[i]->param_ref;
3337 assert(set_parameters[i]->param_ref != 0)((void) sizeof ((set_parameters[i]->param_ref != 0) ? 1 : 0
), __extension__ ({ if (set_parameters[i]->param_ref != 0)
; else __assert_fail ("set_parameters[i]->param_ref != 0"
, "ccv_cnnp_model.c", 3337, __extension__ __PRETTY_FUNCTION__
); }));
3338 if (param_ref >= 0)
3339 {
3340 assert(param_ref + old_rnum < parameter_indices->rnum)((void) sizeof ((param_ref + old_rnum < parameter_indices->
rnum) ? 1 : 0), __extension__ ({ if (param_ref + old_rnum <
parameter_indices->rnum) ; else __assert_fail ("param_ref + old_rnum < parameter_indices->rnum"
, "ccv_cnnp_model.c", 3340, __extension__ __PRETTY_FUNCTION__
); }));
3341 *(int*)ccv_array_get(parameter_indices, old_rnum)((void*)(((char*)((parameter_indices)->data)) + (size_t)(parameter_indices
)->rsize * (size_t)(old_rnum))) = *(int*)ccv_array_get(parameter_indices, param_ref + old_rnum)((void*)(((char*)((parameter_indices)->data)) + (size_t)(parameter_indices
)->rsize * (size_t)(param_ref + old_rnum)));
3342 parameter_indices->rnum = old_rnum + 1;
3343 }
3344 }
3345 // We may have duplicated indices, but that is OK, we will set it twice.
3346 for (i = 0; i < parameter_indices->rnum; i++)
3347 {
3348 const int d = *(int*)ccv_array_get(parameter_indices, i)((void*)(((char*)((parameter_indices)->data)) + (size_t)(parameter_indices
)->rsize * (size_t)(i)));
3349 if (_ccv_cnnp_set_minimizer_for_parameter(symbolic_graph, compiled_data, compiled_data->update_nodes, compiled_data->updated_parameters, compiled_data->saved_aux, parallel_count, minimizer, saved_aux_size, max_saved_aux_size, d))
3350 flag = 1;
3351 }
3352 ccv_array_free(parameter_indices);
3353 } else {
3354 for (i = 0; i < parameter_size; i++)
3355 if (_ccv_cnnp_set_minimizer_for_parameter(symbolic_graph, compiled_data, compiled_data->update_nodes, compiled_data->updated_parameters, compiled_data->saved_aux, parallel_count, minimizer, saved_aux_size, max_saved_aux_size, i))
3356 flag = 1;
3357 if (compiled_data->minimize.parameters)
3358 if (_ccv_cnnp_apply_parameters_with_minimizer(model))
3359 flag = 1;
3360 }
3361 if (flag)
3362 {
3363 // If saved_aux_size doesn't match, we need to remove / add new saved_aux to the graph. But first, free up apply gradients graph.
3364 if (compiled_data->graph_mode == CCV_CNNP_MODEL_GRAPH_FIT_MODE)
3365 _ccv_cnnp_compiled_data_graph_free(compiled_data);
3366 _ccv_cnnp_compiled_data_apply_gradients_free(compiled_data);
3367 }
3368}
3369
3370void ccv_cnnp_model_set_compile_params(ccv_cnnp_model_t* const model, const ccv_nnc_symbolic_graph_compile_param_t compile_params)
3371{
3372 ccv_cnnp_compiled_data_t* const compiled_data = model->compiled_data;
3373 assert(compiled_data)((void) sizeof ((compiled_data) ? 1 : 0), __extension__ ({ if
(compiled_data) ; else __assert_fail ("compiled_data", "ccv_cnnp_model.c"
, 3373, __extension__ __PRETTY_FUNCTION__); }));
3374 compiled_data->compile_params = compile_params;
3375}
3376
3377void ccv_cnnp_model_dot(const ccv_cnnp_model_t* const model, const int flags, FILE** const outs, const int out_size)
3378{
3379 if (model->graph && out_size > 0)
3380 ccv_nnc_symbolic_graph_dot(model->graph, flags, outs[0]);
3381 if (model->compiled_data && model->compiled_data->graph && out_size > 1)
3382 ccv_nnc_graph_dot(model->compiled_data->graph, flags, outs[1]);
3383 if (model->compiled_data && model->compiled_data->backward.accum && out_size > 2)
3384 ccv_nnc_graph_dot(model->compiled_data->backward.accum, flags, outs[2]);
3385 if (model->compiled_data && model->compiled_data->apply_gradients.graph && out_size > 3)
3386 ccv_nnc_graph_dot(model->compiled_data->apply_gradients.graph, flags, outs[3]);
3387}
3388
3389void ccv_cnnp_model_format(const ccv_cnnp_model_t* const model, const ccv_nnc_symbolic_graph_format_f format_fn, void* const context)
3390{
3391 if (model->graph)
3392 ccv_nnc_symbolic_graph_format(model->graph, 0, 0, 0, 0, format_fn, context);
3393}
3394
3395static void _ccv_cnnp_compiled_data_free(const ccv_cnnp_model_t* const model, ccv_cnnp_compiled_data_t* const compiled_data)
3396{
3397 int i;
3398 const int parameter_size = compiled_data->parameters->rnum;
3399 ccv_array_free(compiled_data->parameters);
3400 if (compiled_data->parameter_flags)
3401 ccfreefree(compiled_data->parameter_flags);
3402 const int internal_size = compiled_data->internals->rnum;
3403 ccv_array_free(compiled_data->internals);
3404 assert(compiled_data->ids.parameters->rnum == parameter_size)((void) sizeof ((compiled_data->ids.parameters->rnum ==
parameter_size) ? 1 : 0), __extension__ ({ if (compiled_data
->ids.parameters->rnum == parameter_size) ; else __assert_fail
("compiled_data->ids.parameters->rnum == parameter_size"
, "ccv_cnnp_model.c", 3404, __extension__ __PRETTY_FUNCTION__
); }));
3405 assert(compiled_data->ids.internals->rnum == internal_size)((void) sizeof ((compiled_data->ids.internals->rnum == internal_size
) ? 1 : 0), __extension__ ({ if (compiled_data->ids.internals
->rnum == internal_size) ; else __assert_fail ("compiled_data->ids.internals->rnum == internal_size"
, "ccv_cnnp_model.c", 3405, __extension__ __PRETTY_FUNCTION__
); }));
3406 for (i = 0; i < parameter_size; i++)
3407 ccfreefree(*(char**)ccv_array_get(compiled_data->ids.parameters, i)((void*)(((char*)((compiled_data->ids.parameters)->data
)) + (size_t)(compiled_data->ids.parameters)->rsize * (
size_t)(i))));
3408 ccv_array_free(compiled_data->ids.parameters);
3409 for (i = 0; i < internal_size; i++)
3410 ccfreefree(*(char**)ccv_array_get(compiled_data->ids.internals, i)((void*)(((char*)((compiled_data->ids.internals)->data)
) + (size_t)(compiled_data->ids.internals)->rsize * (size_t
)(i))));
3411 ccv_array_free(compiled_data->ids.internals);
3412 const int parallel_count = compiled_data->parallel_count > 0 ? compiled_data->parallel_count : _ccv_cnnp_model_root_parallel_count(model);
3413 if (compiled_data->tensors.parameters)
3414 {
3415 for (i = 0; i < parameter_size * parallel_count; i++)
3416 // If it is not marked as not belonging, we can free it.
3417 if (!((uintptr_t)compiled_data->tensors.parameters[i] & (uintptr_t)1))
3418 if (compiled_data->tensors.parameters[i])
3419 ccv_nnc_tensor_free(compiled_data->tensors.parameters[i]);
3420 for (i = 0; i < internal_size * parallel_count; i++)
3421 if (compiled_data->tensors.internals[i])
3422 ccv_nnc_tensor_free(compiled_data->tensors.internals[i]);
3423 ccfreefree(compiled_data->tensors.parameters);
3424 }
3425 if (compiled_data->tensors.gradients)
3426 {
3427 for (i = 0; i < parameter_size * parallel_count; i++)
3428 {
3429 if (compiled_data->tensors.gradients[i])
3430 ccv_nnc_tensor_free(compiled_data->tensors.gradients[i]);
3431 if (compiled_data->tensors.accum_gradients[i])
3432 ccv_nnc_tensor_free(compiled_data->tensors.accum_gradients[i]);
3433 }
3434 ccfreefree(compiled_data->tensors.gradients);
3435 }
3436 if (compiled_data->minimize.parameters)
3437 {
3438 for (i = 0; i < compiled_data->minimize.parameters->rnum; i++)
3439 ccfreefree(*(ccv_cnnp_set_minimizer_for_parameter_t**)ccv_array_get(compiled_data->minimize.parameters, i)((void*)(((char*)((compiled_data->minimize.parameters)->
data)) + (size_t)(compiled_data->minimize.parameters)->
rsize * (size_t)(i))));
3440 ccv_array_free(compiled_data->minimize.parameters);
3441 }
3442 if (compiled_data->rewindables)
3443 ccv_array_free(compiled_data->rewindables);
3444 if (compiled_data->tensors_init.v)
3445 ccfreefree(CCV_NNC_INIT_V(compiled_data->tensors_init.v)((uint32_t*)((uintptr_t)(compiled_data->tensors_init.v) &
~(uintptr_t)1)));
3446 if (compiled_data->evaluate.tos)
3447 ccfreefree(compiled_data->evaluate.tos);
3448 compiled_data->evaluate.tos = 0;
3449 if (compiled_data->stream_map)
3450 {
3451 khiter_t k;
3452 for (k = kh_begin(compiled_data->stream_map)(khint_t)(0); k != kh_end(compiled_data->stream_map)((compiled_data->stream_map)->n_buckets); ++k)
3453 {
3454 if (!kh_exist(compiled_data->stream_map, k)(!(((compiled_data->stream_map)->flags[(k)>>4]>>
(((k)&0xfU)<<1))&3)))
3455 continue;
3456 ccv_nnc_stream_context_t* const stream = kh_val(compiled_data->stream_map, k)((compiled_data->stream_map)->vals[k]);
3457 ccv_nnc_stream_context_free(stream);
3458 }
3459 kh_destroy(stream_map, compiled_data->stream_map)kh_destroy_stream_map(compiled_data->stream_map);
3460 }
3461 _ccv_cnnp_compiled_data_graph_free(compiled_data);
3462 _ccv_cnnp_compiled_data_gradient_free(compiled_data);
3463 _ccv_cnnp_compiled_data_backward_free(compiled_data);
3464 _ccv_cnnp_compiled_data_apply_gradients_free(compiled_data);
3465 if (compiled_data->gradient_checkpoints)
3466 {
3467 for (i = 0; i < compiled_data->gradient_checkpoints->rnum; i++)
3468 {
3469 ccv_cnnp_model_gradient_checkpoint_t* const checkpoint = (ccv_cnnp_model_gradient_checkpoint_t*)ccv_array_get(compiled_data->gradient_checkpoints, i)((void*)(((char*)((compiled_data->gradient_checkpoints)->
data)) + (size_t)(compiled_data->gradient_checkpoints)->
rsize * (size_t)(i)));
3470 assert(checkpoint->inputs)((void) sizeof ((checkpoint->inputs) ? 1 : 0), __extension__
({ if (checkpoint->inputs) ; else __assert_fail ("checkpoint->inputs"
, "ccv_cnnp_model.c", 3470, __extension__ __PRETTY_FUNCTION__
); }));
3471 ccfreefree(checkpoint->inputs);
3472 ccv_array_free(checkpoint->tensor_symbols);
3473 }
3474 ccv_array_free(compiled_data->gradient_checkpoints);
3475 }
3476 ccv_nnc_xpu_alloc_destroy(&compiled_data->xpu_alloc);
3477 ccfreefree(compiled_data);
3478}
3479
3480void ccv_cnnp_model_free(ccv_cnnp_model_t* const model)
3481{
3482 ccv_cnnp_model_pin_memory(model, 0);
3483 ccv_cnnp_model_deinit(model);
3484 if (model->isa->dealloc)
3485 model->isa->dealloc(model);
3486 if (model->io)
3487 {
3488 int i;
3489 for (i = 0; i < model->io->rnum; i++)
3490 {
3491 ccv_cnnp_model_io_t model_io = *(ccv_cnnp_model_io_t*)ccv_array_get(model->io, i)((void*)(((char*)((model->io)->data)) + (size_t)(model->
io)->rsize * (size_t)(i)));
3492 if (model_io->outgoings)
3493 ccv_array_free(model_io->outgoings);
3494 if (model_io->incomings)
3495 ccv_array_free(model_io->incomings);
3496 if (model_io->dependencies)
3497 ccv_array_free(model_io->dependencies);
3498 ccfreefree(model_io);
3499 }
3500 ccv_array_free(model->io);
3501 }
3502 if (model->parameter_indices)
3503 ccv_array_free(model->parameter_indices);
3504 if (model->inputs)
3505 ccfreefree(model->inputs);
3506 if (model->graph)
3507 ccv_nnc_symbolic_graph_free(model->graph);
3508 if (model->compiled_data)
3509 _ccv_cnnp_compiled_data_free(model, model->compiled_data);
3510 if (model->name)
3511 ccfreefree(model->name);
3512 ccfreefree(model);
3513}
3514
3515void ccv_cnnp_model_cancel(ccv_cnnp_model_t* const model)
3516{
3517 ccv_cnnp_compiled_data_t* const compiled_data = model->compiled_data;
3518 if (!compiled_data)
3519 return;
3520 if (compiled_data->graph)
3521 ccv_nnc_graph_cancel(compiled_data->graph);
3522 if (compiled_data->apply_gradients.graph)
3523 ccv_nnc_graph_cancel(compiled_data->apply_gradients.graph);
3524}
3525
3526void ccv_cnnp_model_async_enter(ccv_cnnp_model_t* const model)
3527{
3528 ccv_cnnp_compiled_data_t* const compiled_data = model->compiled_data;
3529 if (!compiled_data)
3530 return;
3531 if (compiled_data->graph)
3532 ccv_nnc_graph_async_enter(compiled_data->graph);
3533 if (compiled_data->apply_gradients.graph)
3534 ccv_nnc_graph_async_enter(compiled_data->apply_gradients.graph);
3535}
3536
3537void ccv_cnnp_model_set_flags(ccv_cnnp_model_t* const model, const int flags)
3538{
3539 model->exec_flags = flags;
3540}
3541
3542int ccv_cnnp_model_flags(ccv_cnnp_model_t* const model)
3543{
3544 return model->exec_flags;
3545}