ccv_cnnp_model.c

Bug Summary

File:	nnc/ccv_cnnp_model.c
Warning:	line 2889, column 25 Array access (via field 'vals') results in a null pointer dereference
Annotated Source Code

Press '?' to see keyboard shortcuts
Show analyzer invocation
clang -cc1 -cc1 -triple x86_64-unknown-linux-gnu -analyze -disable-free -clear-ast-before-backend -disable-llvm-verifier -discard-value-names -main-file-name ccv_cnnp_model.c -analyzer-checker=core -analyzer-checker=apiModeling -analyzer-checker=unix -analyzer-checker=deadcode -analyzer-checker=security.insecureAPI.UncheckedReturn -analyzer-checker=security.insecureAPI.getpw -analyzer-checker=security.insecureAPI.gets -analyzer-checker=security.insecureAPI.mktemp -analyzer-checker=security.insecureAPI.mkstemp -analyzer-checker=security.insecureAPI.vfork -analyzer-checker=nullability.NullPassedToNonnull -analyzer-checker=nullability.NullReturnedFromNonnull -analyzer-output plist -w -setup-static-analyzer -mrelocation-model pic -pic-level 2 -pic-is-pie -mframe-pointer=none -fmath-errno -ffp-contract=on -fno-rounding-math -mconstructor-aliases -funwind-tables=2 -target-cpu x86-64 -target-feature +sse2 -tune-cpu generic -debugger-tuning=gdb -fdebug-compilation-dir=/home/liu/actions-runner/_work/ccv/ccv/lib/nnc -fcoverage-compilation-dir=/home/liu/actions-runner/_work/ccv/ccv/lib/nnc -resource-dir /usr/local/lib/clang/19 -I ../ -I /usr/local/cuda/include -D HAVE_CBLAS -D HAVE_LIBPNG -D HAVE_LIBJPEG -D HAVE_FFTW3 -D HAVE_PTHREAD -D HAVE_LIBLINEAR -D HAVE_TESSERACT -D HAVE_AVCODEC -D HAVE_AVFORMAT -D HAVE_AVUTIL -D HAVE_SWSCALE -D HAVE_SSE2 -D HAVE_GSL -D HAVE_CUDA -D HAVE_CUDNN -D HAVE_NCCL -D USE_SYSTEM_CUB -I /usr/local/include -internal-isystem /usr/local/lib/clang/19/include -internal-isystem /usr/local/include -internal-isystem /usr/lib/gcc/x86_64-linux-gnu/12/../../../../x86_64-linux-gnu/include -internal-externc-isystem /usr/include/x86_64-linux-gnu -internal-externc-isystem /include -internal-externc-isystem /usr/include -O3 -ferror-limit 19 -fgnuc-version=4.2.1 -fskip-odr-check-in-gmf -vectorize-loops -vectorize-slp -analyzer-output=html -faddrsig -D__GCC_HAVE_DWARF2_CFI_ASM=1 -o /home/liu/actions-runner/_work/ccv/ccv/_analyze/2026-09-11-054550-2650730-1 -x c ccv_cnnp_model.c
1#include "ccv_nnc.h"
2#include "ccv_nnc_easy.h"
3#include "ccv_nnc_internal.h"
4#include "ccv_internal.h"
5#include "_ccv_cnnp_model.h"
6#include "_ccv_nnc_graph.h"
7#include "_ccv_nnc_symbolic_graph.h"
8#ifdef HAVE_CUDA1
9#include "gpu/ccv_nnc_compat.h"
10#elif defined(HAVE_MPS)
11#include "mps/ccv_nnc_mps.h"
12#endif
13 
14// MARK - Level-5 API
15 
16ccv_cnnp_model_io_t ccv_cnnp_model_apply(ccv_cnnp_model_t* const model, const ccv_cnnp_model_io_t* const inputs, const int input_size)
17{
18	if (!model->io)
19		model->io = ccv_array_new(sizeof(ccv_cnnp_model_io_t), 1, 0);
20	ccv_cnnp_model_io_t model_io = ccmallocmalloc(sizeof(struct ccv_cnnp_model_io_s) + sizeof(ccv_nnc_tensor_symbol_t) * model->output_size);
21	model_io->param_ref = 0;
22	model_io->param_sel = 0;
23	model_io->visit = 0;
24	model_io->model = model;
25	model_io->dependencies = 0;
26	model_io->dependents = 0;
27	model_io->outgoings = 0;
28	model_io->outputs = (ccv_nnc_tensor_symbol_t*)(model_io + 1);
29	ccv_array_push(model->io, &model_io);
30	if (input_size > 0)
31	{
32		model_io->incomings = ccv_array_new(sizeof(ccv_cnnp_model_io_t), input_size, 0);
33		ccv_array_resize(model_io->incomings, input_size);
34		int i;
35		memcpy(ccv_array_get(model_io->incomings, 0)((void*)(((char*)((model_io->incomings)->data)) + (size_t
)(model_io->incomings)->rsize * (size_t)(0))), inputs, sizeof(ccv_cnnp_model_io_t) * input_size);
36		for (i = 0; i < input_size; i++)
37		{
38			if (!inputs[i]->outgoings)
39				inputs[i]->outgoings = ccv_array_new(sizeof(ccv_cnnp_model_io_t), 1, 0);
40			ccv_array_push(inputs[i]->outgoings, &model_io);
41		}
42	} else {
43		model_io->incomings = 0;
44	}
45	return model_io;
46}
47 
48void ccv_cnnp_model_add_dependencies(ccv_cnnp_model_io_t model_io, const ccv_cnnp_model_io_t* const dependencies, const int dependency_size)
49{
50	assert(dependency_size > 0)((void) sizeof ((dependency_size > 0) ? 1 : 0), __extension__
 ({ if (dependency_size > 0) ; else __assert_fail ("dependency_size > 0"
, "ccv_cnnp_model.c", 50, __extension__ __PRETTY_FUNCTION__);
 }));
51	if (!model_io->dependencies)
52		model_io->dependencies = ccv_array_new(sizeof(ccv_cnnp_model_io_t), dependency_size, 0);
53	int i, j;
54	for (i = 0; i < dependency_size; i++)
55	{
56		int flag = 0;
57		// Check if it is already exist or not.
58		for (j = 0; !flag && j < model_io->dependencies->rnum; j++)
59			if (*(ccv_cnnp_model_io_t*)ccv_array_get(model_io->dependencies, j)((void*)(((char*)((model_io->dependencies)->data)) + (size_t
)(model_io->dependencies)->rsize * (size_t)(j))) == dependencies[i])
60				flag = 1;
61		if (flag)
62			continue;
63		ccv_array_push(model_io->dependencies, dependencies + i);
64		++dependencies[i]->dependents;
65	}
66}
67 
68int ccv_cnnp_model_output_size(const ccv_cnnp_model_t* const model)
69{
70	return model->output_size;
71}
72 
73int ccv_cnnp_model_is_trainable(const ccv_cnnp_model_t* const model)
74{
75	// If the model is compiled, it is default to 1 unless it is not.
76	if (model->compiled_data)
77		return model->is_trainable >= 0 ? model->is_trainable : 1;
78	return model->is_trainable;
79}
80 
81ccv_cnnp_model_io_t ccv_cnnp_model_parameters(ccv_cnnp_model_t* const model, const int selector, const int index)
82{
83	if (!model->io)
84		model->io = ccv_array_new(sizeof(ccv_cnnp_model_io_t), 1, 0);
85	ccv_cnnp_model_io_t model_io = ccmallocmalloc(sizeof(struct ccv_cnnp_model_io_s));
86	model_io->param_ref = index >= 0 ? index + 1 : ALL_PARAMETERS-1;
87	model_io->param_sel = selector >= 0 ? selector + 1 : ALL_PARAMETERS-1;
88	model_io->visit = 0;
89	model_io->model = model;
90	model_io->outputs = 0;
91	model_io->dependencies = 0;
92	model_io->dependents = 0;
93	model_io->incomings = 0;
94	model_io->outgoings = 0;
95	ccv_array_push(model->io, &model_io);
96	return model_io;
97}
98 
99void ccv_cnnp_model_notify_hook(ccv_cnnp_model_t* const model, ccv_cnnp_model_notify_f func, void* const context)
100{
101	model->notify_hook.func = func;
102	model->notify_hook.context = context;
103}
104 
105void ccv_cnnp_model_notify(const ccv_cnnp_model_t* const model, const int tag, void* const payload)
106{
107	if (model->notify_hook.func)
108		model->notify_hook.func(model, tag, payload, model->notify_hook.context);
109	if (model->isa->notify)
110		model->isa->notify(model, tag, payload);
111}
112 
113static int _ccv_nnc_array_dedup_graph_exec_symbols(ccv_nnc_graph_exec_symbol_t* const graph_exec_symbols, int graph_exec_symbol_size)
114{
115	int i, j;
116	for (i = 0; i < graph_exec_symbol_size; i++)
117	{
118		ccv_nnc_graph_exec_symbol_t* const graph_exec_symbol = graph_exec_symbols + i;
119		// Check whether this tensor symbol has any duplicate.
120		for (j = i + 1; j < graph_exec_symbol_size;)
121		{
122			ccv_nnc_graph_exec_symbol_t* const other_symbol = graph_exec_symbols + j;
123			// If there is a same tensor symbol, remove it.
124			if (other_symbol->d == graph_exec_symbol->d && other_symbol->graph == graph_exec_symbol->graph)
125			{
126				if (j + 1 < graph_exec_symbol_size)
127					*other_symbol = graph_exec_symbols[graph_exec_symbol_size - 1];
128				--graph_exec_symbol_size;
129				continue;
130			}
131			++j;
132		}
133	}
134	return graph_exec_symbol_size;
135}
136 
137void ccv_cnnp_model_add_to_array(void* const context, const ccv_nnc_tensor_symbol_t symbol, const int is_trainable)
138{
139	ccv_cnnp_model_add_to_array_context_t* const add_to_array_context = (ccv_cnnp_model_add_to_array_context_t*)context;
140	ccv_cnnp_model_t* const model = add_to_array_context->sequence->model;
141	int i;
142	if (add_to_array_context->add_parameter_indices && !model->parameter_indices)
143		model->parameter_indices = ccv_array_new(sizeof(int), 0, 0);
144	for (i = 0; i < add_to_array_context->symbols->rnum; i++)
145	{
146		const ccv_nnc_tensor_symbol_t other_symbol = *(ccv_nnc_tensor_symbol_t*)ccv_array_get(add_to_array_context->symbols, i)((void*)(((char*)((add_to_array_context->symbols)->data
)) + (size_t)(add_to_array_context->symbols)->rsize * (
size_t)(i)));
147		if (other_symbol.d == symbol.d && other_symbol.graph == symbol.graph)
148		{
149			// Only add to parameter_indices if it is trainable.
150			if (add_to_array_context->add_parameter_indices)
151				ccv_array_add_unique_int(model->parameter_indices, i);
152			// Found it, return, don't add it.
153			return;
154		}
155	}
156	// Only add to parameter_indices if it is trainable.
157	if (add_to_array_context->add_parameter_indices)
158		ccv_array_push(model->parameter_indices, &add_to_array_context->symbols->rnum);
159	// This is a new one, no need to add_unique_int, it is unique.
160	ccv_array_push(add_to_array_context->symbols, &symbol);
161	if (add_to_array_context->trainables)
162		ccv_array_push(add_to_array_context->trainables, &is_trainable);
163	char id[2048];
164	id[0] = add_to_array_context->prefix;
165	id[1] = '-';
166	int total_len = 2;
167	for (i = 0; i < add_to_array_context->sequence->sequences->rnum; i++)
168	{
169		const ccv_cnnp_model_name_t* const name = (ccv_cnnp_model_name_t*)ccv_array_get(add_to_array_context->sequence->sequences, i)((void*)(((char*)((add_to_array_context->sequence->sequences
)->data)) + (size_t)(add_to_array_context->sequence->
sequences)->rsize * (size_t)(i)));
170		int len;
171		if (name->name && name->name[0] != '\0')
172			len = snprintf(id + total_len, 2048 - total_len, "%s-%d-", name->name, name->sequence);
173		else
174			len = snprintf(id + total_len, 2048 - total_len, "%d-", name->sequence);
175		total_len += len;
176		if (total_len >= 2047)
177			break;
178	}
179	if (total_len < 2047)
180		total_len += snprintf(id + total_len, 2048 - total_len, "%d", add_to_array_context->sequence->it);
181	assert(total_len < 2048)((void) sizeof ((total_len < 2048) ? 1 : 0), __extension__
 ({ if (total_len < 2048) ; else __assert_fail ("total_len < 2048"
, "ccv_cnnp_model.c", 181, __extension__ __PRETTY_FUNCTION__)
; }));
182	char *heap_id = (char*)ccmallocmalloc(total_len + 1);
183	memcpy(heap_id, id, total_len + 1);
184	ccv_array_push(add_to_array_context->ids, &heap_id);
185	++add_to_array_context->sequence->it;
186}
187 
188static void _ccv_cnnp_compiled_data_init(ccv_cnnp_compiled_data_t* const compiled_data, const int output_size, ccv_array_t* const gradient_checkpoints)
189{
190	compiled_data->f = compiled_data->fits + output_size;
191	compiled_data->xpu_alloc.mp_hdr = -1;
192	compiled_data->xpu_alloc.freed = kh_init(dy_str)kh_init_dy_str();
193	compiled_data->xpu_alloc.allocd = kh_init(dy_alloc)kh_init_dy_alloc();
194	compiled_data->gradient_checkpoints = gradient_checkpoints;
195}
196 
197static int _ccv_cnnp_model_root_parallel_count(const ccv_cnnp_model_t* const model)
198{
199	return ccv_max(model->parallel_count, 1)({ typeof (model->parallel_count) _a = (model->parallel_count
); typeof (1) _b = (1); (_a > _b) ? _a : _b; });
200}
201 
202static int _ccv_cnnp_model_effective_parallel_count(const ccv_cnnp_model_t* const model)
203{
204	int parallel_count = _ccv_cnnp_model_root_parallel_count(model);
205	if (model->graph && model->graph->data_parallel.count > parallel_count)
206		parallel_count = model->graph->data_parallel.count;
207	return parallel_count;
208}
209 
210static int _ccv_cnnp_compiled_data_parallel_count(const ccv_cnnp_model_t* const model, const ccv_cnnp_compiled_data_t* const compiled_data)
211{
212	return compiled_data->parallel_count > 0 ? compiled_data->parallel_count : _ccv_cnnp_model_effective_parallel_count(model);
213}
214 
215ccv_nnc_tensor_symbol_t ccv_cnnp_model_get_symbol(ccv_cnnp_model_t* const self, const ccv_nnc_tensor_symbol_t symbol)
216{
217	assert(self->data)((void) sizeof ((self->data) ? 1 : 0), __extension__ ({ if
 (self->data) ; else __assert_fail ("self->data", "ccv_cnnp_model.c"
, 217, __extension__ __PRETTY_FUNCTION__); }));
218	ccv_cnnp_model_build_data_t* const build_data = (ccv_cnnp_model_build_data_t*)self->data;
219	if (build_data->parallel_count <= 1 || build_data->parallel_rank == 0)
220		return symbol;
221	const int rank = build_data->parallel_rank;
222	assert(rank > 0)((void) sizeof ((rank > 0) ? 1 : 0), __extension__ ({ if (
rank > 0) ; else __assert_fail ("rank > 0", "ccv_cnnp_model.c"
, 222, __extension__ __PRETTY_FUNCTION__); }));
223	assert(rank < build_data->parallel_count)((void) sizeof ((rank < build_data->parallel_count) ? 1
 : 0), __extension__ ({ if (rank < build_data->parallel_count
) ; else __assert_fail ("rank < build_data->parallel_count"
, "ccv_cnnp_model.c", 223, __extension__ __PRETTY_FUNCTION__)
; }));
224	ccv_nnc_symbolic_graph_t* const graph = (ccv_nnc_symbolic_graph_t*)symbol.graph;
225	ccv_nnc_tensor_symbol_t copy = ccv_nnc_tensor_symbol_copy(graph, symbol, rank);
226	if (copy.d != CCV_NNC_NO_TENSOR_SYMBOL)
227		return copy;
228	ccv_nnc_tensor_param_t params = ccv_nnc_tensor_symbol_params(graph, symbol);
229	if (CCV_TENSOR_GET_MEMORY(params.type)((params.type) & 0x3) == CCV_TENSOR_GPU_MEMORY)
230		CCV_TENSOR_SET_DEVICE_ID(params.type, rank)(params.type) = (((params.type) & ~0xfff00) | (((rank) &
 0xfff) << 8));
231	copy = ccv_nnc_tensor_symbol_new(graph, params, 0);
232	ccv_nnc_tensor_symbol_set_copy(graph, symbol, rank, copy);
233	return copy;
234}
235 
236typedef struct {
237	void* old_graph_exec_symbol_new_hook_context;
238	ccv_nnc_graph_exec_symbol_new_hook_f old_graph_exec_symbol_new_hook;
239	ccv_nnc_symbolic_graph_t* graph;
240	ccv_cnnp_model_build_data_t* build_data;
241} ccv_cnnp_model_set_exec_flags_context_t;
242 
243static void _ccv_cnnp_model_set_exec_flags(void* context, const ccv_nnc_graph_exec_symbol_t symbol, const ccv_nnc_cmd_t cmd, const ccv_nnc_tensor_symbol_t* const inputs, const int input_size, const ccv_nnc_tensor_symbol_t* const outputs, const int output_size, const char* const name)
244{
245	ccv_cnnp_model_set_exec_flags_context_t* flags_context = (ccv_cnnp_model_set_exec_flags_context_t*)context;
246	if (flags_context->build_data->exec_flags)
247		ccv_nnc_graph_exec_symbol_set_flags(flags_context->graph, symbol, flags_context->build_data->exec_flags);
248	if (flags_context->old_graph_exec_symbol_new_hook)
249		flags_context->old_graph_exec_symbol_new_hook(flags_context->old_graph_exec_symbol_new_hook_context, symbol, cmd, inputs, input_size, outputs, output_size, name);
250}
251 
252static void _ccv_cnnp_model_compile(ccv_cnnp_model_t* const model, const ccv_nnc_tensor_param_t* const inputs, const int input_size, const ccv_nnc_cmd_t loss)
253{
254	assert(model->graph)((void) sizeof ((model->graph) ? 1 : 0), __extension__ ({ if
 (model->graph) ; else __assert_fail ("model->graph", "ccv_cnnp_model.c"
, 254, __extension__ __PRETTY_FUNCTION__); }));
255	model->inputs = ccmallocmalloc(sizeof(ccv_nnc_tensor_symbol_t) * input_size);
256	int i;
257	for (i = 0; i < input_size; i++)
258		model->inputs[i] = ccv_nnc_tensor_symbol_new(model->graph, inputs[i], 0);
259	ccv_array_t* const parameters = ccv_array_new(sizeof(ccv_nnc_tensor_symbol_t), 0, 0);
260	ccv_array_t* const parameter_ids = ccv_array_new(sizeof(char*), 0, 0);
261	ccv_array_t* const parameter_trainables = ccv_array_new(sizeof(int), 0, 0);
262	ccv_cnnp_model_sequence_t model_sequence = {
263		.bank = kh_init(ccv_cnnp_model_name_bank)kh_init_ccv_cnnp_model_name_bank()
264	};
265	ccv_cnnp_model_add_to_array_context_t add_to_parameter_context = {
266		.add_parameter_indices = 1,
267		.prefix = 't',
268		.sequence = &model_sequence,
269		.symbols = parameters,
270		.ids = parameter_ids,
271		.trainables = parameter_trainables,
272	};
273	ccv_array_t* const internals = ccv_array_new(sizeof(ccv_nnc_tensor_symbol_t), 0, 0);
274	ccv_array_t* const internal_ids = ccv_array_new(sizeof(char*), 0, 0);
275	ccv_cnnp_model_add_to_array_context_t add_to_output_context = {
276		.add_parameter_indices = 0,
277		.prefix = 'r',
278		.sequence = &model_sequence,
279		.symbols = internals,
280		.ids = internal_ids,
281		.trainables = 0,
282	};
283	ccv_cnnp_model_build_data_t build_data = {
284		.exec_flags = 0,
285		.is_trainable = model->is_trainable >= 0 ? model->is_trainable : 1,
286		.parallel_count = 1,
287		.parallel_rank = 0,
288		.model_sequence = &model_sequence,
289		.add_to_array = ccv_cnnp_model_add_to_array,
290		.parameters = parameters,
291		.context = {
292			.add_to_parameter = &add_to_parameter_context,
293			.add_to_output = &add_to_output_context,
294		},
295		.gradient_checkpoints = 0,
296	};
297	model->data = &build_data;
298	ccv_cnnp_model_set_exec_flags_context_t flags_context = {
299		.graph = model->graph,
300		.build_data = &build_data,
301		.old_graph_exec_symbol_new_hook = 0,
302		.old_graph_exec_symbol_new_hook_context = 0
303	};
304	flags_context.old_graph_exec_symbol_new_hook_context = ccv_nnc_graph_exec_symbol_new_hook(model->graph, _ccv_cnnp_model_set_exec_flags, &flags_context, &flags_context.old_graph_exec_symbol_new_hook);
305	ccv_cnnp_model_build(model, model->graph, model->inputs, input_size, 0, 0);
306	// Reset back to previous hook.
307	ccv_nnc_graph_exec_symbol_new_hook(model->graph, flags_context.old_graph_exec_symbol_new_hook, flags_context.old_graph_exec_symbol_new_hook_context, 0);
308	for (i = 0; i < model->output_size; i++)
309	{
310		const ccv_nnc_tensor_symbol_t output = model->outputs[i];
311		const ccv_nnc_tensor_symbol_t alias_to = ccv_nnc_tensor_symbol_alias_to(model->graph, output);
312		if (alias_to.d == CCV_NNC_NO_TENSOR_SYMBOL)
313			continue;
314		// If output is an alias, insert data transform regardless for result correctness (we cannot bind an alias). You can check ccv_nnc_tensor_bind_symbol method
315		// to see that we can correctly bind a tensor which from it, has aliases, but we cannot bind an alias tensor correctly (this is expected, sort of, to be
316		// honest, because we cannot handle cases of alias is part of the original tensor but bind differently).
317		const ccv_nnc_tensor_param_t output_params = ccv_nnc_tensor_symbol_params(model->graph, output);
318		model->outputs[i] = ccv_nnc_tensor_symbol_new(model->graph, output_params, 0);
319		ccv_nnc_graph_exec_symbol_t make_contiguous = ccv_nnc_graph_exec_symbol_new(model->graph, CMD_FORMAT_TRANSFORM_FORWARD()ccv_nnc_cmd(CCV_NNC_FORMAT_TRANSFORM_FORWARD, 0, ccv_nnc_cmd_auto
, 0), &output, 1, model->outputs + i, 1, "contiguous");
320		ccv_nnc_graph_exec_symbol_set_flags(model->graph, make_contiguous, CCV_NNC_GRAPH_EXEC_DISABLE_OPT);
321	}
322	model->data = 0;
323	kh_destroy(ccv_cnnp_model_name_bank, model_sequence.bank)kh_destroy_ccv_cnnp_model_name_bank(model_sequence.bank);
324	if (model_sequence.sequences)
325		ccv_array_free(model_sequence.sequences);
326	// Check if there are parameters that are not trainables. If there are, we will allocate uint64 bitmap to record that.
327	int not_trainables = 0;
328	// Assert no parameter is alias.
329	for (i = 0; i < parameters->rnum; i++)
330	{
331		const ccv_nnc_tensor_symbol_t parameter = *(ccv_nnc_tensor_symbol_t*)ccv_array_get(parameters, i)((void*)(((char*)((parameters)->data)) + (size_t)(parameters
)->rsize * (size_t)(i)));
332		const ccv_nnc_tensor_symbol_t alias_to = ccv_nnc_tensor_symbol_alias_to(parameter.graph, parameter);
333		assert(alias_to.graph == 0)((void) sizeof ((alias_to.graph == 0) ? 1 : 0), __extension__
 ({ if (alias_to.graph == 0) ; else __assert_fail ("alias_to.graph == 0"
, "ccv_cnnp_model.c", 333, __extension__ __PRETTY_FUNCTION__)
; })); // Cannot find the one alias to.
334		if (*(int*)ccv_array_get(parameter_trainables, i)((void*)(((char*)((parameter_trainables)->data)) + (size_t
)(parameter_trainables)->rsize * (size_t)(i))) == 0)
335			not_trainables = 1;
336	}
337	assert(parameters->rnum == parameter_trainables->rnum)((void) sizeof ((parameters->rnum == parameter_trainables->
rnum) ? 1 : 0), __extension__ ({ if (parameters->rnum == parameter_trainables
->rnum) ; else __assert_fail ("parameters->rnum == parameter_trainables->rnum"
, "ccv_cnnp_model.c", 337, __extension__ __PRETTY_FUNCTION__)
; }));
338	uint64_t* parameter_flags = 0;
339	if (not_trainables)
340	{
341		parameter_flags = (uint64_t*)cccalloccalloc(((parameters->rnum + 63) >> 6), sizeof(uint64_t));
342		for (i = 0; i < parameter_trainables->rnum; i++)
343			if (*(int*)ccv_array_get(parameter_trainables, i)((void*)(((char*)((parameter_trainables)->data)) + (size_t
)(parameter_trainables)->rsize * (size_t)(i))))
344				parameter_flags[i >> 6] |= ((uint64_t)1 << (i & 63));
345	}
346	ccv_array_free(parameter_trainables);
347	// Assert no internal is alias.
348	for (i = 0; i < internals->rnum; i++)
349	{
350		const ccv_nnc_tensor_symbol_t internal = *(ccv_nnc_tensor_symbol_t*)ccv_array_get(internals, i)((void*)(((char*)((internals)->data)) + (size_t)(internals
)->rsize * (size_t)(i)));
351		const ccv_nnc_tensor_symbol_t alias_to = ccv_nnc_tensor_symbol_alias_to(internal.graph, internal);
352		assert(alias_to.graph == 0)((void) sizeof ((alias_to.graph == 0) ? 1 : 0), __extension__
 ({ if (alias_to.graph == 0) ; else __assert_fail ("alias_to.graph == 0"
, "ccv_cnnp_model.c", 352, __extension__ __PRETTY_FUNCTION__)
; })); // Cannot find the one alias to.
353	}
354	const int output_size = model->output_size;
355	ccv_nnc_graph_exec_symbol_autogen(model->graph, 0, 0, CCV_NNC_AUTOGEN_ALL_EXECS | CCV_NNC_AUTOGEN_SOURCES_AND_DESTINATIONS);
356	const int parameters_rnum = parameters->rnum;
357	if (input_size > 0)
358	{
359		ccv_array_resize(parameters, parameters_rnum + input_size);
360		memcpy(ccv_array_get(parameters, parameters_rnum)((void*)(((char*)((parameters)->data)) + (size_t)(parameters
)->rsize * (size_t)(parameters_rnum))), model->inputs, input_size * sizeof(ccv_nnc_tensor_symbol_t));
361	}
362	ccv_nnc_symbolic_graph_simplify(model->graph,
363		SYMBOLIC_GRAPH_PASSES(CCV_NNC_SIMPLIFY_COMMON_SUBEXPRESSION_ELIMINATION,(const int []){CCV_NNC_SIMPLIFY_COMMON_SUBEXPRESSION_ELIMINATION
, CCV_NNC_SIMPLIFY_DATA_TRANSFER_OPT, CCV_NNC_SIMPLIFY_OPS_FUSION
, CCV_NNC_SIMPLIFY_GRAPH_PRUNING}, (1 +1 +1 +1 +1 +0 +0 +0 +0
 +0 +0 +0 +0 +0 +0 +0 +0 +0 +0 +0 +0 +0 +0 +0 +0 +0 +0 +0 +0 +
0 +0 +0 +0 +0 +0 +0 +0 +0 +0 +0 +0 +0 +0 +0 +0 +0 +0 +0 +0 +0
 +0 +0 +0 +0 +0 +0 +0 +0 +0 +0 +0 +0 +0 +0 -1)
364			CCV_NNC_SIMPLIFY_DATA_TRANSFER_OPT,(const int []){CCV_NNC_SIMPLIFY_COMMON_SUBEXPRESSION_ELIMINATION
, CCV_NNC_SIMPLIFY_DATA_TRANSFER_OPT, CCV_NNC_SIMPLIFY_OPS_FUSION
, CCV_NNC_SIMPLIFY_GRAPH_PRUNING}, (1 +1 +1 +1 +1 +0 +0 +0 +0
 +0 +0 +0 +0 +0 +0 +0 +0 +0 +0 +0 +0 +0 +0 +0 +0 +0 +0 +0 +0 +
0 +0 +0 +0 +0 +0 +0 +0 +0 +0 +0 +0 +0 +0 +0 +0 +0 +0 +0 +0 +0
 +0 +0 +0 +0 +0 +0 +0 +0 +0 +0 +0 +0 +0 +0 -1)
365			CCV_NNC_SIMPLIFY_OPS_FUSION,(const int []){CCV_NNC_SIMPLIFY_COMMON_SUBEXPRESSION_ELIMINATION
, CCV_NNC_SIMPLIFY_DATA_TRANSFER_OPT, CCV_NNC_SIMPLIFY_OPS_FUSION
, CCV_NNC_SIMPLIFY_GRAPH_PRUNING}, (1 +1 +1 +1 +1 +0 +0 +0 +0
 +0 +0 +0 +0 +0 +0 +0 +0 +0 +0 +0 +0 +0 +0 +0 +0 +0 +0 +0 +0 +
0 +0 +0 +0 +0 +0 +0 +0 +0 +0 +0 +0 +0 +0 +0 +0 +0 +0 +0 +0 +0
 +0 +0 +0 +0 +0 +0 +0 +0 +0 +0 +0 +0 +0 +0 -1)
366			CCV_NNC_SIMPLIFY_GRAPH_PRUNING)(const int []){CCV_NNC_SIMPLIFY_COMMON_SUBEXPRESSION_ELIMINATION
, CCV_NNC_SIMPLIFY_DATA_TRANSFER_OPT, CCV_NNC_SIMPLIFY_OPS_FUSION
, CCV_NNC_SIMPLIFY_GRAPH_PRUNING}, (1 +1 +1 +1 +1 +0 +0 +0 +0
 +0 +0 +0 +0 +0 +0 +0 +0 +0 +0 +0 +0 +0 +0 +0 +0 +0 +0 +0 +0 +
0 +0 +0 +0 +0 +0 +0 +0 +0 +0 +0 +0 +0 +0 +0 +0 +0 +0 +0 +0 +0
 +0 +0 +0 +0 +0 +0 +0 +0 +0 +0 +0 +0 +0 +0 -1),
367		ccv_array_get(parameters, 0)((void*)(((char*)((parameters)->data)) + (size_t)(parameters
)->rsize * (size_t)(0))), parameters_rnum + input_size,
368		model->outputs, output_size,
369		SYMBOLIC_GRAPH_SOURCES(model->graph)ccv_nnc_symbolic_graph_sources(model->graph), ccv_nnc_symbolic_graph_source_size
(model->graph), SYMBOLIC_GRAPH_DESTINATIONS(model->graph)ccv_nnc_symbolic_graph_destinations(model->graph), ccv_nnc_symbolic_graph_destination_size
(model->graph));
370	ccv_nnc_graph_exec_symbol_autogen(model->graph, 0, 0, CCV_NNC_AUTOGEN_SOURCES_AND_DESTINATIONS);
371	// Size it down.
372	parameters->rnum = parameters_rnum;
373	ccv_cnnp_compiled_data_t* compiled_data = model->compiled_data = cccalloccalloc(1, sizeof(ccv_cnnp_compiled_data_t) + sizeof(ccv_nnc_tensor_symbol_t) * (output_size * 2 - 1));
374	_ccv_cnnp_compiled_data_init(compiled_data, output_size, build_data.gradient_checkpoints);
375	const int evaluate_to_size = compiled_data->evaluate.to_size = ccv_nnc_symbolic_graph_destination_size(model->graph);
376	assert(evaluate_to_size > 0)((void) sizeof ((evaluate_to_size > 0) ? 1 : 0), __extension__
 ({ if (evaluate_to_size > 0) ; else __assert_fail ("evaluate_to_size > 0"
, "ccv_cnnp_model.c", 376, __extension__ __PRETTY_FUNCTION__)
; }));
377	compiled_data->evaluate.tos = ccmallocmalloc(sizeof(ccv_nnc_graph_exec_symbol_t) * evaluate_to_size);
378	memcpy(compiled_data->evaluate.tos, ccv_nnc_symbolic_graph_destinations(model->graph), sizeof(ccv_nnc_graph_exec_symbol_t) * evaluate_to_size);
379	compiled_data->loss = loss;
380	if (loss.cmd == CCV_NNC_NOOP)
381	{
382		// If no loss function provided, there is no fits.
383		for (i = 0; i < output_size; i++)
384		{
385			compiled_data->fits[i] = NO_TENSOR_SYMBOL(const ccv_nnc_tensor_symbol_t){.d = CCV_NNC_NO_TENSOR_SYMBOL
};
386			const ccv_nnc_tensor_symbol_t alias_to = ccv_nnc_tensor_symbol_alias_to(model->graph, model->outputs[i]);
387			if (alias_to.d < 0)
388				compiled_data->f[i] = model->outputs[i];
389			else { // We cannot differentiate against an alias, therefore, we have to verify this output is full, and we can diff against the original.
390				int ofs[CCV_NNC_MAX_DIM_ALLOC(12)];
391				int inc[CCV_NNC_MAX_DIM_ALLOC(12)];
392				ccv_nnc_tensor_symbol_alias_params(model->graph, model->outputs[i], ofs, inc);
393				int j;
394				for (j = 0; j < CCV_NNC_MAX_DIM_ALLOC(12); j++)
395					{ assert(ofs[j] == 0)((void) sizeof ((ofs[j] == 0) ? 1 : 0), __extension__ ({ if (
ofs[j] == 0) ; else __assert_fail ("ofs[j] == 0", "ccv_cnnp_model.c"
, 395, __extension__ __PRETTY_FUNCTION__); })); } // There is no ofs.
396				compiled_data->f[i] = alias_to; // Unfortunately, I cannot assert the size yet.
397			}
398		}
399	} else {
400		for (i = 0; i < output_size; i++)
401		{
402			const ccv_nnc_tensor_param_t info = ccv_nnc_tensor_symbol_params(model->graph, model->outputs[i]);
403			const ccv_nnc_tensor_symbol_t fit = compiled_data->fits[i] = ccv_nnc_tensor_symbol_new(model->graph, info, 0);
404			compiled_data->f[i] = ccv_nnc_tensor_symbol_new(model->graph, ccv_nnc_tensor_auto, 0);
405			ccv_nnc_graph_exec_symbol_new(model->graph, loss, TENSOR_SYMBOL_LIST(model->outputs[i], fit)(const ccv_nnc_tensor_symbol_t []){model->outputs[i], fit}
, (1 +1 +1 +0 +0 +0 +0 +0 +0 +0 +0 +0 +0 +0 +0 +0 +0 +0 +0 +0
 +0 +0 +0 +0 +0 +0 +0 +0 +0 +0 +0 +0 +0 +0 +0 +0 +0 +0 +0 +0 +
0 +0 +0 +0 +0 +0 +0 +0 +0 +0 +0 +0 +0 +0 +0 +0 +0 +0 +0 +0 +0
 +0 +0 +0 -1), TENSOR_SYMBOL_LIST(compiled_data->f[i])(const ccv_nnc_tensor_symbol_t []){compiled_data->f[i]}, (
1 +1 +0 +0 +0 +0 +0 +0 +0 +0 +0 +0 +0 +0 +0 +0 +0 +0 +0 +0 +0
 +0 +0 +0 +0 +0 +0 +0 +0 +0 +0 +0 +0 +0 +0 +0 +0 +0 +0 +0 +0 +
0 +0 +0 +0 +0 +0 +0 +0 +0 +0 +0 +0 +0 +0 +0 +0 +0 +0 +0 +0 +0
 +0 +0 -1), 0);
406		}
407	}
408	if (loss.cmd != CCV_NNC_NOOP)
409	{
410		ccv_nnc_graph_exec_symbol_autogen(model->graph, 0, 0, CCV_NNC_AUTOGEN_ALL_EXECS | CCV_NNC_AUTOGEN_SOURCES_AND_DESTINATIONS);
411		ccv_nnc_symbolic_graph_simplify(model->graph,
412			SYMBOLIC_GRAPH_PASSES(CCV_NNC_SIMPLIFY_OPS_FUSION)(const int []){CCV_NNC_SIMPLIFY_OPS_FUSION}, (1 +1 +0 +0 +0 +
0 +0 +0 +0 +0 +0 +0 +0 +0 +0 +0 +0 +0 +0 +0 +0 +0 +0 +0 +0 +0
 +0 +0 +0 +0 +0 +0 +0 +0 +0 +0 +0 +0 +0 +0 +0 +0 +0 +0 +0 +0 +
0 +0 +0 +0 +0 +0 +0 +0 +0 +0 +0 +0 +0 +0 +0 +0 +0 +0 -1), // Only do Ops fusion, in this way, we can fuse the loss function.
413			0, 0, // No need to provide binds at this point.
414			compiled_data->f, model->output_size,
415			SYMBOLIC_GRAPH_SOURCES(model->graph)ccv_nnc_symbolic_graph_sources(model->graph), ccv_nnc_symbolic_graph_source_size
(model->graph), SYMBOLIC_GRAPH_DESTINATIONS(model->graph)ccv_nnc_symbolic_graph_destinations(model->graph), ccv_nnc_symbolic_graph_destination_size
(model->graph));
416		ccv_nnc_graph_exec_symbol_autogen(model->graph, 0, 0, CCV_NNC_AUTOGEN_SOURCES_AND_DESTINATIONS);
417	}
418	// If inputs are from GPU, stream type is GPU.
419	compiled_data->parameters = parameters;
420	compiled_data->parameter_flags = parameter_flags;
421	compiled_data->internals = internals;
422	compiled_data->ids.parameters = parameter_ids;
423	compiled_data->ids.internals = internal_ids;
424	ccv_cnnp_model_gradient_checkpoints_cleanup_after_build(compiled_data, model->graph);
425}
426 
427static void _ccv_cnnp_graph_push_graph_exec_symbol(void* context, const ccv_nnc_graph_exec_symbol_t symbol, const ccv_nnc_cmd_t cmd, const ccv_nnc_tensor_symbol_t* const inputs, const int input_size, const ccv_nnc_tensor_symbol_t* const outputs, const int output_size, const char* const name)
428{
429	ccv_array_t* const stack = (ccv_array_t*)context;
430	ccv_array_push(stack, &symbol.d);
431}
432 
433static void _ccv_nnc_tensor_symbol_reinit(const ccv_nnc_symbolic_graph_t* const src_graph, ccv_nnc_symbolic_graph_t* const dest_graph, const int src_index, const int dest_index)
434{
435	int src_alias_ref = src_index;
436	const ccv_nnc_tensor_symbol_info_t* src_info = (ccv_nnc_tensor_symbol_info_t*)ccv_array_get(src_graph->tensor_symbol_info, src_alias_ref)((void*)(((char*)((src_graph->tensor_symbol_info)->data
)) + (size_t)(src_graph->tensor_symbol_info)->rsize * (
size_t)(src_alias_ref)));
437	while (src_info->alias_ref)
438	{
439		src_alias_ref = src_info->alias_ref - 1;
440		src_info = (ccv_nnc_tensor_symbol_info_t*)ccv_array_get(src_graph->tensor_symbol_info, src_alias_ref)((void*)(((char*)((src_graph->tensor_symbol_info)->data
)) + (size_t)(src_graph->tensor_symbol_info)->rsize * (
size_t)(src_alias_ref)));
441	}
442	int dest_alias_ref = dest_index;
443	const ccv_nnc_tensor_symbol_info_t* dest_info = (ccv_nnc_tensor_symbol_info_t*)ccv_array_get(dest_graph->tensor_symbol_info, dest_alias_ref)((void*)(((char*)((dest_graph->tensor_symbol_info)->data
)) + (size_t)(dest_graph->tensor_symbol_info)->rsize * (
size_t)(dest_alias_ref)));
444	while (dest_info->alias_ref)
445	{
446		dest_alias_ref = dest_info->alias_ref - 1;
447		dest_info = (ccv_nnc_tensor_symbol_info_t*)ccv_array_get(dest_graph->tensor_symbol_info, dest_alias_ref)((void*)(((char*)((dest_graph->tensor_symbol_info)->data
)) + (size_t)(dest_graph->tensor_symbol_info)->rsize * (
size_t)(dest_alias_ref)));
448	}
449	assert((src_alias_ref != src_index) == (dest_alias_ref != dest_index))((void) sizeof (((src_alias_ref != src_index) == (dest_alias_ref
 != dest_index)) ? 1 : 0), __extension__ ({ if ((src_alias_ref
 != src_index) == (dest_alias_ref != dest_index)) ; else __assert_fail
 ("(src_alias_ref != src_index) == (dest_alias_ref != dest_index)"
, "ccv_cnnp_model.c", 449, __extension__ __PRETTY_FUNCTION__)
; }));
450	if (src_alias_ref != src_index)
451	{
452		const ccv_nnc_tensor_symbol_t src_alias_symbol = {
453			.d = src_alias_ref,
454			.graph = src_graph
455		};
456		const ccv_nnc_tensor_symbol_t dest_alias_symbol = {
457			.d = dest_alias_ref,
458			.graph = dest_graph
459		};
460		const ccv_nnc_tensor_param_t params = ccv_nnc_tensor_symbol_params(src_graph, src_alias_symbol);
461		ccv_nnc_tensor_symbol_set(dest_graph, dest_alias_symbol, params);
462	}
463	const ccv_nnc_tensor_symbol_t src_symbol = {
464		.d = src_index,
465		.graph = src_graph
466	};
467	const ccv_nnc_tensor_symbol_t dest_symbol = {
468		.d = dest_index,
469		.graph = dest_graph
470	};
471	const ccv_nnc_tensor_param_t params = ccv_nnc_tensor_symbol_params(src_graph, src_symbol);
472	ccv_nnc_tensor_symbol_set(dest_graph, dest_symbol, params);
473	int ofs[CCV_NNC_MAX_DIM_ALLOC(12)];
474	int inc[CCV_NNC_MAX_DIM_ALLOC(12)];
475	if (0 == ccv_nnc_tensor_symbol_alias_params(src_graph, src_symbol, ofs, inc))
476		ccv_nnc_tensor_symbol_alias_set(dest_graph, dest_symbol, ofs, inc);
477}
478 
479static int _ccv_nnc_tensor_symbol_check_dim(const ccv_nnc_symbolic_graph_t* const src_graph, ccv_nnc_symbolic_graph_t* const dest_graph, const int src_index, const int dest_index)
480{
481	const ccv_nnc_tensor_symbol_t src_symbol = {
482		.d = src_index,
483		.graph = src_graph
484	};
485	const ccv_nnc_tensor_param_t src_params = ccv_nnc_tensor_symbol_params(src_graph, src_symbol);
486	const ccv_nnc_tensor_symbol_t dest_symbol = {
487		.d = dest_index,
488		.graph = dest_graph
489	};
490	const ccv_nnc_tensor_param_t dest_params = ccv_nnc_tensor_symbol_params(dest_graph, dest_symbol);
491	if (src_params.dim[0] == 0 || dest_params.dim[0] == 0)
492		return 1;
493	return memcmp(src_params.dim, dest_params.dim, sizeof(src_params.dim)) == 0;
494}
495 
496static void _ccv_cnnp_model_gradient_init(ccv_cnnp_model_t* const model, const int gradient_mode, const uint64_t disable_outgrad, ccv_nnc_tensor_t* const* const fits, const int fit_size);
497static void _ccv_cnnp_compiled_data_graph_free(ccv_cnnp_compiled_data_t* const compiled_data);
498 
499typedef struct {
500	int parallel_count;
501	ccv_nnc_symbolic_graph_t* graph;
502	ccv_nnc_graph_exec_arena_t* graph_exec_arena;
503} ccv_nnc_graph_exec_update_t;
504 
505static void _ccv_cnnp_cmd_update_for_execs(void* const context, const ccv_nnc_graph_exec_symbol_t symbol, const ccv_nnc_cmd_t cmd, const ccv_nnc_hint_t hint)
506{
507	ccv_nnc_graph_exec_update_t* const graph_exec_update = (ccv_nnc_graph_exec_update_t*)context;
508	ccv_nnc_graph_exec_arena_t* const graph_exec_arena = graph_exec_update->graph_exec_arena;
509	ccv_nnc_graph_exec_t graph_exec = ccv_nnc_graph_exec_from_symbol(graph_exec_arena, symbol);
510	ccv_nnc_graph_exec_set(graph_exec.graph, graph_exec, cmd);
511	ccv_nnc_graph_exec_set_hint(graph_exec.graph, graph_exec, hint);
512	const ccv_nnc_symbolic_graph_t* const graph = graph_exec_update->graph;
513	const int parallel_count = graph_exec_update->parallel_count;
514	int i;
515	for (i = 1; i < parallel_count; i++)
516	{
517		const ccv_nnc_graph_exec_t copy = ccv_nnc_graph_exec_from_symbol(graph_exec_arena, ccv_nnc_graph_exec_symbol_copy(graph, symbol, i));
518		if (!CCV_NO_GRAPH_EXEC(copy)((copy).graph == 0))
519		{
520			ccv_nnc_graph_exec_set(copy.graph, copy, cmd);
521			ccv_nnc_graph_exec_set_hint(copy.graph, copy, hint);
522		}
523	}
524}
525 
526void ccv_cnnp_model_absorb(ccv_cnnp_model_t* const model, ccv_cnnp_model_t* const init, const ccv_nnc_tensor_param_t* const inputs, const int input_size)
527{
528	assert(model->graph)((void) sizeof ((model->graph) ? 1 : 0), __extension__ ({ if
 (model->graph) ; else __assert_fail ("model->graph", "ccv_cnnp_model.c"
, 528, __extension__ __PRETTY_FUNCTION__); }));
529	assert(model->compiled_data)((void) sizeof ((model->compiled_data) ? 1 : 0), __extension__
 ({ if (model->compiled_data) ; else __assert_fail ("model->compiled_data"
, "ccv_cnnp_model.c", 529, __extension__ __PRETTY_FUNCTION__)
; }));
530	assert(!init->graph)((void) sizeof ((!init->graph) ? 1 : 0), __extension__ ({ if
 (!init->graph) ; else __assert_fail ("!init->graph", "ccv_cnnp_model.c"
, 530, __extension__ __PRETTY_FUNCTION__); }));
531	ccv_cnnp_compiled_data_t* const compiled_data = model->compiled_data;
532	init->graph = ccv_nnc_symbolic_graph_new();
533	ccv_array_t* const stack = ccv_array_new(sizeof(int), 0, 0);
534	ccv_nnc_graph_exec_symbol_new_hook(init->graph, _ccv_cnnp_graph_push_graph_exec_symbol, stack, 0);
535	_ccv_cnnp_model_compile(init, inputs, input_size, compiled_data->loss);
536	init->parallel_count = model->parallel_count;
537	init->memory_compression = model->memory_compression;
538	init->memory_reduction = model->memory_reduction;
539	init->gradient_checkpointing = model->gradient_checkpointing;
540	init->compiled_data->stream_type = model->compiled_data->stream_type;
541	init->compiled_data->minimize.minimizer = model->compiled_data->minimize.minimizer;
542	init->compiled_data->minimize.max_saved_aux_size = model->compiled_data->minimize.max_saved_aux_size;
543	if (model->compiled_data->gradient_mode != CCV_CNNP_COMPILED_DATA_GRADIENT_NONE)
544		_ccv_cnnp_model_gradient_init(init, model->compiled_data->gradient_mode, model->compiled_data->disable_outgrad, 0, 0);
545	ccv_nnc_graph_exec_symbol_new_hook(init->graph, 0, 0, 0);
546	ccv_nnc_symbolic_graph_tensor_auto(init->graph, TRAVERSE_FULL0,0,0,0);
547	int i, j;
548	// Verify parameters, internals and saved_aux in both graph has the same dimensionality.
549	for (i = 0; i < compiled_data->parameters->rnum; i++)
550	{
551		const int d = ((ccv_nnc_tensor_symbol_t*)ccv_array_get(compiled_data->parameters, i)((void*)(((char*)((compiled_data->parameters)->data)) +
 (size_t)(compiled_data->parameters)->rsize * (size_t)(
i))))->d;
552		assert(_ccv_nnc_tensor_symbol_check_dim(model->graph, init->graph, d, d))((void) sizeof ((_ccv_nnc_tensor_symbol_check_dim(model->graph
, init->graph, d, d)) ? 1 : 0), __extension__ ({ if (_ccv_nnc_tensor_symbol_check_dim
(model->graph, init->graph, d, d)) ; else __assert_fail
 ("_ccv_nnc_tensor_symbol_check_dim(model->graph, init->graph, d, d)"
, "ccv_cnnp_model.c", 552, __extension__ __PRETTY_FUNCTION__)
; }));
553	}
554	for (i = 0; i < compiled_data->internals->rnum; i++)
555	{
556		const int d = ((ccv_nnc_tensor_symbol_t*)ccv_array_get(compiled_data->internals, i)((void*)(((char*)((compiled_data->internals)->data)) + (
size_t)(compiled_data->internals)->rsize * (size_t)(i))
))->d;
557		assert(_ccv_nnc_tensor_symbol_check_dim(model->graph, init->graph, d, d))((void) sizeof ((_ccv_nnc_tensor_symbol_check_dim(model->graph
, init->graph, d, d)) ? 1 : 0), __extension__ ({ if (_ccv_nnc_tensor_symbol_check_dim
(model->graph, init->graph, d, d)) ; else __assert_fail
 ("_ccv_nnc_tensor_symbol_check_dim(model->graph, init->graph, d, d)"
, "ccv_cnnp_model.c", 557, __extension__ __PRETTY_FUNCTION__)
; }));
558	}
559	// Update inputs.
560	assert(model->input_size == init->input_size)((void) sizeof ((model->input_size == init->input_size)
 ? 1 : 0), __extension__ ({ if (model->input_size == init->
input_size) ; else __assert_fail ("model->input_size == init->input_size"
, "ccv_cnnp_model.c", 560, __extension__ __PRETTY_FUNCTION__)
; }));
561	for (i = 0; i < model->input_size; i++)
562		if (model->inputs[i].d >= 0)
563		{
564			assert(init->inputs[i].d >= 0)((void) sizeof ((init->inputs[i].d >= 0) ? 1 : 0), __extension__
 ({ if (init->inputs[i].d >= 0) ; else __assert_fail ("init->inputs[i].d >= 0"
, "ccv_cnnp_model.c", 564, __extension__ __PRETTY_FUNCTION__)
; }));
565			_ccv_nnc_tensor_symbol_reinit(init->graph, model->graph, init->inputs[i].d, model->inputs[i].d);
566		}
567	// Update outputs.
568	assert(model->output_size == init->output_size)((void) sizeof ((model->output_size == init->output_size
) ? 1 : 0), __extension__ ({ if (model->output_size == init
->output_size) ; else __assert_fail ("model->output_size == init->output_size"
, "ccv_cnnp_model.c", 568, __extension__ __PRETTY_FUNCTION__)
; }));
569	for (i = 0; i < model->output_size; i++)
570	{
571		if (model->outputs[i].d >= 0)
572		{
573			assert(init->outputs[i].d >= 0)((void) sizeof ((init->outputs[i].d >= 0) ? 1 : 0), __extension__
 ({ if (init->outputs[i].d >= 0) ; else __assert_fail (
"init->outputs[i].d >= 0", "ccv_cnnp_model.c", 573, __extension__
 __PRETTY_FUNCTION__); }));
574			_ccv_nnc_tensor_symbol_reinit(init->graph, model->graph, init->outputs[i].d, model->outputs[i].d);
575		}
576		if (model->outputs[i].d != model->compiled_data->f[i].d)
577		{
578			assert(init->outputs[i].d != init->compiled_data->f[i].d)((void) sizeof ((init->outputs[i].d != init->compiled_data
->f[i].d) ? 1 : 0), __extension__ ({ if (init->outputs[
i].d != init->compiled_data->f[i].d) ; else __assert_fail
 ("init->outputs[i].d != init->compiled_data->f[i].d"
, "ccv_cnnp_model.c", 578, __extension__ __PRETTY_FUNCTION__)
; }));
579			if (model->compiled_data->f[i].d >= 0)
580			{
581				assert(init->compiled_data->f[i].d >= 0)((void) sizeof ((init->compiled_data->f[i].d >= 0) ?
 1 : 0), __extension__ ({ if (init->compiled_data->f[i]
.d >= 0) ; else __assert_fail ("init->compiled_data->f[i].d >= 0"
, "ccv_cnnp_model.c", 581, __extension__ __PRETTY_FUNCTION__)
; }));
582				_ccv_nnc_tensor_symbol_reinit(init->graph, model->graph, init->compiled_data->f[i].d, model->compiled_data->f[i].d);
583			}
584		}
585	}
586	// Go through the graph to set tensor on matching symbols
587	for (i = 0; i < stack->rnum; i++)
588	{
589		const int d = *(int*)ccv_array_get(stack, i)((void*)(((char*)((stack)->data)) + (size_t)(stack)->rsize
 * (size_t)(i)));
590		// If exceed range, skip.
591		if (d >= ccv_nnc_graph_exec_symbol_count(init->graph) ||
592			d >= ccv_nnc_graph_exec_symbol_count(model->graph))
593			continue;
594		const ccv_nnc_graph_exec_symbol_t src_symbol = {
595			.d = d,
596			.graph = init->graph
597		};
598		const ccv_nnc_graph_exec_symbol_t dest_symbol = {
599			.d = d,
600			.graph = model->graph
601		};
602		const ccv_nnc_cmd_t src_cmd = ccv_nnc_graph_exec_symbol_cmd(init->graph, src_symbol);
603		const ccv_nnc_cmd_t dest_cmd = ccv_nnc_graph_exec_symbol_cmd(model->graph, dest_symbol);
604		// If the name doesn't match, skip.
605		if (dest_cmd.cmd != src_cmd.cmd && src_cmd.cmd != CCV_NNC_NOOP)
606			continue;
607		// Now get all the inputs and outputs, if matches, set them.
608		const int* src_inputs;
609		int src_input_size;
610		const int* src_outputs;
611		int src_output_size;
612		ccv_nnc_graph_exec_symbol_io(init->graph, src_symbol, &src_inputs, &src_input_size, &src_outputs, &src_output_size);
613		const int* dest_inputs;
614		int dest_input_size;
615		const int* dest_outputs;
616		int dest_output_size;
617		ccv_nnc_graph_exec_symbol_io(model->graph, dest_symbol, &dest_inputs, &dest_input_size, &dest_outputs, &dest_output_size);
618		// We may have unmatched input / output size because this is the minimizer and it has
619		// different saved_aux (for example, when we shrunk with CMD_NOOP).
620		if (src_input_size != dest_input_size)
621			continue;
622		if (src_output_size != dest_output_size)
623			continue;
624		ccv_nnc_graph_exec_symbol_set(model->graph, dest_symbol, src_cmd);
625		// There may be mismatches of the source tensor symbols and destination tensor symbols. The reason is because
626		// we may later passed-in the minimizer, therefore, we may allocate tensors for minimizer later in the original
627		// graph whereas in the newly created graph, it is streamlined (the minimizer exists from the beginning). That
628		// will make the order of tensor symbols creation different, therefore, exact which tensor is which wrong as
629		// well. However, set a new minimizer won't change the exec symbol ordering, because we never create new exec
630		// symbols after gradient init step. Changing a new minimizer just updated that exec symbols setting, it is not
631		// a new exec symbol.
632		for (j = 0; j < src_input_size; j++)
633			if (src_inputs[j] >= 0)
634				_ccv_nnc_tensor_symbol_reinit(init->graph, model->graph, src_inputs[j], dest_inputs[j]);
635		for (j = 0; j < src_output_size; j++)
636			if (src_outputs[j] >= 0)
637				_ccv_nnc_tensor_symbol_reinit(init->graph, model->graph, src_outputs[j], dest_outputs[j]);
638	}
639	ccv_array_free(stack);
640	// After this, we get all tensors in the model graph resolved through tensor_auto.
641	ccv_nnc_symbolic_graph_tensor_auto(model->graph, TRAVERSE_FULL0,0,0,0);
642	// Verify symbols we get matches.
643	const int parameter_size = compiled_data->parameters->rnum;
644	for (i = 0; i < parameter_size; i++)
645		{ assert(((ccv_nnc_tensor_symbol_t*)ccv_array_get(compiled_data->parameters, i))->d == ((ccv_nnc_tensor_symbol_t*)ccv_array_get(init->compiled_data->parameters, i))->d)((void) sizeof ((((ccv_nnc_tensor_symbol_t*)((void*)(((char*)
((compiled_data->parameters)->data)) + (size_t)(compiled_data
->parameters)->rsize * (size_t)(i))))->d == ((ccv_nnc_tensor_symbol_t
*)((void*)(((char*)((init->compiled_data->parameters)->
data)) + (size_t)(init->compiled_data->parameters)->
rsize * (size_t)(i))))->d) ? 1 : 0), __extension__ ({ if (
((ccv_nnc_tensor_symbol_t*)((void*)(((char*)((compiled_data->
parameters)->data)) + (size_t)(compiled_data->parameters
)->rsize * (size_t)(i))))->d == ((ccv_nnc_tensor_symbol_t
*)((void*)(((char*)((init->compiled_data->parameters)->
data)) + (size_t)(init->compiled_data->parameters)->
rsize * (size_t)(i))))->d) ; else __assert_fail ("((ccv_nnc_tensor_symbol_t*)ccv_array_get(compiled_data->parameters, i))->d == ((ccv_nnc_tensor_symbol_t*)ccv_array_get(init->compiled_data->parameters, i))->d"
, "ccv_cnnp_model.c", 645, __extension__ __PRETTY_FUNCTION__)
; })); }
646	const int internal_size = compiled_data->internals->rnum;
647	for (i = 0; i < internal_size; i++)
648		{ assert(((ccv_nnc_tensor_symbol_t*)ccv_array_get(compiled_data->internals, i))->d == ((ccv_nnc_tensor_symbol_t*)ccv_array_get(init->compiled_data->internals, i))->d)((void) sizeof ((((ccv_nnc_tensor_symbol_t*)((void*)(((char*)
((compiled_data->internals)->data)) + (size_t)(compiled_data
->internals)->rsize * (size_t)(i))))->d == ((ccv_nnc_tensor_symbol_t
*)((void*)(((char*)((init->compiled_data->internals)->
data)) + (size_t)(init->compiled_data->internals)->rsize
 * (size_t)(i))))->d) ? 1 : 0), __extension__ ({ if (((ccv_nnc_tensor_symbol_t
*)((void*)(((char*)((compiled_data->internals)->data)) +
 (size_t)(compiled_data->internals)->rsize * (size_t)(i
))))->d == ((ccv_nnc_tensor_symbol_t*)((void*)(((char*)((init
->compiled_data->internals)->data)) + (size_t)(init->
compiled_data->internals)->rsize * (size_t)(i))))->d
) ; else __assert_fail ("((ccv_nnc_tensor_symbol_t*)ccv_array_get(compiled_data->internals, i))->d == ((ccv_nnc_tensor_symbol_t*)ccv_array_get(init->compiled_data->internals, i))->d"
, "ccv_cnnp_model.c", 648, __extension__ __PRETTY_FUNCTION__)
; })); }
649	// Go through compiled data.
650	if (compiled_data->tensor_arena)
651	{
652		const int flag = ccv_nnc_tensor_arena_reinit(compiled_data->tensor_arena, model->graph);
653		if (flag == 0 && compiled_data->graph_exec_arena)
654		{
655			ccv_nnc_graph_exec_reinit(compiled_data->graph_exec_arena, compiled_data->graph, model->graph);
656			// Since we will reinit, if we previously set is_test, we need to set it again.
657			if (compiled_data->is_test)
658			{
659				const int parallel_count = ccv_max(model->parallel_count, 1)({ typeof (model->parallel_count) _a = (model->parallel_count
); typeof (1) _b = (1); (_a > _b) ? _a : _b; });
660				ccv_nnc_graph_exec_update_t update = {
661					.parallel_count = parallel_count,
662					.graph = model->graph,
663					.graph_exec_arena = compiled_data->graph_exec_arena,
664				};
665				ccv_cnnp_model_set_is_test(model, 1, _ccv_cnnp_cmd_update_for_execs, &update);
666			}
667		} else
668			// Free-up tensor arena & graph exec arena.
669			_ccv_cnnp_compiled_data_graph_free(compiled_data);
670	}
671	// There are other compiled graphs, for accum and apply gradients.
672	// However, the main conclusion is, these absorb operations shouldn't impact parameters.
673	// Thus, it won't impact the shape of gradients (only outgrad). Since for outgrad, we
674	// don't allocate ourselves, it is not a concern. For normal gradients, the shape cannot
675	// be changed otherwise parameters' shape will be meaningless. The same goes to internals.
676	// That is why we don't update these compiled graphs at all this point.
677	// Free the model, we've already "absorbed" it.
678	ccv_cnnp_model_free(init);
679}
680 
681void ccv_cnnp_model_compile(ccv_cnnp_model_t* const model, const ccv_nnc_tensor_param_t* const inputs, const int input_size, const ccv_nnc_cmd_t minimizer, const ccv_nnc_cmd_t loss)
682{
683	assert(input_size == model->input_size || model->input_size == 0)((void) sizeof ((input_size == model->input_size || model->
input_size == 0) ? 1 : 0), __extension__ ({ if (input_size ==
 model->input_size || model->input_size == 0) ; else __assert_fail
 ("input_size == model->input_size || model->input_size == 0"
, "ccv_cnnp_model.c", 683, __extension__ __PRETTY_FUNCTION__)
; }));
684	if (model->input_size == 0)
685		model->input_size = input_size;
686	if (!model->graph) // The graph is not compiled yet.
687	{
688		model->graph = ccv_nnc_symbolic_graph_new();
689		_ccv_cnnp_model_compile(model, inputs, input_size, loss);
690		assert(model->compiled_data)((void) sizeof ((model->compiled_data) ? 1 : 0), __extension__
 ({ if (model->compiled_data) ; else __assert_fail ("model->compiled_data"
, "ccv_cnnp_model.c", 690, __extension__ __PRETTY_FUNCTION__)
; }));
691		int i, flag = 0;
692		for (i = 0; !flag && i < input_size; i++)
693			flag = (CCV_TENSOR_GET_MEMORY(inputs[i].type)((inputs[i].type) & 0x3) == CCV_TENSOR_GPU_MEMORY);
694		// If inputs are from GPU, stream type is GPU.
695		model->compiled_data->stream_type = flag ? CCV_STREAM_CONTEXT_GPU : CCV_STREAM_CONTEXT_CPU;
696		model->compiled_data->minimize.minimizer = minimizer;
697		model->compiled_data->minimize.max_saved_aux_size = ccv_nnc_minimizer_saved_aux_size(minimizer);
698	} else {
699		// Now, finally fill in this part. If the graph is already compiled, we make a copy of the model.
700		// And then absorb the "new model" to the old one.
701		ccv_cnnp_model_t* const init = ccv_cnnp_model_copy(model, model->is_trainable);
702		ccv_cnnp_model_absorb(model, init, inputs, input_size);
703		// Reset minimizer.
704		ccv_cnnp_model_set_minimizer(model, minimizer, 1, 0, 0);
705	}
706}
707 
708ccv_cnnp_model_t* ccv_cnnp_model_copy(const ccv_cnnp_model_t* const model, const int is_trainable)
709{
710	ccv_cnnp_model_t* const new_model = _ccv_cnnp_model_copy(model, 0);
711	new_model->is_trainable = is_trainable;
712	return new_model;
713}
714 
715void ccv_cnnp_model_tensor_auto(ccv_cnnp_model_t* const model, ccv_nnc_tensor_param_t* const outputs, const int output_size)
716{
717	assert(model->graph)((void) sizeof ((model->graph) ? 1 : 0), __extension__ ({ if
 (model->graph) ; else __assert_fail ("model->graph", "ccv_cnnp_model.c"
, 717, __extension__ __PRETTY_FUNCTION__); }));
718	assert(output_size == model->output_size)((void) sizeof ((output_size == model->output_size) ? 1 : 0
), __extension__ ({ if (output_size == model->output_size)
 ; else __assert_fail ("output_size == model->output_size"
, "ccv_cnnp_model.c", 718, __extension__ __PRETTY_FUNCTION__)
; }));
719	ccv_nnc_symbolic_graph_t* const graph = model->graph;
720	ccv_nnc_symbolic_graph_tensor_auto(graph, TRAVERSE_FULL0,0,0,0);
721	int i;
722	for (i = 0; i < output_size; i++)
723	{
724		assert(model->outputs[i].d != CCV_NNC_NO_TENSOR_SYMBOL)((void) sizeof ((model->outputs[i].d != CCV_NNC_NO_TENSOR_SYMBOL
) ? 1 : 0), __extension__ ({ if (model->outputs[i].d != CCV_NNC_NO_TENSOR_SYMBOL
) ; else __assert_fail ("model->outputs[i].d != CCV_NNC_NO_TENSOR_SYMBOL"
, "ccv_cnnp_model.c", 724, __extension__ __PRETTY_FUNCTION__)
; }));
725		outputs[i] = ccv_nnc_tensor_symbol_params(graph, model->outputs[i]);
726	}
727}
728 
729void ccv_cnnp_model_set_workspace_size(ccv_cnnp_model_t* const model, size_t workspace_size)
730{
731	if (workspace_size == model->workspace_size)
732		return;
733	model->workspace_size = workspace_size;
734	ccv_cnnp_compiled_data_t* const compiled_data = model->compiled_data;
735	if (compiled_data && compiled_data->graph)
736		ccv_nnc_graph_autotune(compiled_data->graph, workspace_size, 0, TRAVERSE_FULL0,0,0,0);
737}
738 
739size_t ccv_cnnp_model_workspace_size(ccv_cnnp_model_t* const model)
740{
741	return model->workspace_size;
742}
743 
744void ccv_cnnp_model_set_data_parallel(ccv_cnnp_model_t* const model, const int parallel)
745{
746	if (parallel == 0)
747		model->parallel_count = ccv_nnc_device_count(CCV_STREAM_CONTEXT_GPU);
748	else
749		model->parallel_count = parallel;
750	ccv_cnnp_compiled_data_t* const compiled_data = model->compiled_data;
751	if (compiled_data)
752		{ assert(!compiled_data->graph)((void) sizeof ((!compiled_data->graph) ? 1 : 0), __extension__
 ({ if (!compiled_data->graph) ; else __assert_fail ("!compiled_data->graph"
, "ccv_cnnp_model.c", 752, __extension__ __PRETTY_FUNCTION__)
; })); }
753}
754 
755void ccv_cnnp_model_set_max_concurrency(ccv_cnnp_model_t* const model, const int max_stream_count)
756{
757	model->max_stream_count = max_stream_count;
758	ccv_cnnp_compiled_data_t* const compiled_data = model->compiled_data;
759	if (compiled_data)
760		{ assert(!compiled_data->graph)((void) sizeof ((!compiled_data->graph) ? 1 : 0), __extension__
 ({ if (!compiled_data->graph) ; else __assert_fail ("!compiled_data->graph"
, "ccv_cnnp_model.c", 760, __extension__ __PRETTY_FUNCTION__)
; })); }
761}
762 
763void ccv_cnnp_model_set_memory_compression(ccv_cnnp_model_t* const model, const int memory_compression)
764{
765	model->memory_compression = memory_compression;
766	ccv_cnnp_compiled_data_t* const compiled_data = model->compiled_data;
767	if (compiled_data)
768		{ assert(!compiled_data->graph)((void) sizeof ((!compiled_data->graph) ? 1 : 0), __extension__
 ({ if (!compiled_data->graph) ; else __assert_fail ("!compiled_data->graph"
, "ccv_cnnp_model.c", 768, __extension__ __PRETTY_FUNCTION__)
; })); }
769}
770 
771void ccv_cnnp_model_set_memory_reduction(ccv_cnnp_model_t* const model, const int memory_reduction)
772{
773	model->memory_reduction = memory_reduction;
774	ccv_cnnp_compiled_data_t* const compiled_data = model->compiled_data;
775	if (compiled_data)
776		{ assert(!compiled_data->graph)((void) sizeof ((!compiled_data->graph) ? 1 : 0), __extension__
 ({ if (!compiled_data->graph) ; else __assert_fail ("!compiled_data->graph"
, "ccv_cnnp_model.c", 776, __extension__ __PRETTY_FUNCTION__)
; })); }
777}
778 
779void ccv_cnnp_model_set_start_index(ccv_cnnp_model_t* const model, const int start_index)
780{
781	assert(start_index >= 0)((void) sizeof ((start_index >= 0) ? 1 : 0), __extension__
 ({ if (start_index >= 0) ; else __assert_fail ("start_index >= 0"
, "ccv_cnnp_model.c", 781, __extension__ __PRETTY_FUNCTION__)
; }));
782	assert(!model->graph && !model->parameter_indices)((void) sizeof ((!model->graph && !model->parameter_indices
) ? 1 : 0), __extension__ ({ if (!model->graph && !
model->parameter_indices) ; else __assert_fail ("!model->graph && !model->parameter_indices"
, "ccv_cnnp_model.c", 782, __extension__ __PRETTY_FUNCTION__)
; }));
783	model->start_index = start_index;
784}
785 
786int ccv_cnnp_model_start_index(ccv_cnnp_model_t* const model)
787{
788	return model->start_index;
789}
790 
791void ccv_cnnp_model_set_gradient_checkpointing(ccv_cnnp_model_t* const model, const int gradient_checkpointing)
792{
793	model->gradient_checkpointing = gradient_checkpointing;
794}
795 
796int ccv_cnnp_model_gradient_checkpointing(ccv_cnnp_model_t* const model)
797{
798	return model->gradient_checkpointing;
799}
800 
801typedef struct {
802	int parallel_count;
803	ccv_nnc_symbolic_graph_t* graph;
804	ccv_cnnp_compiled_data_t* compiled_data;
805	ccv_nnc_tensor_arena_t* tensor_arena;
806} ccv_nnc_tensor_init_states_t;
807 
808static int _ccv_cnnp_any_to_init(const ccv_cnnp_compiled_data_t* const compiled_data)
809{
810	int i;
811	const uint32_t* const init_v = CCV_NNC_INIT_V(compiled_data->tensors_init.v)((uint32_t*)((uintptr_t)(compiled_data->tensors_init.v) &
 ~(uintptr_t)1));
812	for (i = 0; i < compiled_data->parameters->rnum; i++)
813	{
814		const int d = ((ccv_nnc_tensor_symbol_t*)ccv_array_get(compiled_data->parameters, i)((void*)(((char*)((compiled_data->parameters)->data)) +
 (size_t)(compiled_data->parameters)->rsize * (size_t)(
i))))->d;
815		if (!(init_v[d >> 5] & (1u << (d & 0x1f))))
816			return 1;
817	}
818	for (i = 0; i < compiled_data->internals->rnum; i++)
819	{
820		const int d = ((ccv_nnc_tensor_symbol_t*)ccv_array_get(compiled_data->internals, i)((void*)(((char*)((compiled_data->internals)->data)) + (
size_t)(compiled_data->internals)->rsize * (size_t)(i))
))->d;
821		if (!(init_v[d >> 5] & (1u << (d & 0x1f))))
822			return 1;
823	}
824	return 0;
825}
826 
827static void _ccv_cnnp_init_states_for_tensors(void* const context, const ccv_nnc_cmd_t cmd, const ccv_nnc_hint_t hint, const int flags, ccv_nnc_tensor_t* const input, const ccv_nnc_tensor_symbol_t output_symbol)
828{
829	ccv_nnc_tensor_init_states_t* const tensor_init_states = (ccv_nnc_tensor_init_states_t*)context;
830	ccv_nnc_tensor_arena_t* const tensor_arena = tensor_init_states->tensor_arena;
831	ccv_nnc_tensor_t* const output_tensor = ccv_nnc_tensor_from_symbol(tensor_arena, output_symbol);
832	if (!output_tensor)
833		return;
834	const int d = output_symbol.d;
835	assert(d < tensor_init_states->compiled_data->tensors_init.size)((void) sizeof ((d < tensor_init_states->compiled_data->
tensors_init.size) ? 1 : 0), __extension__ ({ if (d < tensor_init_states
->compiled_data->tensors_init.size) ; else __assert_fail
 ("d < tensor_init_states->compiled_data->tensors_init.size"
, "ccv_cnnp_model.c", 835, __extension__ __PRETTY_FUNCTION__)
; }));
836  uint32_t* const init_v = CCV_NNC_INIT_V(tensor_init_states->compiled_data->tensors_init.v)((uint32_t*)((uintptr_t)(tensor_init_states->compiled_data
->tensors_init.v) & ~(uintptr_t)1));
837	if (init_v[d >> 5] & (1u << (d & 0x1f)))
838		return;
839	init_v[d >> 5] |= (1u << (d & 0x1f));
840	ccv_nnc_cmd_exec(cmd, hint, flags, &input, input ? 1 : 0, &output_tensor, 1, 0);
841	const ccv_nnc_symbolic_graph_t* const graph = tensor_init_states->graph;
842	const int parallel_count = tensor_init_states->parallel_count;
843	int i;
844	for (i = 1; i < parallel_count; i++)
845	{
846		ccv_nnc_tensor_t* const copy = ccv_nnc_tensor_from_symbol(tensor_arena, ccv_nnc_tensor_symbol_copy(graph, output_symbol, i));
847		if (copy)
848			ccv_nnc_cmd_exec(CMD_DATA_TRANSFER_FORWARD()ccv_nnc_cmd(CCV_NNC_DATA_TRANSFER_FORWARD, 0, ccv_nnc_cmd_auto
, 0), ccv_nnc_no_hint, 0, &output_tensor, 1, &copy, 1, 0);
849	}
850}
851 
852// This method can only handle cases we added new tensors and exec, never delete. This invariant is true because
853// we setup everything (including calling simplify method) in ccv_cnnp_model_compile method, before this rewind setup.
854static void _ccv_cnnp_model_rewind_graph(ccv_cnnp_model_t* const model)
855{
856	assert(model->graph)((void) sizeof ((model->graph) ? 1 : 0), __extension__ ({ if
 (model->graph) ; else __assert_fail ("model->graph", "ccv_cnnp_model.c"
, 856, __extension__ __PRETTY_FUNCTION__); }));
857	assert(model->compiled_data)((void) sizeof ((model->compiled_data) ? 1 : 0), __extension__
 ({ if (model->compiled_data) ; else __assert_fail ("model->compiled_data"
, "ccv_cnnp_model.c", 857, __extension__ __PRETTY_FUNCTION__)
; }));
858	ccv_cnnp_compiled_data_t* const compiled_data = model->compiled_data;
859	assert(compiled_data->rewindables)((void) sizeof ((compiled_data->rewindables) ? 1 : 0), __extension__
 ({ if (compiled_data->rewindables) ; else __assert_fail (
"compiled_data->rewindables", "ccv_cnnp_model.c", 859, __extension__
 __PRETTY_FUNCTION__); }));
860	int i;
861	for (i = 0; i < compiled_data->rewindables->rnum; i++)
862	{
863		const ccv_cnnp_rewind_symbol_t* const rewind_symbol = (ccv_cnnp_rewind_symbol_t*)ccv_array_get(compiled_data->rewindables, i)((void*)(((char*)((compiled_data->rewindables)->data)) +
 (size_t)(compiled_data->rewindables)->rsize * (size_t)
(i)));
864		if (rewind_symbol->type == CCV_CNNP_REWIND_GRAPH_EXEC)
865			ccv_nnc_graph_exec_symbol_free(model->graph, rewind_symbol->graph_exec);
866		else if (rewind_symbol->type == CCV_CNNP_REWIND_TENSOR)
867			ccv_nnc_tensor_symbol_free(model->graph, rewind_symbol->tensor);
868	}
869	ccv_array_clear(compiled_data->rewindables);
870	ccv_nnc_graph_exec_symbol_autogen(model->graph, 0, 0, CCV_NNC_AUTOGEN_SOURCES_AND_DESTINATIONS);
871}
872 
873static void _ccv_cnnp_model_tensor_symbol_new_hook(void* context, const ccv_nnc_tensor_symbol_t symbol, const ccv_nnc_tensor_param_t info, const char* const name)
874{
875	const ccv_cnnp_rewind_symbol_t rewind_symbol = {
876		.type = CCV_CNNP_REWIND_TENSOR,
877		.tensor = symbol
878	};
879	ccv_array_t* const rewind_symbols = (ccv_array_t*)context;
880	ccv_array_push(rewind_symbols, &rewind_symbol);
881}
882 
883static void _ccv_cnnp_model_tensor_symbol_alias_new_hook(void* context, const ccv_nnc_tensor_symbol_t symbol, const ccv_nnc_tensor_symbol_t from_symbol, const int ofs[CCV_NNC_MAX_DIM_ALLOC(12)], const int inc[CCV_NNC_MAX_DIM_ALLOC(12)], const ccv_nnc_tensor_param_t info, const char* const name)
884{
885	const ccv_cnnp_rewind_symbol_t rewind_symbol = {
886		.type = CCV_CNNP_REWIND_TENSOR,
887		.tensor = symbol
888	};
889	ccv_array_t* const rewind_symbols = (ccv_array_t*)context;
890	ccv_array_push(rewind_symbols, &rewind_symbol);
891}
892 
893static void _ccv_cnnp_model_graph_exec_symbol_new_hook(void* context, const ccv_nnc_graph_exec_symbol_t symbol, const ccv_nnc_cmd_t cmd, const ccv_nnc_tensor_symbol_t* const inputs, const int input_size, const ccv_nnc_tensor_symbol_t* const outputs, const int output_size, const char* const name)
894{
895	const ccv_cnnp_rewind_symbol_t rewind_symbol = {
896		.type = CCV_CNNP_REWIND_GRAPH_EXEC,
897		.graph_exec = symbol
898	};
899	ccv_array_t* const rewind_symbols = (ccv_array_t*)context;
900	ccv_array_push(rewind_symbols, &rewind_symbol);
901}
902 
903static void _ccv_cnnp_model_graph_symbol_exec_set_for_graph_exec_arena(const ccv_nnc_graph_exec_arena_t* const graph_exec_arena, const int parallel_count, const ccv_nnc_graph_exec_symbol_t exec_symbol, const ccv_nnc_cmd_t cmd, ccv_nnc_symbolic_graph_t* const symbolic_graph)
904{
905	ccv_nnc_graph_exec_t const update_exec = ccv_nnc_graph_exec_from_symbol(graph_exec_arena, exec_symbol);
906	if (!CCV_NO_GRAPH_EXEC(update_exec)((update_exec).graph == 0))
907		ccv_nnc_graph_exec_set(update_exec.graph, update_exec, cmd);
908	int i;
909	for (i = 1; i < parallel_count; i++)
910	{
911		ccv_nnc_graph_exec_symbol_t copy_symbol = ccv_nnc_graph_exec_symbol_copy(symbolic_graph, exec_symbol, i);
912		const ccv_nnc_graph_exec_t copy = ccv_nnc_graph_exec_from_symbol(graph_exec_arena, copy_symbol);
913		if (!CCV_NO_GRAPH_EXEC(copy)((copy).graph == 0))
914			ccv_nnc_graph_exec_set(copy.graph, copy, cmd);
915	}
916}
917 
918static void _ccv_cnnp_model_graph_exec_symbol_set(ccv_nnc_symbolic_graph_t* const symbolic_graph, ccv_cnnp_compiled_data_t* const compiled_data, const int parallel_count, const ccv_nnc_graph_exec_symbol_t exec_symbol, const ccv_nnc_cmd_t cmd)
919{
920	assert(compiled_data)((void) sizeof ((compiled_data) ? 1 : 0), __extension__ ({ if
 (compiled_data) ; else __assert_fail ("compiled_data", "ccv_cnnp_model.c"
, 920, __extension__ __PRETTY_FUNCTION__); }));
921	assert(symbolic_graph)((void) sizeof ((symbolic_graph) ? 1 : 0), __extension__ ({ if
 (symbolic_graph) ; else __assert_fail ("symbolic_graph", "ccv_cnnp_model.c"
, 921, __extension__ __PRETTY_FUNCTION__); }));
922	ccv_nnc_graph_exec_symbol_set(symbolic_graph, exec_symbol, cmd);
923	int i;
924	for (i = 1; i < parallel_count; i++)
925	{
926		ccv_nnc_graph_exec_symbol_t copy_symbol = ccv_nnc_graph_exec_symbol_copy(symbolic_graph, exec_symbol, i);
927		if (copy_symbol.graph)
928			ccv_nnc_graph_exec_symbol_set(symbolic_graph, copy_symbol, cmd);
929	}
930	ccv_nnc_graph_exec_arena_t* const graph_exec_arena = compiled_data->graph_exec_arena;
931	if (graph_exec_arena)
932		_ccv_cnnp_model_graph_symbol_exec_set_for_graph_exec_arena(graph_exec_arena, parallel_count, exec_symbol, cmd, symbolic_graph);
933	// Skip backward graph exec arena because it is for a specific accum symbolic graph, not the main graph (model->graph)
934	ccv_nnc_graph_exec_arena_t* const gradient_graph_exec_arena = compiled_data->apply_gradients.graph_exec_arena;
935	if (gradient_graph_exec_arena)
936		_ccv_cnnp_model_graph_symbol_exec_set_for_graph_exec_arena(gradient_graph_exec_arena, parallel_count, exec_symbol, cmd, symbolic_graph);
937}
938 
939static int _ccv_cnnp_set_minimizer_for_parameter(ccv_nnc_symbolic_graph_t* const graph, ccv_cnnp_compiled_data_t* const compiled_data, ccv_nnc_graph_exec_symbol_t* const update_nodes, ccv_nnc_tensor_symbol_t* const updated_parameters, ccv_nnc_tensor_symbol_map_t* const saved_aux, const int parallel_count, const ccv_nnc_cmd_t minimizer, const int saved_aux_size, const int max_saved_aux_size, const int parameter_indice)
940{
941	int this_parameter_flag = 0;
942	if (update_nodes[parameter_indice].d == CCV_NNC_NO_TENSOR_SYMBOL)
943		return this_parameter_flag;
944	const ccv_nnc_cmd_t old_minimizer = ccv_nnc_graph_exec_symbol_cmd(graph, update_nodes[parameter_indice]);
945	int j, k;
946	// For no-op, we can preserve previous saved_aux_size.
947	if (old_minimizer.cmd != minimizer.cmd && minimizer.cmd != CCV_NNC_NOOP)
948	{
949		// If the old minimizer is a noop, then the old_saved_aux_size should be whatever its previous
950		// saved_aux_size is, otherwise we will reinit the saved_aux repeatedly if you switch between
951		// noop and a minimizer. We don't want that because we do that in high-level frameworks to
952		// make sure some model parameters don't update if we don't want them to.
953		int old_saved_aux_size;
954		if (old_minimizer.cmd == CCV_NNC_NOOP)
955		{
956			int input_size;
957			ccv_nnc_graph_exec_symbol_io(graph, update_nodes[parameter_indice], 0, &input_size, 0, 0);
958			if (input_size < 2) // This is not legit.
959				old_saved_aux_size = ccv_nnc_minimizer_saved_aux_size(old_minimizer);
960			else // See ccv_nnc_minimizer_saved_aux_size, the saved_aux is inputs excluding gradients and parameters.
961				old_saved_aux_size = input_size - 2;
962		} else
963			old_saved_aux_size = ccv_nnc_minimizer_saved_aux_size(old_minimizer);
964		if (old_saved_aux_size != saved_aux_size)
965		{
966			this_parameter_flag = 1;
967			if (saved_aux_size > old_saved_aux_size)
968			{
969				// Allocate new tensor symbols.
970				const ccv_nnc_tensor_param_t info = ccv_nnc_tensor_symbol_params(graph, updated_parameters[parameter_indice]);
971				for (j = old_saved_aux_size; j < saved_aux_size; j++)
972				{
973					saved_aux[parameter_indice * max_saved_aux_size + j].source = ccv_nnc_tensor_symbol_new(graph, info, 0);
974					saved_aux[parameter_indice * max_saved_aux_size + j].destination = ccv_nnc_tensor_symbol_new(graph, info, 0);
975					const int device_id = CCV_TENSOR_GET_DEVICE_ID(info.type)(((info.type) & 0xfff00) >> 8);
976					for (k = 1; k < parallel_count; k++)
977					{
978						ccv_nnc_tensor_param_t dev_info = info;
979						if (k != device_id)
980							CCV_TENSOR_SET_DEVICE_ID(dev_info.type, k)(dev_info.type) = (((dev_info.type) & ~0xfff00) | (((k) &
 0xfff) << 8));
981						else
982							CCV_TENSOR_SET_DEVICE_ID(dev_info.type, 0)(dev_info.type) = (((dev_info.type) & ~0xfff00) | (((0) &
 0xfff) << 8));
983						const ccv_nnc_tensor_symbol_t src_copy = ccv_nnc_tensor_symbol_new(graph, dev_info, 0);
984						const ccv_nnc_tensor_symbol_t dest_copy = ccv_nnc_tensor_symbol_new(graph, dev_info, 0);
985						ccv_nnc_tensor_symbol_set_copy(graph, saved_aux[parameter_indice * max_saved_aux_size + j].source, k, src_copy);
986						ccv_nnc_tensor_symbol_set_copy(graph, saved_aux[parameter_indice * max_saved_aux_size + j].destination, k, dest_copy);
987					}
988				}
989			} else {
990				for (j = saved_aux_size; j < old_saved_aux_size; j++)
991				{
992					for (k = 1; k < parallel_count; k++)
993					{
994						const ccv_nnc_tensor_symbol_t src_copy = ccv_nnc_tensor_symbol_copy(graph, saved_aux[parameter_indice * max_saved_aux_size + j].source, k);
995						if (src_copy.d >= 0)
996						{
997							ccv_nnc_tensor_symbol_free(graph, src_copy);
998							ccv_nnc_tensor_symbol_set_copy(graph, saved_aux[parameter_indice * max_saved_aux_size + j].source, k, NO_TENSOR_SYMBOL(const ccv_nnc_tensor_symbol_t){.d = CCV_NNC_NO_TENSOR_SYMBOL
});
999						}
1000						const ccv_nnc_tensor_symbol_t dest_copy = ccv_nnc_tensor_symbol_copy(graph, saved_aux[parameter_indice * max_saved_aux_size + j].destination, k);
1001						if (dest_copy.d >= 0)
1002						{
1003							ccv_nnc_tensor_symbol_free(graph, dest_copy);
1004							ccv_nnc_tensor_symbol_set_copy(graph, saved_aux[parameter_indice * max_saved_aux_size + j].destination, k, NO_TENSOR_SYMBOL(const ccv_nnc_tensor_symbol_t){.d = CCV_NNC_NO_TENSOR_SYMBOL
});
1005						}
1006					}
1007					ccv_nnc_tensor_symbol_free(graph, saved_aux[parameter_indice * max_saved_aux_size + j].source);
1008					ccv_nnc_tensor_symbol_free(graph, saved_aux[parameter_indice * max_saved_aux_size + j].destination);
1009					saved_aux[parameter_indice * max_saved_aux_size + j].source = saved_aux[parameter_indice * max_saved_aux_size + j].destination = NO_TENSOR_SYMBOL(const ccv_nnc_tensor_symbol_t){.d = CCV_NNC_NO_TENSOR_SYMBOL
};
1010				}
1011			}
1012		}
1013	}
1014	_ccv_cnnp_model_graph_exec_symbol_set(graph, compiled_data, parallel_count, update_nodes[parameter_indice], minimizer);
1015	if (this_parameter_flag)
1016	{
1017		ccv_nnc_tensor_symbol_t update_inputs[saved_aux_size + 2];
1018		ccv_nnc_tensor_symbol_t update_outputs[saved_aux_size + 1];
1019		const int* inputs = 0;
1020		int input_size = 0;
1021		ccv_nnc_graph_exec_symbol_io(graph, update_nodes[parameter_indice], &inputs, &input_size, 0, 0);
1022		assert(input_size >= 1)((void) sizeof ((input_size >= 1) ? 1 : 0), __extension__ (
{ if (input_size >= 1) ; else __assert_fail ("input_size >= 1"
, "ccv_cnnp_model.c", 1022, __extension__ __PRETTY_FUNCTION__
); }));
1023		update_inputs[0].d = inputs[0];
1024		update_inputs[0].graph = graph;
1025		update_inputs[1].d = inputs[1];
1026		update_inputs[1].graph = graph;
1027		update_outputs[0] = updated_parameters[parameter_indice];
1028		for (j = 0; j < saved_aux_size; j++)
1029		{
1030			update_inputs[j + 2] = saved_aux[parameter_indice * max_saved_aux_size + j].source;
1031			update_outputs[j + 1] = saved_aux[parameter_indice * max_saved_aux_size + j].destination;
1032		}
1033		ccv_nnc_graph_exec_symbol_set_io(graph, update_nodes[parameter_indice], update_inputs, saved_aux_size + 2, update_outputs, saved_aux_size + 1);
1034		for (k = 1; k < parallel_count; k++)
1035		{
1036			const ccv_nnc_graph_exec_symbol_t copy = ccv_nnc_graph_exec_symbol_copy(graph, update_nodes[parameter_indice], k);
1037			assert(copy.d >= 0)((void) sizeof ((copy.d >= 0) ? 1 : 0), __extension__ ({ if
 (copy.d >= 0) ; else __assert_fail ("copy.d >= 0", "ccv_cnnp_model.c"
, 1037, __extension__ __PRETTY_FUNCTION__); }));
1038			ccv_nnc_graph_exec_symbol_io(graph, copy, &inputs, &input_size, 0, 0);
1039			assert(input_size >= 1)((void) sizeof ((input_size >= 1) ? 1 : 0), __extension__ (
{ if (input_size >= 1) ; else __assert_fail ("input_size >= 1"
, "ccv_cnnp_model.c", 1039, __extension__ __PRETTY_FUNCTION__
); }));
1040			update_inputs[0].d = inputs[0];
1041			update_inputs[0].graph = graph;
1042			update_inputs[1].d = inputs[1];
1043			update_inputs[1].graph = graph;
1044			update_outputs[0] = ccv_nnc_tensor_symbol_copy(graph, updated_parameters[parameter_indice], k);
1045			for (j = 0; j < saved_aux_size; j++)
1046			{
1047				update_inputs[j + 2] = ccv_nnc_tensor_symbol_copy(graph, saved_aux[parameter_indice * max_saved_aux_size + j].source, k);
1048				update_outputs[j + 1] = ccv_nnc_tensor_symbol_copy(graph, saved_aux[parameter_indice * max_saved_aux_size + j].destination, k);
1049			}
1050			ccv_nnc_graph_exec_symbol_set_io(graph, copy, update_inputs, saved_aux_size + 2, update_outputs, saved_aux_size + 1);
1051		}
1052	}
1053	return this_parameter_flag;
1054}
1055 
1056typedef struct {
1057	int parameter_size;
1058	ccv_nnc_cmd_t minimizer;
1059	ccv_cnnp_model_io_t parameters[1];
1060} ccv_cnnp_set_minimizer_for_parameter_t;
1061 
1062static int _ccv_cnnp_apply_parameters_with_minimizer(ccv_cnnp_model_t* const model)
1063{
1064	ccv_cnnp_compiled_data_t* const compiled_data = model->compiled_data;
1065	assert(compiled_data)((void) sizeof ((compiled_data) ? 1 : 0), __extension__ ({ if
 (compiled_data) ; else __assert_fail ("compiled_data", "ccv_cnnp_model.c"
, 1065, __extension__ __PRETTY_FUNCTION__); }));
1066	const int max_saved_aux_size = compiled_data->minimize.max_saved_aux_size;
1067	// We update all parameters, at this point, we have one minimizer.
1068	const int parameter_size = compiled_data->parameters->rnum;
1069	ccv_nnc_graph_exec_symbol_t* const update_nodes = compiled_data->update_nodes;
1070	ccv_nnc_symbolic_graph_t* const symbolic_graph = model->graph;
1071	assert(symbolic_graph)((void) sizeof ((symbolic_graph) ? 1 : 0), __extension__ ({ if
 (symbolic_graph) ; else __assert_fail ("symbolic_graph", "ccv_cnnp_model.c"
, 1071, __extension__ __PRETTY_FUNCTION__); }));
1072	const int parallel_count = _ccv_cnnp_model_root_parallel_count(model);
1073	assert(_ccv_cnnp_model_effective_parallel_count(model) == parallel_count && "local replicated stateful models only support forward / no-grad evaluation for now")((void) sizeof ((_ccv_cnnp_model_effective_parallel_count(model
) == parallel_count && "local replicated stateful models only support forward / no-grad evaluation for now"
) ? 1 : 0), __extension__ ({ if (_ccv_cnnp_model_effective_parallel_count
(model) == parallel_count && "local replicated stateful models only support forward / no-grad evaluation for now"
) ; else __assert_fail ("_ccv_cnnp_model_effective_parallel_count(model) == parallel_count && \"local replicated stateful models only support forward / no-grad evaluation for now\""
, "ccv_cnnp_model.c", 1073, __extension__ __PRETTY_FUNCTION__
); }));
1074	ccv_array_t* const parameters = compiled_data->minimize.parameters;
1075	ccv_array_t* const parameter_indices = ccv_array_new(sizeof(int), 0, 0);
1076	int i, j, flag = 0;
1077	for (i = 0; i < parameters->rnum; i++)
1078	{
1079		ccv_cnnp_set_minimizer_for_parameter_t* const set_minimizer_for_parameter = *(ccv_cnnp_set_minimizer_for_parameter_t**)ccv_array_get(parameters, i)((void*)(((char*)((parameters)->data)) + (size_t)(parameters
)->rsize * (size_t)(i)));
1080		for (j = 0; j < set_minimizer_for_parameter->parameter_size; j++)
1081		{
1082			const int param_sel = set_minimizer_for_parameter->parameters[j]->param_sel > 0 ? set_minimizer_for_parameter->parameters[j]->param_sel - 1 : set_minimizer_for_parameter->parameters[j]->param_sel;
1083			assert(set_minimizer_for_parameter->parameters[j]->param_sel != 0)((void) sizeof ((set_minimizer_for_parameter->parameters[j
]->param_sel != 0) ? 1 : 0), __extension__ ({ if (set_minimizer_for_parameter
->parameters[j]->param_sel != 0) ; else __assert_fail (
"set_minimizer_for_parameter->parameters[j]->param_sel != 0"
, "ccv_cnnp_model.c", 1083, __extension__ __PRETTY_FUNCTION__
); }));
1084			const int old_rnum = parameter_indices->rnum;
1085			ccv_cnnp_model_add_to_parameter_indices(set_minimizer_for_parameter->parameters[j]->model, param_sel, parameter_indices);
1086			const int param_ref = set_minimizer_for_parameter->parameters[j]->param_ref > 0 ? set_minimizer_for_parameter->parameters[j]->param_ref - 1 : set_minimizer_for_parameter->parameters[j]->param_ref;
1087			assert(set_minimizer_for_parameter->parameters[j]->param_ref != 0)((void) sizeof ((set_minimizer_for_parameter->parameters[j
]->param_ref != 0) ? 1 : 0), __extension__ ({ if (set_minimizer_for_parameter
->parameters[j]->param_ref != 0) ; else __assert_fail (
"set_minimizer_for_parameter->parameters[j]->param_ref != 0"
, "ccv_cnnp_model.c", 1087, __extension__ __PRETTY_FUNCTION__
); }));
1088			if (param_ref >= 0)
1089			{
1090				assert(param_ref + old_rnum < parameter_indices->rnum)((void) sizeof ((param_ref + old_rnum < parameter_indices->
rnum) ? 1 : 0), __extension__ ({ if (param_ref + old_rnum <
 parameter_indices->rnum) ; else __assert_fail ("param_ref + old_rnum < parameter_indices->rnum"
, "ccv_cnnp_model.c", 1090, __extension__ __PRETTY_FUNCTION__
); }));
1091				*(int*)ccv_array_get(parameter_indices, old_rnum)((void*)(((char*)((parameter_indices)->data)) + (size_t)(parameter_indices
)->rsize * (size_t)(old_rnum))) = *(int*)ccv_array_get(parameter_indices, param_ref + old_rnum)((void*)(((char*)((parameter_indices)->data)) + (size_t)(parameter_indices
)->rsize * (size_t)(param_ref + old_rnum)));
1092				parameter_indices->rnum = old_rnum + 1;
1093			}
1094		}
1095		const int saved_aux_size = ccv_nnc_minimizer_saved_aux_size(set_minimizer_for_parameter->minimizer);
1096		// We may have duplicated indices, but that is OK, we will set it twice.
1097		for (j = 0; j < parameter_indices->rnum; j++)
1098		{
1099			const int d = *(int*)ccv_array_get(parameter_indices, j)((void*)(((char*)((parameter_indices)->data)) + (size_t)(parameter_indices
)->rsize * (size_t)(j)));
1100			assert(d <= parameter_size)((void) sizeof ((d <= parameter_size) ? 1 : 0), __extension__
 ({ if (d <= parameter_size) ; else __assert_fail ("d <= parameter_size"
, "ccv_cnnp_model.c", 1100, __extension__ __PRETTY_FUNCTION__
); }));
1101			if (_ccv_cnnp_set_minimizer_for_parameter(symbolic_graph, compiled_data, update_nodes, compiled_data->updated_parameters, compiled_data->saved_aux, parallel_count, set_minimizer_for_parameter->minimizer, saved_aux_size, max_saved_aux_size, d))
1102				flag = 1;
1103		}
1104		ccv_array_clear(parameter_indices);
1105	}
1106	ccv_array_free(parameter_indices);
1107	return flag;
1108}
1109 
1110static void _ccv_cnnp_scatter_saved_aux(ccv_nnc_tensor_symbol_map_t* const saved_aux, const int parameter_size, const int old_saved_aux_size, const int new_saved_aux_size)
1111{
1112	if (new_saved_aux_size == old_saved_aux_size)
1113		return;
1114	assert(new_saved_aux_size > old_saved_aux_size)((void) sizeof ((new_saved_aux_size > old_saved_aux_size) ?
 1 : 0), __extension__ ({ if (new_saved_aux_size > old_saved_aux_size
) ; else __assert_fail ("new_saved_aux_size > old_saved_aux_size"
, "ccv_cnnp_model.c", 1114, __extension__ __PRETTY_FUNCTION__
); }));
1115	int i, j;
1116	for (i = parameter_size - 1; i >= 0; i--)
1117	{
1118		for (j = new_saved_aux_size - 1; j >= old_saved_aux_size; j--)
1119			saved_aux[i * new_saved_aux_size + j].source = saved_aux[i * new_saved_aux_size + j].destination = NO_TENSOR_SYMBOL(const ccv_nnc_tensor_symbol_t){.d = CCV_NNC_NO_TENSOR_SYMBOL
};
1120		for (j = old_saved_aux_size - 1; j >= 0; j--)
1121			saved_aux[i * new_saved_aux_size + j] = saved_aux[i * old_saved_aux_size + j];
1122	}
1123}
1124 
1125static void _ccv_cnnp_model_set_rewindables(ccv_cnnp_model_t* const model)
1126{
1127	ccv_cnnp_compiled_data_t* const compiled_data = model->compiled_data;
1128	assert(compiled_data)((void) sizeof ((compiled_data) ? 1 : 0), __extension__ ({ if
 (compiled_data) ; else __assert_fail ("compiled_data", "ccv_cnnp_model.c"
, 1128, __extension__ __PRETTY_FUNCTION__); }));
1129	if (!compiled_data->rewindables)
1130		compiled_data->rewindables = ccv_array_new(sizeof(ccv_cnnp_rewind_symbol_t), 0, 0);
1131	ccv_nnc_tensor_symbol_new_hook(model->graph, _ccv_cnnp_model_tensor_symbol_new_hook, compiled_data->rewindables, 0);
1132	ccv_nnc_tensor_symbol_alias_new_hook(model->graph, _ccv_cnnp_model_tensor_symbol_alias_new_hook, compiled_data->rewindables, 0);
1133	ccv_nnc_graph_exec_symbol_new_hook(model->graph, _ccv_cnnp_model_graph_exec_symbol_new_hook, compiled_data->rewindables, 0);
1134}
1135 
1136static void _ccv_cnnp_model_gradient_init(ccv_cnnp_model_t* const model, const int gradient_mode, const uint64_t disable_outgrad, ccv_nnc_tensor_t* const* const fits, const int fit_size)
1137{
1138	ccv_cnnp_compiled_data_t* const compiled_data = model->compiled_data;
1139	assert(compiled_data->gradient_mode == CCV_CNNP_COMPILED_DATA_GRADIENT_NONE)((void) sizeof ((compiled_data->gradient_mode == CCV_CNNP_COMPILED_DATA_GRADIENT_NONE
) ? 1 : 0), __extension__ ({ if (compiled_data->gradient_mode
 == CCV_CNNP_COMPILED_DATA_GRADIENT_NONE) ; else __assert_fail
 ("compiled_data->gradient_mode == CCV_CNNP_COMPILED_DATA_GRADIENT_NONE"
, "ccv_cnnp_model.c", 1139, __extension__ __PRETTY_FUNCTION__
); }));
1140	assert(gradient_mode != CCV_CNNP_COMPILED_DATA_GRADIENT_NONE)((void) sizeof ((gradient_mode != CCV_CNNP_COMPILED_DATA_GRADIENT_NONE
) ? 1 : 0), __extension__ ({ if (gradient_mode != CCV_CNNP_COMPILED_DATA_GRADIENT_NONE
) ; else __assert_fail ("gradient_mode != CCV_CNNP_COMPILED_DATA_GRADIENT_NONE"
, "ccv_cnnp_model.c", 1140, __extension__ __PRETTY_FUNCTION__
); }));
1141	const int evaluate_to_size = compiled_data->evaluate.to_size;
1142	assert(evaluate_to_size > 0)((void) sizeof ((evaluate_to_size > 0) ? 1 : 0), __extension__
 ({ if (evaluate_to_size > 0) ; else __assert_fail ("evaluate_to_size > 0"
, "ccv_cnnp_model.c", 1142, __extension__ __PRETTY_FUNCTION__
); }));
1143	const int parallel_count = _ccv_cnnp_model_root_parallel_count(model);
1144	assert(_ccv_cnnp_model_effective_parallel_count(model) == parallel_count && "local replicated stateful models only support forward / no-grad evaluation for now")((void) sizeof ((_ccv_cnnp_model_effective_parallel_count(model
) == parallel_count && "local replicated stateful models only support forward / no-grad evaluation for now"
) ? 1 : 0), __extension__ ({ if (_ccv_cnnp_model_effective_parallel_count
(model) == parallel_count && "local replicated stateful models only support forward / no-grad evaluation for now"
) ; else __assert_fail ("_ccv_cnnp_model_effective_parallel_count(model) == parallel_count && \"local replicated stateful models only support forward / no-grad evaluation for now\""
, "ccv_cnnp_model.c", 1144, __extension__ __PRETTY_FUNCTION__
); }));
1145	compiled_data->evaluate.tos = ccreallocrealloc(compiled_data->evaluate.tos, sizeof(ccv_nnc_graph_exec_symbol_t) * evaluate_to_size * parallel_count + sizeof(ccv_nnc_graph_exec_t) * evaluate_to_size * parallel_count);
1146	compiled_data->evaluate.to_ops = (ccv_nnc_graph_exec_t*)(compiled_data->evaluate.tos + evaluate_to_size * parallel_count);
1147	int i, j;
1148	const int output_size = model->output_size;
1149	assert(!fits || fit_size == output_size * parallel_count)((void) sizeof ((!fits || fit_size == output_size * parallel_count
) ? 1 : 0), __extension__ ({ if (!fits || fit_size == output_size
 * parallel_count) ; else __assert_fail ("!fits || fit_size == output_size * parallel_count"
, "ccv_cnnp_model.c", 1149, __extension__ __PRETTY_FUNCTION__
); }));
1150	if (fits)
1151		for (i = 0; i < output_size; i++)
1152			ccv_nnc_tensor_symbol_set(model->graph, compiled_data->fits[i], fits[i]->info);
1153	const int max_saved_aux_size = compiled_data->minimize.max_saved_aux_size;
1154	const int parameter_size = compiled_data->parameters->rnum;
1155	compiled_data->updated_parameters = (ccv_nnc_tensor_symbol_t*)ccmallocmalloc(sizeof(ccv_nnc_tensor_symbol_t) * parameter_size + sizeof(ccv_nnc_graph_exec_symbol_t) * parameter_size + sizeof(ccv_nnc_tensor_symbol_map_t) * max_saved_aux_size * parameter_size);
1156	compiled_data->update_nodes = (ccv_nnc_graph_exec_symbol_t*)(compiled_data->updated_parameters + parameter_size);
1157	compiled_data->saved_aux = (ccv_nnc_tensor_symbol_map_t*)(compiled_data->update_nodes + parameter_size);
1158	int parameter_size_maybe_more = parameter_size;
1159	compiled_data->disable_outgrad = disable_outgrad;
1160	int outgrad_size;
1161	if (gradient_mode == CCV_CNNP_COMPILED_DATA_GRADIENT_TRAINABLES || model->input_size == 0)
1162		outgrad_size = 0;
1163	else if (disable_outgrad == CCV_CNNP_DISABLE_OUTGRAD_NONE) // Compute minimize with gradients including inputs.
1164		outgrad_size = model->input_size;
1165	else {
1166		assert(disable_outgrad != CCV_CNNP_DISABLE_OUTGRAD_ALL)((void) sizeof ((disable_outgrad != CCV_CNNP_DISABLE_OUTGRAD_ALL
) ? 1 : 0), __extension__ ({ if (disable_outgrad != CCV_CNNP_DISABLE_OUTGRAD_ALL
) ; else __assert_fail ("disable_outgrad != CCV_CNNP_DISABLE_OUTGRAD_ALL"
, "ccv_cnnp_model.c", 1166, __extension__ __PRETTY_FUNCTION__
); })); // If it is disable all, gradient mode won't be this.
1167		outgrad_size = 0;
1168		for (i = 0; i < model->input_size; i++)
1169			if (!(disable_outgrad & ((uint64_t)1 << i)))
1170				++outgrad_size;
1171	}
1172	compiled_data->outgrad_size = outgrad_size;
1173	parameter_size_maybe_more += outgrad_size;
1174	compiled_data->gradients = (ccv_nnc_tensor_symbol_t*)ccmallocmalloc(sizeof(ccv_nnc_tensor_symbol_t) * parameter_size_maybe_more + sizeof(ccv_nnc_graph_exec_symbol_t) * parameter_size_maybe_more * parallel_count);
1175	compiled_data->outgrads = parameter_size_maybe_more > parameter_size ? compiled_data->gradients + parameter_size : 0;
1176	compiled_data->backward.tos = (ccv_nnc_graph_exec_symbol_t*)(compiled_data->gradients + parameter_size_maybe_more);
1177	compiled_data->backward.to_size = parameter_size_maybe_more;
1178	ccv_nnc_tensor_symbol_t* parameters = (ccv_nnc_tensor_symbol_t*)ccv_array_get(compiled_data->parameters, 0)((void*)(((char*)((compiled_data->parameters)->data)) +
 (size_t)(compiled_data->parameters)->rsize * (size_t)(
0)));
1179	if (compiled_data->parameter_flags)
1180	{
1181		parameters = (ccv_nnc_tensor_symbol_t*)ccmallocmalloc(sizeof(ccv_nnc_tensor_symbol_t) * parameter_size);
1182		for (i = 0; i < parameter_size; i++)
1183			if (compiled_data->parameter_flags[i >> 6] & ((uint64_t)1 << (i & 63)))
1184				parameters[i] = *(ccv_nnc_tensor_symbol_t*)ccv_array_get(compiled_data->parameters, i)((void*)(((char*)((compiled_data->parameters)->data)) +
 (size_t)(compiled_data->parameters)->rsize * (size_t)(
i)));
1185			else
1186				parameters[i] = NO_TENSOR_SYMBOL(const ccv_nnc_tensor_symbol_t){.d = CCV_NNC_NO_TENSOR_SYMBOL
};
1187	}
1188	if (gradient_mode == CCV_CNNP_COMPILED_DATA_GRADIENT_TRAINABLES || model->input_size == 0)
1189		ccv_nnc_symbolic_graph_minimize(model->graph, compiled_data->minimize.minimizer, compiled_data->f, output_size, parameters, parameter_size, 0, 0, SYMBOLIC_GRAPH_SOURCES(model->graph)ccv_nnc_symbolic_graph_sources(model->graph), ccv_nnc_symbolic_graph_source_size
(model->graph), SYMBOLIC_GRAPH_DESTINATIONS(model->graph)ccv_nnc_symbolic_graph_destinations(model->graph), ccv_nnc_symbolic_graph_destination_size
(model->graph), compiled_data->gradients, compiled_data->updated_parameters, compiled_data->saved_aux, compiled_data->update_nodes);
1190	else if (disable_outgrad == CCV_CNNP_DISABLE_OUTGRAD_NONE) // Compute minimize with gradients including inputs.
1191		ccv_nnc_symbolic_graph_minimize(model->graph, compiled_data->minimize.minimizer, compiled_data->f, output_size, parameters, parameter_size, model->inputs, model->input_size, SYMBOLIC_GRAPH_SOURCES(model->graph)ccv_nnc_symbolic_graph_sources(model->graph), ccv_nnc_symbolic_graph_source_size
(model->graph), SYMBOLIC_GRAPH_DESTINATIONS(model->graph)ccv_nnc_symbolic_graph_destinations(model->graph), ccv_nnc_symbolic_graph_destination_size
(model->graph), compiled_data->gradients, compiled_data->updated_parameters, compiled_data->saved_aux, compiled_data->update_nodes);
1192	else { // Compute minimize with gradients including selected inputs.
1193		assert(model->input_size > 0)((void) sizeof ((model->input_size > 0) ? 1 : 0), __extension__
 ({ if (model->input_size > 0) ; else __assert_fail ("model->input_size > 0"
, "ccv_cnnp_model.c", 1193, __extension__ __PRETTY_FUNCTION__
); }));
1194		assert(disable_outgrad != CCV_CNNP_DISABLE_OUTGRAD_ALL)((void) sizeof ((disable_outgrad != CCV_CNNP_DISABLE_OUTGRAD_ALL
) ? 1 : 0), __extension__ ({ if (disable_outgrad != CCV_CNNP_DISABLE_OUTGRAD_ALL
) ; else __assert_fail ("disable_outgrad != CCV_CNNP_DISABLE_OUTGRAD_ALL"
, "ccv_cnnp_model.c", 1194, __extension__ __PRETTY_FUNCTION__
); })); // If it is disable all, gradient mode won't be this.
1195		assert(outgrad_size > 0)((void) sizeof ((outgrad_size > 0) ? 1 : 0), __extension__
 ({ if (outgrad_size > 0) ; else __assert_fail ("outgrad_size > 0"
, "ccv_cnnp_model.c", 1195, __extension__ __PRETTY_FUNCTION__
); }));
1196		ccv_nnc_tensor_symbol_t outgrads[outgrad_size];
1197		j = 0;
1198		for (i = 0; i < model->input_size; i++)
1199			if (!(disable_outgrad & ((uint64_t)1 << i)))
1200				outgrads[j++] = model->inputs[i];
1201		ccv_nnc_symbolic_graph_minimize(model->graph, compiled_data->minimize.minimizer, compiled_data->f, output_size, parameters, parameter_size, outgrads, outgrad_size, SYMBOLIC_GRAPH_SOURCES(model->graph)ccv_nnc_symbolic_graph_sources(model->graph), ccv_nnc_symbolic_graph_source_size
(model->graph), SYMBOLIC_GRAPH_DESTINATIONS(model->graph)ccv_nnc_symbolic_graph_destinations(model->graph), ccv_nnc_symbolic_graph_destination_size
(model->graph), compiled_data->gradients, compiled_data->updated_parameters, compiled_data->saved_aux, compiled_data->update_nodes);
1202	}
1203	if (compiled_data->parameter_flags)
1204		ccfreefree(parameters);
1205	_ccv_cnnp_scatter_saved_aux(compiled_data->saved_aux, parameter_size, ccv_nnc_minimizer_saved_aux_size(compiled_data->minimize.minimizer), compiled_data->minimize.max_saved_aux_size);
1206	if (compiled_data->minimize.parameters)
1207		_ccv_cnnp_apply_parameters_with_minimizer(model);
1208	// Go through gradient checkpoints to generate tensor inputs for backward pass just before executing the backward pass.
1209	ccv_cnnp_model_apply_gradient_checkpoints(compiled_data, model->graph);
1210	for (i = 0; i < output_size; i++)
1211	{
1212		const ccv_nnc_tensor_symbol_t df = ccv_nnc_tensor_symbol_for_backward(model->graph, compiled_data->f[i]);
1213		// Init this to 1 so we can backprop.
1214		ccv_nnc_tensor_symbol_set_flags(model->graph, df, CCV_NNC_TENSOR_SYMBOL_INIT_ONES);
1215	}
1216	compiled_data->backward.to_size = 0;
1217	for (i = 0; i < parameter_size_maybe_more; i++)
1218		if (compiled_data->gradients[i].d != CCV_NNC_NO_TENSOR_SYMBOL)
1219			compiled_data->backward.tos[compiled_data->backward.to_size++] = ccv_nnc_graph_exec_symbol_for_backward(model->graph, compiled_data->gradients[i]);
1220	ccv_nnc_graph_exec_symbol_autogen(model->graph, 0, 0, CCV_NNC_AUTOGEN_ALL_EXECS);
1221	ccv_nnc_symbolic_graph_set_destinations(model->graph, compiled_data->update_nodes, parameter_size);
1222	for (i = 0; i < parameter_size_maybe_more - parameter_size; i++)
1223	{
1224		if (compiled_data->outgrads[i].d < 0) // When we go through input, we might find zero-length inputs, and for these, we cannot have any outgrads.
1225			continue;
1226		const ccv_nnc_graph_exec_symbol_t outgrad = ccv_nnc_graph_exec_symbol_for_backward(model->graph, compiled_data->outgrads[i]);
1227		const int* tos;
1228		int to_size;
1229		ccv_nnc_graph_exec_symbol_to(model->graph, outgrad, &tos, &to_size);
1230		if (to_size == 0) // If this is the end (no minimizers afterwards). We need to attach this as a destination. Otherwise this is covered in update_nodes.
1231		{
1232			const ccv_nnc_graph_exec_symbol_t* destinations = ccv_nnc_symbolic_graph_destinations(model->graph);
1233			const int destination_count = ccv_nnc_symbolic_graph_destination_size(model->graph);
1234			int flag = 0;
1235			const int outgrad_destination_start = ccv_max(0, destination_count - i)({ typeof (0) _a = (0); typeof (destination_count - i) _b = (
destination_count - i); (_a > _b) ? _a : _b; });
1236			for (j = i - 1; !flag && j >= 0; j--)
1237				if (j + outgrad_destination_start < destination_count)
1238					flag = (destinations[j + outgrad_destination_start].d == outgrad.d);
1239			if (!flag) // Only if we cannot find it, we add it.
1240				ccv_nnc_symbolic_graph_add_destination(model->graph, outgrad);
1241		}
1242	}
1243	if (parallel_count > 1)
1244	{
1245		ccv_nnc_symbolic_graph_data_parallel(model->graph, parallel_count,
1246			0, 0,
1247			compiled_data->gradients, parameter_size /* No need to deal with outgrads, we don't allreduce outgrads */,
1248			compiled_data->gradients /* We only care about gradients before allreduce, thus, update our current pointers */,
1249			0, 0, 0,
1250			CCV_NNC_PARALLEL_REDUCE_OP_SUM,
1251			SYMBOLIC_GRAPH_SOURCES(model->graph)ccv_nnc_symbolic_graph_sources(model->graph), ccv_nnc_symbolic_graph_source_size
(model->graph), SYMBOLIC_GRAPH_DESTINATIONS(model->graph)ccv_nnc_symbolic_graph_destinations(model->graph), ccv_nnc_symbolic_graph_destination_size
(model->graph));
1252		ccv_nnc_graph_exec_symbol_autogen(model->graph, 0, 0, CCV_NNC_AUTOGEN_SOURCES_AND_DESTINATIONS);
1253		for (i = 0; i < evaluate_to_size; i++)
1254			for (j = 1; j < parallel_count; j++)
1255			{
1256				const ccv_nnc_graph_exec_symbol_t copy = ccv_nnc_graph_exec_symbol_copy(model->graph, compiled_data->evaluate.tos[i], j);
1257				if (copy.d != CCV_NNC_NO_GRAPH_EXEC_SYMBOL)
1258					compiled_data->evaluate.tos[compiled_data->evaluate.to_size++] = copy;
1259			}
1260		const int backward_to_size = compiled_data->backward.to_size;
1261		for (i = 0; i < backward_to_size; i++)
1262			for (j = 1; j < parallel_count; j++)
1263			{
1264				const ccv_nnc_graph_exec_symbol_t copy = ccv_nnc_graph_exec_symbol_copy(model->graph, compiled_data->backward.tos[i], j);
1265				if (copy.d != CCV_NNC_NO_GRAPH_EXEC_SYMBOL)
1266					compiled_data->backward.tos[compiled_data->backward.to_size++] = copy;
1267			}
1268	}
1269	// Only use memory compression if we are in gradient parameter mode.
1270	if (gradient_mode == CCV_CNNP_COMPILED_DATA_GRADIENT_TRAINABLES || gradient_mode == CCV_CNNP_COMPILED_DATA_GRADIENT_TRAINABLES_AND_INPUTS)
1271	{
1272		if (model->memory_compression)
1273			ccv_nnc_symbolic_graph_memory_compression(model->graph, SYMBOLIC_GRAPH_SOURCES(model->graph)ccv_nnc_symbolic_graph_sources(model->graph), ccv_nnc_symbolic_graph_source_size
(model->graph), SYMBOLIC_GRAPH_DESTINATIONS(model->graph)ccv_nnc_symbolic_graph_destinations(model->graph), ccv_nnc_symbolic_graph_destination_size
(model->graph));
1274		if (model->memory_reduction)
1275			ccv_nnc_symbolic_graph_memory_reduction(model->graph, SYMBOLIC_GRAPH_SOURCES(model->graph)ccv_nnc_symbolic_graph_sources(model->graph), ccv_nnc_symbolic_graph_source_size
(model->graph), SYMBOLIC_GRAPH_DESTINATIONS(model->graph)ccv_nnc_symbolic_graph_destinations(model->graph), ccv_nnc_symbolic_graph_destination_size
(model->graph));
1276	}
1277	compiled_data->backward.to_size = _ccv_nnc_array_dedup_graph_exec_symbols(compiled_data->backward.tos, compiled_data->backward.to_size);
1278	compiled_data->gradient_mode = gradient_mode;
1279}
1280 
1281void ccv_cnnp_model_tensors_init_0(const ccv_cnnp_model_t* const model, ccv_cnnp_compiled_data_t* const compiled_data)
1282{
1283	assert(!compiled_data->tensors.parameters)((void) sizeof ((!compiled_data->tensors.parameters) ? 1 :
 0), __extension__ ({ if (!compiled_data->tensors.parameters
) ; else __assert_fail ("!compiled_data->tensors.parameters"
, "ccv_cnnp_model.c", 1283, __extension__ __PRETTY_FUNCTION__
); }));
1284	const int parameter_size = compiled_data->parameters->rnum;
1285	const int parallel_count = _ccv_cnnp_model_effective_parallel_count(model);
1286	compiled_data->parallel_count = parallel_count;
1287	const int internal_size = compiled_data->internals->rnum;
1288	compiled_data->tensors_init.size = ccv_nnc_tensor_symbol_count(model->graph);
1289	compiled_data->tensors_init.v = cccalloccalloc(((compiled_data->tensors_init.size + 31) >> 5), sizeof(uint32_t));
1290	compiled_data->tensors.parameters = (ccv_nnc_tensor_t**)cccalloccalloc((parameter_size + internal_size) * parallel_count, sizeof(ccv_nnc_tensor_t*));
1291	compiled_data->tensors.internals = compiled_data->tensors.parameters + parameter_size * parallel_count;
1292}
1293 
1294int ccv_cnnp_model_tensors_any_to_alloc(const ccv_cnnp_model_t* const model, ccv_cnnp_compiled_data_t* const compiled_data)
1295{
1296	int i, j;
1297	const int parameter_size = compiled_data->parameters->rnum;
1298	const int parallel_count = _ccv_cnnp_compiled_data_parallel_count(model, compiled_data);
1299	const int internal_size = compiled_data->internals->rnum;
1300	for (i = 0; i < parameter_size; i++)
1301	{
1302		// parameters has to be allocated all together.
1303		if (compiled_data->tensors.parameters[i])
1304		{
1305			for (j = 1; j < parallel_count; j++)
1306				{ assert(compiled_data->tensors.parameters[i + j * parameter_size])((void) sizeof ((compiled_data->tensors.parameters[i + j *
 parameter_size]) ? 1 : 0), __extension__ ({ if (compiled_data
->tensors.parameters[i + j * parameter_size]) ; else __assert_fail
 ("compiled_data->tensors.parameters[i + j * parameter_size]"
, "ccv_cnnp_model.c", 1306, __extension__ __PRETTY_FUNCTION__
); })); }
1307			continue;
1308		}
1309		return 1;
1310	}
1311	for (i = 0; i < internal_size; i++)
1312	{
1313		if (!compiled_data->tensors.internals[i])
1314			return 1;
1315		for (j = 1; j < parallel_count; j++)
1316			if (!compiled_data->tensors.internals[i + j * internal_size])
1317				return 1;
1318	}
1319	return 0;
1320}
1321 
1322void ccv_cnnp_model_tensors_init_1(const ccv_cnnp_model_t* const model, ccv_cnnp_compiled_data_t* const compiled_data)
1323{
1324	int i, j;
1325	const int parameter_size = compiled_data->parameters->rnum;
1326	const int parallel_count = _ccv_cnnp_compiled_data_parallel_count(model, compiled_data);
1327	compiled_data->parallel_count = parallel_count;
1328	const int internal_size = compiled_data->internals->rnum;
1329	for (i = 0; i < parameter_size; i++)
1330	{
1331		// parameters has to be allocated all together.
1332		if (compiled_data->tensors.parameters[i])
1333		{
1334			for (j = 1; j < parallel_count; j++)
1335				{ assert(compiled_data->tensors.parameters[i + j * parameter_size])((void) sizeof ((compiled_data->tensors.parameters[i + j *
 parameter_size]) ? 1 : 0), __extension__ ({ if (compiled_data
->tensors.parameters[i + j * parameter_size]) ; else __assert_fail
 ("compiled_data->tensors.parameters[i + j * parameter_size]"
, "ccv_cnnp_model.c", 1335, __extension__ __PRETTY_FUNCTION__
); })); }
1336			continue;
1337		}
1338		const ccv_nnc_tensor_symbol_t parameter = *(ccv_nnc_tensor_symbol_t*)ccv_array_get(compiled_data->parameters, i)((void*)(((char*)((compiled_data->parameters)->data)) +
 (size_t)(compiled_data->parameters)->rsize * (size_t)(
i)));
1339		ccv_nnc_tensor_param_t info = ccv_nnc_tensor_symbol_params(parameter.graph, parameter);
1340		if (CCV_TENSOR_GET_DEVICE(info.type)((info.type) & 0xfff00) == CCV_COMPUTE_DEVICE_ANY)
1341			CCV_TENSOR_SET_DEVICE_ID(info.type, 0)(info.type) = (((info.type) & ~0xfff00) | (((0) & 0xfff
) << 8));
1342		const int device_id = CCV_TENSOR_GET_DEVICE_ID(info.type)(((info.type) & 0xfff00) >> 8);
1343		compiled_data->tensors.parameters[i] = ccv_nnc_tensor_new(0, info, 0);
1344		for (j = 1; j < parallel_count; j++)
1345		{
1346			if (j != device_id)
1347				CCV_TENSOR_SET_DEVICE_ID(info.type, j)(info.type) = (((info.type) & ~0xfff00) | (((j) & 0xfff
) << 8));
1348			else
1349				CCV_TENSOR_SET_DEVICE_ID(info.type, 0)(info.type) = (((info.type) & ~0xfff00) | (((0) & 0xfff
) << 8));
1350			compiled_data->tensors.parameters[i + j * parameter_size] = ccv_nnc_tensor_new(0, info, 0);
1351		}
1352	}
1353	const uint32_t* const init_v = CCV_NNC_INIT_V(compiled_data->tensors_init.v)((uint32_t*)((uintptr_t)(compiled_data->tensors_init.v) &
 ~(uintptr_t)1));
1354	for (i = 0; i < internal_size; i++)
1355	{
1356		const ccv_nnc_tensor_symbol_t retained = *(ccv_nnc_tensor_symbol_t*)ccv_array_get(compiled_data->internals, i)((void*)(((char*)((compiled_data->internals)->data)) + (
size_t)(compiled_data->internals)->rsize * (size_t)(i))
);
1357		const int d = retained.d;
1358		if (init_v[d >> 5] & (1u << (d & 0x1f)))
1359			continue;
1360		ccv_nnc_tensor_param_t info = ccv_nnc_tensor_symbol_params(retained.graph, retained);
1361		if (CCV_TENSOR_GET_DEVICE(info.type)((info.type) & 0xfff00) == CCV_COMPUTE_DEVICE_ANY)
1362			CCV_TENSOR_SET_DEVICE_ID(info.type, 0)(info.type) = (((info.type) & ~0xfff00) | (((0) & 0xfff
) << 8));
1363		const int device_id = CCV_TENSOR_GET_DEVICE_ID(info.type)(((info.type) & 0xfff00) >> 8);
1364		if (!compiled_data->tensors.internals[i])
1365			compiled_data->tensors.internals[i] = ccv_nnc_tensor_new(0, info, 0);
1366		for (j = 1; j < parallel_count; j++)
1367		{
1368			if (j != device_id)
1369				CCV_TENSOR_SET_DEVICE_ID(info.type, j)(info.type) = (((info.type) & ~0xfff00) | (((j) & 0xfff
) << 8));
1370			else
1371				CCV_TENSOR_SET_DEVICE_ID(info.type, 0)(info.type) = (((info.type) & ~0xfff00) | (((0) & 0xfff
) << 8));
1372			if (!compiled_data->tensors.internals[i + j * internal_size])
1373				compiled_data->tensors.internals[i + j * internal_size] = ccv_nnc_tensor_new(0, info, 0);
1374		}
1375	}
1376	compiled_data->tensors_init.v = CCV_NNC_INIT_V(compiled_data->tensors_init.v)((uint32_t*)((uintptr_t)(compiled_data->tensors_init.v) &
 ~(uintptr_t)1)); // Remove 1 if any.
1377}
1378 
1379static void _ccv_cnnp_model_tensors_init(const ccv_cnnp_model_t* const model, ccv_cnnp_compiled_data_t* const compiled_data)
1380{
1381	ccv_cnnp_model_tensors_init_0(model, compiled_data);
1382	ccv_cnnp_model_tensors_init_1(model, compiled_data);
1383}
1384 
1385static void _ccv_cnnp_model_copy_tensors(const uint32_t* const tensors_init, const ccv_nnc_tensor_symbol_t* const tensor_symbols, ccv_nnc_tensor_t* const* const tensors, const int tensor_size, const int parallel_count)
1386{
1387	assert(parallel_count > 0)((void) sizeof ((parallel_count > 0) ? 1 : 0), __extension__
 ({ if (parallel_count > 0) ; else __assert_fail ("parallel_count > 0"
, "ccv_cnnp_model.c", 1387, __extension__ __PRETTY_FUNCTION__
); }));
1388	int i, j;
1389	for (i = 0; i < tensor_size; i++)
1390	{
1391		if (!tensors[i])
1392			continue;
1393		const int d = tensor_symbols[i].d;
1394		if (!(tensors_init[d >> 5] & (1u << (d & 0x1f))))
1395			continue;
1396		for (j = 1; j < parallel_count; j++)
1397			if (tensors[i + j * tensor_size])
1398			{
1399				ccv_nnc_tensor_t* const input = CCV_NNC_TENSOR(tensors[i])((ccv_nnc_tensor_t*)((uintptr_t)(tensors[i]) & ~(uintptr_t
)1));
1400				ccv_nnc_tensor_t* const output = CCV_NNC_TENSOR(tensors[i + j * tensor_size])((ccv_nnc_tensor_t*)((uintptr_t)(tensors[i + j * tensor_size]
) & ~(uintptr_t)1));
1401				ccv_nnc_cmd_exec(CMD_DATA_TRANSFER_FORWARD()ccv_nnc_cmd(CCV_NNC_DATA_TRANSFER_FORWARD, 0, ccv_nnc_cmd_auto
, 0), ccv_nnc_no_hint, 0, &input, 1, &output, 1, 0);
1402			}
1403	}
1404}
1405 
1406static void _ccv_cnnp_model_remove_nocopies(const ccv_nnc_symbolic_graph_t* const graph, const ccv_nnc_tensor_symbol_t* const tensor_symbols, ccv_nnc_tensor_t** const tensors, const int tensor_size, const int parallel_count)
1407{
1408	assert(parallel_count > 0)((void) sizeof ((parallel_count > 0) ? 1 : 0), __extension__
 ({ if (parallel_count > 0) ; else __assert_fail ("parallel_count > 0"
, "ccv_cnnp_model.c", 1408, __extension__ __PRETTY_FUNCTION__
); }));
1409	int i, j;
1410	for (i = 0; i < tensor_size; i++)
1411	{
1412		const ccv_nnc_tensor_symbol_t tensor_symbol = tensor_symbols[i];
1413		for (j = 1; j < parallel_count; j++)
1414		{
1415			const ccv_nnc_tensor_symbol_t copy = ccv_nnc_tensor_symbol_copy(graph, tensor_symbol, j);
1416			ccv_nnc_tensor_t* copy_tensor = tensors[i + j * tensor_size];
1417			if (copy_tensor && copy.d == CCV_NNC_NO_TENSOR_SYMBOL)
1418			{ // We shouldn't allocate this, free it up.
1419				ccv_nnc_tensor_free(tensors[i + j * tensor_size]);
1420				tensors[i + j * tensor_size] = 0;
1421			}
1422		}
1423	}
1424}
1425 
1426static void _ccv_cnnp_model_bind_tensors(const ccv_nnc_symbolic_graph_t* const graph, const ccv_nnc_tensor_symbol_t* const tensor_symbols, ccv_nnc_tensor_t* const* const tensors, const int tensor_size, const int parallel_count, ccv_array_t* const tensor_binds)
1427{
1428	assert(parallel_count > 0)((void) sizeof ((parallel_count > 0) ? 1 : 0), __extension__
 ({ if (parallel_count > 0) ; else __assert_fail ("parallel_count > 0"
, "ccv_cnnp_model.c", 1428, __extension__ __PRETTY_FUNCTION__
); }));
1429	int i, j;
1430	for (i = 0; i < tensor_size; i++)
1431	{
1432		ccv_nnc_tensor_symbol_t tensor_symbol = tensor_symbols[i];
1433		if (tensor_symbol.d == CCV_NNC_NO_TENSOR_SYMBOL)
1434			continue;
1435		if (graph)
1436		{
1437			const ccv_nnc_tensor_symbol_t alias_to = ccv_nnc_tensor_symbol_alias_to(graph, tensor_symbol);
1438			if (alias_to.d != CCV_NNC_NO_TENSOR_SYMBOL)
1439				tensor_symbol = alias_to;
1440		}
1441		ccv_nnc_tensor_t* const tensor = CCV_NNC_TENSOR(tensors[i])((ccv_nnc_tensor_t*)((uintptr_t)(tensors[i]) & ~(uintptr_t
)1));
1442		if (tensor && tensor_symbol.d != CCV_NNC_NO_TENSOR_SYMBOL)
1443		{
1444			const ccv_nnc_tensor_bind_t retained_bind = {
1445				.symbol = tensor_symbol,
1446				.tensor = tensor
1447			};
1448			ccv_array_push(tensor_binds, &retained_bind);
1449		}
1450		for (j = 1; j < parallel_count; j++)
1451		{
1452			const ccv_nnc_tensor_symbol_t copy = ccv_nnc_tensor_symbol_copy(graph, tensor_symbol, j);
1453			ccv_nnc_tensor_t* copy_tensor = tensors[i + j * tensor_size];
1454			if (copy_tensor && copy.d != CCV_NNC_NO_TENSOR_SYMBOL)
1455			{
1456				const ccv_nnc_tensor_bind_t bind = {
1457					.symbol = copy,
1458					.tensor = tensors[i + j * tensor_size]
1459				};
1460				ccv_array_push(tensor_binds, &bind);
1461			}
1462		}
1463	}
1464}
1465 
1466static void _ccv_cnnp_compiled_data_graph_free(ccv_cnnp_compiled_data_t* const compiled_data)
1467{
1468	if (compiled_data->graph)
1469		ccv_nnc_graph_free(compiled_data->graph);
1470	compiled_data->graph = 0;
1471	compiled_data->is_test = 0;
1472	if (compiled_data->tensor_arena)
1473		ccv_nnc_tensor_arena_free(compiled_data->tensor_arena);
1474	compiled_data->tensor_arena = 0;
1475	if (compiled_data->graph_exec_arena)
1476		ccv_nnc_graph_exec_arena_free(compiled_data->graph_exec_arena);
1477	compiled_data->graph_exec_arena = 0;
1478	if (compiled_data->backward.from_ops)
1479		ccfreefree(compiled_data->backward.from_ops);
1480	compiled_data->backward.from_ops = 0;
1481	if (compiled_data->evaluate.schedule)
1482		ccv_nnc_graph_static_schedule_free(compiled_data->evaluate.schedule);
1483	compiled_data->evaluate.schedule = 0;
1484	if (compiled_data->backward.schedule)
1485		ccv_nnc_graph_static_schedule_free(compiled_data->backward.schedule);
1486	compiled_data->backward.schedule = 0;
1487}
1488 
1489static void _ccv_cnnp_compiled_data_gradient_free(ccv_cnnp_compiled_data_t* const compiled_data)
1490{
1491	if (compiled_data->gradients)
1492		ccfreefree(compiled_data->gradients);
1493	compiled_data->gradients = 0;
1494	if (compiled_data->updated_parameters)
1495		ccfreefree(compiled_data->updated_parameters);
1496	compiled_data->updated_parameters = 0;
1497	compiled_data->update_nodes = 0;
1498	compiled_data->saved_aux = 0;
1499}
1500 
1501static void _ccv_cnnp_compiled_data_backward_free(ccv_cnnp_compiled_data_t* const compiled_data)
1502{
1503	if (compiled_data->backward.gradients)
1504		ccfreefree(compiled_data->backward.gradients);
1505	compiled_data->backward.gradients = 0;
1506	if (compiled_data->backward.accum)
1507		ccv_nnc_graph_free(compiled_data->backward.accum);
1508	compiled_data->backward.accum = 0;
1509	if (compiled_data->backward.tensor_arena)
1510		ccv_nnc_tensor_arena_free(compiled_data->backward.tensor_arena);
1511	compiled_data->backward.tensor_arena = 0;
1512	if (compiled_data->backward.graph_exec_arena)
1513		ccv_nnc_graph_exec_arena_free(compiled_data->backward.graph_exec_arena);
1514	compiled_data->backward.graph_exec_arena = 0;
1515}
1516 
1517static void _ccv_cnnp_compiled_data_apply_gradients_free(ccv_cnnp_compiled_data_t* const compiled_data)
1518{
1519	if (compiled_data->apply_gradients.graph)
1520		ccv_nnc_graph_free(compiled_data->apply_gradients.graph);
1521	compiled_data->apply_gradients.graph = 0;
1522	if (compiled_data->apply_gradients.tensor_arena)
1523		ccv_nnc_tensor_arena_free(compiled_data->apply_gradients.tensor_arena);
1524	compiled_data->apply_gradients.tensor_arena = 0;
1525	if (compiled_data->apply_gradients.graph_exec_arena)
1526		ccv_nnc_graph_exec_arena_free(compiled_data->apply_gradients.graph_exec_arena);
1527	compiled_data->apply_gradients.graph_exec_arena = 0;
1528}
1529 
1530// Compile the graph to run ccv_cnnp_model_fit
1531static void _ccv_cnnp_model_fit_jit(ccv_cnnp_model_t* const model, ccv_nnc_tensor_t* const* const inputs, const int input_size, ccv_nnc_tensor_t* const* const fits, const int fit_size, ccv_nnc_tensor_t* const* const outputs, const int output_size)
1532{
1533	int i, j;
1534	ccv_cnnp_compiled_data_t* const compiled_data = model->compiled_data;
1535	assert(!compiled_data->graph || compiled_data->graph_mode != CCV_CNNP_MODEL_GRAPH_FIT_MODE)((void) sizeof ((!compiled_data->graph || compiled_data->
graph_mode != CCV_CNNP_MODEL_GRAPH_FIT_MODE) ? 1 : 0), __extension__
 ({ if (!compiled_data->graph || compiled_data->graph_mode
 != CCV_CNNP_MODEL_GRAPH_FIT_MODE) ; else __assert_fail ("!compiled_data->graph || compiled_data->graph_mode != CCV_CNNP_MODEL_GRAPH_FIT_MODE"
, "ccv_cnnp_model.c", 1535, __extension__ __PRETTY_FUNCTION__
); }));
1536	compiled_data->graph_mode = CCV_CNNP_MODEL_GRAPH_FIT_MODE;
1537	const int parallel_count = _ccv_cnnp_model_root_parallel_count(model);
1538	assert(output_size == model->output_size * parallel_count)((void) sizeof ((output_size == model->output_size * parallel_count
) ? 1 : 0), __extension__ ({ if (output_size == model->output_size
 * parallel_count) ; else __assert_fail ("output_size == model->output_size * parallel_count"
, "ccv_cnnp_model.c", 1538, __extension__ __PRETTY_FUNCTION__
); }));
1539	assert(!fits || output_size == fit_size)((void) sizeof ((!fits || output_size == fit_size) ? 1 : 0), __extension__
 ({ if (!fits || output_size == fit_size) ; else __assert_fail
 ("!fits || output_size == fit_size", "ccv_cnnp_model.c", 1539
, __extension__ __PRETTY_FUNCTION__); }));
1540	assert(output_size > 0)((void) sizeof ((output_size > 0) ? 1 : 0), __extension__ (
{ if (output_size > 0) ; else __assert_fail ("output_size > 0"
, "ccv_cnnp_model.c", 1540, __extension__ __PRETTY_FUNCTION__
); }));
1541	if (compiled_data->gradient_mode == CCV_CNNP_COMPILED_DATA_GRADIENT_NONE)
1542	{
1543		_ccv_cnnp_model_set_rewindables(model);
1544		_ccv_cnnp_model_gradient_init(model, CCV_CNNP_COMPILED_DATA_GRADIENT_TRAINABLES, CCV_CNNP_DISABLE_OUTGRAD_ALL, fits, fit_size);
1545	} else if (compiled_data->gradient_mode != CCV_CNNP_COMPILED_DATA_GRADIENT_TRAINABLES) {
1546		_ccv_cnnp_model_rewind_graph(model);
1547		_ccv_cnnp_compiled_data_gradient_free(compiled_data);
1548		compiled_data->gradient_mode = CCV_CNNP_COMPILED_DATA_GRADIENT_NONE;
1549		_ccv_cnnp_model_gradient_init(model, CCV_CNNP_COMPILED_DATA_GRADIENT_TRAINABLES, CCV_CNNP_DISABLE_OUTGRAD_ALL, fits, fit_size);
1550	}
1551	const int tensors_init = !!compiled_data->tensors_init.v;
1552	if (!tensors_init)
1553		_ccv_cnnp_model_tensors_init(model, compiled_data);
1554	else if ((uintptr_t)compiled_data->tensors_init.v & (uintptr_t)1)
1555	// Check if it is not fully allocated, if it is not, init_1.
1556		ccv_cnnp_model_tensors_init_1(model, compiled_data);
1557	ccv_array_t* const tensor_binds = ccv_array_new(sizeof(ccv_nnc_tensor_bind_t), 0, 0);
1558	assert((input_size % parallel_count) == 0)((void) sizeof (((input_size % parallel_count) == 0) ? 1 : 0)
, __extension__ ({ if ((input_size % parallel_count) == 0) ; else
 __assert_fail ("(input_size % parallel_count) == 0", "ccv_cnnp_model.c"
, 1558, __extension__ __PRETTY_FUNCTION__); }));
1559	assert((output_size % parallel_count) == 0)((void) sizeof (((output_size % parallel_count) == 0) ? 1 : 0
), __extension__ ({ if ((output_size % parallel_count) == 0) ;
 else __assert_fail ("(output_size % parallel_count) == 0", "ccv_cnnp_model.c"
, 1559, __extension__ __PRETTY_FUNCTION__); }));
1560	assert((fit_size % parallel_count) == 0)((void) sizeof (((fit_size % parallel_count) == 0) ? 1 : 0), __extension__
 ({ if ((fit_size % parallel_count) == 0) ; else __assert_fail
 ("(fit_size % parallel_count) == 0", "ccv_cnnp_model.c", 1560
, __extension__ __PRETTY_FUNCTION__); }));
1561	const int input_size_per_p = input_size / parallel_count;
1562	_ccv_cnnp_model_bind_tensors(model->graph, model->inputs, inputs, input_size_per_p, parallel_count, tensor_binds);
1563	const int output_size_per_p = output_size / parallel_count;
1564	_ccv_cnnp_model_bind_tensors(model->graph, model->outputs, outputs, output_size_per_p, parallel_count, tensor_binds);
1565	const int fit_size_per_p = fit_size / parallel_count;
1566	_ccv_cnnp_model_bind_tensors(model->graph, compiled_data->fits, fits, fit_size_per_p, parallel_count, tensor_binds);
1567	const int parameter_size = compiled_data->parameters->rnum;
1568	_ccv_cnnp_model_bind_tensors(model->graph, (ccv_nnc_tensor_symbol_t*)ccv_array_get(compiled_data->parameters, 0)((void*)(((char*)((compiled_data->parameters)->data)) +
 (size_t)(compiled_data->parameters)->rsize * (size_t)(
0))), compiled_data->tensors.parameters, parameter_size, parallel_count, tensor_binds);
1569	_ccv_cnnp_model_bind_tensors(model->graph, compiled_data->updated_parameters, compiled_data->tensors.parameters, parameter_size, parallel_count, tensor_binds);
1570	const int internal_size = compiled_data->internals->rnum;
1571	_ccv_cnnp_model_remove_nocopies(model->graph, (ccv_nnc_tensor_symbol_t*)ccv_array_get(compiled_data->internals, 0)((void*)(((char*)((compiled_data->internals)->data)) + (
size_t)(compiled_data->internals)->rsize * (size_t)(0))
), compiled_data->tensors.internals, internal_size, parallel_count);
1572	_ccv_cnnp_model_bind_tensors(model->graph, (ccv_nnc_tensor_symbol_t*)ccv_array_get(compiled_data->internals, 0)((void*)(((char*)((compiled_data->internals)->data)) + (
size_t)(compiled_data->internals)->rsize * (size_t)(0))
), compiled_data->tensors.internals, internal_size, parallel_count, tensor_binds);
1573	ccv_nnc_symbolic_graph_compile(model->graph, compiled_data->compile_params, (ccv_nnc_tensor_bind_t*)ccv_array_get(tensor_binds, 0)((void*)(((char*)((tensor_binds)->data)) + (size_t)(tensor_binds
)->rsize * (size_t)(0))), tensor_binds->rnum, 0, 0, SYMBOLIC_GRAPH_SOURCES(model->graph)ccv_nnc_symbolic_graph_sources(model->graph), ccv_nnc_symbolic_graph_source_size
(model->graph), SYMBOLIC_GRAPH_DESTINATIONS(model->graph)ccv_nnc_symbolic_graph_destinations(model->graph), ccv_nnc_symbolic_graph_destination_size
(model->graph), &compiled_data->graph, &compiled_data->tensor_arena, &compiled_data->graph_exec_arena);
1574	ccv_array_free(tensor_binds);
1575	const uint32_t* const init_v = CCV_NNC_INIT_V(compiled_data->tensors_init.v)((uint32_t*)((uintptr_t)(compiled_data->tensors_init.v) &
 ~(uintptr_t)1));
1576	if (tensors_init && parallel_count > 1)
1577		_ccv_cnnp_model_copy_tensors(init_v, (ccv_nnc_tensor_symbol_t*)ccv_array_get(compiled_data->parameters, 0)((void*)(((char*)((compiled_data->parameters)->data)) +
 (size_t)(compiled_data->parameters)->rsize * (size_t)(
0))), compiled_data->tensors.parameters, compiled_data->parameters->rnum, parallel_count);
1578	// If tensor is not init'ed, we need to init states first.
1579	if (_ccv_cnnp_any_to_init(compiled_data))
1580	{
1581		ccv_nnc_tensor_init_states_t tensor_init_states = {
1582			.parallel_count = parallel_count,
1583			.graph = model->graph,
1584			.compiled_data = compiled_data,
1585			.tensor_arena = compiled_data->tensor_arena
1586		};
1587		ccv_cnnp_model_init_states(model, model->graph, _ccv_cnnp_init_states_for_tensors, &tensor_init_states);
1588	}
1589	compiled_data->is_test = 0;
1590	const int saved_aux_size = ccv_nnc_minimizer_saved_aux_size(compiled_data->minimize.minimizer);
1591	// No need to set because it is default to training mode.
1592	// ccv_cnnp_model_set_is_test(model, 0, _ccv_cnnp_cmd_update_for_execs, &update);
1593	for (i = 0; i < saved_aux_size * parameter_size; i++)
1594	{
1595		if (compiled_data->saved_aux[i].source.d == CCV_NNC_NO_TENSOR_SYMBOL)
1596			continue;
1597		ccv_nnc_tensor_t* const tensor = ccv_nnc_tensor_from_symbol(compiled_data->tensor_arena, compiled_data->saved_aux[i].source);
1598		ccv_nnc_cmd_exec(CMD_SET_FORWARD(0)ccv_nnc_cmd(CCV_NNC_SET_FORWARD, 0, (ccv_nnc_cmd_param_t){.size
={.dim={1,1,1}},.blas={.a={0,}}}, 0), ccv_nnc_no_hint, 0, 0, 0, &tensor, 1, 0);
1599		for (j = 1; j < parallel_count; j++)
1600		{
1601			ccv_nnc_tensor_t* const copy = ccv_nnc_tensor_from_symbol(compiled_data->tensor_arena, ccv_nnc_tensor_symbol_copy(model->graph, compiled_data->saved_aux[i].source, j));
1602			if (copy)
1603				ccv_nnc_cmd_exec(CMD_SET_FORWARD(0)ccv_nnc_cmd(CCV_NNC_SET_FORWARD, 0, (ccv_nnc_cmd_param_t){.size
={.dim={1,1,1}},.blas={.a={0,}}}, 0), ccv_nnc_no_hint, 0, 0, 0, &copy, 1, 0);
1604		}
1605	}
1606	const int evaluate_to_size = compiled_data->evaluate.to_size;
1607	compiled_data->evaluate.to_op_size = 0;
1608	for (i = 0; i < evaluate_to_size; i++)
1609	{
1610		ccv_nnc_graph_exec_t const to = ccv_nnc_graph_exec_from_symbol(compiled_data->graph_exec_arena, compiled_data->evaluate.tos[i]);
1611		if (to.graph)
1612			compiled_data->evaluate.to_ops[compiled_data->evaluate.to_op_size++] = to;
1613	}
1614	ccv_nnc_graph_set_default_static_schedule(compiled_data->graph, compiled_data->stream_type, model->max_stream_count);
1615	ccv_nnc_graph_autotune(compiled_data->graph, model->workspace_size, 0, TRAVERSE_FULL0,0,0,0);
1616}
1617 
1618ccv_nnc_stream_context_t* ccv_cnnp_model_default_stream(const ccv_cnnp_model_t* const model)
1619{
1620	const ccv_cnnp_compiled_data_t* const compiled_data = model->compiled_data;
1621	if (!compiled_data || !compiled_data->graph)
1622		return 0;
1623	return ccv_nnc_graph_default_stream(compiled_data->graph);
1624}
1625 
1626uint64_t ccv_cnnp_model_memory_size(const ccv_cnnp_model_t* const model)
1627{
1628	const ccv_cnnp_compiled_data_t* const compiled_data = model->compiled_data;
1629	if (!compiled_data || !compiled_data->tensor_arena)
1630		return 0;
1631	return ccv_nnc_tensor_arena_size(compiled_data->tensor_arena);
1632}
1633 
1634int ccv_cnnp_model_pin_memory(ccv_cnnp_model_t* const model, const int pin_memory)
1635{
1636#ifdef HAVE_MPS
1637	ccv_cnnp_compiled_data_t* const compiled_data = model->compiled_data;
1638	if (!compiled_data)
1639		return 0;
1640	int i, status = 0;
1641	if (!pin_memory)
1642	{
1643		if (!compiled_data->pinned_refs)
1644			return 0;
1645		for (i = 0; i < compiled_data->pinned_refs->rnum; i++)
1646			if (mpunpinmemory(*(void**)ccv_array_get(compiled_data->pinned_refs, i)((void*)(((char*)((compiled_data->pinned_refs)->data)) +
 (size_t)(compiled_data->pinned_refs)->rsize * (size_t)
(i)))) != 0)
1647				status = -1;
1648		ccv_array_free(compiled_data->pinned_refs);
1649		compiled_data->pinned_refs = 0;
1650		return status;
1651	}
1652	if (!compiled_data->tensors.parameters || compiled_data->pinned_refs)
1653		return 0;
1654	compiled_data->pinned_refs = ccv_array_new(sizeof(void*), 0, 0);
1655	const int parallel_count = compiled_data->parallel_count > 0 ? compiled_data->parallel_count : _ccv_cnnp_model_root_parallel_count(model);
1656	for (i = 0; i < compiled_data->parameters->rnum * parallel_count; i++)
1657	{
1658		ccv_nnc_tensor_t* const tensor = CCV_NNC_TENSOR(compiled_data->tensors.parameters[i])((ccv_nnc_tensor_t*)((uintptr_t)(compiled_data->tensors.parameters
[i]) & ~(uintptr_t)1));
1659		if (!tensor || CCV_TENSOR_GET_MEMORY(tensor->info.type)((tensor->info.type) & 0x3) != CCV_TENSOR_GPU_MEMORY || !tensor->data.u8)
1660			continue;
1661		int result;
1662		void* const mapping = mppinmemory(tensor->data.u8, &result);
1663		if (mapping)
1664			ccv_array_push(compiled_data->pinned_refs, &mapping);
1665		if (result != 0)
1666			status = -1; // Best effort: keep going after a failed mapping.
1667	}
1668	return status;
1669#else
1670	return 0;
1671#endif
1672}
1673 
1674static void _ccv_cnnp_bind_tensors_to_arena(ccv_nnc_tensor_arena_t* const tensor_arena, const ccv_nnc_symbolic_graph_t* const graph, const ccv_nnc_tensor_symbol_t* const tensor_symbols, ccv_nnc_tensor_t* const* const tensors, const int tensor_size, const int parallel_count)
1675{
1676	int i, j;
1677	for (i = 0; i < tensor_size; i++)
1678	{
1679		ccv_nnc_tensor_symbol_t tensor_symbol = tensor_symbols[i];
1680		if (tensor_symbol.d == CCV_NNC_NO_TENSOR_SYMBOL)
1681			continue;
1682		if (graph)
1683		{
1684			const ccv_nnc_tensor_symbol_t alias_to = ccv_nnc_tensor_symbol_alias_to(graph, tensor_symbol);
1685			if (alias_to.d != CCV_NNC_NO_TENSOR_SYMBOL)
1686				tensor_symbol = alias_to;
1687		}
1688		ccv_nnc_tensor_bind_symbol(tensor_arena, tensor_symbol, tensors[i]);
1689		for (j = 1; j < parallel_count; j++)
1690		{
1691			const ccv_nnc_tensor_symbol_t copy = ccv_nnc_tensor_symbol_copy(graph, tensor_symbol, j);
1692			if (copy.d != CCV_NNC_NO_TENSOR_SYMBOL)
1693				ccv_nnc_tensor_bind_symbol(tensor_arena, copy, tensors[i + tensor_size * j]);
1694		}
1695	}
1696}
1697 
1698void ccv_cnnp_model_fit(ccv_cnnp_model_t* const model, ccv_nnc_tensor_t* const* const inputs, const int input_size, ccv_nnc_tensor_t* const* const fits, const int fit_size, ccv_nnc_tensor_t* const* const outputs, const int output_size, ccv_nnc_tensor_tape_t* const tensor_tape, ccv_nnc_stream_context_t* const stream_context)
1699{
1700	ccv_cnnp_compiled_data_t* const compiled_data = model->compiled_data;
1701	assert(compiled_data)((void) sizeof ((compiled_data) ? 1 : 0), __extension__ ({ if
 (compiled_data) ; else __assert_fail ("compiled_data", "ccv_cnnp_model.c"
, 1701, __extension__ __PRETTY_FUNCTION__); }));
1702	const int parallel_count = ccv_max(model->parallel_count, 1)({ typeof (model->parallel_count) _a = (model->parallel_count
); typeof (1) _b = (1); (_a > _b) ? _a : _b; });
1703	assert(output_size == model->output_size * parallel_count)((void) sizeof ((output_size == model->output_size * parallel_count
) ? 1 : 0), __extension__ ({ if (output_size == model->output_size
 * parallel_count) ; else __assert_fail ("output_size == model->output_size * parallel_count"
, "ccv_cnnp_model.c", 1703, __extension__ __PRETTY_FUNCTION__
); }));
1704	assert(input_size == model->input_size * parallel_count)((void) sizeof ((input_size == model->input_size * parallel_count
) ? 1 : 0), __extension__ ({ if (input_size == model->input_size
 * parallel_count) ; else __assert_fail ("input_size == model->input_size * parallel_count"
, "ccv_cnnp_model.c", 1704, __extension__ __PRETTY_FUNCTION__
); }));
1705	assert(!fits || fit_size == output_size)((void) sizeof ((!fits || fit_size == output_size) ? 1 : 0), __extension__
 ({ if (!fits || fit_size == output_size) ; else __assert_fail
 ("!fits || fit_size == output_size", "ccv_cnnp_model.c", 1705
, __extension__ __PRETTY_FUNCTION__); }));
1706	assert(model->graph)((void) sizeof ((model->graph) ? 1 : 0), __extension__ ({ if
 (model->graph) ; else __assert_fail ("model->graph", "ccv_cnnp_model.c"
, 1706, __extension__ __PRETTY_FUNCTION__); }));
1707	if (!compiled_data->graph || compiled_data->graph_mode != CCV_CNNP_MODEL_GRAPH_FIT_MODE)
1708	{
1709		_ccv_cnnp_compiled_data_graph_free(compiled_data);
1710		_ccv_cnnp_compiled_data_backward_free(compiled_data);
1711		_ccv_cnnp_compiled_data_apply_gradients_free(compiled_data);
1712		// Compile the symbolic graph down only when needed.
1713		_ccv_cnnp_model_fit_jit(model, inputs, input_size, fits, fit_size, outputs, output_size);
1714	} else {
1715		assert((input_size % parallel_count) == 0)((void) sizeof (((input_size % parallel_count) == 0) ? 1 : 0)
, __extension__ ({ if ((input_size % parallel_count) == 0) ; else
 __assert_fail ("(input_size % parallel_count) == 0", "ccv_cnnp_model.c"
, 1715, __extension__ __PRETTY_FUNCTION__); }));
1716		assert((output_size % parallel_count) == 0)((void) sizeof (((output_size % parallel_count) == 0) ? 1 : 0
), __extension__ ({ if ((output_size % parallel_count) == 0) ;
 else __assert_fail ("(output_size % parallel_count) == 0", "ccv_cnnp_model.c"
, 1716, __extension__ __PRETTY_FUNCTION__); }));
1717		assert((fit_size % parallel_count) == 0)((void) sizeof (((fit_size % parallel_count) == 0) ? 1 : 0), __extension__
 ({ if ((fit_size % parallel_count) == 0) ; else __assert_fail
 ("(fit_size % parallel_count) == 0", "ccv_cnnp_model.c", 1717
, __extension__ __PRETTY_FUNCTION__); }));
1718		const int input_size_per_p = input_size / parallel_count;
1719		_ccv_cnnp_bind_tensors_to_arena(compiled_data->tensor_arena, model->graph, model->inputs, inputs, input_size_per_p, parallel_count);
1720		const int output_size_per_p = output_size / parallel_count;
1721		_ccv_cnnp_bind_tensors_to_arena(compiled_data->tensor_arena, model->graph, model->outputs, outputs, output_size_per_p, parallel_count);
1722		const int fit_size_per_p = fit_size / parallel_count;
1723		_ccv_cnnp_bind_tensors_to_arena(compiled_data->tensor_arena, model->graph, compiled_data->fits, fits, fit_size_per_p, parallel_count);
1724	}
1725	if (compiled_data->is_test)
1726	{
1727		compiled_data->is_test = 0;
1728		ccv_nnc_graph_exec_update_t update = {
1729			.parallel_count = parallel_count,
1730			.graph = model->graph,
1731			.graph_exec_arena = compiled_data->graph_exec_arena,
1732		};
1733		ccv_cnnp_model_set_is_test(model, 0, _ccv_cnnp_cmd_update_for_execs, &update);
1734	}
1735	ccv_nnc_graph_run_with_schedule(compiled_data->graph, 0, 0, tensor_tape, stream_context);
1736}
1737 
1738// Compile the graph to run ccv_cnnp_model_evaluate with require_grad = false (MULTISTAGE_MODE_NO_GRAD).
1739static void _ccv_cnnp_model_multistage_no_grad_jit(ccv_cnnp_model_t* const model, ccv_nnc_tensor_t* const* const inputs, const int input_size, ccv_nnc_tensor_t* const* const outputs, const int output_size)
1740{
1741	ccv_cnnp_compiled_data_t* const compiled_data = model->compiled_data;
1742	compiled_data->graph_mode = CCV_CNNP_MODEL_GRAPH_MULTISTAGE_MODE_NO_GRAD;
1743	const int parallel_count = ccv_max(model->parallel_count, 1)({ typeof (model->parallel_count) _a = (model->parallel_count
); typeof (1) _b = (1); (_a > _b) ? _a : _b; });
1744	assert(output_size == model->output_size * parallel_count)((void) sizeof ((output_size == model->output_size * parallel_count
) ? 1 : 0), __extension__ ({ if (output_size == model->output_size
 * parallel_count) ; else __assert_fail ("output_size == model->output_size * parallel_count"
, "ccv_cnnp_model.c", 1744, __extension__ __PRETTY_FUNCTION__
); }));
1745	assert(output_size > 0)((void) sizeof ((output_size > 0) ? 1 : 0), __extension__ (
{ if (output_size > 0) ; else __assert_fail ("output_size > 0"
, "ccv_cnnp_model.c", 1745, __extension__ __PRETTY_FUNCTION__
); }));
1746	// If the gradient is not initialized, continue to setup parallel process. We don't init gradient here, but rather,
1747	// we setup proper rewindables so the graph can be rewinded to previous state before we run data parallel.
1748	if (parallel_count > 1 && compiled_data->gradient_mode == CCV_CNNP_COMPILED_DATA_GRADIENT_NONE)
1749	{
1750		const int evaluate_to_size = compiled_data->evaluate.to_size;
1751		compiled_data->evaluate.tos = ccreallocrealloc(compiled_data->evaluate.tos, sizeof(ccv_nnc_graph_exec_symbol_t) * evaluate_to_size * parallel_count + sizeof(ccv_nnc_graph_exec_t) * evaluate_to_size * parallel_count);
1752		_ccv_cnnp_model_set_rewindables(model);
1753		ccv_nnc_symbolic_graph_data_parallel(model->graph, parallel_count,
1754			0, 0,
1755			0, 0, 0,
1756			0, 0, 0,
1757			CCV_NNC_PARALLEL_REDUCE_OP_SUM,
1758			SYMBOLIC_GRAPH_SOURCES(model->graph)ccv_nnc_symbolic_graph_sources(model->graph), ccv_nnc_symbolic_graph_source_size
(model->graph), SYMBOLIC_GRAPH_DESTINATIONS(model->graph)ccv_nnc_symbolic_graph_destinations(model->graph), ccv_nnc_symbolic_graph_destination_size
(model->graph));
1759		ccv_nnc_graph_exec_symbol_autogen(model->graph, 0, 0, CCV_NNC_AUTOGEN_SOURCES_AND_DESTINATIONS);
1760		int i, j;
1761		for (i = 0; i < evaluate_to_size; i++)
1762			for (j = 1; j < parallel_count; j++)
1763			{
1764				const ccv_nnc_graph_exec_symbol_t copy = ccv_nnc_graph_exec_symbol_copy(model->graph, compiled_data->evaluate.tos[i], j);
1765				if (copy.d != CCV_NNC_NO_GRAPH_EXEC_SYMBOL)
1766					compiled_data->evaluate.tos[compiled_data->evaluate.to_size++] = copy;
1767			}
1768	}
1769	const int tensors_init = !!compiled_data->tensors_init.v;
1770	if (!tensors_init)
1771		_ccv_cnnp_model_tensors_init(model, compiled_data);
1772	else if ((uintptr_t)compiled_data->tensors_init.v & (uintptr_t)1)
1773	// Check if it is not fully allocated, if it is not, init_1.
1774		ccv_cnnp_model_tensors_init_1(model, compiled_data);
1775	const int tensor_parallel_count = _ccv_cnnp_compiled_data_parallel_count(model, compiled_data);
1776	ccv_array_t* const tensor_binds = ccv_array_new(sizeof(ccv_nnc_tensor_bind_t), 0, 0);
1777	assert((input_size % parallel_count) == 0)((void) sizeof (((input_size % parallel_count) == 0) ? 1 : 0)
, __extension__ ({ if ((input_size % parallel_count) == 0) ; else
 __assert_fail ("(input_size % parallel_count) == 0", "ccv_cnnp_model.c"
, 1777, __extension__ __PRETTY_FUNCTION__); }));
1778	assert((output_size % parallel_count) == 0)((void) sizeof (((output_size % parallel_count) == 0) ? 1 : 0
), __extension__ ({ if ((output_size % parallel_count) == 0) ;
 else __assert_fail ("(output_size % parallel_count) == 0", "ccv_cnnp_model.c"
, 1778, __extension__ __PRETTY_FUNCTION__); }));
1779	const int input_size_per_p = input_size / parallel_count;
1780	_ccv_cnnp_model_bind_tensors(model->graph, model->inputs, inputs, input_size_per_p, parallel_count, tensor_binds);
1781	const int output_size_per_p = output_size / parallel_count;
1782	_ccv_cnnp_model_bind_tensors(model->graph, model->outputs, outputs, output_size_per_p, parallel_count, tensor_binds);
1783	const int parameter_size = compiled_data->parameters->rnum;
1784	_ccv_cnnp_model_bind_tensors(model->graph, (ccv_nnc_tensor_symbol_t*)ccv_array_get(compiled_data->parameters, 0)((void*)(((char*)((compiled_data->parameters)->data)) +
 (size_t)(compiled_data->parameters)->rsize * (size_t)(
0))), compiled_data->tensors.parameters, parameter_size, tensor_parallel_count, tensor_binds);
1785	const int internal_size = compiled_data->internals->rnum;
1786	_ccv_cnnp_model_remove_nocopies(model->graph, (ccv_nnc_tensor_symbol_t*)ccv_array_get(compiled_data->internals, 0)((void*)(((char*)((compiled_data->internals)->data)) + (
size_t)(compiled_data->internals)->rsize * (size_t)(0))
), compiled_data->tensors.internals, internal_size, tensor_parallel_count);
1787	_ccv_cnnp_model_bind_tensors(model->graph, (ccv_nnc_tensor_symbol_t*)ccv_array_get(compiled_data->internals, 0)((void*)(((char*)((compiled_data->internals)->data)) + (
size_t)(compiled_data->internals)->rsize * (size_t)(0))
), compiled_data->tensors.internals, internal_size, tensor_parallel_count, tensor_binds);
1788	// If we generated gradient for the graph, only compile part of the graph because the rest is irrelevant for evaluation.
1789	ccv_nnc_symbolic_graph_compile(model->graph, compiled_data->compile_params, (ccv_nnc_tensor_bind_t*)ccv_array_get(tensor_binds, 0)((void*)(((char*)((tensor_binds)->data)) + (size_t)(tensor_binds
)->rsize * (size_t)(0))), tensor_binds->rnum, 0, 0, SYMBOLIC_GRAPH_SOURCES(model->graph)ccv_nnc_symbolic_graph_sources(model->graph), ccv_nnc_symbolic_graph_source_size
(model->graph), compiled_data->evaluate.tos, compiled_data->evaluate.to_size, &compiled_data->graph, &compiled_data->tensor_arena, &compiled_data->graph_exec_arena);
1790	ccv_array_free(tensor_binds);
1791	const uint32_t* const init_v = CCV_NNC_INIT_V(compiled_data->tensors_init.v)((uint32_t*)((uintptr_t)(compiled_data->tensors_init.v) &
 ~(uintptr_t)1));
1792	// If tensor is not init'ed, we need to init states first.
1793	if (tensors_init && tensor_parallel_count > 1)
1794		_ccv_cnnp_model_copy_tensors(init_v, (ccv_nnc_tensor_symbol_t*)ccv_array_get(compiled_data->parameters, 0)((void*)(((char*)((compiled_data->parameters)->data)) +
 (size_t)(compiled_data->parameters)->rsize * (size_t)(
0))), compiled_data->tensors.parameters, compiled_data->parameters->rnum, tensor_parallel_count);
1795	if (_ccv_cnnp_any_to_init(compiled_data))
1796	{
1797		ccv_nnc_tensor_init_states_t tensor_init_states = {
1798			.parallel_count = tensor_parallel_count,
1799			.graph = model->graph,
1800			.compiled_data = compiled_data,
1801			.tensor_arena = compiled_data->tensor_arena
1802		};
1803		ccv_cnnp_model_init_states(model, model->graph, _ccv_cnnp_init_states_for_tensors, &tensor_init_states);
1804	}
1805	compiled_data->is_test = 1;
1806	ccv_nnc_graph_exec_update_t update = {
1807		.parallel_count = parallel_count,
1808		.graph = model->graph,
1809		.graph_exec_arena = compiled_data->graph_exec_arena,
1810	};
1811	ccv_cnnp_model_set_is_test(model, 1, _ccv_cnnp_cmd_update_for_execs, &update);
1812	ccv_nnc_graph_set_default_static_schedule(compiled_data->graph, compiled_data->stream_type, model->max_stream_count);
1813	ccv_nnc_graph_autotune(compiled_data->graph, model->workspace_size, 0, TRAVERSE_FULL0,0,0,0);
1814}
1815 
1816static void _ccv_cnnp_model_gradient_tensors_init(const ccv_cnnp_model_t* const model, ccv_cnnp_compiled_data_t* const compiled_data)
1817{
1818	assert(!compiled_data->tensors.gradients)((void) sizeof ((!compiled_data->tensors.gradients) ? 1 : 0
), __extension__ ({ if (!compiled_data->tensors.gradients)
 ; else __assert_fail ("!compiled_data->tensors.gradients"
, "ccv_cnnp_model.c", 1818, __extension__ __PRETTY_FUNCTION__
); }));
1819	const int parameter_size = compiled_data->parameters->rnum;
1820	const int parallel_count = ccv_max(model->parallel_count, 1)({ typeof (model->parallel_count) _a = (model->parallel_count
); typeof (1) _b = (1); (_a > _b) ? _a : _b; });
1821	compiled_data->tensors.gradients = (ccv_nnc_tensor_t**)ccmallocmalloc(sizeof(ccv_nnc_tensor_t*) * parameter_size * 2 * parallel_count);
1822	compiled_data->tensors.accum_gradients = compiled_data->tensors.gradients + parameter_size * parallel_count;
1823	int i, j;
1824	for (i = 0; i < parameter_size; i++)
1825	{
1826		if (compiled_data->parameter_flags && !(compiled_data->parameter_flags[i >> 6] & ((uint64_t)1 << (i & 63))))
1827		{
1828			compiled_data->tensors.gradients[i] = 0;
1829			compiled_data->tensors.accum_gradients[i] = 0;
1830			for (j = 1; j < parallel_count; j++)
1831			{
1832				compiled_data->tensors.gradients[i + j * parameter_size] = 0;
1833				compiled_data->tensors.accum_gradients[i + j * parameter_size] = 0;
1834			}
1835			continue;
1836		}
1837		const ccv_nnc_tensor_symbol_t parameter = *(ccv_nnc_tensor_symbol_t*)ccv_array_get(compiled_data->parameters, i)((void*)(((char*)((compiled_data->parameters)->data)) +
 (size_t)(compiled_data->parameters)->rsize * (size_t)(
i)));
1838		ccv_nnc_tensor_param_t info = ccv_nnc_tensor_symbol_params(parameter.graph, parameter);
1839		if (CCV_TENSOR_GET_DEVICE(info.type)((info.type) & 0xfff00) == CCV_COMPUTE_DEVICE_ANY)
1840			CCV_TENSOR_SET_DEVICE_ID(info.type, 0)(info.type) = (((info.type) & ~0xfff00) | (((0) & 0xfff
) << 8));
1841		const int device_id = CCV_TENSOR_GET_DEVICE_ID(info.type)(((info.type) & 0xfff00) >> 8);
1842		compiled_data->tensors.gradients[i] = ccv_nnc_tensor_new(0, info, 0);
1843		compiled_data->tensors.accum_gradients[i] = 0; // delay the accumulated gradient allocation until when we need it.
1844		for (j = 1; j < parallel_count; j++)
1845		{
1846			if (j != device_id)
1847				CCV_TENSOR_SET_DEVICE_ID(info.type, j)(info.type) = (((info.type) & ~0xfff00) | (((j) & 0xfff
) << 8));
1848			else
1849				CCV_TENSOR_SET_DEVICE_ID(info.type, 0)(info.type) = (((info.type) & ~0xfff00) | (((0) & 0xfff
) << 8));
1850			compiled_data->tensors.gradients[i + j * parameter_size] = ccv_nnc_tensor_new(0, info, 0);
1851			compiled_data->tensors.accum_gradients[i + j * parameter_size] = 0;
1852		}
1853	}
1854}
1855 
1856static int _ccv_cnnp_is_disable_outgrad_all(const uint64_t disable_outgrad, const int input_size)
1857{
1858	if (disable_outgrad == CCV_CNNP_DISABLE_OUTGRAD_ALL)
1859		return 1;
1860	if (disable_outgrad == CCV_CNNP_DISABLE_OUTGRAD_NONE)
1861		return 0;
1862	int i;
1863	for (i = 0; i < input_size; i++)
1864		if (!(disable_outgrad & ((uint64_t)1 << i)))
1865			return 0;
1866	return 1;
1867}
1868 
1869// Compile the graph to run ccv_cnnp_model_evaluate with requires_grad = true (MULTISTAGE_MODE).
1870// Particularly, this method compiles the evaluation and backprop graph (the main graph).
1871static void _ccv_cnnp_model_multistage_jit_0(ccv_cnnp_model_t* const model, const uint64_t disable_outgrad, const int is_test, ccv_nnc_tensor_t* const* const inputs, const int input_size, ccv_nnc_tensor_t* const* const outputs, const int output_size)
1872{
1873	int i, j;
1874	ccv_cnnp_compiled_data_t* const compiled_data = model->compiled_data;
1875	const int target_gradient_mode = _ccv_cnnp_is_disable_outgrad_all(disable_outgrad, model->input_size) ? CCV_CNNP_COMPILED_DATA_GRADIENT_TRAINABLES : CCV_CNNP_COMPILED_DATA_GRADIENT_TRAINABLES_AND_INPUTS;
1876	assert(!compiled_data->graph || compiled_data->graph_mode != CCV_CNNP_MODEL_GRAPH_MULTISTAGE_MODE || compiled_data->gradient_mode != target_gradient_mode)((void) sizeof ((!compiled_data->graph || compiled_data->
graph_mode != CCV_CNNP_MODEL_GRAPH_MULTISTAGE_MODE || compiled_data
->gradient_mode != target_gradient_mode) ? 1 : 0), __extension__
 ({ if (!compiled_data->graph || compiled_data->graph_mode
 != CCV_CNNP_MODEL_GRAPH_MULTISTAGE_MODE || compiled_data->
gradient_mode != target_gradient_mode) ; else __assert_fail (
"!compiled_data->graph || compiled_data->graph_mode != CCV_CNNP_MODEL_GRAPH_MULTISTAGE_MODE || compiled_data->gradient_mode != target_gradient_mode"
, "ccv_cnnp_model.c", 1876, __extension__ __PRETTY_FUNCTION__
); }));
1877	compiled_data->graph_mode = CCV_CNNP_MODEL_GRAPH_MULTISTAGE_MODE;
1878	const int parallel_count = ccv_max(model->parallel_count, 1)({ typeof (model->parallel_count) _a = (model->parallel_count
); typeof (1) _b = (1); (_a > _b) ? _a : _b; });
1879	assert(output_size == model->output_size * parallel_count)((void) sizeof ((output_size == model->output_size * parallel_count
) ? 1 : 0), __extension__ ({ if (output_size == model->output_size
 * parallel_count) ; else __assert_fail ("output_size == model->output_size * parallel_count"
, "ccv_cnnp_model.c", 1879, __extension__ __PRETTY_FUNCTION__
); }));
1880	assert(output_size > 0)((void) sizeof ((output_size > 0) ? 1 : 0), __extension__ (
{ if (output_size > 0) ; else __assert_fail ("output_size > 0"
, "ccv_cnnp_model.c", 1880, __extension__ __PRETTY_FUNCTION__
); }));
1881	// There shouldn't be a loss function if we evaluate with multistage jit.
1882	assert(compiled_data->loss.cmd == CCV_NNC_NOOP)((void) sizeof ((compiled_data->loss.cmd == CCV_NNC_NOOP) ?
 1 : 0), __extension__ ({ if (compiled_data->loss.cmd == CCV_NNC_NOOP
) ; else __assert_fail ("compiled_data->loss.cmd == CCV_NNC_NOOP"
, "ccv_cnnp_model.c", 1882, __extension__ __PRETTY_FUNCTION__
); }));
1883	if (compiled_data->gradient_mode == CCV_CNNP_COMPILED_DATA_GRADIENT_NONE)
1884	{
1885		_ccv_cnnp_model_set_rewindables(model);
1886		_ccv_cnnp_model_gradient_init(model, target_gradient_mode, disable_outgrad, 0, 0); // The type of outputs and fits should be the same. We only use type here.
1887	} else if (compiled_data->gradient_mode != target_gradient_mode) {
1888		_ccv_cnnp_model_rewind_graph(model);
1889		_ccv_cnnp_compiled_data_gradient_free(compiled_data);
1890		compiled_data->gradient_mode = CCV_CNNP_COMPILED_DATA_GRADIENT_NONE;
1891		_ccv_cnnp_model_gradient_init(model, target_gradient_mode, disable_outgrad, 0, 0); // The type of outputs and fits should be the same. We only use type here.
1892	}
1893	const int tensors_init = !!compiled_data->tensors_init.v;
1894	if (!tensors_init)
1895		_ccv_cnnp_model_tensors_init(model, compiled_data);
1896	else if ((uintptr_t)compiled_data->tensors_init.v & (uintptr_t)1)
1897	// Check if it is not fully allocated, if it is not, init_1.
1898		ccv_cnnp_model_tensors_init_1(model, compiled_data);
1899	ccv_array_t* const tensor_binds = ccv_array_new(sizeof(ccv_nnc_tensor_bind_t), 0, 0);
1900	assert((input_size % parallel_count) == 0)((void) sizeof (((input_size % parallel_count) == 0) ? 1 : 0)
, __extension__ ({ if ((input_size % parallel_count) == 0) ; else
 __assert_fail ("(input_size % parallel_count) == 0", "ccv_cnnp_model.c"
, 1900, __extension__ __PRETTY_FUNCTION__); }));
1901	assert((output_size % parallel_count) == 0)((void) sizeof (((output_size % parallel_count) == 0) ? 1 : 0
), __extension__ ({ if ((output_size % parallel_count) == 0) ;
 else __assert_fail ("(output_size % parallel_count) == 0", "ccv_cnnp_model.c"
, 1901, __extension__ __PRETTY_FUNCTION__); }));
1902	const int input_size_per_p = input_size / parallel_count;
1903	_ccv_cnnp_model_bind_tensors(model->graph, model->inputs, inputs, input_size_per_p, parallel_count, tensor_binds);
1904	const int output_size_per_p = output_size / parallel_count;
1905	_ccv_cnnp_model_bind_tensors(model->graph, model->outputs, outputs, output_size_per_p, parallel_count, tensor_binds);
1906	const int parameter_size = compiled_data->parameters->rnum;
1907	_ccv_cnnp_model_bind_tensors(model->graph, (ccv_nnc_tensor_symbol_t*)ccv_array_get(compiled_data->parameters, 0)((void*)(((char*)((compiled_data->parameters)->data)) +
 (size_t)(compiled_data->parameters)->rsize * (size_t)(
0))), compiled_data->tensors.parameters, parameter_size, parallel_count, tensor_binds);
1908	const int internal_size = compiled_data->internals->rnum;
1909	_ccv_cnnp_model_remove_nocopies(model->graph, (ccv_nnc_tensor_symbol_t*)ccv_array_get(compiled_data->internals, 0)((void*)(((char*)((compiled_data->internals)->data)) + (
size_t)(compiled_data->internals)->rsize * (size_t)(0))
), compiled_data->tensors.internals, internal_size, parallel_count);
1910	_ccv_cnnp_model_bind_tensors(model->graph, (ccv_nnc_tensor_symbol_t*)ccv_array_get(compiled_data->internals, 0)((void*)(((char*)((compiled_data->internals)->data)) + (
size_t)(compiled_data->internals)->rsize * (size_t)(0))
), compiled_data->tensors.internals, internal_size, parallel_count, tensor_binds);
1911	if (!compiled_data->tensors.gradients)
1912		_ccv_cnnp_model_gradient_tensors_init(model, compiled_data);
1913	_ccv_cnnp_model_bind_tensors(model->graph, compiled_data->gradients, compiled_data->tensors.gradients, parameter_size, parallel_count, tensor_binds);
1914	if (compiled_data->backward.to_size > 0)
1915		ccv_nnc_symbolic_graph_compile(model->graph, compiled_data->compile_params, (ccv_nnc_tensor_bind_t*)ccv_array_get(tensor_binds, 0)((void*)(((char*)((tensor_binds)->data)) + (size_t)(tensor_binds
)->rsize * (size_t)(0))), tensor_binds->rnum, 0, 0, SYMBOLIC_GRAPH_SOURCES(model->graph)ccv_nnc_symbolic_graph_sources(model->graph), ccv_nnc_symbolic_graph_source_size
(model->graph), compiled_data->backward.tos, compiled_data->backward.to_size, &compiled_data->graph, &compiled_data->tensor_arena, &compiled_data->graph_exec_arena);
1916	else
1917		ccv_nnc_symbolic_graph_compile(model->graph, compiled_data->compile_params, (ccv_nnc_tensor_bind_t*)ccv_array_get(tensor_binds, 0)((void*)(((char*)((tensor_binds)->data)) + (size_t)(tensor_binds
)->rsize * (size_t)(0))), tensor_binds->rnum, 0, 0, SYMBOLIC_GRAPH_SOURCES(model->graph)ccv_nnc_symbolic_graph_sources(model->graph), ccv_nnc_symbolic_graph_source_size
(model->graph), compiled_data->evaluate.tos, compiled_data->evaluate.to_size, &compiled_data->graph, &compiled_data->tensor_arena, &compiled_data->graph_exec_arena);
1918	ccv_array_free(tensor_binds);
1919	const uint32_t* const init_v = CCV_NNC_INIT_V(compiled_data->tensors_init.v)((uint32_t*)((uintptr_t)(compiled_data->tensors_init.v) &
 ~(uintptr_t)1));
1920	if (tensors_init && parallel_count > 1)
1921		_ccv_cnnp_model_copy_tensors(init_v, (ccv_nnc_tensor_symbol_t*)ccv_array_get(compiled_data->parameters, 0)((void*)(((char*)((compiled_data->parameters)->data)) +
 (size_t)(compiled_data->parameters)->rsize * (size_t)(
0))), compiled_data->tensors.parameters, compiled_data->parameters->rnum, parallel_count);
1922	// If tensor is not init'ed, we need to init states first.
1923	if (_ccv_cnnp_any_to_init(compiled_data))
1924	{
1925		ccv_nnc_tensor_init_states_t tensor_init_states = {
1926			.parallel_count = parallel_count,
1927			.graph = model->graph,
1928			.compiled_data = compiled_data,
1929			.tensor_arena = compiled_data->tensor_arena
1930		};
1931		ccv_cnnp_model_init_states(model, model->graph, _ccv_cnnp_init_states_for_tensors, &tensor_init_states);
1932	}
1933	compiled_data->is_test = is_test;
1934	ccv_nnc_graph_exec_update_t update = {
1935		.parallel_count = parallel_count,
1936		.graph = model->graph,
1937		.graph_exec_arena = compiled_data->graph_exec_arena,
1938	};
1939	ccv_cnnp_model_set_is_test(model, is_test, _ccv_cnnp_cmd_update_for_execs, &update);
1940	const int evaluate_to_size = compiled_data->evaluate.to_size;
1941	compiled_data->evaluate.to_op_size = 0;
1942	ccv_array_t* const backward_from = ccv_array_new(sizeof(int), 0, 0);
1943	for (i = 0; i < evaluate_to_size; i++)
1944	{
1945		ccv_nnc_graph_exec_t const to_op = ccv_nnc_graph_exec_from_symbol(compiled_data->graph_exec_arena, compiled_data->evaluate.tos[i]);
1946		if (to_op.graph)
1947			compiled_data->evaluate.to_ops[compiled_data->evaluate.to_op_size++] = to_op;
1948		const int* tos;
1949		int to_size;
1950		ccv_nnc_graph_exec_symbol_to(model->graph, compiled_data->evaluate.tos[i], &tos, &to_size);
1951		for (j = 0; j < to_size; j++)
1952		{
1953			ccv_nnc_graph_exec_t const to_op = ccv_nnc_graph_exec_from_symbol(compiled_data->graph_exec_arena, (ccv_nnc_graph_exec_symbol_t){
1954				.d = tos[j],
1955				.graph = model->graph
1956			});
1957			if (to_op.graph)
1958				ccv_array_add_unique_int(backward_from, to_op.d);
1959		}
1960	}
1961	assert(backward_from->rnum > 0)((void) sizeof ((backward_from->rnum > 0) ? 1 : 0), __extension__
 ({ if (backward_from->rnum > 0) ; else __assert_fail (
"backward_from->rnum > 0", "ccv_cnnp_model.c", 1961, __extension__
 __PRETTY_FUNCTION__); }));
1962	compiled_data->backward.from_op_size = backward_from->rnum;
1963	compiled_data->backward.from_ops = (ccv_nnc_graph_exec_t*)ccmallocmalloc(sizeof(ccv_nnc_graph_exec_t) * backward_from->rnum);
1964	for (i = 0; i < backward_from->rnum; i++)
1965		compiled_data->backward.from_ops[i] = (ccv_nnc_graph_exec_t){
1966			.d = *(int*)ccv_array_get(backward_from, i)((void*)(((char*)((backward_from)->data)) + (size_t)(backward_from
)->rsize * (size_t)(i))),
1967			.graph = compiled_data->graph,
1968		};
1969	// If there are any set node (to set some tensors to 0) inserted through backward pass, these won't be executed if we just do sources -> evaluate.to_ops, backward.from_ops -> destinations. We need this logic to find out these nodes and explicitly adding them to backward.from_ops.
1970	ccv_nnc_graph_exec_info_t* const exec_info = (ccv_nnc_graph_exec_info_t*)ccv_array_get(compiled_data->graph->exec_info, 0)((void*)(((char*)((compiled_data->graph->exec_info)->
data)) + (size_t)(compiled_data->graph->exec_info)->
rsize * (size_t)(0)));
1971	const int exec_info_size = compiled_data->graph->exec_info->rnum;
1972	uint32_t* const visited = cccalloccalloc((exec_info_size + 31) >> 5, sizeof(uint32_t));
1973	const ccv_nnc_graph_exec_t* const sources = (ccv_nnc_graph_exec_t*)ccv_array_get(compiled_data->graph->sources, 0)((void*)(((char*)((compiled_data->graph->sources)->data
)) + (size_t)(compiled_data->graph->sources)->rsize *
 (size_t)(0)));
1974	const int source_size = compiled_data->graph->sources->rnum;
1975	ccv_nnc_graph_visit_t* visit = ccv_nnc_graph_visit_new(compiled_data->graph, exec_info, exec_info_size, sources, source_size, compiled_data->evaluate.to_ops, compiled_data->evaluate.to_op_size, 0)({ ccv_nnc_graph_visit_t* _visit_ = (ccv_nnc_graph_visit_t*)malloc
(sizeof(ccv_nnc_graph_visit_t) + sizeof(_visit_->node[0]) *
 ((exec_info_size) - 1)); _visit_->size = 0; do { typedef struct
 { int8_t d; int8_t r; uint16_t c; int32_t edges; } ccv_nnc_incoming_t
; int _i_, _j_; int _incoming_edges_ = 0; for (_i_ = 0; _i_ <
 (exec_info_size); _i_++) _incoming_edges_ += ((exec_info)[_i_
].outgoings) ? (exec_info)[_i_].outgoings->rnum : 0; const
 int _heap_mem_ = ((exec_info_size) + _incoming_edges_ > 1024
); ccv_nnc_incoming_t* _incomings_; if (_heap_mem_) _incomings_
 = (ccv_nnc_incoming_t*)malloc(sizeof(ccv_nnc_incoming_t) * (
exec_info_size) + sizeof(int32_t) * ((exec_info_size) * 2 + _incoming_edges_
)); else _incomings_ = (ccv_nnc_incoming_t*)__builtin_alloca (
sizeof(ccv_nnc_incoming_t) * (exec_info_size) + sizeof(int32_t
) * ((exec_info_size) * 2 + _incoming_edges_)); memset(_incomings_
, 0, sizeof(ccv_nnc_incoming_t) * (exec_info_size)); int32_t*
 _exists_[2] = { (int32_t*)(_incomings_ + (exec_info_size)), (
int32_t*)(_incomings_ + (exec_info_size)) + (exec_info_size),
 }; int32_t* const _edges_ = _exists_[1] + (exec_info_size); for
 (_i_ = 0; _i_ < (source_size); _i_++) { ((void) sizeof ((
(sources)[_i_].graph == compiled_data->graph) ? 1 : 0), __extension__
 ({ if ((sources)[_i_].graph == compiled_data->graph) ; else
 __assert_fail ("(sources)[_i_].graph == compiled_data->graph"
, "ccv_cnnp_model.c", 1975, __extension__ __PRETTY_FUNCTION__
); })); _incomings_[(sources)[_i_].d].r = 1; _exists_[0][_i_]
 = (sources)[_i_].d; } int _exist_size_[2] = { (source_size),
 0, }; int _p_ = 0, _q_ = 1; while (_exist_size_[_p_] > 0)
 { _exist_size_[_q_] = 0; for (_i_ = 0; _i_ < _exist_size_
[_p_]; _i_++) { const int32_t _idx_ = _exists_[_p_][_i_]; if (
_incomings_[_idx_].r != 1) continue; _incomings_[_idx_].r = 2
; if ((exec_info)[_idx_].outgoings) for (_j_ = 0; _j_ < (exec_info
)[_idx_].outgoings->rnum; _j_++) { const int d = *(int*)((
void*)(((char*)(((exec_info)[_idx_].outgoings)->data)) + (
size_t)((exec_info)[_idx_].outgoings)->rsize * (size_t)(_j_
))); ++_incomings_[d].c; if (_incomings_[d].r != 0) continue;
 _incomings_[d].r = 1; ((void) sizeof ((_exist_size_[_q_] <
 (exec_info_size)) ? 1 : 0), __extension__ ({ if (_exist_size_
[_q_] < (exec_info_size)) ; else __assert_fail ("_exist_size_[_q_] < (exec_info_size)"
, "ccv_cnnp_model.c", 1975, __extension__ __PRETTY_FUNCTION__
); })); _exists_[_q_][_exist_size_[_q_]] = d; ++_exist_size_[
_q_]; } } ((_i_) = (_p_), (_p_) = (_q_), (_q_) = (_i_)); } for
 (_i_ = 0; _i_ < (source_size); _i_++) { ((void) sizeof ((
(sources)[_i_].graph == compiled_data->graph) ? 1 : 0), __extension__
 ({ if ((sources)[_i_].graph == compiled_data->graph) ; else
 __assert_fail ("(sources)[_i_].graph == compiled_data->graph"
, "ccv_cnnp_model.c", 1975, __extension__ __PRETTY_FUNCTION__
); })); _incomings_[(sources)[_i_].d].r = 3; _exists_[0][_i_]
 = (sources)[_i_].d; } _exist_size_[0] = (source_size); _exist_size_
[1] = 0; _p_ = 0, _q_ = 1; int _bump_ = 1; while (_exist_size_
[_p_] > 0) { _exist_size_[_q_] = 0; for (_i_ = 0; _i_ <
 _exist_size_[_p_]; _i_++) { const int32_t _idx_ = _exists_[_p_
][_i_]; if (_incomings_[_idx_].r != 3) continue; _incomings_[
_idx_].r = 4; if ((exec_info)[_idx_].outgoings) for (_j_ = 0;
 _j_ < (exec_info)[_idx_].outgoings->rnum; _j_++) { const
 int d = *(int*)((void*)(((char*)(((exec_info)[_idx_].outgoings
)->data)) + (size_t)((exec_info)[_idx_].outgoings)->rsize
 * (size_t)(_j_))); if (_incomings_[d].edges == 0) { _incomings_
[d].edges = _bump_; _bump_ += _incomings_[d].c; _incomings_[d
].c = 0; } _edges_[_incomings_[d].edges - 1 + _incomings_[d].
c] = _idx_; ++_incomings_[d].c; if (_incomings_[d].r != 2) continue
; _incomings_[d].r = 3; ((void) sizeof ((_exist_size_[_q_] <
 (exec_info_size)) ? 1 : 0), __extension__ ({ if (_exist_size_
[_q_] < (exec_info_size)) ; else __assert_fail ("_exist_size_[_q_] < (exec_info_size)"
, "ccv_cnnp_model.c", 1975, __extension__ __PRETTY_FUNCTION__
); })); _exists_[_q_][_exist_size_[_q_]] = d; ++_exist_size_[
_q_]; } } ((_i_) = (_p_), (_p_) = (_q_), (_q_) = (_i_)); } for
 (_i_ = 0; _i_ < (compiled_data->evaluate.to_op_size); _i_
++) { ((void) sizeof (((compiled_data->evaluate.to_ops)[_i_
].graph == compiled_data->graph) ? 1 : 0), __extension__ (
{ if ((compiled_data->evaluate.to_ops)[_i_].graph == compiled_data
->graph) ; else __assert_fail ("(compiled_data->evaluate.to_ops)[_i_].graph == compiled_data->graph"
, "ccv_cnnp_model.c", 1975, __extension__ __PRETTY_FUNCTION__
); })); _incomings_[(compiled_data->evaluate.to_ops)[_i_].
d].r = 5; _exists_[0][_i_] = (compiled_data->evaluate.to_ops
)[_i_].d; } _exist_size_[0] = (compiled_data->evaluate.to_op_size
); _exist_size_[1] = 0; _p_ = 0, _q_ = 1; while (_exist_size_
[_p_] > 0) { _exist_size_[_q_] = 0; for (_i_ = 0; _i_ <
 _exist_size_[_p_]; _i_++) { const int32_t _idx_ = _exists_[_p_
][_i_]; if (_incomings_[_idx_].r != 5) continue; _incomings_[
_idx_].r = 6; if (_incomings_[_idx_].edges > 0) for (_j_ =
 0; _j_ < _incomings_[_idx_].c; _j_++) { const int d = _edges_
[_incomings_[_idx_].edges - 1 + _j_]; if (_incomings_[d].r !=
 4) continue; _incomings_[d].r = 5; ((void) sizeof ((_exist_size_
[_q_] < (exec_info_size)) ? 1 : 0), __extension__ ({ if (_exist_size_
[_q_] < (exec_info_size)) ; else __assert_fail ("_exist_size_[_q_] < (exec_info_size)"
, "ccv_cnnp_model.c", 1975, __extension__ __PRETTY_FUNCTION__
); })); _exists_[_q_][_exist_size_[_q_]] = d; ++_exist_size_[
_q_]; } } ((_i_) = (_p_), (_p_) = (_q_), (_q_) = (_i_)); } for
 (_i_ = 0; _i_ < (compiled_data->evaluate.to_op_size); _i_
++) { ((void) sizeof (((compiled_data->evaluate.to_ops)[_i_
].graph == compiled_data->graph) ? 1 : 0), __extension__ (
{ if ((compiled_data->evaluate.to_ops)[_i_].graph == compiled_data
->graph) ; else __assert_fail ("(compiled_data->evaluate.to_ops)[_i_].graph == compiled_data->graph"
, "ccv_cnnp_model.c", 1975, __extension__ __PRETTY_FUNCTION__
); })); _incomings_[(compiled_data->evaluate.to_ops)[_i_].
d].d = 1; } for (_i_ = 0; _i_ < (source_size); _i_++) { ((
void) sizeof (((sources)[_i_].graph == compiled_data->graph
) ? 1 : 0), __extension__ ({ if ((sources)[_i_].graph == compiled_data
->graph) ; else __assert_fail ("(sources)[_i_].graph == compiled_data->graph"
, "ccv_cnnp_model.c", 1975, __extension__ __PRETTY_FUNCTION__
); })); _exists_[0][_i_] = (sources)[_i_].d; } _p_ = 0; _q_ =
 1; _exist_size_[0] = (source_size); _exist_size_[1] = 0; int
 _d_ = 0; while (_exist_size_[_p_] > 0) { _exist_size_[_q_
] = 0; for (_i_ = 0; _i_ < _exist_size_[_p_];) { const int32_t
 _idx_ = _exists_[_p_][_i_]; _visit_->node[_visit_->size
].index = ((_idx_)); _visit_->node[_visit_->size].term =
 ((_incomings_[_idx_].d)); ++_visit_->size;; if (_incomings_
[_idx_].d) { ++_d_; _incomings_[_idx_].r = 7; } if ((exec_info
)[_idx_].outgoings) { if ((exec_info)[_idx_].outgoings->rnum
 == 1) { const int d = *(int*)((void*)(((char*)(((exec_info)[
_idx_].outgoings)->data)) + (size_t)((exec_info)[_idx_].outgoings
)->rsize * (size_t)(0))); --_incomings_[d].c; if (_incomings_
[d].c == 0 && _incomings_[d].r == 6 && _d_ <
 (compiled_data->evaluate.to_op_size)) { _exists_[_p_][_i_
] = d; continue; } } else for (_j_ = 0; _j_ < (exec_info)[
_idx_].outgoings->rnum; _j_++) { const int d = *(int*)((void
*)(((char*)(((exec_info)[_idx_].outgoings)->data)) + (size_t
)((exec_info)[_idx_].outgoings)->rsize * (size_t)(_j_))); --
_incomings_[d].c; if (_incomings_[d].c == 0 && _incomings_
[d].r == 6 && _d_ < (compiled_data->evaluate.to_op_size
)) { ((void) sizeof ((_exist_size_[_q_] < (exec_info_size)
) ? 1 : 0), __extension__ ({ if (_exist_size_[_q_] < (exec_info_size
)) ; else __assert_fail ("_exist_size_[_q_] < (exec_info_size)"
, "ccv_cnnp_model.c", 1975, __extension__ __PRETTY_FUNCTION__
); })); _exists_[_q_][_exist_size_[_q_]] = d; ++_exist_size_[
_q_]; } } } ++_i_; } ((_i_) = (_p_), (_p_) = (_q_), (_q_) = (
_i_)); } for (_i_ = 0; _i_ < (compiled_data->evaluate.to_op_size
); _i_++) { ((void) sizeof (((compiled_data->evaluate.to_ops
)[_i_].graph == compiled_data->graph) ? 1 : 0), __extension__
 ({ if ((compiled_data->evaluate.to_ops)[_i_].graph == compiled_data
->graph) ; else __assert_fail ("(compiled_data->evaluate.to_ops)[_i_].graph == compiled_data->graph"
, "ccv_cnnp_model.c", 1975, __extension__ __PRETTY_FUNCTION__
); })); if (_incomings_[(compiled_data->evaluate.to_ops)[_i_
].d].r == 7) continue; if (!(0)) { ((void) sizeof ((_incomings_
[(compiled_data->evaluate.to_ops)[_i_].d].c == 0) ? 1 : 0)
, __extension__ ({ if (_incomings_[(compiled_data->evaluate
.to_ops)[_i_].d].c == 0) ; else __assert_fail ("_incomings_[(compiled_data->evaluate.to_ops)[_i_].d].c == 0"
, "ccv_cnnp_model.c", 1975, __extension__ __PRETTY_FUNCTION__
); })); } else if (_incomings_[(compiled_data->evaluate.to_ops
)[_i_].d].c > 0) continue; _visit_->node[_visit_->size
].index = (((compiled_data->evaluate.to_ops)[_i_].d)); _visit_
->node[_visit_->size].term = ((_incomings_[(compiled_data
->evaluate.to_ops)[_i_].d].d)); ++_visit_->size;; } if (
_heap_mem_) free(_incomings_); } while (0);; ((void) sizeof (
(_visit_->size <= (exec_info_size)) ? 1 : 0), __extension__
 ({ if (_visit_->size <= (exec_info_size)) ; else __assert_fail
 ("_visit_->size <= (exec_info_size)", "ccv_cnnp_model.c"
, 1975, __extension__ __PRETTY_FUNCTION__); })); _visit_; });
1976	ccv_nnc_graph_visit_for(visit, exec_info, node, idx){ int _i_; for (_i_ = 0; _i_ < (visit)->size; _i_++) { const
 int idx __attribute__((unused)) = (visit)->node[_i_].index
; const int _node_unused_ __attribute__((unused)) = (visit)->
node[_i_].term; typeof ((exec_info)) const node __attribute__
((unused)) = (exec_info) + idx; {
1977		visited[(idx >> 5)] |= (1u << (idx & 31));
1978	} ccv_nnc_graph_visit_endfor} }
1979	ccv_nnc_graph_visit_free(visit);
1980	const ccv_nnc_graph_exec_t* const destinations = (ccv_nnc_graph_exec_t*)ccv_array_get(compiled_data->graph->destinations, 0)((void*)(((char*)((compiled_data->graph->destinations)->
data)) + (size_t)(compiled_data->graph->destinations)->
rsize * (size_t)(0)));
1981	const int destination_size = compiled_data->graph->destinations->rnum;
1982	visit = ccv_nnc_graph_visit_new(compiled_data->graph, exec_info, exec_info_size, compiled_data->backward.from_ops, compiled_data->backward.from_op_size, destinations, destination_size, 0)({ ccv_nnc_graph_visit_t* _visit_ = (ccv_nnc_graph_visit_t*)malloc
(sizeof(ccv_nnc_graph_visit_t) + sizeof(_visit_->node[0]) *
 ((exec_info_size) - 1)); _visit_->size = 0; do { typedef struct
 { int8_t d; int8_t r; uint16_t c; int32_t edges; } ccv_nnc_incoming_t
; int _i_, _j_; int _incoming_edges_ = 0; for (_i_ = 0; _i_ <
 (exec_info_size); _i_++) _incoming_edges_ += ((exec_info)[_i_
].outgoings) ? (exec_info)[_i_].outgoings->rnum : 0; const
 int _heap_mem_ = ((exec_info_size) + _incoming_edges_ > 1024
); ccv_nnc_incoming_t* _incomings_; if (_heap_mem_) _incomings_
 = (ccv_nnc_incoming_t*)malloc(sizeof(ccv_nnc_incoming_t) * (
exec_info_size) + sizeof(int32_t) * ((exec_info_size) * 2 + _incoming_edges_
)); else _incomings_ = (ccv_nnc_incoming_t*)__builtin_alloca (
sizeof(ccv_nnc_incoming_t) * (exec_info_size) + sizeof(int32_t
) * ((exec_info_size) * 2 + _incoming_edges_)); memset(_incomings_
, 0, sizeof(ccv_nnc_incoming_t) * (exec_info_size)); int32_t*
 _exists_[2] = { (int32_t*)(_incomings_ + (exec_info_size)), (
int32_t*)(_incomings_ + (exec_info_size)) + (exec_info_size),
 }; int32_t* const _edges_ = _exists_[1] + (exec_info_size); for
 (_i_ = 0; _i_ < (compiled_data->backward.from_op_size)
; _i_++) { ((void) sizeof (((compiled_data->backward.from_ops
)[_i_].graph == compiled_data->graph) ? 1 : 0), __extension__
 ({ if ((compiled_data->backward.from_ops)[_i_].graph == compiled_data
->graph) ; else __assert_fail ("(compiled_data->backward.from_ops)[_i_].graph == compiled_data->graph"
, "ccv_cnnp_model.c", 1982, __extension__ __PRETTY_FUNCTION__
); })); _incomings_[(compiled_data->backward.from_ops)[_i_
].d].r = 1; _exists_[0][_i_] = (compiled_data->backward.from_ops
)[_i_].d; } int _exist_size_[2] = { (compiled_data->backward
.from_op_size), 0, }; int _p_ = 0, _q_ = 1; while (_exist_size_
[_p_] > 0) { _exist_size_[_q_] = 0; for (_i_ = 0; _i_ <
 _exist_size_[_p_]; _i_++) { const int32_t _idx_ = _exists_[_p_
][_i_]; if (_incomings_[_idx_].r != 1) continue; _incomings_[
_idx_].r = 2; if ((exec_info)[_idx_].outgoings) for (_j_ = 0;
 _j_ < (exec_info)[_idx_].outgoings->rnum; _j_++) { const
 int d = *(int*)((void*)(((char*)(((exec_info)[_idx_].outgoings
)->data)) + (size_t)((exec_info)[_idx_].outgoings)->rsize
 * (size_t)(_j_))); ++_incomings_[d].c; if (_incomings_[d].r !=
 0) continue; _incomings_[d].r = 1; ((void) sizeof ((_exist_size_
[_q_] < (exec_info_size)) ? 1 : 0), __extension__ ({ if (_exist_size_
[_q_] < (exec_info_size)) ; else __assert_fail ("_exist_size_[_q_] < (exec_info_size)"
, "ccv_cnnp_model.c", 1982, __extension__ __PRETTY_FUNCTION__
); })); _exists_[_q_][_exist_size_[_q_]] = d; ++_exist_size_[
_q_]; } } ((_i_) = (_p_), (_p_) = (_q_), (_q_) = (_i_)); } for
 (_i_ = 0; _i_ < (compiled_data->backward.from_op_size)
; _i_++) { ((void) sizeof (((compiled_data->backward.from_ops
)[_i_].graph == compiled_data->graph) ? 1 : 0), __extension__
 ({ if ((compiled_data->backward.from_ops)[_i_].graph == compiled_data
->graph) ; else __assert_fail ("(compiled_data->backward.from_ops)[_i_].graph == compiled_data->graph"
, "ccv_cnnp_model.c", 1982, __extension__ __PRETTY_FUNCTION__
); })); _incomings_[(compiled_data->backward.from_ops)[_i_
].d].r = 3; _exists_[0][_i_] = (compiled_data->backward.from_ops
)[_i_].d; } _exist_size_[0] = (compiled_data->backward.from_op_size
); _exist_size_[1] = 0; _p_ = 0, _q_ = 1; int _bump_ = 1; while
 (_exist_size_[_p_] > 0) { _exist_size_[_q_] = 0; for (_i_
 = 0; _i_ < _exist_size_[_p_]; _i_++) { const int32_t _idx_
 = _exists_[_p_][_i_]; if (_incomings_[_idx_].r != 3) continue
; _incomings_[_idx_].r = 4; if ((exec_info)[_idx_].outgoings)
 for (_j_ = 0; _j_ < (exec_info)[_idx_].outgoings->rnum
; _j_++) { const int d = *(int*)((void*)(((char*)(((exec_info
)[_idx_].outgoings)->data)) + (size_t)((exec_info)[_idx_].
outgoings)->rsize * (size_t)(_j_))); if (_incomings_[d].edges
 == 0) { _incomings_[d].edges = _bump_; _bump_ += _incomings_
[d].c; _incomings_[d].c = 0; } _edges_[_incomings_[d].edges -
 1 + _incomings_[d].c] = _idx_; ++_incomings_[d].c; if (_incomings_
[d].r != 2) continue; _incomings_[d].r = 3; ((void) sizeof ((
_exist_size_[_q_] < (exec_info_size)) ? 1 : 0), __extension__
 ({ if (_exist_size_[_q_] < (exec_info_size)) ; else __assert_fail
 ("_exist_size_[_q_] < (exec_info_size)", "ccv_cnnp_model.c"
, 1982, __extension__ __PRETTY_FUNCTION__); })); _exists_[_q_
][_exist_size_[_q_]] = d; ++_exist_size_[_q_]; } } ((_i_) = (
_p_), (_p_) = (_q_), (_q_) = (_i_)); } for (_i_ = 0; _i_ <
 (destination_size); _i_++) { ((void) sizeof (((destinations)
[_i_].graph == compiled_data->graph) ? 1 : 0), __extension__
 ({ if ((destinations)[_i_].graph == compiled_data->graph)
 ; else __assert_fail ("(destinations)[_i_].graph == compiled_data->graph"
, "ccv_cnnp_model.c", 1982, __extension__ __PRETTY_FUNCTION__
); })); _incomings_[(destinations)[_i_].d].r = 5; _exists_[0]
[_i_] = (destinations)[_i_].d; } _exist_size_[0] = (destination_size
); _exist_size_[1] = 0; _p_ = 0, _q_ = 1; while (_exist_size_
[_p_] > 0) { _exist_size_[_q_] = 0; for (_i_ = 0; _i_ <
 _exist_size_[_p_]; _i_++) { const int32_t _idx_ = _exists_[_p_
][_i_]; if (_incomings_[_idx_].r != 5) continue; _incomings_[
_idx_].r = 6; if (_incomings_[_idx_].edges > 0) for (_j_ =
 0; _j_ < _incomings_[_idx_].c; _j_++) { const int d = _edges_
[_incomings_[_idx_].edges - 1 + _j_]; if (_incomings_[d].r !=
 4) continue; _incomings_[d].r = 5; ((void) sizeof ((_exist_size_
[_q_] < (exec_info_size)) ? 1 : 0), __extension__ ({ if (_exist_size_
[_q_] < (exec_info_size)) ; else __assert_fail ("_exist_size_[_q_] < (exec_info_size)"
, "ccv_cnnp_model.c", 1982, __extension__ __PRETTY_FUNCTION__
); })); _exists_[_q_][_exist_size_[_q_]] = d; ++_exist_size_[
_q_]; } } ((_i_) = (_p_), (_p_) = (_q_), (_q_) = (_i_)); } for
 (_i_ = 0; _i_ < (destination_size); _i_++) { ((void) sizeof
 (((destinations)[_i_].graph == compiled_data->graph) ? 1 :
 0), __extension__ ({ if ((destinations)[_i_].graph == compiled_data
->graph) ; else __assert_fail ("(destinations)[_i_].graph == compiled_data->graph"
, "ccv_cnnp_model.c", 1982, __extension__ __PRETTY_FUNCTION__
); })); _incomings_[(destinations)[_i_].d].d = 1; } for (_i_ =
 0; _i_ < (compiled_data->backward.from_op_size); _i_++
) { ((void) sizeof (((compiled_data->backward.from_ops)[_i_
].graph == compiled_data->graph) ? 1 : 0), __extension__ (
{ if ((compiled_data->backward.from_ops)[_i_].graph == compiled_data
->graph) ; else __assert_fail ("(compiled_data->backward.from_ops)[_i_].graph == compiled_data->graph"
, "ccv_cnnp_model.c", 1982, __extension__ __PRETTY_FUNCTION__
); })); _exists_[0][_i_] = (compiled_data->backward.from_ops
)[_i_].d; } _p_ = 0; _q_ = 1; _exist_size_[0] = (compiled_data
->backward.from_op_size); _exist_size_[1] = 0; int _d_ = 0
; while (_exist_size_[_p_] > 0) { _exist_size_[_q_] = 0; for
 (_i_ = 0; _i_ < _exist_size_[_p_];) { const int32_t _idx_
 = _exists_[_p_][_i_]; _visit_->node[_visit_->size].index
 = ((_idx_)); _visit_->node[_visit_->size].term = ((_incomings_
[_idx_].d)); ++_visit_->size;; if (_incomings_[_idx_].d) {
 ++_d_; _incomings_[_idx_].r = 7; } if ((exec_info)[_idx_].outgoings
) { if ((exec_info)[_idx_].outgoings->rnum == 1) { const int
 d = *(int*)((void*)(((char*)(((exec_info)[_idx_].outgoings)->
data)) + (size_t)((exec_info)[_idx_].outgoings)->rsize * (
size_t)(0))); --_incomings_[d].c; if (_incomings_[d].c == 0 &&
 _incomings_[d].r == 6 && _d_ < (destination_size)
) { _exists_[_p_][_i_] = d; continue; } } else for (_j_ = 0; _j_
 < (exec_info)[_idx_].outgoings->rnum; _j_++) { const int
 d = *(int*)((void*)(((char*)(((exec_info)[_idx_].outgoings)->
data)) + (size_t)((exec_info)[_idx_].outgoings)->rsize * (
size_t)(_j_))); --_incomings_[d].c; if (_incomings_[d].c == 0
 && _incomings_[d].r == 6 && _d_ < (destination_size
)) { ((void) sizeof ((_exist_size_[_q_] < (exec_info_size)
) ? 1 : 0), __extension__ ({ if (_exist_size_[_q_] < (exec_info_size
)) ; else __assert_fail ("_exist_size_[_q_] < (exec_info_size)"
, "ccv_cnnp_model.c", 1982, __extension__ __PRETTY_FUNCTION__
); })); _exists_[_q_][_exist_size_[_q_]] = d; ++_exist_size_[
_q_]; } } } ++_i_; } ((_i_) = (_p_), (_p_) = (_q_), (_q_) = (
_i_)); } for (_i_ = 0; _i_ < (destination_size); _i_++) { (
(void) sizeof (((destinations)[_i_].graph == compiled_data->
graph) ? 1 : 0), __extension__ ({ if ((destinations)[_i_].graph
 == compiled_data->graph) ; else __assert_fail ("(destinations)[_i_].graph == compiled_data->graph"
, "ccv_cnnp_model.c", 1982, __extension__ __PRETTY_FUNCTION__
); })); if (_incomings_[(destinations)[_i_].d].r == 7) continue
; if (!(0)) { ((void) sizeof ((_incomings_[(destinations)[_i_
].d].c == 0) ? 1 : 0), __extension__ ({ if (_incomings_[(destinations
)[_i_].d].c == 0) ; else __assert_fail ("_incomings_[(destinations)[_i_].d].c == 0"
, "ccv_cnnp_model.c", 1982, __extension__ __PRETTY_FUNCTION__
); })); } else if (_incomings_[(destinations)[_i_].d].c > 0
) continue; _visit_->node[_visit_->size].index = (((destinations
)[_i_].d)); _visit_->node[_visit_->size].term = ((_incomings_
[(destinations)[_i_].d].d)); ++_visit_->size;; } if (_heap_mem_
) free(_incomings_); } while (0);; ((void) sizeof ((_visit_->
size <= (exec_info_size)) ? 1 : 0), __extension__ ({ if (_visit_
->size <= (exec_info_size)) ; else __assert_fail ("_visit_->size <= (exec_info_size)"
, "ccv_cnnp_model.c", 1982, __extension__ __PRETTY_FUNCTION__
); })); _visit_; });
1983	ccv_nnc_graph_visit_for(visit, exec_info, node, idx){ int _i_; for (_i_ = 0; _i_ < (visit)->size; _i_++) { const
 int idx __attribute__((unused)) = (visit)->node[_i_].index
; const int _node_unused_ __attribute__((unused)) = (visit)->
node[_i_].term; typeof ((exec_info)) const node __attribute__
((unused)) = (exec_info) + idx; {
1984		visited[(idx >> 5)] |= (1u << (idx & 31));
1985	} ccv_nnc_graph_visit_endfor} }
1986	ccv_nnc_graph_visit_free(visit);
1987	visit = ccv_nnc_graph_visit_new(compiled_data->graph, exec_info, exec_info_size, sources, source_size, destinations, destination_size, 0)({ ccv_nnc_graph_visit_t* _visit_ = (ccv_nnc_graph_visit_t*)malloc
(sizeof(ccv_nnc_graph_visit_t) + sizeof(_visit_->node[0]) *
 ((exec_info_size) - 1)); _visit_->size = 0; do { typedef struct
 { int8_t d; int8_t r; uint16_t c; int32_t edges; } ccv_nnc_incoming_t
; int _i_, _j_; int _incoming_edges_ = 0; for (_i_ = 0; _i_ <
 (exec_info_size); _i_++) _incoming_edges_ += ((exec_info)[_i_
].outgoings) ? (exec_info)[_i_].outgoings->rnum : 0; const
 int _heap_mem_ = ((exec_info_size) + _incoming_edges_ > 1024
); ccv_nnc_incoming_t* _incomings_; if (_heap_mem_) _incomings_
 = (ccv_nnc_incoming_t*)malloc(sizeof(ccv_nnc_incoming_t) * (
exec_info_size) + sizeof(int32_t) * ((exec_info_size) * 2 + _incoming_edges_
)); else _incomings_ = (ccv_nnc_incoming_t*)__builtin_alloca (
sizeof(ccv_nnc_incoming_t) * (exec_info_size) + sizeof(int32_t
) * ((exec_info_size) * 2 + _incoming_edges_)); memset(_incomings_
, 0, sizeof(ccv_nnc_incoming_t) * (exec_info_size)); int32_t*
 _exists_[2] = { (int32_t*)(_incomings_ + (exec_info_size)), (
int32_t*)(_incomings_ + (exec_info_size)) + (exec_info_size),
 }; int32_t* const _edges_ = _exists_[1] + (exec_info_size); for
 (_i_ = 0; _i_ < (source_size); _i_++) { ((void) sizeof ((
(sources)[_i_].graph == compiled_data->graph) ? 1 : 0), __extension__
 ({ if ((sources)[_i_].graph == compiled_data->graph) ; else
 __assert_fail ("(sources)[_i_].graph == compiled_data->graph"
, "ccv_cnnp_model.c", 1987, __extension__ __PRETTY_FUNCTION__
); })); _incomings_[(sources)[_i_].d].r = 1; _exists_[0][_i_]
 = (sources)[_i_].d; } int _exist_size_[2] = { (source_size),
 0, }; int _p_ = 0, _q_ = 1; while (_exist_size_[_p_] > 0)
 { _exist_size_[_q_] = 0; for (_i_ = 0; _i_ < _exist_size_
[_p_]; _i_++) { const int32_t _idx_ = _exists_[_p_][_i_]; if (
_incomings_[_idx_].r != 1) continue; _incomings_[_idx_].r = 2
; if ((exec_info)[_idx_].outgoings) for (_j_ = 0; _j_ < (exec_info
)[_idx_].outgoings->rnum; _j_++) { const int d = *(int*)((
void*)(((char*)(((exec_info)[_idx_].outgoings)->data)) + (
size_t)((exec_info)[_idx_].outgoings)->rsize * (size_t)(_j_
))); ++_incomings_[d].c; if (_incomings_[d].r != 0) continue;
 _incomings_[d].r = 1; ((void) sizeof ((_exist_size_[_q_] <
 (exec_info_size)) ? 1 : 0), __extension__ ({ if (_exist_size_
[_q_] < (exec_info_size)) ; else __assert_fail ("_exist_size_[_q_] < (exec_info_size)"
, "ccv_cnnp_model.c", 1987, __extension__ __PRETTY_FUNCTION__
); })); _exists_[_q_][_exist_size_[_q_]] = d; ++_exist_size_[
_q_]; } } ((_i_) = (_p_), (_p_) = (_q_), (_q_) = (_i_)); } for
 (_i_ = 0; _i_ < (source_size); _i_++) { ((void) sizeof ((
(sources)[_i_].graph == compiled_data->graph) ? 1 : 0), __extension__
 ({ if ((sources)[_i_].graph == compiled_data->graph) ; else
 __assert_fail ("(sources)[_i_].graph == compiled_data->graph"
, "ccv_cnnp_model.c", 1987, __extension__ __PRETTY_FUNCTION__
); })); _incomings_[(sources)[_i_].d].r = 3; _exists_[0][_i_]
 = (sources)[_i_].d; } _exist_size_[0] = (source_size); _exist_size_
[1] = 0; _p_ = 0, _q_ = 1; int _bump_ = 1; while (_exist_size_
[_p_] > 0) { _exist_size_[_q_] = 0; for (_i_ = 0; _i_ <
 _exist_size_[_p_]; _i_++) { const int32_t _idx_ = _exists_[_p_
][_i_]; if (_incomings_[_idx_].r != 3) continue; _incomings_[
_idx_].r = 4; if ((exec_info)[_idx_].outgoings) for (_j_ = 0;
 _j_ < (exec_info)[_idx_].outgoings->rnum; _j_++) { const
 int d = *(int*)((void*)(((char*)(((exec_info)[_idx_].outgoings
)->data)) + (size_t)((exec_info)[_idx_].outgoings)->rsize
 * (size_t)(_j_))); if (_incomings_[d].edges == 0) { _incomings_
[d].edges = _bump_; _bump_ += _incomings_[d].c; _incomings_[d
].c = 0; } _edges_[_incomings_[d].edges - 1 + _incomings_[d].
c] = _idx_; ++_incomings_[d].c; if (_incomings_[d].r != 2) continue
; _incomings_[d].r = 3; ((void) sizeof ((_exist_size_[_q_] <
 (exec_info_size)) ? 1 : 0), __extension__ ({ if (_exist_size_
[_q_] < (exec_info_size)) ; else __assert_fail ("_exist_size_[_q_] < (exec_info_size)"
, "ccv_cnnp_model.c", 1987, __extension__ __PRETTY_FUNCTION__
); })); _exists_[_q_][_exist_size_[_q_]] = d; ++_exist_size_[
_q_]; } } ((_i_) = (_p_), (_p_) = (_q_), (_q_) = (_i_)); } for
 (_i_ = 0; _i_ < (destination_size); _i_++) { ((void) sizeof
 (((destinations)[_i_].graph == compiled_data->graph) ? 1 :
 0), __extension__ ({ if ((destinations)[_i_].graph == compiled_data
->graph) ; else __assert_fail ("(destinations)[_i_].graph == compiled_data->graph"
, "ccv_cnnp_model.c", 1987, __extension__ __PRETTY_FUNCTION__
); })); _incomings_[(destinations)[_i_].d].r = 5; _exists_[0]
[_i_] = (destinations)[_i_].d; } _exist_size_[0] = (destination_size
); _exist_size_[1] = 0; _p_ = 0, _q_ = 1; while (_exist_size_
[_p_] > 0) { _exist_size_[_q_] = 0; for (_i_ = 0; _i_ <
 _exist_size_[_p_]; _i_++) { const int32_t _idx_ = _exists_[_p_
][_i_]; if (_incomings_[_idx_].r != 5) continue; _incomings_[
_idx_].r = 6; if (_incomings_[_idx_].edges > 0) for (_j_ =
 0; _j_ < _incomings_[_idx_].c; _j_++) { const int d = _edges_
[_incomings_[_idx_].edges - 1 + _j_]; if (_incomings_[d].r !=
 4) continue; _incomings_[d].r = 5; ((void) sizeof ((_exist_size_
[_q_] < (exec_info_size)) ? 1 : 0), __extension__ ({ if (_exist_size_
[_q_] < (exec_info_size)) ; else __assert_fail ("_exist_size_[_q_] < (exec_info_size)"
, "ccv_cnnp_model.c", 1987, __extension__ __PRETTY_FUNCTION__
); })); _exists_[_q_][_exist_size_[_q_]] = d; ++_exist_size_[
_q_]; } } ((_i_) = (_p_), (_p_) = (_q_), (_q_) = (_i_)); } for
 (_i_ = 0; _i_ < (destination_size); _i_++) { ((void) sizeof
 (((destinations)[_i_].graph == compiled_data->graph) ? 1 :
 0), __extension__ ({ if ((destinations)[_i_].graph == compiled_data
->graph) ; else __assert_fail ("(destinations)[_i_].graph == compiled_data->graph"
, "ccv_cnnp_model.c", 1987, __extension__ __PRETTY_FUNCTION__
); })); _incomings_[(destinations)[_i_].d].d = 1; } for (_i_ =
 0; _i_ < (source_size); _i_++) { ((void) sizeof (((sources
)[_i_].graph == compiled_data->graph) ? 1 : 0), __extension__
 ({ if ((sources)[_i_].graph == compiled_data->graph) ; else
 __assert_fail ("(sources)[_i_].graph == compiled_data->graph"
, "ccv_cnnp_model.c", 1987, __extension__ __PRETTY_FUNCTION__
); })); _exists_[0][_i_] = (sources)[_i_].d; } _p_ = 0; _q_ =
 1; _exist_size_[0] = (source_size); _exist_size_[1] = 0; int
 _d_ = 0; while (_exist_size_[_p_] > 0) { _exist_size_[_q_
] = 0; for (_i_ = 0; _i_ < _exist_size_[_p_];) { const int32_t
 _idx_ = _exists_[_p_][_i_]; _visit_->node[_visit_->size
].index = ((_idx_)); _visit_->node[_visit_->size].term =
 ((_incomings_[_idx_].d)); ++_visit_->size;; if (_incomings_
[_idx_].d) { ++_d_; _incomings_[_idx_].r = 7; } if ((exec_info
)[_idx_].outgoings) { if ((exec_info)[_idx_].outgoings->rnum
 == 1) { const int d = *(int*)((void*)(((char*)(((exec_info)[
_idx_].outgoings)->data)) + (size_t)((exec_info)[_idx_].outgoings
)->rsize * (size_t)(0))); --_incomings_[d].c; if (_incomings_
[d].c == 0 && _incomings_[d].r == 6 && _d_ <
 (destination_size)) { _exists_[_p_][_i_] = d; continue; } } else
 for (_j_ = 0; _j_ < (exec_info)[_idx_].outgoings->rnum
; _j_++) { const int d = *(int*)((void*)(((char*)(((exec_info
)[_idx_].outgoings)->data)) + (size_t)((exec_info)[_idx_].
outgoings)->rsize * (size_t)(_j_))); --_incomings_[d].c; if
 (_incomings_[d].c == 0 && _incomings_[d].r == 6 &&
 _d_ < (destination_size)) { ((void) sizeof ((_exist_size_
[_q_] < (exec_info_size)) ? 1 : 0), __extension__ ({ if (_exist_size_
[_q_] < (exec_info_size)) ; else __assert_fail ("_exist_size_[_q_] < (exec_info_size)"
, "ccv_cnnp_model.c", 1987, __extension__ __PRETTY_FUNCTION__
); })); _exists_[_q_][_exist_size_[_q_]] = d; ++_exist_size_[
_q_]; } } } ++_i_; } ((_i_) = (_p_), (_p_) = (_q_), (_q_) = (
_i_)); } for (_i_ = 0; _i_ < (destination_size); _i_++) { (
(void) sizeof (((destinations)[_i_].graph == compiled_data->
graph) ? 1 : 0), __extension__ ({ if ((destinations)[_i_].graph
 == compiled_data->graph) ; else __assert_fail ("(destinations)[_i_].graph == compiled_data->graph"
, "ccv_cnnp_model.c", 1987, __extension__ __PRETTY_FUNCTION__
); })); if (_incomings_[(destinations)[_i_].d].r == 7) continue
; if (!(0)) { ((void) sizeof ((_incomings_[(destinations)[_i_
].d].c == 0) ? 1 : 0), __extension__ ({ if (_incomings_[(destinations
)[_i_].d].c == 0) ; else __assert_fail ("_incomings_[(destinations)[_i_].d].c == 0"
, "ccv_cnnp_model.c", 1987, __extension__ __PRETTY_FUNCTION__
); })); } else if (_incomings_[(destinations)[_i_].d].c > 0
) continue; _visit_->node[_visit_->size].index = (((destinations
)[_i_].d)); _visit_->node[_visit_->size].term = ((_incomings_
[(destinations)[_i_].d].d)); ++_visit_->size;; } if (_heap_mem_
) free(_incomings_); } while (0);; ((void) sizeof ((_visit_->
size <= (exec_info_size)) ? 1 : 0), __extension__ ({ if (_visit_
->size <= (exec_info_size)) ; else __assert_fail ("_visit_->size <= (exec_info_size)"
, "ccv_cnnp_model.c", 1987, __extension__ __PRETTY_FUNCTION__
); })); _visit_; });
1988	// Find any missing nodes to be added as source. Right now, these are only set nodes.
1989	ccv_nnc_graph_visit_for(visit, exec_info, node, idx){ int _i_; for (_i_ = 0; _i_ < (visit)->size; _i_++) { const
 int idx __attribute__((unused)) = (visit)->node[_i_].index
; const int _node_unused_ __attribute__((unused)) = (visit)->
node[_i_].term; typeof ((exec_info)) const node __attribute__
((unused)) = (exec_info) + idx; {
1990		if (!(visited[(idx >> 5)] & (1u << (idx & 31))))
1991		{
1992			assert(exec_info[idx].cmd.cmd == CCV_NNC_SET_FORWARD)((void) sizeof ((exec_info[idx].cmd.cmd == CCV_NNC_SET_FORWARD
) ? 1 : 0), __extension__ ({ if (exec_info[idx].cmd.cmd == CCV_NNC_SET_FORWARD
) ; else __assert_fail ("exec_info[idx].cmd.cmd == CCV_NNC_SET_FORWARD"
, "ccv_cnnp_model.c", 1992, __extension__ __PRETTY_FUNCTION__
); }));
1993			if (exec_info[idx].cmd.info.blas.a[0] == 0) // Special-casing for empty out the tensor set function, not for the set grad to 1 one.
1994				ccv_array_add_unique_int(backward_from, idx);
1995		}
1996	} ccv_nnc_graph_visit_endfor} }
1997	ccv_nnc_graph_visit_free(visit);
1998	ccfreefree(visited);
1999	if (backward_from->rnum != compiled_data->backward.from_op_size) // If it doesn't match, need to redo this.
2000	{
2001		compiled_data->backward.from_op_size = backward_from->rnum;
2002		compiled_data->backward.from_ops = (ccv_nnc_graph_exec_t*)ccreallocrealloc(compiled_data->backward.from_ops, sizeof(ccv_nnc_graph_exec_t) * backward_from->rnum);
2003		for (i = 0; i < backward_from->rnum; i++)
2004			compiled_data->backward.from_ops[i] = (ccv_nnc_graph_exec_t){
2005				.d = *(int*)ccv_array_get(backward_from, i)((void*)(((char*)((backward_from)->data)) + (size_t)(backward_from
)->rsize * (size_t)(i))),
2006				.graph = compiled_data->graph,
2007			};
2008	}
2009	ccv_array_free(backward_from);
2010	ccv_nnc_graph_set_default_static_schedule(compiled_data->graph, compiled_data->stream_type, model->max_stream_count);
2011	ccv_nnc_graph_autotune(compiled_data->graph, model->workspace_size, 0, TRAVERSE_FULL0,0,0,0);
2012}
2013 
2014void ccv_cnnp_model_dry_run(ccv_cnnp_model_t* const model, const ccv_cnnp_evaluate_param_t params, ccv_nnc_tensor_t* const* const inputs, const int input_size, ccv_nnc_tensor_t* const* const outputs, const int output_size)
2015{
2016	ccv_cnnp_compiled_data_t* const compiled_data = model->compiled_data;
2017	assert(compiled_data)((void) sizeof ((compiled_data) ? 1 : 0), __extension__ ({ if
 (compiled_data) ; else __assert_fail ("compiled_data", "ccv_cnnp_model.c"
, 2017, __extension__ __PRETTY_FUNCTION__); }));
2018	const int parallel_count = ccv_max(model->parallel_count, 1)({ typeof (model->parallel_count) _a = (model->parallel_count
); typeof (1) _b = (1); (_a > _b) ? _a : _b; });
2019	assert(output_size == model->output_size * parallel_count)((void) sizeof ((output_size == model->output_size * parallel_count
) ? 1 : 0), __extension__ ({ if (output_size == model->output_size
 * parallel_count) ; else __assert_fail ("output_size == model->output_size * parallel_count"
, "ccv_cnnp_model.c", 2019, __extension__ __PRETTY_FUNCTION__
); }));
2020	assert(input_size == model->input_size * parallel_count)((void) sizeof ((input_size == model->input_size * parallel_count
) ? 1 : 0), __extension__ ({ if (input_size == model->input_size
 * parallel_count) ; else __assert_fail ("input_size == model->input_size * parallel_count"
, "ccv_cnnp_model.c", 2020, __extension__ __PRETTY_FUNCTION__
); }));
2021	assert(model->graph)((void) sizeof ((model->graph) ? 1 : 0), __extension__ ({ if
 (model->graph) ; else __assert_fail ("model->graph", "ccv_cnnp_model.c"
, 2021, __extension__ __PRETTY_FUNCTION__); }));
2022	const int target_gradient_mode = _ccv_cnnp_is_disable_outgrad_all(params.disable_outgrad, model->input_size) ? CCV_CNNP_COMPILED_DATA_GRADIENT_TRAINABLES : CCV_CNNP_COMPILED_DATA_GRADIENT_TRAINABLES_AND_INPUTS;
2023	const int mode_mismatch = (params.requires_grad && (compiled_data->graph_mode != CCV_CNNP_MODEL_GRAPH_MULTISTAGE_MODE || compiled_data->gradient_mode != target_gradient_mode || compiled_data->disable_outgrad != params.disable_outgrad));
2024	if (!compiled_data->graph || mode_mismatch)
2025	{
2026		_ccv_cnnp_compiled_data_graph_free(compiled_data);
2027		if (mode_mismatch) // If mode mismatch, we need to redo the backward as well (no need to redo apply_gradients, it doesn't require target_gradient_mode or disable_outgrad.
2028			_ccv_cnnp_compiled_data_backward_free(compiled_data);
2029		if (params.requires_grad)
2030			_ccv_cnnp_model_multistage_jit_0(model, params.disable_outgrad, params.is_test, inputs, input_size, outputs, output_size);
2031		else
2032			_ccv_cnnp_model_multistage_no_grad_jit(model, inputs, input_size, outputs, output_size);
2033	} else {
2034		ccv_nnc_tensor_arena_clear_bindings(compiled_data->tensor_arena);
2035		assert((input_size % parallel_count) == 0)((void) sizeof (((input_size % parallel_count) == 0) ? 1 : 0)
, __extension__ ({ if ((input_size % parallel_count) == 0) ; else
 __assert_fail ("(input_size % parallel_count) == 0", "ccv_cnnp_model.c"
, 2035, __extension__ __PRETTY_FUNCTION__); }));
2036		const int input_size_per_p = input_size / parallel_count;
2037		_ccv_cnnp_bind_tensors_to_arena(compiled_data->tensor_arena, model->graph, model->inputs, inputs, input_size_per_p, parallel_count);
2038		assert((output_size % parallel_count) == 0)((void) sizeof (((output_size % parallel_count) == 0) ? 1 : 0
), __extension__ ({ if ((output_size % parallel_count) == 0) ;
 else __assert_fail ("(output_size % parallel_count) == 0", "ccv_cnnp_model.c"
, 2038, __extension__ __PRETTY_FUNCTION__); }));
2039		const int output_size_per_p = output_size / parallel_count;
2040		_ccv_cnnp_bind_tensors_to_arena(compiled_data->tensor_arena, model->graph, model->outputs, outputs, output_size_per_p, parallel_count);
2041	}
2042	if (compiled_data->is_test != params.is_test)
2043	{
2044		compiled_data->is_test = params.is_test;
2045		ccv_nnc_graph_exec_update_t update = {
2046			.parallel_count = parallel_count,
2047			.graph = model->graph,
2048			.graph_exec_arena = compiled_data->graph_exec_arena,
2049		};
2050		ccv_cnnp_model_set_is_test(model, params.is_test, _ccv_cnnp_cmd_update_for_execs, &update);
2051	}
2052}
2053 
2054void ccv_cnnp_model_evaluate(ccv_cnnp_model_t* const model, const ccv_cnnp_evaluate_param_t params, ccv_nnc_tensor_t* const* const inputs, const int input_size, ccv_nnc_tensor_t* const* const outputs, const int output_size, ccv_nnc_tensor_tape_t* const tensor_tape, ccv_nnc_stream_context_t* const stream_context)
2055{
2056	ccv_cnnp_compiled_data_t* const compiled_data = model->compiled_data;
2057	assert(compiled_data)((void) sizeof ((compiled_data) ? 1 : 0), __extension__ ({ if
 (compiled_data) ; else __assert_fail ("compiled_data", "ccv_cnnp_model.c"
, 2057, __extension__ __PRETTY_FUNCTION__); }));
2058	ccv_cnnp_model_dry_run(model, params, inputs, input_size, outputs, output_size);
2059	if (compiled_data->graph_mode == CCV_CNNP_MODEL_GRAPH_MULTISTAGE_MODE_NO_GRAD)
2060		ccv_nnc_graph_run_with_schedule(compiled_data->graph, 0, 0, tensor_tape, stream_context);
2061	else {
2062		if (!compiled_data->evaluate.schedule)
2063			compiled_data->evaluate.schedule = ccv_nnc_graph_static_schedule_new(compiled_data->graph, compiled_data->stream_type, model->max_stream_count, 0, 0, compiled_data->evaluate.to_ops, compiled_data->evaluate.to_op_size);
2064		ccv_nnc_graph_run_with_schedule(compiled_data->graph, 0, compiled_data->evaluate.schedule, tensor_tape, stream_context);
2065	}
2066}
2067 
2068// Compile the graph to run ccv_cnnp_model_backward after ccv_cnnp_model_evaluate with requires_grad = true (MULTISTAGE_MODE).
2069// Particularly, this method compiles the accumulator graph.
2070static void _ccv_cnnp_model_multistage_jit_1(ccv_cnnp_model_t* const model)
2071{
2072	ccv_cnnp_compiled_data_t* const compiled_data = model->compiled_data;
2073	assert(compiled_data)((void) sizeof ((compiled_data) ? 1 : 0), __extension__ ({ if
 (compiled_data) ; else __assert_fail ("compiled_data", "ccv_cnnp_model.c"
, 2073, __extension__ __PRETTY_FUNCTION__); }));
2074	assert(compiled_data->graph_mode == CCV_CNNP_MODEL_GRAPH_MULTISTAGE_MODE)((void) sizeof ((compiled_data->graph_mode == CCV_CNNP_MODEL_GRAPH_MULTISTAGE_MODE
) ? 1 : 0), __extension__ ({ if (compiled_data->graph_mode
 == CCV_CNNP_MODEL_GRAPH_MULTISTAGE_MODE) ; else __assert_fail
 ("compiled_data->graph_mode == CCV_CNNP_MODEL_GRAPH_MULTISTAGE_MODE"
, "ccv_cnnp_model.c", 2074, __extension__ __PRETTY_FUNCTION__
); }));
2075	ccv_nnc_symbolic_graph_t* accum = ccv_nnc_symbolic_graph_new();
2076	const int parallel_count = ccv_max(model->parallel_count, 1)({ typeof (model->parallel_count) _a = (model->parallel_count
); typeof (1) _b = (1); (_a > _b) ? _a : _b; });
2077	const int parameter_size = compiled_data->parameters->rnum;
2078	int i, j;
2079	compiled_data->backward.gradients = (ccv_nnc_tensor_symbol_t*)ccmallocmalloc(sizeof(ccv_nnc_tensor_symbol_t) * parameter_size * parallel_count * 3);
2080	compiled_data->backward.accum_gradients = compiled_data->backward.gradients + parameter_size * parallel_count;
2081	compiled_data->backward.updated_accum_gradients = compiled_data->backward.accum_gradients + parameter_size * parallel_count;
2082	for (i = 0; i < parameter_size; i++)
2083		for (j = 0; j < parallel_count; j++)
2084			if (compiled_data->tensors.gradients[i + j * parameter_size])
2085			{
2086				const ccv_nnc_tensor_param_t info = compiled_data->tensors.gradients[i + j * parameter_size]->info;
2087				// Now, the old gradient is the accumulated gradient, getting new gradient tensor setup so we can collect them.
2088				compiled_data->tensors.accum_gradients[i + j * parameter_size] = compiled_data->tensors.gradients[i + j * parameter_size];
2089				compiled_data->tensors.gradients[i + j * parameter_size] = ccv_nnc_tensor_new(0, info, 0);
2090				ccv_nnc_tensor_symbol_t inputs[2];
2091				inputs[0] = compiled_data->backward.accum_gradients[i + j * parameter_size] = ccv_nnc_tensor_symbol_new(accum, info, 0);
2092				inputs[1] = compiled_data->backward.gradients[i + j * parameter_size] = ccv_nnc_tensor_symbol_new(accum, info, 0);
2093				ccv_nnc_tensor_symbol_t output = compiled_data->backward.updated_accum_gradients[i + j * parameter_size] = ccv_nnc_tensor_symbol_new(accum, info, 0);
2094				ccv_nnc_graph_exec_symbol_new(accum, CMD_EWSUM_FORWARD()ccv_nnc_cmd(CCV_NNC_EWSUM_FORWARD, 0, ccv_nnc_cmd_auto, 0), inputs, 2, &output, 1, 0);
2095			} else {
2096				compiled_data->backward.accum_gradients[i + j * parameter_size] = NO_TENSOR_SYMBOL(const ccv_nnc_tensor_symbol_t){.d = CCV_NNC_NO_TENSOR_SYMBOL
};
2097				compiled_data->backward.gradients[i + j * parameter_size] = NO_TENSOR_SYMBOL(const ccv_nnc_tensor_symbol_t){.d = CCV_NNC_NO_TENSOR_SYMBOL
};
2098				compiled_data->backward.updated_accum_gradients[i + j * parameter_size] = NO_TENSOR_SYMBOL(const ccv_nnc_tensor_symbol_t){.d = CCV_NNC_NO_TENSOR_SYMBOL
};
2099			}
2100	ccv_nnc_graph_exec_symbol_autogen(accum, 0, 0, CCV_NNC_AUTOGEN_ALL_EXECS | CCV_NNC_AUTOGEN_SOURCES_AND_DESTINATIONS);
2101	if (ccv_nnc_symbolic_graph_source_size(accum) == 0)
2102	{
2103		ccv_nnc_symbolic_graph_free(accum);
2104		// Create empty graph.
2105		compiled_data->backward.accum = ccv_nnc_graph_new();
2106		ccv_nnc_graph_topsort(compiled_data->backward.accum, 0, 0);
2107		return;
2108	}
2109	ccv_array_t* const tensor_binds = ccv_array_new(sizeof(ccv_nnc_tensor_bind_t), 0, 0);
2110	_ccv_cnnp_model_bind_tensors(accum, compiled_data->backward.accum_gradients, compiled_data->tensors.accum_gradients, parameter_size * parallel_count, 1, tensor_binds);
2111	_ccv_cnnp_model_bind_tensors(accum, compiled_data->backward.gradients, compiled_data->tensors.gradients, parameter_size * parallel_count, 1, tensor_binds);
2112	_ccv_cnnp_model_bind_tensors(accum, compiled_data->backward.updated_accum_gradients, compiled_data->tensors.accum_gradients, parameter_size * parallel_count, 1, tensor_binds);
2113	ccv_nnc_symbolic_graph_compile(accum, compiled_data->compile_params, (ccv_nnc_tensor_bind_t*)ccv_array_get(tensor_binds, 0)((void*)(((char*)((tensor_binds)->data)) + (size_t)(tensor_binds
)->rsize * (size_t)(0))), tensor_binds->rnum, 0, 0, SYMBOLIC_GRAPH_SOURCES(accum)ccv_nnc_symbolic_graph_sources(accum), ccv_nnc_symbolic_graph_source_size
(accum), SYMBOLIC_GRAPH_DESTINATIONS(accum)ccv_nnc_symbolic_graph_destinations(accum), ccv_nnc_symbolic_graph_destination_size
(accum), &compiled_data->backward.accum, &compiled_data->backward.tensor_arena, &compiled_data->backward.graph_exec_arena);
2114	ccv_nnc_symbolic_graph_free(accum);
2115	ccv_array_free(tensor_binds);
2116	ccv_nnc_graph_set_default_static_schedule(compiled_data->backward.accum, compiled_data->stream_type, model->max_stream_count);
2117}
2118 
2119void ccv_cnnp_model_backward(ccv_cnnp_model_t* const model, ccv_nnc_tensor_t* const* const ingrads, const int ingrad_size, ccv_nnc_tensor_t* const* const outgrads, const int outgrad_size, ccv_nnc_tensor_tape_t* const tensor_tape, ccv_nnc_stream_context_t* const stream_context)
2120{
2121	ccv_cnnp_compiled_data_t* const compiled_data = model->compiled_data;
2122	assert(compiled_data)((void) sizeof ((compiled_data) ? 1 : 0), __extension__ ({ if
 (compiled_data) ; else __assert_fail ("compiled_data", "ccv_cnnp_model.c"
, 2122, __extension__ __PRETTY_FUNCTION__); }));
2123	assert(compiled_data->graph_mode == CCV_CNNP_MODEL_GRAPH_MULTISTAGE_MODE)((void) sizeof ((compiled_data->graph_mode == CCV_CNNP_MODEL_GRAPH_MULTISTAGE_MODE
) ? 1 : 0), __extension__ ({ if (compiled_data->graph_mode
 == CCV_CNNP_MODEL_GRAPH_MULTISTAGE_MODE) ; else __assert_fail
 ("compiled_data->graph_mode == CCV_CNNP_MODEL_GRAPH_MULTISTAGE_MODE"
, "ccv_cnnp_model.c", 2123, __extension__ __PRETTY_FUNCTION__
); }));
2124	const int parallel_count = ccv_max(model->parallel_count, 1)({ typeof (model->parallel_count) _a = (model->parallel_count
); typeof (1) _b = (1); (_a > _b) ? _a : _b; });
2125	assert(ingrad_size == 0 || ingrad_size == model->output_size * parallel_count)((void) sizeof ((ingrad_size == 0 || ingrad_size == model->
output_size * parallel_count) ? 1 : 0), __extension__ ({ if (
ingrad_size == 0 || ingrad_size == model->output_size * parallel_count
) ; else __assert_fail ("ingrad_size == 0 || ingrad_size == model->output_size * parallel_count"
, "ccv_cnnp_model.c", 2125, __extension__ __PRETTY_FUNCTION__
); }));
2126	if (outgrad_size > 0)
2127		{ assert(outgrad_size == compiled_data->outgrad_size * parallel_count)((void) sizeof ((outgrad_size == compiled_data->outgrad_size
 * parallel_count) ? 1 : 0), __extension__ ({ if (outgrad_size
 == compiled_data->outgrad_size * parallel_count) ; else __assert_fail
 ("outgrad_size == compiled_data->outgrad_size * parallel_count"
, "ccv_cnnp_model.c", 2127, __extension__ __PRETTY_FUNCTION__
); })); }
2128	assert(model->graph)((void) sizeof ((model->graph) ? 1 : 0), __extension__ ({ if
 (model->graph) ; else __assert_fail ("model->graph", "ccv_cnnp_model.c"
, 2128, __extension__ __PRETTY_FUNCTION__); }));
2129	assert(compiled_data->graph)((void) sizeof ((compiled_data->graph) ? 1 : 0), __extension__
 ({ if (compiled_data->graph) ; else __assert_fail ("compiled_data->graph"
, "ccv_cnnp_model.c", 2129, __extension__ __PRETTY_FUNCTION__
); }));
2130	const int parameter_size = compiled_data->parameters->rnum;
2131	// If we need to accumulate the gradients now, do jit on accumulator.
2132	if (compiled_data->backward.count > 0)
2133	{
2134		if (!compiled_data->backward.accum)
2135			_ccv_cnnp_model_multistage_jit_1(model);
2136		else if (compiled_data->backward.count == 1) {
2137			//  On this round, we need to switch accumulated gradients with gradients (so we can do accumulation properly).
2138			int i;
2139			for (i = 0; i < parameter_size * parallel_count; i++)
2140			{
2141				ccv_nnc_tensor_t* tensor;
2142				CCV_SWAP(compiled_data->tensors.accum_gradients[i], compiled_data->tensors.gradients[i], tensor)((tensor) = (compiled_data->tensors.accum_gradients[i]), (
compiled_data->tensors.accum_gradients[i]) = (compiled_data
->tensors.gradients[i]), (compiled_data->tensors.gradients
[i]) = (tensor));
2143			}
2144			if (compiled_data->backward.tensor_arena)
2145			{
2146				ccv_nnc_tensor_arena_clear_bindings(compiled_data->backward.tensor_arena);
2147				// Do rebind in case we messed up the binding (we switch accum_gradients and gradients).
2148				_ccv_cnnp_bind_tensors_to_arena(compiled_data->backward.tensor_arena, 0, compiled_data->backward.gradients, compiled_data->tensors.gradients, parameter_size * parallel_count, 1);
2149				_ccv_cnnp_bind_tensors_to_arena(compiled_data->backward.tensor_arena, 0, compiled_data->backward.accum_gradients, compiled_data->tensors.accum_gradients, parameter_size * parallel_count, 1);
2150				_ccv_cnnp_bind_tensors_to_arena(compiled_data->backward.tensor_arena, 0, compiled_data->backward.updated_accum_gradients, compiled_data->tensors.accum_gradients, parameter_size * parallel_count, 1);
2151			}
2152		}
2153	}
2154	const int ingrad_size_per_p = model->output_size;
2155	const int outgrad_size_per_p = compiled_data->outgrad_size;
2156	int i, j;
2157	for (i = 0; i < ingrad_size_per_p; i++)
2158	{
2159		const ccv_nnc_tensor_symbol_t ingrad = ccv_nnc_tensor_symbol_for_backward(model->graph, compiled_data->f[i]);
2160		if (!ingrad_size || !ingrads || ingrads[i] == 0)
2161		{
2162			// Set it to 1 if it is not specified.
2163			ccv_nnc_tensor_t* const ingrad_tensor = ccv_nnc_tensor_from_symbol(compiled_data->tensor_arena, ingrad);
2164			if (ingrad_tensor)
2165				ccv_nnc_cmd_exec(CMD_SET_FORWARD(1)ccv_nnc_cmd(CCV_NNC_SET_FORWARD, 0, (ccv_nnc_cmd_param_t){.size
={.dim={1,1,1}},.blas={.a={1,}}}, 0), ccv_nnc_no_hint, 0, 0, 0, TENSOR_LIST(ingrad_tensor)(ccv_nnc_tensor_t* []){ingrad_tensor}, (1 +1 +0 +0 +0 +0 +0 +
0 +0 +0 +0 +0 +0 +0 +0 +0 +0 +0 +0 +0 +0 +0 +0 +0 +0 +0 +0 +0
 +0 +0 +0 +0 +0 +0 +0 +0 +0 +0 +0 +0 +0 +0 +0 +0 +0 +0 +0 +0 +
0 +0 +0 +0 +0 +0 +0 +0 +0 +0 +0 +0 +0 +0 +0 +0 -1), stream_context);
2166			for (j = 1; j < parallel_count; j++)
2167			{
2168				ccv_nnc_tensor_t* const ingrad_tensor = ccv_nnc_tensor_from_symbol(compiled_data->tensor_arena, ccv_nnc_tensor_symbol_copy(model->graph, ingrad, j));
2169				if (ingrad_tensor)
2170					ccv_nnc_cmd_exec(CMD_SET_FORWARD(1)ccv_nnc_cmd(CCV_NNC_SET_FORWARD, 0, (ccv_nnc_cmd_param_t){.size
={.dim={1,1,1}},.blas={.a={1,}}}, 0), ccv_nnc_no_hint, 0, 0, 0, TENSOR_LIST(ingrad_tensor)(ccv_nnc_tensor_t* []){ingrad_tensor}, (1 +1 +0 +0 +0 +0 +0 +
0 +0 +0 +0 +0 +0 +0 +0 +0 +0 +0 +0 +0 +0 +0 +0 +0 +0 +0 +0 +0
 +0 +0 +0 +0 +0 +0 +0 +0 +0 +0 +0 +0 +0 +0 +0 +0 +0 +0 +0 +0 +
0 +0 +0 +0 +0 +0 +0 +0 +0 +0 +0 +0 +0 +0 +0 +0 -1), stream_context);
2171			}
2172		} else {
2173			// Make sure the length matches, in case it is an alias.
2174			assert(ccv_nnc_tensor_count(ingrads[i]->info) == ccv_nnc_tensor_count(ccv_nnc_tensor_symbol_params(model->graph, ingrad)))((void) sizeof ((ccv_nnc_tensor_count(ingrads[i]->info) ==
 ccv_nnc_tensor_count(ccv_nnc_tensor_symbol_params(model->
graph, ingrad))) ? 1 : 0), __extension__ ({ if (ccv_nnc_tensor_count
(ingrads[i]->info) == ccv_nnc_tensor_count(ccv_nnc_tensor_symbol_params
(model->graph, ingrad))) ; else __assert_fail ("ccv_nnc_tensor_count(ingrads[i]->info) == ccv_nnc_tensor_count(ccv_nnc_tensor_symbol_params(model->graph, ingrad))"
, "ccv_cnnp_model.c", 2174, __extension__ __PRETTY_FUNCTION__
); }));
2175			ccv_nnc_tensor_bind_symbol(compiled_data->tensor_arena, ingrad, ingrads[i]);
2176			for (j = 1; j < parallel_count; j++)
2177				ccv_nnc_tensor_bind_symbol(compiled_data->tensor_arena, ccv_nnc_tensor_symbol_copy(model->graph, ingrad, j), ingrads[i + ingrad_size_per_p * j]);
2178		}
2179	}
2180	if (outgrad_size > 0)
2181	{
2182		assert(compiled_data->gradient_mode == CCV_CNNP_COMPILED_DATA_GRADIENT_TRAINABLES_AND_INPUTS && "shouldn't pass disable_outgrad to ccv_cnnp_model_evaluate before if you plan to compute outgrad")((void) sizeof ((compiled_data->gradient_mode == CCV_CNNP_COMPILED_DATA_GRADIENT_TRAINABLES_AND_INPUTS
 && "shouldn't pass disable_outgrad to ccv_cnnp_model_evaluate before if you plan to compute outgrad"
) ? 1 : 0), __extension__ ({ if (compiled_data->gradient_mode
 == CCV_CNNP_COMPILED_DATA_GRADIENT_TRAINABLES_AND_INPUTS &&
 "shouldn't pass disable_outgrad to ccv_cnnp_model_evaluate before if you plan to compute outgrad"
) ; else __assert_fail ("compiled_data->gradient_mode == CCV_CNNP_COMPILED_DATA_GRADIENT_TRAINABLES_AND_INPUTS && \"shouldn't pass disable_outgrad to ccv_cnnp_model_evaluate before if you plan to compute outgrad\""
, "ccv_cnnp_model.c", 2182, __extension__ __PRETTY_FUNCTION__
); }));
2183		for (i = 0; i < outgrad_size_per_p; i++)
2184			if (outgrads[i])
2185			{
2186				const ccv_nnc_tensor_symbol_t outgrad = compiled_data->outgrads[i];
2187				ccv_nnc_tensor_bind_symbol(compiled_data->tensor_arena, outgrad, outgrads[i]);
2188				for (j = 1; j < parallel_count; j++)
2189					ccv_nnc_tensor_bind_symbol(compiled_data->tensor_arena, ccv_nnc_tensor_symbol_copy(model->graph, outgrad, j), outgrads[i + outgrad_size_per_p * j]);
2190			}
2191	} else {
2192		assert(compiled_data->gradient_mode == CCV_CNNP_COMPILED_DATA_GRADIENT_TRAINABLES ||((void) sizeof ((compiled_data->gradient_mode == CCV_CNNP_COMPILED_DATA_GRADIENT_TRAINABLES
 || compiled_data->gradient_mode == CCV_CNNP_COMPILED_DATA_GRADIENT_TRAINABLES_AND_INPUTS
) ? 1 : 0), __extension__ ({ if (compiled_data->gradient_mode
 == CCV_CNNP_COMPILED_DATA_GRADIENT_TRAINABLES || compiled_data
->gradient_mode == CCV_CNNP_COMPILED_DATA_GRADIENT_TRAINABLES_AND_INPUTS
) ; else __assert_fail ("compiled_data->gradient_mode == CCV_CNNP_COMPILED_DATA_GRADIENT_TRAINABLES || compiled_data->gradient_mode == CCV_CNNP_COMPILED_DATA_GRADIENT_TRAINABLES_AND_INPUTS"
, "ccv_cnnp_model.c", 2193, __extension__ __PRETTY_FUNCTION__
); }))
2193			compiled_data->gradient_mode == CCV_CNNP_COMPILED_DATA_GRADIENT_TRAINABLES_AND_INPUTS)((void) sizeof ((compiled_data->gradient_mode == CCV_CNNP_COMPILED_DATA_GRADIENT_TRAINABLES
 || compiled_data->gradient_mode == CCV_CNNP_COMPILED_DATA_GRADIENT_TRAINABLES_AND_INPUTS
) ? 1 : 0), __extension__ ({ if (compiled_data->gradient_mode
 == CCV_CNNP_COMPILED_DATA_GRADIENT_TRAINABLES || compiled_data
->gradient_mode == CCV_CNNP_COMPILED_DATA_GRADIENT_TRAINABLES_AND_INPUTS
) ; else __assert_fail ("compiled_data->gradient_mode == CCV_CNNP_COMPILED_DATA_GRADIENT_TRAINABLES || compiled_data->gradient_mode == CCV_CNNP_COMPILED_DATA_GRADIENT_TRAINABLES_AND_INPUTS"
, "ccv_cnnp_model.c", 2193, __extension__ __PRETTY_FUNCTION__
); }));
2194	}
2195	// We need to rebind here because in ccv_cnnp_evaluate, we clear bindings, that will reset all bindings for the gradients.
2196	// For parameters and internals these are fine because when we clear bindings, it restores to original bindings, which are these
2197	// parameters and internals. The same cannot be said for gradients due to the accum_gradients switching.
2198	_ccv_cnnp_bind_tensors_to_arena(compiled_data->tensor_arena, model->graph, compiled_data->gradients, compiled_data->tensors.gradients, parameter_size, parallel_count);
2199	if (!compiled_data->backward.schedule)
2200		compiled_data->backward.schedule = ccv_nnc_graph_static_schedule_new(compiled_data->graph, compiled_data->stream_type, model->max_stream_count, compiled_data->backward.from_ops, compiled_data->backward.from_op_size, 0, 0);
2201	// Run the backward pass.
2202	ccv_nnc_graph_run_with_schedule(compiled_data->graph, 0, compiled_data->backward.schedule, tensor_tape, stream_context);
2203	// If we need to run accumulation round, do that now.
2204	if (compiled_data->backward.count > 0)
2205		ccv_nnc_graph_run_with_schedule(compiled_data->backward.accum, 0, 0, 0, stream_context);
2206	// Update the count, this determines whether we need to accumulate or not.
2207	++compiled_data->backward.count;
2208}
2209 
2210// Compile the graph to run ccv_cnnp_model_apply_gradients after ccv_cnnp_model_backward (MULTISTAGE_MODE).
2211// Particularly, this method compiles the parameter update graph.
2212static void _ccv_cnnp_model_multistage_jit_2(ccv_cnnp_model_t* const model)
2213{
2214	ccv_cnnp_compiled_data_t* const compiled_data = model->compiled_data;
2215	assert(compiled_data->graph_mode == CCV_CNNP_MODEL_GRAPH_MULTISTAGE_MODE)((void) sizeof ((compiled_data->graph_mode == CCV_CNNP_MODEL_GRAPH_MULTISTAGE_MODE
) ? 1 : 0), __extension__ ({ if (compiled_data->graph_mode
 == CCV_CNNP_MODEL_GRAPH_MULTISTAGE_MODE) ; else __assert_fail
 ("compiled_data->graph_mode == CCV_CNNP_MODEL_GRAPH_MULTISTAGE_MODE"
, "ccv_cnnp_model.c", 2215, __extension__ __PRETTY_FUNCTION__
); }));
2216	const int parallel_count = ccv_max(model->parallel_count, 1)({ typeof (model->parallel_count) _a = (model->parallel_count
); typeof (1) _b = (1); (_a > _b) ? _a : _b; });
2217	const int parameter_size = compiled_data->parameters->rnum;
2218	ccv_array_t* const tensor_binds = ccv_array_new(sizeof(ccv_nnc_tensor_bind_t), 0, 0);
2219	_ccv_cnnp_model_bind_tensors(model->graph, (ccv_nnc_tensor_symbol_t*)ccv_array_get(compiled_data->parameters, 0)((void*)(((char*)((compiled_data->parameters)->data)) +
 (size_t)(compiled_data->parameters)->rsize * (size_t)(
0))), compiled_data->tensors.parameters, parameter_size, parallel_count, tensor_binds);
2220	_ccv_cnnp_model_bind_tensors(model->graph, compiled_data->updated_parameters, compiled_data->tensors.parameters, parameter_size, parallel_count, tensor_binds);
2221	// Bind accumulated gradients.
2222	if (compiled_data->backward.count > 1)
2223		_ccv_cnnp_model_bind_tensors(model->graph, compiled_data->gradients, compiled_data->tensors.accum_gradients, parameter_size, parallel_count, tensor_binds);
2224	else
2225		_ccv_cnnp_model_bind_tensors(model->graph, compiled_data->gradients, compiled_data->tensors.gradients, parameter_size, parallel_count, tensor_binds);
2226	ccv_array_t* const apply_gradients_from = ccv_array_new(sizeof(int), 0, 0);
2227	int i, j;
2228	for (i = 0; i < compiled_data->backward.to_size; i++)
2229	{
2230		const int* tos;
2231		int to_size;
2232		ccv_nnc_graph_exec_symbol_to(model->graph, compiled_data->backward.tos[i], &tos, &to_size);
2233		for (j = 0; j < to_size; j++)
2234		{
2235			// Check if this is already show up in the backward graph, if that is the case, it won't be in the apply
2236			// gradients graph.
2237			const ccv_nnc_graph_exec_t exec = ccv_nnc_graph_exec_from_symbol(compiled_data->graph_exec_arena, (ccv_nnc_graph_exec_symbol_t){
2238				.d = tos[j],
2239				.graph = model->graph,
2240			});
2241			if (!exec.graph)
2242				ccv_array_add_unique_int(apply_gradients_from, tos[j]);
2243		}
2244	}
2245	const int from_size = apply_gradients_from->rnum;
2246	if (from_size == 0)
2247	{
2248		ccv_array_free(apply_gradients_from);
2249		ccv_array_free(tensor_binds);
2250		return;
2251	}
2252	ccv_nnc_graph_exec_symbol_t* const froms = (ccv_nnc_graph_exec_symbol_t*)ccmallocmalloc(sizeof(ccv_nnc_graph_exec_symbol_t) * from_size);
2253	for (i = 0; i < from_size; i++)
2254		froms[i] = (ccv_nnc_graph_exec_symbol_t){
2255			.d = *(int*)ccv_array_get(apply_gradients_from, i)((void*)(((char*)((apply_gradients_from)->data)) + (size_t
)(apply_gradients_from)->rsize * (size_t)(i))),
2256			.graph = model->graph
2257		};
2258	ccv_array_free(apply_gradients_from);
2259	// It can only ends with updates on the parameters.
2260	ccv_array_t* const tos = ccv_array_new(sizeof(ccv_nnc_graph_exec_symbol_t), parameter_size * parallel_count, 0);
2261	for (i = 0;  i < parameter_size; i++)
2262	{
2263		if (compiled_data->update_nodes[i].d == CCV_NNC_NO_TENSOR_SYMBOL)
2264			continue;
2265		ccv_array_push(tos, &compiled_data->update_nodes[i]);
2266		for (j = 1; j < parallel_count; j++)
2267		{
2268			const ccv_nnc_graph_exec_symbol_t copy = ccv_nnc_graph_exec_symbol_copy(model->graph, compiled_data->update_nodes[i], j);
2269			ccv_array_push(tos, &copy);
2270		}
2271	}
2272	ccv_nnc_symbolic_graph_compile(model->graph, compiled_data->compile_params, (ccv_nnc_tensor_bind_t*)ccv_array_get(tensor_binds, 0)((void*)(((char*)((tensor_binds)->data)) + (size_t)(tensor_binds
)->rsize * (size_t)(0))), tensor_binds->rnum, 0, 0, froms, from_size, (ccv_nnc_graph_exec_symbol_t*)ccv_array_get(tos, 0)((void*)(((char*)((tos)->data)) + (size_t)(tos)->rsize *
 (size_t)(0))), tos->rnum, &compiled_data->apply_gradients.graph, &compiled_data->apply_gradients.tensor_arena, &compiled_data->apply_gradients.graph_exec_arena);
2273	ccv_array_free(tos);
2274	ccv_array_free(tensor_binds);
2275	ccfreefree(froms);
2276	const int max_saved_aux_size = compiled_data->minimize.max_saved_aux_size;
2277	for (i = 0; i < max_saved_aux_size * parameter_size; i++)
2278	{
2279		// Skip on no tensor.
2280		if (compiled_data->saved_aux[i].source.d == CCV_NNC_NO_TENSOR_SYMBOL)
2281			continue;
2282		ccv_nnc_tensor_t* const tensor = ccv_nnc_tensor_from_symbol(compiled_data->apply_gradients.tensor_arena, compiled_data->saved_aux[i].source);
2283		ccv_nnc_cmd_exec(CMD_SET_FORWARD(0)ccv_nnc_cmd(CCV_NNC_SET_FORWARD, 0, (ccv_nnc_cmd_param_t){.size
={.dim={1,1,1}},.blas={.a={0,}}}, 0), ccv_nnc_no_hint, 0, 0, 0, &tensor, 1, 0);
2284		for (j = 1; j < parallel_count; j++)
2285		{
2286			ccv_nnc_tensor_t* const copy = ccv_nnc_tensor_from_symbol(compiled_data->apply_gradients.tensor_arena, ccv_nnc_tensor_symbol_copy(model->graph, compiled_data->saved_aux[i].source, j));
2287			if (copy)
2288				ccv_nnc_cmd_exec(CMD_SET_FORWARD(0)ccv_nnc_cmd(CCV_NNC_SET_FORWARD, 0, (ccv_nnc_cmd_param_t){.size
={.dim={1,1,1}},.blas={.a={0,}}}, 0), ccv_nnc_no_hint, 0, 0, 0, &copy, 1, 0);
2289		}
2290	}
2291	ccv_nnc_graph_set_default_static_schedule(compiled_data->apply_gradients.graph, compiled_data->stream_type, model->max_stream_count);
2292}
2293 
2294void ccv_cnnp_model_apply_gradients(ccv_cnnp_model_t* const model, ccv_nnc_stream_context_t* const stream_context)
2295{
2296	ccv_cnnp_compiled_data_t* const compiled_data = model->compiled_data;
2297	assert(compiled_data)((void) sizeof ((compiled_data) ? 1 : 0), __extension__ ({ if
 (compiled_data) ; else __assert_fail ("compiled_data", "ccv_cnnp_model.c"
, 2297, __extension__ __PRETTY_FUNCTION__); }));
2298	assert(compiled_data->graph_mode == CCV_CNNP_MODEL_GRAPH_MULTISTAGE_MODE)((void) sizeof ((compiled_data->graph_mode == CCV_CNNP_MODEL_GRAPH_MULTISTAGE_MODE
) ? 1 : 0), __extension__ ({ if (compiled_data->graph_mode
 == CCV_CNNP_MODEL_GRAPH_MULTISTAGE_MODE) ; else __assert_fail
 ("compiled_data->graph_mode == CCV_CNNP_MODEL_GRAPH_MULTISTAGE_MODE"
, "ccv_cnnp_model.c", 2298, __extension__ __PRETTY_FUNCTION__
); }));
2299	const int parallel_count = ccv_max(model->parallel_count, 1)({ typeof (model->parallel_count) _a = (model->parallel_count
); typeof (1) _b = (1); (_a > _b) ? _a : _b; });
2300	assert(model->graph)((void) sizeof ((model->graph) ? 1 : 0), __extension__ ({ if
 (model->graph) ; else __assert_fail ("model->graph", "ccv_cnnp_model.c"
, 2300, __extension__ __PRETTY_FUNCTION__); }));
2301	assert(compiled_data->graph)((void) sizeof ((compiled_data->graph) ? 1 : 0), __extension__
 ({ if (compiled_data->graph) ; else __assert_fail ("compiled_data->graph"
, "ccv_cnnp_model.c", 2301, __extension__ __PRETTY_FUNCTION__
); }));
2302	// Skip if there is no backward pass.
2303	if (compiled_data->backward.count <= 0)
2304		return;
2305	// Skip if there is no parameters.
2306	if (compiled_data->parameters->rnum == 0)
2307	{
2308		compiled_data->backward.count = 0;
2309		return;
2310	}
2311	if (!compiled_data->apply_gradients.graph)
2312		_ccv_cnnp_model_multistage_jit_2(model);
2313	else {
2314		const int parameter_size = compiled_data->parameters->rnum;
2315		ccv_nnc_tensor_arena_clear_bindings(compiled_data->apply_gradients.tensor_arena);
2316		// Change to bind accum_gradients if we do gradient accumulation (run backward more than once).
2317		if (compiled_data->backward.count > 1)
2318			_ccv_cnnp_bind_tensors_to_arena(compiled_data->apply_gradients.tensor_arena, model->graph, compiled_data->gradients, compiled_data->tensors.accum_gradients, parameter_size, parallel_count);
2319		else
2320			_ccv_cnnp_bind_tensors_to_arena(compiled_data->apply_gradients.tensor_arena, model->graph, compiled_data->gradients, compiled_data->tensors.gradients, parameter_size, parallel_count);
2321	}
2322	if (compiled_data->apply_gradients.graph)
2323		ccv_nnc_graph_run_with_schedule(compiled_data->apply_gradients.graph, 0, 0, 0, stream_context);
2324	// Reset backward count to 0.
2325	compiled_data->backward.count = 0;
2326}
2327 
2328void ccv_cnnp_model_set_parameter(ccv_cnnp_model_t* const model, const ccv_cnnp_model_io_t parameter, const ccv_nnc_tensor_t* const tensor)
2329{
2330	ccv_cnnp_compiled_data_t* const compiled_data = model->compiled_data;
2331	const int param_sel = parameter->param_sel > 0 ? parameter->param_sel - 1 : parameter->param_sel;
2332	assert(parameter->param_sel != 0)((void) sizeof ((parameter->param_sel != 0) ? 1 : 0), __extension__
 ({ if (parameter->param_sel != 0) ; else __assert_fail ("parameter->param_sel != 0"
, "ccv_cnnp_model.c", 2332, __extension__ __PRETTY_FUNCTION__
); }));
2333	const int tensors_init = !!compiled_data->tensors_init.v;
2334	int this_tensor_init = tensors_init;
2335	if (!tensors_init)
2336		ccv_cnnp_model_tensors_init_0(model, compiled_data);
2337	else if ((uintptr_t)compiled_data->tensors_init.v & (uintptr_t)1)
2338	// Check if it is not fully allocated, if it is not, init_1.
2339		this_tensor_init = 0;
2340	ccv_array_t* const parameter_indices = ccv_array_new(sizeof(int), 0, 0);
2341	ccv_cnnp_model_add_to_parameter_indices(parameter->model, param_sel, parameter_indices);
2342	const int param_ref = parameter->param_ref > 0 ? parameter->param_ref - 1 : parameter->param_ref;
2343	if (param_ref < 0)
2344		{ assert(parameter_indices->rnum == 1)((void) sizeof ((parameter_indices->rnum == 1) ? 1 : 0), __extension__
 ({ if (parameter_indices->rnum == 1) ; else __assert_fail
 ("parameter_indices->rnum == 1", "ccv_cnnp_model.c", 2344
, __extension__ __PRETTY_FUNCTION__); })); }
2345	else
2346		{ assert(param_ref < parameter_indices->rnum)((void) sizeof ((param_ref < parameter_indices->rnum) ?
 1 : 0), __extension__ ({ if (param_ref < parameter_indices
->rnum) ; else __assert_fail ("param_ref < parameter_indices->rnum"
, "ccv_cnnp_model.c", 2346, __extension__ __PRETTY_FUNCTION__
); })); }
2347	const int d = *(int*)ccv_array_get(parameter_indices, param_ref >= 0 ? param_ref : 0)((void*)(((char*)((parameter_indices)->data)) + (size_t)(parameter_indices
)->rsize * (size_t)(param_ref >= 0 ? param_ref : 0)));
2348	ccv_array_free(parameter_indices);
2349	const int parameter_size = compiled_data->parameters->rnum;
2350	assert(d >= 0)((void) sizeof ((d >= 0) ? 1 : 0), __extension__ ({ if (d >=
 0) ; else __assert_fail ("d >= 0", "ccv_cnnp_model.c", 2350
, __extension__ __PRETTY_FUNCTION__); }));
2351	assert(d < parameter_size)((void) sizeof ((d < parameter_size) ? 1 : 0), __extension__
 ({ if (d < parameter_size) ; else __assert_fail ("d < parameter_size"
, "ccv_cnnp_model.c", 2351, __extension__ __PRETTY_FUNCTION__
); }));
2352	const int parallel_count = _ccv_cnnp_compiled_data_parallel_count(model, compiled_data);
2353	int i;
2354	if (!this_tensor_init)
2355	{
2356		if (compiled_data->tensors.parameters[d])
2357		{
2358			for (i = 1; i < parallel_count; i++)
2359				{ assert(compiled_data->tensors.parameters[d + i * parameter_size])((void) sizeof ((compiled_data->tensors.parameters[d + i *
 parameter_size]) ? 1 : 0), __extension__ ({ if (compiled_data
->tensors.parameters[d + i * parameter_size]) ; else __assert_fail
 ("compiled_data->tensors.parameters[d + i * parameter_size]"
, "ccv_cnnp_model.c", 2359, __extension__ __PRETTY_FUNCTION__
); })); }
2360			this_tensor_init = 1;
2361		} else {
2362			const ccv_nnc_tensor_symbol_t parameter = *(ccv_nnc_tensor_symbol_t*)ccv_array_get(compiled_data->parameters, d)((void*)(((char*)((compiled_data->parameters)->data)) +
 (size_t)(compiled_data->parameters)->rsize * (size_t)(
d)));
2363			ccv_nnc_tensor_param_t info = ccv_nnc_tensor_symbol_params(parameter.graph, parameter);
2364			if (CCV_TENSOR_GET_DEVICE(info.type)((info.type) & 0xfff00) == CCV_COMPUTE_DEVICE_ANY)
2365				CCV_TENSOR_SET_DEVICE_ID(info.type, 0)(info.type) = (((info.type) & ~0xfff00) | (((0) & 0xfff
) << 8));
2366			const int device_id = CCV_TENSOR_GET_DEVICE_ID(info.type)(((info.type) & 0xfff00) >> 8);
2367			compiled_data->tensors.parameters[d] = ccv_nnc_tensor_new(0, info, 0);
2368			for (i = 1; i < parallel_count; i++)
2369			{
2370				if (i != device_id)
2371					CCV_TENSOR_SET_DEVICE_ID(info.type, i)(info.type) = (((info.type) & ~0xfff00) | (((i) & 0xfff
) << 8));
2372				else
2373					CCV_TENSOR_SET_DEVICE_ID(info.type, 0)(info.type) = (((info.type) & ~0xfff00) | (((0) & 0xfff
) << 8));
2374				compiled_data->tensors.parameters[d + i * parameter_size] = ccv_nnc_tensor_new(0, info, 0);
2375			}
2376		}
2377	}
2378	ccv_nnc_tensor_t* const dest = CCV_NNC_TENSOR(compiled_data->tensors.parameters[d])((ccv_nnc_tensor_t*)((uintptr_t)(compiled_data->tensors.parameters
[d]) & ~(uintptr_t)1));
2379	assert(dest)((void) sizeof ((dest) ? 1 : 0), __extension__ ({ if (dest) ;
 else __assert_fail ("dest", "ccv_cnnp_model.c", 2379, __extension__
 __PRETTY_FUNCTION__); }));
2380	ccv_nnc_cmd_exec(CMD_DATA_TRANSFER_FORWARD()ccv_nnc_cmd(CCV_NNC_DATA_TRANSFER_FORWARD, 0, ccv_nnc_cmd_auto
, 0), ccv_nnc_no_hint, 0, TENSOR_LIST((ccv_nnc_tensor_t*)tensor)(ccv_nnc_tensor_t* []){(ccv_nnc_tensor_t*)tensor}, (1 +1 +0 +
0 +0 +0 +0 +0 +0 +0 +0 +0 +0 +0 +0 +0 +0 +0 +0 +0 +0 +0 +0 +0
 +0 +0 +0 +0 +0 +0 +0 +0 +0 +0 +0 +0 +0 +0 +0 +0 +0 +0 +0 +0 +
0 +0 +0 +0 +0 +0 +0 +0 +0 +0 +0 +0 +0 +0 +0 +0 +0 +0 +0 +0 -1
), TENSOR_LIST(dest)(ccv_nnc_tensor_t* []){dest}, (1 +1 +0 +0 +0 +0 +0 +0 +0 +0 +
0 +0 +0 +0 +0 +0 +0 +0 +0 +0 +0 +0 +0 +0 +0 +0 +0 +0 +0 +0 +0
 +0 +0 +0 +0 +0 +0 +0 +0 +0 +0 +0 +0 +0 +0 +0 +0 +0 +0 +0 +0 +
0 +0 +0 +0 +0 +0 +0 +0 +0 +0 +0 +0 +0 -1), 0);
2381	for (i = 1; i < parallel_count; i++)
2382	{
2383		ccv_nnc_tensor_t* const copy_tensor = CCV_NNC_TENSOR(compiled_data->tensors.parameters[d + i * parameter_size])((ccv_nnc_tensor_t*)((uintptr_t)(compiled_data->tensors.parameters
[d + i * parameter_size]) & ~(uintptr_t)1));
2384		if (copy_tensor)
2385			ccv_nnc_cmd_exec(CMD_DATA_TRANSFER_FORWARD()ccv_nnc_cmd(CCV_NNC_DATA_TRANSFER_FORWARD, 0, ccv_nnc_cmd_auto
, 0), ccv_nnc_no_hint, 0, TENSOR_LIST(dest)(ccv_nnc_tensor_t* []){dest}, (1 +1 +0 +0 +0 +0 +0 +0 +0 +0 +
0 +0 +0 +0 +0 +0 +0 +0 +0 +0 +0 +0 +0 +0 +0 +0 +0 +0 +0 +0 +0
 +0 +0 +0 +0 +0 +0 +0 +0 +0 +0 +0 +0 +0 +0 +0 +0 +0 +0 +0 +0 +
0 +0 +0 +0 +0 +0 +0 +0 +0 +0 +0 +0 +0 -1), TENSOR_LIST(copy_tensor)(ccv_nnc_tensor_t* []){copy_tensor}, (1 +1 +0 +0 +0 +0 +0 +0 +
0 +0 +0 +0 +0 +0 +0 +0 +0 +0 +0 +0 +0 +0 +0 +0 +0 +0 +0 +0 +0
 +0 +0 +0 +0 +0 +0 +0 +0 +0 +0 +0 +0 +0 +0 +0 +0 +0 +0 +0 +0 +
0 +0 +0 +0 +0 +0 +0 +0 +0 +0 +0 +0 +0 +0 +0 -1), 0);
2386	}
2387	// Mark this symbol as init'ed.
2388	const int s = ((ccv_nnc_tensor_symbol_t*)ccv_array_get(compiled_data->parameters, d)((void*)(((char*)((compiled_data->parameters)->data)) +
 (size_t)(compiled_data->parameters)->rsize * (size_t)(
d))))->d;
2389	uint32_t* const init_v = CCV_NNC_INIT_V(compiled_data->tensors_init.v)((uint32_t*)((uintptr_t)(compiled_data->tensors_init.v) &
 ~(uintptr_t)1));
2390	init_v[s >> 5] |= (1u << (s & 0x1f));
2391	// If we just allocated this tensor, now it is time to check if we need to mark it as fully allocated.
2392	if (!this_tensor_init)
2393	{
2394		if (ccv_cnnp_model_tensors_any_to_alloc(model, compiled_data))
2395			compiled_data->tensors_init.v = (uint32_t*)((uintptr_t)compiled_data->tensors_init.v | (uintptr_t)1);
2396		else // Remove the flag.
2397			compiled_data->tensors_init.v = CCV_NNC_INIT_V(compiled_data->tensors_init.v)((uint32_t*)((uintptr_t)(compiled_data->tensors_init.v) &
 ~(uintptr_t)1));
2398	}
2399}
2400 
2401void ccv_cnnp_model_parameter_copy(ccv_cnnp_model_t* const model, const ccv_cnnp_model_io_t parameter, ccv_nnc_tensor_t* const tensor)
2402{
2403	ccv_cnnp_compiled_data_t* const compiled_data = model->compiled_data;
2404	const int param_sel = parameter->param_sel > 0 ? parameter->param_sel - 1 : parameter->param_sel;
2405	assert(parameter->param_sel != 0)((void) sizeof ((parameter->param_sel != 0) ? 1 : 0), __extension__
 ({ if (parameter->param_sel != 0) ; else __assert_fail ("parameter->param_sel != 0"
, "ccv_cnnp_model.c", 2405, __extension__ __PRETTY_FUNCTION__
); }));
2406	assert(compiled_data->tensors.parameters)((void) sizeof ((compiled_data->tensors.parameters) ? 1 : 0
), __extension__ ({ if (compiled_data->tensors.parameters)
 ; else __assert_fail ("compiled_data->tensors.parameters"
, "ccv_cnnp_model.c", 2406, __extension__ __PRETTY_FUNCTION__
); }));
2407	ccv_array_t* const parameter_indices = ccv_array_new(sizeof(int), 0, 0);
2408	ccv_cnnp_model_add_to_parameter_indices(parameter->model, param_sel, parameter_indices);
2409	const int param_ref = parameter->param_ref > 0 ? parameter->param_ref - 1 : parameter->param_ref;
2410	if (param_ref < 0)
2411		{ assert(parameter_indices->rnum == 1)((void) sizeof ((parameter_indices->rnum == 1) ? 1 : 0), __extension__
 ({ if (parameter_indices->rnum == 1) ; else __assert_fail
 ("parameter_indices->rnum == 1", "ccv_cnnp_model.c", 2411
, __extension__ __PRETTY_FUNCTION__); })); }
2412	else
2413		{ assert(param_ref < parameter_indices->rnum)((void) sizeof ((param_ref < parameter_indices->rnum) ?
 1 : 0), __extension__ ({ if (param_ref < parameter_indices
->rnum) ; else __assert_fail ("param_ref < parameter_indices->rnum"
, "ccv_cnnp_model.c", 2413, __extension__ __PRETTY_FUNCTION__
); })); }
2414	const int d = *(int*)ccv_array_get(parameter_indices, param_ref >= 0 ? param_ref : 0)((void*)(((char*)((parameter_indices)->data)) + (size_t)(parameter_indices
)->rsize * (size_t)(param_ref >= 0 ? param_ref : 0)));
2415	ccv_array_free(parameter_indices);
2416	const int parameter_size = compiled_data->parameters->rnum;
2417	assert(d >= 0)((void) sizeof ((d >= 0) ? 1 : 0), __extension__ ({ if (d >=
 0) ; else __assert_fail ("d >= 0", "ccv_cnnp_model.c", 2417
, __extension__ __PRETTY_FUNCTION__); }));
2418	assert(d < parameter_size)((void) sizeof ((d < parameter_size) ? 1 : 0), __extension__
 ({ if (d < parameter_size) ; else __assert_fail ("d < parameter_size"
, "ccv_cnnp_model.c", 2418, __extension__ __PRETTY_FUNCTION__
); }));
2419	// We don't need to consider parallel_count, every parameter on each device is identical.
2420	ccv_nnc_tensor_t* const src = CCV_NNC_TENSOR(compiled_data->tensors.parameters[d])((ccv_nnc_tensor_t*)((uintptr_t)(compiled_data->tensors.parameters
[d]) & ~(uintptr_t)1));
2421	assert(src)((void) sizeof ((src) ? 1 : 0), __extension__ ({ if (src) ; else
 __assert_fail ("src", "ccv_cnnp_model.c", 2421, __extension__
 __PRETTY_FUNCTION__); }));
2422	ccv_nnc_cmd_exec(CMD_DATA_TRANSFER_FORWARD()ccv_nnc_cmd(CCV_NNC_DATA_TRANSFER_FORWARD, 0, ccv_nnc_cmd_auto
, 0), ccv_nnc_no_hint, 0, TENSOR_LIST(src)(ccv_nnc_tensor_t* []){src}, (1 +1 +0 +0 +0 +0 +0 +0 +0 +0 +0
 +0 +0 +0 +0 +0 +0 +0 +0 +0 +0 +0 +0 +0 +0 +0 +0 +0 +0 +0 +0 +
0 +0 +0 +0 +0 +0 +0 +0 +0 +0 +0 +0 +0 +0 +0 +0 +0 +0 +0 +0 +0
 +0 +0 +0 +0 +0 +0 +0 +0 +0 +0 +0 +0 -1), TENSOR_LIST(tensor)(ccv_nnc_tensor_t* []){tensor}, (1 +1 +0 +0 +0 +0 +0 +0 +0 +0
 +0 +0 +0 +0 +0 +0 +0 +0 +0 +0 +0 +0 +0 +0 +0 +0 +0 +0 +0 +0 +
0 +0 +0 +0 +0 +0 +0 +0 +0 +0 +0 +0 +0 +0 +0 +0 +0 +0 +0 +0 +0
 +0 +0 +0 +0 +0 +0 +0 +0 +0 +0 +0 +0 +0 -1), 0);
2423}
2424 
2425ccv_nnc_tensor_param_t ccv_cnnp_model_parameter_tensor_params(ccv_cnnp_model_t* const model, const ccv_cnnp_model_io_t parameter)
2426{
2427	ccv_cnnp_compiled_data_t* const compiled_data = model->compiled_data;
2428	const int param_sel = parameter->param_sel > 0 ? parameter->param_sel - 1 : parameter->param_sel;
2429	assert(parameter->param_sel != 0)((void) sizeof ((parameter->param_sel != 0) ? 1 : 0), __extension__
 ({ if (parameter->param_sel != 0) ; else __assert_fail ("parameter->param_sel != 0"
, "ccv_cnnp_model.c", 2429, __extension__ __PRETTY_FUNCTION__
); }));
2430	assert(compiled_data->tensors.parameters)((void) sizeof ((compiled_data->tensors.parameters) ? 1 : 0
), __extension__ ({ if (compiled_data->tensors.parameters)
 ; else __assert_fail ("compiled_data->tensors.parameters"
, "ccv_cnnp_model.c", 2430, __extension__ __PRETTY_FUNCTION__
); }));
2431	ccv_array_t* const parameter_indices = ccv_array_new(sizeof(int), 0, 0);
2432	ccv_cnnp_model_add_to_parameter_indices(parameter->model, param_sel, parameter_indices);
2433	const int param_ref = parameter->param_ref > 0 ? parameter->param_ref - 1 : parameter->param_ref;
2434	if (param_ref < 0)
2435		{ assert(parameter_indices->rnum == 1)((void) sizeof ((parameter_indices->rnum == 1) ? 1 : 0), __extension__
 ({ if (parameter_indices->rnum == 1) ; else __assert_fail
 ("parameter_indices->rnum == 1", "ccv_cnnp_model.c", 2435
, __extension__ __PRETTY_FUNCTION__); })); }
2436	else
2437		{ assert(param_ref < parameter_indices->rnum)((void) sizeof ((param_ref < parameter_indices->rnum) ?
 1 : 0), __extension__ ({ if (param_ref < parameter_indices
->rnum) ; else __assert_fail ("param_ref < parameter_indices->rnum"
, "ccv_cnnp_model.c", 2437, __extension__ __PRETTY_FUNCTION__
); })); }
2438	const int d = *(int*)ccv_array_get(parameter_indices, param_ref >= 0 ? param_ref : 0)((void*)(((char*)((parameter_indices)->data)) + (size_t)(parameter_indices
)->rsize * (size_t)(param_ref >= 0 ? param_ref : 0)));
2439	ccv_array_free(parameter_indices);
2440	const int parameter_size = compiled_data->parameters->rnum;
2441	assert(d >= 0)((void) sizeof ((d >= 0) ? 1 : 0), __extension__ ({ if (d >=
 0) ; else __assert_fail ("d >= 0", "ccv_cnnp_model.c", 2441
, __extension__ __PRETTY_FUNCTION__); }));
2442	assert(d < parameter_size)((void) sizeof ((d < parameter_size) ? 1 : 0), __extension__
 ({ if (d < parameter_size) ; else __assert_fail ("d < parameter_size"
, "ccv_cnnp_model.c", 2442, __extension__ __PRETTY_FUNCTION__
); }));
2443	// We don't need to consider parallel_count, every parameter on each device is identical.
2444	ccv_nnc_tensor_t* const tensor = CCV_NNC_TENSOR(compiled_data->tensors.parameters[d])((ccv_nnc_tensor_t*)((uintptr_t)(compiled_data->tensors.parameters
[d]) & ~(uintptr_t)1));
2445	assert(tensor)((void) sizeof ((tensor) ? 1 : 0), __extension__ ({ if (tensor
) ; else __assert_fail ("tensor", "ccv_cnnp_model.c", 2445, __extension__
 __PRETTY_FUNCTION__); }));
2446	return tensor->info;
2447}
2448 
2449const char* ccv_cnnp_model_parameter_name(ccv_cnnp_model_t* const model, const ccv_cnnp_model_io_t parameter)
2450{
2451	ccv_cnnp_compiled_data_t* const compiled_data = model->compiled_data;
2452	const int param_sel = parameter->param_sel > 0 ? parameter->param_sel - 1 : parameter->param_sel;
2453	assert(parameter->param_sel != 0)((void) sizeof ((parameter->param_sel != 0) ? 1 : 0), __extension__
 ({ if (parameter->param_sel != 0) ; else __assert_fail ("parameter->param_sel != 0"
, "ccv_cnnp_model.c", 2453, __extension__ __PRETTY_FUNCTION__
); }));
2454	ccv_array_t* const parameter_indices = ccv_array_new(sizeof(int), 0, 0);
2455	ccv_cnnp_model_add_to_parameter_indices(parameter->model, param_sel, parameter_indices);
2456	const int param_ref = parameter->param_ref > 0 ? parameter->param_ref - 1 : parameter->param_ref;
2457	if (param_ref < 0)
2458		{ assert(parameter_indices->rnum == 1)((void) sizeof ((parameter_indices->rnum == 1) ? 1 : 0), __extension__
 ({ if (parameter_indices->rnum == 1) ; else __assert_fail
 ("parameter_indices->rnum == 1", "ccv_cnnp_model.c", 2458
, __extension__ __PRETTY_FUNCTION__); })); }
2459	else
2460		{ assert(param_ref < parameter_indices->rnum)((void) sizeof ((param_ref < parameter_indices->rnum) ?
 1 : 0), __extension__ ({ if (param_ref < parameter_indices
->rnum) ; else __assert_fail ("param_ref < parameter_indices->rnum"
, "ccv_cnnp_model.c", 2460, __extension__ __PRETTY_FUNCTION__
); })); }
2461	const int d = *(int*)ccv_array_get(parameter_indices, param_ref >= 0 ? param_ref : 0)((void*)(((char*)((parameter_indices)->data)) + (size_t)(parameter_indices
)->rsize * (size_t)(param_ref >= 0 ? param_ref : 0)));
2462	ccv_array_free(parameter_indices);
2463	const int parameter_size = compiled_data->parameters->rnum;
2464	assert(d >= 0)((void) sizeof ((d >= 0) ? 1 : 0), __extension__ ({ if (d >=
 0) ; else __assert_fail ("d >= 0", "ccv_cnnp_model.c", 2464
, __extension__ __PRETTY_FUNCTION__); }));
2465	assert(d < parameter_size)((void) sizeof ((d < parameter_size) ? 1 : 0), __extension__
 ({ if (d < parameter_size) ; else __assert_fail ("d < parameter_size"
, "ccv_cnnp_model.c", 2465, __extension__ __PRETTY_FUNCTION__
); }));
2466	return *(char**)ccv_array_get(compiled_data->ids.parameters, d)((void*)(((char*)((compiled_data->ids.parameters)->data
)) + (size_t)(compiled_data->ids.parameters)->rsize * (
size_t)(d)));
2467}
2468 
2469int ccv_cnnp_model_parameter_count(ccv_cnnp_model_t* const model)
2470{
2471	assert(model->compiled_data)((void) sizeof ((model->compiled_data) ? 1 : 0), __extension__
 ({ if (model->compiled_data) ; else __assert_fail ("model->compiled_data"
, "ccv_cnnp_model.c", 2471, __extension__ __PRETTY_FUNCTION__
); }));
2472	ccv_cnnp_compiled_data_t* const compiled_data = model->compiled_data;
2473	return compiled_data->parameters->rnum;
2474}
2475 
2476uint64_t ccv_cnnp_model_parameters_size(ccv_cnnp_model_t* const model)
2477{
2478	assert(model->compiled_data)((void) sizeof ((model->compiled_data) ? 1 : 0), __extension__
 ({ if (model->compiled_data) ; else __assert_fail ("model->compiled_data"
, "ccv_cnnp_model.c", 2478, __extension__ __PRETTY_FUNCTION__
); }));
2479	ccv_cnnp_compiled_data_t* const compiled_data = model->compiled_data;
2480	const int parameter_size = compiled_data->parameters->rnum;
2481	int i;
2482	const ccv_nnc_symbolic_graph_t* const graph = model->graph;
2483	uint64_t size = 0;
2484	const int tensors_init = !!compiled_data->tensors_init.v;
2485	uint32_t* const init_v = tensors_init ? CCV_NNC_INIT_V(compiled_data->tensors_init.v)((uint32_t*)((uintptr_t)(compiled_data->tensors_init.v) &
 ~(uintptr_t)1)) : 0;
2486	for (i = 0; i < parameter_size; i++)
2487	{
2488		const int d = ((ccv_nnc_tensor_symbol_t*)ccv_array_get(compiled_data->parameters, i)((void*)(((char*)((compiled_data->parameters)->data)) +
 (size_t)(compiled_data->parameters)->rsize * (size_t)(
i))))->d;
2489		if (tensors_init && compiled_data->tensors.parameters && (init_v[d >> 5] | (1u << (d & 0x1f))) && compiled_data->tensors.parameters[i])
2490		{
2491			ccv_nnc_tensor_param_t params = compiled_data->tensors.parameters[i]->info;
2492			size += ccv_nnc_tensor_data_size(params);
2493			continue;
2494		}
2495		ccv_nnc_tensor_param_t params = ccv_nnc_tensor_symbol_params(graph, (ccv_nnc_tensor_symbol_t){
2496			.graph = graph,
2497			.d = d
2498		});
2499		size += ccv_nnc_tensor_data_size(params);
2500	}
2501	return size;
2502}
2503 
2504int ccv_cnnp_model_parameters_move(ccv_cnnp_model_t* const model, char** const names, ccv_nnc_tensor_t** const tensors, const int count, int type)
2505{
2506	assert(model->compiled_data)((void) sizeof ((model->compiled_data) ? 1 : 0), __extension__
 ({ if (model->compiled_data) ; else __assert_fail ("model->compiled_data"
, "ccv_cnnp_model.c", 2506, __extension__ __PRETTY_FUNCTION__
); }));
2507	ccv_cnnp_compiled_data_t* const compiled_data = model->compiled_data;
2508	if (count != compiled_data->parameters->rnum)
2509		return 0;
2510	if (CCV_TENSOR_GET_DEVICE(type)((type) & 0xfff00) == CCV_COMPUTE_DEVICE_ANY)
2511		CCV_TENSOR_SET_DEVICE_ID(type, 0)(type) = (((type) & ~0xfff00) | (((0) & 0xfff) <<
 8));
2512	int i;
2513	// We don't need to consider parallel_count, every parameter on each device is identical.
2514	for (i = 0; i < count; i++)
2515	{
2516		ccv_nnc_tensor_t* tensor = compiled_data->tensors.parameters[i];
2517		if ((uintptr_t)tensor & (uintptr_t)1) // If it is not owned. We don't do anything.
2518		{
2519			tensors[i] = 0;
2520			continue;
2521		}
2522		tensor = CCV_NNC_TENSOR(tensor)((ccv_nnc_tensor_t*)((uintptr_t)(tensor) & ~(uintptr_t)1)
);
2523		if (tensor->info.type == type)
2524			tensors[i] = tensor;
2525		else {
2526			ccv_nnc_tensor_param_t info = tensor->info;
2527			info.type = type;
2528			tensors[i] = ccv_nnc_tensor_new(0, info, 0); // Create this tensor, don't initiate copy yet.
2529		}
2530	}
2531	for (i = 0; i < count; i++)
2532	{
2533		ccv_nnc_tensor_t* tensor = compiled_data->tensors.parameters[i];
2534		if ((uintptr_t)tensor & (uintptr_t)1) // If it is not owned. We don't do anything.
2535			continue;
2536		tensor = CCV_NNC_TENSOR(tensor)((ccv_nnc_tensor_t*)((uintptr_t)(tensor) & ~(uintptr_t)1)
);
2537		// Now initiate transfer. We should do this one on a stream.
2538		if (tensor->info.type != type)
2539			ccv_nnc_cmd_exec(CMD_DATA_TRANSFER_FORWARD()ccv_nnc_cmd(CCV_NNC_DATA_TRANSFER_FORWARD, 0, ccv_nnc_cmd_auto
, 0), ccv_nnc_no_hint, 0, TENSOR_LIST(tensor)(ccv_nnc_tensor_t* []){tensor}, (1 +1 +0 +0 +0 +0 +0 +0 +0 +0
 +0 +0 +0 +0 +0 +0 +0 +0 +0 +0 +0 +0 +0 +0 +0 +0 +0 +0 +0 +0 +
0 +0 +0 +0 +0 +0 +0 +0 +0 +0 +0 +0 +0 +0 +0 +0 +0 +0 +0 +0 +0
 +0 +0 +0 +0 +0 +0 +0 +0 +0 +0 +0 +0 +0 -1), TENSOR_LIST(tensors[i])(ccv_nnc_tensor_t* []){tensors[i]}, (1 +1 +0 +0 +0 +0 +0 +0 +
0 +0 +0 +0 +0 +0 +0 +0 +0 +0 +0 +0 +0 +0 +0 +0 +0 +0 +0 +0 +0
 +0 +0 +0 +0 +0 +0 +0 +0 +0 +0 +0 +0 +0 +0 +0 +0 +0 +0 +0 +0 +
0 +0 +0 +0 +0 +0 +0 +0 +0 +0 +0 +0 +0 +0 +0 -1), 0);
2540	}
2541	// Copy names and remove parameters.
2542	for (i = 0; i < count; i++)
2543	{
2544		ccv_nnc_tensor_t* const tensor = compiled_data->tensors.parameters[i];
2545		if ((uintptr_t)tensor & (uintptr_t)1) // If it is not owned. We don't do anything.
2546		{
2547			names[i] = 0;
2548			continue;
2549		}
2550		const char* const name = *(char**)ccv_array_get(compiled_data->ids.parameters, i)((void*)(((char*)((compiled_data->ids.parameters)->data
)) + (size_t)(compiled_data->ids.parameters)->rsize * (
size_t)(i)));
2551		const size_t name_len = ccv_min(strnlen(name, 1023), 1023)({ typeof (strnlen(name, 1023)) _a = (strnlen(name, 1023)); typeof
 (1023) _b = (1023); (_a < _b) ? _a : _b; });
2552		names[i] = ccmallocmalloc(name_len + 1);
2553		names[i][name_len] = 0;
2554		memcpy(names[i], name, name_len);
2555		if (tensor->info.type == type)
2556			compiled_data->tensors.parameters[i] = 0; // Only move when it is moved.
2557	}
2558	return 1;
2559}
2560 
2561KHASH_MAP_INIT_STR(ccv_cnnp_parameter_id, int)typedef struct kh_ccv_cnnp_parameter_id_s { khint_t n_buckets
, size, n_occupied, upper_bound; khint32_t *flags; kh_cstr_t *
keys; int *vals; } kh_ccv_cnnp_parameter_id_t; static inline __attribute__
 ((__unused__)) kh_ccv_cnnp_parameter_id_t *kh_init_ccv_cnnp_parameter_id
(void) { return (kh_ccv_cnnp_parameter_id_t*)calloc(1,sizeof(
kh_ccv_cnnp_parameter_id_t)); } static inline __attribute__ (
(__unused__)) void kh_destroy_ccv_cnnp_parameter_id(kh_ccv_cnnp_parameter_id_t
 *h) { if (h) { free((void *)h->keys); free(h->flags); free
((void *)h->vals); free(h); } } static inline __attribute__
 ((__unused__)) void kh_clear_ccv_cnnp_parameter_id(kh_ccv_cnnp_parameter_id_t
 *h) { if (h && h->flags) { memset(h->flags, 0xaa
, ((h->n_buckets) < 16? 1 : (h->n_buckets)>>4)
 * sizeof(khint32_t)); h->size = h->n_occupied = 0; } }
 static inline __attribute__ ((__unused__)) khint_t kh_get_ccv_cnnp_parameter_id
(const kh_ccv_cnnp_parameter_id_t *h, kh_cstr_t key) { if (h->
n_buckets) { khint_t k, i, last, mask, step = 0; mask = h->
n_buckets - 1; k = __ac_X31_hash_string(key); i = k & mask
; last = i; while (!((h->flags[i>>4]>>((i&
0xfU)<<1))&2) && (((h->flags[i>>4]
>>((i&0xfU)<<1))&1) || !(strcmp(h->keys
[i], key) == 0))) { i = (i + (++step)) & mask; if (i == last
) return h->n_buckets; } return ((h->flags[i>>4]>>
((i&0xfU)<<1))&3)? h->n_buckets : i; } else return
 0; } static inline __attribute__ ((__unused__)) int kh_resize_ccv_cnnp_parameter_id
(kh_ccv_cnnp_parameter_id_t *h, khint_t new_n_buckets) { khint32_t
 *new_flags = 0; khint_t j = 1; { (--(new_n_buckets), (new_n_buckets
)|=(new_n_buckets)>>1, (new_n_buckets)|=(new_n_buckets)
>>2, (new_n_buckets)|=(new_n_buckets)>>4, (new_n_buckets
)|=(new_n_buckets)>>8, (new_n_buckets)|=(new_n_buckets)
>>16, ++(new_n_buckets)); if (new_n_buckets < 4) new_n_buckets
 = 4; if (h->size >= (khint_t)(new_n_buckets * __ac_HASH_UPPER
 + 0.5)) j = 0; else { new_flags = (khint32_t*)malloc(((new_n_buckets
) < 16? 1 : (new_n_buckets)>>4) * sizeof(khint32_t))
; if (!new_flags) return -1; memset(new_flags, 0xaa, ((new_n_buckets
) < 16? 1 : (new_n_buckets)>>4) * sizeof(khint32_t))
; if (h->n_buckets < new_n_buckets) { kh_cstr_t *new_keys
 = (kh_cstr_t*)realloc((void *)h->keys,new_n_buckets * sizeof
(kh_cstr_t)); if (!new_keys) { free(new_flags); return -1; } h
->keys = new_keys; if (1) { int *new_vals = (int*)realloc(
(void *)h->vals,new_n_buckets * sizeof(int)); if (!new_vals
) { free(new_flags); return -1; } h->vals = new_vals; } } }
 } if (j) { for (j = 0; j != h->n_buckets; ++j) { if (((h->
flags[j>>4]>>((j&0xfU)<<1))&3) == 0
) { kh_cstr_t key = h->keys[j]; int val; khint_t new_mask;
 new_mask = new_n_buckets - 1; if (1) val = h->vals[j]; (h
->flags[j>>4]|=1ul<<((j&0xfU)<<1)); while
 (1) { khint_t k, i, step = 0; k = __ac_X31_hash_string(key);
 i = k & new_mask; while (!((new_flags[i>>4]>>
((i&0xfU)<<1))&2)) i = (i + (++step)) & new_mask
; (new_flags[i>>4]&=~(2ul<<((i&0xfU)<<
1))); if (i < h->n_buckets && ((h->flags[i>>
4]>>((i&0xfU)<<1))&3) == 0) { { kh_cstr_t
 tmp = h->keys[i]; h->keys[i] = key; key = tmp; } if (1
) { int tmp = h->vals[i]; h->vals[i] = val; val = tmp; }
 (h->flags[i>>4]|=1ul<<((i&0xfU)<<1)
); } else { h->keys[i] = key; if (1) h->vals[i] = val; break
; } } } } if (h->n_buckets > new_n_buckets) { h->keys
 = (kh_cstr_t*)realloc((void *)h->keys,new_n_buckets * sizeof
(kh_cstr_t)); if (1) h->vals = (int*)realloc((void *)h->
vals,new_n_buckets * sizeof(int)); } free(h->flags); h->
flags = new_flags; h->n_buckets = new_n_buckets; h->n_occupied
 = h->size; h->upper_bound = (khint_t)(h->n_buckets *
 __ac_HASH_UPPER + 0.5); } return 0; } static inline __attribute__
 ((__unused__)) khint_t kh_put_ccv_cnnp_parameter_id(kh_ccv_cnnp_parameter_id_t
 *h, kh_cstr_t key, int *ret) { khint_t x; if (h->n_occupied
 >= h->upper_bound) { if (h->n_buckets > (h->size
<<1)) { if (kh_resize_ccv_cnnp_parameter_id(h, h->n_buckets
 - 1) < 0) { *ret = -1; return h->n_buckets; } } else if
 (kh_resize_ccv_cnnp_parameter_id(h, h->n_buckets + 1) <
 0) { *ret = -1; return h->n_buckets; } } { khint_t k, i, site
, last, mask = h->n_buckets - 1, step = 0; x = site = h->
n_buckets; k = __ac_X31_hash_string(key); i = k & mask; if
 (((h->flags[i>>4]>>((i&0xfU)<<1))&
2)) x = i; else { last = i; while (!((h->flags[i>>4]
>>((i&0xfU)<<1))&2) && (((h->flags
[i>>4]>>((i&0xfU)<<1))&1) || !(strcmp
(h->keys[i], key) == 0))) { if (((h->flags[i>>4]>>
((i&0xfU)<<1))&1)) site = i; i = (i + (++step))
 & mask; if (i == last) { x = site; break; } } if (x == h
->n_buckets) { if (((h->flags[i>>4]>>((i&
0xfU)<<1))&2) && site != h->n_buckets) x
 = site; else x = i; } } } if (((h->flags[x>>4]>>
((x&0xfU)<<1))&2)) { h->keys[x] = key; (h->
flags[x>>4]&=~(3ul<<((x&0xfU)<<1)))
; ++h->size; ++h->n_occupied; *ret = 1; } else if (((h->
flags[x>>4]>>((x&0xfU)<<1))&1)) { h
->keys[x] = key; (h->flags[x>>4]&=~(3ul<<
((x&0xfU)<<1))); ++h->size; *ret = 2; } else *ret
 = 0; return x; } static inline __attribute__ ((__unused__)) void
 kh_del_ccv_cnnp_parameter_id(kh_ccv_cnnp_parameter_id_t *h, khint_t
 x) { if (x != h->n_buckets && !((h->flags[x>>
4]>>((x&0xfU)<<1))&3)) { (h->flags[x>>
4]|=1ul<<((x&0xfU)<<1)); --h->size; } }
25
←
Null pointer value stored to field 'vals'→
30
←
Taking true branch→
31
←
Taking false branch→
32
←
Calling 'kh_resize_ccv_cnnp_parameter_id'→
33
←
Taking true branch→
34
←
Assuming the condition is false→
35
←
Taking false branch→
36
←
'?' condition is true→
37
←
Assuming 'new_flags' is null→
38
←
Taking true branch→
39
←
Returning without writing to 'h->vals'→
40
←
Returning from 'kh_resize_ccv_cnnp_parameter_id'→
41
←
Taking true branch→
42
←
Returning without writing to 'h->vals'→
2562 
2563void ccv_cnnp_model_set_parameters_from_key_values(ccv_cnnp_model_t* const model, char* const* const names, ccv_nnc_tensor_t** const tensors, const int count, const int invalidates)
2564{
2565	assert(model->compiled_data)((void) sizeof ((model->compiled_data) ? 1 : 0), __extension__
 ({ if (model->compiled_data) ; else __assert_fail ("model->compiled_data"
, "ccv_cnnp_model.c", 2565, __extension__ __PRETTY_FUNCTION__
); }));
2566	ccv_cnnp_compiled_data_t* const compiled_data = model->compiled_data;
2567	int i;
2568	khash_t(ccv_cnnp_parameter_id)kh_ccv_cnnp_parameter_id_t* id_map = 0;
2569	if (count != compiled_data->parameters->rnum)
2570	{
2571		id_map = kh_init(ccv_cnnp_parameter_id)kh_init_ccv_cnnp_parameter_id();
2572		// Build the map between name and the index.
2573		for (i = 0; i < count; i++)
2574		{
2575			int ret;
2576			const khiter_t k = kh_put(ccv_cnnp_parameter_id, id_map, names[i], &ret)kh_put_ccv_cnnp_parameter_id(id_map, names[i], &ret);
2577			assert(ret != 0)((void) sizeof ((ret != 0) ? 1 : 0), __extension__ ({ if (ret
 != 0) ; else __assert_fail ("ret != 0", "ccv_cnnp_model.c", 2577
, __extension__ __PRETTY_FUNCTION__); }));
2578			kh_val(id_map, k)((id_map)->vals[k]) = i;
2579		}
2580	}
2581	const int parameter_size = compiled_data->parameters->rnum;
2582	int* copy_back = 0;
2583	const int tensors_init = !!compiled_data->tensors_init.v;
2584	if (!tensors_init)
2585		ccv_cnnp_model_tensors_init_0(model, compiled_data);
2586	const int parallel_count = ccv_max(model->parallel_count, 1)({ typeof (model->parallel_count) _a = (model->parallel_count
); typeof (1) _b = (1); (_a > _b) ? _a : _b; });
2587	uint32_t* const init_v = CCV_NNC_INIT_V(compiled_data->tensors_init.v)((uint32_t*)((uintptr_t)(compiled_data->tensors_init.v) &
 ~(uintptr_t)1));
2588	for (i = 0; i < parameter_size; i++)
2589	{
2590		int j = i;
2591		const char* const name = *(char**)ccv_array_get(compiled_data->ids.parameters, i)((void*)(((char*)((compiled_data->ids.parameters)->data
)) + (size_t)(compiled_data->ids.parameters)->rsize * (
size_t)(i)));
2592		if (i >= 0 || strncmp(name, names[i], 1023) != 0)
2593		{
2594			// Build the map.
2595			if (id_map == 0)
2596			{
2597				id_map = kh_init(ccv_cnnp_parameter_id)kh_init_ccv_cnnp_parameter_id();
2598				for (j = 0; j < count; j++)
2599				{
2600					int ret;
2601					const khiter_t k = kh_put(ccv_cnnp_parameter_id, id_map, names[j], &ret)kh_put_ccv_cnnp_parameter_id(id_map, names[j], &ret);
2602					assert(ret != 0)((void) sizeof ((ret != 0) ? 1 : 0), __extension__ ({ if (ret
 != 0) ; else __assert_fail ("ret != 0", "ccv_cnnp_model.c", 2602
, __extension__ __PRETTY_FUNCTION__); }));
2603					kh_val(id_map, k)((id_map)->vals[k]) = j;
2604				}
2605			}
2606			const khiter_t k = kh_get(ccv_cnnp_parameter_id, id_map, name)kh_get_ccv_cnnp_parameter_id(id_map, name);
2607			if (k == kh_end(id_map)((id_map)->n_buckets)) // Cannot find the name, skip.
2608				continue;
2609			j = kh_val(id_map, k)((id_map)->vals[k]);
2610		}
2611		if (compiled_data->tensors.parameters[i]) // Cannot be a shared parameter to read.
2612			{ assert(!((uintptr_t)compiled_data->tensors.parameters[i] & (uintptr_t)1))((void) sizeof ((!((uintptr_t)compiled_data->tensors.parameters
[i] & (uintptr_t)1)) ? 1 : 0), __extension__ ({ if (!((uintptr_t
)compiled_data->tensors.parameters[i] & (uintptr_t)1))
 ; else __assert_fail ("!((uintptr_t)compiled_data->tensors.parameters[i] & (uintptr_t)1)"
, "ccv_cnnp_model.c", 2612, __extension__ __PRETTY_FUNCTION__
); })); }
2613		const ccv_nnc_tensor_symbol_t parameter = *(ccv_nnc_tensor_symbol_t*)ccv_array_get(compiled_data->parameters, i)((void*)(((char*)((compiled_data->parameters)->data)) +
 (size_t)(compiled_data->parameters)->rsize * (size_t)(
i)));
2614		ccv_nnc_tensor_param_t info = ccv_nnc_tensor_symbol_params(parameter.graph, parameter);
2615		if (CCV_TENSOR_GET_DEVICE(info.type)((info.type) & 0xfff00) == CCV_COMPUTE_DEVICE_ANY)
2616			CCV_TENSOR_SET_DEVICE_ID(info.type, 0)(info.type) = (((info.type) & ~0xfff00) | (((0) & 0xfff
) << 8));
2617		const int d = parameter.d;
2618		if (info.type == tensors[j]->info.type && invalidates) // Can move.
2619		{
2620			// Deallocate it if needed.
2621			if (!((uintptr_t)compiled_data->tensors.parameters[i] & (uintptr_t)1))
2622				if (compiled_data->tensors.parameters[i])
2623					ccv_nnc_tensor_free(compiled_data->tensors.parameters[i]);
2624			compiled_data->tensors.parameters[i] = tensors[j];
2625			tensors[j] = 0;
2626		} else {
2627			if (!compiled_data->tensors.parameters[i])
2628			{ // Not allocated, to allocate first.
2629				// Create new one, make sure we create this by having the right parameters.
2630				const int type = info.type;
2631				info = tensors[j]->info;
2632				info.type = type; // Revert back the type.
2633				compiled_data->tensors.parameters[i] = ccv_nnc_tensor_new(0, info, 0);
2634			}
2635			if (!copy_back)
2636				copy_back = (int*)cccalloccalloc(parameter_size, sizeof(int));
2637			copy_back[i] = j + 1;
2638		}
2639		init_v[d >> 5] |= (1u << (d & 0x1f));
2640		// Create this tensor for other data parallel allocations.
2641		info = compiled_data->tensors.parameters[i]->info; // In case we loaded a different info.
2642		const int device_id = CCV_TENSOR_GET_DEVICE_ID(info.type)(((info.type) & 0xfff00) >> 8);
2643		for (j = 1; j < parallel_count; j++)
2644			if (!compiled_data->tensors.parameters[i + j * parameter_size])
2645			{
2646				if (j != device_id)
2647					CCV_TENSOR_SET_DEVICE_ID(info.type, j)(info.type) = (((info.type) & ~0xfff00) | (((j) & 0xfff
) << 8));
2648				else
2649					CCV_TENSOR_SET_DEVICE_ID(info.type, 0)(info.type) = (((info.type) & ~0xfff00) | (((0) & 0xfff
) << 8));
2650				compiled_data->tensors.parameters[i + j * parameter_size] = ccv_nnc_tensor_new(0, info, 0);
2651			}
2652			// No need to copy over, this is done in ccv_cnnp_model.c's copy_tensors method.
2653	}
2654	if (id_map)
2655		kh_destroy(ccv_cnnp_parameter_id, id_map)kh_destroy_ccv_cnnp_parameter_id(id_map);
2656	// Now do the transfer.
2657	if (copy_back)
2658	{
2659		for (i = 0; i < parameter_size; i++)
2660		{
2661			ccv_nnc_tensor_t* const tensor = CCV_NNC_TENSOR(compiled_data->tensors.parameters[i])((ccv_nnc_tensor_t*)((uintptr_t)(compiled_data->tensors.parameters
[i]) & ~(uintptr_t)1));
2662			if (copy_back[i] == 0)
2663				continue;
2664			const int j = copy_back[i] - 1;
2665			ccv_nnc_cmd_exec(CMD_DATA_TRANSFER_FORWARD()ccv_nnc_cmd(CCV_NNC_DATA_TRANSFER_FORWARD, 0, ccv_nnc_cmd_auto
, 0), ccv_nnc_no_hint, 0, TENSOR_LIST(tensors[j])(ccv_nnc_tensor_t* []){tensors[j]}, (1 +1 +0 +0 +0 +0 +0 +0 +
0 +0 +0 +0 +0 +0 +0 +0 +0 +0 +0 +0 +0 +0 +0 +0 +0 +0 +0 +0 +0
 +0 +0 +0 +0 +0 +0 +0 +0 +0 +0 +0 +0 +0 +0 +0 +0 +0 +0 +0 +0 +
0 +0 +0 +0 +0 +0 +0 +0 +0 +0 +0 +0 +0 +0 +0 -1), TENSOR_LIST(tensor)(ccv_nnc_tensor_t* []){tensor}, (1 +1 +0 +0 +0 +0 +0 +0 +0 +0
 +0 +0 +0 +0 +0 +0 +0 +0 +0 +0 +0 +0 +0 +0 +0 +0 +0 +0 +0 +0 +
0 +0 +0 +0 +0 +0 +0 +0 +0 +0 +0 +0 +0 +0 +0 +0 +0 +0 +0 +0 +0
 +0 +0 +0 +0 +0 +0 +0 +0 +0 +0 +0 +0 +0 -1), 0);
2666		}
2667		ccfreefree(copy_back);
2668	}
2669}
2670 
2671ccv_cnnp_model_io_t ccv_cnnp_model_parameter_first(ccv_cnnp_model_t* const model, ccv_cnnp_model_parameters_filter_f first, void* const context)
2672{
2673	ccv_cnnp_compiled_data_t* const compiled_data = model->compiled_data;
2674	assert(compiled_data)((void) sizeof ((compiled_data) ? 1 : 0), __extension__ ({ if
 (compiled_data) ; else __assert_fail ("compiled_data", "ccv_cnnp_model.c"
, 2674, __extension__ __PRETTY_FUNCTION__); }));
2675	const int parameter_size = compiled_data->parameters->rnum;
2676	int i;
2677	for (i = 0; i < parameter_size; i++)
2678	{
2679		const char* const name = *(char**)ccv_array_get(compiled_data->ids.parameters, i)((void*)(((char*)((compiled_data->ids.parameters)->data
)) + (size_t)(compiled_data->ids.parameters)->rsize * (
size_t)(i)));
2680		if (first(model, name, context))
2681			return ccv_cnnp_model_parameters(model, -1, i);
2682	}
2683	return 0;
2684}
2685 
2686ccv_array_t* ccv_cnnp_model_parameters_filter(ccv_cnnp_model_t* const model, ccv_cnnp_model_parameters_filter_f filter, void* const context)
2687{
2688	ccv_cnnp_compiled_data_t* const compiled_data = model->compiled_data;
2689	assert(compiled_data)((void) sizeof ((compiled_data) ? 1 : 0), __extension__ ({ if
 (compiled_data) ; else __assert_fail ("compiled_data", "ccv_cnnp_model.c"
, 2689, __extension__ __PRETTY_FUNCTION__); }));
2690	ccv_array_t* const parameters = ccv_array_new(sizeof(ccv_cnnp_model_io_t), 0, 0);
2691	const int parameter_size = compiled_data->parameters->rnum;
2692	int i;
2693	for (i = 0; i < parameter_size; i++)
2694	{
2695		const char* const name = *(char**)ccv_array_get(compiled_data->ids.parameters, i)((void*)(((char*)((compiled_data->ids.parameters)->data
)) + (size_t)(compiled_data->ids.parameters)->rsize * (
size_t)(i)));
2696		if (filter(model, name, context))
2697		{
2698			ccv_cnnp_model_io_t parameter = ccv_cnnp_model_parameters(model, -1, i);
2699			ccv_array_push(parameters, &parameter);
2700		}
2701	}
2702	return parameters;
2703 
2704}
2705 
2706CCV_WARN_UNUSED(ccv_cnnp_model_io_t)ccv_cnnp_model_io_t __attribute__((warn_unused_result)) ccv_cnnp_model_parameter_first_uninit(ccv_cnnp_model_t* const model)
2707{
2708	ccv_cnnp_compiled_data_t* const compiled_data = model->compiled_data;
2709	assert(compiled_data)((void) sizeof ((compiled_data) ? 1 : 0), __extension__ ({ if
 (compiled_data) ; else __assert_fail ("compiled_data", "ccv_cnnp_model.c"
, 2709, __extension__ __PRETTY_FUNCTION__); }));
2710	const int tensors_init = !!compiled_data->tensors_init.v;
2711	if (!tensors_init) // If nothing initialized, we return parameter 0.
2712		return ccv_cnnp_model_parameters(model, -1, 0);
2713	const int parameter_size = compiled_data->parameters->rnum;
2714	int i;
2715	const uint32_t* const init_v = CCV_NNC_INIT_V(compiled_data->tensors_init.v)((uint32_t*)((uintptr_t)(compiled_data->tensors_init.v) &
 ~(uintptr_t)1));
2716	for (i = 0; i < parameter_size; i++)
2717	{
2718		const int d = ((ccv_nnc_tensor_symbol_t*)ccv_array_get(compiled_data->parameters, i)((void*)(((char*)((compiled_data->parameters)->data)) +
 (size_t)(compiled_data->parameters)->rsize * (size_t)(
i))))->d;
2719		if (!(init_v[d >> 5] & (1u << (d & 0x1f))))
2720			return ccv_cnnp_model_parameters(model, -1, i);
2721	}
2722	return 0;
2723}
2724 
2725static ccv_array_t* _ccv_cnnp_model_parameter_indices(const ccv_cnnp_model_t* const model, const ccv_cnnp_model_io_t parameters, int* const param_ref)
2726{
2727	const int to_param_sel = parameters->param_sel > 0 ? parameters->param_sel - 1 : parameters->param_sel;
2728	assert(parameters->param_sel != 0)((void) sizeof ((parameters->param_sel != 0) ? 1 : 0), __extension__
 ({ if (parameters->param_sel != 0) ; else __assert_fail (
"parameters->param_sel != 0", "ccv_cnnp_model.c", 2728, __extension__
 __PRETTY_FUNCTION__); }));
2729	ccv_array_t* const to_parameter_indices = ccv_array_new(sizeof(int), 0, 0);
2730	ccv_cnnp_model_add_to_parameter_indices(parameters->model, to_param_sel, to_parameter_indices);
2731	*param_ref = parameters->param_ref > 0 ? parameters->param_ref - 1 : parameters->param_ref;
2732	return to_parameter_indices;
2733}
2734 
2735static void _ccv_cnnp_model_to_parameter_indices_and_from_parameter_indices(ccv_cnnp_model_t* const model, const ccv_cnnp_model_io_t parameters, const ccv_cnnp_model_t* const from_model, const ccv_cnnp_model_io_t from_parameters, ccv_array_t** const parameter_indices, int* const param_ref, ccv_array_t** const from_parameter_indices, int* const from_param_ref, const int only_init_0)
2736{
2737	// If the model is not compiled yet. Compile them now.
2738	if (!model->graph)
2739	{
2740		model->graph = ccv_nnc_symbolic_graph_new();
2741		assert(from_model->compiled_data)((void) sizeof ((from_model->compiled_data) ? 1 : 0), __extension__
 ({ if (from_model->compiled_data) ; else __assert_fail ("from_model->compiled_data"
, "ccv_cnnp_model.c", 2741, __extension__ __PRETTY_FUNCTION__
); }));
2742		const int input_size = from_model->input_size;
2743		ccv_nnc_tensor_param_t input_params[input_size];
2744		int i;
2745		for (i = 0; i < input_size; i++)
2746			input_params[i] = ccv_nnc_tensor_symbol_params(from_model->graph, from_model->inputs[i]);
2747		_ccv_cnnp_model_compile(model, input_params, input_size, from_model->compiled_data->loss);
2748		model->parallel_count = from_model->parallel_count;
2749		model->memory_compression = from_model->memory_compression;
2750		model->memory_reduction = from_model->memory_reduction;
2751		model->gradient_checkpointing = from_model->gradient_checkpointing;
2752		model->compiled_data->stream_type = from_model->compiled_data->stream_type;
2753		model->compiled_data->minimize.minimizer = from_model->compiled_data->minimize.minimizer;
2754		model->compiled_data->minimize.max_saved_aux_size = from_model->compiled_data->minimize.max_saved_aux_size;
2755	}
2756	ccv_cnnp_compiled_data_t* const to_compiled_data = model->compiled_data;
2757	assert(to_compiled_data)((void) sizeof ((to_compiled_data) ? 1 : 0), __extension__ ({
 if (to_compiled_data) ; else __assert_fail ("to_compiled_data"
, "ccv_cnnp_model.c", 2757, __extension__ __PRETTY_FUNCTION__
); }));
2758	const int to_tensors_init = !!to_compiled_data->tensors_init.v;
2759	if (!to_tensors_init)
2760	{
2761		if (only_init_0)
2762			ccv_cnnp_model_tensors_init_0(model, to_compiled_data);
2763		else
2764			_ccv_cnnp_model_tensors_init(model, to_compiled_data);
2765	} else if (!only_init_0 && (uintptr_t)to_compiled_data->tensors_init.v & (uintptr_t)1)
2766		// Check if it is not fully allocated, if it is not, init_1.
2767			ccv_cnnp_model_tensors_init_1(model, to_compiled_data);
2768	assert(to_compiled_data->tensors.parameters)((void) sizeof ((to_compiled_data->tensors.parameters) ? 1
 : 0), __extension__ ({ if (to_compiled_data->tensors.parameters
) ; else __assert_fail ("to_compiled_data->tensors.parameters"
, "ccv_cnnp_model.c", 2768, __extension__ __PRETTY_FUNCTION__
); }));
2769	*parameter_indices = _ccv_cnnp_model_parameter_indices(model, parameters, param_ref);
2770	*from_parameter_indices = _ccv_cnnp_model_parameter_indices(from_model, from_parameters, from_param_ref);
2771	if (*from_param_ref < 0 && *param_ref >= 0)
2772		{ assert((*from_parameter_indices)->rnum == 1)((void) sizeof (((*from_parameter_indices)->rnum == 1) ? 1
 : 0), __extension__ ({ if ((*from_parameter_indices)->rnum
 == 1) ; else __assert_fail ("(*from_parameter_indices)->rnum == 1"
, "ccv_cnnp_model.c", 2772, __extension__ __PRETTY_FUNCTION__
); })); }
2773	else if (*from_param_ref >= 0)
2774		{ assert(*from_param_ref < (*from_parameter_indices)->rnum)((void) sizeof ((*from_param_ref < (*from_parameter_indices
)->rnum) ? 1 : 0), __extension__ ({ if (*from_param_ref <
 (*from_parameter_indices)->rnum) ; else __assert_fail ("*from_param_ref < (*from_parameter_indices)->rnum"
, "ccv_cnnp_model.c", 2774, __extension__ __PRETTY_FUNCTION__
); })); }
2775	if (*param_ref < 0 && *from_param_ref >= 0)
2776		{ assert((*parameter_indices)->rnum == 1)((void) sizeof (((*parameter_indices)->rnum == 1) ? 1 : 0)
, __extension__ ({ if ((*parameter_indices)->rnum == 1) ; else
 __assert_fail ("(*parameter_indices)->rnum == 1", "ccv_cnnp_model.c"
, 2776, __extension__ __PRETTY_FUNCTION__); })); }
2777	else if (*param_ref >= 0)
2778		{ assert(*param_ref < (*parameter_indices)->rnum)((void) sizeof ((*param_ref < (*parameter_indices)->rnum
) ? 1 : 0), __extension__ ({ if (*param_ref < (*parameter_indices
)->rnum) ; else __assert_fail ("*param_ref < (*parameter_indices)->rnum"
, "ccv_cnnp_model.c", 2778, __extension__ __PRETTY_FUNCTION__
); })); }
2779}
2780 
2781void ccv_cnnp_model_set_parameters(ccv_cnnp_model_t* const model, const ccv_cnnp_model_io_t parameters, const ccv_cnnp_model_t* const from_model, const ccv_cnnp_model_io_t from_parameters)
2782{
2783	ccv_array_t* to_parameter_indices;
2784	int to_param_ref;
2785	ccv_array_t* from_parameter_indices;
2786	int from_param_ref;
2787	_ccv_cnnp_model_to_parameter_indices_and_from_parameter_indices(model, parameters, from_model, from_parameters, &to_parameter_indices, &to_param_ref, &from_parameter_indices, &from_param_ref, 0);
2788	// Should be exactly the same tensor.
2789	if (to_param_ref < 0 && from_param_ref < 0)
2790		{ assert(from_parameter_indices->rnum == to_parameter_indices->rnum)((void) sizeof ((from_parameter_indices->rnum == to_parameter_indices
->rnum) ? 1 : 0), __extension__ ({ if (from_parameter_indices
->rnum == to_parameter_indices->rnum) ; else __assert_fail
 ("from_parameter_indices->rnum == to_parameter_indices->rnum"
, "ccv_cnnp_model.c", 2790, __extension__ __PRETTY_FUNCTION__
); })); }
2791	// To models.
2792	ccv_cnnp_compiled_data_t* const to_compiled_data = model->compiled_data;
2793	assert(to_compiled_data)((void) sizeof ((to_compiled_data) ? 1 : 0), __extension__ ({
 if (to_compiled_data) ; else __assert_fail ("to_compiled_data"
, "ccv_cnnp_model.c", 2793, __extension__ __PRETTY_FUNCTION__
); }));
2794	// From models.
2795	const ccv_cnnp_compiled_data_t* const from_compiled_data = from_model->compiled_data;
2796	const int parallel_count = ccv_max(model->parallel_count, 1)({ typeof (model->parallel_count) _a = (model->parallel_count
); typeof (1) _b = (1); (_a > _b) ? _a : _b; });
2797	const int to_parameter_size = to_compiled_data->parameters->rnum;
2798	const int rnum = (to_param_ref < 0 && from_param_ref < 0) ? from_parameter_indices->rnum : 1;
2799	int i, j;
2800	const uint32_t* const from_init_v = CCV_NNC_INIT_V(from_compiled_data->tensors_init.v)((uint32_t*)((uintptr_t)(from_compiled_data->tensors_init.
v) & ~(uintptr_t)1));
2801  uint32_t* const to_init_v = CCV_NNC_INIT_V(to_compiled_data->tensors_init.v)((uint32_t*)((uintptr_t)(to_compiled_data->tensors_init.v)
 & ~(uintptr_t)1));
2802	for (i = 0; i < rnum; i++)
2803	{
2804		const int src_d = *(int*)ccv_array_get(from_parameter_indices,from_param_ref >= 0 ? from_param_ref : i)((void*)(((char*)((from_parameter_indices)->data)) + (size_t
)(from_parameter_indices)->rsize * (size_t)(from_param_ref
 >= 0 ? from_param_ref : i)));
2805		assert(src_d >= 0)((void) sizeof ((src_d >= 0) ? 1 : 0), __extension__ ({ if
 (src_d >= 0) ; else __assert_fail ("src_d >= 0", "ccv_cnnp_model.c"
, 2805, __extension__ __PRETTY_FUNCTION__); }));
2806		assert(src_d < from_compiled_data->parameters->rnum)((void) sizeof ((src_d < from_compiled_data->parameters
->rnum) ? 1 : 0), __extension__ ({ if (src_d < from_compiled_data
->parameters->rnum) ; else __assert_fail ("src_d < from_compiled_data->parameters->rnum"
, "ccv_cnnp_model.c", 2806, __extension__ __PRETTY_FUNCTION__
); }));
2807		const int s = ((ccv_nnc_tensor_symbol_t*)ccv_array_get(from_compiled_data->parameters, src_d)((void*)(((char*)((from_compiled_data->parameters)->data
)) + (size_t)(from_compiled_data->parameters)->rsize * (
size_t)(src_d))))->d;
2808		// If the original is not init'ed. We cannot copy from.
2809		if (!(from_init_v[s >> 5] & (1u << (s & 0x1f))))
2810			continue;
2811		const int dest_d = *(int*)ccv_array_get(to_parameter_indices, to_param_ref >= 0 ? to_param_ref : i)((void*)(((char*)((to_parameter_indices)->data)) + (size_t
)(to_parameter_indices)->rsize * (size_t)(to_param_ref >=
 0 ? to_param_ref : i)));
2812		assert(dest_d >= 0)((void) sizeof ((dest_d >= 0) ? 1 : 0), __extension__ ({ if
 (dest_d >= 0) ; else __assert_fail ("dest_d >= 0", "ccv_cnnp_model.c"
, 2812, __extension__ __PRETTY_FUNCTION__); }));
2813		assert(dest_d < to_compiled_data->parameters->rnum)((void) sizeof ((dest_d < to_compiled_data->parameters->
rnum) ? 1 : 0), __extension__ ({ if (dest_d < to_compiled_data
->parameters->rnum) ; else __assert_fail ("dest_d < to_compiled_data->parameters->rnum"
, "ccv_cnnp_model.c", 2813, __extension__ __PRETTY_FUNCTION__
); }));
2814		ccv_nnc_tensor_t* const src = CCV_NNC_TENSOR(from_compiled_data->tensors.parameters[src_d])((ccv_nnc_tensor_t*)((uintptr_t)(from_compiled_data->tensors
.parameters[src_d]) & ~(uintptr_t)1));
2815		assert(src)((void) sizeof ((src) ? 1 : 0), __extension__ ({ if (src) ; else
 __assert_fail ("src", "ccv_cnnp_model.c", 2815, __extension__
 __PRETTY_FUNCTION__); }));
2816		ccv_nnc_tensor_t* const dest = CCV_NNC_TENSOR(to_compiled_data->tensors.parameters[dest_d])((ccv_nnc_tensor_t*)((uintptr_t)(to_compiled_data->tensors
.parameters[dest_d]) & ~(uintptr_t)1));
2817		assert(dest)((void) sizeof ((dest) ? 1 : 0), __extension__ ({ if (dest) ;
 else __assert_fail ("dest", "ccv_cnnp_model.c", 2817, __extension__
 __PRETTY_FUNCTION__); }));
2818		ccv_nnc_cmd_exec(CMD_DATA_TRANSFER_FORWARD()ccv_nnc_cmd(CCV_NNC_DATA_TRANSFER_FORWARD, 0, ccv_nnc_cmd_auto
, 0), ccv_nnc_no_hint, 0, TENSOR_LIST(src)(ccv_nnc_tensor_t* []){src}, (1 +1 +0 +0 +0 +0 +0 +0 +0 +0 +0
 +0 +0 +0 +0 +0 +0 +0 +0 +0 +0 +0 +0 +0 +0 +0 +0 +0 +0 +0 +0 +
0 +0 +0 +0 +0 +0 +0 +0 +0 +0 +0 +0 +0 +0 +0 +0 +0 +0 +0 +0 +0
 +0 +0 +0 +0 +0 +0 +0 +0 +0 +0 +0 +0 -1), TENSOR_LIST(dest)(ccv_nnc_tensor_t* []){dest}, (1 +1 +0 +0 +0 +0 +0 +0 +0 +0 +
0 +0 +0 +0 +0 +0 +0 +0 +0 +0 +0 +0 +0 +0 +0 +0 +0 +0 +0 +0 +0
 +0 +0 +0 +0 +0 +0 +0 +0 +0 +0 +0 +0 +0 +0 +0 +0 +0 +0 +0 +0 +
0 +0 +0 +0 +0 +0 +0 +0 +0 +0 +0 +0 +0 -1), 0);
2819		for (j = 1; j < parallel_count; j++)
2820		{
2821			ccv_nnc_tensor_t* const copy_tensor = CCV_NNC_TENSOR(to_compiled_data->tensors.parameters[dest_d + j * to_parameter_size])((ccv_nnc_tensor_t*)((uintptr_t)(to_compiled_data->tensors
.parameters[dest_d + j * to_parameter_size]) & ~(uintptr_t
)1));
2822			if (copy_tensor)
2823				ccv_nnc_cmd_exec(CMD_DATA_TRANSFER_FORWARD()ccv_nnc_cmd(CCV_NNC_DATA_TRANSFER_FORWARD, 0, ccv_nnc_cmd_auto
, 0), ccv_nnc_no_hint, 0, TENSOR_LIST(dest)(ccv_nnc_tensor_t* []){dest}, (1 +1 +0 +0 +0 +0 +0 +0 +0 +0 +
0 +0 +0 +0 +0 +0 +0 +0 +0 +0 +0 +0 +0 +0 +0 +0 +0 +0 +0 +0 +0
 +0 +0 +0 +0 +0 +0 +0 +0 +0 +0 +0 +0 +0 +0 +0 +0 +0 +0 +0 +0 +
0 +0 +0 +0 +0 +0 +0 +0 +0 +0 +0 +0 +0 -1), TENSOR_LIST(copy_tensor)(ccv_nnc_tensor_t* []){copy_tensor}, (1 +1 +0 +0 +0 +0 +0 +0 +
0 +0 +0 +0 +0 +0 +0 +0 +0 +0 +0 +0 +0 +0 +0 +0 +0 +0 +0 +0 +0
 +0 +0 +0 +0 +0 +0 +0 +0 +0 +0 +0 +0 +0 +0 +0 +0 +0 +0 +0 +0 +
0 +0 +0 +0 +0 +0 +0 +0 +0 +0 +0 +0 +0 +0 +0 -1), 0);
2824		}
2825		// Mark this symbol as init'ed.
2826		const int d = ((ccv_nnc_tensor_symbol_t*)ccv_array_get(to_compiled_data->parameters, dest_d)((void*)(((char*)((to_compiled_data->parameters)->data)
) + (size_t)(to_compiled_data->parameters)->rsize * (size_t
)(dest_d))))->d;
2827		to_init_v[d >> 5] |= (1u << (d & 0x1f));
2828	}
2829	ccv_array_free(to_parameter_indices);
2830	ccv_array_free(from_parameter_indices);
2831}
2832 
2833void ccv_cnnp_model_share_parameters(ccv_cnnp_model_t* const model, const ccv_cnnp_model_io_t parameters, const ccv_cnnp_model_t* const from_model, const ccv_cnnp_model_io_t from_parameters, ccv_cnnp_model_parameters_renamer_f renamer, void* const context)
2834{
2835	ccv_array_t* to_parameter_indices;
2836	int to_param_ref;
2837	ccv_array_t* from_parameter_indices;
2838	int from_param_ref;
2839	_ccv_cnnp_model_to_parameter_indices_and_from_parameter_indices(model, parameters, from_model, from_parameters, &to_parameter_indices, &to_param_ref, &from_parameter_indices, &from_param_ref, 1);
2840	// Should be exactly the same tensor.
2841	if (renamer == 0 && to_param_ref < 0 && from_param_ref < 0)
1
Assuming 'renamer' is not equal to null→
2842		{ assert(from_parameter_indices->rnum == to_parameter_indices->rnum)((void) sizeof ((from_parameter_indices->rnum == to_parameter_indices
->rnum) ? 1 : 0), __extension__ ({ if (from_parameter_indices
->rnum == to_parameter_indices->rnum) ; else __assert_fail
 ("from_parameter_indices->rnum == to_parameter_indices->rnum"
, "ccv_cnnp_model.c", 2842, __extension__ __PRETTY_FUNCTION__
); })); }
2843	// To models.
2844	ccv_cnnp_compiled_data_t* const to_compiled_data = model->compiled_data;
2845	assert(to_compiled_data)((void) sizeof ((to_compiled_data) ? 1 : 0), __extension__ ({
 if (to_compiled_data) ; else __assert_fail ("to_compiled_data"
, "ccv_cnnp_model.c", 2845, __extension__ __PRETTY_FUNCTION__
); }));
2
←
Assuming 'to_compiled_data' is non-null→
3
←
Taking true branch→
2846	// From models.
2847	const ccv_cnnp_compiled_data_t* const from_compiled_data = from_model->compiled_data;
2848	const int parallel_count = ccv_max(model->parallel_count, 1)({ typeof (model->parallel_count) _a = (model->parallel_count
); typeof (1) _b = (1); (_a > _b) ? _a : _b; });
4
←
Assuming '_a' is <= '_b'→
5
←
'?' condition is false→
2849	assert(parallel_count == ccv_max(from_model->parallel_count, 1))((void) sizeof ((parallel_count == ({ typeof (from_model->
parallel_count) _a = (from_model->parallel_count); typeof (
1) _b = (1); (_a > _b) ? _a : _b; })) ? 1 : 0), __extension__
 ({ if (parallel_count == ({ typeof (from_model->parallel_count
) _a = (from_model->parallel_count); typeof (1) _b = (1); (
_a > _b) ? _a : _b; })) ; else __assert_fail ("parallel_count == ccv_max(from_model->parallel_count, 1)"
, "ccv_cnnp_model.c", 2849, __extension__ __PRETTY_FUNCTION__
); })); // Should have the same parallel count can share parameters.
6
←
Assuming '_a' is <= '_b'→
7
←
'?' condition is false→
8
←
Taking true branch→
2850	const int from_parameter_size = from_compiled_data->parameters->rnum;
2851	const int to_parameter_size = to_compiled_data->parameters->rnum;
2852	const int rnum = (to_param_ref < 0 && from_param_ref < 0) ? to_parameter_indices->rnum : 1;
9
←
Assuming 'to_param_ref' is >= 0→
2853	int i, j;
2854	khash_t(ccv_cnnp_parameter_id)kh_ccv_cnnp_parameter_id_t* id_map = 0;
2855	char* updated_name = 0;
2856	const uint32_t* const from_init_v = CCV_NNC_INIT_V(from_compiled_data->tensors_init.v)((uint32_t*)((uintptr_t)(from_compiled_data->tensors_init.
v) & ~(uintptr_t)1));
2857  uint32_t* const to_init_v = CCV_NNC_INIT_V(to_compiled_data->tensors_init.v)((uint32_t*)((uintptr_t)(to_compiled_data->tensors_init.v)
 & ~(uintptr_t)1));
2858	for (i = 0; i < rnum; i++)
2859	{
2860		int src_d = (from_param_ref >= 0 ? from_param_ref : i) < from_parameter_indices->rnum ? *(int*)ccv_array_get(from_parameter_indices,from_param_ref >= 0 ? from_param_ref : i)((void*)(((char*)((from_parameter_indices)->data)) + (size_t
)(from_parameter_indices)->rsize * (size_t)(from_param_ref
 >= 0 ? from_param_ref : i))) : from_parameter_size;
10
←
Assuming 'from_param_ref' is < 0→
11
←
'?' condition is false→
12
←
Assuming the condition is false→
13
←
'?' condition is false→
2861		// Need to figure out how to use the renamer here.
2862		const int dest_d = *(int*)ccv_array_get(to_parameter_indices, to_param_ref >= 0 ? to_param_ref : i)((void*)(((char*)((to_parameter_indices)->data)) + (size_t
)(to_parameter_indices)->rsize * (size_t)(to_param_ref >=
 0 ? to_param_ref : i)));
14
←
'?' condition is true→
2863		assert(dest_d >= 0)((void) sizeof ((dest_d >= 0) ? 1 : 0), __extension__ ({ if
 (dest_d >= 0) ; else __assert_fail ("dest_d >= 0", "ccv_cnnp_model.c"
, 2863, __extension__ __PRETTY_FUNCTION__); }));
15
←
Assuming 'dest_d' is >= 0→
16
←
Taking true branch→
2864		assert(dest_d < to_parameter_size)((void) sizeof ((dest_d < to_parameter_size) ? 1 : 0), __extension__
 ({ if (dest_d < to_parameter_size) ; else __assert_fail (
"dest_d < to_parameter_size", "ccv_cnnp_model.c", 2864, __extension__
 __PRETTY_FUNCTION__); }));
17
←
Assuming 'dest_d' is < 'to_parameter_size'→
18
←
Taking true branch→
2865		if (renamer18.1
'renamer' is non-null
)
2866		{
2867			const char* const src_name = (src_d18.2
'src_d' is >= 'from_parameter_size'
 < from_parameter_size && src_d >= 0) ? *(char**)ccv_array_get(from_compiled_data->ids.parameters, src_d)((void*)(((char*)((from_compiled_data->ids.parameters)->
data)) + (size_t)(from_compiled_data->ids.parameters)->
rsize * (size_t)(src_d))) : 0;
2868			const char* const dest_name = *(char**)ccv_array_get(to_compiled_data->ids.parameters, dest_d)((void*)(((char*)((to_compiled_data->ids.parameters)->data
)) + (size_t)(to_compiled_data->ids.parameters)->rsize *
 (size_t)(dest_d)));
2869			if (!updated_name18.3
'updated_name' is null
)
19
←
Taking true branch→
2870				updated_name = (char*)ccmallocmalloc(1024);
2871			const size_t src_name_len = src_name19.1
'src_name' is equal to null
 == 0 ? 0 : ccv_min(strnlen(src_name, 1023), 1023)({ typeof (strnlen(src_name, 1023)) _a = (strnlen(src_name, 1023
)); typeof (1023) _b = (1023); (_a < _b) ? _a : _b; });
20
←
'?' condition is true→
2872			if (src_name_len20.1
'src_name_len' is <= 0
 > 0)
21
←
Taking false branch→
2873				memcpy(updated_name, src_name, src_name_len);
2874			updated_name[src_name_len] = 0;
2875			if (renamer(context, dest_name, updated_name, 1024) != 0)
22
←
Assuming the condition is false→
2876				continue; // Skip this.
2877			if (src_name22.1
'src_name' is equal to null
 != 0 && memcmp(updated_name, src_name, src_name_len) == 0 && strnlen(updated_name, 1023) == src_name_len)
2878			{
2879				// Nothing changed.
2880			} else {
2881				if (!id_map22.2
'id_map' is null
)
23
←
Taking true branch→
2882				{
2883					id_map = kh_init(ccv_cnnp_parameter_id)kh_init_ccv_cnnp_parameter_id();
24
←
Calling 'kh_init_ccv_cnnp_parameter_id'→
26
←
Returning from 'kh_init_ccv_cnnp_parameter_id'→
2884					for (j = 0; j < from_parameter_size; j++)
27
←
Assuming 'j' is < 'from_parameter_size'→
28
←
Loop condition is true.  Entering loop body→
2885					{
2886						int ret;
2887						const khiter_t k = kh_put(ccv_cnnp_parameter_id, id_map, *(char**)ccv_array_get(from_compiled_data->ids.parameters, j), &ret)kh_put_ccv_cnnp_parameter_id(id_map, *(char**)((void*)(((char
*)((from_compiled_data->ids.parameters)->data)) + (size_t
)(from_compiled_data->ids.parameters)->rsize * (size_t)
(j))), &ret);
29
←
Calling 'kh_put_ccv_cnnp_parameter_id'→
43
←
Returning from 'kh_put_ccv_cnnp_parameter_id'→
2888						assert(ret != 0)((void) sizeof ((ret != 0) ? 1 : 0), __extension__ ({ if (ret
 != 0) ; else __assert_fail ("ret != 0", "ccv_cnnp_model.c", 2888
, __extension__ __PRETTY_FUNCTION__); }));
44
←
Taking true branch→
2889						kh_val(id_map, k)((id_map)->vals[k]) = j;
45
←
Array access (via field 'vals') results in a null pointer dereference
2890					}
2891				}
2892				const khiter_t k = kh_get(ccv_cnnp_parameter_id, id_map, updated_name)kh_get_ccv_cnnp_parameter_id(id_map, updated_name);
2893				if (k == kh_end(id_map)((id_map)->n_buckets)) // Cannot find the name, skip.
2894					continue;
2895				src_d = kh_val(id_map, k)((id_map)->vals[k]);
2896				assert(src_d >= 0)((void) sizeof ((src_d >= 0) ? 1 : 0), __extension__ ({ if
 (src_d >= 0) ; else __assert_fail ("src_d >= 0", "ccv_cnnp_model.c"
, 2896, __extension__ __PRETTY_FUNCTION__); }));
2897				assert(src_d < from_parameter_size)((void) sizeof ((src_d < from_parameter_size) ? 1 : 0), __extension__
 ({ if (src_d < from_parameter_size) ; else __assert_fail (
"src_d < from_parameter_size", "ccv_cnnp_model.c", 2897, __extension__
 __PRETTY_FUNCTION__); }));
2898			}
2899		}
2900		assert(src_d >= 0)((void) sizeof ((src_d >= 0) ? 1 : 0), __extension__ ({ if
 (src_d >= 0) ; else __assert_fail ("src_d >= 0", "ccv_cnnp_model.c"
, 2900, __extension__ __PRETTY_FUNCTION__); }));
2901		assert(src_d < from_parameter_size)((void) sizeof ((src_d < from_parameter_size) ? 1 : 0), __extension__
 ({ if (src_d < from_parameter_size) ; else __assert_fail (
"src_d < from_parameter_size", "ccv_cnnp_model.c", 2901, __extension__
 __PRETTY_FUNCTION__); }));
2902		const int s = ((ccv_nnc_tensor_symbol_t*)ccv_array_get(from_compiled_data->parameters, src_d)((void*)(((char*)((from_compiled_data->parameters)->data
)) + (size_t)(from_compiled_data->parameters)->rsize * (
size_t)(src_d))))->d;
2903		// If the original is not init'ed. We cannot share from.
2904		if (!(from_init_v[s >> 5] & (1u << (s & 0x1f))))
2905			continue;
2906		for (j = 0; j < parallel_count; j++)
2907		{
2908			ccv_nnc_tensor_t* const src = CCV_NNC_TENSOR(from_compiled_data->tensors.parameters[src_d + j * from_parameter_size])((ccv_nnc_tensor_t*)((uintptr_t)(from_compiled_data->tensors
.parameters[src_d + j * from_parameter_size]) & ~(uintptr_t
)1));
2909			assert(src)((void) sizeof ((src) ? 1 : 0), __extension__ ({ if (src) ; else
 __assert_fail ("src", "ccv_cnnp_model.c", 2909, __extension__
 __PRETTY_FUNCTION__); }));
2910			ccv_nnc_tensor_t* const dest = to_compiled_data->tensors.parameters[dest_d + j * to_parameter_size];
2911			if (dest && !((uintptr_t)dest & (uintptr_t)1))
2912				ccv_nnc_tensor_free(dest);
2913			to_compiled_data->tensors.parameters[dest_d + j * to_parameter_size] = (ccv_nnc_tensor_t*)((uintptr_t)src | (uintptr_t)1);
2914		}
2915		// Mark this symbol as init'ed.
2916		const int d = ((ccv_nnc_tensor_symbol_t*)ccv_array_get(to_compiled_data->parameters, dest_d)((void*)(((char*)((to_compiled_data->parameters)->data)
) + (size_t)(to_compiled_data->parameters)->rsize * (size_t
)(dest_d))))->d;
2917		to_init_v[d >> 5] |= (1u << (d & 0x1f));
2918	}
2919	ccv_array_free(to_parameter_indices);
2920	ccv_array_free(from_parameter_indices);
2921	if (id_map)
2922		kh_destroy(ccv_cnnp_parameter_id, id_map)kh_destroy_ccv_cnnp_parameter_id(id_map);
2923	if (updated_name)
2924		ccfreefree(updated_name);
2925	// Mark it as incomplete so we will call init_1.
2926	if (ccv_cnnp_model_tensors_any_to_alloc(model, to_compiled_data))
2927		to_compiled_data->tensors_init.v = (uint32_t*)((uintptr_t)to_compiled_data->tensors_init.v | (uintptr_t)1);
2928	else // Remove the flag.
2929		to_compiled_data->tensors_init.v = CCV_NNC_INIT_V(to_compiled_data->tensors_init.v)((uint32_t*)((uintptr_t)(to_compiled_data->tensors_init.v)
 & ~(uintptr_t)1));
2930}
2931 
2932ccv_nnc_stream_context_t* ccv_cnnp_compiled_data_get_stream(ccv_cnnp_compiled_data_t* const compiled_data, const int type)
2933{
2934	if (!compiled_data->stream_map)
2935		compiled_data->stream_map = kh_init(stream_map)kh_init_stream_map();
2936	int ret = 0;
2937	khiter_t k = kh_put(stream_map, compiled_data->stream_map, type, &ret)kh_put_stream_map(compiled_data->stream_map, type, &ret
);
2938	assert(ret >= 0)((void) sizeof ((ret >= 0) ? 1 : 0), __extension__ ({ if (
ret >= 0) ; else __assert_fail ("ret >= 0", "ccv_cnnp_model.c"
, 2938, __extension__ __PRETTY_FUNCTION__); }));
2939	ccv_nnc_stream_context_t* stream = kh_val(compiled_data->stream_map, k)((compiled_data->stream_map)->vals[k]);
2940	// If ret == 0, the key already exist, we can return directly, otherwise, create and return.
2941	if (ret != 0)
2942	{
2943		stream = ccv_nnc_stream_context_new(type);
2944		kh_val(compiled_data->stream_map, k)((compiled_data->stream_map)->vals[k]) = stream;
2945	}
2946	return stream;
2947}
2948 
2949void ccv_cnnp_model_parameters_zip_map(ccv_cnnp_model_t* const model, const ccv_cnnp_model_io_t parameters, const ccv_nnc_cmd_t cmd, const ccv_nnc_hint_t hint, const int flags, ccv_nnc_tensor_t* const* const aux_ins, const int aux_in_size, ccv_nnc_tensor_t* const* const aux_outs, const int aux_out_size, ccv_nnc_stream_context_t* const stream_context, const ccv_cnnp_model_t* const from_model, const ccv_cnnp_model_io_t from_parameters)
2950{
2951	ccv_array_t* to_parameter_indices;
2952	int to_param_ref;
2953	ccv_array_t* from_parameter_indices;
2954	int from_param_ref;
2955	_ccv_cnnp_model_to_parameter_indices_and_from_parameter_indices(model, parameters, from_model, from_parameters, &to_parameter_indices, &to_param_ref, &from_parameter_indices, &from_param_ref, 0);
2956	// Should be exactly the same tensor.
2957	if (to_param_ref < 0 && from_param_ref < 0)
2958		{ assert(from_parameter_indices->rnum == to_parameter_indices->rnum)((void) sizeof ((from_parameter_indices->rnum == to_parameter_indices
->rnum) ? 1 : 0), __extension__ ({ if (from_parameter_indices
->rnum == to_parameter_indices->rnum) ; else __assert_fail
 ("from_parameter_indices->rnum == to_parameter_indices->rnum"
, "ccv_cnnp_model.c", 2958, __extension__ __PRETTY_FUNCTION__
); })); }
2959	// To models.
2960	ccv_cnnp_compiled_data_t* const to_compiled_data = model->compiled_data;
2961	assert(to_compiled_data)((void) sizeof ((to_compiled_data) ? 1 : 0), __extension__ ({
 if (to_compiled_data) ; else __assert_fail ("to_compiled_data"
, "ccv_cnnp_model.c", 2961, __extension__ __PRETTY_FUNCTION__
); }));
2962	// From models.
2963	const ccv_cnnp_compiled_data_t* const from_compiled_data = from_model->compiled_data;
2964	const int parallel_count = ccv_max(model->parallel_count, 1)({ typeof (model->parallel_count) _a = (model->parallel_count
); typeof (1) _b = (1); (_a > _b) ? _a : _b; });
2965	const int to_parameter_size = to_compiled_data->parameters->rnum;
2966	const int rnum = (to_param_ref < 0 && from_param_ref < 0) ? from_parameter_indices->rnum : 1;
2967	assert(aux_in_size >= 0)((void) sizeof ((aux_in_size >= 0) ? 1 : 0), __extension__
 ({ if (aux_in_size >= 0) ; else __assert_fail ("aux_in_size >= 0"
, "ccv_cnnp_model.c", 2967, __extension__ __PRETTY_FUNCTION__
); }));
2968	assert(aux_out_size >= 0)((void) sizeof ((aux_out_size >= 0) ? 1 : 0), __extension__
 ({ if (aux_out_size >= 0) ; else __assert_fail ("aux_out_size >= 0"
, "ccv_cnnp_model.c", 2968, __extension__ __PRETTY_FUNCTION__
); }));
2969	int i, j;
2970	ccv_nnc_tensor_t* inputs[aux_in_size + 2];
2971	ccv_nnc_tensor_t* outputs[aux_out_size + 1];
2972	for (i = 0; i < aux_in_size; i++)
2973		inputs[i + 2] = aux_ins[i];
2974	for (i = 0; i < aux_out_size; i++)
2975		outputs[i + 1] = aux_outs[i];
2976	const uint32_t* const from_init_v = CCV_NNC_INIT_V(from_compiled_data->tensors_init.v)((uint32_t*)((uintptr_t)(from_compiled_data->tensors_init.
v) & ~(uintptr_t)1));
2977	uint32_t* const to_init_v = CCV_NNC_INIT_V(to_compiled_data->tensors_init.v)((uint32_t*)((uintptr_t)(to_compiled_data->tensors_init.v)
 & ~(uintptr_t)1));
2978	for (i = 0; i < rnum; i++)
2979	{
2980		const int src_d = *(int*)ccv_array_get(from_parameter_indices,from_param_ref >= 0 ? from_param_ref : i)((void*)(((char*)((from_parameter_indices)->data)) + (size_t
)(from_parameter_indices)->rsize * (size_t)(from_param_ref
 >= 0 ? from_param_ref : i)));
2981		assert(src_d >= 0)((void) sizeof ((src_d >= 0) ? 1 : 0), __extension__ ({ if
 (src_d >= 0) ; else __assert_fail ("src_d >= 0", "ccv_cnnp_model.c"
, 2981, __extension__ __PRETTY_FUNCTION__); }));
2982		assert(src_d < from_compiled_data->parameters->rnum)((void) sizeof ((src_d < from_compiled_data->parameters
->rnum) ? 1 : 0), __extension__ ({ if (src_d < from_compiled_data
->parameters->rnum) ; else __assert_fail ("src_d < from_compiled_data->parameters->rnum"
, "ccv_cnnp_model.c", 2982, __extension__ __PRETTY_FUNCTION__
); }));
2983		const int s = ((ccv_nnc_tensor_symbol_t*)ccv_array_get(from_compiled_data->parameters, src_d)((void*)(((char*)((from_compiled_data->parameters)->data
)) + (size_t)(from_compiled_data->parameters)->rsize * (
size_t)(src_d))))->d;
2984		// If the original is not init'ed. We cannot copy from.
2985		if (!(from_init_v[s >> 5] & (1u << (s & 0x1f))))
2986			continue;
2987		const int dest_d = *(int*)ccv_array_get(to_parameter_indices, to_param_ref >= 0 ? to_param_ref : i)((void*)(((char*)((to_parameter_indices)->data)) + (size_t
)(to_parameter_indices)->rsize * (size_t)(to_param_ref >=
 0 ? to_param_ref : i)));
2988		assert(dest_d >= 0)((void) sizeof ((dest_d >= 0) ? 1 : 0), __extension__ ({ if
 (dest_d >= 0) ; else __assert_fail ("dest_d >= 0", "ccv_cnnp_model.c"
, 2988, __extension__ __PRETTY_FUNCTION__); }));
2989		assert(dest_d < to_compiled_data->parameters->rnum)((void) sizeof ((dest_d < to_compiled_data->parameters->
rnum) ? 1 : 0), __extension__ ({ if (dest_d < to_compiled_data
->parameters->rnum) ; else __assert_fail ("dest_d < to_compiled_data->parameters->rnum"
, "ccv_cnnp_model.c", 2989, __extension__ __PRETTY_FUNCTION__
); }));
2990		if (parallel_count > 1)
2991		{
2992			ccv_nnc_stream_context_t* streams[parallel_count];
2993			ccv_nnc_stream_signal_t* signal;
2994			if (stream_context)
2995				signal = ccv_nnc_stream_context_emit_signal_new(stream_context);
2996			for (j = 0; j < parallel_count; j++)
2997			{
2998				ccv_nnc_tensor_t* const src = CCV_NNC_TENSOR(from_compiled_data->tensors.parameters[src_d + j * to_parameter_size])((ccv_nnc_tensor_t*)((uintptr_t)(from_compiled_data->tensors
.parameters[src_d + j * to_parameter_size]) & ~(uintptr_t
)1));
2999				ccv_nnc_tensor_t* const dest = CCV_NNC_TENSOR(to_compiled_data->tensors.parameters[dest_d + j * to_parameter_size])((ccv_nnc_tensor_t*)((uintptr_t)(to_compiled_data->tensors
.parameters[dest_d + j * to_parameter_size]) & ~(uintptr_t
)1));
3000				if (!dest || !src)
3001				{
3002					streams[j] = 0;
3003					continue;
3004				}
3005				// At the moment, can only handle them on the same device.
3006				assert(CCV_TENSOR_GET_MEMORY(src->info.type) == CCV_TENSOR_GET_MEMORY(dest->info.type))((void) sizeof ((((src->info.type) & 0x3) == ((dest->
info.type) & 0x3)) ? 1 : 0), __extension__ ({ if (((src->
info.type) & 0x3) == ((dest->info.type) & 0x3)) ; else
 __assert_fail ("CCV_TENSOR_GET_MEMORY(src->info.type) == CCV_TENSOR_GET_MEMORY(dest->info.type)"
, "ccv_cnnp_model.c", 3006, __extension__ __PRETTY_FUNCTION__
); }));
3007				assert(CCV_TENSOR_GET_DEVICE_ID(src->info.type) == CCV_TENSOR_GET_DEVICE_ID(dest->info.type))((void) sizeof (((((src->info.type) & 0xfff00) >>
 8) == (((dest->info.type) & 0xfff00) >> 8)) ? 1
 : 0), __extension__ ({ if ((((src->info.type) & 0xfff00
) >> 8) == (((dest->info.type) & 0xfff00) >>
 8)) ; else __assert_fail ("CCV_TENSOR_GET_DEVICE_ID(src->info.type) == CCV_TENSOR_GET_DEVICE_ID(dest->info.type)"
, "ccv_cnnp_model.c", 3007, __extension__ __PRETTY_FUNCTION__
); }));
3008				const int stream_type = CCV_TENSOR_GET_MEMORY(src->info.type)((src->info.type) & 0x3) == CCV_TENSOR_GPU_MEMORY ? CCV_STREAM_CONTEXT_GPU : CCV_STREAM_CONTEXT_CPU;
3009				const int device_id = CCV_TENSOR_GET_DEVICE_ID(src->info.type)(((src->info.type) & 0xfff00) >> 8);
3010				int type = stream_type;
3011				CCV_STREAM_SET_DEVICE_ID(type, device_id)(type) = (((type) & ~0xfff00) | (((device_id) & 0xfff
) << 8));
3012				ccv_nnc_stream_context_t* const stream_0 = ccv_cnnp_compiled_data_get_stream(to_compiled_data, type);
3013				// Wait signal to finish.
3014				if (stream_context)
3015					ccv_nnc_stream_context_wait_signal(stream_0, signal);
3016				inputs[0] = outputs[0] = dest;
3017				inputs[1] = src;
3018				ccv_nnc_cmd_exec(cmd, hint, flags, inputs, aux_in_size + 2, outputs, aux_out_size + 1, stream_0);
3019				if (stream_context)
3020				{
3021					ccv_nnc_stream_signal_t* const signal = ccv_nnc_stream_context_emit_signal_new(stream_0);
3022					ccv_nnc_stream_context_wait_signal(stream_context, signal);
3023				}
3024				streams[j] = stream_0;
3025			}
3026			// If this should be blocking, blocking it.
3027			if (!stream_context)
3028				for (j = 0; j < parallel_count; j++)
3029					if (streams[j])
3030						ccv_nnc_stream_context_wait(streams[j]);
3031		} else {
3032			ccv_nnc_tensor_t* const src = CCV_NNC_TENSOR(from_compiled_data->tensors.parameters[src_d])((ccv_nnc_tensor_t*)((uintptr_t)(from_compiled_data->tensors
.parameters[src_d]) & ~(uintptr_t)1));
3033			assert(src)((void) sizeof ((src) ? 1 : 0), __extension__ ({ if (src) ; else
 __assert_fail ("src", "ccv_cnnp_model.c", 3033, __extension__
 __PRETTY_FUNCTION__); }));
3034			ccv_nnc_tensor_t* const dest = CCV_NNC_TENSOR(to_compiled_data->tensors.parameters[dest_d])((ccv_nnc_tensor_t*)((uintptr_t)(to_compiled_data->tensors
.parameters[dest_d]) & ~(uintptr_t)1));
3035			assert(dest)((void) sizeof ((dest) ? 1 : 0), __extension__ ({ if (dest) ;
 else __assert_fail ("dest", "ccv_cnnp_model.c", 3035, __extension__
 __PRETTY_FUNCTION__); }));
3036			inputs[0] = outputs[0] = dest;
3037			inputs[1] = src;
3038			ccv_nnc_cmd_exec(cmd, hint, flags, inputs, aux_in_size + 2, outputs, aux_out_size + 1, stream_context);
3039		}
3040		// Mark this symbol as init'ed.
3041		const int d = ((ccv_nnc_tensor_symbol_t*)ccv_array_get(to_compiled_data->parameters, dest_d)((void*)(((char*)((to_compiled_data->parameters)->data)
) + (size_t)(to_compiled_data->parameters)->rsize * (size_t
)(dest_d))))->d;
3042		to_init_v[d >> 5] |= (1u << (d & 0x1f));
3043	}
3044	ccv_array_free(to_parameter_indices);
3045	ccv_array_free(from_parameter_indices);
3046}
3047 
3048void ccv_cnnp_model_parameters_map(ccv_cnnp_model_t* const model, const ccv_cnnp_model_io_t parameters, const ccv_nnc_cmd_t cmd, const ccv_nnc_hint_t hint, const int flags, ccv_nnc_tensor_t* const* const aux_ins, const int aux_in_size, ccv_nnc_tensor_t* const* const aux_outs, const int aux_out_size, ccv_nnc_stream_context_t* const stream_context)
3049{
3050	int to_param_ref;
3051	ccv_array_t* const to_parameter_indices = _ccv_cnnp_model_parameter_indices(model, parameters, &to_param_ref);
3052	// To models.
3053	ccv_cnnp_compiled_data_t* const to_compiled_data = model->compiled_data;
3054	assert(to_compiled_data)((void) sizeof ((to_compiled_data) ? 1 : 0), __extension__ ({
 if (to_compiled_data) ; else __assert_fail ("to_compiled_data"
, "ccv_cnnp_model.c", 3054, __extension__ __PRETTY_FUNCTION__
); }));
3055	// Tensor has to be inited already.
3056	assert(!!to_compiled_data->tensors_init.v)((void) sizeof ((!!to_compiled_data->tensors_init.v) ? 1 :
 0), __extension__ ({ if (!!to_compiled_data->tensors_init
.v) ; else __assert_fail ("!!to_compiled_data->tensors_init.v"
, "ccv_cnnp_model.c", 3056, __extension__ __PRETTY_FUNCTION__
); }));
3057	assert(to_compiled_data->tensors.parameters)((void) sizeof ((to_compiled_data->tensors.parameters) ? 1
 : 0), __extension__ ({ if (to_compiled_data->tensors.parameters
) ; else __assert_fail ("to_compiled_data->tensors.parameters"
, "ccv_cnnp_model.c", 3057, __extension__ __PRETTY_FUNCTION__
); }));
3058	// From models.
3059	const int parallel_count = ccv_max(model->parallel_count, 1)({ typeof (model->parallel_count) _a = (model->parallel_count
); typeof (1) _b = (1); (_a > _b) ? _a : _b; });
3060	const int to_parameter_size = to_compiled_data->parameters->rnum;
3061	const int rnum = (to_param_ref < 0) ? to_parameter_indices->rnum : 1;
3062	assert(aux_in_size >= 0)((void) sizeof ((aux_in_size >= 0) ? 1 : 0), __extension__
 ({ if (aux_in_size >= 0) ; else __assert_fail ("aux_in_size >= 0"
, "ccv_cnnp_model.c", 3062, __extension__ __PRETTY_FUNCTION__
); }));
3063	assert(aux_out_size >= 0)((void) sizeof ((aux_out_size >= 0) ? 1 : 0), __extension__
 ({ if (aux_out_size >= 0) ; else __assert_fail ("aux_out_size >= 0"
, "ccv_cnnp_model.c", 3063, __extension__ __PRETTY_FUNCTION__
); }));
3064	int i, j;
3065	ccv_nnc_tensor_t* inputs[aux_in_size + 1];
3066	ccv_nnc_tensor_t* outputs[aux_out_size + 1];
3067	for (i = 0; i < aux_in_size; i++)
3068		inputs[i + 1] = aux_ins[i];
3069	for (i = 0; i < aux_out_size; i++)
3070		outputs[i + 1] = aux_outs[i];
3071	for (i = 0; i < rnum; i++)
3072	{
3073		const int dest_d = *(int*)ccv_array_get(to_parameter_indices, to_param_ref >= 0 ? to_param_ref : i)((void*)(((char*)((to_parameter_indices)->data)) + (size_t
)(to_parameter_indices)->rsize * (size_t)(to_param_ref >=
 0 ? to_param_ref : i)));
3074		assert(dest_d >= 0)((void) sizeof ((dest_d >= 0) ? 1 : 0), __extension__ ({ if
 (dest_d >= 0) ; else __assert_fail ("dest_d >= 0", "ccv_cnnp_model.c"
, 3074, __extension__ __PRETTY_FUNCTION__); }));
3075		assert(dest_d < to_compiled_data->parameters->rnum)((void) sizeof ((dest_d < to_compiled_data->parameters->
rnum) ? 1 : 0), __extension__ ({ if (dest_d < to_compiled_data
->parameters->rnum) ; else __assert_fail ("dest_d < to_compiled_data->parameters->rnum"
, "ccv_cnnp_model.c", 3075, __extension__ __PRETTY_FUNCTION__
); }));
3076		if (parallel_count > 1)
3077		{
3078			ccv_nnc_stream_context_t* streams[parallel_count];
3079			ccv_nnc_stream_signal_t* signal;
3080			if (stream_context)
3081				signal = ccv_nnc_stream_context_emit_signal_new(stream_context);
3082			for (j = 0; j < parallel_count; j++)
3083			{
3084				ccv_nnc_tensor_t* const dest = CCV_NNC_TENSOR(to_compiled_data->tensors.parameters[dest_d + j * to_parameter_size])((ccv_nnc_tensor_t*)((uintptr_t)(to_compiled_data->tensors
.parameters[dest_d + j * to_parameter_size]) & ~(uintptr_t
)1));
3085				if (!dest)
3086				{
3087					streams[j] = 0;
3088					continue;
3089				}
3090				const int stream_type = CCV_TENSOR_GET_MEMORY(dest->info.type)((dest->info.type) & 0x3) == CCV_TENSOR_GPU_MEMORY ? CCV_STREAM_CONTEXT_GPU : CCV_STREAM_CONTEXT_CPU;
3091				const int device_id = CCV_TENSOR_GET_DEVICE_ID(dest->info.type)(((dest->info.type) & 0xfff00) >> 8);
3092				int type = stream_type;
3093				CCV_STREAM_SET_DEVICE_ID(type, device_id)(type) = (((type) & ~0xfff00) | (((device_id) & 0xfff
) << 8));
3094				ccv_nnc_stream_context_t* const stream_0 = ccv_cnnp_compiled_data_get_stream(to_compiled_data, type);
3095				// Wait signal to finish.
3096				if (stream_context)
3097					ccv_nnc_stream_context_wait_signal(stream_0, signal);
3098				inputs[0] = outputs[0] = dest;
3099				ccv_nnc_cmd_exec(cmd, hint, flags, inputs, aux_in_size + 1, outputs, aux_out_size + 1, stream_0);
3100				if (stream_context)
3101				{
3102					ccv_nnc_stream_signal_t* const signal = ccv_nnc_stream_context_emit_signal_new(stream_0);
3103					ccv_nnc_stream_context_wait_signal(stream_context, signal);
3104				}
3105				streams[j] = stream_0;
3106			}
3107			// If this should be blocking, blocking it.
3108			if (!stream_context)
3109				for (j = 0; j < parallel_count; j++)
3110					if (streams[j])
3111						ccv_nnc_stream_context_wait(streams[j]);
3112		} else {
3113			ccv_nnc_tensor_t* const dest = CCV_NNC_TENSOR(to_compiled_data->tensors.parameters[dest_d])((ccv_nnc_tensor_t*)((uintptr_t)(to_compiled_data->tensors
.parameters[dest_d]) & ~(uintptr_t)1));
3114			assert(dest)((void) sizeof ((dest) ? 1 : 0), __extension__ ({ if (dest) ;
 else __assert_fail ("dest", "ccv_cnnp_model.c", 3114, __extension__
 __PRETTY_FUNCTION__); }));
3115			inputs[0] = outputs[0] = dest;
3116			ccv_nnc_cmd_exec(cmd, hint, flags, inputs, aux_in_size + 1, outputs, aux_out_size + 1, stream_context);
3117		}
3118		// No need to mark this symbol as init'ed, it is already.
3119	}
3120	ccv_array_free(to_parameter_indices);
3121}
3122 
3123void ccv_cnnp_model_parameter_gradients_map(ccv_cnnp_model_t* const model, const ccv_cnnp_model_io_t parameters, const ccv_nnc_cmd_t cmd, const ccv_nnc_hint_t hint, const int flags, ccv_nnc_tensor_t* const* const aux_ins, const int aux_in_size, ccv_nnc_tensor_t* const* const aux_outs, const int aux_out_size, ccv_nnc_stream_context_t* const stream_context)
3124{
3125	int to_param_ref;
3126	ccv_array_t* const to_parameter_indices = _ccv_cnnp_model_parameter_indices(model, parameters, &to_param_ref);
3127	// To models.
3128	ccv_cnnp_compiled_data_t* const to_compiled_data = model->compiled_data;
3129	assert(to_compiled_data)((void) sizeof ((to_compiled_data) ? 1 : 0), __extension__ ({
 if (to_compiled_data) ; else __assert_fail ("to_compiled_data"
, "ccv_cnnp_model.c", 3129, __extension__ __PRETTY_FUNCTION__
); }));
3130	// Tensor has to be inited already.
3131	assert(!!to_compiled_data->tensors_init.v)((void) sizeof ((!!to_compiled_data->tensors_init.v) ? 1 :
 0), __extension__ ({ if (!!to_compiled_data->tensors_init
.v) ; else __assert_fail ("!!to_compiled_data->tensors_init.v"
, "ccv_cnnp_model.c", 3131, __extension__ __PRETTY_FUNCTION__
); }));
3132	ccv_nnc_tensor_t** tensor_gradients;
3133	if (to_compiled_data->backward.count > 1)
3134		tensor_gradients = to_compiled_data->tensors.accum_gradients;
3135	else
3136		tensor_gradients = to_compiled_data->tensors.gradients;
3137	assert(tensor_gradients)((void) sizeof ((tensor_gradients) ? 1 : 0), __extension__ ({
 if (tensor_gradients) ; else __assert_fail ("tensor_gradients"
, "ccv_cnnp_model.c", 3137, __extension__ __PRETTY_FUNCTION__
); }));
3138	// From models.
3139	const int parallel_count = ccv_max(model->parallel_count, 1)({ typeof (model->parallel_count) _a = (model->parallel_count
); typeof (1) _b = (1); (_a > _b) ? _a : _b; });
3140	const int to_parameter_size = to_compiled_data->parameters->rnum;
3141	const int rnum = (to_param_ref < 0) ? to_parameter_indices->rnum : 1;
3142	assert(aux_in_size >= 0)((void) sizeof ((aux_in_size >= 0) ? 1 : 0), __extension__
 ({ if (aux_in_size >= 0) ; else __assert_fail ("aux_in_size >= 0"
, "ccv_cnnp_model.c", 3142, __extension__ __PRETTY_FUNCTION__
); }));
3143	assert(aux_out_size >= 0)((void) sizeof ((aux_out_size >= 0) ? 1 : 0), __extension__
 ({ if (aux_out_size >= 0) ; else __assert_fail ("aux_out_size >= 0"
, "ccv_cnnp_model.c", 3143, __extension__ __PRETTY_FUNCTION__
); }));
3144	int i, j;
3145	ccv_nnc_tensor_t* inputs[aux_in_size + 1];
3146	ccv_nnc_tensor_t* outputs[aux_out_size + 1];
3147	for (i = 0; i < aux_in_size; i++)
3148		inputs[i + 1] = aux_ins[i];
3149	for (i = 0; i < aux_out_size; i++)
3150		outputs[i + 1] = aux_outs[i];
3151	for (i = 0; i < rnum; i++)
3152	{
3153		const int dest_d = *(int*)ccv_array_get(to_parameter_indices, to_param_ref >= 0 ? to_param_ref : i)((void*)(((char*)((to_parameter_indices)->data)) + (size_t
)(to_parameter_indices)->rsize * (size_t)(to_param_ref >=
 0 ? to_param_ref : i)));
3154		assert(dest_d >= 0)((void) sizeof ((dest_d >= 0) ? 1 : 0), __extension__ ({ if
 (dest_d >= 0) ; else __assert_fail ("dest_d >= 0", "ccv_cnnp_model.c"
, 3154, __extension__ __PRETTY_FUNCTION__); }));
3155		assert(dest_d < to_compiled_data->parameters->rnum)((void) sizeof ((dest_d < to_compiled_data->parameters->
rnum) ? 1 : 0), __extension__ ({ if (dest_d < to_compiled_data
->parameters->rnum) ; else __assert_fail ("dest_d < to_compiled_data->parameters->rnum"
, "ccv_cnnp_model.c", 3155, __extension__ __PRETTY_FUNCTION__
); }));
3156		if (parallel_count > 1)
3157		{
3158			ccv_nnc_stream_context_t* streams[parallel_count];
3159			ccv_nnc_stream_signal_t* signal;
3160			if (stream_context)
3161				signal = ccv_nnc_stream_context_emit_signal_new(stream_context);
3162			for (j = 0; j < parallel_count; j++)
3163			{
3164				ccv_nnc_tensor_t* const dest = tensor_gradients[dest_d + j * to_parameter_size];
3165				if (!dest)
3166				{
3167					streams[j] = 0;
3168					continue;
3169				}
3170				const int stream_type = CCV_TENSOR_GET_MEMORY(dest->info.type)((dest->info.type) & 0x3) == CCV_TENSOR_GPU_MEMORY ? CCV_STREAM_CONTEXT_GPU : CCV_STREAM_CONTEXT_CPU;
3171				const int device_id = CCV_TENSOR_GET_DEVICE_ID(dest->info.type)(((dest->info.type) & 0xfff00) >> 8);
3172				int type = stream_type;
3173				CCV_STREAM_SET_DEVICE_ID(type, device_id)(type) = (((type) & ~0xfff00) | (((device_id) & 0xfff
) << 8));
3174				ccv_nnc_stream_context_t* const stream_0 = ccv_cnnp_compiled_data_get_stream(to_compiled_data, type);
3175				// Wait signal to finish.
3176				if (stream_context)
3177					ccv_nnc_stream_context_wait_signal(stream_0, signal);
3178				inputs[0] = outputs[0] = dest;
3179				ccv_nnc_cmd_exec(cmd, hint, flags, inputs, aux_in_size + 1, outputs, aux_out_size + 1, stream_0);
3180				if (stream_context)
3181				{
3182					ccv_nnc_stream_signal_t* const signal = ccv_nnc_stream_context_emit_signal_new(stream_0);
3183					ccv_nnc_stream_context_wait_signal(stream_context, signal);
3184				}
3185				streams[j] = stream_0;
3186			}
3187			// If this should be blocking, blocking it.
3188			if (!stream_context)
3189				for (j = 0; j < parallel_count; j++)
3190					if (streams[j])
3191						ccv_nnc_stream_context_wait(streams[j]);
3192		} else {
3193			ccv_nnc_tensor_t* const dest = tensor_gradients[dest_d];
3194			if (!dest)
3195				continue;
3196			assert(dest)((void) sizeof ((dest) ? 1 : 0), __extension__ ({ if (dest) ;
 else __assert_fail ("dest", "ccv_cnnp_model.c", 3196, __extension__
 __PRETTY_FUNCTION__); }));
3197			inputs[0] = outputs[0] = dest;
3198			ccv_nnc_cmd_exec(cmd, hint, flags, inputs, aux_in_size + 1, outputs, aux_out_size + 1, stream_context);
3199		}
3200		// No need to mark this symbol as init'ed, it is already.
3201	}
3202	ccv_array_free(to_parameter_indices);
3203}
3204 
3205void ccv_cnnp_model_parameters_to_unified_memory(ccv_cnnp_model_t* const model, const ccv_cnnp_model_io_t parameters, ccv_nnc_stream_context_t* const stream_context)
3206{
3207	// Only CUDA backend has this feature.
3208#ifdef HAVE_CUDA1
3209	int to_param_ref;
3210	ccv_array_t* const to_parameter_indices = _ccv_cnnp_model_parameter_indices(model, parameters, &to_param_ref);
3211	// To models.
3212	ccv_cnnp_compiled_data_t* const compiled_data = model->compiled_data;
3213	assert(compiled_data)((void) sizeof ((compiled_data) ? 1 : 0), __extension__ ({ if
 (compiled_data) ; else __assert_fail ("compiled_data", "ccv_cnnp_model.c"
, 3213, __extension__ __PRETTY_FUNCTION__); }));
3214	// Tensor has to be inited already.
3215	assert(!!compiled_data->tensors_init.v)((void) sizeof ((!!compiled_data->tensors_init.v) ? 1 : 0)
, __extension__ ({ if (!!compiled_data->tensors_init.v) ; else
 __assert_fail ("!!compiled_data->tensors_init.v", "ccv_cnnp_model.c"
, 3215, __extension__ __PRETTY_FUNCTION__); }));
3216	assert(compiled_data->tensors.parameters)((void) sizeof ((compiled_data->tensors.parameters) ? 1 : 0
), __extension__ ({ if (compiled_data->tensors.parameters)
 ; else __assert_fail ("compiled_data->tensors.parameters"
, "ccv_cnnp_model.c", 3216, __extension__ __PRETTY_FUNCTION__
); }));
3217	// From models.
3218	const int parallel_count = ccv_max(model->parallel_count, 1)({ typeof (model->parallel_count) _a = (model->parallel_count
); typeof (1) _b = (1); (_a > _b) ? _a : _b; });
3219	const int rnum = (to_param_ref < 0) ? to_parameter_indices->rnum : 1;
3220	int i;
3221	for (i = 0; i < rnum; i++)
3222	{
3223		const int dest_d = *(int*)ccv_array_get(to_parameter_indices, to_param_ref >= 0 ? to_param_ref : i)((void*)(((char*)((to_parameter_indices)->data)) + (size_t
)(to_parameter_indices)->rsize * (size_t)(to_param_ref >=
 0 ? to_param_ref : i)));
3224		assert(dest_d >= 0)((void) sizeof ((dest_d >= 0) ? 1 : 0), __extension__ ({ if
 (dest_d >= 0) ; else __assert_fail ("dest_d >= 0", "ccv_cnnp_model.c"
, 3224, __extension__ __PRETTY_FUNCTION__); }));
3225		assert(dest_d < compiled_data->parameters->rnum)((void) sizeof ((dest_d < compiled_data->parameters->
rnum) ? 1 : 0), __extension__ ({ if (dest_d < compiled_data
->parameters->rnum) ; else __assert_fail ("dest_d < compiled_data->parameters->rnum"
, "ccv_cnnp_model.c", 3225, __extension__ __PRETTY_FUNCTION__
); }));
3226		if (parallel_count > 1)
3227		{
3228			assert(0 && "Cannot support this when data parallel is in effect.")((void) sizeof ((0 && "Cannot support this when data parallel is in effect."
) ? 1 : 0), __extension__ ({ if (0 && "Cannot support this when data parallel is in effect."
) ; else __assert_fail ("0 && \"Cannot support this when data parallel is in effect.\""
, "ccv_cnnp_model.c", 3228, __extension__ __PRETTY_FUNCTION__
); }));
3229		} else {
3230			ccv_nnc_tensor_t* const src = CCV_NNC_TENSOR(compiled_data->tensors.parameters[dest_d])((ccv_nnc_tensor_t*)((uintptr_t)(compiled_data->tensors.parameters
[dest_d]) & ~(uintptr_t)1));
3231			assert(src)((void) sizeof ((src) ? 1 : 0), __extension__ ({ if (src) ; else
 __assert_fail ("src", "ccv_cnnp_model.c", 3231, __extension__
 __PRETTY_FUNCTION__); }));
3232			ccv_nnc_tensor_param_t params = src->info;
3233			if (CCV_TENSOR_GET_MEMORY(params.type)((params.type) & 0x3) != CCV_TENSOR_GPU_MEMORY)
3234				continue;
3235			const size_t size = ccv_nnc_tensor_data_size(params);
3236			if (size <= 0)
3237				continue;
3238			const int should_free = !((uintptr_t)compiled_data->tensors.parameters[dest_d] & (uintptr_t)1);
3239			const int tfb = (CCV_TENSOR_GET_MEMORY(params.type)((params.type) & 0x3) == CCV_TENSOR_CPU_MEMORY && params.format == CCV_TENSOR_FORMAT_NHWC && params.dim[2] > 0 && params.dim[2] <= CCV_MAX_CHANNEL(0xFFF) && params.dim[0] > 0 && params.dim[1] > 0 && params.dim[3] == 0);
3240			ccv_nnc_tensor_t* const tensor = (ccv_nnc_tensor_t*)ccmallocmalloc(sizeof(ccv_nnc_tensor_t));
3241			tensor->dataof = 0;
3242			tensor->alias_ref = 0;
3243			tensor->sig = 0;
3244			tensor->refcount = 1;
3245			tensor->info = params;
3246			if (tfb)
3247			{
3248				tensor->type = CCV_NO_DATA_ALLOC | CCV_MATRIX_DENSE | CCV_GET_DATA_TYPE(params.datatype)((params.datatype) & 0xFF000) | params.dim[2];
3249				// This corresponding to mat->step
3250				tensor->info.dim[4] = CCV_GET_STEP(params.dim[1], (CCV_GET_DATA_TYPE(params.datatype) | params.dim[2]))(((params.dim[1]) * _ccv_get_data_type_size[(((((params.datatype
) & 0xFF000) | params.dim[2])) & 0xFF000) >> 12
] * (((((params.datatype) & 0xFF000) | params.dim[2])) &
 0xFFF) + 3) & -4);
3251			} else // This won't be recognized by ccv_dense_matrix_t
3252				tensor->type = CCV_NO_DATA_ALLOC | CCV_MATRIX_DENSE | CCV_GET_DATA_TYPE(params.datatype)((params.datatype) & 0xFF000);
3253			// Remove this flag so it can be deallocated as usual.
3254			tensor->type &= ~CCV_NO_DATA_ALLOC;
3255			assert(CCV_TENSOR_GET_DEVICE(params.type) != CCV_COMPUTE_DEVICE_ANY)((void) sizeof ((((params.type) & 0xfff00) != CCV_COMPUTE_DEVICE_ANY
) ? 1 : 0), __extension__ ({ if (((params.type) & 0xfff00
) != CCV_COMPUTE_DEVICE_ANY) ; else __assert_fail ("CCV_TENSOR_GET_DEVICE(params.type) != CCV_COMPUTE_DEVICE_ANY"
, "ccv_cnnp_model.c", 3255, __extension__ __PRETTY_FUNCTION__
); }));
3256			void* ptr = cumallocmanaged(CCV_TENSOR_GET_DEVICE_ID(params.type)(((params.type) & 0xfff00) >> 8), size);
3257			if (ptr) // If allocated successfully. Otherwise we go through the fallback path.
3258			{
3259				tensor->data.u8 = (uint8_t*)ptr;
3260				tensor->type |= CCV_MAPPED_MEM; // This denotes the tensor is mapped to CPU, and would prefer a explicit prefetch call.
3261			} else {
3262				// Allocation failed.
3263				ccfreefree(tensor);
3264				continue;
3265			}
3266			// TODO: Cannot run this on the stream context yet, due to allocation and deallocations.
3267			ccv_nnc_cmd_exec(CMD_DATA_TRANSFER_FORWARD()ccv_nnc_cmd(CCV_NNC_DATA_TRANSFER_FORWARD, 0, ccv_nnc_cmd_auto
, 0), ccv_nnc_no_hint, 0, &src, 1, &tensor, 1, 0);
3268			cumemadvisereadmostly(CCV_TENSOR_GET_DEVICE_ID(params.type)(((params.type) & 0xfff00) >> 8), tensor->data.u8, size);
3269			compiled_data->tensors.parameters[dest_d] = tensor;
3270			// Can free out the old one.
3271			if (should_free)
3272				ccv_nnc_tensor_free(src);
3273		}
3274		// No need to mark this symbol as init'ed, it is already.
3275	}
3276	ccv_array_free(to_parameter_indices);
3277#endif
3278}
3279 
3280ccv_nnc_cmd_t ccv_cnnp_model_minimizer(ccv_cnnp_model_t* const model)
3281{
3282	ccv_cnnp_compiled_data_t* const compiled_data = model->compiled_data;
3283	assert(compiled_data)((void) sizeof ((compiled_data) ? 1 : 0), __extension__ ({ if
 (compiled_data) ; else __assert_fail ("compiled_data", "ccv_cnnp_model.c"
, 3283, __extension__ __PRETTY_FUNCTION__); }));
3284	return compiled_data->minimize.minimizer;
3285}
3286 
3287void ccv_cnnp_model_set_minimizer(ccv_cnnp_model_t* const model, const ccv_nnc_cmd_t minimizer, const int reset, const ccv_cnnp_model_io_t* const set_parameters, const int set_parameter_size)
3288{
3289	ccv_cnnp_compiled_data_t* const compiled_data = model->compiled_data;
3290	assert(compiled_data)((void) sizeof ((compiled_data) ? 1 : 0), __extension__ ({ if
 (compiled_data) ; else __assert_fail ("compiled_data", "ccv_cnnp_model.c"
, 3290, __extension__ __PRETTY_FUNCTION__); }));
3291	const int parameter_size = compiled_data->parameters->rnum;
3292	if (parameter_size == 0)
3293		return;
3294	if (reset)
3295		{ assert(set_parameters == 0 && set_parameter_size == 0)((void) sizeof ((set_parameters == 0 && set_parameter_size
 == 0) ? 1 : 0), __extension__ ({ if (set_parameters == 0 &&
 set_parameter_size == 0) ; else __assert_fail ("set_parameters == 0 && set_parameter_size == 0"
, "ccv_cnnp_model.c", 3295, __extension__ __PRETTY_FUNCTION__
); })); }
3296	const int old_max_saved_aux_size = compiled_data->minimize.max_saved_aux_size;
3297	const int saved_aux_size = ccv_nnc_minimizer_saved_aux_size(minimizer);
3298	if (saved_aux_size > compiled_data->minimize.max_saved_aux_size)
3299		compiled_data->minimize.max_saved_aux_size = saved_aux_size;
3300	const int max_saved_aux_size = compiled_data->minimize.max_saved_aux_size;
3301	// We update all parameters, at this point, we have one minimizer.
3302	if (set_parameters == 0 || set_parameter_size == 0)
3303		compiled_data->minimize.minimizer = minimizer;
3304	int i;
3305	if (set_parameters && set_parameter_size)
3306	{
3307		// I need to save what's the minimizer along with this.
3308		if (!compiled_data->minimize.parameters)
3309			compiled_data->minimize.parameters = ccv_array_new(sizeof(ccv_cnnp_set_minimizer_for_parameter_t*), 1, 0);
3310		ccv_cnnp_set_minimizer_for_parameter_t* const set_minimizer_for_parameter = ccmallocmalloc(sizeof(ccv_cnnp_set_minimizer_for_parameter_t) + (set_parameter_size - 1) * sizeof(ccv_cnnp_model_io_t));
3311		set_minimizer_for_parameter->minimizer = minimizer;
3312		set_minimizer_for_parameter->parameter_size = set_parameter_size;
3313		memcpy(set_minimizer_for_parameter->parameters, set_parameters, sizeof(ccv_cnnp_model_io_t) * set_parameter_size);
3314		ccv_array_push(compiled_data->minimize.parameters, &set_minimizer_for_parameter);
3315	}
3316	// If reset is true, clear the parameters array.
3317	if (reset && compiled_data->minimize.parameters)
3318	{
3319		for (i = 0; i < compiled_data->minimize.parameters->rnum; i++)
3320			ccfreefree(*(ccv_cnnp_set_minimizer_for_parameter_t**)ccv_array_get(compiled_data->minimize.parameters, i)((void*)(((char*)((compiled_data->minimize.parameters)->
data)) + (size_t)(compiled_data->minimize.parameters)->
rsize * (size_t)(i))));
3321		ccv_array_clear(compiled_data->minimize.parameters);
3322	}
3323	if (!compiled_data->update_nodes)
3324		return;
3325	ccv_nnc_symbolic_graph_t* const symbolic_graph = model->graph;
3326	assert(symbolic_graph)((void) sizeof ((symbolic_graph) ? 1 : 0), __extension__ ({ if
 (symbolic_graph) ; else __assert_fail ("symbolic_graph", "ccv_cnnp_model.c"
, 3326, __extension__ __PRETTY_FUNCTION__); }));
3327	if (saved_aux_size > old_max_saved_aux_size)
3328	{
3329		assert(compiled_data->updated_parameters)((void) sizeof ((compiled_data->updated_parameters) ? 1 : 0
), __extension__ ({ if (compiled_data->updated_parameters)
 ; else __assert_fail ("compiled_data->updated_parameters"
, "ccv_cnnp_model.c", 3329, __extension__ __PRETTY_FUNCTION__
); }));
3330		// Reallocate first, move them around later.
3331		compiled_data->updated_parameters = (ccv_nnc_tensor_symbol_t*)ccreallocrealloc(compiled_data->updated_parameters, sizeof(ccv_nnc_tensor_symbol_t) * parameter_size + sizeof(ccv_nnc_graph_exec_symbol_t) * parameter_size + sizeof(ccv_nnc_tensor_symbol_map_t) * saved_aux_size * parameter_size);
3332		compiled_data->update_nodes = (ccv_nnc_graph_exec_symbol_t*)(compiled_data->updated_parameters + parameter_size);
3333		compiled_data->saved_aux = (ccv_nnc_tensor_symbol_map_t*)(compiled_data->update_nodes + parameter_size);
3334		// We need to do this from back to front because saved_aux_size > old_saved_aux_size, it could overlap.
3335		_ccv_cnnp_scatter_saved_aux(compiled_data->saved_aux, parameter_size, old_max_saved_aux_size, saved_aux_size);
3336	}
3337	int flag = 0;
3338	const int parallel_count = ccv_max(model->parallel_count, 1)({ typeof (model->parallel_count) _a = (model->parallel_count
); typeof (1) _b = (1); (_a > _b) ? _a : _b; });
3339	if (set_parameters && set_parameter_size)
3340	{
3341		ccv_array_t* const parameter_indices = ccv_array_new(sizeof(int), 0, 0);
3342		for (i = 0; i < set_parameter_size; i++)
3343		{
3344			const int param_sel = set_parameters[i]->param_sel > 0 ? set_parameters[i]->param_sel - 1 : set_parameters[i]->param_sel;
3345			assert(set_parameters[i]->param_sel != 0)((void) sizeof ((set_parameters[i]->param_sel != 0) ? 1 : 0
), __extension__ ({ if (set_parameters[i]->param_sel != 0)
 ; else __assert_fail ("set_parameters[i]->param_sel != 0"
, "ccv_cnnp_model.c", 3345, __extension__ __PRETTY_FUNCTION__
); }));
3346			const int old_rnum = parameter_indices->rnum;
3347			ccv_cnnp_model_add_to_parameter_indices(set_parameters[i]->model, param_sel, parameter_indices);
3348			const int param_ref = set_parameters[i]->param_ref > 0 ? set_parameters[i]->param_ref - 1 : set_parameters[i]->param_ref;
3349			assert(set_parameters[i]->param_ref != 0)((void) sizeof ((set_parameters[i]->param_ref != 0) ? 1 : 0
), __extension__ ({ if (set_parameters[i]->param_ref != 0)
 ; else __assert_fail ("set_parameters[i]->param_ref != 0"
, "ccv_cnnp_model.c", 3349, __extension__ __PRETTY_FUNCTION__
); }));
3350			if (param_ref >= 0)
3351			{
3352				assert(param_ref + old_rnum < parameter_indices->rnum)((void) sizeof ((param_ref + old_rnum < parameter_indices->
rnum) ? 1 : 0), __extension__ ({ if (param_ref + old_rnum <
 parameter_indices->rnum) ; else __assert_fail ("param_ref + old_rnum < parameter_indices->rnum"
, "ccv_cnnp_model.c", 3352, __extension__ __PRETTY_FUNCTION__
); }));
3353				*(int*)ccv_array_get(parameter_indices, old_rnum)((void*)(((char*)((parameter_indices)->data)) + (size_t)(parameter_indices
)->rsize * (size_t)(old_rnum))) = *(int*)ccv_array_get(parameter_indices, param_ref + old_rnum)((void*)(((char*)((parameter_indices)->data)) + (size_t)(parameter_indices
)->rsize * (size_t)(param_ref + old_rnum)));
3354				parameter_indices->rnum = old_rnum + 1;
3355			}
3356		}
3357		// We may have duplicated indices, but that is OK, we will set it twice.
3358		for (i = 0; i < parameter_indices->rnum; i++)
3359		{
3360			const int d = *(int*)ccv_array_get(parameter_indices, i)((void*)(((char*)((parameter_indices)->data)) + (size_t)(parameter_indices
)->rsize * (size_t)(i)));
3361			if (_ccv_cnnp_set_minimizer_for_parameter(symbolic_graph, compiled_data, compiled_data->update_nodes, compiled_data->updated_parameters, compiled_data->saved_aux, parallel_count, minimizer, saved_aux_size, max_saved_aux_size, d))
3362				flag = 1;
3363		}
3364		ccv_array_free(parameter_indices);
3365	} else {
3366		for (i = 0; i < parameter_size; i++)
3367			if (_ccv_cnnp_set_minimizer_for_parameter(symbolic_graph, compiled_data, compiled_data->update_nodes, compiled_data->updated_parameters, compiled_data->saved_aux, parallel_count, minimizer, saved_aux_size, max_saved_aux_size, i))
3368				flag = 1;
3369		if (compiled_data->minimize.parameters)
3370			if (_ccv_cnnp_apply_parameters_with_minimizer(model))
3371				flag = 1;
3372	}
3373	if (flag)
3374	{
3375		// If saved_aux_size doesn't match, we need to remove / add new saved_aux to the graph. But first, free up apply gradients graph.
3376		if (compiled_data->graph_mode == CCV_CNNP_MODEL_GRAPH_FIT_MODE)
3377			_ccv_cnnp_compiled_data_graph_free(compiled_data);
3378		_ccv_cnnp_compiled_data_apply_gradients_free(compiled_data);
3379	}
3380}
3381 
3382void ccv_cnnp_model_set_compile_params(ccv_cnnp_model_t* const model, const ccv_nnc_symbolic_graph_compile_param_t compile_params)
3383{
3384	ccv_cnnp_compiled_data_t* const compiled_data = model->compiled_data;
3385	assert(compiled_data)((void) sizeof ((compiled_data) ? 1 : 0), __extension__ ({ if
 (compiled_data) ; else __assert_fail ("compiled_data", "ccv_cnnp_model.c"
, 3385, __extension__ __PRETTY_FUNCTION__); }));
3386	compiled_data->compile_params = compile_params;
3387}
3388 
3389void ccv_cnnp_model_dot(const ccv_cnnp_model_t* const model, const int flags, FILE** const outs, const int out_size)
3390{
3391	if (model->graph && out_size > 0)
3392		ccv_nnc_symbolic_graph_dot(model->graph, flags, outs[0]);
3393	if (model->compiled_data && model->compiled_data->graph && out_size > 1)
3394		ccv_nnc_graph_dot(model->compiled_data->graph, flags, outs[1]);
3395	if (model->compiled_data && model->compiled_data->backward.accum && out_size > 2)
3396		ccv_nnc_graph_dot(model->compiled_data->backward.accum, flags, outs[2]);
3397	if (model->compiled_data && model->compiled_data->apply_gradients.graph && out_size > 3)
3398		ccv_nnc_graph_dot(model->compiled_data->apply_gradients.graph, flags, outs[3]);
3399}
3400 
3401void ccv_cnnp_model_format(const ccv_cnnp_model_t* const model, const ccv_nnc_symbolic_graph_format_f format_fn, void* const context)
3402{
3403	if (model->graph)
3404		ccv_nnc_symbolic_graph_format(model->graph, 0, 0, 0, 0, format_fn, context);
3405}
3406 
3407static void _ccv_cnnp_compiled_data_free(const ccv_cnnp_model_t* const model, ccv_cnnp_compiled_data_t* const compiled_data)
3408{
3409	int i;
3410	const int parameter_size = compiled_data->parameters->rnum;
3411	ccv_array_free(compiled_data->parameters);
3412	if (compiled_data->parameter_flags)
3413		ccfreefree(compiled_data->parameter_flags);
3414	const int internal_size = compiled_data->internals->rnum;
3415	ccv_array_free(compiled_data->internals);
3416	assert(compiled_data->ids.parameters->rnum == parameter_size)((void) sizeof ((compiled_data->ids.parameters->rnum ==
 parameter_size) ? 1 : 0), __extension__ ({ if (compiled_data
->ids.parameters->rnum == parameter_size) ; else __assert_fail
 ("compiled_data->ids.parameters->rnum == parameter_size"
, "ccv_cnnp_model.c", 3416, __extension__ __PRETTY_FUNCTION__
); }));
3417	assert(compiled_data->ids.internals->rnum == internal_size)((void) sizeof ((compiled_data->ids.internals->rnum == internal_size
) ? 1 : 0), __extension__ ({ if (compiled_data->ids.internals
->rnum == internal_size) ; else __assert_fail ("compiled_data->ids.internals->rnum == internal_size"
, "ccv_cnnp_model.c", 3417, __extension__ __PRETTY_FUNCTION__
); }));
3418	for (i = 0; i < parameter_size; i++)
3419		ccfreefree(*(char**)ccv_array_get(compiled_data->ids.parameters, i)((void*)(((char*)((compiled_data->ids.parameters)->data
)) + (size_t)(compiled_data->ids.parameters)->rsize * (
size_t)(i))));
3420	ccv_array_free(compiled_data->ids.parameters);
3421	for (i = 0; i < internal_size; i++)
3422		ccfreefree(*(char**)ccv_array_get(compiled_data->ids.internals, i)((void*)(((char*)((compiled_data->ids.internals)->data)
) + (size_t)(compiled_data->ids.internals)->rsize * (size_t
)(i))));
3423	ccv_array_free(compiled_data->ids.internals);
3424	const int parallel_count = compiled_data->parallel_count > 0 ? compiled_data->parallel_count : _ccv_cnnp_model_root_parallel_count(model);
3425	if (compiled_data->tensors.parameters)
3426	{
3427		for (i = 0; i < parameter_size * parallel_count; i++)
3428			// If it is not marked as not belonging, we can free it.
3429			if (!((uintptr_t)compiled_data->tensors.parameters[i] & (uintptr_t)1))
3430				if (compiled_data->tensors.parameters[i])
3431					ccv_nnc_tensor_free(compiled_data->tensors.parameters[i]);
3432		for (i = 0; i < internal_size * parallel_count; i++)
3433			if (compiled_data->tensors.internals[i])
3434				ccv_nnc_tensor_free(compiled_data->tensors.internals[i]);
3435		ccfreefree(compiled_data->tensors.parameters);
3436	}
3437	if (compiled_data->tensors.gradients)
3438	{
3439		for (i = 0; i < parameter_size * parallel_count; i++)
3440		{
3441			if (compiled_data->tensors.gradients[i])
3442				ccv_nnc_tensor_free(compiled_data->tensors.gradients[i]);
3443			if (compiled_data->tensors.accum_gradients[i])
3444				ccv_nnc_tensor_free(compiled_data->tensors.accum_gradients[i]);
3445		}
3446		ccfreefree(compiled_data->tensors.gradients);
3447	}
3448	if (compiled_data->minimize.parameters)
3449	{
3450		for (i = 0; i < compiled_data->minimize.parameters->rnum; i++)
3451			ccfreefree(*(ccv_cnnp_set_minimizer_for_parameter_t**)ccv_array_get(compiled_data->minimize.parameters, i)((void*)(((char*)((compiled_data->minimize.parameters)->
data)) + (size_t)(compiled_data->minimize.parameters)->
rsize * (size_t)(i))));
3452		ccv_array_free(compiled_data->minimize.parameters);
3453	}
3454	if (compiled_data->rewindables)
3455		ccv_array_free(compiled_data->rewindables);
3456	if (compiled_data->tensors_init.v)
3457		ccfreefree(CCV_NNC_INIT_V(compiled_data->tensors_init.v)((uint32_t*)((uintptr_t)(compiled_data->tensors_init.v) &
 ~(uintptr_t)1)));
3458	if (compiled_data->evaluate.tos)
3459		ccfreefree(compiled_data->evaluate.tos);
3460	compiled_data->evaluate.tos = 0;
3461	if (compiled_data->stream_map)
3462	{
3463		khiter_t k;
3464		for (k = kh_begin(compiled_data->stream_map)(khint_t)(0); k != kh_end(compiled_data->stream_map)((compiled_data->stream_map)->n_buckets); ++k)
3465		{
3466			if (!kh_exist(compiled_data->stream_map, k)(!(((compiled_data->stream_map)->flags[(k)>>4]>>
(((k)&0xfU)<<1))&3)))
3467				continue;
3468			ccv_nnc_stream_context_t* const stream = kh_val(compiled_data->stream_map, k)((compiled_data->stream_map)->vals[k]);
3469			ccv_nnc_stream_context_free(stream);
3470		}
3471		kh_destroy(stream_map, compiled_data->stream_map)kh_destroy_stream_map(compiled_data->stream_map);
3472	}
3473	_ccv_cnnp_compiled_data_graph_free(compiled_data);
3474	_ccv_cnnp_compiled_data_gradient_free(compiled_data);
3475	_ccv_cnnp_compiled_data_backward_free(compiled_data);
3476	_ccv_cnnp_compiled_data_apply_gradients_free(compiled_data);
3477	if (compiled_data->gradient_checkpoints)
3478	{
3479		for (i = 0; i < compiled_data->gradient_checkpoints->rnum; i++)
3480		{
3481			ccv_cnnp_model_gradient_checkpoint_t* const checkpoint = (ccv_cnnp_model_gradient_checkpoint_t*)ccv_array_get(compiled_data->gradient_checkpoints, i)((void*)(((char*)((compiled_data->gradient_checkpoints)->
data)) + (size_t)(compiled_data->gradient_checkpoints)->
rsize * (size_t)(i)));
3482			assert(checkpoint->inputs)((void) sizeof ((checkpoint->inputs) ? 1 : 0), __extension__
 ({ if (checkpoint->inputs) ; else __assert_fail ("checkpoint->inputs"
, "ccv_cnnp_model.c", 3482, __extension__ __PRETTY_FUNCTION__
); }));
3483			ccfreefree(checkpoint->inputs);
3484			ccv_array_free(checkpoint->tensor_symbols);
3485		}
3486		ccv_array_free(compiled_data->gradient_checkpoints);
3487	}
3488	ccv_nnc_xpu_alloc_destroy(&compiled_data->xpu_alloc);
3489	ccfreefree(compiled_data);
3490}
3491 
3492void ccv_cnnp_model_free(ccv_cnnp_model_t* const model)
3493{
3494	ccv_cnnp_model_pin_memory(model, 0);
3495	ccv_cnnp_model_deinit(model);
3496	if (model->isa->dealloc)
3497		model->isa->dealloc(model);
3498	if (model->io)
3499	{
3500		int i;
3501		for (i = 0; i < model->io->rnum; i++)
3502		{
3503			ccv_cnnp_model_io_t model_io = *(ccv_cnnp_model_io_t*)ccv_array_get(model->io, i)((void*)(((char*)((model->io)->data)) + (size_t)(model->
io)->rsize * (size_t)(i)));
3504			if (model_io->outgoings)
3505				ccv_array_free(model_io->outgoings);
3506			if (model_io->incomings)
3507				ccv_array_free(model_io->incomings);
3508			if (model_io->dependencies)
3509				ccv_array_free(model_io->dependencies);
3510			ccfreefree(model_io);
3511		}
3512		ccv_array_free(model->io);
3513	}
3514	if (model->parameter_indices)
3515		ccv_array_free(model->parameter_indices);
3516	if (model->inputs)
3517		ccfreefree(model->inputs);
3518	if (model->graph)
3519		ccv_nnc_symbolic_graph_free(model->graph);
3520	if (model->compiled_data)
3521		_ccv_cnnp_compiled_data_free(model, model->compiled_data);
3522	if (model->name)
3523		ccfreefree(model->name);
3524	ccfreefree(model);
3525}
3526 
3527void ccv_cnnp_model_cancel(ccv_cnnp_model_t* const model)
3528{
3529	ccv_cnnp_compiled_data_t* const compiled_data = model->compiled_data;
3530	if (!compiled_data)
3531		return;
3532	if (compiled_data->graph)
3533		ccv_nnc_graph_cancel(compiled_data->graph);
3534	if (compiled_data->apply_gradients.graph)
3535		ccv_nnc_graph_cancel(compiled_data->apply_gradients.graph);
3536}
3537 
3538void ccv_cnnp_model_async_enter(ccv_cnnp_model_t* const model)
3539{
3540	ccv_cnnp_compiled_data_t* const compiled_data = model->compiled_data;
3541	if (!compiled_data)
3542		return;
3543	if (compiled_data->graph)
3544		ccv_nnc_graph_async_enter(compiled_data->graph);
3545	if (compiled_data->apply_gradients.graph)
3546		ccv_nnc_graph_async_enter(compiled_data->apply_gradients.graph);
3547}
3548 
3549void ccv_cnnp_model_set_flags(ccv_cnnp_model_t* const model, const int flags)
3550{
3551	model->exec_flags = flags;
3552}
3553 
3554int ccv_cnnp_model_flags(ccv_cnnp_model_t* const model)
3555{
3556	return model->exec_flags;
3557}