/home/liu/actions-runner/_work/ccv/ccv/lib/nnc/_ccv_cnnp_model.h
Line | Count | Source |
1 | | /********************************************************** |
2 | | * C-based/Cached/Core Computer Vision Library |
3 | | * Liu Liu, 2010-02-01 |
4 | | **********************************************************/ |
5 | | |
6 | | /********************************************************** |
7 | | * CCV - Neural Network Collection |
8 | | **********************************************************/ |
9 | | |
10 | | #ifndef GUARD_ccv_cnnp_model_internal_h |
11 | | #define GUARD_ccv_cnnp_model_internal_h |
12 | | |
13 | | #include "ccv_nnc.h" |
14 | | #include "_ccv_nnc_stream.h" |
15 | | #include "_ccv_nnc_xpu_alloc.h" |
16 | | #include "3rdparty/khash/khash.h" |
17 | | |
18 | | typedef void(*ccv_cnnp_cmd_updater_f)(void* const context, const ccv_nnc_graph_exec_symbol_t symbol, const ccv_nnc_cmd_t cmd, const ccv_nnc_hint_t hint); |
19 | | typedef void(*ccv_cnnp_add_to_array_f)(void* const context, const ccv_nnc_tensor_symbol_t symbol, const int is_trainable); |
20 | | /** |
21 | | * This is the virtual table of the model. |
22 | | */ |
23 | | typedef struct { |
24 | | void (*deinit)(ccv_cnnp_model_t* const self); /**< It can be nil. */ |
25 | | void (*dealloc)(ccv_cnnp_model_t* const self); /**< It can be nil. This is different from deinit because you should only free other models in this method. */ |
26 | | void (*build)(ccv_cnnp_model_t* const self, ccv_nnc_symbolic_graph_t* const graph, const ccv_nnc_tensor_symbol_t* const inputs, const int input_size, ccv_nnc_tensor_symbol_t* const outputs, const int output_size); /**< Call this graph to build computation. No need to specify input size or output size, as it is defined along in the model already. */ |
27 | | void (*init_states)(ccv_cnnp_model_t* const self, ccv_nnc_symbolic_graph_t* const graph, const ccv_cnnp_state_initializer_f initializer, void* const context); /**< This is called to init ccv_nnc_tensor_symbol_t with a exec. */ |
28 | | void (*add_to_parameter)(ccv_cnnp_model_t* const self, const ccv_cnnp_add_to_array_f add_to_array, void* const parameters, const int is_trainable); /**< This is called to add ccv_nnc_tensor_symbol_t to as list of parameters. */ |
29 | | void (*add_to_output)(ccv_cnnp_model_t* const self, const ccv_cnnp_add_to_array_f add_to_array, void* const outputs); /**< This is called to add ccv_nnc_tensor_symbol_t to as list of outputs for retention. The final outputs are already added. This method is optional for any additional values we want to retain. */ |
30 | | ccv_cnnp_model_t* (*copy)(const ccv_cnnp_model_t* const self, void* const context); /**< This is called to make a deep copy of itself. */ |
31 | | void (*set_is_test)(ccv_cnnp_model_t* const self, const int is_test, const ccv_cnnp_cmd_updater_f updater, void* const context); /**< This is called when it is switched between test or training. */ |
32 | | void (*add_to_parameter_indices)(ccv_cnnp_model_t* const self, const int index, ccv_array_t* const parameter_indices); /**< This is called when we try to get parameter indices out of a given model */ |
33 | | void (*notify)(const ccv_cnnp_model_t* const self, const int tag, void* const payload); /**< This is called when we want to notify something to this model. */ |
34 | | } ccv_cnnp_model_vtab_t; |
35 | | |
36 | | struct ccv_cnnp_model_io_s { |
37 | | int param_ref; // Reference to parameter in the model, starts with 1. 0 means no such thing. |
38 | | int param_sel; // Selector to parameter in the model, starts with 1. 0 means no selector. |
39 | | int visit; // Temporary bits stored in the ccv_cnnp_model_io_t object, whoever uses it should clean it up. |
40 | | ccv_cnnp_model_t* model; // Reference back to the model who holds it. This is required because the model is the one whole holds the io. |
41 | | ccv_array_t* incomings; // Array of ccv_cnnp_model_io_t. The order is important because it impacts the order of symbols. |
42 | | ccv_array_t* dependencies; // Array of ccv_cnnp_model_io_t. |
43 | | int dependents; // Number of dependents. |
44 | | ccv_array_t* outgoings; // Array of ccv_cnnp_model_io_t. |
45 | | ccv_nnc_tensor_symbol_t* outputs; // This is different from the outputs from a model. A model could be reused, causing the outputs on that model to be the most recent one. This keeps the outputs of each. |
46 | | }; |
47 | | |
48 | | enum { |
49 | | CCV_CNNP_MODEL_GRAPH_FIT_MODE, // This mode computes loss, backprop, and then apply gradients. |
50 | | CCV_CNNP_MODEL_GRAPH_MULTISTAGE_MODE_NO_GRAD, // This mode allows you to only use ccv_cnnp_model_evaluate (others require gradient). |
51 | | CCV_CNNP_MODEL_GRAPH_MULTISTAGE_MODE, // This mode allows you to use ccv_cnnp_model_evaluate, ccv_cnnp_model_backward, ccv_cnnp_model_apply_gradients separately. |
52 | | }; |
53 | | |
54 | | enum { |
55 | | CCV_CNNP_COMPILED_DATA_GRADIENT_NONE, |
56 | | CCV_CNNP_COMPILED_DATA_GRADIENT_TRAINABLES, |
57 | | CCV_CNNP_COMPILED_DATA_GRADIENT_TRAINABLES_AND_INPUTS, |
58 | | }; |
59 | | |
60 | | enum { |
61 | | CCV_CNNP_REWIND_GRAPH_EXEC, |
62 | | CCV_CNNP_REWIND_TENSOR, |
63 | | }; |
64 | | |
65 | | typedef struct { |
66 | | int type; |
67 | | union { |
68 | | ccv_nnc_tensor_symbol_t tensor; |
69 | | ccv_nnc_graph_exec_symbol_t graph_exec; |
70 | | }; |
71 | | } ccv_cnnp_rewind_symbol_t; |
72 | | |
73 | 1.55k | #define CCV_NNC_TENSOR(tv) ((ccv_nnc_tensor_t*)((uintptr_t)(tv) & ~(uintptr_t)1)) |
74 | 884 | #define CCV_NNC_INIT_V(v) ((uint32_t*)((uintptr_t)(v) & ~(uintptr_t)1)) |
75 | | |
76 | | // This contains relevant information after model compilation. |
77 | | typedef struct { |
78 | | int graph_mode; |
79 | | int gradient_mode; // Have init gradient graph. |
80 | | int is_test; |
81 | | int stream_type; |
82 | | int outgrad_size; |
83 | | uint64_t disable_outgrad; |
84 | | ccv_nnc_symbolic_graph_compile_param_t compile_params; |
85 | | ccv_nnc_xpu_alloc_t xpu_alloc; |
86 | | ccv_nnc_graph_t* graph; |
87 | | ccv_nnc_tensor_arena_t* tensor_arena; |
88 | | ccv_nnc_graph_exec_arena_t* graph_exec_arena; |
89 | | khash_t(stream_map)* stream_map; // Keeps track of streams on both GPU / CPU and devices so it can be used properly during execution. |
90 | | ccv_array_t* parameters; |
91 | | uint64_t* parameter_flags; |
92 | | ccv_array_t* internals; // Additional symbols need to retain. |
93 | | ccv_nnc_tensor_symbol_t* gradients; |
94 | | ccv_nnc_tensor_symbol_t* outgrads; |
95 | | ccv_nnc_tensor_symbol_t* updated_parameters; |
96 | | ccv_nnc_graph_exec_symbol_t* update_nodes; |
97 | | ccv_nnc_tensor_symbol_map_t* saved_aux; |
98 | | ccv_array_t* rewindables; |
99 | | ccv_array_t* gradient_checkpoints; |
100 | | struct { |
101 | | int size; |
102 | | uint32_t* v; // If the last is 1, we know it is incomplete (thus, the tensors_init_1 hasn't been called yet. This is to save RAM usage. |
103 | | } tensors_init; |
104 | | struct { |
105 | | ccv_nnc_tensor_t** internals; // Additional need to retained tensors. |
106 | | ccv_nnc_tensor_t** parameters; |
107 | | ccv_nnc_tensor_t** gradients; |
108 | | ccv_nnc_tensor_t** accum_gradients; |
109 | | } tensors; |
110 | | struct { |
111 | | ccv_array_t* parameters; |
112 | | ccv_array_t* internals; |
113 | | } ids; |
114 | | struct { |
115 | | int to_op_size; |
116 | | int to_size; |
117 | | ccv_nnc_graph_exec_t* to_ops; |
118 | | ccv_nnc_graph_exec_symbol_t* tos; |
119 | | ccv_nnc_graph_static_schedule_t* schedule; // The partial schedule for running evaluate step. |
120 | | } evaluate; // Data related to ccv_cnnp_model_evaluate |
121 | | struct { |
122 | | int count; // Called backward how many times. Starting with 0. |
123 | | int from_op_size; |
124 | | ccv_nnc_graph_exec_t* from_ops; // These are the ops in the main graph. |
125 | | int to_size; |
126 | | ccv_nnc_graph_exec_symbol_t* tos; |
127 | | ccv_nnc_graph_t* accum; // The graph to accumulate gradients. |
128 | | ccv_nnc_tensor_arena_t* tensor_arena; |
129 | | ccv_nnc_graph_exec_arena_t* graph_exec_arena; |
130 | | ccv_nnc_tensor_symbol_t* gradients; // The new gradients. |
131 | | ccv_nnc_tensor_symbol_t* accum_gradients; // The old accumulate gradients. |
132 | | ccv_nnc_tensor_symbol_t* updated_accum_gradients; // The new accumulate gradients. |
133 | | ccv_nnc_graph_static_schedule_t* schedule; // The partial schedule for running backward step. |
134 | | } backward; |
135 | | struct { |
136 | | ccv_nnc_graph_t* graph; |
137 | | ccv_nnc_tensor_arena_t* tensor_arena; |
138 | | ccv_nnc_graph_exec_arena_t* graph_exec_arena; |
139 | | } apply_gradients; |
140 | | struct { |
141 | | ccv_nnc_cmd_t minimizer; |
142 | | ccv_array_t* parameters; |
143 | | int max_saved_aux_size; |
144 | | } minimize; |
145 | | ccv_nnc_cmd_t loss; |
146 | | ccv_nnc_tensor_symbol_t* f; |
147 | | ccv_nnc_tensor_symbol_t fits[1]; |
148 | | } ccv_cnnp_compiled_data_t; |
149 | | |
150 | | struct ccv_cnnp_model_s { |
151 | | const ccv_cnnp_model_vtab_t* isa; |
152 | | int input_size; // This is the best effort number, mostly just for subclass to use. |
153 | | int output_size; |
154 | | int max_stream_count; |
155 | | int deinit_state; // If it is 1, it is already deinit. |
156 | | ccv_array_t* io; // The opaque io that can be nil. |
157 | | ccv_array_t* parameter_indices; // The indexes for parameters in the final model. |
158 | | ccv_nnc_symbolic_graph_t* graph; |
159 | | ccv_nnc_tensor_symbol_t* inputs; // Unlike outputs, which is not dynamically allocated, inputs is dynamically allocated, and may be 0. |
160 | | ccv_nnc_tensor_symbol_t* outputs; |
161 | | char* name; |
162 | | struct { |
163 | | ccv_cnnp_model_notify_f func; |
164 | | void* context; |
165 | | } notify_hook; |
166 | | ccv_cnnp_compiled_data_t* compiled_data; |
167 | | int parallel_count; // How many parallel devices. |
168 | | int memory_compression; // Whether to enable memory compression for training phase. |
169 | | int gradient_checkpointing; // Whether to enable gradient checkpointing for training phase. |
170 | | int is_trainable; // Whether this model can be trained or not. |
171 | | int memory_reduction; // Whether to enable memory reduction techniques for training phase. |
172 | | int exec_flags; // The flags to be applied to the execution nodes. |
173 | | size_t workspace_size; // Set the default workspace size. |
174 | | struct { |
175 | | ccv_cnnp_model_io_reader_f reader; |
176 | | ccv_cnnp_model_io_writer_f writer; |
177 | | } rw; |
178 | | void* data; // Temporary storage for some internal functions. |
179 | | }; |
180 | | |
181 | | KHASH_MAP_INIT_STR(ccv_cnnp_model_name_bank, int) |
182 | | |
183 | | typedef struct { |
184 | | int sequence; |
185 | | khash_t(ccv_cnnp_model_name_bank)* bank; |
186 | | const char* name; |
187 | | } ccv_cnnp_model_name_t; |
188 | | |
189 | | typedef struct { |
190 | | int it; |
191 | | ccv_cnnp_model_t* model; |
192 | | khash_t(ccv_cnnp_model_name_bank)* bank; |
193 | | ccv_array_t* sequences; |
194 | | } ccv_cnnp_model_sequence_t; |
195 | | |
196 | | static inline void ccv_cnnp_model_push(ccv_cnnp_model_t* const self, ccv_cnnp_model_sequence_t* const model_sequence) |
197 | 2.74k | { |
198 | | // Reset to 0. |
199 | 2.74k | if (!model_sequence->sequences) |
200 | 2.29k | model_sequence->sequences = ccv_array_new(sizeof(ccv_cnnp_model_name_t), 1, 0); |
201 | 2.74k | khash_t(ccv_cnnp_model_name_bank)* bank = model_sequence->sequences->rnum > 0 ? ((ccv_cnnp_model_name_t*)192 ccv_array_get192 (model_sequence->sequences, model_sequence->sequences->rnum - 1))->bank : model_sequence->bank2.55k ; |
202 | 2.74k | int ret; |
203 | 2.74k | khiter_t k = kh_put(ccv_cnnp_model_name_bank, bank, self->name ? self->name : "", &ret); |
204 | 2.74k | int sequence; |
205 | 2.74k | if (ret != 0) |
206 | 2.48k | sequence = kh_val(bank, k) = 0; |
207 | 264 | else |
208 | 264 | sequence = ++kh_val(bank, k); |
209 | 2.74k | ccv_cnnp_model_name_t name = { |
210 | 2.74k | .bank = kh_init(ccv_cnnp_model_name_bank), |
211 | 2.74k | .name = self->name, |
212 | 2.74k | .sequence = sequence, |
213 | 2.74k | }; |
214 | 2.74k | ccv_array_push(model_sequence->sequences, &name); |
215 | 2.74k | model_sequence->model = self; |
216 | 2.74k | } Unexecuted instantiation: ccv_nnc_dynamic_graph_evaluate.c:ccv_cnnp_model_push ccv_cnnp_model.c:ccv_cnnp_model_push Line | Count | Source | 197 | 1.24k | { | 198 | | // Reset to 0. | 199 | 1.24k | if (!model_sequence->sequences) | 200 | 1.24k | model_sequence->sequences = ccv_array_new(sizeof(ccv_cnnp_model_name_t), 1, 0); | 201 | 1.24k | khash_t(ccv_cnnp_model_name_bank)* bank = model_sequence->sequences->rnum > 0 ? ((ccv_cnnp_model_name_t*)0 ccv_array_get0 (model_sequence->sequences, model_sequence->sequences->rnum - 1))->bank : model_sequence->bank; | 202 | 1.24k | int ret; | 203 | 1.24k | khiter_t k = kh_put(ccv_cnnp_model_name_bank, bank, self->name ? self->name : "", &ret); | 204 | 1.24k | int sequence; | 205 | 1.24k | if (ret != 0) | 206 | 1.24k | sequence = kh_val(bank, k) = 0; | 207 | 0 | else | 208 | 0 | sequence = ++kh_val(bank, k); | 209 | 1.24k | ccv_cnnp_model_name_t name = { | 210 | 1.24k | .bank = kh_init(ccv_cnnp_model_name_bank), | 211 | 1.24k | .name = self->name, | 212 | 1.24k | .sequence = sequence, | 213 | 1.24k | }; | 214 | 1.24k | ccv_array_push(model_sequence->sequences, &name); | 215 | 1.24k | model_sequence->model = self; | 216 | 1.24k | } |
Unexecuted instantiation: ccv_cnnp_model_io.c:ccv_cnnp_model_push ccv_cnnp_model_core.c:ccv_cnnp_model_push Line | Count | Source | 197 | 1.50k | { | 198 | | // Reset to 0. | 199 | 1.50k | if (!model_sequence->sequences) | 200 | 1.04k | model_sequence->sequences = ccv_array_new(sizeof(ccv_cnnp_model_name_t), 1, 0); | 201 | 1.50k | khash_t(ccv_cnnp_model_name_bank)* bank = model_sequence->sequences->rnum > 0 ? ((ccv_cnnp_model_name_t*)192 ccv_array_get192 (model_sequence->sequences, model_sequence->sequences->rnum - 1))->bank : model_sequence->bank1.31k ; | 202 | 1.50k | int ret; | 203 | 1.50k | khiter_t k = kh_put(ccv_cnnp_model_name_bank, bank, self->name ? self->name : "", &ret); | 204 | 1.50k | int sequence; | 205 | 1.50k | if (ret != 0) | 206 | 1.24k | sequence = kh_val(bank, k) = 0; | 207 | 264 | else | 208 | 264 | sequence = ++kh_val(bank, k); | 209 | 1.50k | ccv_cnnp_model_name_t name = { | 210 | 1.50k | .bank = kh_init(ccv_cnnp_model_name_bank), | 211 | 1.50k | .name = self->name, | 212 | 1.50k | .sequence = sequence, | 213 | 1.50k | }; | 214 | 1.50k | ccv_array_push(model_sequence->sequences, &name); | 215 | 1.50k | model_sequence->model = self; | 216 | 1.50k | } |
Unexecuted instantiation: ccv_cnnp_model_addons.c:ccv_cnnp_model_push Unexecuted instantiation: ccv_cnnp_model_gradient_checkpointing.c:ccv_cnnp_model_push |
217 | | |
218 | | static inline void ccv_cnnp_model_pop(const ccv_cnnp_model_t* const self, ccv_cnnp_model_sequence_t* const model_sequence) |
219 | 2.74k | { |
220 | 2.74k | khash_t(ccv_cnnp_model_name_bank)* const bank = ((ccv_cnnp_model_name_t*)ccv_array_get(model_sequence->sequences, model_sequence->sequences->rnum - 1))->bank; |
221 | 2.74k | kh_destroy(ccv_cnnp_model_name_bank, bank); |
222 | 2.74k | --model_sequence->sequences->rnum; |
223 | 2.74k | assert(model_sequence->sequences->rnum >= 0); |
224 | 2.74k | model_sequence->model = 0; |
225 | 2.74k | } Unexecuted instantiation: ccv_nnc_dynamic_graph_evaluate.c:ccv_cnnp_model_pop ccv_cnnp_model.c:ccv_cnnp_model_pop Line | Count | Source | 219 | 1.24k | { | 220 | 1.24k | khash_t(ccv_cnnp_model_name_bank)* const bank = ((ccv_cnnp_model_name_t*)ccv_array_get(model_sequence->sequences, model_sequence->sequences->rnum - 1))->bank; | 221 | 1.24k | kh_destroy(ccv_cnnp_model_name_bank, bank); | 222 | 1.24k | --model_sequence->sequences->rnum; | 223 | 1.24k | assert(model_sequence->sequences->rnum >= 0); | 224 | 1.24k | model_sequence->model = 0; | 225 | 1.24k | } |
Unexecuted instantiation: ccv_cnnp_model_io.c:ccv_cnnp_model_pop ccv_cnnp_model_core.c:ccv_cnnp_model_pop Line | Count | Source | 219 | 1.50k | { | 220 | 1.50k | khash_t(ccv_cnnp_model_name_bank)* const bank = ((ccv_cnnp_model_name_t*)ccv_array_get(model_sequence->sequences, model_sequence->sequences->rnum - 1))->bank; | 221 | 1.50k | kh_destroy(ccv_cnnp_model_name_bank, bank); | 222 | 1.50k | --model_sequence->sequences->rnum; | 223 | 1.50k | assert(model_sequence->sequences->rnum >= 0); | 224 | 1.50k | model_sequence->model = 0; | 225 | 1.50k | } |
Unexecuted instantiation: ccv_cnnp_model_addons.c:ccv_cnnp_model_pop Unexecuted instantiation: ccv_cnnp_model_gradient_checkpointing.c:ccv_cnnp_model_pop |
226 | | |
227 | | static inline ccv_cnnp_model_t* _ccv_cnnp_model_copy(const ccv_cnnp_model_t* const model, void* const context) |
228 | 4.30k | { |
229 | 4.30k | assert(model->isa->copy); |
230 | 4.30k | ccv_cnnp_model_t* const copy = model->isa->copy(model, context); |
231 | 4.30k | copy->parallel_count = model->parallel_count; |
232 | 4.30k | copy->memory_compression = model->memory_compression; |
233 | 4.30k | copy->memory_reduction = model->memory_reduction; |
234 | 4.30k | copy->max_stream_count = model->max_stream_count; |
235 | 4.30k | copy->gradient_checkpointing = model->gradient_checkpointing; |
236 | 4.30k | return copy; |
237 | 4.30k | } Unexecuted instantiation: ccv_nnc_dynamic_graph_evaluate.c:_ccv_cnnp_model_copy ccv_cnnp_model.c:_ccv_cnnp_model_copy Line | Count | Source | 228 | 2.20k | { | 229 | 2.20k | assert(model->isa->copy); | 230 | 2.20k | ccv_cnnp_model_t* const copy = model->isa->copy(model, context); | 231 | 2.20k | copy->parallel_count = model->parallel_count; | 232 | 2.20k | copy->memory_compression = model->memory_compression; | 233 | 2.20k | copy->memory_reduction = model->memory_reduction; | 234 | 2.20k | copy->max_stream_count = model->max_stream_count; | 235 | 2.20k | copy->gradient_checkpointing = model->gradient_checkpointing; | 236 | 2.20k | return copy; | 237 | 2.20k | } |
Unexecuted instantiation: ccv_cnnp_model_io.c:_ccv_cnnp_model_copy ccv_cnnp_model_core.c:_ccv_cnnp_model_copy Line | Count | Source | 228 | 2.09k | { | 229 | 2.09k | assert(model->isa->copy); | 230 | 2.09k | ccv_cnnp_model_t* const copy = model->isa->copy(model, context); | 231 | 2.09k | copy->parallel_count = model->parallel_count; | 232 | 2.09k | copy->memory_compression = model->memory_compression; | 233 | 2.09k | copy->memory_reduction = model->memory_reduction; | 234 | 2.09k | copy->max_stream_count = model->max_stream_count; | 235 | 2.09k | copy->gradient_checkpointing = model->gradient_checkpointing; | 236 | 2.09k | return copy; | 237 | 2.09k | } |
Unexecuted instantiation: ccv_cnnp_model_addons.c:_ccv_cnnp_model_copy Unexecuted instantiation: ccv_cnnp_model_gradient_checkpointing.c:_ccv_cnnp_model_copy |
238 | | |
239 | | static inline void ccv_cnnp_model_copy_name(ccv_cnnp_model_t* const self, const char* const name) |
240 | 5.27k | { |
241 | 5.27k | if (name) |
242 | 216 | { |
243 | 216 | const size_t len = strnlen(name, 63); |
244 | 216 | const size_t n = len + 1; |
245 | 216 | self->name = (char*)ccmalloc(n); |
246 | | // Don't use strndup because this way I can have custom allocator (for ccmalloc). |
247 | 216 | memcpy(self->name, name, n); |
248 | 216 | self->name[len] = 0; |
249 | 216 | } |
250 | 5.27k | } Unexecuted instantiation: ccv_nnc_dynamic_graph_evaluate.c:ccv_cnnp_model_copy_name Unexecuted instantiation: ccv_cnnp_model.c:ccv_cnnp_model_copy_name Unexecuted instantiation: ccv_cnnp_model_io.c:ccv_cnnp_model_copy_name ccv_cnnp_model_core.c:ccv_cnnp_model_copy_name Line | Count | Source | 240 | 1.31k | { | 241 | 1.31k | if (name) | 242 | 84 | { | 243 | 84 | const size_t len = strnlen(name, 63); | 244 | 84 | const size_t n = len + 1; | 245 | 84 | self->name = (char*)ccmalloc(n); | 246 | | // Don't use strndup because this way I can have custom allocator (for ccmalloc). | 247 | 84 | memcpy(self->name, name, n); | 248 | 84 | self->name[len] = 0; | 249 | 84 | } | 250 | 1.31k | } |
ccv_cnnp_model_addons.c:ccv_cnnp_model_copy_name Line | Count | Source | 240 | 3.96k | { | 241 | 3.96k | if (name) | 242 | 132 | { | 243 | 132 | const size_t len = strnlen(name, 63); | 244 | 132 | const size_t n = len + 1; | 245 | 132 | self->name = (char*)ccmalloc(n); | 246 | | // Don't use strndup because this way I can have custom allocator (for ccmalloc). | 247 | 132 | memcpy(self->name, name, n); | 248 | 132 | self->name[len] = 0; | 249 | 132 | } | 250 | 3.96k | } |
Unexecuted instantiation: ccv_cnnp_model_gradient_checkpointing.c:ccv_cnnp_model_copy_name |
251 | | |
252 | | static inline void ccv_cnnp_model_add_to_parameter(ccv_cnnp_model_t* const self, const ccv_cnnp_add_to_array_f add_to_array, void* const parameters, const int is_trainable) |
253 | 2.63k | { |
254 | 2.63k | if (self->isa->add_to_parameter) |
255 | 2.63k | self->isa->add_to_parameter(self, add_to_array, parameters, is_trainable); |
256 | 2.63k | } Unexecuted instantiation: ccv_nnc_dynamic_graph_evaluate.c:ccv_cnnp_model_add_to_parameter ccv_cnnp_model.c:ccv_cnnp_model_add_to_parameter Line | Count | Source | 253 | 1.20k | { | 254 | 1.20k | if (self->isa->add_to_parameter) | 255 | 1.20k | self->isa->add_to_parameter(self, add_to_array, parameters, is_trainable); | 256 | 1.20k | } |
Unexecuted instantiation: ccv_cnnp_model_io.c:ccv_cnnp_model_add_to_parameter ccv_cnnp_model_core.c:ccv_cnnp_model_add_to_parameter Line | Count | Source | 253 | 1.42k | { | 254 | 1.42k | if (self->isa->add_to_parameter) | 255 | 1.42k | self->isa->add_to_parameter(self, add_to_array, parameters, is_trainable); | 256 | 1.42k | } |
Unexecuted instantiation: ccv_cnnp_model_addons.c:ccv_cnnp_model_add_to_parameter Unexecuted instantiation: ccv_cnnp_model_gradient_checkpointing.c:ccv_cnnp_model_add_to_parameter |
257 | | |
258 | | static inline void ccv_cnnp_model_add_to_output(ccv_cnnp_model_t* const self, const ccv_cnnp_add_to_array_f add_to_array, void* const outputs) |
259 | 2.63k | { |
260 | 2.63k | if (self->isa->add_to_output) |
261 | 176 | self->isa->add_to_output(self, add_to_array, outputs); |
262 | 2.63k | } Unexecuted instantiation: ccv_nnc_dynamic_graph_evaluate.c:ccv_cnnp_model_add_to_output ccv_cnnp_model.c:ccv_cnnp_model_add_to_output Line | Count | Source | 259 | 1.20k | { | 260 | 1.20k | if (self->isa->add_to_output) | 261 | 0 | self->isa->add_to_output(self, add_to_array, outputs); | 262 | 1.20k | } |
Unexecuted instantiation: ccv_cnnp_model_io.c:ccv_cnnp_model_add_to_output ccv_cnnp_model_core.c:ccv_cnnp_model_add_to_output Line | Count | Source | 259 | 1.42k | { | 260 | 1.42k | if (self->isa->add_to_output) | 261 | 176 | self->isa->add_to_output(self, add_to_array, outputs); | 262 | 1.42k | } |
Unexecuted instantiation: ccv_cnnp_model_addons.c:ccv_cnnp_model_add_to_output Unexecuted instantiation: ccv_cnnp_model_gradient_checkpointing.c:ccv_cnnp_model_add_to_output |
263 | | |
264 | | static inline void ccv_cnnp_model_deinit(ccv_cnnp_model_t* const self) |
265 | 8.58k | { |
266 | 8.58k | if (self->deinit_state) |
267 | 3.13k | return; |
268 | 5.44k | if (self->isa->deinit) |
269 | 1.39k | self->isa->deinit(self); |
270 | 5.44k | self->deinit_state = 1; |
271 | 5.44k | } Unexecuted instantiation: ccv_nnc_dynamic_graph_evaluate.c:ccv_cnnp_model_deinit ccv_cnnp_model.c:ccv_cnnp_model_deinit Line | Count | Source | 265 | 5.44k | { | 266 | 5.44k | if (self->deinit_state) | 267 | 3.13k | return; | 268 | 2.30k | if (self->isa->deinit) | 269 | 1.07k | self->isa->deinit(self); | 270 | 2.30k | self->deinit_state = 1; | 271 | 2.30k | } |
Unexecuted instantiation: ccv_cnnp_model_io.c:ccv_cnnp_model_deinit ccv_cnnp_model_core.c:ccv_cnnp_model_deinit Line | Count | Source | 265 | 3.13k | { | 266 | 3.13k | if (self->deinit_state) | 267 | 0 | return; | 268 | 3.13k | if (self->isa->deinit) | 269 | 315 | self->isa->deinit(self); | 270 | 3.13k | self->deinit_state = 1; | 271 | 3.13k | } |
Unexecuted instantiation: ccv_cnnp_model_addons.c:ccv_cnnp_model_deinit Unexecuted instantiation: ccv_cnnp_model_gradient_checkpointing.c:ccv_cnnp_model_deinit |
272 | | |
273 | | typedef struct { |
274 | | int exec_flags; |
275 | | int is_trainable; |
276 | | int is_gradient_checkpointing; |
277 | | ccv_cnnp_model_sequence_t* model_sequence; |
278 | | ccv_cnnp_add_to_array_f add_to_array; |
279 | | ccv_array_t* parameters; |
280 | | struct { |
281 | | void* add_to_parameter; |
282 | | void* add_to_output; |
283 | | } context; |
284 | | ccv_array_t* gradient_checkpoints; |
285 | | } ccv_cnnp_model_build_data_t; // Host temporary data for building models. |
286 | | |
287 | | typedef struct { |
288 | | int input_size; |
289 | | int output_size; |
290 | | int is_trainable; |
291 | | ccv_cnnp_model_t* model; |
292 | | void (*build)(ccv_cnnp_model_t* const self, ccv_nnc_symbolic_graph_t* const graph, const ccv_nnc_tensor_symbol_t* const inputs, const int input_size, ccv_nnc_tensor_symbol_t* const outputs, const int output_size); |
293 | | ccv_array_t* tensor_symbols; |
294 | | ccv_nnc_tensor_symbol_t* inputs; |
295 | | ccv_nnc_tensor_symbol_t* outputs; |
296 | | } ccv_cnnp_model_gradient_checkpoint_t; |
297 | | |
298 | | static inline ccv_nnc_tensor_symbol_t ccv_cnnp_parameter_from_indice(ccv_cnnp_model_t* const self, const int indice) |
299 | 2 | { |
300 | 2 | assert(self->data); |
301 | 2 | ccv_cnnp_model_build_data_t* const build_data = (ccv_cnnp_model_build_data_t*)self->data; |
302 | 2 | assert(indice < build_data->parameters->rnum); |
303 | 2 | return *(ccv_nnc_tensor_symbol_t*)ccv_array_get(build_data->parameters, indice); |
304 | 2 | } Unexecuted instantiation: ccv_nnc_dynamic_graph_evaluate.c:ccv_cnnp_parameter_from_indice Unexecuted instantiation: ccv_cnnp_model.c:ccv_cnnp_parameter_from_indice Unexecuted instantiation: ccv_cnnp_model_io.c:ccv_cnnp_parameter_from_indice ccv_cnnp_model_core.c:ccv_cnnp_parameter_from_indice Line | Count | Source | 299 | 2 | { | 300 | 2 | assert(self->data); | 301 | 2 | ccv_cnnp_model_build_data_t* const build_data = (ccv_cnnp_model_build_data_t*)self->data; | 302 | 2 | assert(indice < build_data->parameters->rnum); | 303 | 2 | return *(ccv_nnc_tensor_symbol_t*)ccv_array_get(build_data->parameters, indice); | 304 | 2 | } |
Unexecuted instantiation: ccv_cnnp_model_addons.c:ccv_cnnp_parameter_from_indice Unexecuted instantiation: ccv_cnnp_model_gradient_checkpointing.c:ccv_cnnp_parameter_from_indice |
305 | | |
306 | | typedef struct { |
307 | | int record; |
308 | | ccv_array_t* tensor_symbols; |
309 | | void* old_tensor_symbol_new_hook_context; |
310 | | ccv_nnc_tensor_symbol_new_hook_f old_tensor_symbol_new_hook; |
311 | | void* old_tensor_symbol_alias_new_hook_context; |
312 | | ccv_nnc_tensor_symbol_alias_new_hook_f old_tensor_symbol_alias_new_hook; |
313 | | } ccv_cnnp_model_gradient_checkpoint_build_context_t; |
314 | | |
315 | | static void _ccv_cnnp_model_gradient_checkpoint_tensor_symbol_new_hook(void* context, const ccv_nnc_tensor_symbol_t symbol, const ccv_nnc_tensor_param_t info, const char* const name) |
316 | 30 | { |
317 | 30 | ccv_cnnp_model_gradient_checkpoint_build_context_t* const build_context = (ccv_cnnp_model_gradient_checkpoint_build_context_t*)context; |
318 | 30 | if (build_context->record) |
319 | 30 | ccv_array_push(build_context->tensor_symbols, &symbol); |
320 | 30 | if (build_context->old_tensor_symbol_new_hook) |
321 | 0 | build_context->old_tensor_symbol_new_hook(build_context->old_tensor_symbol_new_hook_context, symbol, info, name); |
322 | 30 | } Unexecuted instantiation: ccv_nnc_dynamic_graph_evaluate.c:_ccv_cnnp_model_gradient_checkpoint_tensor_symbol_new_hook ccv_cnnp_model.c:_ccv_cnnp_model_gradient_checkpoint_tensor_symbol_new_hook Line | Count | Source | 316 | 30 | { | 317 | 30 | ccv_cnnp_model_gradient_checkpoint_build_context_t* const build_context = (ccv_cnnp_model_gradient_checkpoint_build_context_t*)context; | 318 | 30 | if (build_context->record) | 319 | 30 | ccv_array_push(build_context->tensor_symbols, &symbol); | 320 | 30 | if (build_context->old_tensor_symbol_new_hook) | 321 | 0 | build_context->old_tensor_symbol_new_hook(build_context->old_tensor_symbol_new_hook_context, symbol, info, name); | 322 | 30 | } |
Unexecuted instantiation: ccv_cnnp_model_io.c:_ccv_cnnp_model_gradient_checkpoint_tensor_symbol_new_hook Unexecuted instantiation: ccv_cnnp_model_core.c:_ccv_cnnp_model_gradient_checkpoint_tensor_symbol_new_hook Unexecuted instantiation: ccv_cnnp_model_addons.c:_ccv_cnnp_model_gradient_checkpoint_tensor_symbol_new_hook Unexecuted instantiation: ccv_cnnp_model_gradient_checkpointing.c:_ccv_cnnp_model_gradient_checkpoint_tensor_symbol_new_hook |
323 | | |
324 | | static void _ccv_cnnp_model_gradient_checkpoint_tensor_symbol_alias_new_hook(void* context, const ccv_nnc_tensor_symbol_t symbol, const ccv_nnc_tensor_symbol_t from_symbol, const int ofs[CCV_NNC_MAX_DIM_ALLOC], const int inc[CCV_NNC_MAX_DIM_ALLOC], const ccv_nnc_tensor_param_t info, const char* const name) |
325 | 0 | { |
326 | 0 | ccv_cnnp_model_gradient_checkpoint_build_context_t* const build_context = (ccv_cnnp_model_gradient_checkpoint_build_context_t*)context; |
327 | 0 | if (build_context->record) |
328 | 0 | ccv_array_push(build_context->tensor_symbols, &symbol); |
329 | 0 | if (build_context->old_tensor_symbol_alias_new_hook) |
330 | 0 | build_context->old_tensor_symbol_alias_new_hook(build_context->old_tensor_symbol_alias_new_hook_context, symbol, from_symbol, ofs, inc, info, name); |
331 | 0 | } Unexecuted instantiation: ccv_nnc_dynamic_graph_evaluate.c:_ccv_cnnp_model_gradient_checkpoint_tensor_symbol_alias_new_hook Unexecuted instantiation: ccv_cnnp_model.c:_ccv_cnnp_model_gradient_checkpoint_tensor_symbol_alias_new_hook Unexecuted instantiation: ccv_cnnp_model_io.c:_ccv_cnnp_model_gradient_checkpoint_tensor_symbol_alias_new_hook Unexecuted instantiation: ccv_cnnp_model_core.c:_ccv_cnnp_model_gradient_checkpoint_tensor_symbol_alias_new_hook Unexecuted instantiation: ccv_cnnp_model_addons.c:_ccv_cnnp_model_gradient_checkpoint_tensor_symbol_alias_new_hook Unexecuted instantiation: ccv_cnnp_model_gradient_checkpointing.c:_ccv_cnnp_model_gradient_checkpoint_tensor_symbol_alias_new_hook |
332 | | |
333 | | static inline void ccv_cnnp_model_build(ccv_cnnp_model_t* const self, ccv_nnc_symbolic_graph_t* const graph, const ccv_nnc_tensor_symbol_t* const inputs, const int input_size, ccv_nnc_tensor_symbol_t* const outputs, const int output_size) |
334 | 5.24k | { |
335 | 5.24k | assert(self->data); |
336 | 5.24k | ccv_cnnp_model_build_data_t* const build_data = (ccv_cnnp_model_build_data_t*)self->data; |
337 | 5.24k | const int old_exec_flags = build_data->exec_flags; |
338 | 5.24k | const int old_is_trainable = build_data->is_trainable; |
339 | 5.24k | if (self->exec_flags) |
340 | 0 | build_data->exec_flags |= self->exec_flags; |
341 | 5.24k | if (self->is_trainable >= 0) |
342 | 5.22k | build_data->is_trainable = self->is_trainable; |
343 | 5.24k | if (self->name && self->name[0] != '\0'252 ) |
344 | 252 | ccv_cnnp_model_push(self, build_data->model_sequence); |
345 | 5.24k | if (self->gradient_checkpointing == 1 && !build_data->is_gradient_checkpointing2 ) |
346 | 2 | { |
347 | 2 | build_data->is_gradient_checkpointing = 1; |
348 | | // Prepare to record gradient checkpoint. We will log the build function, inputs, what are the tensors / graph execs we created. |
349 | 2 | if (!build_data->gradient_checkpoints) |
350 | 2 | build_data->gradient_checkpoints = ccv_array_new(sizeof(ccv_cnnp_model_gradient_checkpoint_t), 0, 0); |
351 | 2 | ccv_cnnp_model_gradient_checkpoint_build_context_t build_context = { |
352 | 2 | .record = 1, |
353 | 2 | .tensor_symbols = ccv_array_new(sizeof(ccv_nnc_tensor_symbol_t), 0, 0), |
354 | 2 | }; |
355 | 2 | build_context.old_tensor_symbol_new_hook_context = ccv_nnc_tensor_symbol_new_hook(graph, _ccv_cnnp_model_gradient_checkpoint_tensor_symbol_new_hook, &build_context, &build_context.old_tensor_symbol_new_hook); |
356 | 2 | build_context.old_tensor_symbol_alias_new_hook_context = ccv_nnc_tensor_symbol_alias_new_hook(graph, _ccv_cnnp_model_gradient_checkpoint_tensor_symbol_alias_new_hook, &build_context, &build_context.old_tensor_symbol_alias_new_hook); |
357 | 2 | if (outputs && output_size0 ) |
358 | 0 | { |
359 | 0 | assert(output_size == self->output_size); |
360 | 0 | self->isa->build(self, graph, inputs, input_size, outputs, output_size); |
361 | 0 | memcpy(self->outputs, outputs, sizeof(ccv_nnc_tensor_symbol_t) * output_size); |
362 | 0 | } else |
363 | 2 | self->isa->build(self, graph, inputs, input_size, self->outputs, self->output_size); |
364 | 2 | ccv_nnc_tensor_symbol_new_hook(graph, build_context.old_tensor_symbol_new_hook, build_context.old_tensor_symbol_new_hook_context, 0); |
365 | 2 | ccv_nnc_tensor_symbol_alias_new_hook(graph, build_context.old_tensor_symbol_alias_new_hook, build_context.old_tensor_symbol_alias_new_hook_context, 0); |
366 | 2 | ccv_cnnp_model_gradient_checkpoint_t checkpoint = { |
367 | 2 | .input_size = input_size, |
368 | 2 | .output_size = (outputs && output_size > 00 ) ? output_size0 : self->output_size, |
369 | 2 | .is_trainable = build_data->is_trainable, |
370 | 2 | .model = self, |
371 | 2 | .build = self->isa->build, |
372 | 2 | .tensor_symbols = build_context.tensor_symbols, |
373 | 2 | .inputs = ccmalloc(sizeof(ccv_nnc_tensor_symbol_t) * (input_size + ((outputs && output_size > 00 ) ? output_size0 : self->output_size))), |
374 | 2 | }; |
375 | 2 | checkpoint.outputs = checkpoint.inputs + input_size; |
376 | 2 | if (input_size > 0) |
377 | 2 | memcpy(checkpoint.inputs, inputs, sizeof(ccv_nnc_tensor_symbol_t) * input_size); |
378 | 2 | if (outputs && output_size > 00 ) |
379 | 0 | memcpy(checkpoint.outputs, outputs, sizeof(ccv_nnc_tensor_symbol_t) * output_size); |
380 | 2 | else if (self->outputs && self->output_size > 0) |
381 | 2 | memcpy(checkpoint.outputs, self->outputs, sizeof(ccv_nnc_tensor_symbol_t) * self->output_size); |
382 | 2 | ccv_array_push(build_data->gradient_checkpoints, &checkpoint); |
383 | 2 | build_data->is_gradient_checkpointing = 0; |
384 | 5.24k | } else { |
385 | | // If we want to disable gradient checkpointing for this model, we simply not log any new tensors created here so there is no mapping for these. |
386 | 5.24k | int old_record; |
387 | 5.24k | ccv_cnnp_model_gradient_checkpoint_build_context_t* build_context = 0; |
388 | 5.24k | if (build_data->is_gradient_checkpointing) |
389 | 36 | { |
390 | 36 | if (self->gradient_checkpointing == -1) |
391 | 0 | { |
392 | 0 | ccv_nnc_tensor_symbol_new_hook_f old_tensor_symbol_new_hook; |
393 | 0 | build_context = ccv_nnc_tensor_symbol_new_hook(graph, 0, 0, &old_tensor_symbol_new_hook); |
394 | | // Set back the build_context. |
395 | 0 | ccv_nnc_tensor_symbol_new_hook(graph, old_tensor_symbol_new_hook, build_context, 0); |
396 | 0 | old_record = build_context->record; |
397 | 0 | build_context->record = 0; |
398 | 36 | } else if (self->gradient_checkpointing == 1) { // Force to turn on gradient checkpointing if it is inside a gradient checkpointing = -1. |
399 | 0 | ccv_nnc_tensor_symbol_new_hook_f old_tensor_symbol_new_hook; |
400 | 0 | build_context = ccv_nnc_tensor_symbol_new_hook(graph, 0, 0, &old_tensor_symbol_new_hook); |
401 | | // Set back the build_context. |
402 | 0 | ccv_nnc_tensor_symbol_new_hook(graph, old_tensor_symbol_new_hook, build_context, 0); |
403 | 0 | old_record = build_context->record; |
404 | 0 | build_context->record = 1; |
405 | 0 | } |
406 | 36 | } |
407 | | // No push checkpoint, easy. |
408 | 5.24k | if (outputs && output_size2.93k ) |
409 | 2.93k | { |
410 | 2.93k | assert(output_size == self->output_size); |
411 | 2.93k | self->isa->build(self, graph, inputs, input_size, outputs, output_size); |
412 | 2.93k | memcpy(self->outputs, outputs, sizeof(ccv_nnc_tensor_symbol_t) * output_size); |
413 | 2.93k | } else |
414 | 2.30k | self->isa->build(self, graph, inputs, input_size, self->outputs, self->output_size); |
415 | 5.24k | if (build_context) // Restore previous state even if our gradient checkpointing controlled whether to turn on recording or not. |
416 | 0 | build_context->record = old_record; |
417 | 5.24k | } |
418 | | // Skip if there is none. This helps to load parameters to a different model when only changes non-parameterized settings (add reshapes, permutations etc). |
419 | | // If it is named, we have to push too. |
420 | 5.24k | if (self->isa->add_to_parameter || self->isa->add_to_output2.60k ) |
421 | 2.63k | { |
422 | | // If we already pushed, no need to push again. |
423 | 2.63k | if (!(self->name && self->name[0] != '\0'138 )) |
424 | 2.49k | ccv_cnnp_model_push(self, build_data->model_sequence); |
425 | 2.63k | build_data->model_sequence->it = 0; |
426 | 2.63k | ccv_cnnp_model_add_to_parameter(self, build_data->add_to_array, build_data->context.add_to_parameter, build_data->is_trainable); |
427 | 2.63k | build_data->model_sequence->it = 0; |
428 | 2.63k | ccv_cnnp_model_add_to_output(self, build_data->add_to_array, build_data->context.add_to_output); |
429 | 2.63k | ccv_cnnp_model_pop(self, build_data->model_sequence); |
430 | 2.63k | } else if (2.60k self->name2.60k && self->name[0] != '\0'114 ) |
431 | 114 | ccv_cnnp_model_pop(self, build_data->model_sequence); |
432 | 5.24k | build_data->exec_flags = old_exec_flags; |
433 | 5.24k | build_data->is_trainable = old_is_trainable; |
434 | 5.24k | } Unexecuted instantiation: ccv_nnc_dynamic_graph_evaluate.c:ccv_cnnp_model_build ccv_cnnp_model.c:ccv_cnnp_model_build Line | Count | Source | 334 | 2.30k | { | 335 | 2.30k | assert(self->data); | 336 | 2.30k | ccv_cnnp_model_build_data_t* const build_data = (ccv_cnnp_model_build_data_t*)self->data; | 337 | 2.30k | const int old_exec_flags = build_data->exec_flags; | 338 | 2.30k | const int old_is_trainable = build_data->is_trainable; | 339 | 2.30k | if (self->exec_flags) | 340 | 0 | build_data->exec_flags |= self->exec_flags; | 341 | 2.30k | if (self->is_trainable >= 0) | 342 | 2.29k | build_data->is_trainable = self->is_trainable; | 343 | 2.30k | if (self->name && self->name[0] != '\0'39 ) | 344 | 39 | ccv_cnnp_model_push(self, build_data->model_sequence); | 345 | 2.30k | if (self->gradient_checkpointing == 1 && !build_data->is_gradient_checkpointing2 ) | 346 | 2 | { | 347 | 2 | build_data->is_gradient_checkpointing = 1; | 348 | | // Prepare to record gradient checkpoint. We will log the build function, inputs, what are the tensors / graph execs we created. | 349 | 2 | if (!build_data->gradient_checkpoints) | 350 | 2 | build_data->gradient_checkpoints = ccv_array_new(sizeof(ccv_cnnp_model_gradient_checkpoint_t), 0, 0); | 351 | 2 | ccv_cnnp_model_gradient_checkpoint_build_context_t build_context = { | 352 | 2 | .record = 1, | 353 | 2 | .tensor_symbols = ccv_array_new(sizeof(ccv_nnc_tensor_symbol_t), 0, 0), | 354 | 2 | }; | 355 | 2 | build_context.old_tensor_symbol_new_hook_context = ccv_nnc_tensor_symbol_new_hook(graph, _ccv_cnnp_model_gradient_checkpoint_tensor_symbol_new_hook, &build_context, &build_context.old_tensor_symbol_new_hook); | 356 | 2 | build_context.old_tensor_symbol_alias_new_hook_context = ccv_nnc_tensor_symbol_alias_new_hook(graph, _ccv_cnnp_model_gradient_checkpoint_tensor_symbol_alias_new_hook, &build_context, &build_context.old_tensor_symbol_alias_new_hook); | 357 | 2 | if (outputs && output_size0 ) | 358 | 0 | { | 359 | 0 | assert(output_size == self->output_size); | 360 | 0 | self->isa->build(self, graph, inputs, input_size, outputs, output_size); | 361 | 0 | memcpy(self->outputs, outputs, sizeof(ccv_nnc_tensor_symbol_t) * output_size); | 362 | 0 | } else | 363 | 2 | self->isa->build(self, graph, inputs, input_size, self->outputs, self->output_size); | 364 | 2 | ccv_nnc_tensor_symbol_new_hook(graph, build_context.old_tensor_symbol_new_hook, build_context.old_tensor_symbol_new_hook_context, 0); | 365 | 2 | ccv_nnc_tensor_symbol_alias_new_hook(graph, build_context.old_tensor_symbol_alias_new_hook, build_context.old_tensor_symbol_alias_new_hook_context, 0); | 366 | 2 | ccv_cnnp_model_gradient_checkpoint_t checkpoint = { | 367 | 2 | .input_size = input_size, | 368 | 2 | .output_size = (outputs && output_size > 00 ) ? output_size0 : self->output_size, | 369 | 2 | .is_trainable = build_data->is_trainable, | 370 | 2 | .model = self, | 371 | 2 | .build = self->isa->build, | 372 | 2 | .tensor_symbols = build_context.tensor_symbols, | 373 | 2 | .inputs = ccmalloc(sizeof(ccv_nnc_tensor_symbol_t) * (input_size + ((outputs && output_size > 00 ) ? output_size0 : self->output_size))), | 374 | 2 | }; | 375 | 2 | checkpoint.outputs = checkpoint.inputs + input_size; | 376 | 2 | if (input_size > 0) | 377 | 2 | memcpy(checkpoint.inputs, inputs, sizeof(ccv_nnc_tensor_symbol_t) * input_size); | 378 | 2 | if (outputs && output_size > 00 ) | 379 | 0 | memcpy(checkpoint.outputs, outputs, sizeof(ccv_nnc_tensor_symbol_t) * output_size); | 380 | 2 | else if (self->outputs && self->output_size > 0) | 381 | 2 | memcpy(checkpoint.outputs, self->outputs, sizeof(ccv_nnc_tensor_symbol_t) * self->output_size); | 382 | 2 | ccv_array_push(build_data->gradient_checkpoints, &checkpoint); | 383 | 2 | build_data->is_gradient_checkpointing = 0; | 384 | 2.30k | } else { | 385 | | // If we want to disable gradient checkpointing for this model, we simply not log any new tensors created here so there is no mapping for these. | 386 | 2.30k | int old_record; | 387 | 2.30k | ccv_cnnp_model_gradient_checkpoint_build_context_t* build_context = 0; | 388 | 2.30k | if (build_data->is_gradient_checkpointing) | 389 | 0 | { | 390 | 0 | if (self->gradient_checkpointing == -1) | 391 | 0 | { | 392 | 0 | ccv_nnc_tensor_symbol_new_hook_f old_tensor_symbol_new_hook; | 393 | 0 | build_context = ccv_nnc_tensor_symbol_new_hook(graph, 0, 0, &old_tensor_symbol_new_hook); | 394 | | // Set back the build_context. | 395 | 0 | ccv_nnc_tensor_symbol_new_hook(graph, old_tensor_symbol_new_hook, build_context, 0); | 396 | 0 | old_record = build_context->record; | 397 | 0 | build_context->record = 0; | 398 | 0 | } else if (self->gradient_checkpointing == 1) { // Force to turn on gradient checkpointing if it is inside a gradient checkpointing = -1. | 399 | 0 | ccv_nnc_tensor_symbol_new_hook_f old_tensor_symbol_new_hook; | 400 | 0 | build_context = ccv_nnc_tensor_symbol_new_hook(graph, 0, 0, &old_tensor_symbol_new_hook); | 401 | | // Set back the build_context. | 402 | 0 | ccv_nnc_tensor_symbol_new_hook(graph, old_tensor_symbol_new_hook, build_context, 0); | 403 | 0 | old_record = build_context->record; | 404 | 0 | build_context->record = 1; | 405 | 0 | } | 406 | 0 | } | 407 | | // No push checkpoint, easy. | 408 | 2.30k | if (outputs && output_size0 ) | 409 | 0 | { | 410 | 0 | assert(output_size == self->output_size); | 411 | 0 | self->isa->build(self, graph, inputs, input_size, outputs, output_size); | 412 | 0 | memcpy(self->outputs, outputs, sizeof(ccv_nnc_tensor_symbol_t) * output_size); | 413 | 0 | } else | 414 | 2.30k | self->isa->build(self, graph, inputs, input_size, self->outputs, self->output_size); | 415 | 2.30k | if (build_context) // Restore previous state even if our gradient checkpointing controlled whether to turn on recording or not. | 416 | 0 | build_context->record = old_record; | 417 | 2.30k | } | 418 | | // Skip if there is none. This helps to load parameters to a different model when only changes non-parameterized settings (add reshapes, permutations etc). | 419 | | // If it is named, we have to push too. | 420 | 2.30k | if (self->isa->add_to_parameter || self->isa->add_to_output1.09k ) | 421 | 1.20k | { | 422 | | // If we already pushed, no need to push again. | 423 | 1.20k | if (!(self->name && self->name[0] != '\0'3 )) | 424 | 1.20k | ccv_cnnp_model_push(self, build_data->model_sequence); | 425 | 1.20k | build_data->model_sequence->it = 0; | 426 | 1.20k | ccv_cnnp_model_add_to_parameter(self, build_data->add_to_array, build_data->context.add_to_parameter, build_data->is_trainable); | 427 | 1.20k | build_data->model_sequence->it = 0; | 428 | 1.20k | ccv_cnnp_model_add_to_output(self, build_data->add_to_array, build_data->context.add_to_output); | 429 | 1.20k | ccv_cnnp_model_pop(self, build_data->model_sequence); | 430 | 1.20k | } else if (1.09k self->name1.09k && self->name[0] != '\0'36 ) | 431 | 36 | ccv_cnnp_model_pop(self, build_data->model_sequence); | 432 | 2.30k | build_data->exec_flags = old_exec_flags; | 433 | 2.30k | build_data->is_trainable = old_is_trainable; | 434 | 2.30k | } |
Unexecuted instantiation: ccv_cnnp_model_io.c:ccv_cnnp_model_build ccv_cnnp_model_core.c:ccv_cnnp_model_build Line | Count | Source | 334 | 2.93k | { | 335 | 2.93k | assert(self->data); | 336 | 2.93k | ccv_cnnp_model_build_data_t* const build_data = (ccv_cnnp_model_build_data_t*)self->data; | 337 | 2.93k | const int old_exec_flags = build_data->exec_flags; | 338 | 2.93k | const int old_is_trainable = build_data->is_trainable; | 339 | 2.93k | if (self->exec_flags) | 340 | 0 | build_data->exec_flags |= self->exec_flags; | 341 | 2.93k | if (self->is_trainable >= 0) | 342 | 2.92k | build_data->is_trainable = self->is_trainable; | 343 | 2.93k | if (self->name && self->name[0] != '\0'213 ) | 344 | 213 | ccv_cnnp_model_push(self, build_data->model_sequence); | 345 | 2.93k | if (self->gradient_checkpointing == 1 && !build_data->is_gradient_checkpointing0 ) | 346 | 0 | { | 347 | 0 | build_data->is_gradient_checkpointing = 1; | 348 | | // Prepare to record gradient checkpoint. We will log the build function, inputs, what are the tensors / graph execs we created. | 349 | 0 | if (!build_data->gradient_checkpoints) | 350 | 0 | build_data->gradient_checkpoints = ccv_array_new(sizeof(ccv_cnnp_model_gradient_checkpoint_t), 0, 0); | 351 | 0 | ccv_cnnp_model_gradient_checkpoint_build_context_t build_context = { | 352 | 0 | .record = 1, | 353 | 0 | .tensor_symbols = ccv_array_new(sizeof(ccv_nnc_tensor_symbol_t), 0, 0), | 354 | 0 | }; | 355 | 0 | build_context.old_tensor_symbol_new_hook_context = ccv_nnc_tensor_symbol_new_hook(graph, _ccv_cnnp_model_gradient_checkpoint_tensor_symbol_new_hook, &build_context, &build_context.old_tensor_symbol_new_hook); | 356 | 0 | build_context.old_tensor_symbol_alias_new_hook_context = ccv_nnc_tensor_symbol_alias_new_hook(graph, _ccv_cnnp_model_gradient_checkpoint_tensor_symbol_alias_new_hook, &build_context, &build_context.old_tensor_symbol_alias_new_hook); | 357 | 0 | if (outputs && output_size) | 358 | 0 | { | 359 | 0 | assert(output_size == self->output_size); | 360 | 0 | self->isa->build(self, graph, inputs, input_size, outputs, output_size); | 361 | 0 | memcpy(self->outputs, outputs, sizeof(ccv_nnc_tensor_symbol_t) * output_size); | 362 | 0 | } else | 363 | 0 | self->isa->build(self, graph, inputs, input_size, self->outputs, self->output_size); | 364 | 0 | ccv_nnc_tensor_symbol_new_hook(graph, build_context.old_tensor_symbol_new_hook, build_context.old_tensor_symbol_new_hook_context, 0); | 365 | 0 | ccv_nnc_tensor_symbol_alias_new_hook(graph, build_context.old_tensor_symbol_alias_new_hook, build_context.old_tensor_symbol_alias_new_hook_context, 0); | 366 | 0 | ccv_cnnp_model_gradient_checkpoint_t checkpoint = { | 367 | 0 | .input_size = input_size, | 368 | 0 | .output_size = (outputs && output_size > 0) ? output_size : self->output_size, | 369 | 0 | .is_trainable = build_data->is_trainable, | 370 | 0 | .model = self, | 371 | 0 | .build = self->isa->build, | 372 | 0 | .tensor_symbols = build_context.tensor_symbols, | 373 | 0 | .inputs = ccmalloc(sizeof(ccv_nnc_tensor_symbol_t) * (input_size + ((outputs && output_size > 0) ? output_size : self->output_size))), | 374 | 0 | }; | 375 | 0 | checkpoint.outputs = checkpoint.inputs + input_size; | 376 | 0 | if (input_size > 0) | 377 | 0 | memcpy(checkpoint.inputs, inputs, sizeof(ccv_nnc_tensor_symbol_t) * input_size); | 378 | 0 | if (outputs && output_size > 0) | 379 | 0 | memcpy(checkpoint.outputs, outputs, sizeof(ccv_nnc_tensor_symbol_t) * output_size); | 380 | 0 | else if (self->outputs && self->output_size > 0) | 381 | 0 | memcpy(checkpoint.outputs, self->outputs, sizeof(ccv_nnc_tensor_symbol_t) * self->output_size); | 382 | 0 | ccv_array_push(build_data->gradient_checkpoints, &checkpoint); | 383 | 0 | build_data->is_gradient_checkpointing = 0; | 384 | 2.93k | } else { | 385 | | // If we want to disable gradient checkpointing for this model, we simply not log any new tensors created here so there is no mapping for these. | 386 | 2.93k | int old_record; | 387 | 2.93k | ccv_cnnp_model_gradient_checkpoint_build_context_t* build_context = 0; | 388 | 2.93k | if (build_data->is_gradient_checkpointing) | 389 | 36 | { | 390 | 36 | if (self->gradient_checkpointing == -1) | 391 | 0 | { | 392 | 0 | ccv_nnc_tensor_symbol_new_hook_f old_tensor_symbol_new_hook; | 393 | 0 | build_context = ccv_nnc_tensor_symbol_new_hook(graph, 0, 0, &old_tensor_symbol_new_hook); | 394 | | // Set back the build_context. | 395 | 0 | ccv_nnc_tensor_symbol_new_hook(graph, old_tensor_symbol_new_hook, build_context, 0); | 396 | 0 | old_record = build_context->record; | 397 | 0 | build_context->record = 0; | 398 | 36 | } else if (self->gradient_checkpointing == 1) { // Force to turn on gradient checkpointing if it is inside a gradient checkpointing = -1. | 399 | 0 | ccv_nnc_tensor_symbol_new_hook_f old_tensor_symbol_new_hook; | 400 | 0 | build_context = ccv_nnc_tensor_symbol_new_hook(graph, 0, 0, &old_tensor_symbol_new_hook); | 401 | | // Set back the build_context. | 402 | 0 | ccv_nnc_tensor_symbol_new_hook(graph, old_tensor_symbol_new_hook, build_context, 0); | 403 | 0 | old_record = build_context->record; | 404 | 0 | build_context->record = 1; | 405 | 0 | } | 406 | 36 | } | 407 | | // No push checkpoint, easy. | 408 | 2.93k | if (outputs && output_size2.93k ) | 409 | 2.93k | { | 410 | 2.93k | assert(output_size == self->output_size); | 411 | 2.93k | self->isa->build(self, graph, inputs, input_size, outputs, output_size); | 412 | 2.93k | memcpy(self->outputs, outputs, sizeof(ccv_nnc_tensor_symbol_t) * output_size); | 413 | 2.93k | } else | 414 | 4 | self->isa->build(self, graph, inputs, input_size, self->outputs, self->output_size); | 415 | 2.93k | if (build_context) // Restore previous state even if our gradient checkpointing controlled whether to turn on recording or not. | 416 | 0 | build_context->record = old_record; | 417 | 2.93k | } | 418 | | // Skip if there is none. This helps to load parameters to a different model when only changes non-parameterized settings (add reshapes, permutations etc). | 419 | | // If it is named, we have to push too. | 420 | 2.93k | if (self->isa->add_to_parameter || self->isa->add_to_output1.51k ) | 421 | 1.42k | { | 422 | | // If we already pushed, no need to push again. | 423 | 1.42k | if (!(self->name && self->name[0] != '\0'135 )) | 424 | 1.29k | ccv_cnnp_model_push(self, build_data->model_sequence); | 425 | 1.42k | build_data->model_sequence->it = 0; | 426 | 1.42k | ccv_cnnp_model_add_to_parameter(self, build_data->add_to_array, build_data->context.add_to_parameter, build_data->is_trainable); | 427 | 1.42k | build_data->model_sequence->it = 0; | 428 | 1.42k | ccv_cnnp_model_add_to_output(self, build_data->add_to_array, build_data->context.add_to_output); | 429 | 1.42k | ccv_cnnp_model_pop(self, build_data->model_sequence); | 430 | 1.51k | } else if (self->name && self->name[0] != '\0'78 ) | 431 | 78 | ccv_cnnp_model_pop(self, build_data->model_sequence); | 432 | 2.93k | build_data->exec_flags = old_exec_flags; | 433 | 2.93k | build_data->is_trainable = old_is_trainable; | 434 | 2.93k | } |
Unexecuted instantiation: ccv_cnnp_model_addons.c:ccv_cnnp_model_build Unexecuted instantiation: ccv_cnnp_model_gradient_checkpointing.c:ccv_cnnp_model_build |
435 | | |
436 | | static inline void ccv_cnnp_model_init_states(ccv_cnnp_model_t* const self, ccv_nnc_symbolic_graph_t* const graph, const ccv_cnnp_state_initializer_f initializer, void* const context) |
437 | 552 | { |
438 | 552 | if (self->isa->init_states) |
439 | 333 | self->isa->init_states(self, graph, initializer, context); |
440 | 552 | } Unexecuted instantiation: ccv_nnc_dynamic_graph_evaluate.c:ccv_cnnp_model_init_states ccv_cnnp_model.c:ccv_cnnp_model_init_states Line | Count | Source | 437 | 45 | { | 438 | 45 | if (self->isa->init_states) | 439 | 45 | self->isa->init_states(self, graph, initializer, context); | 440 | 45 | } |
Unexecuted instantiation: ccv_cnnp_model_io.c:ccv_cnnp_model_init_states ccv_cnnp_model_core.c:ccv_cnnp_model_init_states Line | Count | Source | 437 | 507 | { | 438 | 507 | if (self->isa->init_states) | 439 | 288 | self->isa->init_states(self, graph, initializer, context); | 440 | 507 | } |
Unexecuted instantiation: ccv_cnnp_model_addons.c:ccv_cnnp_model_init_states Unexecuted instantiation: ccv_cnnp_model_gradient_checkpointing.c:ccv_cnnp_model_init_states |
441 | | |
442 | | static inline void ccv_cnnp_model_set_is_test(ccv_cnnp_model_t* const self, const int is_test, const ccv_cnnp_cmd_updater_f updater, void* const context) |
443 | 1.17k | { |
444 | 1.17k | if (self->isa->set_is_test) |
445 | 251 | self->isa->set_is_test(self, is_test, updater, context); |
446 | 1.17k | } Unexecuted instantiation: ccv_nnc_dynamic_graph_evaluate.c:ccv_cnnp_model_set_is_test ccv_cnnp_model.c:ccv_cnnp_model_set_is_test Line | Count | Source | 443 | 156 | { | 444 | 156 | if (self->isa->set_is_test) | 445 | 106 | self->isa->set_is_test(self, is_test, updater, context); | 446 | 156 | } |
Unexecuted instantiation: ccv_cnnp_model_io.c:ccv_cnnp_model_set_is_test ccv_cnnp_model_core.c:ccv_cnnp_model_set_is_test Line | Count | Source | 443 | 1.01k | { | 444 | 1.01k | if (self->isa->set_is_test) | 445 | 145 | self->isa->set_is_test(self, is_test, updater, context); | 446 | 1.01k | } |
Unexecuted instantiation: ccv_cnnp_model_addons.c:ccv_cnnp_model_set_is_test Unexecuted instantiation: ccv_cnnp_model_gradient_checkpointing.c:ccv_cnnp_model_set_is_test |
447 | | |
448 | | static inline void ccv_cnnp_model_add_to_parameter_indices(ccv_cnnp_model_t* const self, const int index, ccv_array_t* const parameter_indices) |
449 | 13.3k | { |
450 | 13.3k | if (self->isa->add_to_parameter_indices) |
451 | 3.59k | self->isa->add_to_parameter_indices(self, index, parameter_indices); |
452 | 9.74k | else { |
453 | 9.74k | int i; |
454 | 9.74k | if (!self->parameter_indices) |
455 | 4.64k | return; |
456 | 5.10k | if (index == -1) |
457 | 188 | for (i = 0; 93 i < self->parameter_indices->rnum; i++95 ) |
458 | 95 | ccv_array_push(parameter_indices, ccv_array_get(self->parameter_indices, i)); |
459 | 5.01k | else if (index < self->parameter_indices->rnum) |
460 | 5.01k | ccv_array_push(parameter_indices, ccv_array_get(self->parameter_indices, index)); |
461 | 5.10k | } |
462 | 13.3k | } Unexecuted instantiation: ccv_nnc_dynamic_graph_evaluate.c:ccv_cnnp_model_add_to_parameter_indices ccv_cnnp_model.c:ccv_cnnp_model_add_to_parameter_indices Line | Count | Source | 449 | 394 | { | 450 | 394 | if (self->isa->add_to_parameter_indices) | 451 | 344 | self->isa->add_to_parameter_indices(self, index, parameter_indices); | 452 | 50 | else { | 453 | 50 | int i; | 454 | 50 | if (!self->parameter_indices) | 455 | 0 | return; | 456 | 50 | if (index == -1) | 457 | 82 | for (i = 0; 40 i < self->parameter_indices->rnum; i++42 ) | 458 | 42 | ccv_array_push(parameter_indices, ccv_array_get(self->parameter_indices, i)); | 459 | 10 | else if (index < self->parameter_indices->rnum) | 460 | 10 | ccv_array_push(parameter_indices, ccv_array_get(self->parameter_indices, index)); | 461 | 50 | } | 462 | 394 | } |
Unexecuted instantiation: ccv_cnnp_model_io.c:ccv_cnnp_model_add_to_parameter_indices ccv_cnnp_model_core.c:ccv_cnnp_model_add_to_parameter_indices Line | Count | Source | 449 | 12.9k | { | 450 | 12.9k | if (self->isa->add_to_parameter_indices) | 451 | 3.25k | self->isa->add_to_parameter_indices(self, index, parameter_indices); | 452 | 9.69k | else { | 453 | 9.69k | int i; | 454 | 9.69k | if (!self->parameter_indices) | 455 | 4.64k | return; | 456 | 5.05k | if (index == -1) | 457 | 106 | for (i = 0; 53 i < self->parameter_indices->rnum; i++53 ) | 458 | 53 | ccv_array_push(parameter_indices, ccv_array_get(self->parameter_indices, i)); | 459 | 5.00k | else if (index < self->parameter_indices->rnum) | 460 | 5.00k | ccv_array_push(parameter_indices, ccv_array_get(self->parameter_indices, index)); | 461 | 5.05k | } | 462 | 12.9k | } |
Unexecuted instantiation: ccv_cnnp_model_addons.c:ccv_cnnp_model_add_to_parameter_indices Unexecuted instantiation: ccv_cnnp_model_gradient_checkpointing.c:ccv_cnnp_model_add_to_parameter_indices |
463 | | |
464 | | typedef struct { |
465 | | uint8_t add_parameter_indices; |
466 | | char prefix; |
467 | | ccv_cnnp_model_sequence_t* sequence; |
468 | | ccv_array_t* symbols; |
469 | | ccv_array_t* ids; |
470 | | ccv_array_t* trainables; |
471 | | } ccv_cnnp_model_add_to_array_context_t; |
472 | | |
473 | | void ccv_cnnp_model_tensors_init_0(const ccv_cnnp_model_t* const model, ccv_cnnp_compiled_data_t* const compiled_data); |
474 | | void ccv_cnnp_model_tensors_init_1(const ccv_cnnp_model_t* const model, ccv_cnnp_compiled_data_t* const compiled_data); |
475 | | int ccv_cnnp_model_tensors_any_to_alloc(const ccv_cnnp_model_t* const model, ccv_cnnp_compiled_data_t* const compiled_data); |
476 | | ccv_nnc_stream_context_t* ccv_cnnp_compiled_data_get_stream(ccv_cnnp_compiled_data_t* const compiled_data, const int type); |
477 | | void ccv_cnnp_model_gradient_checkpoints_cleanup_after_build(ccv_cnnp_compiled_data_t* const compiled_data, ccv_nnc_symbolic_graph_t* const graph); |
478 | | void ccv_cnnp_model_apply_gradient_checkpoints(ccv_cnnp_compiled_data_t* const compiled_data, ccv_nnc_symbolic_graph_t* const graph); |
479 | | void ccv_cnnp_model_add_to_array(void* const context, const ccv_nnc_tensor_symbol_t symbol, const int is_trainable); |
480 | | |
481 | | #endif |