File: | nnc/ccv_nnc_dynamic_graph.c |
Warning: | line 676, column 8 Branch condition evaluates to a garbage value |
Press '?' to see keyboard shortcuts
Keyboard shortcuts:
1 | #include "ccv_nnc.h" | ||||
2 | #include "ccv_nnc_easy.h" | ||||
3 | #include "ccv_nnc_internal.h" | ||||
4 | #include "ccv_nnc_easy.h" | ||||
5 | #include "ccv_internal.h" | ||||
6 | #include "_ccv_nnc_dynamic_graph.h" | ||||
7 | |||||
8 | // MARK - Level-4 API | ||||
9 | |||||
10 | ccv_nnc_dynamic_graph_t* ccv_nnc_dynamic_graph_new(void) | ||||
11 | { | ||||
12 | ccv_nnc_dynamic_graph_t* graph = ccmallocmalloc(sizeof(ccv_nnc_dynamic_graph_t)); | ||||
13 | graph->no_grad = 0; | ||||
14 | graph->reuse_var = -1; | ||||
15 | graph->vars = ccv_array_new(sizeof(ccv_nnc_tensor_variable_t), 1, 0); | ||||
16 | graph->binds = ccv_array_new(sizeof(ccv_nnc_tensor_variable_graph_bind_t), 1, 0); | ||||
17 | graph->tape = ccv_nnc_symbolic_graph_new(); | ||||
18 | graph->xpu_alloc.mp_hdr = -1; | ||||
19 | graph->xpu_alloc.freed = kh_init(dy_str)kh_init_dy_str(); | ||||
20 | graph->xpu_alloc.allocd = kh_init(dy_alloc)kh_init_dy_alloc(); | ||||
21 | // These may not be used as frequent, init as needed. | ||||
22 | graph->stateful_execs = 0; | ||||
23 | graph->reuse_stateful_exec = -1; | ||||
24 | graph->stream_map = 0; | ||||
25 | graph->ws = 0; | ||||
26 | return graph; | ||||
27 | } | ||||
28 | |||||
29 | static void _ccv_nnc_tensor_variable_free(ccv_nnc_dynamic_graph_t* const graph, const ccv_nnc_tensor_variable_t tensor_variable, const int zeroing) | ||||
30 | { | ||||
31 | const int index = tensor_variable->index; | ||||
32 | if (tensor_variable->tensor_view) | ||||
33 | { | ||||
34 | if (tensor_variable->destructor_hook.func) | ||||
35 | tensor_variable->destructor_hook.func(graph, (ccv_nnc_tensor_t*)tensor_variable->tensor_view, tensor_variable->destructor_hook.context); | ||||
36 | if (!CCV_NNC_IS_EXTERN_TENSOR_VIEW(tensor_variable->tensor_view)((uintptr_t)(tensor_variable->tensor_view) & 1)) | ||||
37 | { | ||||
38 | if (CCV_IS_TENSOR_VIEW(tensor_variable->tensor_view)((*(int*)(tensor_variable->tensor_view)) & CCV_TENSOR_VIEW )) | ||||
39 | ccv_nnc_tensor_view_free(tensor_variable->tensor_view); | ||||
40 | else { | ||||
41 | if (!tensor_variable->alias_index_ref && // Return this memory to the graph. | ||||
42 | CCV_TENSOR_GET_MEMORY(tensor_variable->tensor_view->info.type)((tensor_variable->tensor_view->info.type) & 0x3) == CCV_TENSOR_GPU_MEMORY && tensor_variable->tensor_view->data.u8) | ||||
43 | ccv_nnc_xpu_free(&graph->xpu_alloc, tensor_variable->tensor_view->data.u8); | ||||
44 | ccv_nnc_tensor_free((ccv_nnc_tensor_t*)tensor_variable->tensor_view); | ||||
45 | } | ||||
46 | } | ||||
47 | } | ||||
48 | ccfreefree(tensor_variable); | ||||
49 | if (zeroing) | ||||
50 | *(ccv_nnc_tensor_variable_t*)ccv_array_get(graph->vars, index)((void*)(((char*)((graph->vars)->data)) + (size_t)(graph ->vars)->rsize * (size_t)(index))) = 0; | ||||
51 | int i; | ||||
52 | for (i = graph->vars->rnum - 1; i >= 0; i--) | ||||
53 | if (*(ccv_nnc_tensor_variable_t*)ccv_array_get(graph->vars, i)((void*)(((char*)((graph->vars)->data)) + (size_t)(graph ->vars)->rsize * (size_t)(i))) != 0) | ||||
54 | { | ||||
55 | graph->vars->rnum = i + 1; | ||||
56 | break; | ||||
57 | } | ||||
58 | if (index < graph->vars->rnum && | ||||
59 | (index < graph->reuse_var || graph->reuse_var < 0)) | ||||
60 | graph->reuse_var = index; | ||||
61 | else if (graph->reuse_var >= graph->vars->rnum) | ||||
62 | graph->reuse_var = -1; | ||||
63 | } | ||||
64 | |||||
65 | static void _ccv_nnc_tensor_variable_graph_bind_free(ccv_nnc_dynamic_graph_t* const graph, ccv_nnc_tensor_variable_graph_bind_t* const bind, const int zeroing) | ||||
66 | { | ||||
67 | bind->index = CCV_NNC_TENSOR_NO_VARIABLE; | ||||
68 | if (bind->sources) | ||||
69 | ccv_array_free(bind->sources); | ||||
70 | if (bind->destinations) | ||||
71 | ccv_array_free(bind->destinations); | ||||
72 | if (bind->tensor_view) | ||||
73 | { | ||||
74 | if (bind->destructor_hook.func) | ||||
75 | bind->destructor_hook.func(graph, (ccv_nnc_tensor_t*)bind->tensor_view, bind->destructor_hook.context); | ||||
76 | if (!CCV_NNC_IS_EXTERN_TENSOR_VIEW(bind->tensor_view)((uintptr_t)(bind->tensor_view) & 1)) | ||||
77 | { | ||||
78 | if (CCV_IS_TENSOR_VIEW(bind->tensor_view)((*(int*)(bind->tensor_view)) & CCV_TENSOR_VIEW)) | ||||
79 | ccv_nnc_tensor_view_free(bind->tensor_view); | ||||
80 | else { | ||||
81 | if (!bind->alias_ref && // Return this memory to the graph. | ||||
82 | CCV_TENSOR_GET_MEMORY(bind->tensor_view->info.type)((bind->tensor_view->info.type) & 0x3) == CCV_TENSOR_GPU_MEMORY && bind->tensor_view->data.u8) | ||||
83 | ccv_nnc_xpu_free(&graph->xpu_alloc, bind->tensor_view->data.u8); | ||||
84 | ccv_nnc_tensor_free((ccv_nnc_tensor_t*)bind->tensor_view); | ||||
85 | } | ||||
86 | } | ||||
87 | } | ||||
88 | if (zeroing) | ||||
89 | { | ||||
90 | bind->sources = 0; | ||||
91 | bind->destinations = 0; | ||||
92 | bind->tensor_view = 0; | ||||
93 | bind->destructor_hook.func = 0; | ||||
94 | bind->destructor_hook.context = 0; | ||||
95 | } | ||||
96 | } | ||||
97 | |||||
98 | void ccv_nnc_dynamic_graph_free(ccv_nnc_dynamic_graph_t* const graph) | ||||
99 | { | ||||
100 | int i; | ||||
101 | for (i = 0; i < graph->vars->rnum; i++) | ||||
102 | { | ||||
103 | ccv_nnc_tensor_variable_t tensor_variable = *(ccv_nnc_tensor_variable_t*)ccv_array_get(graph->vars, i)((void*)(((char*)((graph->vars)->data)) + (size_t)(graph ->vars)->rsize * (size_t)(i))); | ||||
104 | if (tensor_variable) | ||||
105 | _ccv_nnc_tensor_variable_free(graph, tensor_variable, 0); | ||||
106 | } | ||||
107 | ccv_array_free(graph->vars); | ||||
108 | for (i = 0; i < graph->binds->rnum; i++) | ||||
109 | _ccv_nnc_tensor_variable_graph_bind_free(graph, (ccv_nnc_tensor_variable_graph_bind_t*)ccv_array_get(graph->binds, i)((void*)(((char*)((graph->binds)->data)) + (size_t)(graph ->binds)->rsize * (size_t)(i))), 0); | ||||
110 | ccv_array_free(graph->binds); | ||||
111 | ccv_nnc_symbolic_graph_free(graph->tape); | ||||
112 | if (graph->ws) | ||||
113 | ccv_array_free(graph->ws); | ||||
114 | if (graph->stateful_execs) | ||||
115 | { | ||||
116 | for (i = 0; i < graph->stateful_execs->rnum; i++) | ||||
117 | { | ||||
118 | ccv_nnc_stateful_exec_t* const stateful_exec = *(ccv_nnc_stateful_exec_t**)ccv_array_get(graph->stateful_execs, i)((void*)(((char*)((graph->stateful_execs)->data)) + (size_t )(graph->stateful_execs)->rsize * (size_t)(i))); | ||||
119 | if (stateful_exec) | ||||
120 | ccfreefree(stateful_exec); | ||||
121 | } | ||||
122 | ccv_array_free(graph->stateful_execs); | ||||
123 | } | ||||
124 | if (graph->stream_map) | ||||
125 | { | ||||
126 | khiter_t k; | ||||
127 | for (k = kh_begin(graph->stream_map)(khint_t)(0); k != kh_end(graph->stream_map)((graph->stream_map)->n_buckets); ++k) | ||||
128 | { | ||||
129 | if (!kh_exist(graph->stream_map, k)(!(((graph->stream_map)->flags[(k)>>4]>>((( k)&0xfU)<<1))&3))) | ||||
130 | continue; | ||||
131 | ccv_nnc_stream_context_t* const stream = kh_val(graph->stream_map, k)((graph->stream_map)->vals[k]); | ||||
132 | ccv_nnc_stream_context_free(stream); | ||||
133 | } | ||||
134 | kh_destroy(stream_map, graph->stream_map)kh_destroy_stream_map(graph->stream_map); | ||||
135 | } | ||||
136 | ccv_nnc_xpu_alloc_destroy(&graph->xpu_alloc); | ||||
137 | ccfreefree(graph); | ||||
138 | } | ||||
139 | |||||
140 | void ccv_nnc_tensor_variable_set(ccv_nnc_dynamic_graph_t* const graph, const ccv_nnc_tensor_variable_t tensor_variable, ccv_nnc_tensor_t* const tensor) | ||||
141 | { | ||||
142 | assert(!tensor_variable->alias_index_ref)((void) sizeof ((!tensor_variable->alias_index_ref) ? 1 : 0 ), __extension__ ({ if (!tensor_variable->alias_index_ref) ; else __assert_fail ("!tensor_variable->alias_index_ref" , "ccv_nnc_dynamic_graph.c", 142, __extension__ __PRETTY_FUNCTION__ ); })); | ||||
143 | if (tensor_variable->tensor_view && !CCV_NNC_IS_EXTERN_TENSOR_VIEW(tensor_variable->tensor_view)((uintptr_t)(tensor_variable->tensor_view) & 1)) | ||||
144 | { | ||||
145 | assert(!CCV_IS_TENSOR_VIEW(tensor_variable->tensor_view))((void) sizeof ((!((*(int*)(tensor_variable->tensor_view)) & CCV_TENSOR_VIEW)) ? 1 : 0), __extension__ ({ if (!((*( int*)(tensor_variable->tensor_view)) & CCV_TENSOR_VIEW )) ; else __assert_fail ("!CCV_IS_TENSOR_VIEW(tensor_variable->tensor_view)" , "ccv_nnc_dynamic_graph.c", 145, __extension__ __PRETTY_FUNCTION__ ); })); | ||||
146 | ccv_nnc_tensor_free((ccv_nnc_tensor_t*)tensor_variable->tensor_view); | ||||
147 | } | ||||
148 | tensor_variable->info = tensor->info; | ||||
149 | tensor_variable->tensor_view = (ccv_nnc_tensor_view_t*)((uintptr_t)tensor | 1); | ||||
150 | } | ||||
151 | |||||
152 | void ccv_nnc_tensor_variable_destructor_hook(ccv_nnc_dynamic_graph_t* const graph, const ccv_nnc_tensor_variable_t tensor_variable, ccv_nnc_tensor_variable_destructor_f func, void* const context) | ||||
153 | { | ||||
154 | tensor_variable->destructor_hook.func = func; | ||||
155 | tensor_variable->destructor_hook.context = context; | ||||
156 | } | ||||
157 | |||||
158 | inline static void _ccv_nnc_tensor_variable_init(ccv_nnc_dynamic_graph_t* const graph, ccv_nnc_tensor_variable_t tensor_variable, const ccv_nnc_tensor_param_t info) | ||||
159 | { | ||||
160 | tensor_variable->alias_index_ref = 0; | ||||
161 | tensor_variable->alias_off = 0; | ||||
162 | tensor_variable->destructor_hook.func = 0; | ||||
163 | tensor_variable->destructor_hook.context = 0; | ||||
164 | tensor_variable->info = info; | ||||
165 | tensor_variable->symbol = NO_TENSOR_SYMBOL(const ccv_nnc_tensor_symbol_t){.d = CCV_NNC_NO_TENSOR_SYMBOL }; | ||||
166 | tensor_variable->tensor_view = 0; | ||||
167 | if (graph->reuse_var >= 0) | ||||
168 | { | ||||
169 | const int reuse_var = graph->reuse_var; | ||||
170 | assert(reuse_var < graph->vars->rnum)((void) sizeof ((reuse_var < graph->vars->rnum) ? 1 : 0), __extension__ ({ if (reuse_var < graph->vars->rnum ) ; else __assert_fail ("reuse_var < graph->vars->rnum" , "ccv_nnc_dynamic_graph.c", 170, __extension__ __PRETTY_FUNCTION__ ); })); | ||||
171 | tensor_variable->index = reuse_var; | ||||
172 | *(ccv_nnc_tensor_variable_t*)ccv_array_get(graph->vars, reuse_var)((void*)(((char*)((graph->vars)->data)) + (size_t)(graph ->vars)->rsize * (size_t)(reuse_var))) = tensor_variable; | ||||
173 | int i; | ||||
174 | graph->reuse_var = -1; | ||||
175 | for (i = reuse_var + 1; i < graph->vars->rnum && graph->reuse_var < 0; i++) | ||||
176 | if (*(ccv_nnc_tensor_variable_t*)ccv_array_get(graph->vars, i)((void*)(((char*)((graph->vars)->data)) + (size_t)(graph ->vars)->rsize * (size_t)(i))) == 0) | ||||
177 | graph->reuse_var = i; | ||||
178 | } else { | ||||
179 | tensor_variable->index = graph->vars->rnum; | ||||
180 | ccv_array_push(graph->vars, &tensor_variable); | ||||
181 | } | ||||
182 | } | ||||
183 | |||||
184 | ccv_nnc_tensor_variable_t ccv_nnc_tensor_variable_new_impl(ccv_nnc_dynamic_graph_t* const graph, const ccv_nnc_tensor_param_t info) | ||||
185 | { | ||||
186 | ccv_nnc_tensor_variable_t tensor_variable = ccmallocmalloc(sizeof(struct ccv_nnc_tensor_variable_s)); | ||||
187 | tensor_variable->type = CCV_NNC_TENSOR_VARIABLE; | ||||
188 | _ccv_nnc_tensor_variable_init(graph, tensor_variable, info); | ||||
189 | return tensor_variable; | ||||
190 | } | ||||
191 | |||||
192 | ccv_nnc_tensor_variable_t ccv_nnc_tensor_constant_new_impl(ccv_nnc_dynamic_graph_t* const graph, const ccv_nnc_tensor_param_t info) | ||||
193 | { | ||||
194 | ccv_nnc_tensor_variable_t tensor_variable = ccmallocmalloc(sizeof(struct ccv_nnc_tensor_variable_s)); | ||||
195 | tensor_variable->type = CCV_NNC_TENSOR_CONSTANT; | ||||
196 | _ccv_nnc_tensor_variable_init(graph, tensor_variable, info); | ||||
197 | return tensor_variable; | ||||
198 | } | ||||
199 | |||||
200 | int ccv_nnc_tensor_variable_is_constant(const ccv_nnc_dynamic_graph_t* const graph, const ccv_nnc_tensor_variable_t tensor_variable) | ||||
201 | { | ||||
202 | return tensor_variable->type == CCV_NNC_TENSOR_CONSTANT; | ||||
203 | } | ||||
204 | |||||
205 | ccv_nnc_tensor_param_t ccv_nnc_tensor_variable_params(ccv_nnc_dynamic_graph_t* const graph, const ccv_nnc_tensor_variable_t tensor_variable) | ||||
206 | { | ||||
207 | return tensor_variable->info; | ||||
208 | } | ||||
209 | |||||
210 | ccv_nnc_tensor_variable_t ccv_nnc_tensor_variable_alias_new(ccv_nnc_dynamic_graph_t* const graph, const ccv_nnc_tensor_variable_t tensor_variable, const int ofs[CCV_NNC_MAX_DIM_ALLOC(12)], const int stride[CCV_NNC_MAX_DIM_ALLOC(12)], const ccv_nnc_tensor_param_t info) | ||||
211 | { | ||||
212 | ccv_nnc_tensor_variable_t variable_alias = ccmallocmalloc(sizeof(struct ccv_nnc_tensor_variable_s)); | ||||
213 | variable_alias->type = tensor_variable->type; | ||||
214 | // If the tensor variable is an alias itself, we point directly to its original. | ||||
215 | if (tensor_variable->alias_index_ref) | ||||
216 | { | ||||
217 | variable_alias->alias_index_ref = tensor_variable->alias_index_ref; | ||||
218 | // The tensor variable need to be fully specified if I am doing alias an alias. | ||||
219 | assert(!ccv_nnc_is_tensor_auto(tensor_variable->info))((void) sizeof ((!ccv_nnc_is_tensor_auto(tensor_variable-> info)) ? 1 : 0), __extension__ ({ if (!ccv_nnc_is_tensor_auto (tensor_variable->info)) ; else __assert_fail ("!ccv_nnc_is_tensor_auto(tensor_variable->info)" , "ccv_nnc_dynamic_graph.c", 219, __extension__ __PRETTY_FUNCTION__ ); })); | ||||
220 | int i; | ||||
221 | int no_stride = 1; | ||||
222 | for (i = 0; no_stride && i < CCV_NNC_MAX_DIM_ALLOC(12); i++) | ||||
223 | no_stride = (tensor_variable->stride[i] == 0); | ||||
224 | int stride_from_dim[CCV_NNC_MAX_DIM_ALLOC(12)]; | ||||
225 | int* to_stride; | ||||
226 | if (no_stride) | ||||
227 | { | ||||
228 | ccv_nnc_tensor_get_stride(tensor_variable->info.dim, stride_from_dim); | ||||
229 | to_stride = stride_from_dim; | ||||
230 | } else | ||||
231 | to_stride = tensor_variable->stride; | ||||
232 | // If we provide stride, or reshape to a different size, assert the tensor variable itself is contiguous (otherwise we cannot satisfy the reshape requirements). | ||||
233 | const int different_dim = ccv_nnc_tensor_nd(info.dim) != ccv_nnc_tensor_nd(tensor_variable->info.dim); | ||||
234 | if (different_dim || (stride[0] != 0 && memcmp(stride, to_stride, sizeof(int) * CCV_NNC_MAX_DIM_ALLOC(12)) != 0)) | ||||
235 | { assert(ccv_nnc_tensor_view_is_contiguous(tensor_variable->info.dim, to_stride))((void) sizeof ((ccv_nnc_tensor_view_is_contiguous(tensor_variable ->info.dim, to_stride)) ? 1 : 0), __extension__ ({ if (ccv_nnc_tensor_view_is_contiguous (tensor_variable->info.dim, to_stride)) ; else __assert_fail ("ccv_nnc_tensor_view_is_contiguous(tensor_variable->info.dim, to_stride)" , "ccv_nnc_dynamic_graph.c", 235, __extension__ __PRETTY_FUNCTION__ ); })); } | ||||
236 | // Need to compute alias off, that is the alias off of the tensor variable plus its ofs. | ||||
237 | const off_t off = ccv_nnc_tensor_view_offset(tensor_variable->info.datatype, to_stride, tensor_variable->ofs); | ||||
238 | variable_alias->alias_off = tensor_variable->alias_off + off; | ||||
239 | // If we don't provide stride, copy the stride from previous variable. | ||||
240 | if (stride[0] == 0) | ||||
241 | { | ||||
242 | if (different_dim) | ||||
243 | ccv_nnc_tensor_get_stride(info.dim, variable_alias->stride); | ||||
244 | else | ||||
245 | memcpy(variable_alias->stride, to_stride, sizeof(int) * CCV_NNC_MAX_DIM_ALLOC(12)); | ||||
246 | } else | ||||
247 | memcpy(variable_alias->stride, stride, sizeof(int) * CCV_NNC_MAX_DIM_ALLOC(12)); | ||||
248 | } else { | ||||
249 | variable_alias->alias_index_ref = tensor_variable->index + 1; | ||||
250 | variable_alias->alias_off = 0; | ||||
251 | memcpy(variable_alias->stride, stride, sizeof(int) * CCV_NNC_MAX_DIM_ALLOC(12)); | ||||
252 | } | ||||
253 | variable_alias->info = info; | ||||
254 | variable_alias->symbol = NO_TENSOR_SYMBOL(const ccv_nnc_tensor_symbol_t){.d = CCV_NNC_NO_TENSOR_SYMBOL }; | ||||
255 | variable_alias->destructor_hook.func = 0; | ||||
256 | variable_alias->destructor_hook.context = 0; | ||||
257 | variable_alias->tensor_view = 0; | ||||
258 | memcpy(variable_alias->ofs, ofs, sizeof(int) * CCV_NNC_MAX_DIM_ALLOC(12)); | ||||
259 | if (graph->reuse_var >= 0) | ||||
260 | { | ||||
261 | const int reuse_var = graph->reuse_var; | ||||
262 | assert(reuse_var < graph->vars->rnum)((void) sizeof ((reuse_var < graph->vars->rnum) ? 1 : 0), __extension__ ({ if (reuse_var < graph->vars->rnum ) ; else __assert_fail ("reuse_var < graph->vars->rnum" , "ccv_nnc_dynamic_graph.c", 262, __extension__ __PRETTY_FUNCTION__ ); })); | ||||
263 | variable_alias->index = reuse_var; | ||||
264 | *(ccv_nnc_tensor_variable_t*)ccv_array_get(graph->vars, reuse_var)((void*)(((char*)((graph->vars)->data)) + (size_t)(graph ->vars)->rsize * (size_t)(reuse_var))) = variable_alias; | ||||
265 | int i; | ||||
266 | graph->reuse_var = -1; | ||||
267 | for (i = reuse_var + 1; i < graph->vars->rnum && graph->reuse_var < 0; i++) | ||||
268 | if (*(ccv_nnc_tensor_variable_t*)ccv_array_get(graph->vars, i)((void*)(((char*)((graph->vars)->data)) + (size_t)(graph ->vars)->rsize * (size_t)(i))) == 0) | ||||
269 | graph->reuse_var = i; | ||||
270 | } else { | ||||
271 | variable_alias->index = graph->vars->rnum; | ||||
272 | ccv_array_push(graph->vars, &variable_alias); | ||||
273 | } | ||||
274 | return variable_alias; | ||||
275 | } | ||||
276 | |||||
277 | int ccv_nnc_tensor_variable_alias_params(const ccv_nnc_dynamic_graph_t* const graph, const ccv_nnc_tensor_variable_t tensor_variable, int ofs[CCV_NNC_MAX_DIM_ALLOC(12)], int stride[CCV_NNC_MAX_DIM_ALLOC(12)]) | ||||
278 | { | ||||
279 | if (!tensor_variable->alias_index_ref) | ||||
280 | return -1; | ||||
281 | if (ofs) | ||||
282 | memcpy(ofs, tensor_variable->ofs, sizeof(int) * CCV_NNC_MAX_DIM_ALLOC(12)); | ||||
283 | if (stride) | ||||
284 | memcpy(stride, tensor_variable->stride, sizeof(int) * CCV_NNC_MAX_DIM_ALLOC(12)); | ||||
285 | return 0; | ||||
286 | } | ||||
287 | |||||
288 | ccv_nnc_tensor_t* ccv_nnc_tensor_from_variable_impl(ccv_nnc_dynamic_graph_t* const graph, const ccv_nnc_tensor_variable_t tensor_variable, ccv_nnc_stream_context_t* const stream_context) | ||||
289 | { | ||||
290 | if (tensor_variable->tensor_view) | ||||
291 | { | ||||
292 | if (tensor_variable->alias_index_ref) | ||||
293 | { | ||||
294 | const int alias_index = tensor_variable->alias_index_ref - 1; | ||||
295 | assert(alias_index >= 0)((void) sizeof ((alias_index >= 0) ? 1 : 0), __extension__ ({ if (alias_index >= 0) ; else __assert_fail ("alias_index >= 0" , "ccv_nnc_dynamic_graph.c", 295, __extension__ __PRETTY_FUNCTION__ ); })); | ||||
296 | ccv_nnc_tensor_variable_t variable_to = *(ccv_nnc_tensor_variable_t*)ccv_array_get(graph->vars, alias_index)((void*)(((char*)((graph->vars)->data)) + (size_t)(graph ->vars)->rsize * (size_t)(alias_index))); | ||||
297 | if (CCV_IS_TENSOR_VIEW(tensor_variable->tensor_view)((*(int*)(tensor_variable->tensor_view)) & CCV_TENSOR_VIEW )) | ||||
298 | { | ||||
299 | ccv_nnc_tensor_view_t* const tv = tensor_variable->tensor_view; | ||||
300 | // We cannot have an alias with custom set tensor, otherwise the pointer update is invalid. | ||||
301 | assert(!CCV_NNC_IS_EXTERN_TENSOR_VIEW(tv))((void) sizeof ((!((uintptr_t)(tv) & 1)) ? 1 : 0), __extension__ ({ if (!((uintptr_t)(tv) & 1)) ; else __assert_fail ("!CCV_NNC_IS_EXTERN_TENSOR_VIEW(tv)" , "ccv_nnc_dynamic_graph.c", 301, __extension__ __PRETTY_FUNCTION__ ); })); | ||||
302 | // Update the tensor_view pointer every time access it, because the underlying variable it alias to have changed. | ||||
303 | ccv_nnc_tensor_data(CCV_NNC_TENSOR_VIEW(variable_to->tensor_view)((ccv_nnc_tensor_view_t*)((uintptr_t)(variable_to->tensor_view ) & ~(uintptr_t)1))->info, CCV_NNC_TENSOR_VIEW(variable_to->tensor_view)((ccv_nnc_tensor_view_t*)((uintptr_t)(variable_to->tensor_view ) & ~(uintptr_t)1))->data.u8, tv->off + tensor_variable->alias_off, &tv->data, &tv->dataof); | ||||
304 | } else { | ||||
305 | ccv_nnc_tensor_t* const tv = (ccv_nnc_tensor_t*)tensor_variable->tensor_view; | ||||
306 | // We cannot have an alias with custom set tensor, otherwise the pointer update is invalid. | ||||
307 | assert(!CCV_NNC_IS_EXTERN_TENSOR_VIEW(tv))((void) sizeof ((!((uintptr_t)(tv) & 1)) ? 1 : 0), __extension__ ({ if (!((uintptr_t)(tv) & 1)) ; else __assert_fail ("!CCV_NNC_IS_EXTERN_TENSOR_VIEW(tv)" , "ccv_nnc_dynamic_graph.c", 307, __extension__ __PRETTY_FUNCTION__ ); })); | ||||
308 | // Update the tensor_view pointer every time access it, because the underlying variable it alias to have changed. | ||||
309 | ccv_nnc_tensor_data(CCV_NNC_TENSOR_VIEW(variable_to->tensor_view)((ccv_nnc_tensor_view_t*)((uintptr_t)(variable_to->tensor_view ) & ~(uintptr_t)1))->info, CCV_NNC_TENSOR_VIEW(variable_to->tensor_view)((ccv_nnc_tensor_view_t*)((uintptr_t)(variable_to->tensor_view ) & ~(uintptr_t)1))->data.u8, tensor_variable->alias_off, &tv->data, &tv->dataof); | ||||
310 | } | ||||
311 | } | ||||
312 | return (ccv_nnc_tensor_t*)CCV_NNC_TENSOR_VIEW(tensor_variable->tensor_view)((ccv_nnc_tensor_view_t*)((uintptr_t)(tensor_variable->tensor_view ) & ~(uintptr_t)1)); | ||||
313 | } | ||||
314 | if (!tensor_variable->alias_index_ref) | ||||
315 | { | ||||
316 | // If we haven't allocated tensor_variable, we cannot allocate them now (because no shape specified), return 0. | ||||
317 | if (ccv_nnc_is_tensor_auto(tensor_variable->info)) | ||||
318 | return 0; | ||||
319 | void* ptr = 0; | ||||
320 | const size_t data_size = ccv_nnc_tensor_data_size(tensor_variable->info); | ||||
321 | if (CCV_TENSOR_GET_MEMORY(tensor_variable->info.type)((tensor_variable->info.type) & 0x3) == CCV_TENSOR_GPU_MEMORY && data_size > 0) | ||||
322 | ptr = ccv_nnc_xpu_alloc(&graph->xpu_alloc, CCV_TENSOR_GET_DEVICE_ID(tensor_variable->info.type)(((tensor_variable->info.type) & 0xfff00) >> 8), stream_context, data_size); | ||||
323 | tensor_variable->tensor_view = (ccv_nnc_tensor_view_t*)ccv_nnc_tensor_new(ptr, tensor_variable->info, 0); | ||||
324 | if (tensor_variable->info.dim[0] > 0) | ||||
325 | { assert(tensor_variable->tensor_view->data.u8)((void) sizeof ((tensor_variable->tensor_view->data.u8) ? 1 : 0), __extension__ ({ if (tensor_variable->tensor_view ->data.u8) ; else __assert_fail ("tensor_variable->tensor_view->data.u8" , "ccv_nnc_dynamic_graph.c", 325, __extension__ __PRETTY_FUNCTION__ ); })); } | ||||
326 | return (ccv_nnc_tensor_t*)tensor_variable->tensor_view; | ||||
327 | } | ||||
328 | const int alias_index = tensor_variable->alias_index_ref - 1; | ||||
329 | assert(alias_index >= 0)((void) sizeof ((alias_index >= 0) ? 1 : 0), __extension__ ({ if (alias_index >= 0) ; else __assert_fail ("alias_index >= 0" , "ccv_nnc_dynamic_graph.c", 329, __extension__ __PRETTY_FUNCTION__ ); })); | ||||
330 | ccv_nnc_tensor_variable_t variable_to = *(ccv_nnc_tensor_variable_t*)ccv_array_get(graph->vars, alias_index)((void*)(((char*)((graph->vars)->data)) + (size_t)(graph ->vars)->rsize * (size_t)(alias_index))); | ||||
331 | assert(!variable_to->alias_index_ref)((void) sizeof ((!variable_to->alias_index_ref) ? 1 : 0), __extension__ ({ if (!variable_to->alias_index_ref) ; else __assert_fail ("!variable_to->alias_index_ref", "ccv_nnc_dynamic_graph.c" , 331, __extension__ __PRETTY_FUNCTION__); })); | ||||
332 | if (!variable_to->tensor_view) | ||||
333 | { | ||||
334 | // If we haven't allocated variable_to, we cannot allocate them now (because no shape specified), return 0. | ||||
335 | if (ccv_nnc_is_tensor_auto(variable_to->info)) | ||||
336 | return 0; | ||||
337 | void* ptr = 0; | ||||
338 | assert(variable_to->info.type == tensor_variable->info.type)((void) sizeof ((variable_to->info.type == tensor_variable ->info.type) ? 1 : 0), __extension__ ({ if (variable_to-> info.type == tensor_variable->info.type) ; else __assert_fail ("variable_to->info.type == tensor_variable->info.type" , "ccv_nnc_dynamic_graph.c", 338, __extension__ __PRETTY_FUNCTION__ ); })); | ||||
339 | const size_t data_size = ccv_nnc_tensor_data_size(variable_to->info); | ||||
340 | if (CCV_TENSOR_GET_MEMORY(variable_to->info.type)((variable_to->info.type) & 0x3) == CCV_TENSOR_GPU_MEMORY && data_size > 0) | ||||
341 | ptr = ccv_nnc_xpu_alloc(&graph->xpu_alloc, CCV_TENSOR_GET_DEVICE_ID(variable_to->info.type)(((variable_to->info.type) & 0xfff00) >> 8), stream_context, data_size); | ||||
342 | variable_to->tensor_view = (ccv_nnc_tensor_view_t*)ccv_nnc_tensor_new(ptr, variable_to->info, 0); | ||||
343 | assert(variable_to->tensor_view->data.u8)((void) sizeof ((variable_to->tensor_view->data.u8) ? 1 : 0), __extension__ ({ if (variable_to->tensor_view->data .u8) ; else __assert_fail ("variable_to->tensor_view->data.u8" , "ccv_nnc_dynamic_graph.c", 343, __extension__ __PRETTY_FUNCTION__ ); })); | ||||
344 | } | ||||
345 | int i; | ||||
346 | int no_ofs = 1; | ||||
347 | for (i = 0; no_ofs && i < CCV_NNC_MAX_DIM_ALLOC(12); i++) | ||||
348 | no_ofs = (tensor_variable->ofs[i] == 0); | ||||
349 | int no_stride = 1; | ||||
350 | for (i = 0; no_stride && i < CCV_NNC_MAX_DIM_ALLOC(12); i++) | ||||
351 | no_stride = (tensor_variable->stride[i] == 0); | ||||
352 | int stride_is_packed = no_stride; | ||||
353 | if (!no_stride) // We have stride, now if it is packed. | ||||
354 | stride_is_packed = ccv_nnc_is_tensor_stride_packed(tensor_variable->stride, tensor_variable->info.dim); | ||||
355 | assert(CCV_GET_DATA_TYPE_SIZE(tensor_variable->info.datatype) * ccv_nnc_tensor_count(tensor_variable->info) + tensor_variable->alias_off <= CCV_GET_DATA_TYPE_SIZE(variable_to->info.datatype) * ccv_nnc_tensor_count(variable_to->info))((void) sizeof ((_ccv_get_data_type_size[((tensor_variable-> info.datatype) & 0xFF000) >> 12] * ccv_nnc_tensor_count (tensor_variable->info) + tensor_variable->alias_off <= _ccv_get_data_type_size[((variable_to->info.datatype) & 0xFF000) >> 12] * ccv_nnc_tensor_count(variable_to-> info)) ? 1 : 0), __extension__ ({ if (_ccv_get_data_type_size [((tensor_variable->info.datatype) & 0xFF000) >> 12] * ccv_nnc_tensor_count(tensor_variable->info) + tensor_variable ->alias_off <= _ccv_get_data_type_size[((variable_to-> info.datatype) & 0xFF000) >> 12] * ccv_nnc_tensor_count (variable_to->info)) ; else __assert_fail ("CCV_GET_DATA_TYPE_SIZE(tensor_variable->info.datatype) * ccv_nnc_tensor_count(tensor_variable->info) + tensor_variable->alias_off <= CCV_GET_DATA_TYPE_SIZE(variable_to->info.datatype) * ccv_nnc_tensor_count(variable_to->info)" , "ccv_nnc_dynamic_graph.c", 355, __extension__ __PRETTY_FUNCTION__ ); })); | ||||
356 | // Allowing vector type to be normal tensor, rather than a tensor view. We cannot have any offset though. | ||||
357 | if (no_ofs && !stride_is_packed) | ||||
358 | stride_is_packed = ccv_nnc_tensor_view_is_contiguous(tensor_variable->info.dim, tensor_variable->stride); | ||||
359 | if (no_ofs && stride_is_packed) | ||||
360 | tensor_variable->tensor_view = (ccv_nnc_tensor_view_t*)ccv_nnc_tensor_new(CCV_NNC_TENSOR_VIEW(variable_to->tensor_view)((ccv_nnc_tensor_view_t*)((uintptr_t)(variable_to->tensor_view ) & ~(uintptr_t)1))->data.u8, tensor_variable->info, 0); | ||||
361 | else { | ||||
362 | if (no_stride) | ||||
363 | ccv_nnc_tensor_get_stride(tensor_variable->info.dim, tensor_variable->stride); | ||||
364 | tensor_variable->tensor_view = ccv_nnc_tensor_view_new((ccv_nnc_tensor_t*)CCV_NNC_TENSOR_VIEW(variable_to->tensor_view)((ccv_nnc_tensor_view_t*)((uintptr_t)(variable_to->tensor_view ) & ~(uintptr_t)1)), tensor_variable->info, tensor_variable->ofs, tensor_variable->stride); | ||||
365 | } | ||||
366 | if (tensor_variable->alias_off) | ||||
367 | ccv_nnc_tensor_data_add(tensor_variable->tensor_view->info, tensor_variable->alias_off, &tensor_variable->tensor_view->data, &tensor_variable->tensor_view->dataof); | ||||
368 | return (ccv_nnc_tensor_t*)tensor_variable->tensor_view; | ||||
369 | } | ||||
370 | |||||
371 | static void _ccv_nnc_tensor_symbol_extra_new(ccv_nnc_dynamic_graph_t* const graph, const ccv_nnc_tensor_variable_t tensor_variable, const ccv_nnc_tensor_symbol_t symbol) | ||||
372 | { | ||||
373 | if (symbol.d >= graph->binds->rnum) | ||||
374 | { | ||||
375 | const int rnum = graph->binds->rnum; | ||||
376 | ccv_array_resize(graph->binds, symbol.d + 1); | ||||
377 | int i; | ||||
378 | for (i = rnum; i < graph->binds->rnum; i++) | ||||
379 | ((ccv_nnc_tensor_variable_graph_bind_t*)ccv_array_get(graph->binds, i)((void*)(((char*)((graph->binds)->data)) + (size_t)(graph ->binds)->rsize * (size_t)(i))))->index = CCV_NNC_TENSOR_NO_VARIABLE; | ||||
380 | } | ||||
381 | ccv_nnc_tensor_variable_graph_bind_t* const bind = (ccv_nnc_tensor_variable_graph_bind_t*)ccv_array_get(graph->binds, symbol.d)((void*)(((char*)((graph->binds)->data)) + (size_t)(graph ->binds)->rsize * (size_t)(symbol.d))); | ||||
382 | bind->type = tensor_variable->type; | ||||
383 | bind->index = tensor_variable->index; | ||||
384 | if (tensor_variable->alias_index_ref) | ||||
385 | { | ||||
386 | const ccv_nnc_tensor_symbol_t alias_to = ccv_nnc_tensor_symbol_alias_to(graph->tape, (ccv_nnc_tensor_symbol_t){ | ||||
387 | .d = symbol.d, | ||||
388 | .graph = graph->tape | ||||
389 | }); | ||||
390 | assert(alias_to.d >= 0 && alias_to.d < graph->binds->rnum)((void) sizeof ((alias_to.d >= 0 && alias_to.d < graph->binds->rnum) ? 1 : 0), __extension__ ({ if (alias_to .d >= 0 && alias_to.d < graph->binds->rnum ) ; else __assert_fail ("alias_to.d >= 0 && alias_to.d < graph->binds->rnum" , "ccv_nnc_dynamic_graph.c", 390, __extension__ __PRETTY_FUNCTION__ ); })); | ||||
391 | bind->alias_ref = alias_to.d + 1; | ||||
392 | } else | ||||
393 | bind->alias_ref = 0; | ||||
394 | if (bind->sources) | ||||
395 | ccv_array_free(bind->sources); | ||||
396 | bind->sources = 0; | ||||
397 | if (bind->destinations) | ||||
398 | ccv_array_free(bind->destinations); | ||||
399 | bind->destinations = 0; | ||||
400 | bind->destructor_hook.func = 0; | ||||
401 | bind->destructor_hook.context = 0; | ||||
402 | bind->tensor_view = 0; | ||||
403 | } | ||||
404 | |||||
405 | static ccv_nnc_tensor_symbol_t _ccv_nnc_tensor_symbol_from_variable(ccv_nnc_dynamic_graph_t* const graph, const ccv_nnc_tensor_variable_t tensor_variable) | ||||
406 | { | ||||
407 | if (tensor_variable->symbol.d >= 0) | ||||
408 | return tensor_variable->symbol; | ||||
409 | if (!tensor_variable->alias_index_ref) | ||||
410 | { | ||||
411 | const ccv_nnc_tensor_symbol_t symbol = tensor_variable->symbol = ccv_nnc_tensor_symbol_new(graph->tape, tensor_variable->info, 0); | ||||
412 | _ccv_nnc_tensor_symbol_extra_new(graph, tensor_variable, symbol); | ||||
413 | return symbol; | ||||
414 | } | ||||
415 | const int alias_index = tensor_variable->alias_index_ref - 1; | ||||
416 | assert(alias_index >= 0)((void) sizeof ((alias_index >= 0) ? 1 : 0), __extension__ ({ if (alias_index >= 0) ; else __assert_fail ("alias_index >= 0" , "ccv_nnc_dynamic_graph.c", 416, __extension__ __PRETTY_FUNCTION__ ); })); | ||||
417 | ccv_nnc_tensor_variable_t variable_to = *(ccv_nnc_tensor_variable_t*)ccv_array_get(graph->vars, alias_index)((void*)(((char*)((graph->vars)->data)) + (size_t)(graph ->vars)->rsize * (size_t)(alias_index))); | ||||
418 | assert(!variable_to->alias_index_ref)((void) sizeof ((!variable_to->alias_index_ref) ? 1 : 0), __extension__ ({ if (!variable_to->alias_index_ref) ; else __assert_fail ("!variable_to->alias_index_ref", "ccv_nnc_dynamic_graph.c" , 418, __extension__ __PRETTY_FUNCTION__); })); | ||||
419 | int no_stride = 1; | ||||
420 | int i; | ||||
421 | for (i = 0; no_stride && i < CCV_NNC_MAX_DIM_ALLOC(12); i++) | ||||
422 | no_stride = (tensor_variable->stride[i] == 0); | ||||
423 | if (no_stride) | ||||
424 | ccv_nnc_tensor_get_stride(tensor_variable->info.dim, tensor_variable->stride); | ||||
425 | const ccv_nnc_tensor_symbol_t symbol = tensor_variable->symbol = ccv_nnc_tensor_symbol_alias_new(graph->tape, _ccv_nnc_tensor_symbol_from_variable(graph, variable_to), tensor_variable->ofs, tensor_variable->stride, tensor_variable->info, 0); | ||||
426 | _ccv_nnc_tensor_symbol_extra_new(graph, tensor_variable, symbol); | ||||
427 | return symbol; | ||||
428 | } | ||||
429 | |||||
430 | // Return the tensor variable that is old (the provided tensor variable will have a new setting). | ||||
431 | ccv_nnc_tensor_variable_t ccv_nnc_tensor_variable_exchange_new(ccv_nnc_dynamic_graph_t* const graph, ccv_nnc_tensor_variable_t tensor_variable) | ||||
432 | { | ||||
433 | struct ccv_nnc_tensor_variable_s x = *tensor_variable; | ||||
434 | ccv_nnc_tensor_variable_t new_variable; | ||||
435 | // Need to handle alias. | ||||
436 | if (x.alias_index_ref) | ||||
437 | new_variable = ccv_nnc_tensor_variable_alias_new(graph, *(ccv_nnc_tensor_variable_t*)ccv_array_get(graph->vars, x.alias_index_ref - 1)((void*)(((char*)((graph->vars)->data)) + (size_t)(graph ->vars)->rsize * (size_t)(x.alias_index_ref - 1))), x.ofs, x.stride, x.info); | ||||
438 | else | ||||
439 | new_variable = ccv_nnc_tensor_variable_new(graph, x.info)ccv_nnc_tensor_variable_new_impl(graph, x.info); | ||||
440 | *tensor_variable = *new_variable; | ||||
441 | *new_variable = x; | ||||
442 | // The index should be the same though. | ||||
443 | const int index = new_variable->index; | ||||
444 | new_variable->index = tensor_variable->index; | ||||
445 | if (new_variable->symbol.d != CCV_NNC_NO_TENSOR_SYMBOL) | ||||
446 | { | ||||
447 | ccv_nnc_tensor_variable_graph_bind_t* const bind = (ccv_nnc_tensor_variable_graph_bind_t*)ccv_array_get(graph->binds, new_variable->symbol.d)((void*)(((char*)((graph->binds)->data)) + (size_t)(graph ->binds)->rsize * (size_t)(new_variable->symbol.d))); | ||||
448 | bind->index = new_variable->index; | ||||
449 | } | ||||
450 | tensor_variable->index = index; | ||||
451 | return new_variable; | ||||
452 | } | ||||
453 | |||||
454 | void ccv_nnc_dynamic_graph_set_max_concurrency(ccv_nnc_dynamic_graph_t* const dynamic_graph, const int max_stream_count) | ||||
455 | { | ||||
456 | dynamic_graph->max_stream_count = max_stream_count; | ||||
457 | } | ||||
458 | |||||
459 | int ccv_nnc_dynamic_graph_set_no_grad(ccv_nnc_dynamic_graph_t* const dynamic_graph, const int no_grad) | ||||
460 | { | ||||
461 | if (dynamic_graph->no_grad == no_grad) | ||||
462 | return -1; | ||||
463 | dynamic_graph->no_grad = no_grad; | ||||
464 | return 0; | ||||
465 | } | ||||
466 | |||||
467 | static ccv_nnc_stream_context_t* _ccv_nnc_dynamic_graph_get_stream(ccv_nnc_dynamic_graph_t* const graph, const int type) | ||||
468 | { | ||||
469 | if (!graph->stream_map) | ||||
470 | graph->stream_map = kh_init(stream_map)kh_init_stream_map(); | ||||
471 | int ret = 0; | ||||
472 | khiter_t k = kh_put(stream_map, graph->stream_map, type, &ret)kh_put_stream_map(graph->stream_map, type, &ret); | ||||
473 | assert(ret >= 0)((void) sizeof ((ret >= 0) ? 1 : 0), __extension__ ({ if ( ret >= 0) ; else __assert_fail ("ret >= 0", "ccv_nnc_dynamic_graph.c" , 473, __extension__ __PRETTY_FUNCTION__); })); | ||||
474 | ccv_nnc_stream_context_t* stream = kh_val(graph->stream_map, k)((graph->stream_map)->vals[k]); | ||||
475 | // If ret == 0, the key already exist, we can return directly, otherwise, create and return. | ||||
476 | if (ret != 0) | ||||
477 | { | ||||
478 | stream = ccv_nnc_stream_context_new(type); | ||||
479 | kh_val(graph->stream_map, k)((graph->stream_map)->vals[k]) = stream; | ||||
480 | } | ||||
481 | return stream; | ||||
482 | } | ||||
483 | |||||
484 | typedef struct { | ||||
485 | ccv_nnc_dynamic_graph_t* graph; | ||||
486 | int stream_type; | ||||
487 | } ccv_nnc_dynamic_graph_neighbor_context_discovery_t; | ||||
488 | |||||
489 | static ccv_nnc_stream_context_t* _ccv_nnc_dynamic_graph_neighbor_context_discovery(const int device_id, void* const context) | ||||
490 | { | ||||
491 | ccv_nnc_dynamic_graph_neighbor_context_discovery_t* const discovery = (ccv_nnc_dynamic_graph_neighbor_context_discovery_t*)context; | ||||
492 | int type = discovery->stream_type; | ||||
493 | CCV_STREAM_SET_DEVICE_ID(type, device_id)(type) = (((type) & ~0xfff00) | (((device_id) & 0xfff ) << 8)); | ||||
494 | return _ccv_nnc_dynamic_graph_get_stream(discovery->graph, type); | ||||
495 | } | ||||
496 | |||||
497 | void ccv_nnc_dynamic_graph_exec_ret(ccv_nnc_dynamic_graph_t* const graph, const ccv_nnc_cmd_t cmd, const ccv_nnc_hint_t hint, const int flags, const ccv_nnc_tensor_variable_t* const inputs, const int input_size, ccv_nnc_tensor_variable_t* const outputs, const int output_size, const int parallel, ccv_nnc_stream_context_t* const stream_context, ccv_nnc_graph_exec_symbol_t* const graph_execs) | ||||
498 | { | ||||
499 | int i, j; | ||||
500 | for (i = 0; i < input_size; i++) | ||||
| |||||
501 | if (inputs[i] && !inputs[i]->alias_index_ref) | ||||
502 | { assert(inputs[i]->tensor_view)((void) sizeof ((inputs[i]->tensor_view) ? 1 : 0), __extension__ ({ if (inputs[i]->tensor_view) ; else __assert_fail ("inputs[i]->tensor_view" , "ccv_nnc_dynamic_graph.c", 502, __extension__ __PRETTY_FUNCTION__ ); })); } | ||||
503 | ccv_nnc_tensor_t* input_tensors[ccv_max(1, input_size)({ typeof (1) _a = (1); typeof (input_size) _b = (input_size) ; (_a > _b) ? _a : _b; })]; | ||||
504 | for (i = 0; i
| ||||
505 | input_tensors[i] = inputs[i] ? ccv_nnc_tensor_from_variable(graph, inputs[i], stream_context)ccv_nnc_tensor_from_variable_impl(graph, inputs[i], stream_context ) : 0; | ||||
506 | ccv_nnc_tensor_symbol_t input_symbols[ccv_max(1, input_size)({ typeof (1) _a = (1); typeof (input_size) _b = (input_size) ; (_a > _b) ? _a : _b; })]; | ||||
507 | for (i = 0; i
| ||||
508 | input_symbols[i] = inputs[i] ? _ccv_nnc_tensor_symbol_from_variable(graph, inputs[i]) : NO_TENSOR_SYMBOL(const ccv_nnc_tensor_symbol_t){.d = CCV_NNC_NO_TENSOR_SYMBOL }; | ||||
509 | ccv_array_t* input_sources[ccv_max(1, input_size)({ typeof (1) _a = (1); typeof (input_size) _b = (input_size) ; (_a > _b) ? _a : _b; })]; | ||||
510 | ccv_array_t* input_alias_sources[ccv_max(1, input_size)({ typeof (1) _a = (1); typeof (input_size) _b = (input_size) ; (_a > _b) ? _a : _b; })]; | ||||
511 | for (i = 0; i
| ||||
512 | { | ||||
513 | input_sources[i] = input_symbols[i].d != CCV_NNC_NO_TENSOR_SYMBOL ? ((ccv_nnc_tensor_variable_graph_bind_t*)ccv_array_get(graph->binds, input_symbols[i].d)((void*)(((char*)((graph->binds)->data)) + (size_t)(graph ->binds)->rsize * (size_t)(input_symbols[i].d))))->sources : 0; | ||||
514 | if (inputs[i] && inputs[i]->alias_index_ref) | ||||
515 | { | ||||
516 | const int alias_index_ref = inputs[i]->alias_index_ref - 1; | ||||
517 | assert(alias_index_ref >= 0)((void) sizeof ((alias_index_ref >= 0) ? 1 : 0), __extension__ ({ if (alias_index_ref >= 0) ; else __assert_fail ("alias_index_ref >= 0" , "ccv_nnc_dynamic_graph.c", 517, __extension__ __PRETTY_FUNCTION__ ); })); | ||||
518 | ccv_nnc_tensor_variable_t variable_to = *(ccv_nnc_tensor_variable_t*)ccv_array_get(graph->vars, alias_index_ref)((void*)(((char*)((graph->vars)->data)) + (size_t)(graph ->vars)->rsize * (size_t)(alias_index_ref))); | ||||
519 | input_alias_sources[i] = ((ccv_nnc_tensor_variable_graph_bind_t*)ccv_array_get(graph->binds, variable_to->symbol.d)((void*)(((char*)((graph->binds)->data)) + (size_t)(graph ->binds)->rsize * (size_t)(variable_to->symbol.d))))->sources; | ||||
520 | } else | ||||
521 | input_alias_sources[i] = 0; | ||||
522 | } | ||||
523 | const int parallel_count = ccv_max(1, parallel)({ typeof (1) _a = (1); typeof (parallel) _b = (parallel); (_a > _b) ? _a : _b; }); | ||||
524 | assert(input_size % parallel_count == 0)((void) sizeof ((input_size % parallel_count == 0) ? 1 : 0), __extension__ ({ if (input_size % parallel_count == 0) ; else __assert_fail ("input_size % parallel_count == 0", "ccv_nnc_dynamic_graph.c" , 524, __extension__ __PRETTY_FUNCTION__); })); | ||||
525 | const int per_input_size = input_size / parallel_count; | ||||
526 | assert(output_size % parallel_count == 0)((void) sizeof ((output_size % parallel_count == 0) ? 1 : 0), __extension__ ({ if (output_size % parallel_count == 0) ; else __assert_fail ("output_size % parallel_count == 0", "ccv_nnc_dynamic_graph.c" , 526, __extension__ __PRETTY_FUNCTION__); })); | ||||
527 | const int per_output_size = output_size / parallel_count; | ||||
528 | int output_auto = 0; | ||||
529 | for (i = 0; !output_auto
| ||||
530 | output_auto = outputs[i] ? ccv_nnc_is_tensor_auto(outputs[i]->info) : 0; | ||||
531 | // One extra step, infer the parameters for outputs. | ||||
532 | if (output_auto
| ||||
533 | { | ||||
534 | ccv_nnc_tensor_param_t input_params[ccv_max(1, per_input_size)({ typeof (1) _a = (1); typeof (per_input_size) _b = (per_input_size ); (_a > _b) ? _a : _b; })]; | ||||
535 | ccv_nnc_tensor_param_t output_params[ccv_max(1, per_output_size)({ typeof (1) _a = (1); typeof (per_output_size) _b = (per_output_size ); (_a > _b) ? _a : _b; })]; | ||||
536 | for (i = 0; i
| ||||
537 | { | ||||
538 | for (j = 0; j
| ||||
539 | input_params[j] = inputs[j + i * per_input_size] ? inputs[j + i * per_input_size]->info : ccv_nnc_tensor_auto; | ||||
540 | for (j = 0; j
| ||||
541 | output_params[j] = outputs[j + i * per_output_size] ? outputs[j + i * per_output_size]->info : ccv_nnc_tensor_auto; | ||||
542 | ccv_nnc_hint_tensor_auto(cmd, input_params, per_input_size, hint, output_params, per_output_size); | ||||
543 | for (j = 0; j < per_output_size; j++) | ||||
544 | if (outputs[j + i * per_output_size]) | ||||
545 | outputs[j + i * per_output_size]->info = output_params[j]; | ||||
546 | } | ||||
547 | } | ||||
548 | int freeable_size = 0; | ||||
549 | ccv_nnc_tensor_variable_t freeables[ccv_max(1, output_size)({ typeof (1) _a = (1); typeof (output_size) _b = (output_size ); (_a > _b) ? _a : _b; })]; | ||||
550 | // Refresh the symbol if it is binded to an existing exec. Otherwise we cannot keep the SSA guarantee. | ||||
551 | for (i = 0; i
| ||||
552 | { | ||||
553 | // First, go over to see whether there is enforce inplace. | ||||
554 | int enforce_idx = -1; | ||||
555 | for (j = 0; enforce_idx
| ||||
556 | if (inputs[j] && ccv_nnc_cmd_enforce_inplace(cmd, j, input_size, i, output_size)) | ||||
557 | enforce_idx = j; | ||||
558 | if (enforce_idx
| ||||
559 | { assert(outputs[i] == inputs[enforce_idx] && outputs[i]->symbol.d != CCV_NNC_NO_TENSOR_SYMBOL)((void) sizeof ((outputs[i] == inputs[enforce_idx] && outputs[i]->symbol.d != CCV_NNC_NO_TENSOR_SYMBOL) ? 1 : 0 ), __extension__ ({ if (outputs[i] == inputs[enforce_idx] && outputs[i]->symbol.d != CCV_NNC_NO_TENSOR_SYMBOL) ; else __assert_fail ("outputs[i] == inputs[enforce_idx] && outputs[i]->symbol.d != CCV_NNC_NO_TENSOR_SYMBOL" , "ccv_nnc_dynamic_graph.c", 559, __extension__ __PRETTY_FUNCTION__ ); })); } | ||||
560 | // We don't allow or check "allow inplace" yet. That logic will be at odds with backward logic. | ||||
561 | if (outputs[i] && outputs[i]->symbol.d != CCV_NNC_NO_TENSOR_SYMBOL) | ||||
562 | { | ||||
563 | const ccv_nnc_tensor_variable_graph_bind_t* const bind = (ccv_nnc_tensor_variable_graph_bind_t*)ccv_array_get(graph->binds, outputs[i]->symbol.d)((void*)(((char*)((graph->binds)->data)) + (size_t)(graph ->binds)->rsize * (size_t)(outputs[i]->symbol.d))); | ||||
564 | if (enforce_idx >= 0) | ||||
565 | { assert(!bind->destinations || bind->destinations->rnum == 0)((void) sizeof ((!bind->destinations || bind->destinations ->rnum == 0) ? 1 : 0), __extension__ ({ if (!bind->destinations || bind->destinations->rnum == 0) ; else __assert_fail ("!bind->destinations || bind->destinations->rnum == 0" , "ccv_nnc_dynamic_graph.c", 565, __extension__ __PRETTY_FUNCTION__ ); })); } | ||||
566 | if (bind->sources && bind->sources->rnum > 0) | ||||
567 | { | ||||
568 | const ccv_nnc_tensor_variable_t old_var = freeables[freeable_size++] = ccv_nnc_tensor_variable_exchange_new(graph, outputs[i]); | ||||
569 | // If this is enforce output, make sure the tensor view is taken by the output. | ||||
570 | if (enforce_idx >= 0) | ||||
571 | { | ||||
572 | outputs[i]->destructor_hook = old_var->destructor_hook; | ||||
573 | outputs[i]->tensor_view = old_var->tensor_view; // Make sure the tensor view is taken over by the output. | ||||
574 | old_var->tensor_view = 0; | ||||
575 | } | ||||
576 | } | ||||
577 | } | ||||
578 | } | ||||
579 | ccv_nnc_tensor_t* output_tensors[ccv_max(1, per_output_size)({ typeof (1) _a = (1); typeof (per_output_size) _b = (per_output_size ); (_a > _b) ? _a : _b; })]; | ||||
580 | if (parallel_count
| ||||
581 | { | ||||
582 | const int max_device_id_size = per_input_size + per_output_size; | ||||
583 | assert(max_device_id_size > 0)((void) sizeof ((max_device_id_size > 0) ? 1 : 0), __extension__ ({ if (max_device_id_size > 0) ; else __assert_fail ("max_device_id_size > 0" , "ccv_nnc_dynamic_graph.c", 583, __extension__ __PRETTY_FUNCTION__ ); })); | ||||
584 | int device_ids[max_device_id_size]; | ||||
585 | ccv_nnc_stream_context_t* streams[parallel_count]; | ||||
586 | ccv_nnc_stream_signal_t* signal; | ||||
587 | if (stream_context) | ||||
588 | signal = ccv_nnc_stream_context_emit_signal_new(stream_context); | ||||
589 | for (i = 0; i < parallel_count; i++) | ||||
590 | { | ||||
591 | int flag = 0; | ||||
592 | for (j = 0; !flag && j < per_input_size; j++) | ||||
593 | if (input_tensors[i * per_input_size + j]) | ||||
594 | flag = (CCV_TENSOR_GET_MEMORY(input_tensors[i * per_input_size + j]->info.type)((input_tensors[i * per_input_size + j]->info.type) & 0x3 ) == CCV_TENSOR_GPU_MEMORY); | ||||
595 | for (j = 0; j < per_output_size; j++) | ||||
596 | { | ||||
597 | output_tensors[j] = outputs[j + i * per_output_size] ? ccv_nnc_tensor_from_variable(graph, outputs[j + i * per_output_size], stream_context)ccv_nnc_tensor_from_variable_impl(graph, outputs[j + i * per_output_size ], stream_context) : 0; | ||||
598 | if (output_tensors[j] && !flag) | ||||
599 | flag = (CCV_TENSOR_GET_MEMORY(output_tensors[j]->info.type)((output_tensors[j]->info.type) & 0x3) == CCV_TENSOR_GPU_MEMORY); | ||||
600 | } | ||||
601 | const int stream_type = flag ? CCV_STREAM_CONTEXT_GPU : CCV_STREAM_CONTEXT_CPU; | ||||
602 | const int tensor_type = flag ? CCV_TENSOR_GPU_MEMORY : CCV_TENSOR_CPU_MEMORY; | ||||
603 | const int device_id_size = ccv_nnc_device_ids_for_io(input_tensors + i * per_input_size, per_input_size, output_tensors, per_output_size, tensor_type, device_ids, max_device_id_size); | ||||
604 | ccv_nnc_stream_context_t* stream_0 = 0; | ||||
605 | for (j = 0; j < device_id_size; j++) | ||||
606 | { | ||||
607 | int type = stream_type; | ||||
608 | CCV_STREAM_SET_DEVICE_ID(type, device_ids[j])(type) = (((type) & ~0xfff00) | (((device_ids[j]) & 0xfff ) << 8)); | ||||
609 | ccv_nnc_stream_context_t* const stream = _ccv_nnc_dynamic_graph_get_stream(graph, type); | ||||
610 | if (!stream_0) | ||||
611 | stream_0 = stream; | ||||
612 | } | ||||
613 | // Wait signal to finish. | ||||
614 | if (stream_context) | ||||
615 | { | ||||
616 | if (stream_0) | ||||
617 | ccv_nnc_stream_context_wait_signal(stream_0, signal); | ||||
618 | else | ||||
619 | ccv_nnc_stream_context_wait(stream_context); | ||||
620 | } | ||||
621 | if (stream_0) | ||||
622 | { | ||||
623 | ccv_nnc_dynamic_graph_neighbor_context_discovery_t discovery = { | ||||
624 | .graph = graph, | ||||
625 | .stream_type = stream_type | ||||
626 | }; | ||||
627 | ccv_nnc_stream_context_set_neighbor_discovery(stream_0, _ccv_nnc_dynamic_graph_neighbor_context_discovery, &discovery); | ||||
628 | } | ||||
629 | PRINT(CCV_CLI_INFO, "%s: [%d] -> [%d]\n", ccv_nnc_cmd_name(cmd.cmd), per_input_size, per_output_size)do { if ((CCV_CLI_INFO & ccv_cli_get_output_levels())) { printf ("%s: [%d] -> [%d]\n", ccv_nnc_cmd_name(cmd.cmd), per_input_size , per_output_size); fflush(stdout); } } while (0); | ||||
630 | int k; | ||||
631 | for (k = 0; k < per_input_size; k++) | ||||
632 | { | ||||
633 | PRINT(CCV_CLI_INFO, "|-> %d. %p (%p:%d)", k + 1, input_tensors[k + i * per_input_size], (input_tensors[k + i * per_input_size] ? input_tensors[k + i * per_input_size]->data.u8 : 0), (input_tensors[k + i * per_input_size] ? CCV_TENSOR_GET_DEVICE_ID(input_tensors[k + i * per_input_size]->info.type) : -1))do { if ((CCV_CLI_INFO & ccv_cli_get_output_levels())) { printf ("|-> %d. %p (%p:%d)", k + 1, input_tensors[k + i * per_input_size ], (input_tensors[k + i * per_input_size] ? input_tensors[k + i * per_input_size]->data.u8 : 0), (input_tensors[k + i * per_input_size] ? (((input_tensors[k + i * per_input_size]-> info.type) & 0xfff00) >> 8) : -1)); fflush(stdout); } } while (0); | ||||
634 | if (input_tensors[k + i * per_input_size] && CCV_CLI_OUTPUT_LEVEL_IS(CCV_CLI_INFO)(CCV_CLI_INFO & ccv_cli_get_output_levels())) | ||||
635 | ccv_nnc_print_tensor_info(input_tensors[k + i * per_input_size]); | ||||
636 | PRINT(CCV_CLI_INFO, "\n")do { if ((CCV_CLI_INFO & ccv_cli_get_output_levels())) { printf ("\n"); fflush(stdout); } } while (0); | ||||
637 | } | ||||
638 | for (k = 0; k < per_output_size; k++) | ||||
639 | { | ||||
640 | PRINT(CCV_CLI_INFO, "|<- %d. %p (%p:%d)", k + 1, output_tensors[k], (output_tensors[k] ? output_tensors[k]->data.u8 : 0), (output_tensors[k] ? CCV_TENSOR_GET_DEVICE_ID(output_tensors[k]->info.type) : -1))do { if ((CCV_CLI_INFO & ccv_cli_get_output_levels())) { printf ("|<- %d. %p (%p:%d)", k + 1, output_tensors[k], (output_tensors [k] ? output_tensors[k]->data.u8 : 0), (output_tensors[k] ? (((output_tensors[k]->info.type) & 0xfff00) >> 8 ) : -1)); fflush(stdout); } } while (0); | ||||
641 | if (output_tensors[k] && CCV_CLI_OUTPUT_LEVEL_IS(CCV_CLI_INFO)(CCV_CLI_INFO & ccv_cli_get_output_levels())) | ||||
642 | ccv_nnc_print_tensor_shape(output_tensors[k]); | ||||
643 | PRINT(CCV_CLI_INFO, "\n")do { if ((CCV_CLI_INFO & ccv_cli_get_output_levels())) { printf ("\n"); fflush(stdout); } } while (0); | ||||
644 | } | ||||
645 | const int status = ccv_nnc_cmd_exec(cmd, hint, flags, input_tensors + i * per_input_size, per_input_size, output_tensors, per_output_size, stream_0); | ||||
646 | if (status != 0) | ||||
647 | PRINT(CCV_CLI_INFO, "Invalid Status: %d\n", status)do { if ((CCV_CLI_INFO & ccv_cli_get_output_levels())) { printf ("Invalid Status: %d\n", status); fflush(stdout); } } while ( 0); | ||||
648 | if (CCV_CLI_OUTPUT_LEVEL_IS(CCV_CLI_VERBOSE)(CCV_CLI_VERBOSE & ccv_cli_get_output_levels())) | ||||
649 | { | ||||
650 | for (k = 0; k < per_output_size; k++) | ||||
651 | { | ||||
652 | PRINT(CCV_CLI_VERBOSE, "POST: |<- %d. %p (%p:%d)", k + 1, output_tensors[k], (output_tensors[k] ? output_tensors[k]->data.u8 : 0), (output_tensors[k] ? CCV_TENSOR_GET_DEVICE_ID(output_tensors[k]->info.type) : -1))do { if ((CCV_CLI_VERBOSE & ccv_cli_get_output_levels())) { printf("POST: |<- %d. %p (%p:%d)", k + 1, output_tensors [k], (output_tensors[k] ? output_tensors[k]->data.u8 : 0), (output_tensors[k] ? (((output_tensors[k]->info.type) & 0xfff00) >> 8) : -1)); fflush(stdout); } } while (0); | ||||
653 | if (output_tensors[k]) | ||||
654 | ccv_nnc_print_tensor_info(output_tensors[k]); | ||||
655 | PRINT(CCV_CLI_VERBOSE, "\n")do { if ((CCV_CLI_VERBOSE & ccv_cli_get_output_levels())) { printf("\n"); fflush(stdout); } } while (0); | ||||
656 | } | ||||
657 | } | ||||
658 | if (stream_context && stream_0) | ||||
659 | { | ||||
660 | ccv_nnc_stream_signal_t* const signal = ccv_nnc_stream_context_emit_signal_new(stream_0); | ||||
661 | ccv_nnc_stream_context_wait_signal(stream_context, signal); | ||||
662 | } | ||||
663 | streams[i] = stream_0; | ||||
664 | } | ||||
665 | if (!stream_context) | ||||
666 | for (i = 0; i < parallel_count; i++) | ||||
667 | if (streams[i]) | ||||
668 | ccv_nnc_stream_context_wait(streams[i]); | ||||
669 | } else { | ||||
670 | for (i = 0; i < per_output_size; i++) | ||||
671 | output_tensors[i] = outputs[i] ? ccv_nnc_tensor_from_variable(graph, outputs[i], stream_context)ccv_nnc_tensor_from_variable_impl(graph, outputs[i], stream_context ) : 0; | ||||
672 | PRINT(CCV_CLI_INFO, "%s: [%d] -> [%d]\n", ccv_nnc_cmd_name(cmd.cmd), per_input_size, per_output_size)do { if ((CCV_CLI_INFO & ccv_cli_get_output_levels())) { printf ("%s: [%d] -> [%d]\n", ccv_nnc_cmd_name(cmd.cmd), per_input_size , per_output_size); fflush(stdout); } } while (0); | ||||
673 | for (i = 0; i < per_input_size; i++) | ||||
674 | { | ||||
675 | PRINT(CCV_CLI_INFO, "|-> %d. %p (%p:%d)", i + 1, input_tensors[i], (input_tensors[i] ? input_tensors[i]->data.u8 : 0), (input_tensors[i] ? CCV_TENSOR_GET_DEVICE_ID(input_tensors[i]->info.type) : -1))do { if ((CCV_CLI_INFO & ccv_cli_get_output_levels())) { printf ("|-> %d. %p (%p:%d)", i + 1, input_tensors[i], (input_tensors [i] ? input_tensors[i]->data.u8 : 0), (input_tensors[i] ? ( ((input_tensors[i]->info.type) & 0xfff00) >> 8) : -1)); fflush(stdout); } } while (0); | ||||
676 | if (input_tensors[i] && CCV_CLI_OUTPUT_LEVEL_IS(CCV_CLI_INFO)(CCV_CLI_INFO & ccv_cli_get_output_levels())) | ||||
| |||||
677 | ccv_nnc_print_tensor_info(input_tensors[i]); | ||||
678 | PRINT(CCV_CLI_INFO, "\n")do { if ((CCV_CLI_INFO & ccv_cli_get_output_levels())) { printf ("\n"); fflush(stdout); } } while (0); | ||||
679 | } | ||||
680 | ccv_nnc_cmd_exec(cmd, hint, flags, input_tensors, per_input_size, output_tensors, per_output_size, stream_context); | ||||
681 | for (i = 0; i < per_output_size; i++) | ||||
682 | { | ||||
683 | PRINT(CCV_CLI_INFO, "|<- %d. %p (%p:%d)", i + 1, output_tensors[i], (output_tensors[i] ? output_tensors[i]->data.u8 : 0), (output_tensors[i] ? CCV_TENSOR_GET_DEVICE_ID(output_tensors[i]->info.type) : -1))do { if ((CCV_CLI_INFO & ccv_cli_get_output_levels())) { printf ("|<- %d. %p (%p:%d)", i + 1, output_tensors[i], (output_tensors [i] ? output_tensors[i]->data.u8 : 0), (output_tensors[i] ? (((output_tensors[i]->info.type) & 0xfff00) >> 8 ) : -1)); fflush(stdout); } } while (0); | ||||
684 | if (output_tensors[i] && CCV_CLI_OUTPUT_LEVEL_IS(CCV_CLI_INFO)(CCV_CLI_INFO & ccv_cli_get_output_levels())) | ||||
685 | ccv_nnc_print_tensor_info(output_tensors[i]); | ||||
686 | PRINT(CCV_CLI_INFO, "\n")do { if ((CCV_CLI_INFO & ccv_cli_get_output_levels())) { printf ("\n"); fflush(stdout); } } while (0); | ||||
687 | } | ||||
688 | } | ||||
689 | int inputs_are_constants = 1; | ||||
690 | for (i = 0; inputs_are_constants && i < input_size; i++) | ||||
691 | if (inputs[i] && inputs[i]->type != CCV_NNC_TENSOR_CONSTANT) | ||||
692 | inputs_are_constants = 0; | ||||
693 | if (input_size > 0 && !inputs_are_constants && !graph->no_grad) // No need to record the execution if there is no input or we disabled gradient computation. | ||||
694 | { | ||||
695 | ccv_nnc_tensor_symbol_t output_symbols[ccv_max(1, output_size)({ typeof (1) _a = (1); typeof (output_size) _b = (output_size ); (_a > _b) ? _a : _b; })]; | ||||
696 | for (i = 0; i < output_size; i++) | ||||
697 | if (outputs[i]) | ||||
698 | { | ||||
699 | assert(outputs[i]->type != CCV_NNC_TENSOR_CONSTANT)((void) sizeof ((outputs[i]->type != CCV_NNC_TENSOR_CONSTANT ) ? 1 : 0), __extension__ ({ if (outputs[i]->type != CCV_NNC_TENSOR_CONSTANT ) ; else __assert_fail ("outputs[i]->type != CCV_NNC_TENSOR_CONSTANT" , "ccv_nnc_dynamic_graph.c", 699, __extension__ __PRETTY_FUNCTION__ ); })); | ||||
700 | output_symbols[i] = _ccv_nnc_tensor_symbol_from_variable(graph, outputs[i]); | ||||
701 | } else | ||||
702 | output_symbols[i] = NO_TENSOR_SYMBOL(const ccv_nnc_tensor_symbol_t){.d = CCV_NNC_NO_TENSOR_SYMBOL }; | ||||
703 | int t; | ||||
704 | for (t = 0; t < parallel_count; t++) | ||||
705 | { | ||||
706 | ccv_nnc_graph_exec_symbol_t graph_exec = ccv_nnc_graph_exec_symbol_new(graph->tape, cmd, input_symbols + t * per_input_size, per_input_size, output_symbols + t * per_output_size, per_output_size, 0); | ||||
707 | if (graph_execs) | ||||
708 | graph_execs[t] = graph_exec; | ||||
709 | // This needs to be done before we set the new sources on the outputs. | ||||
710 | for (i = 0; i < per_input_size; i++) | ||||
711 | { | ||||
712 | ccv_array_t* const input_source = input_sources[i + t * per_input_size]; | ||||
713 | if (input_source) | ||||
714 | for (j = 0; j < input_source->rnum; j++) | ||||
715 | ccv_nnc_graph_exec_symbol_concat(graph->tape, (ccv_nnc_graph_exec_symbol_t){ | ||||
716 | .d = *(int*)ccv_array_get(input_source, j)((void*)(((char*)((input_source)->data)) + (size_t)(input_source )->rsize * (size_t)(j))), | ||||
717 | .graph = graph->tape | ||||
718 | }, graph_exec); | ||||
719 | ccv_array_t* const input_alias_source = input_alias_sources[i + t * per_input_size]; | ||||
720 | if (input_alias_source) | ||||
721 | for (j = 0; j < input_alias_source->rnum; j++) | ||||
722 | ccv_nnc_graph_exec_symbol_concat(graph->tape, (ccv_nnc_graph_exec_symbol_t){ | ||||
723 | .d = *(int*)ccv_array_get(input_alias_source, j)((void*)(((char*)((input_alias_source)->data)) + (size_t)( input_alias_source)->rsize * (size_t)(j))), | ||||
724 | .graph = graph->tape | ||||
725 | }, graph_exec); | ||||
726 | } | ||||
727 | for (i = 0; i < per_input_size; i++) | ||||
728 | { | ||||
729 | ccv_nnc_tensor_variable_t const input = inputs[i + t * per_input_size]; | ||||
730 | if (!input || input->type == CCV_NNC_TENSOR_CONSTANT) | ||||
731 | continue; | ||||
732 | ccv_nnc_tensor_variable_graph_bind_t* const bind = (ccv_nnc_tensor_variable_graph_bind_t*)ccv_array_get(graph->binds, input_symbols[i + t * per_input_size].d)((void*)(((char*)((graph->binds)->data)) + (size_t)(graph ->binds)->rsize * (size_t)(input_symbols[i + t * per_input_size ].d))); | ||||
733 | if (!bind->destinations) | ||||
734 | bind->destinations = ccv_array_new(sizeof(int), 1, 0); | ||||
735 | ccv_array_add_unique_int(bind->destinations, graph_exec.d); | ||||
736 | if (input->alias_index_ref) | ||||
737 | { | ||||
738 | const int alias_index = input->alias_index_ref - 1; | ||||
739 | assert(alias_index >= 0)((void) sizeof ((alias_index >= 0) ? 1 : 0), __extension__ ({ if (alias_index >= 0) ; else __assert_fail ("alias_index >= 0" , "ccv_nnc_dynamic_graph.c", 739, __extension__ __PRETTY_FUNCTION__ ); })); | ||||
740 | ccv_nnc_tensor_variable_t variable_to = *(ccv_nnc_tensor_variable_t*)ccv_array_get(graph->vars, alias_index)((void*)(((char*)((graph->vars)->data)) + (size_t)(graph ->vars)->rsize * (size_t)(alias_index))); | ||||
741 | ccv_nnc_tensor_variable_graph_bind_t* const root_bind = (ccv_nnc_tensor_variable_graph_bind_t*)ccv_array_get(graph->binds, variable_to->symbol.d)((void*)(((char*)((graph->binds)->data)) + (size_t)(graph ->binds)->rsize * (size_t)(variable_to->symbol.d))); | ||||
742 | if (!root_bind->destinations) | ||||
743 | root_bind->destinations = ccv_array_new(sizeof(int), 1, 0); | ||||
744 | ccv_array_add_unique_int(root_bind->destinations, graph_exec.d); | ||||
745 | } | ||||
746 | } | ||||
747 | for (i = 0; i < per_output_size; i++) | ||||
748 | { | ||||
749 | ccv_nnc_tensor_variable_t const output = outputs[i + t * per_output_size]; | ||||
750 | if (!output) | ||||
751 | continue; | ||||
752 | ccv_nnc_tensor_variable_graph_bind_t* const bind = (ccv_nnc_tensor_variable_graph_bind_t*)ccv_array_get(graph->binds, output_symbols[i + t * per_output_size].d)((void*)(((char*)((graph->binds)->data)) + (size_t)(graph ->binds)->rsize * (size_t)(output_symbols[i + t * per_output_size ].d))); | ||||
753 | assert(!bind->sources)((void) sizeof ((!bind->sources) ? 1 : 0), __extension__ ( { if (!bind->sources) ; else __assert_fail ("!bind->sources" , "ccv_nnc_dynamic_graph.c", 753, __extension__ __PRETTY_FUNCTION__ ); })); // This is a new symbol, therefore, no binded sources associated yet. | ||||
754 | bind->sources = ccv_array_new(sizeof(int), 1, 0); | ||||
755 | ccv_array_add_unique_int(bind->sources, graph_exec.d); | ||||
756 | if (output->alias_index_ref) | ||||
757 | { | ||||
758 | const int alias_index = output->alias_index_ref - 1; | ||||
759 | assert(alias_index >= 0)((void) sizeof ((alias_index >= 0) ? 1 : 0), __extension__ ({ if (alias_index >= 0) ; else __assert_fail ("alias_index >= 0" , "ccv_nnc_dynamic_graph.c", 759, __extension__ __PRETTY_FUNCTION__ ); })); | ||||
760 | ccv_nnc_tensor_variable_t variable_to = *(ccv_nnc_tensor_variable_t*)ccv_array_get(graph->vars, alias_index)((void*)(((char*)((graph->vars)->data)) + (size_t)(graph ->vars)->rsize * (size_t)(alias_index))); | ||||
761 | ccv_nnc_tensor_variable_graph_bind_t* const root_bind = (ccv_nnc_tensor_variable_graph_bind_t*)ccv_array_get(graph->binds, variable_to->symbol.d)((void*)(((char*)((graph->binds)->data)) + (size_t)(graph ->binds)->rsize * (size_t)(variable_to->symbol.d))); | ||||
762 | if (!root_bind->sources) | ||||
763 | root_bind->sources = ccv_array_new(sizeof(int), 1, 0); | ||||
764 | ccv_array_add_unique_int(root_bind->sources, graph_exec.d); | ||||
765 | } | ||||
766 | } | ||||
767 | } | ||||
768 | } | ||||
769 | // Now, able to free some of the reused outputs. | ||||
770 | for (i = 0; i < freeable_size; i++) | ||||
771 | ccv_nnc_tensor_variable_free(graph, freeables[i]); | ||||
772 | } | ||||
773 | |||||
774 | int ccv_nnc_dynamic_graph_exec(ccv_nnc_dynamic_graph_t* const graph, const ccv_nnc_cmd_t cmd, const ccv_nnc_hint_t hint, const int flags, const ccv_nnc_tensor_variable_t* const inputs, const int input_size, ccv_nnc_tensor_variable_t* const outputs, const int output_size, const int parallel, ccv_nnc_stream_context_t* const stream_context) | ||||
775 | { | ||||
776 | ccv_nnc_dynamic_graph_exec_ret(graph, cmd, hint, flags, inputs, input_size, outputs, output_size, parallel, stream_context, 0); | ||||
777 | return CCV_NNC_EXEC_SUCCESS; | ||||
778 | } | ||||
779 | |||||
780 | static int _ccv_nnc_tensor_variable_is_only_output(ccv_nnc_dynamic_graph_t* const graph, const ccv_nnc_tensor_variable_graph_bind_t* bind, const int symbol_d) | ||||
781 | { | ||||
782 | if (bind->alias_ref) | ||||
783 | bind = (ccv_nnc_tensor_variable_graph_bind_t*)ccv_array_get(graph->binds, bind->alias_ref - 1)((void*)(((char*)((graph->binds)->data)) + (size_t)(graph ->binds)->rsize * (size_t)(bind->alias_ref - 1))); | ||||
784 | if (!bind->sources || bind->sources->rnum == 0) | ||||
785 | return 1; | ||||
786 | int i; | ||||
787 | for (i = 0; i < bind->sources->rnum; i++) | ||||
788 | { | ||||
789 | const int exec_symbol_d = *(int*)ccv_array_get(bind->sources, i)((void*)(((char*)((bind->sources)->data)) + (size_t)(bind ->sources)->rsize * (size_t)(i))); | ||||
790 | const ccv_nnc_graph_exec_symbol_t exec_symbol = { | ||||
791 | .d = exec_symbol_d, | ||||
792 | .graph = graph->tape | ||||
793 | }; | ||||
794 | const int* outputs; int output_size; | ||||
795 | ccv_nnc_graph_exec_symbol_io(graph->tape, exec_symbol, 0, 0, &outputs, &output_size); | ||||
796 | int j; | ||||
797 | for (j = 0; j < output_size; j++) | ||||
798 | if (outputs[j] >= 0 && outputs[j] != symbol_d) // If output is me, it is the only output. | ||||
799 | { | ||||
800 | assert(outputs[j] < graph->binds->rnum)((void) sizeof ((outputs[j] < graph->binds->rnum) ? 1 : 0), __extension__ ({ if (outputs[j] < graph->binds-> rnum) ; else __assert_fail ("outputs[j] < graph->binds->rnum" , "ccv_nnc_dynamic_graph.c", 800, __extension__ __PRETTY_FUNCTION__ ); })); | ||||
801 | const ccv_nnc_tensor_variable_graph_bind_t* other_bind = (ccv_nnc_tensor_variable_graph_bind_t*)ccv_array_get(graph->binds, outputs[j])((void*)(((char*)((graph->binds)->data)) + (size_t)(graph ->binds)->rsize * (size_t)(outputs[j]))); | ||||
802 | // This is in use and is it not a constant symbol. | ||||
803 | if (other_bind->index >= 0 && other_bind->type != CCV_NNC_TENSOR_CONSTANT) | ||||
804 | return 0; | ||||
805 | if (other_bind->alias_ref) // If this is alias, use its original's destinations. | ||||
806 | other_bind = (ccv_nnc_tensor_variable_graph_bind_t*)ccv_array_get(graph->binds, other_bind->alias_ref - 1)((void*)(((char*)((graph->binds)->data)) + (size_t)(graph ->binds)->rsize * (size_t)(other_bind->alias_ref - 1 ))); | ||||
807 | // The original is in use and is it not a constant symbol. | ||||
808 | if (other_bind->index >= 0 && other_bind->type != CCV_NNC_TENSOR_CONSTANT) | ||||
809 | return 0; | ||||
810 | if (other_bind->destinations && other_bind->destinations->rnum > 0) | ||||
811 | return 0; | ||||
812 | } | ||||
813 | } | ||||
814 | return 1; | ||||
815 | } | ||||
816 | |||||
817 | static void _ccv_nnc_update_bind_destinations_when_free(ccv_nnc_dynamic_graph_t* const graph, const int freed_exec_symbol_d, ccv_array_t* const binds, ccv_nnc_tensor_variable_graph_bind_t* const bind, const int tensor_index, ccv_array_t* const ws) | ||||
818 | { | ||||
819 | int i; | ||||
820 | if (bind->destinations) | ||||
821 | { | ||||
822 | int flag = 0; | ||||
823 | for (i = 0; !flag && i < bind->destinations->rnum; i++) | ||||
824 | { | ||||
825 | const int exec_symbol_d = *(int*)ccv_array_get(bind->destinations, i)((void*)(((char*)((bind->destinations)->data)) + (size_t )(bind->destinations)->rsize * (size_t)(i))); | ||||
826 | if (exec_symbol_d == freed_exec_symbol_d) | ||||
827 | { | ||||
828 | if (i < bind->destinations->rnum - 1) | ||||
829 | *(int*)ccv_array_get(bind->destinations, i)((void*)(((char*)((bind->destinations)->data)) + (size_t )(bind->destinations)->rsize * (size_t)(i))) = *(int*)ccv_array_get(bind->destinations, bind->destinations->rnum - 1)((void*)(((char*)((bind->destinations)->data)) + (size_t )(bind->destinations)->rsize * (size_t)(bind->destinations ->rnum - 1))); | ||||
830 | --bind->destinations->rnum; | ||||
831 | flag = 1; | ||||
832 | } | ||||
833 | } | ||||
834 | // This symbol can be freed. | ||||
835 | if (flag && bind->index == CCV_NNC_TENSOR_NO_VARIABLE_BUT_USED) | ||||
836 | { | ||||
837 | ccv_nnc_tensor_variable_graph_bind_t* root_bind = bind; | ||||
838 | if (bind->alias_ref) | ||||
839 | { | ||||
840 | root_bind = (ccv_nnc_tensor_variable_graph_bind_t*)ccv_array_get(binds, bind->alias_ref - 1)((void*)(((char*)((binds)->data)) + (size_t)(binds)->rsize * (size_t)(bind->alias_ref - 1))); | ||||
841 | if (root_bind->index == CCV_NNC_TENSOR_NO_VARIABLE) | ||||
842 | root_bind = bind; | ||||
843 | } | ||||
844 | // If the alias_ref is not freed, we cannot free this, unless it is very clear there is no reference to this any more. | ||||
845 | // It is possible because exec will be freed already, thus, it is safe to remove this alias out. | ||||
846 | if (root_bind->index == CCV_NNC_TENSOR_NO_VARIABLE_BUT_USED && | ||||
847 | ((!root_bind->sources || root_bind->sources->rnum == 0) || _ccv_nnc_tensor_variable_is_only_output(graph, bind, tensor_index)) && | ||||
848 | root_bind->destinations->rnum == 0) | ||||
849 | { | ||||
850 | if (root_bind->sources) | ||||
851 | for (i = 0; i < root_bind->sources->rnum; i++) | ||||
852 | ccv_array_add_unique_int(ws, *(int*)ccv_array_get(root_bind->sources, i)((void*)(((char*)((root_bind->sources)->data)) + (size_t )(root_bind->sources)->rsize * (size_t)(i)))); | ||||
853 | _ccv_nnc_tensor_variable_graph_bind_free(graph, bind, 1); | ||||
854 | ccv_nnc_tensor_symbol_free(graph->tape, (ccv_nnc_tensor_symbol_t){ | ||||
855 | .d = tensor_index, | ||||
856 | .graph = graph->tape | ||||
857 | }); | ||||
858 | } else if (bind->index == CCV_NNC_TENSOR_NO_VARIABLE_BUT_USED && // Handle the case the bind is already freed, and it doesn't have any sources or destinations. | ||||
859 | bind->alias_ref && (!bind->sources || bind->sources->rnum == 0) && (!bind->destinations || bind->destinations->rnum == 0)) { | ||||
860 | _ccv_nnc_tensor_variable_graph_bind_free(graph, bind, 1); | ||||
861 | ccv_nnc_tensor_symbol_free(graph->tape, (ccv_nnc_tensor_symbol_t){ | ||||
862 | .d = tensor_index, | ||||
863 | .graph = graph->tape | ||||
864 | }); | ||||
865 | } | ||||
866 | } | ||||
867 | } | ||||
868 | } | ||||
869 | |||||
870 | static void _ccv_nnc_update_bind_sources_when_free(ccv_nnc_dynamic_graph_t* const graph, const int freed_exec_symbol_d, ccv_array_t* const binds, ccv_nnc_tensor_variable_graph_bind_t* const bind, const int tensor_index, ccv_array_t* const ws) | ||||
871 | { | ||||
872 | int i; | ||||
873 | if (bind->sources) | ||||
874 | { | ||||
875 | int flag = 0; | ||||
876 | for (i = 0; !flag && i < bind->sources->rnum; i++) | ||||
877 | { | ||||
878 | const int exec_symbol_d = *(int*)ccv_array_get(bind->sources, i)((void*)(((char*)((bind->sources)->data)) + (size_t)(bind ->sources)->rsize * (size_t)(i))); | ||||
879 | if (exec_symbol_d == freed_exec_symbol_d) | ||||
880 | { | ||||
881 | if (i < bind->sources->rnum - 1) | ||||
882 | *(int*)ccv_array_get(bind->sources, i)((void*)(((char*)((bind->sources)->data)) + (size_t)(bind ->sources)->rsize * (size_t)(i))) = *(int*)ccv_array_get(bind->sources, bind->sources->rnum - 1)((void*)(((char*)((bind->sources)->data)) + (size_t)(bind ->sources)->rsize * (size_t)(bind->sources->rnum - 1))); | ||||
883 | --bind->sources->rnum; | ||||
884 | flag = 1; | ||||
885 | } | ||||
886 | } | ||||
887 | if (flag && !bind->alias_ref && bind->index >= 0 && bind->type == CCV_NNC_TENSOR_CONSTANT && // If it is detached (constant but previously has sources). Now can check again. | ||||
888 | (bind->sources->rnum == 0 || _ccv_nnc_tensor_variable_is_only_output(graph, bind, tensor_index)) && | ||||
889 | (!bind->destinations || bind->destinations->rnum == 0)) | ||||
890 | { | ||||
891 | // If this is constant, set it to be no symbol again. | ||||
892 | ccv_nnc_tensor_variable_t tv = *(ccv_nnc_tensor_variable_t*)ccv_array_get(graph->vars, bind->index)((void*)(((char*)((graph->vars)->data)) + (size_t)(graph ->vars)->rsize * (size_t)(bind->index))); | ||||
893 | tv->symbol = NO_TENSOR_SYMBOL(const ccv_nnc_tensor_symbol_t){.d = CCV_NNC_NO_TENSOR_SYMBOL }; | ||||
894 | _ccv_nnc_tensor_variable_graph_bind_free(graph, bind, 1); | ||||
895 | ccv_nnc_tensor_symbol_free(graph->tape, (ccv_nnc_tensor_symbol_t){ | ||||
896 | .d = tensor_index, | ||||
897 | .graph = graph->tape | ||||
898 | }); | ||||
899 | } else if (flag && bind->index == CCV_NNC_TENSOR_NO_VARIABLE_BUT_USED) { | ||||
900 | // This symbol can be freed. | ||||
901 | ccv_nnc_tensor_variable_graph_bind_t* root_bind = bind; | ||||
902 | if (bind->alias_ref) | ||||
903 | { | ||||
904 | root_bind = (ccv_nnc_tensor_variable_graph_bind_t*)ccv_array_get(binds, bind->alias_ref - 1)((void*)(((char*)((binds)->data)) + (size_t)(binds)->rsize * (size_t)(bind->alias_ref - 1))); | ||||
905 | if (root_bind->index == CCV_NNC_TENSOR_NO_VARIABLE) | ||||
906 | root_bind = bind; | ||||
907 | } | ||||
908 | // If the alias_ref is not freed, we cannot free this, unless it is very clear there is no reference to this any more. | ||||
909 | // It is possible because exec will be freed already, thus, it is safe to remove this alias out. | ||||
910 | if (root_bind->index == CCV_NNC_TENSOR_NO_VARIABLE_BUT_USED && | ||||
911 | (root_bind->sources->rnum == 0 || _ccv_nnc_tensor_variable_is_only_output(graph, bind, tensor_index)) && | ||||
912 | (!root_bind->destinations || root_bind->destinations->rnum == 0)) | ||||
913 | { | ||||
914 | for (i = 0; i < root_bind->sources->rnum; i++) | ||||
915 | ccv_array_add_unique_int(ws, *(int*)ccv_array_get(root_bind->sources, i)((void*)(((char*)((root_bind->sources)->data)) + (size_t )(root_bind->sources)->rsize * (size_t)(i)))); | ||||
916 | _ccv_nnc_tensor_variable_graph_bind_free(graph, bind, 1); | ||||
917 | ccv_nnc_tensor_symbol_free(graph->tape, (ccv_nnc_tensor_symbol_t){ | ||||
918 | .d = tensor_index, | ||||
919 | .graph = graph->tape | ||||
920 | }); | ||||
921 | } else if (bind->index == CCV_NNC_TENSOR_NO_VARIABLE_BUT_USED && // Handle the case the bind is already freed, and it doesn't have any sources or destinations. | ||||
922 | bind->alias_ref && (!bind->sources || bind->sources->rnum == 0) && (!bind->destinations || bind->destinations->rnum == 0)) { | ||||
923 | _ccv_nnc_tensor_variable_graph_bind_free(graph, bind, 1); | ||||
924 | ccv_nnc_tensor_symbol_free(graph->tape, (ccv_nnc_tensor_symbol_t){ | ||||
925 | .d = tensor_index, | ||||
926 | .graph = graph->tape | ||||
927 | }); | ||||
928 | } | ||||
929 | } | ||||
930 | } | ||||
931 | } | ||||
932 | |||||
933 | static void _ccv_nnc_update_bind_sources_destinations_when_free(ccv_nnc_dynamic_graph_t* const graph, const int freed_exec_symbol_d, ccv_array_t* const binds, const int* const inputs, const int input_size, const int* const outputs, const int output_size, ccv_array_t* const ws) | ||||
934 | { | ||||
935 | int i; | ||||
936 | for (i = 0; i < input_size; i++) | ||||
937 | if (inputs[i] >= 0 && inputs[i] < binds->rnum) | ||||
938 | { | ||||
939 | ccv_nnc_tensor_variable_graph_bind_t* const bind = (ccv_nnc_tensor_variable_graph_bind_t*)ccv_array_get(binds, inputs[i])((void*)(((char*)((binds)->data)) + (size_t)(binds)->rsize * (size_t)(inputs[i]))); | ||||
940 | if (bind->index == CCV_NNC_TENSOR_NO_VARIABLE) | ||||
941 | continue; | ||||
942 | if (bind->alias_ref) | ||||
943 | { | ||||
944 | const int alias_to = bind->alias_ref - 1; | ||||
945 | ccv_nnc_tensor_variable_graph_bind_t* const root_bind = (ccv_nnc_tensor_variable_graph_bind_t*)ccv_array_get(binds, alias_to)((void*)(((char*)((binds)->data)) + (size_t)(binds)->rsize * (size_t)(alias_to))); | ||||
946 | if (root_bind && root_bind->index != CCV_NNC_TENSOR_NO_VARIABLE) | ||||
947 | _ccv_nnc_update_bind_destinations_when_free(graph, freed_exec_symbol_d, binds, root_bind, alias_to, ws); | ||||
948 | } | ||||
949 | _ccv_nnc_update_bind_destinations_when_free(graph, freed_exec_symbol_d, binds, bind, inputs[i], ws); | ||||
950 | } | ||||
951 | // Note that this works because there is no overlap of inputs / outputs. (What about alias?). | ||||
952 | for (i = 0; i < output_size; i++) | ||||
953 | if (outputs[i] >= 0 && outputs[i] < binds->rnum) | ||||
954 | { | ||||
955 | ccv_nnc_tensor_variable_graph_bind_t* const bind = (ccv_nnc_tensor_variable_graph_bind_t*)ccv_array_get(binds, outputs[i])((void*)(((char*)((binds)->data)) + (size_t)(binds)->rsize * (size_t)(outputs[i]))); | ||||
956 | if (bind->index == CCV_NNC_TENSOR_NO_VARIABLE) | ||||
957 | continue; | ||||
958 | if (bind->alias_ref) | ||||
959 | { | ||||
960 | const int alias_to = bind->alias_ref - 1; | ||||
961 | ccv_nnc_tensor_variable_graph_bind_t* const root_bind = (ccv_nnc_tensor_variable_graph_bind_t*)ccv_array_get(binds, alias_to)((void*)(((char*)((binds)->data)) + (size_t)(binds)->rsize * (size_t)(alias_to))); | ||||
962 | if (root_bind && root_bind->index != CCV_NNC_TENSOR_NO_VARIABLE) | ||||
963 | _ccv_nnc_update_bind_sources_when_free(graph, freed_exec_symbol_d, binds, root_bind, alias_to, ws); | ||||
964 | } | ||||
965 | _ccv_nnc_update_bind_sources_when_free(graph, freed_exec_symbol_d, binds, bind, outputs[i], ws); | ||||
966 | } | ||||
967 | } | ||||
968 | |||||
969 | static void _ccv_nnc_stateful_exec_free_if_possible(ccv_nnc_dynamic_graph_t* const graph, const ccv_nnc_graph_exec_symbol_t symbol) | ||||
970 | { | ||||
971 | if (!graph->stateful_execs) | ||||
972 | return; | ||||
973 | assert(symbol.d >= 0)((void) sizeof ((symbol.d >= 0) ? 1 : 0), __extension__ ({ if (symbol.d >= 0) ; else __assert_fail ("symbol.d >= 0" , "ccv_nnc_dynamic_graph.c", 973, __extension__ __PRETTY_FUNCTION__ ); })); | ||||
974 | ccv_array_t* const stateful_execs = graph->stateful_execs; | ||||
975 | ccv_nnc_cmd_t cmd = ccv_nnc_graph_exec_symbol_cmd(graph->tape, symbol); | ||||
976 | ccv_nnc_stateful_exec_t* const stateful_exec = (ccv_nnc_stateful_exec_t*)cmd.data; | ||||
977 | if (!stateful_exec) | ||||
978 | return; | ||||
979 | // If there is no backward, no need to apply gradients. | ||||
980 | // Otherwise, if we applied gradients, we can free it as well. | ||||
981 | // We don't free this stateful exec because apply gradients doesn't require any variables alive. | ||||
982 | if (!stateful_exec->did_backward_but_not_apply_gradients) | ||||
983 | { | ||||
984 | const int index = stateful_exec->index; | ||||
985 | ccfreefree(stateful_exec); | ||||
986 | if (index < graph->reuse_stateful_exec || graph->reuse_stateful_exec < 0) | ||||
987 | graph->reuse_stateful_exec = index; | ||||
988 | *(ccv_nnc_stateful_exec_t**)ccv_array_get(stateful_execs, index)((void*)(((char*)((stateful_execs)->data)) + (size_t)(stateful_execs )->rsize * (size_t)(index))) = 0; | ||||
989 | } else | ||||
990 | stateful_exec->should_free = 1; | ||||
991 | } | ||||
992 | |||||
993 | static int _ccv_nnc_tensor_bind_trace_forward_to_free(ccv_nnc_dynamic_graph_t* const graph, const ccv_nnc_tensor_variable_t tensor_variable, ccv_nnc_tensor_variable_graph_bind_t* const bind, ccv_nnc_tensor_variable_graph_bind_t* const root_bind, int* const ws_start, const int assuming_no_source) // assuming_no_source means we are going to remove sources if possible, thus, it is irrelevant. | ||||
994 | { | ||||
995 | int can_free_symbol = 0; | ||||
996 | const int sources_and_is_only_output = (root_bind->sources && root_bind->sources->rnum > 0) && _ccv_nnc_tensor_variable_is_only_output(graph, bind, tensor_variable->symbol.d); | ||||
997 | if (!root_bind->sources || root_bind->sources->rnum == 0 || sources_and_is_only_output || assuming_no_source) | ||||
998 | { | ||||
999 | int i, j; | ||||
1000 | can_free_symbol = 1; // Assume we can free this symbol. | ||||
1001 | if (!graph->ws) | ||||
1002 | graph->ws = ccv_array_new(sizeof(int), root_bind->destinations ? root_bind->destinations->rnum : 0, 0); | ||||
1003 | ccv_array_t* const ws = graph->ws; | ||||
1004 | ccv_array_clear(ws); | ||||
1005 | if (root_bind->destinations) | ||||
1006 | for (i = 0; i < root_bind->destinations->rnum; i++) | ||||
1007 | ccv_array_add_unique_int(ws, *(int*)ccv_array_get(root_bind->destinations, i)((void*)(((char*)((root_bind->destinations)->data)) + ( size_t)(root_bind->destinations)->rsize * (size_t)(i)))); | ||||
1008 | const int ws_init_size = ws->rnum; | ||||
1009 | *ws_start = ws_init_size; | ||||
1010 | // Add all sources from root_bind, in case it has been freed (during update bind sources / destinations when free. | ||||
1011 | if (root_bind->sources) | ||||
1012 | for (i = 0; i < root_bind->sources->rnum; i++) | ||||
1013 | ccv_array_add_unique_int(ws, *(int*)ccv_array_get(root_bind->sources, i)((void*)(((char*)((root_bind->sources)->data)) + (size_t )(root_bind->sources)->rsize * (size_t)(i)))); | ||||
1014 | // If we cannot loop over any exec symbols (this is not in use). It is simple to determine whether we want | ||||
1015 | // to free it or not: if this is an alias and the origin is not freed, we cannot free this symbol. | ||||
1016 | if (ws_init_size == 0) | ||||
1017 | can_free_symbol = (!bind->alias_ref || root_bind->index < 0); | ||||
1018 | // Go through all the exec symbols use this tensor, to see whether they have inputs that has other sources. | ||||
1019 | for (i = 0; i < ws_init_size; i++) | ||||
1020 | { | ||||
1021 | const int exec_symbol_d = *(int*)ccv_array_get(ws, i)((void*)(((char*)((ws)->data)) + (size_t)(ws)->rsize * ( size_t)(i))); | ||||
1022 | const ccv_nnc_graph_exec_symbol_t symbol = { | ||||
1023 | .d = exec_symbol_d, | ||||
1024 | .graph = graph->tape | ||||
1025 | }; | ||||
1026 | const int* inputs; int input_size; | ||||
1027 | const int* outputs; int output_size; | ||||
1028 | ccv_nnc_graph_exec_symbol_io(graph->tape, symbol, &inputs, &input_size, &outputs, &output_size); | ||||
1029 | int flag = 0; // flag denotes whether there are cases to keep this exec symbol. | ||||
1030 | if (!root_bind->sources || root_bind->sources->rnum == 0 || assuming_no_source) | ||||
1031 | { | ||||
1032 | // If there is no sources, check if other sources can depend on this exec, if they do, we cannot free this. | ||||
1033 | for (j = 0; !flag && j < input_size; j++) | ||||
1034 | if (inputs[j] >= 0 && inputs[j] < graph->binds->rnum && inputs[j] != tensor_variable->symbol.d) | ||||
1035 | { | ||||
1036 | ccv_nnc_tensor_variable_graph_bind_t* other_bind = (ccv_nnc_tensor_variable_graph_bind_t*)ccv_array_get(graph->binds, inputs[j])((void*)(((char*)((graph->binds)->data)) + (size_t)(graph ->binds)->rsize * (size_t)(inputs[j]))); | ||||
1037 | if (other_bind->index >= 0 && other_bind->type != CCV_NNC_TENSOR_CONSTANT) | ||||
1038 | flag = 1; | ||||
1039 | else { | ||||
1040 | if (other_bind->alias_ref) // If this is alias, use its original's destinations. | ||||
1041 | other_bind = (ccv_nnc_tensor_variable_graph_bind_t*)ccv_array_get(graph->binds, other_bind->alias_ref - 1)((void*)(((char*)((graph->binds)->data)) + (size_t)(graph ->binds)->rsize * (size_t)(other_bind->alias_ref - 1 ))); | ||||
1042 | flag = (other_bind->index >= 0 && other_bind->type != CCV_NNC_TENSOR_CONSTANT) || (other_bind->type != CCV_NNC_TENSOR_CONSTANT && other_bind->sources && other_bind->sources->rnum > 0); // Constant should have no source, or it is detached. | ||||
1043 | } | ||||
1044 | } | ||||
1045 | } else { | ||||
1046 | // If there are sources, check whether we have outputs or not. If we do, we cannot free this. | ||||
1047 | for (j = 0; !flag && j < output_size; j++) | ||||
1048 | if (outputs[j] >= 0 && outputs[j] < graph->binds->rnum) | ||||
1049 | { | ||||
1050 | ccv_nnc_tensor_variable_graph_bind_t* other_bind = (ccv_nnc_tensor_variable_graph_bind_t*)ccv_array_get(graph->binds, outputs[j])((void*)(((char*)((graph->binds)->data)) + (size_t)(graph ->binds)->rsize * (size_t)(outputs[j]))); | ||||
1051 | if (other_bind->index >= 0 && other_bind->type != CCV_NNC_TENSOR_CONSTANT) | ||||
1052 | flag = 1; | ||||
1053 | else { | ||||
1054 | if (other_bind->alias_ref) // If this is alias, use its original's destinations. | ||||
1055 | other_bind = (ccv_nnc_tensor_variable_graph_bind_t*)ccv_array_get(graph->binds, other_bind->alias_ref - 1)((void*)(((char*)((graph->binds)->data)) + (size_t)(graph ->binds)->rsize * (size_t)(other_bind->alias_ref - 1 ))); | ||||
1056 | flag = (other_bind->index >= 0 && other_bind->type != CCV_NNC_TENSOR_CONSTANT) || (other_bind->destinations && other_bind->destinations->rnum > 0); | ||||
1057 | } | ||||
1058 | } | ||||
1059 | } | ||||
1060 | // This exec can be freed if there is no input required or there is no output required. | ||||
1061 | can_free_symbol = (can_free_symbol && !flag); | ||||
1062 | if (!flag) | ||||
1063 | { | ||||
1064 | // Go over inputs and remove all references from binded destinations. | ||||
1065 | // and go over outputs remove all references from binded sources. | ||||
1066 | _ccv_nnc_update_bind_sources_destinations_when_free(graph, exec_symbol_d, graph->binds, inputs, input_size, outputs, output_size, ws); | ||||
1067 | const int* outgoings; int outgoing_size; | ||||
1068 | ccv_nnc_graph_exec_symbol_to(graph->tape, symbol, &outgoings, &outgoing_size); | ||||
1069 | for (j = 0; j < outgoing_size; j++) | ||||
1070 | ccv_array_add_unique_int(ws, outgoings[j]); | ||||
1071 | _ccv_nnc_stateful_exec_free_if_possible(graph, symbol); | ||||
1072 | ccv_nnc_graph_exec_symbol_free(graph->tape, symbol); | ||||
1073 | } | ||||
1074 | } | ||||
1075 | } | ||||
1076 | return can_free_symbol; | ||||
1077 | } | ||||
1078 | |||||
1079 | static void _ccv_nnc_tensor_bind_trace_backward_to_free(ccv_nnc_dynamic_graph_t* const graph, ccv_array_t* const ws, const int ws_start) | ||||
1080 | { | ||||
1081 | int i, j; | ||||
1082 | // Now, go over the outgoings, if it is removed, add more to it. Note that the ws array can grow while iterating over. | ||||
1083 | for (i = ws_start; i < ws->rnum; i++) | ||||
1084 | { | ||||
1085 | const int exec_symbol_d = *(int*)ccv_array_get(ws, i)((void*)(((char*)((ws)->data)) + (size_t)(ws)->rsize * ( size_t)(i))); | ||||
1086 | const ccv_nnc_graph_exec_symbol_t symbol = { | ||||
1087 | .d = exec_symbol_d, | ||||
1088 | .graph = graph->tape | ||||
1089 | }; | ||||
1090 | const int* inputs; int input_size; | ||||
1091 | const int* outputs; int output_size; | ||||
1092 | ccv_nnc_graph_exec_symbol_io(graph->tape, symbol, &inputs, &input_size, &outputs, &output_size); | ||||
1093 | int flag = 0; | ||||
1094 | for (j = 0; !flag && j < input_size; j++) | ||||
1095 | if (inputs[j] >= 0 && inputs[j] < graph->binds->rnum) | ||||
1096 | { | ||||
1097 | ccv_nnc_tensor_variable_graph_bind_t* other_bind = (ccv_nnc_tensor_variable_graph_bind_t*)ccv_array_get(graph->binds, inputs[j])((void*)(((char*)((graph->binds)->data)) + (size_t)(graph ->binds)->rsize * (size_t)(inputs[j]))); | ||||
1098 | if (other_bind->index >= 0 && other_bind->type != CCV_NNC_TENSOR_CONSTANT) | ||||
1099 | flag = 1; | ||||
1100 | else { | ||||
1101 | if (other_bind->alias_ref) // If this is alias, use its original's destinations. | ||||
1102 | other_bind = (ccv_nnc_tensor_variable_graph_bind_t*)ccv_array_get(graph->binds, other_bind->alias_ref - 1)((void*)(((char*)((graph->binds)->data)) + (size_t)(graph ->binds)->rsize * (size_t)(other_bind->alias_ref - 1 ))); | ||||
1103 | flag = (other_bind->index >= 0 && other_bind->type != CCV_NNC_TENSOR_CONSTANT) || (other_bind->type != CCV_NNC_TENSOR_CONSTANT && other_bind->sources && other_bind->sources->rnum > 0); | ||||
1104 | } | ||||
1105 | } | ||||
1106 | if (flag) // If any inputs make free this destination impossible. Check whether all its outputs are done. | ||||
1107 | { | ||||
1108 | int output_flag = 0; | ||||
1109 | for (j = 0; !output_flag && j < output_size; j++) | ||||
1110 | if (outputs[j] >= 0 && outputs[j] < graph->binds->rnum) | ||||
1111 | { | ||||
1112 | ccv_nnc_tensor_variable_graph_bind_t* other_bind = (ccv_nnc_tensor_variable_graph_bind_t*)ccv_array_get(graph->binds, outputs[j])((void*)(((char*)((graph->binds)->data)) + (size_t)(graph ->binds)->rsize * (size_t)(outputs[j]))); | ||||
1113 | if (other_bind->index >= 0 && other_bind->type != CCV_NNC_TENSOR_CONSTANT) | ||||
1114 | output_flag = 1; | ||||
1115 | else { | ||||
1116 | if (other_bind->alias_ref) // If this is alias, use its original's destinations. | ||||
1117 | other_bind = (ccv_nnc_tensor_variable_graph_bind_t*)ccv_array_get(graph->binds, other_bind->alias_ref - 1)((void*)(((char*)((graph->binds)->data)) + (size_t)(graph ->binds)->rsize * (size_t)(other_bind->alias_ref - 1 ))); | ||||
1118 | output_flag = (other_bind->index >= 0 && other_bind->type != CCV_NNC_TENSOR_CONSTANT) || (other_bind->destinations && other_bind->destinations->rnum > 0); | ||||
1119 | } | ||||
1120 | } | ||||
1121 | if (!output_flag) // If no output is used (used means it has a tensor variable, or it has a destination). | ||||
1122 | flag = 0; | ||||
1123 | } | ||||
1124 | // Went over all the inputs, it turns out no more inputs has other references, safe to remove. | ||||
1125 | if (!flag) | ||||
1126 | { | ||||
1127 | _ccv_nnc_update_bind_sources_destinations_when_free(graph, exec_symbol_d, graph->binds, inputs, input_size, outputs, output_size, ws); | ||||
1128 | const int* outgoings; int outgoing_size; | ||||
1129 | ccv_nnc_graph_exec_symbol_to(graph->tape, symbol, &outgoings, &outgoing_size); | ||||
1130 | // It it has outgoings, add that for further inspection. | ||||
1131 | for (j = 0; j < outgoing_size; j++) | ||||
1132 | ccv_array_add_unique_int(ws, outgoings[j]); | ||||
1133 | _ccv_nnc_stateful_exec_free_if_possible(graph, symbol); | ||||
1134 | ccv_nnc_graph_exec_symbol_free(graph->tape, symbol); | ||||
1135 | } | ||||
1136 | } | ||||
1137 | } | ||||
1138 | |||||
1139 | void ccv_nnc_tensor_variable_free(ccv_nnc_dynamic_graph_t* const graph, const ccv_nnc_tensor_variable_t tensor_variable) | ||||
1140 | { | ||||
1141 | // If it contains a symbol, this tensor variable is not a free variable. It is either used as input or output. | ||||
1142 | if (tensor_variable->symbol.d != CCV_NNC_NO_TENSOR_SYMBOL) | ||||
1143 | { | ||||
1144 | // If it is not a free variable, when can we free the symbol and the underlying variable? | ||||
1145 | // 1. There should be no sources (the command generate this tensor should be freed) or the output of these sources is only the current one; | ||||
1146 | // 2. The destinations (the commands that uses this tensor) should have no other inputs, or the other inputs has no binded sources as well. | ||||
1147 | ccv_nnc_tensor_variable_graph_bind_t* const bind = (ccv_nnc_tensor_variable_graph_bind_t*)ccv_array_get(graph->binds, tensor_variable->symbol.d)((void*)(((char*)((graph->binds)->data)) + (size_t)(graph ->binds)->rsize * (size_t)(tensor_variable->symbol.d ))); | ||||
1148 | // There should be no source associated with it no more. | ||||
1149 | // I am free if no exec symbol is producing me or the symbol producing me can only producing me (thus, it is not required to | ||||
1150 | // compute gradient because I am the only variable it can compute gradient for). | ||||
1151 | ccv_nnc_tensor_variable_graph_bind_t* root_bind = bind; | ||||
1152 | if (bind->alias_ref) | ||||
1153 | { | ||||
1154 | const int alias_to = bind->alias_ref - 1; | ||||
1155 | root_bind = (ccv_nnc_tensor_variable_graph_bind_t*)ccv_array_get(graph->binds, alias_to)((void*)(((char*)((graph->binds)->data)) + (size_t)(graph ->binds)->rsize * (size_t)(alias_to))); | ||||
1156 | } | ||||
1157 | int ws_start; | ||||
1158 | const int can_free_symbol = _ccv_nnc_tensor_bind_trace_forward_to_free(graph, tensor_variable, bind, root_bind, &ws_start, 0); | ||||
1159 | if (can_free_symbol) | ||||
1160 | { | ||||
1161 | _ccv_nnc_tensor_variable_graph_bind_free(graph, bind, 1); | ||||
1162 | ccv_nnc_tensor_symbol_free(graph->tape, tensor_variable->symbol); | ||||
1163 | _ccv_nnc_tensor_bind_trace_backward_to_free(graph, graph->ws, ws_start); | ||||
1164 | } else { // If this symbol is not freed, move the tensor view to the bind. | ||||
1165 | // If current bind is an alias, and it doesn't have any sources or destinations. We cannot find this alias | ||||
1166 | // through any exec. This is not only safe to delete, but has to be deleted. We don't need to handle this | ||||
1167 | // if free_symbol is true, because when that happens, root_bind will be deleted, and we will clean up the | ||||
1168 | // alias in that process. | ||||
1169 | if (bind->alias_ref && (!bind->sources || bind->sources->rnum == 0) && (!bind->destinations || bind->destinations->rnum == 0)) | ||||
1170 | { | ||||
1171 | _ccv_nnc_tensor_variable_graph_bind_free(graph, bind, 1); | ||||
1172 | ccv_nnc_tensor_symbol_free(graph->tape, tensor_variable->symbol); | ||||
1173 | } else { | ||||
1174 | bind->index = CCV_NNC_TENSOR_NO_VARIABLE_BUT_USED; // This tensor variable will be freed, but this symbol extra will continue exists. | ||||
1175 | bind->destructor_hook.func = tensor_variable->destructor_hook.func; // Transfer the destructor callback. | ||||
1176 | bind->destructor_hook.context = tensor_variable->destructor_hook.context; // Transfer the destructor callback context. | ||||
1177 | bind->tensor_view = tensor_variable->tensor_view; // Transfer the ownership to the bind. | ||||
1178 | tensor_variable->tensor_view = 0; | ||||
1179 | } | ||||
1180 | } | ||||
1181 | } | ||||
1182 | _ccv_nnc_tensor_variable_free(graph, tensor_variable, 1); | ||||
1183 | } | ||||
1184 | |||||
1185 | void ccv_nnc_tensor_variable_detach(ccv_nnc_dynamic_graph_t* const graph, const ccv_nnc_tensor_variable_t tensor_variable) | ||||
1186 | { | ||||
1187 | // This cannot be an alias. | ||||
1188 | assert(!tensor_variable->alias_index_ref)((void) sizeof ((!tensor_variable->alias_index_ref) ? 1 : 0 ), __extension__ ({ if (!tensor_variable->alias_index_ref) ; else __assert_fail ("!tensor_variable->alias_index_ref" , "ccv_nnc_dynamic_graph.c", 1188, __extension__ __PRETTY_FUNCTION__ ); })); | ||||
1189 | // If no computation done yet, mark this as constant. | ||||
1190 | if (tensor_variable->symbol.d == CCV_NNC_NO_TENSOR_SYMBOL) | ||||
1191 | { | ||||
1192 | tensor_variable->type = CCV_NNC_TENSOR_CONSTANT; | ||||
1193 | return; | ||||
1194 | } | ||||
1195 | // Otherwise, we need to do some book keeping updates to make sure it doesn't participate gradient computation any more. | ||||
1196 | ccv_nnc_tensor_variable_graph_bind_t* const bind = (ccv_nnc_tensor_variable_graph_bind_t*)ccv_array_get(graph->binds, tensor_variable->symbol.d)((void*)(((char*)((graph->binds)->data)) + (size_t)(graph ->binds)->rsize * (size_t)(tensor_variable->symbol.d ))); | ||||
1197 | // Because tensor variable cannot be alias, its bind cannot have alias pointer. | ||||
1198 | assert(!bind->alias_ref)((void) sizeof ((!bind->alias_ref) ? 1 : 0), __extension__ ({ if (!bind->alias_ref) ; else __assert_fail ("!bind->alias_ref" , "ccv_nnc_dynamic_graph.c", 1198, __extension__ __PRETTY_FUNCTION__ ); })); | ||||
1199 | // Go through to break ties between sources and destinations. | ||||
1200 | int i, j; | ||||
1201 | if (bind->sources && bind->destinations) | ||||
1202 | { | ||||
1203 | for (i = 0; i < bind->sources->rnum; i++) | ||||
1204 | { | ||||
1205 | const int s = *(int*)ccv_array_get(bind->sources, i)((void*)(((char*)((bind->sources)->data)) + (size_t)(bind ->sources)->rsize * (size_t)(i))); | ||||
1206 | const int* outputs; int output_size; | ||||
1207 | const ccv_nnc_graph_exec_symbol_t s_symbol = { | ||||
1208 | .d = s, | ||||
1209 | .graph = graph->tape | ||||
1210 | }; | ||||
1211 | ccv_nnc_graph_exec_symbol_io(graph->tape, s_symbol, 0, 0, &outputs, &output_size); | ||||
1212 | for (j = 0; j < bind->destinations->rnum; j++) | ||||
1213 | { | ||||
1214 | const int d = *(int*)ccv_array_get(bind->destinations, j)((void*)(((char*)((bind->destinations)->data)) + (size_t )(bind->destinations)->rsize * (size_t)(j))); | ||||
1215 | const ccv_nnc_graph_exec_symbol_t d_symbol = { | ||||
1216 | .d = d, | ||||
1217 | .graph = graph->tape | ||||
1218 | }; | ||||
1219 | const int* inputs; int input_size; | ||||
1220 | ccv_nnc_graph_exec_symbol_io(graph->tape, d_symbol, &inputs, &input_size, 0, 0); | ||||
1221 | int x, y; | ||||
1222 | int flag = 0; // Whether we find a symbol that connects source and destination but not the current one we detach. If found, we cannot break the tie between s_symbol and d_symbol. | ||||
1223 | for (x = 0; !flag && x < output_size; x++) | ||||
1224 | { | ||||
1225 | ccv_nnc_tensor_symbol_t x_symbol = ccv_nnc_tensor_symbol_alias_to(graph->tape, (ccv_nnc_tensor_symbol_t){ | ||||
1226 | .d = outputs[x], | ||||
1227 | .graph = graph->tape | ||||
1228 | }); | ||||
1229 | if (x_symbol.d == CCV_NNC_NO_TENSOR_SYMBOL) | ||||
1230 | { | ||||
1231 | x_symbol.d = outputs[x]; | ||||
1232 | x_symbol.graph = graph->tape; | ||||
1233 | } | ||||
1234 | if (x_symbol.d == tensor_variable->symbol.d || x_symbol.d == CCV_NNC_NO_TENSOR_SYMBOL) | ||||
1235 | continue; | ||||
1236 | for (y = 0; !flag && y < input_size; y++) | ||||
1237 | { | ||||
1238 | ccv_nnc_tensor_symbol_t y_symbol = ccv_nnc_tensor_symbol_alias_to(graph->tape, (ccv_nnc_tensor_symbol_t){ | ||||
1239 | .d = inputs[y], | ||||
1240 | .graph = graph->tape | ||||
1241 | }); | ||||
1242 | if (y_symbol.d == CCV_NNC_NO_TENSOR_SYMBOL) | ||||
1243 | { | ||||
1244 | y_symbol.d = inputs[y]; | ||||
1245 | y_symbol.graph = graph->tape; | ||||
1246 | } | ||||
1247 | if (y_symbol.d == tensor_variable->symbol.d || y_symbol.d == CCV_NNC_NO_TENSOR_SYMBOL) | ||||
1248 | continue; | ||||
1249 | flag = (x_symbol.d == y_symbol.d); | ||||
1250 | } | ||||
1251 | } | ||||
1252 | if (!flag) | ||||
1253 | ccv_nnc_graph_exec_symbol_disjoin(graph->tape, s_symbol, d_symbol); | ||||
1254 | } | ||||
1255 | } | ||||
1256 | } | ||||
1257 | const int sources_and_is_only_output = (bind->sources && bind->sources->rnum > 0) && _ccv_nnc_tensor_variable_is_only_output(graph, bind, tensor_variable->symbol.d); | ||||
1258 | if (!bind->sources || bind->sources->rnum == 0 || sources_and_is_only_output) | ||||
1259 | { | ||||
1260 | int ws_start = -1; | ||||
1261 | _ccv_nnc_tensor_bind_trace_forward_to_free(graph, tensor_variable, bind, bind, &ws_start, 1); | ||||
1262 | // Because we are detaching from the graph, there is no need to forward trace to see if it is not used and | ||||
1263 | // then to remove the source execs. We can remove them right now, breaking the graph in two. That is why | ||||
1264 | // we called trace backward to free regardless the outcome of the forward to free. | ||||
1265 | if (ws_start == -1) | ||||
1266 | { | ||||
1267 | if (!graph->ws) | ||||
1268 | graph->ws = ccv_array_new(sizeof(int), bind->destinations ? bind->destinations->rnum : 0, 0); | ||||
1269 | ccv_array_t* const ws = graph->ws; | ||||
1270 | ccv_array_clear(ws); | ||||
1271 | if (bind->sources) | ||||
1272 | for (i = 0; i < bind->sources->rnum; i++) | ||||
1273 | ccv_array_add_unique_int(ws, *(int*)ccv_array_get(bind->sources, i)((void*)(((char*)((bind->sources)->data)) + (size_t)(bind ->sources)->rsize * (size_t)(i)))); | ||||
1274 | ws_start = 0; | ||||
1275 | } | ||||
1276 | _ccv_nnc_tensor_bind_trace_backward_to_free(graph, graph->ws, ws_start); | ||||
1277 | } | ||||
1278 | // If now bind has no relevant sources or destinations, we can safely free the underlying tensor symbol. | ||||
1279 | if ((!bind->sources || bind->sources->rnum == 0) && (!bind->destinations || bind->destinations->rnum == 0)) | ||||
1280 | { | ||||
1281 | _ccv_nnc_tensor_variable_graph_bind_free(graph, bind, 1); | ||||
1282 | ccv_nnc_tensor_symbol_free(graph->tape, tensor_variable->symbol); | ||||
1283 | tensor_variable->type = CCV_NNC_TENSOR_CONSTANT; | ||||
1284 | tensor_variable->symbol = NO_TENSOR_SYMBOL(const ccv_nnc_tensor_symbol_t){.d = CCV_NNC_NO_TENSOR_SYMBOL }; | ||||
1285 | return; | ||||
1286 | } | ||||
1287 | // Mark both as constant, such that even if it cannot be freed now, it can be freed as soon as possible later. | ||||
1288 | bind->type = CCV_NNC_TENSOR_CONSTANT; | ||||
1289 | tensor_variable->type = CCV_NNC_TENSOR_CONSTANT; | ||||
1290 | } | ||||
1291 | |||||
1292 | void ccv_nnc_dynamic_graph_has_effect_to_tensor_variables(const ccv_nnc_dynamic_graph_t* const graph, const ccv_nnc_tensor_variable_t* const source_variables, const int source_variable_size, const ccv_nnc_tensor_variable_t* const destination_variables, const int destination_variable_size, uint64_t* const bitmask) | ||||
1293 | { | ||||
1294 | int i, j; | ||||
1295 | ccv_array_t* const sources_destinations = ccv_array_new(sizeof(ccv_nnc_graph_exec_symbol_t), source_variable_size + destination_variable_size, 0); | ||||
1296 | for (i = 0; i < source_variable_size; i++) | ||||
1297 | { | ||||
1298 | if (source_variables[i]->symbol.d < 0) | ||||
1299 | continue; | ||||
1300 | ccv_nnc_tensor_variable_graph_bind_t* const bind = (ccv_nnc_tensor_variable_graph_bind_t*)ccv_array_get(graph->binds, source_variables[i]->symbol.d)((void*)(((char*)((graph->binds)->data)) + (size_t)(graph ->binds)->rsize * (size_t)(source_variables[i]->symbol .d))); | ||||
1301 | if (bind->destinations && bind->destinations->rnum > 0) | ||||
1302 | for (j = 0; j < bind->destinations->rnum; j++) | ||||
1303 | { | ||||
1304 | // It is ok to have duplicate symbols. | ||||
1305 | const int d = *(int*)ccv_array_get(bind->destinations, j)((void*)(((char*)((bind->destinations)->data)) + (size_t )(bind->destinations)->rsize * (size_t)(j))); | ||||
1306 | ccv_nnc_graph_exec_symbol_t symbol = { | ||||
1307 | .d = d, | ||||
1308 | .graph = graph->tape | ||||
1309 | }; | ||||
1310 | ccv_array_push(sources_destinations, &symbol); | ||||
1311 | } | ||||
1312 | } | ||||
1313 | const int source_size = sources_destinations->rnum; | ||||
1314 | for (i = 0; i < destination_variable_size; i++) | ||||
1315 | { | ||||
1316 | if (destination_variables[i]->symbol.d < 0) | ||||
1317 | continue; | ||||
1318 | ccv_nnc_tensor_variable_graph_bind_t* const bind = (ccv_nnc_tensor_variable_graph_bind_t*)ccv_array_get(graph->binds, destination_variables[i]->symbol.d)((void*)(((char*)((graph->binds)->data)) + (size_t)(graph ->binds)->rsize * (size_t)(destination_variables[i]-> symbol.d))); | ||||
1319 | if (bind->sources && bind->sources->rnum > 0) | ||||
1320 | for (j = 0; j < bind->sources->rnum; j++) | ||||
1321 | { | ||||
1322 | // It is ok to have duplicate symbols. | ||||
1323 | const int d = *(int*)ccv_array_get(bind->sources, j)((void*)(((char*)((bind->sources)->data)) + (size_t)(bind ->sources)->rsize * (size_t)(j))); | ||||
1324 | ccv_nnc_graph_exec_symbol_t symbol = { | ||||
1325 | .d = d, | ||||
1326 | .graph = graph->tape | ||||
1327 | }; | ||||
1328 | ccv_array_push(sources_destinations, &symbol); | ||||
1329 | } | ||||
1330 | } | ||||
1331 | const int destination_size = sources_destinations->rnum - source_size; | ||||
1332 | if (source_size == 0 || destination_size == 0) | ||||
1333 | { | ||||
1334 | ccv_array_free(sources_destinations); | ||||
1335 | return; | ||||
1336 | } | ||||
1337 | const int bitmask_size = ((source_size + 63) >> 6); | ||||
1338 | assert(bitmask_size < 256)((void) sizeof ((bitmask_size < 256) ? 1 : 0), __extension__ ({ if (bitmask_size < 256) ; else __assert_fail ("bitmask_size < 256" , "ccv_nnc_dynamic_graph.c", 1338, __extension__ __PRETTY_FUNCTION__ ); })); | ||||
1339 | uint64_t exec_bitmask[bitmask_size]; | ||||
1340 | ccv_nnc_symbolic_graph_sources_to_destinations(graph->tape, (ccv_nnc_graph_exec_symbol_t*)ccv_array_get(sources_destinations, 0)((void*)(((char*)((sources_destinations)->data)) + (size_t )(sources_destinations)->rsize * (size_t)(0))), source_size, (ccv_nnc_graph_exec_symbol_t*)ccv_array_get(sources_destinations, source_size)((void*)(((char*)((sources_destinations)->data)) + (size_t )(sources_destinations)->rsize * (size_t)(source_size))), destination_size, exec_bitmask); | ||||
1341 | int k = 0; | ||||
1342 | for (i = 0; i < source_variable_size; i++) | ||||
1343 | { | ||||
1344 | if (source_variables[i]->symbol.d < 0) | ||||
1345 | { | ||||
1346 | bitmask[i >> 6] &= ~((uint64_t)1 << (i & 63)); | ||||
1347 | continue; | ||||
1348 | } | ||||
1349 | ccv_nnc_tensor_variable_graph_bind_t* const bind = (ccv_nnc_tensor_variable_graph_bind_t*)ccv_array_get(graph->binds, source_variables[i]->symbol.d)((void*)(((char*)((graph->binds)->data)) + (size_t)(graph ->binds)->rsize * (size_t)(source_variables[i]->symbol .d))); | ||||
1350 | int flag = 0; | ||||
1351 | if (bind->destinations && bind->destinations->rnum > 0) | ||||
1352 | { | ||||
1353 | assert(k <= source_size - bind->destinations->rnum)((void) sizeof ((k <= source_size - bind->destinations-> rnum) ? 1 : 0), __extension__ ({ if (k <= source_size - bind ->destinations->rnum) ; else __assert_fail ("k <= source_size - bind->destinations->rnum" , "ccv_nnc_dynamic_graph.c", 1353, __extension__ __PRETTY_FUNCTION__ ); })); | ||||
1354 | for (j = 0; !flag && j < bind->destinations->rnum; j++) | ||||
1355 | flag = (((uint64_t)1 << ((k + j) & 63)) & exec_bitmask[(k + j) >> 6]); | ||||
1356 | k += bind->destinations->rnum; | ||||
1357 | } | ||||
1358 | if (flag) | ||||
1359 | bitmask[i >> 6] |= ((uint64_t)1 << (i & 63)); | ||||
1360 | else | ||||
1361 | bitmask[i >> 6] &= ~((uint64_t)1 << (i & 63)); | ||||
1362 | } | ||||
1363 | ccv_array_free(sources_destinations); | ||||
1364 | } | ||||
1365 | |||||
1366 | int ccv_nnc_dynamic_graph_bookkeeping_count(const ccv_nnc_dynamic_graph_t* const graph, const int type) | ||||
1367 | { | ||||
1368 | return ccv_nnc_symbolic_graph_active_symbol_count(graph->tape, type); | ||||
1369 | } | ||||
1370 | |||||
1371 | void ccv_nnc_dynamic_graph_dot(const ccv_nnc_dynamic_graph_t* const graph, const int flags, FILE* out) | ||||
1372 | { | ||||
1373 | ccv_nnc_symbolic_graph_dot(graph->tape, flags, out); | ||||
1374 | } | ||||
1375 | |||||
1376 | void ccv_nnc_dynamic_graph_format(const ccv_nnc_dynamic_graph_t* const graph, const ccv_nnc_symbolic_graph_format_f format_fn, void* const context) | ||||
1377 | { | ||||
1378 | ccv_nnc_symbolic_graph_format(graph->tape, 0, 0, 0, 0, format_fn, context); | ||||
1379 | } |