Bug Summary

File:nnc/ccv_nnc_cmd.c
Warning:line 596, column 27
Potential memory leak

Annotated Source Code

Press '?' to see keyboard shortcuts

clang -cc1 -cc1 -triple x86_64-unknown-linux-gnu -analyze -disable-free -clear-ast-before-backend -disable-llvm-verifier -discard-value-names -main-file-name ccv_nnc_cmd.c -analyzer-checker=core -analyzer-checker=apiModeling -analyzer-checker=unix -analyzer-checker=deadcode -analyzer-checker=security.insecureAPI.UncheckedReturn -analyzer-checker=security.insecureAPI.getpw -analyzer-checker=security.insecureAPI.gets -analyzer-checker=security.insecureAPI.mktemp -analyzer-checker=security.insecureAPI.mkstemp -analyzer-checker=security.insecureAPI.vfork -analyzer-checker=nullability.NullPassedToNonnull -analyzer-checker=nullability.NullReturnedFromNonnull -analyzer-output plist -w -setup-static-analyzer -mrelocation-model pic -pic-level 2 -pic-is-pie -mframe-pointer=none -fmath-errno -ffp-contract=on -fno-rounding-math -mconstructor-aliases -funwind-tables=2 -target-cpu x86-64 -target-feature +sse2 -tune-cpu generic -debugger-tuning=gdb -fdebug-compilation-dir=/home/liu/actions-runner/_work/ccv/ccv/lib/nnc -fcoverage-compilation-dir=/home/liu/actions-runner/_work/ccv/ccv/lib/nnc -resource-dir /usr/local/lib/clang/19 -I ../ -I /usr/local/cuda/include -D HAVE_CBLAS -D HAVE_LIBPNG -D HAVE_LIBJPEG -D HAVE_FFTW3 -D HAVE_PTHREAD -D HAVE_LIBLINEAR -D HAVE_TESSERACT -D HAVE_AVCODEC -D HAVE_AVFORMAT -D HAVE_AVUTIL -D HAVE_SWSCALE -D HAVE_SSE2 -D HAVE_GSL -D HAVE_CUDA -D HAVE_CUDNN -D HAVE_NCCL -D USE_SYSTEM_CUB -I /usr/local/include -internal-isystem /usr/local/lib/clang/19/include -internal-isystem /usr/local/include -internal-isystem /usr/lib/gcc/x86_64-linux-gnu/12/../../../../x86_64-linux-gnu/include -internal-externc-isystem /usr/include/x86_64-linux-gnu -internal-externc-isystem /include -internal-externc-isystem /usr/include -O3 -ferror-limit 19 -fgnuc-version=4.2.1 -fskip-odr-check-in-gmf -vectorize-loops -vectorize-slp -analyzer-output=html -faddrsig -D__GCC_HAVE_DWARF2_CFI_ASM=1 -o /home/liu/actions-runner/_work/ccv/ccv/_analyze/2026-09-23-222944-215595-1 -x c ccv_nnc_cmd.c
1#include "ccv_nnc.h"
2#include "ccv_nnc_internal.h"
3#include "3rdparty/khash/khash.h"
4#include "ccv_nnc_easy.h"
5#ifdef HAVE_CUDA1
6#include "gpu/ccv_nnc_compat.h"
7#elif defined(HAVE_MPS)
8#include "mps/ccv_nnc_mps.h"
9#endif
10#include <time.h>
11#include <sys/time.h>
12
13typedef struct {
14 const uint32_t cmd;
15 const char* name;
16 ccv_nnc_cmd_registry_t registry;
17 ccv_nnc_cmd_backend_registry_t backends[CCV_NNC_BACKEND_COUNT];
18} ccv_nnc_cmd_init_t;
19
20typedef struct {
21 const uint32_t backend;
22 const char* name;
23} ccv_nnc_cmd_backend_init_t;
24
25// The generated code configures command and its mapping.
26#include "cmd/ccv_nnc_cmd.inc"
27
28void ccv_nnc_init(void)
29{
30 _ccv_nnc_cmd_init();
31}
32
33int ccv_nnc_fork(void)
34{
35#ifdef HAVE_MPS
36 return ccv_nnc_mps_fork();
37#else
38 return 0;
39#endif
40}
41
42void ccv_nnc_join(void)
43{
44#ifdef HAVE_MPS
45 ccv_nnc_mps_join();
46#endif
47}
48
49static uint64_t _ccv_nnc_flags = 0;
50
51uint64_t ccv_nnc_flags(void)
52{
53 return _ccv_nnc_flags;
54}
55
56void ccv_nnc_enable_flag(uint64_t flag)
57{
58 _ccv_nnc_flags |= flag;
59}
60
61void ccv_nnc_disable_flag(uint64_t flag)
62{
63 _ccv_nnc_flags &= ~flag;
64}
65
66const char* ccv_nnc_cmd_name(const uint32_t cmd)
67{
68 switch (cmd)
69 {
70 case CCV_NNC_NOOP:
71 return "CCV_NNC_NOOP";
72 case CCV_NNC_CUSTOM_FORWARD:
73 return "CCV_NNC_CUSTOM_FORWARD";
74 case CCV_NNC_CUSTOM_BACKWARD:
75 return "CCV_NNC_CUSTOM_BACKWARD";
76 case CCV_NNC_GRAPH_FORWARD:
77 return "CCV_NNC_GRAPH_FORWARD";
78 case CCV_NNC_GRAPH_BACKWARD:
79 return "CCV_NNC_GRAPH_BACKWARD";
80 }
81 const int idx = _ccv_nnc_cmd_ph(cmd);
82 assert(idx >= 0)((void) sizeof ((idx >= 0) ? 1 : 0), __extension__ ({ if (
idx >= 0) ; else __assert_fail ("idx >= 0", "ccv_nnc_cmd.c"
, 82, __extension__ __PRETTY_FUNCTION__); }));
83 assert(idx < sizeof(init_map) / sizeof(init_map[0]))((void) sizeof ((idx < sizeof(init_map) / sizeof(init_map[
0])) ? 1 : 0), __extension__ ({ if (idx < sizeof(init_map)
/ sizeof(init_map[0])) ; else __assert_fail ("idx < sizeof(init_map) / sizeof(init_map[0])"
, "ccv_nnc_cmd.c", 83, __extension__ __PRETTY_FUNCTION__); })
);
84 return init_map[idx].name;
85}
86
87const char* ccv_nnc_cmd_backend_name(const uint32_t backend)
88{
89 if (backend == CCV_NNC_NO_BACKEND)
90 return "CCV_NNC_NO_BACKEND";
91 const int idx = _ccv_nnc_cmd_backend_ph(backend);
92 assert(idx >= 0)((void) sizeof ((idx >= 0) ? 1 : 0), __extension__ ({ if (
idx >= 0) ; else __assert_fail ("idx >= 0", "ccv_nnc_cmd.c"
, 92, __extension__ __PRETTY_FUNCTION__); }));
93 assert(idx < CCV_NNC_BACKEND_COUNT)((void) sizeof ((idx < CCV_NNC_BACKEND_COUNT) ? 1 : 0), __extension__
({ if (idx < CCV_NNC_BACKEND_COUNT) ; else __assert_fail (
"idx < CCV_NNC_BACKEND_COUNT", "ccv_nnc_cmd.c", 93, __extension__
__PRETTY_FUNCTION__); }));
94 return backend_init_map[idx].name;
95}
96
97const ccv_nnc_cmd_param_t ccv_nnc_cmd_auto = {};
98
99int ccv_nnc_is_cmd_auto(const ccv_nnc_cmd_param_t params)
100{
101 return (memcmp(&params, &ccv_nnc_cmd_auto, sizeof(ccv_nnc_cmd_param_t)) == 0);
102}
103
104int ccv_nnc_cmd_is_forward(const ccv_nnc_cmd_t cmd)
105{
106 switch (cmd.cmd)
107 {
108 case CCV_NNC_NOOP:
109 return 0;
110 case CCV_NNC_CUSTOM_FORWARD:
111 case CCV_NNC_CUSTOM_BACKWARD:
112 case CCV_NNC_GRAPH_FORWARD:
113 case CCV_NNC_GRAPH_BACKWARD:
114 default:
115 return !(cmd.cmd & 0x1); // If it is even, it is forward
116 }
117}
118
119int ccv_nnc_cmd_is_backward(const ccv_nnc_cmd_t cmd)
120{
121 switch (cmd.cmd)
122 {
123 case CCV_NNC_NOOP:
124 return 0;
125 case CCV_NNC_CUSTOM_FORWARD:
126 case CCV_NNC_CUSTOM_BACKWARD:
127 case CCV_NNC_GRAPH_FORWARD:
128 case CCV_NNC_GRAPH_BACKWARD:
129 default:
130 return !!(cmd.cmd & 0x1); // If it is odd, it is backward
131 }
132}
133
134int ccv_nnc_cmd_ok(const uint32_t cmd, const uint32_t backend)
135{
136 // If it is a custom command, a no op, or a graph op, there is no backend to check.
137 if (cmd == CCV_NNC_NOOP ||
138 cmd == CCV_NNC_GRAPH_FORWARD || cmd == CCV_NNC_GRAPH_BACKWARD ||
139 cmd == CCV_NNC_CUSTOM_FORWARD || cmd == CCV_NNC_CUSTOM_BACKWARD)
140 return 1;
141 const int cmd_idx = _ccv_nnc_cmd_ph(cmd);
142 const int backend_idx = _ccv_nnc_cmd_backend_ph(backend);
143 assert(cmd_idx >= 0 && cmd_idx < sizeof(init_map) / sizeof(init_map[0]))((void) sizeof ((cmd_idx >= 0 && cmd_idx < sizeof
(init_map) / sizeof(init_map[0])) ? 1 : 0), __extension__ ({ if
(cmd_idx >= 0 && cmd_idx < sizeof(init_map) / sizeof
(init_map[0])) ; else __assert_fail ("cmd_idx >= 0 && cmd_idx < sizeof(init_map) / sizeof(init_map[0])"
, "ccv_nnc_cmd.c", 143, __extension__ __PRETTY_FUNCTION__); }
));
144 assert(backend_idx >= 0 && backend_idx < CCV_NNC_BACKEND_COUNT)((void) sizeof ((backend_idx >= 0 && backend_idx <
CCV_NNC_BACKEND_COUNT) ? 1 : 0), __extension__ ({ if (backend_idx
>= 0 && backend_idx < CCV_NNC_BACKEND_COUNT) ;
else __assert_fail ("backend_idx >= 0 && backend_idx < CCV_NNC_BACKEND_COUNT"
, "ccv_nnc_cmd.c", 144, __extension__ __PRETTY_FUNCTION__); }
));
145 const ccv_nnc_cmd_backend_registry_t api_registry = init_map[cmd_idx].backends[backend_idx];
146 // Check if the execution function exists or not.
147 return !!api_registry.exec;
148}
149
150ccv_nnc_cmd_t ccv_nnc_cmd(const uint32_t _cmd, ccv_nnc_cmd_vtab_t* const isa, const ccv_nnc_cmd_param_t params, const int flags)
151{
152 ccv_nnc_cmd_t cmd;
153 cmd.info = params;
154 cmd.backend = CCV_NNC_NO_BACKEND;
155 assert((_cmd == CCV_NNC_CUSTOM_FORWARD && isa) || (_cmd != CCV_NNC_CUSTOM_FORWARD && !isa))((void) sizeof (((_cmd == CCV_NNC_CUSTOM_FORWARD && isa
) || (_cmd != CCV_NNC_CUSTOM_FORWARD && !isa)) ? 1 : 0
), __extension__ ({ if ((_cmd == CCV_NNC_CUSTOM_FORWARD &&
isa) || (_cmd != CCV_NNC_CUSTOM_FORWARD && !isa)) ; else
__assert_fail ("(_cmd == CCV_NNC_CUSTOM_FORWARD && isa) || (_cmd != CCV_NNC_CUSTOM_FORWARD && !isa)"
, "ccv_nnc_cmd.c", 155, __extension__ __PRETTY_FUNCTION__); }
));
156 cmd.cmd = _cmd;
157 cmd.algorithm = -1; // This is default.
158 cmd.isa = isa;
159 cmd.data = 0;
160 return cmd;
161}
162
163const ccv_nnc_hint_t ccv_nnc_no_hint = {};
164
165int ccv_nnc_is_no_hint(const ccv_nnc_hint_t hint)
166{
167 return (memcmp(&hint, &ccv_nnc_no_hint, sizeof(ccv_nnc_hint_t)) == 0);
168}
169
170int ccv_nnc_hint_verify(const ccv_nnc_hint_t hint, const ccv_nnc_cmd_param_t cmd, const ccv_nnc_tensor_param_t a, const ccv_nnc_tensor_param_t b)
171{
172 int i;
173 assert(a.format == b.format)((void) sizeof ((a.format == b.format) ? 1 : 0), __extension__
({ if (a.format == b.format) ; else __assert_fail ("a.format == b.format"
, "ccv_nnc_cmd.c", 173, __extension__ __PRETTY_FUNCTION__); }
));
174 const int nd = ccv_nnc_tensor_nd(a.dim);
175 const int size_nd = ccv_max(2, ccv_nnc_tensor_nd(cmd.size.dim) - 1)({ typeof (2) _a = (2); typeof (ccv_nnc_tensor_nd(cmd.size.dim
) - 1) _b = (ccv_nnc_tensor_nd(cmd.size.dim) - 1); (_a > _b
) ? _a : _b; });
176 assert(size_nd == 2 || size_nd == 3)((void) sizeof ((size_nd == 2 || size_nd == 3) ? 1 : 0), __extension__
({ if (size_nd == 2 || size_nd == 3) ; else __assert_fail ("size_nd == 2 || size_nd == 3"
, "ccv_nnc_cmd.c", 176, __extension__ __PRETTY_FUNCTION__); }
)); // Support 3D convolution.
177 assert(nd == size_nd + 1 || nd == size_nd + 2)((void) sizeof ((nd == size_nd + 1 || nd == size_nd + 2) ? 1 :
0), __extension__ ({ if (nd == size_nd + 1 || nd == size_nd +
2) ; else __assert_fail ("nd == size_nd + 1 || nd == size_nd + 2"
, "ccv_nnc_cmd.c", 177, __extension__ __PRETTY_FUNCTION__); }
));
178 int hw;
179 if ((a.format == CCV_TENSOR_FORMAT_CHWN) ||
180 (a.format == CCV_TENSOR_FORMAT_NHWC && nd == size_nd + 1))
181 hw = 0;
182 else if ((a.format == CCV_TENSOR_FORMAT_NHWC && nd == size_nd + 2) ||
183 (a.format == CCV_TENSOR_FORMAT_NCHW && nd == size_nd + 1))
184 hw = 1;
185 else if (a.format == CCV_TENSOR_FORMAT_NCHW && nd == size_nd + 2)
186 hw = 2;
187 else
188 assert(0 && "unknown format")((void) sizeof ((0 && "unknown format") ? 1 : 0), __extension__
({ if (0 && "unknown format") ; else __assert_fail (
"0 && \"unknown format\"", "ccv_nnc_cmd.c", 188, __extension__
__PRETTY_FUNCTION__); }));
189 for (i = 0; i < size_nd; i++)
190 {
191 if ((hint.border.begin[i] + hint.border.end[i] + a.dim[i + hw] - cmd.size.dim[i]) % hint.stride.dim[i] != 0)
192 return -1;
193 int expected = (hint.border.begin[i] + hint.border.end[i] + a.dim[i + hw] - cmd.size.dim[i]) / hint.stride.dim[i] + 1;
194 if (expected != b.dim[i + hw])
195 return -1;
196 }
197 return 0;
198}
199
200ccv_nnc_hint_t ccv_nnc_hint_auto(const ccv_nnc_cmd_param_t cmd, const ccv_nnc_tensor_param_t a, const ccv_nnc_tensor_param_t b)
201{
202 int i;
203 if (a.format != b.format)
204 return ccv_nnc_no_hint;
205 assert(a.format == b.format)((void) sizeof ((a.format == b.format) ? 1 : 0), __extension__
({ if (a.format == b.format) ; else __assert_fail ("a.format == b.format"
, "ccv_nnc_cmd.c", 205, __extension__ __PRETTY_FUNCTION__); }
));
206 const int a_nd = ccv_nnc_tensor_nd(a.dim);
207 const int b_nd = ccv_nnc_tensor_nd(b.dim);
208 const int size_nd = ccv_max(2, ccv_nnc_tensor_nd(cmd.size.dim) - 1)({ typeof (2) _a = (2); typeof (ccv_nnc_tensor_nd(cmd.size.dim
) - 1) _b = (ccv_nnc_tensor_nd(cmd.size.dim) - 1); (_a > _b
) ? _a : _b; });
209 assert(size_nd == 2 || size_nd == 3)((void) sizeof ((size_nd == 2 || size_nd == 3) ? 1 : 0), __extension__
({ if (size_nd == 2 || size_nd == 3) ; else __assert_fail ("size_nd == 2 || size_nd == 3"
, "ccv_nnc_cmd.c", 209, __extension__ __PRETTY_FUNCTION__); }
)); // Support 3D convolution.
210 // Is not auto hint deducible dimensions.
211 if (a_nd != b_nd || (a_nd != size_nd + 1 && a_nd != size_nd + 2))
212 return ccv_nnc_no_hint;
213 int hw;
214 if ((a.format == CCV_TENSOR_FORMAT_CHWN) ||
215 (a.format == CCV_TENSOR_FORMAT_NHWC && a_nd == size_nd + 1))
216 hw = 0;
217 else if ((a.format == CCV_TENSOR_FORMAT_NHWC && a_nd == size_nd + 2) ||
218 (a.format == CCV_TENSOR_FORMAT_NCHW && a_nd == size_nd + 1))
219 hw = 1;
220 else if (a.format == CCV_TENSOR_FORMAT_NCHW && a_nd == size_nd + 2)
221 hw = 2;
222 else
223 assert(0 && "unknown format")((void) sizeof ((0 && "unknown format") ? 1 : 0), __extension__
({ if (0 && "unknown format") ; else __assert_fail (
"0 && \"unknown format\"", "ccv_nnc_cmd.c", 223, __extension__
__PRETTY_FUNCTION__); }));
224 ccv_nnc_hint_t hint_auto = {};
225 // 0-dim is reserved for channels
226 for (i = 0; i < size_nd; i++)
227 {
228 // Cannot have one of the dim is zero, we cannot auto the hint, return no hint.
229 assert(a.dim[i + hw] && b.dim[i + hw])((void) sizeof ((a.dim[i + hw] && b.dim[i + hw]) ? 1 :
0), __extension__ ({ if (a.dim[i + hw] && b.dim[i + hw
]) ; else __assert_fail ("a.dim[i + hw] && b.dim[i + hw]"
, "ccv_nnc_cmd.c", 229, __extension__ __PRETTY_FUNCTION__); }
));
230 // This is guessed by having a stride that will approximately match the scale.
231 int stride = (a.dim[i + hw] + b.dim[i + hw] / 2) / b.dim[i + hw];
232 hint_auto.stride.dim[i] = stride;
233 int border = (b.dim[i + hw] - 1) * stride - a.dim[i + hw] + cmd.size.dim[i];
234 hint_auto.border.begin[i] = (border + 1) / 2; // Always prefer to have more padding in the beginning, this matches CUDNN behavior.
235 hint_auto.border.end[i] = border - hint_auto.border.begin[i];
236 }
237 return hint_auto;
238}
239
240void ccv_nnc_hint_tensor_auto_forward_from_inputs(const ccv_nnc_cmd_param_t cmd, const ccv_nnc_tensor_param_t* const inputs, const int input_size, const ccv_nnc_hint_t hint, ccv_nnc_tensor_param_t* const outputs, const int output_size)
241{
242 int i;
243 assert(output_size <= input_size)((void) sizeof ((output_size <= input_size) ? 1 : 0), __extension__
({ if (output_size <= input_size) ; else __assert_fail ("output_size <= input_size"
, "ccv_nnc_cmd.c", 243, __extension__ __PRETTY_FUNCTION__); }
));
244 for (i = 0; i < output_size; i++)
245 outputs[i] = inputs[i];
246}
247
248void ccv_nnc_hint_tensor_auto_backward_from_gradient(const ccv_nnc_cmd_param_t cmd, const ccv_nnc_tensor_param_t* const inputs, const int input_size, const ccv_nnc_hint_t hint, ccv_nnc_tensor_param_t* const outputs, const int output_size)
249{
250 int i;
251 for (i = 0; i < output_size; i++)
252 outputs[i] = inputs[0];
253}
254
255void ccv_nnc_hint_tensor_auto_backward_from_inputs(const ccv_nnc_cmd_param_t cmd, const ccv_nnc_tensor_param_t* const inputs, const int input_size, const ccv_nnc_hint_t hint, ccv_nnc_tensor_param_t* const outputs, const int output_size)
256{
257 int i;
258 assert(output_size < input_size)((void) sizeof ((output_size < input_size) ? 1 : 0), __extension__
({ if (output_size < input_size) ; else __assert_fail ("output_size < input_size"
, "ccv_nnc_cmd.c", 258, __extension__ __PRETTY_FUNCTION__); }
));
259 for (i = 0; i < output_size; i++)
260 outputs[i] = inputs[i + 1];
261}
262
263void ccv_nnc_hint_tensor_auto_backward_from_gradient_and_inputs(const ccv_nnc_cmd_param_t cmd, const ccv_nnc_tensor_param_t* const inputs, const int input_size, const ccv_nnc_hint_t hint, ccv_nnc_tensor_param_t* const outputs, const int output_size)
264{
265 int i;
266 outputs[0] = inputs[0];
267 assert(output_size < input_size)((void) sizeof ((output_size < input_size) ? 1 : 0), __extension__
({ if (output_size < input_size) ; else __assert_fail ("output_size < input_size"
, "ccv_nnc_cmd.c", 267, __extension__ __PRETTY_FUNCTION__); }
));
268 for (i = 1; i < output_size; i++)
269 outputs[i] = inputs[i + 1];
270}
271
272void ccv_nnc_hint_tensor_auto(const ccv_nnc_cmd_t cmd, const ccv_nnc_tensor_param_t* const inputs, const int input_size, const ccv_nnc_hint_t hint, ccv_nnc_tensor_param_t* const outputs, const int output_size)
273{
274 // zero out the parameters
275 const ccv_nnc_tensor_param_t z = {};
276 int i;
277 for (i = 0; i < output_size; i++)
278 outputs[i] = z; // Reset the outputs.
279 // Cannot handle these situations.
280 if (cmd.cmd == CCV_NNC_NOOP || cmd.cmd == CCV_NNC_CUSTOM_BACKWARD || cmd.cmd == CCV_NNC_GRAPH_FORWARD || cmd.cmd == CCV_NNC_GRAPH_BACKWARD)
281 return;
282 if (cmd.cmd == CCV_NNC_CUSTOM_FORWARD)
283 {
284 if (cmd.isa->tensor_auto)
285 cmd.isa->tensor_auto(cmd, inputs, input_size, hint, outputs, output_size);
286 return;
287 }
288 const int cmd_idx = _ccv_nnc_cmd_ph(cmd.cmd);
289 const ccv_nnc_cmd_registry_t registry = init_map[cmd_idx].registry;
290 if (registry.tensor_auto)
291 registry.tensor_auto(cmd.info, inputs, input_size, hint, outputs, output_size);
292 else if (ccv_nnc_cmd_is_forward(cmd)) // For forward, the default auto is forward_from_inputs
293 ccv_nnc_hint_tensor_auto_forward_from_inputs(cmd.info, inputs, input_size, hint, outputs, output_size);
294 else // For backward, the default auto is backward_from_inputs
295 ccv_nnc_hint_tensor_auto_backward_from_inputs(cmd.info, inputs, input_size, hint, outputs, output_size);
296}
297
298int ccv_nnc_cmd_allow_inplace(const ccv_nnc_cmd_t cmd, const int input_idx, const int input_size, const int output_idx, const int output_size)
299{
300 if (cmd.cmd == CCV_NNC_NOOP || cmd.cmd == CCV_NNC_CUSTOM_FORWARD || cmd.cmd == CCV_NNC_CUSTOM_BACKWARD || cmd.cmd == CCV_NNC_GRAPH_FORWARD || cmd.cmd == CCV_NNC_GRAPH_BACKWARD)
301 return 0;
302 const int cmd_idx = _ccv_nnc_cmd_ph(cmd.cmd);
303 const ccv_nnc_cmd_registry_t registry = init_map[cmd_idx].registry;
304 if (registry.allow_inplace)
305 return registry.allow_inplace(cmd.info, input_idx, input_size, output_idx, output_size);
306 return 0;
307}
308
309int ccv_nnc_cmd_enforce_inplace(const ccv_nnc_cmd_t cmd, const int input_idx, const int input_size, const int output_idx, const int output_size)
310{
311 if (cmd.cmd == CCV_NNC_NOOP || cmd.cmd == CCV_NNC_CUSTOM_FORWARD || cmd.cmd == CCV_NNC_CUSTOM_BACKWARD || cmd.cmd == CCV_NNC_GRAPH_FORWARD || cmd.cmd == CCV_NNC_GRAPH_BACKWARD)
312 return 0;
313 const int cmd_idx = _ccv_nnc_cmd_ph(cmd.cmd);
314 const ccv_nnc_cmd_registry_t registry = init_map[cmd_idx].registry;
315 if (registry.enforce_inplace)
316 return registry.enforce_inplace(cmd.info, input_idx, input_size, output_idx, output_size);
317 return 0;
318}
319
320// This returns absolute time.
321uint64_t ccv_nnc_cmd_mono_time(void)
322{
323 struct timespec ts;
324 clock_gettime(CLOCK_MONOTONIC1, &ts);
325 return ts.tv_sec * 1000000000ULL + ts.tv_nsec;
326}
327
328uint32_t ccv_nnc_cmd_find_backend(const ccv_nnc_cmd_t cmd, const int tensor_memory, const int tensor_formats, const int tensor_datatypes)
329{
330 if (cmd.cmd == CCV_NNC_NOOP ||
331 cmd.cmd == CCV_NNC_GRAPH_FORWARD || cmd.cmd == CCV_NNC_GRAPH_BACKWARD ||
332 cmd.cmd == CCV_NNC_CUSTOM_FORWARD || cmd.cmd == CCV_NNC_CUSTOM_BACKWARD)
333 return cmd.backend;
334 const int cmd_idx = _ccv_nnc_cmd_ph(cmd.cmd);
335 assert(cmd_idx >= 0 && cmd_idx < sizeof(init_map) / sizeof(init_map[0]))((void) sizeof ((cmd_idx >= 0 && cmd_idx < sizeof
(init_map) / sizeof(init_map[0])) ? 1 : 0), __extension__ ({ if
(cmd_idx >= 0 && cmd_idx < sizeof(init_map) / sizeof
(init_map[0])) ; else __assert_fail ("cmd_idx >= 0 && cmd_idx < sizeof(init_map) / sizeof(init_map[0])"
, "ccv_nnc_cmd.c", 335, __extension__ __PRETTY_FUNCTION__); }
));
336 assert(tensor_memory != 0 && tensor_formats != 0 && tensor_datatypes != 0)((void) sizeof ((tensor_memory != 0 && tensor_formats
!= 0 && tensor_datatypes != 0) ? 1 : 0), __extension__
({ if (tensor_memory != 0 && tensor_formats != 0 &&
tensor_datatypes != 0) ; else __assert_fail ("tensor_memory != 0 && tensor_formats != 0 && tensor_datatypes != 0"
, "ccv_nnc_cmd.c", 336, __extension__ __PRETTY_FUNCTION__); }
));
337 int i;
338 for (i = 0; i < CCV_NNC_BACKEND_COUNT; i++)
339 {
340 const ccv_nnc_cmd_backend_registry_t api_registry = init_map[cmd_idx].backends[i];
341 // We have the exec kernel, and support all the tensor memory types.
342 if (api_registry.exec &&
343 (api_registry.tensor_memory & tensor_memory) == tensor_memory &&
344 (api_registry.tensor_formats & tensor_formats) == tensor_formats &&
345 (api_registry.tensor_datatypes & tensor_datatypes) == tensor_datatypes)
346 return backend_init_map[i].backend;
347 }
348 return cmd.backend;
349}
350
351#define AUTO_TUNE_TRIAL_SIZE(3) (3)
352
353static void _ccv_nnc_cmd_set_device_id(ccv_nnc_tensor_t* const* const inputs, const int input_size, ccv_nnc_tensor_t* const* const outputs, const int output_size, ccv_nnc_stream_context_t* const stream_context)
354{
355#ifdef HAVE_CUDA1
356 if (!stream_context)
357 {
358 int device_id;
359 if (ccv_nnc_device_ids_for_io(inputs, input_size, outputs, output_size, CCV_TENSOR_GPU_MEMORY, &device_id, 1) > 0)
360 cudevice(device_id);
361 }
362#endif
363}
364
365typedef struct {
366 int format;
367 int datatype;
368 int nd;
369 off_t dataof;
370 int dim[CCV_NNC_MAX_DIM_ALLOC(12)];
371 int stride[CCV_NNC_MAX_DIM_ALLOC(12)];
372} ccv_nnc_cmd_autotune_tensor_shape_t;
373
374typedef struct {
375 uint32_t cmd;
376 ccv_nnc_cmd_param_t params;
377 ccv_nnc_hint_t hint;
378 int flags;
379 int input_size;
380 int output_size;
381 size_t workspace_size;
382 ccv_nnc_cmd_autotune_tensor_shape_t* inputs;
383 ccv_nnc_cmd_autotune_tensor_shape_t* outputs;
384} ccv_nnc_cmd_autotune_key_t;
385
386static CCV_WARN_UNUSED(ccv_nnc_cmd_autotune_key_t)ccv_nnc_cmd_autotune_key_t __attribute__((warn_unused_result)
) ccv_nnc_cmd_autotune_key_new(const ccv_nnc_cmd_t cmd, const size_t workspace_size, const ccv_nnc_hint_t hint, const int flags, ccv_nnc_tensor_t* const* const inputs, const int input_size, ccv_nnc_tensor_t* const* const outputs, const int output_size)
387{
388 ccv_nnc_cmd_autotune_key_t key = {
389 .cmd = cmd.cmd,
390 .params = cmd.info,
391 .hint = hint,
392 .workspace_size = workspace_size,
393 .inputs = 0,
394 .input_size = 0,
395 .outputs = 0,
396 .output_size = 0
397 };
398 if (input_size == 0 && output_size
20.1
'output_size' is not equal to 0
 == 0)
20
←
Assuming 'input_size' is equal to 0
→
21
←
Taking false branch
→
399 return key;
400 assert(input_size >= 0 && output_size >= 0)((void) sizeof ((input_size >= 0 && output_size >=
0) ? 1 : 0), __extension__ ({ if (input_size >= 0 &&
output_size >= 0) ; else __assert_fail ("input_size >= 0 && output_size >= 0"
, "ccv_nnc_cmd.c", 400, __extension__ __PRETTY_FUNCTION__); }
));
22
←
Taking true branch
→
401 key.input_size = input_size;
402 key.output_size = output_size;
403 key.inputs = (ccv_nnc_cmd_autotune_tensor_shape_t*)ccmallocmalloc(sizeof(ccv_nnc_cmd_autotune_tensor_shape_t) * (input_size + output_size));
23
←
Memory is allocated
→
404 key.outputs = key.inputs + input_size;
405 int i, j;
406 for (i = 0; i < input_size; i++)
24
←
Loop condition is false. Execution continues on line 428
→
407 {
408 memset(key.inputs[i].dim, 0, sizeof(key.inputs[i].dim));
409 memset(key.inputs[i].stride, 0, sizeof(key.inputs[i].stride));
410 if (!inputs[i])
411 {
412 key.inputs[i].format = 0;
413 key.inputs[i].datatype = 0;
414 key.inputs[i].dataof = 0;
415 key.inputs[i].nd = 0;
416 continue;
417 }
418 key.inputs[i].format = inputs[i]->info.format;
419 key.inputs[i].datatype = inputs[i]->info.datatype;
420 key.inputs[i].dataof = inputs[i]->dataof;
421 const int nd = key.inputs[i].nd = ccv_nnc_tensor_nd(inputs[i]->info.dim);
422 for (j = 0; j < nd; j++)
423 key.inputs[i].dim[j] = inputs[i]->info.dim[j];
424 if (CCV_IS_TENSOR_VIEW(inputs[i])((*(int*)(inputs[i])) & CCV_TENSOR_VIEW))
425 for (j = 0; j < nd; j++)
426 key.inputs[i].stride[j] = ((ccv_nnc_tensor_view_t*)inputs[i])->stride[j];
427 }
428 for (i = 0; i < output_size; i++)
25
←
Loop condition is true. Entering loop body
→
31
←
Loop condition is false. Execution continues on line 450
→
429 {
430 memset(key.outputs[i].dim, 0, sizeof(key.outputs[i].dim));
431 memset(key.outputs[i].stride, 0, sizeof(key.outputs[i].stride));
432 if (!outputs[i])
26
←
Taking false branch
→
433 {
434 key.outputs[i].format = 0;
435 key.outputs[i].datatype = 0;
436 key.outputs[i].dataof = 0;
437 key.outputs[i].nd = 0;
438 continue;
439 }
440 key.outputs[i].format = outputs[i]->info.format;
441 key.outputs[i].datatype = outputs[i]->info.datatype;
442 key.outputs[i].dataof = outputs[i]->dataof;
443 const int nd = key.outputs[i].nd = ccv_nnc_tensor_nd(outputs[i]->info.dim);
444 for (j = 0; j < nd; j++)
27
←
Assuming 'j' is >= 'nd'
→
28
←
Loop condition is false. Execution continues on line 446
→
445 key.outputs[i].dim[j] = outputs[i]->info.dim[j];
446 if (CCV_IS_TENSOR_VIEW(outputs[i])((*(int*)(outputs[i])) & CCV_TENSOR_VIEW))
29
←
Assuming the condition is false
→
30
←
Taking false branch
→
447 for (j = 0; j < nd; j++)
448 key.outputs[i].stride[j] = ((ccv_nnc_tensor_view_t*)outputs[i])->stride[j];
449 }
450 return key;
451}
452
453// autotune cache.
454static inline uint32_t twang_32from64(uint64_t key)
455{
456 key = (~key) + (key << 18);
457 key = key ^ (key >> 31);
458 key = key * 21;
459 key = key ^ (key >> 11);
460 key = key + (key << 6);
461 key = key ^ (key >> 22);
462 return (uint32_t)(key);
463}
464
465static inline khint32_t _kh_autotune_key_executable_hash_func(const ccv_nnc_cmd_autotune_key_t key)
466{
467 uint32_t h = key.cmd;
468 int i, j;
469 uint32_t* data = (uint32_t*)&key.params;
470 for (i = 0; i < sizeof(key.params) / sizeof(uint32_t); i++)
471 h = twang_32from64(((uint64_t)h << 32) | data[i]);
472 data = (uint32_t*)&key.hint;
473 for (i = 0; i < sizeof(key.hint) / sizeof(uint32_t); i++)
474 h = twang_32from64(((uint64_t)h << 32) | data[i]);
475 h = twang_32from64(((uint64_t)h << 32) | key.workspace_size);
476 h = twang_32from64(((uint64_t)h << 32) | key.input_size);
477 h = twang_32from64(((uint64_t)h << 32) | key.output_size);
478 for (i = 0; i < key.input_size; i++)
479 {
480 h = twang_32from64(((uint64_t)h << 32) | key.inputs[i].format);
481 h = twang_32from64(((uint64_t)h << 32) | key.inputs[i].datatype);
482 h = twang_32from64(((uint64_t)h << 32) | key.inputs[i].dataof);
483 h = twang_32from64(((uint64_t)h << 32) | key.inputs[i].nd);
484 for (j = 0; j < key.inputs[i].nd; j++)
485 {
486 h = twang_32from64(((uint64_t)h << 32) | key.inputs[i].dim[j]);
487 h = twang_32from64(((uint64_t)h << 32) | key.inputs[i].stride[j]);
488 }
489 }
490 for (i = 0; i < key.output_size; i++)
491 {
492 h = twang_32from64(((uint64_t)h << 32) | key.outputs[i].format);
493 h = twang_32from64(((uint64_t)h << 32) | key.outputs[i].datatype);
494 h = twang_32from64(((uint64_t)h << 32) | key.outputs[i].dataof);
495 h = twang_32from64(((uint64_t)h << 32) | key.outputs[i].nd);
496 for (j = 0; j < key.outputs[i].nd; j++)
497 {
498 h = twang_32from64(((uint64_t)h << 32) | key.outputs[i].dim[j]);
499 h = twang_32from64(((uint64_t)h << 32) | key.outputs[i].stride[j]);
500 }
501 }
502 return (khint32_t)h;
503}
504
505static inline int _kh_autotune_key_executable_hash_equal(const ccv_nnc_cmd_autotune_key_t a, const ccv_nnc_cmd_autotune_key_t b)
506{
507 if (a.cmd != b.cmd || a.flags != b.flags || a.workspace_size != b.workspace_size || a.input_size != b.input_size || a.output_size != b.output_size)
508 return 0;
509 if (memcmp(&a.params, &b.params, sizeof(a.params)) != 0)
510 return 0;
511 if (memcmp(&a.hint, &b.hint, sizeof(a.hint)) != 0)
512 return 0;
513 int i, j;
514 for (i = 0; i < a.input_size; i++)
515 {
516 if (a.inputs[i].format != b.inputs[i].format || a.inputs[i].datatype != b.inputs[i].datatype || a.inputs[i].nd != b.inputs[i].nd || a.inputs[i].dataof != b.inputs[i].dataof)
517 return 0;
518 for (j = 0; j < a.inputs[i].nd; j++)
519 if (a.inputs[i].dim[j] != b.inputs[i].dim[j] || a.inputs[i].stride[j] != b.inputs[i].stride[j])
520 return 0;
521 }
522 for (i = 0; i < a.output_size; i++)
523 {
524 if (a.outputs[i].format != b.outputs[i].format || a.outputs[i].datatype != b.outputs[i].datatype || a.outputs[i].nd != b.outputs[i].nd || a.outputs[i].dataof != b.outputs[i].dataof)
525 return 0;
526 for (j = 0; j < a.outputs[i].nd; j++)
527 if (a.outputs[i].dim[j] != b.outputs[i].dim[j] || a.outputs[i].stride[j] != b.outputs[i].stride[j])
528 return 0;
529 }
530 return 1;
531}
532
533typedef struct {
534 int backend;
535 int algorithm;
536} ccv_nnc_cmd_autotune_val_t;
537
538KHASH_INIT(autotune_executable_cache, ccv_nnc_cmd_autotune_key_t, ccv_nnc_cmd_autotune_val_t, 1, _kh_autotune_key_executable_hash_func, _kh_autotune_key_executable_hash_equal)typedef struct kh_autotune_executable_cache_s { khint_t n_buckets
, size, n_occupied, upper_bound; khint32_t *flags; ccv_nnc_cmd_autotune_key_t
*keys; ccv_nnc_cmd_autotune_val_t *vals; } kh_autotune_executable_cache_t
; static inline __attribute__ ((__unused__)) kh_autotune_executable_cache_t
*kh_init_autotune_executable_cache(void) { return (kh_autotune_executable_cache_t
*)calloc(1,sizeof(kh_autotune_executable_cache_t)); } static inline
__attribute__ ((__unused__)) void kh_destroy_autotune_executable_cache
(kh_autotune_executable_cache_t *h) { if (h) { free((void *)h
->keys); free(h->flags); free((void *)h->vals); free
(h); } } static inline __attribute__ ((__unused__)) void kh_clear_autotune_executable_cache
(kh_autotune_executable_cache_t *h) { if (h && h->
flags) { memset(h->flags, 0xaa, ((h->n_buckets) < 16
? 1 : (h->n_buckets)>>4) * sizeof(khint32_t)); h->
size = h->n_occupied = 0; } } static inline __attribute__ (
(__unused__)) khint_t kh_get_autotune_executable_cache(const kh_autotune_executable_cache_t
*h, ccv_nnc_cmd_autotune_key_t key) { if (h->n_buckets) {
khint_t k, i, last, mask, step = 0; mask = h->n_buckets -
1; k = _kh_autotune_key_executable_hash_func(key); i = k &
mask; last = i; while (!((h->flags[i>>4]>>((i
&0xfU)<<1))&2) && (((h->flags[i>>
4]>>((i&0xfU)<<1))&1) || !_kh_autotune_key_executable_hash_equal
(h->keys[i], key))) { i = (i + (++step)) & mask; if (i
== last) return h->n_buckets; } return ((h->flags[i>>
4]>>((i&0xfU)<<1))&3)? h->n_buckets : i
; } else return 0; } static inline __attribute__ ((__unused__
)) int kh_resize_autotune_executable_cache(kh_autotune_executable_cache_t
*h, khint_t new_n_buckets) { khint32_t *new_flags = 0; khint_t
j = 1; { (--(new_n_buckets), (new_n_buckets)|=(new_n_buckets
)>>1, (new_n_buckets)|=(new_n_buckets)>>2, (new_n_buckets
)|=(new_n_buckets)>>4, (new_n_buckets)|=(new_n_buckets)
>>8, (new_n_buckets)|=(new_n_buckets)>>16, ++(new_n_buckets
)); if (new_n_buckets < 4) new_n_buckets = 4; if (h->size
>= (khint_t)(new_n_buckets * __ac_HASH_UPPER + 0.5)) j = 0
; else { new_flags = (khint32_t*)malloc(((new_n_buckets) <
16? 1 : (new_n_buckets)>>4) * sizeof(khint32_t)); if (
!new_flags) return -1; memset(new_flags, 0xaa, ((new_n_buckets
) < 16? 1 : (new_n_buckets)>>4) * sizeof(khint32_t))
; if (h->n_buckets < new_n_buckets) { ccv_nnc_cmd_autotune_key_t
*new_keys = (ccv_nnc_cmd_autotune_key_t*)realloc((void *)h->
keys,new_n_buckets * sizeof(ccv_nnc_cmd_autotune_key_t)); if (
!new_keys) { free(new_flags); return -1; } h->keys = new_keys
; if (1) { ccv_nnc_cmd_autotune_val_t *new_vals = (ccv_nnc_cmd_autotune_val_t
*)realloc((void *)h->vals,new_n_buckets * sizeof(ccv_nnc_cmd_autotune_val_t
)); if (!new_vals) { free(new_flags); return -1; } h->vals
= new_vals; } } } } if (j) { for (j = 0; j != h->n_buckets
; ++j) { if (((h->flags[j>>4]>>((j&0xfU)<<
1))&3) == 0) { ccv_nnc_cmd_autotune_key_t key = h->keys
[j]; ccv_nnc_cmd_autotune_val_t val; khint_t new_mask; new_mask
= new_n_buckets - 1; if (1) val = h->vals[j]; (h->flags
[j>>4]|=1ul<<((j&0xfU)<<1)); while (1) {
khint_t k, i, step = 0; k = _kh_autotune_key_executable_hash_func
(key); i = k & new_mask; while (!((new_flags[i>>4]>>
((i&0xfU)<<1))&2)) i = (i + (++step)) & new_mask
; (new_flags[i>>4]&=~(2ul<<((i&0xfU)<<
1))); if (i < h->n_buckets && ((h->flags[i>>
4]>>((i&0xfU)<<1))&3) == 0) { { ccv_nnc_cmd_autotune_key_t
tmp = h->keys[i]; h->keys[i] = key; key = tmp; } if (1
) { ccv_nnc_cmd_autotune_val_t tmp = h->vals[i]; h->vals
[i] = val; val = tmp; } (h->flags[i>>4]|=1ul<<
((i&0xfU)<<1)); } else { h->keys[i] = key; if (1
) h->vals[i] = val; break; } } } } if (h->n_buckets >
new_n_buckets) { h->keys = (ccv_nnc_cmd_autotune_key_t*)realloc
((void *)h->keys,new_n_buckets * sizeof(ccv_nnc_cmd_autotune_key_t
)); if (1) h->vals = (ccv_nnc_cmd_autotune_val_t*)realloc(
(void *)h->vals,new_n_buckets * sizeof(ccv_nnc_cmd_autotune_val_t
)); } free(h->flags); h->flags = new_flags; h->n_buckets
= new_n_buckets; h->n_occupied = h->size; h->upper_bound
= (khint_t)(h->n_buckets * __ac_HASH_UPPER + 0.5); } return
0; } static inline __attribute__ ((__unused__)) khint_t kh_put_autotune_executable_cache
(kh_autotune_executable_cache_t *h, ccv_nnc_cmd_autotune_key_t
key, int *ret) { khint_t x; if (h->n_occupied >= h->
upper_bound) { if (h->n_buckets > (h->size<<1)
) { if (kh_resize_autotune_executable_cache(h, h->n_buckets
- 1) < 0) { *ret = -1; return h->n_buckets; } } else if
(kh_resize_autotune_executable_cache(h, h->n_buckets + 1)
< 0) { *ret = -1; return h->n_buckets; } } { khint_t k
, i, site, last, mask = h->n_buckets - 1, step = 0; x = site
= h->n_buckets; k = _kh_autotune_key_executable_hash_func
(key); i = k & mask; if (((h->flags[i>>4]>>
((i&0xfU)<<1))&2)) x = i; else { last = i; while
(!((h->flags[i>>4]>>((i&0xfU)<<1))&
2) && (((h->flags[i>>4]>>((i&0xfU)
<<1))&1) || !_kh_autotune_key_executable_hash_equal
(h->keys[i], key))) { if (((h->flags[i>>4]>>
((i&0xfU)<<1))&1)) site = i; i = (i + (++step))
& mask; if (i == last) { x = site; break; } } if (x == h
->n_buckets) { if (((h->flags[i>>4]>>((i&
0xfU)<<1))&2) && site != h->n_buckets) x
= site; else x = i; } } } if (((h->flags[x>>4]>>
((x&0xfU)<<1))&2)) { h->keys[x] = key; (h->
flags[x>>4]&=~(3ul<<((x&0xfU)<<1)))
; ++h->size; ++h->n_occupied; *ret = 1; } else if (((h->
flags[x>>4]>>((x&0xfU)<<1))&1)) { h
->keys[x] = key; (h->flags[x>>4]&=~(3ul<<
((x&0xfU)<<1))); ++h->size; *ret = 2; } else *ret
= 0; return x; } static inline __attribute__ ((__unused__)) void
kh_del_autotune_executable_cache(kh_autotune_executable_cache_t
*h, khint_t x) { if (x != h->n_buckets && !((h->
flags[x>>4]>>((x&0xfU)<<1))&3)) { (
h->flags[x>>4]|=1ul<<((x&0xfU)<<1));
--h->size; } }
539
540static khash_t(autotune_executable_cache)kh_autotune_executable_cache_t* g_autotune_executable_cache = 0;
541
542static inline void ccv_nnc_cmd_autotune_key_free(ccv_nnc_cmd_autotune_key_t key)
543{
544 if (key.inputs)
545 ccfreefree(key.inputs);
546}
547
548void ccv_nnc_drain_autotune_cache(void)
549{
550 if (!g_autotune_executable_cache)
551 return;
552 khiter_t k;
553 for (k = kh_begin(g_autotune_executable_cache)(khint_t)(0); k < kh_end(g_autotune_executable_cache)((g_autotune_executable_cache)->n_buckets); k++)
554 {
555 if (!kh_exist(g_autotune_executable_cache, k)(!(((g_autotune_executable_cache)->flags[(k)>>4]>>
(((k)&0xfU)<<1))&3)))
556 continue;
557 ccv_nnc_cmd_autotune_key_free(kh_key(g_autotune_executable_cache, k)((g_autotune_executable_cache)->keys[k]));
558 kh_del(autotune_executable_cache, g_autotune_executable_cache, k)kh_del_autotune_executable_cache(g_autotune_executable_cache,
k);
559 }
560}
561
562ccv_nnc_cmd_t ccv_nnc_cmd_autotune(const ccv_nnc_cmd_t cmd, const size_t max_workspace_size, const ccv_nnc_hint_t hint, const int flags, ccv_nnc_tensor_t* const* const inputs, const int input_size, ccv_nnc_tensor_t* const* const outputs, const int output_size, ccv_nnc_stream_context_t* const stream_context)
563{
564 // This is a custom cmd kernel, no need to autotune.
565 if (cmd.cmd == CCV_NNC_NOOP ||
1
Assuming field 'cmd' is not equal to CCV_NNC_NOOP
→
6
←
Taking false branch
→
566 cmd.cmd == CCV_NNC_GRAPH_FORWARD || cmd.cmd == CCV_NNC_GRAPH_BACKWARD ||
2
←
Assuming field 'cmd' is not equal to CCV_NNC_GRAPH_FORWARD
→
3
←
Assuming field 'cmd' is not equal to CCV_NNC_GRAPH_BACKWARD
→
567 cmd.cmd == CCV_NNC_CUSTOM_FORWARD || cmd.cmd == CCV_NNC_CUSTOM_BACKWARD)
4
←
Assuming field 'cmd' is not equal to CCV_NNC_CUSTOM_FORWARD
→
5
←
Assuming field 'cmd' is not equal to CCV_NNC_CUSTOM_BACKWARD
→
568 return cmd;
569 int i, j, k;
570 // Go through all the backends that supports the same type of memory input / output tensors support.
571 int tensor_memory = 0, tensor_formats = 0, tensor_datatypes = 0;
572 for (i = 0; i < input_size; i++)
7
←
Assuming 'i' is >= 'input_size'
→
8
←
Loop condition is false. Execution continues on line 575
→
573 if (inputs[i])
574 tensor_memory |= CCV_TENSOR_GET_MEMORY(inputs[i]->info.type)((inputs[i]->info.type) & 0x3), tensor_formats |= inputs[i]->info.format, tensor_datatypes |= CCV_GET_DATA_TYPE(inputs[i]->info.datatype)((inputs[i]->info.datatype) & 0xFF000);
575 for (i = 0; i < output_size; i++)
9
←
Assuming 'i' is < 'output_size'
→
10
←
Loop condition is true. Entering loop body
→
13
←
Assuming 'i' is >= 'output_size'
→
14
←
Loop condition is false. Execution continues on line 579
→
576 if (outputs[i])
11
←
Assuming the condition is true
→
12
←
Taking true branch
→
577 tensor_memory |= CCV_TENSOR_GET_MEMORY(outputs[i]->info.type)((outputs[i]->info.type) & 0x3), tensor_formats |= outputs[i]->info.format, tensor_datatypes |= CCV_GET_DATA_TYPE(outputs[i]->info.datatype)((outputs[i]->info.datatype) & 0xFF000);
578 // In this case, we cannot determine the type of the tensor, skip auto-tune.
579 if (!tensor_memory)
15
←
Assuming 'tensor_memory' is not equal to 0
→
16
←
Taking false branch
→
580 return cmd;
581 // Otherwise, we are good to go.
582 ccv_nnc_cmd_t tuned_cmd = cmd;
583 if (!g_autotune_executable_cache)
17
←
Assuming 'g_autotune_executable_cache' is non-null
→
18
←
Taking false branch
→
584 g_autotune_executable_cache = kh_init(autotune_executable_cache)kh_init_autotune_executable_cache();
585 int ret = 0;
586 ccv_nnc_cmd_autotune_key_t key = ccv_nnc_cmd_autotune_key_new(cmd, max_workspace_size, hint, flags, inputs, input_size, outputs, output_size);
19
←
Calling 'ccv_nnc_cmd_autotune_key_new'
→
32
←
Returned allocated memory
→
587 khiter_t kiter = kh_put(autotune_executable_cache, g_autotune_executable_cache, key, &ret)kh_put_autotune_executable_cache(g_autotune_executable_cache,
key, &ret);
588 if (ret
32.1
'ret' is not equal to 0
 == 0)
33
←
Taking false branch
→
589 {
590 ccv_nnc_cmd_autotune_key_free(key);
591 const ccv_nnc_cmd_autotune_val_t val = kh_val(g_autotune_executable_cache, kiter)((g_autotune_executable_cache)->vals[kiter]);
592 tuned_cmd.backend = val.backend;
593 tuned_cmd.algorithm = val.algorithm;
594 return tuned_cmd;
595 }
596 int64_t best_measured = -1;
34
←
Potential memory leak
597 const int cmd_idx = _ccv_nnc_cmd_ph(cmd.cmd);
598 assert(cmd_idx >= 0 && cmd_idx < sizeof(init_map) / sizeof(init_map[0]))((void) sizeof ((cmd_idx >= 0 && cmd_idx < sizeof
(init_map) / sizeof(init_map[0])) ? 1 : 0), __extension__ ({ if
(cmd_idx >= 0 && cmd_idx < sizeof(init_map) / sizeof
(init_map[0])) ; else __assert_fail ("cmd_idx >= 0 && cmd_idx < sizeof(init_map) / sizeof(init_map[0])"
, "ccv_nnc_cmd.c", 598, __extension__ __PRETTY_FUNCTION__); }
));
599 int flag = 0, autotune_available_1 = 0; // This is only applicable if we have only one backend.
600 for (i = 0; i < CCV_NNC_BACKEND_COUNT; i++)
601 {
602 const ccv_nnc_cmd_backend_registry_t api_registry = init_map[cmd_idx].backends[i];
603 // We have the exec kernel, and support all the tensor memory types.
604 if (api_registry.exec &&
605 (api_registry.tensor_memory & tensor_memory) == tensor_memory &&
606 (api_registry.tensor_formats & tensor_formats) == tensor_formats &&
607 (api_registry.tensor_datatypes & tensor_datatypes) == tensor_datatypes)
608 {
609 if (api_registry.autotune)
610 autotune_available_1 = 1;
611 if ((++flag) >= 2) // If we have more than 2 suitable backend, we can do this now.
612 break;
613 }
614 }
615 if (flag == 0)
616 return cmd;
617 _ccv_nnc_cmd_set_device_id(inputs, input_size, outputs, output_size, stream_context);
618 // Allocate inputs / outputs and fill them in.
619 ccv_nnc_tensor_t** copy_inputs;
620 ccv_nnc_tensor_t** copy_outputs;
621 ccv_nnc_tensor_t** allocated_inputs;
622 ccv_nnc_tensor_t** allocated_outputs;
623 ccv_nnc_tensor_view_t** allocated_input_views;
624 ccv_nnc_tensor_view_t** allocated_output_views;
625 if (flag > 1 || autotune_available_1)
626 {
627 copy_inputs = (ccv_nnc_tensor_t**)cccalloccalloc((input_size + output_size) * 3, sizeof(ccv_nnc_tensor_t*));
628 copy_outputs = copy_inputs + input_size;
629 allocated_inputs = copy_outputs + output_size;
630 allocated_outputs = allocated_inputs + input_size;
631 allocated_input_views = (ccv_nnc_tensor_view_t**)(allocated_outputs + output_size);
632 allocated_output_views = allocated_input_views + input_size;
633 int stride[CCV_NNC_MAX_DIM_ALLOC(12)];
634 for (i = 0; i < output_size; i++)
635 if (outputs[i])
636 {
637 for (j = 0; j < input_size; j++)
638 if (inputs[j])
639 {
640 if (outputs[i] == inputs[j])
641 {
642 if (!copy_inputs[j])
643 {
644 allocated_inputs[j] = ccv_nnc_tensor_new(0, inputs[j]->info, 0);
645 if (CCV_IS_TENSOR_VIEW(inputs[j])((*(int*)(inputs[j])) & CCV_TENSOR_VIEW))
646 {
647 ccv_nnc_tensor_get_stride(inputs[j]->info.dim, stride);
648 copy_inputs[j] = (ccv_nnc_tensor_t*)(allocated_input_views[j] = ccv_nnc_tensor_view_new(allocated_inputs[j], inputs[j]->info, DIM_ALLOC()(int [(12)]){}, stride));
649 } else
650 copy_inputs[j] = allocated_inputs[j];
651 }
652 copy_outputs[i] = copy_inputs[j];
653 break;
654 } else if (outputs[i]->data.u8 == inputs[j]->data.u8 &&
655 ccv_nnc_tensor_count(outputs[i]->info) == ccv_nnc_tensor_count(inputs[j]->info)) {
656 if (!copy_inputs[j])
657 {
658 allocated_inputs[j] = ccv_nnc_tensor_new(0, inputs[j]->info, 0);
659 if (CCV_IS_TENSOR_VIEW(inputs[j])((*(int*)(inputs[j])) & CCV_TENSOR_VIEW))
660 {
661 ccv_nnc_tensor_get_stride(inputs[j]->info.dim, stride);
662 copy_inputs[j] = (ccv_nnc_tensor_t*)(allocated_input_views[j] = ccv_nnc_tensor_view_new(allocated_inputs[j], inputs[j]->info, DIM_ALLOC()(int [(12)]){}, stride));
663 } else
664 copy_inputs[j] = allocated_inputs[j];
665 }
666 allocated_outputs[i] = ccv_nnc_tensor_new(copy_inputs[j]->data.u8, outputs[i]->info, 0);
667 if (CCV_IS_TENSOR_VIEW(outputs[i])((*(int*)(outputs[i])) & CCV_TENSOR_VIEW))
668 {
669 ccv_nnc_tensor_get_stride(outputs[i]->info.dim, stride);
670 copy_outputs[i] = (ccv_nnc_tensor_t*)(allocated_output_views[i] = ccv_nnc_tensor_view_new(allocated_outputs[i], outputs[i]->info, DIM_ALLOC()(int [(12)]){}, stride));
671 } else
672 copy_outputs[i] = allocated_outputs[i];
673 break;
674 }
675 }
676 if (!copy_outputs[i])
677 {
678 allocated_outputs[i] = ccv_nnc_tensor_new(0, outputs[i]->info, 0);
679 if (CCV_IS_TENSOR_VIEW(outputs[i])((*(int*)(outputs[i])) & CCV_TENSOR_VIEW))
680 {
681 ccv_nnc_tensor_get_stride(outputs[i]->info.dim, stride);
682 copy_outputs[i] = (ccv_nnc_tensor_t*)(allocated_output_views[i] = ccv_nnc_tensor_view_new(allocated_outputs[i], outputs[i]->info, DIM_ALLOC()(int [(12)]){}, stride));
683 } else
684 copy_outputs[i] = allocated_outputs[i];
685 }
686 }
687 for (i = 0; i < input_size; i++)
688 if (inputs[i] && !copy_inputs[i])
689 copy_inputs[i] = inputs[i];
690 }
691 if (flag == 1)
692 {
693 for (i = 0; i < CCV_NNC_BACKEND_COUNT; i++)
694 {
695 const ccv_nnc_cmd_backend_registry_t api_registry = init_map[cmd_idx].backends[i];
696 // We have the exec kernel, and support all the tensor memory types.
697 if (api_registry.exec &&
698 (api_registry.tensor_memory & tensor_memory) == tensor_memory &&
699 (api_registry.tensor_formats & tensor_formats) == tensor_formats &&
700 (api_registry.tensor_datatypes & tensor_datatypes) == tensor_datatypes)
701 {
702 tuned_cmd.backend = backend_init_map[i].backend;
703 // If a given API exist an autotune function, use that to pick the top algorithm.
704 if (api_registry.autotune)
705 {
706 ccv_nnc_cmd_exec(CMD_DATA_TRANSFER_FORWARD()ccv_nnc_cmd(CCV_NNC_DATA_TRANSFER_FORWARD, 0, ccv_nnc_cmd_auto
, 0), ccv_nnc_no_hint, 0, inputs, input_size, copy_inputs, input_size, stream_context);
707 _ccv_nnc_cmd_set_device_id(copy_inputs, input_size, copy_outputs, output_size, stream_context);
708 tuned_cmd.algorithm = api_registry.autotune(tuned_cmd, max_workspace_size, hint, flags, copy_inputs, input_size, copy_outputs, output_size, stream_context);
709 // Drain the context, autotune can use excessive amount of memory. Need to drain it now.
710 ccv_nnc_stream_context_drain(stream_context);
711 }
712 break;
713 }
714 }
715 if (autotune_available_1)
716 {
717 for (i = 0; i < input_size; i++)
718 {
719 if (allocated_inputs[i])
720 ccv_nnc_tensor_free(allocated_inputs[i]);
721 if (allocated_input_views[i])
722 ccv_nnc_tensor_view_free(allocated_input_views[i]);
723 }
724 for (i = 0; i < output_size; i++)
725 {
726 if (allocated_outputs[i])
727 ccv_nnc_tensor_free(allocated_outputs[i]);
728 if (allocated_output_views[i])
729 ccv_nnc_tensor_view_free(allocated_output_views[i]);
730 }
731 ccfreefree(copy_inputs);
732 }
733 const ccv_nnc_cmd_autotune_val_t val = {
734 .backend = tuned_cmd.backend,
735 .algorithm = tuned_cmd.algorithm
736 };
737 kh_val(g_autotune_executable_cache, kiter)((g_autotune_executable_cache)->vals[kiter]) = val;
738 return tuned_cmd;
739 }
740 // We need to have trial loop through all the data.
741 for (k = 0; k < AUTO_TUNE_TRIAL_SIZE(3); k++)
742 {
743 for (i = 0; i < CCV_NNC_BACKEND_COUNT; i++)
744 {
745 const ccv_nnc_cmd_backend_registry_t api_registry = init_map[cmd_idx].backends[i];
746 // We have the exec kernel, and support all the tensor memory types.
747 if (api_registry.exec &&
748 (api_registry.tensor_memory & tensor_memory) == tensor_memory &&
749 (api_registry.tensor_formats & tensor_formats) == tensor_formats &&
750 (api_registry.tensor_datatypes & tensor_datatypes) == tensor_datatypes)
751 {
752 ccv_nnc_cmd_t candid_cmd = cmd;
753 candid_cmd.backend = backend_init_map[i].backend;
754 // If a given API exist an autotune function, use that to pick the top algorithm.
755 if (api_registry.autotune)
756 {
757 // Assuming k == 0 is sufficient, and we can skip.
758 if (k > 0)
759 continue;
760 ccv_nnc_cmd_exec(CMD_DATA_TRANSFER_FORWARD()ccv_nnc_cmd(CCV_NNC_DATA_TRANSFER_FORWARD, 0, ccv_nnc_cmd_auto
, 0), ccv_nnc_no_hint, 0, inputs, input_size, copy_inputs, input_size, stream_context);
761 _ccv_nnc_cmd_set_device_id(copy_inputs, input_size, copy_outputs, output_size, stream_context);
762 candid_cmd.algorithm = api_registry.autotune(candid_cmd, max_workspace_size, hint, flags, copy_inputs, input_size, copy_outputs, output_size, stream_context);
763 // Drain the context, autotune can use excessive amount of memory. Need to drain it now.
764 ccv_nnc_stream_context_drain(stream_context);
765 uint64_t elapsed = ccv_nnc_cmd_mono_time();
766 // Ready to run.
767 int status = ccv_nnc_cmd_exec(candid_cmd, hint, flags, inputs, input_size, outputs, output_size, stream_context);
768 ccv_nnc_stream_context_wait(stream_context);
769 elapsed = ccv_nnc_cmd_mono_time() - elapsed;
770 if (status == CCV_NNC_EXEC_SUCCESS &&
771 (best_measured == -1 || elapsed < best_measured))
772 {
773 best_measured = elapsed;
774 tuned_cmd = candid_cmd;
775 }
776 } else {
777 // Otherwise loop over the existing algorithms and pick the top one.
778 for (j = 0; j < api_registry.algorithms; j++)
779 {
780 candid_cmd.algorithm = j;
781 ccv_nnc_cmd_exec(CMD_DATA_TRANSFER_FORWARD()ccv_nnc_cmd(CCV_NNC_DATA_TRANSFER_FORWARD, 0, ccv_nnc_cmd_auto
, 0), ccv_nnc_no_hint, 0, inputs, input_size, copy_inputs, input_size, stream_context);
782 _ccv_nnc_cmd_set_device_id(copy_inputs, input_size, copy_outputs, output_size, stream_context);
783 uint64_t elapsed = ccv_nnc_cmd_mono_time();
784 // Ready to run.
785 int status = ccv_nnc_cmd_exec(candid_cmd, hint, flags, copy_inputs, input_size, copy_outputs, output_size, stream_context);
786 elapsed = ccv_nnc_cmd_mono_time() - elapsed;
787 if (status == CCV_NNC_EXEC_SUCCESS &&
788 (best_measured == -1 || elapsed < best_measured))
789 {
790 best_measured = elapsed;
791 tuned_cmd = candid_cmd;
792 }
793 }
794 }
795 }
796 }
797 }
798 for (i = 0; i < input_size; i++)
799 {
800 if (allocated_inputs[i])
801 ccv_nnc_tensor_free(allocated_inputs[i]);
802 if (allocated_input_views[i])
803 ccv_nnc_tensor_view_free(allocated_input_views[i]);
804 }
805 for (i = 0; i < output_size; i++)
806 {
807 if (allocated_outputs[i])
808 ccv_nnc_tensor_free(allocated_outputs[i]);
809 if (allocated_output_views[i])
810 ccv_nnc_tensor_view_free(allocated_output_views[i]);
811 }
812 ccfreefree(copy_inputs);
813 const ccv_nnc_cmd_autotune_val_t val = {
814 .backend = tuned_cmd.backend,
815 .algorithm = tuned_cmd.algorithm
816 };
817 kh_val(g_autotune_executable_cache, kiter)((g_autotune_executable_cache)->vals[kiter]) = val;
818 return tuned_cmd;
819}
820
821int ccv_nnc_cmd_bitmask(const ccv_nnc_cmd_t cmd, const int input_size, const int output_size, const uint64_t* const input_bitmasks, const int input_bitmask_size, const uint64_t* const output_bitmasks, const int output_bitmask_size)
822{
823 // If it is no-op, return true, it can deal with any number of parameters.
824 if (cmd.cmd == CCV_NNC_NOOP)
825 return 1;
826 // If it is a custom command, I cannot check it at all, return false.
827 if (cmd.cmd == CCV_NNC_CUSTOM_FORWARD || cmd.cmd == CCV_NNC_CUSTOM_BACKWARD)
828 return 0;
829 const int cmd_idx = _ccv_nnc_cmd_ph(cmd.cmd);
830 const ccv_nnc_cmd_registry_t cmd_registry = init_map[cmd_idx].registry;
831 if (cmd_registry.bitmask)
832 return cmd_registry.bitmask(cmd.info, input_size, output_size, input_bitmasks, input_bitmask_size, output_bitmasks, output_bitmask_size);
833 // If there is not checking, none can pass.
834 return 0;
835}
836
837int ccv_nnc_device_ids_for_io(ccv_nnc_tensor_t* const* const inputs, const int input_size, ccv_nnc_tensor_t* const* const outputs, const int output_size, const int tensor_type, int* const device_ids, const int max_device_id_size)
838{
839 int i, j;
840 int device_id_size = 0;
841 if (max_device_id_size <= device_id_size)
842 return device_id_size;
843 // The device id of the exec is determined by its outputs.
844 for (i = 0; i < output_size; i++)
845 if (outputs[i] &&
846 CCV_TENSOR_GET_MEMORY(outputs[i]->info.type)((outputs[i]->info.type) & 0x3) == tensor_type &&
847 CCV_TENSOR_GET_DEVICE(outputs[i]->info.type)((outputs[i]->info.type) & 0xfff00) != CCV_COMPUTE_DEVICE_ANY)
848 {
849 const int device_id = CCV_TENSOR_GET_DEVICE_ID(outputs[i]->info.type)(((outputs[i]->info.type) & 0xfff00) >> 8);
850 int flag = 0;
851 for (j = 0; !flag && j < device_id_size; j++)
852 flag = (device_ids[j] == device_id);
853 if (flag)
854 continue;
855 device_ids[device_id_size++] = device_id;
856 if (device_id_size >= max_device_id_size)
857 return device_id_size;
858 }
859 if (device_id_size == 0)
860 {
861 int device_id = -1;
862 for (i = 0; i < input_size; i++)
863 if (inputs[i] &&
864 CCV_TENSOR_GET_MEMORY(inputs[i]->info.type)((inputs[i]->info.type) & 0x3) == tensor_type &&
865 CCV_TENSOR_GET_DEVICE(inputs[i]->info.type)((inputs[i]->info.type) & 0xfff00) != CCV_COMPUTE_DEVICE_ANY &&
866 (device_id < 0 || CCV_TENSOR_GET_DEVICE_ID(inputs[i]->info.type)(((inputs[i]->info.type) & 0xfff00) >> 8) < device_id))
867 device_id = CCV_TENSOR_GET_DEVICE_ID(inputs[i]->info.type)(((inputs[i]->info.type) & 0xfff00) >> 8);
868 if (device_id >= 0)
869 {
870 device_ids[0] = device_id;
871 return 1;
872 }
873 }
874 return device_id_size;
875}
876
877void* ccv_nnc_cmd_aux(const ccv_nnc_cmd_t cmd)
878{
879 if (cmd.cmd == CCV_NNC_NOOP ||
880 cmd.cmd == CCV_NNC_CUSTOM_FORWARD || cmd.cmd == CCV_NNC_CUSTOM_BACKWARD ||
881 cmd.cmd == CCV_NNC_GRAPH_FORWARD || cmd.cmd == CCV_NNC_GRAPH_BACKWARD)
882 return 0;
883 assert(cmd.backend != CCV_NNC_NO_BACKEND)((void) sizeof ((cmd.backend != CCV_NNC_NO_BACKEND) ? 1 : 0),
__extension__ ({ if (cmd.backend != CCV_NNC_NO_BACKEND) ; else
__assert_fail ("cmd.backend != CCV_NNC_NO_BACKEND", "ccv_nnc_cmd.c"
, 883, __extension__ __PRETTY_FUNCTION__); }));
884 const int cmd_idx = _ccv_nnc_cmd_ph(cmd.cmd);
885 assert(cmd_idx >= 0 && cmd_idx < sizeof(init_map) / sizeof(init_map[0]))((void) sizeof ((cmd_idx >= 0 && cmd_idx < sizeof
(init_map) / sizeof(init_map[0])) ? 1 : 0), __extension__ ({ if
(cmd_idx >= 0 && cmd_idx < sizeof(init_map) / sizeof
(init_map[0])) ; else __assert_fail ("cmd_idx >= 0 && cmd_idx < sizeof(init_map) / sizeof(init_map[0])"
, "ccv_nnc_cmd.c", 885, __extension__ __PRETTY_FUNCTION__); }
));
886 const int backend_idx = _ccv_nnc_cmd_backend_ph(cmd.backend);
887 assert(backend_idx >= 0 && backend_idx < CCV_NNC_BACKEND_COUNT)((void) sizeof ((backend_idx >= 0 && backend_idx <
CCV_NNC_BACKEND_COUNT) ? 1 : 0), __extension__ ({ if (backend_idx
>= 0 && backend_idx < CCV_NNC_BACKEND_COUNT) ;
else __assert_fail ("backend_idx >= 0 && backend_idx < CCV_NNC_BACKEND_COUNT"
, "ccv_nnc_cmd.c", 887, __extension__ __PRETTY_FUNCTION__); }
));
888 const ccv_nnc_cmd_backend_registry_t api_registry = init_map[cmd_idx].backends[backend_idx];
889 return api_registry.aux;
890}
891
892int ccv_nnc_cmd_exec(const ccv_nnc_cmd_t cmd, const ccv_nnc_hint_t hint, const int flags, ccv_nnc_tensor_t* const* const inputs, const int input_size, ccv_nnc_tensor_t* const* const outputs, const int output_size, ccv_nnc_stream_context_t* const stream_context)
893{
894 // If it is no-op, return as if succeed already.
895 if (cmd.cmd == CCV_NNC_NOOP)
896 return 0;
897 _ccv_nnc_cmd_set_device_id(inputs, input_size, outputs, output_size, stream_context);
898 // If it is a custom command, just apply it directly.
899 if (cmd.cmd == CCV_NNC_CUSTOM_FORWARD || cmd.cmd == CCV_NNC_CUSTOM_BACKWARD)
900 {
901 int ret = cmd.isa->exec(cmd, hint, flags, inputs, input_size, outputs, output_size, stream_context);
902 if (!stream_context)
903 ccv_nnc_stream_context_drain(stream_context);
904 return ret;
905 }
906 assert(cmd.cmd != CCV_NNC_GRAPH_FORWARD && cmd.cmd != CCV_NNC_GRAPH_BACKWARD)((void) sizeof ((cmd.cmd != CCV_NNC_GRAPH_FORWARD && cmd
.cmd != CCV_NNC_GRAPH_BACKWARD) ? 1 : 0), __extension__ ({ if
(cmd.cmd != CCV_NNC_GRAPH_FORWARD && cmd.cmd != CCV_NNC_GRAPH_BACKWARD
) ; else __assert_fail ("cmd.cmd != CCV_NNC_GRAPH_FORWARD && cmd.cmd != CCV_NNC_GRAPH_BACKWARD"
, "ccv_nnc_cmd.c", 906, __extension__ __PRETTY_FUNCTION__); }
));
907 const int cmd_idx = _ccv_nnc_cmd_ph(cmd.cmd);
908 assert(cmd_idx >= 0 && cmd_idx < sizeof(init_map) / sizeof(init_map[0]))((void) sizeof ((cmd_idx >= 0 && cmd_idx < sizeof
(init_map) / sizeof(init_map[0])) ? 1 : 0), __extension__ ({ if
(cmd_idx >= 0 && cmd_idx < sizeof(init_map) / sizeof
(init_map[0])) ; else __assert_fail ("cmd_idx >= 0 && cmd_idx < sizeof(init_map) / sizeof(init_map[0])"
, "ccv_nnc_cmd.c", 908, __extension__ __PRETTY_FUNCTION__); }
));
909 int i;
910 uint32_t backend = cmd.backend;
911 if (backend == CCV_NNC_NO_BACKEND)
912 {
913 // Find a suitable backend.
914 int tensor_memory = 0, tensor_formats = 0, tensor_datatypes = 0;
915 for (i = 0; i < input_size; i++)
916 if (inputs[i])
917 tensor_memory |= CCV_TENSOR_GET_MEMORY(inputs[i]->info.type)((inputs[i]->info.type) & 0x3), tensor_formats |= inputs[i]->info.format, tensor_datatypes |= CCV_GET_DATA_TYPE(inputs[i]->info.datatype)((inputs[i]->info.datatype) & 0xFF000);
918 for (i = 0; i < output_size; i++)
919 if (outputs[i])
920 tensor_memory |= CCV_TENSOR_GET_MEMORY(outputs[i]->info.type)((outputs[i]->info.type) & 0x3), tensor_formats |= outputs[i]->info.format, tensor_datatypes |= CCV_GET_DATA_TYPE(outputs[i]->info.datatype)((outputs[i]->info.datatype) & 0xFF000);
921 backend = ccv_nnc_cmd_find_backend(cmd, tensor_memory, tensor_formats, tensor_datatypes);
922 }
923 assert(backend != CCV_NNC_NO_BACKEND)((void) sizeof ((backend != CCV_NNC_NO_BACKEND) ? 1 : 0), __extension__
({ if (backend != CCV_NNC_NO_BACKEND) ; else __assert_fail (
"backend != CCV_NNC_NO_BACKEND", "ccv_nnc_cmd.c", 923, __extension__
__PRETTY_FUNCTION__); }));
924 const int backend_idx = _ccv_nnc_cmd_backend_ph(backend);
925 assert(backend_idx >= 0 && backend_idx < CCV_NNC_BACKEND_COUNT)((void) sizeof ((backend_idx >= 0 && backend_idx <
CCV_NNC_BACKEND_COUNT) ? 1 : 0), __extension__ ({ if (backend_idx
>= 0 && backend_idx < CCV_NNC_BACKEND_COUNT) ;
else __assert_fail ("backend_idx >= 0 && backend_idx < CCV_NNC_BACKEND_COUNT"
, "ccv_nnc_cmd.c", 925, __extension__ __PRETTY_FUNCTION__); }
));
926 const ccv_nnc_cmd_backend_registry_t api_registry = init_map[cmd_idx].backends[backend_idx];
927 if (!api_registry.exec)
928 return CCV_NNC_EXEC_NO_KERNEL;
929 // Everything is out, call the underlying implementation.
930 int ret = api_registry.exec(cmd, hint, flags, inputs, input_size, outputs, output_size, stream_context);
931 if (!stream_context)
932 ccv_nnc_stream_context_drain(stream_context);
933 return ret;
934}
935
936int ccv_nnc_cmd_attr(const ccv_nnc_cmd_t cmd, const int flags)
937{
938 // No additional attr for noop.
939 if (cmd.cmd == CCV_NNC_NOOP ||
940 // If it is a custom command, just apply it directly.
941 cmd.cmd == CCV_NNC_CUSTOM_FORWARD || cmd.cmd == CCV_NNC_CUSTOM_BACKWARD ||
942 // If it is sub-graph, there is no additional attr as well.
943 cmd.cmd == CCV_NNC_GRAPH_FORWARD || cmd.cmd == CCV_NNC_GRAPH_BACKWARD)
944 return 0;
945 const int cmd_idx = _ccv_nnc_cmd_ph(cmd.cmd);
946 assert(cmd_idx >= 0 && cmd_idx <sizeof(init_map) / sizeof(init_map[0]))((void) sizeof ((cmd_idx >= 0 && cmd_idx <sizeof
(init_map) / sizeof(init_map[0])) ? 1 : 0), __extension__ ({ if
(cmd_idx >= 0 && cmd_idx <sizeof(init_map) / sizeof
(init_map[0])) ; else __assert_fail ("cmd_idx >= 0 && cmd_idx <sizeof(init_map) / sizeof(init_map[0])"
, "ccv_nnc_cmd.c", 946, __extension__ __PRETTY_FUNCTION__); }
));
947 const ccv_nnc_cmd_registry_t cmd_registry = init_map[cmd_idx].registry;
948 return !!(cmd_registry.flags & flags);
949}
950
951void ccv_nnc_set_profiler(int state)
952{
953#ifdef HAVE_CUDA1
954 cusetprofiler(state);
955#endif
956}
957
958int ccv_nnc_queue_watermark(void)
959{
960#ifdef HAVE_MPS
961 return ccv_nnc_mps_queue_watermark();
962#else
963 return 0;
964#endif
965}
966
967void ccv_nnc_set_queue_watermark(int watermark)
968{
969#ifdef HAVE_MPS
970 // If we need to be memory efficient, we need to bound how many in-flight command buffers there are.
971 ccv_nnc_mps_set_queue_watermark(watermark);
972#endif
973}
974
975void ccv_nnc_set_whole_file_mapping_size_limit(const size_t size_limit)
976{
977#ifdef HAVE_MPS
978 ccv_nnc_mps_set_whole_file_mapping_size_limit(size_limit);
979#endif
980}
981
982void ccv_nnc_set_device_permutation(const int type, const int* const device_map, const int size)
983{
984 if (type != CCV_STREAM_CONTEXT_GPU)
985 return;
986#ifdef HAVE_CUDA1
987 cusetdevicemap(device_map, size);
988#endif
989}
990
991void ccv_nnc_set_binary_artifacts(const char** const paths_to_read, const int paths_to_read_size, const char* const path_to_write)
992{
993#ifdef HAVE_MPS
994 ccv_nnc_mps_set_binary_artifacts(paths_to_read, paths_to_read_size, path_to_write);
995#endif
996}
997
998void ccv_nnc_vacuum(void)
999{
1000#ifdef HAVE_MPS
1001 ccv_nnc_mps_clear_graph_executable_cache();
1002#endif
1003}