Coverage Report

Created: 2017-11-12 13:27

/home/liu/buildslave/linux-x64-runtests/build/lib/nnc/ccv_nnc_graph_while.c
Line
Count
Source (jump to first uncovered line)
1
#include "ccv_nnc.h"
2
#include "ccv_nnc_easy.h"
3
#include "ccv_nnc_internal.h"
4
#include "ccv_internal.h"
5
#ifdef HAVE_CUDA
6
#include "gpu/ccv_nnc_compat.h"
7
#endif
8
#include "_ccv_nnc_graph.h"
9
10
void ccv_nnc_tensor_multiview(ccv_nnc_tensor_t* data[], const uint8_t kind, const uint16_t repeat, const ccv_nnc_graph_t* const graph, ccv_nnc_tensor_multiview_t* const tensor_multiview)
11
27
{
12
27
  assert(kind == CCV_NNC_MULTIVIEW_K0N || kind == CCV_NNC_MULTIVIEW_K1N);
13
27
  assert(repeat > 0);
14
27
  tensor_multiview->type = CCV_TENSOR_MULTIVIEW;
15
27
  tensor_multiview->kind = kind;
16
27
  tensor_multiview->repeat = repeat;
17
27
  tensor_multiview->anchor = (intptr_t)graph;
18
27
  tensor_multiview->it = 0;
19
27
  tensor_multiview->p = 0;
20
27
  tensor_multiview->offset = 0;
21
27
  tensor_multiview->rtvs = 0;
22
27
  tensor_multiview->_heap_data = (repeat + kind <= 
CCV_NNC_MAX_INLINE_UNROLL27
) ?
027
:
ccmalloc0
(sizeof(ccv_nnc_tensor_t*) * (repeat + kind))0
;
23
27
  int i;
24
27
  // Currently, only CCV_NNC_MULTIVIEW_K12 uses 3 tensors.
25
86
  for (i = 0; 
i < repeat + kind86
;
i++59
)
26
59
  {
27
59
    CCV_NNC_MULTIVIEW_DATA(tensor_multiview)[i] = data[i];
28
59
    ccv_nnc_tensor_multiview_t* const mv = (ccv_nnc_tensor_multiview_t*)data[i];
29
59
    if (
data[i] != 59
CCV_NNC_TENSOR_PLACEHOLDER59
&&
CCV_IS_TENSOR_MULTIVIEW54
(mv))
30
7
      mv->p = tensor_multiview;
31
59
  }
32
27
}
33
34
void ccv_nnc_tensor_multiview_free(const ccv_nnc_tensor_multiview_t tensor_multiview)
35
18
{
36
18
  if (tensor_multiview.rtvs)
37
9
    ccv_array_free(tensor_multiview.rtvs);
38
18
  if (tensor_multiview._heap_data)
39
0
    
ccfree0
(tensor_multiview._heap_data)0
;
40
18
}
41
42
void ccv_nnc_tensor_reference_to_multiview(ccv_nnc_tensor_multiview_t* const tensor_multiview, ccv_nnc_tensor_t* const tensor)
43
14
{
44
14
  if (!tensor_multiview->rtvs)
45
9
    tensor_multiview->rtvs = ccv_array_new(sizeof(ccv_nnc_tensor_t*), 0, 0);
46
14
  ccv_array_push(tensor_multiview->rtvs, &tensor);
47
14
}
48
49
void ccv_nnc_tensor_multiview_broadcast(ccv_nnc_tensor_multiview_t* const tensor_multiview)
50
206
{
51
206
  assert(tensor_multiview->it && !CCV_IS_TENSOR_MULTIVIEW(tensor_multiview->it));
52
206
  // Update the pointer on tv only if it is not a single tensor pointer.
53
206
  unsigned char* const data = tensor_multiview->it->data.u8 - tensor_multiview->offset;
54
206
  const ccv_nnc_tensor_multiview_t* c = tensor_multiview;
55
206
  int i;
56
340
  do {
57
340
    if (c->rtvs)
58
195
      
for (i = 0; 72
i < c->rtvs->rnum195
;
i++123
)
59
123
      {
60
123
        ccv_nnc_tensor_t* tensor = *(ccv_nnc_tensor_t**)ccv_array_get(c->rtvs, i);
61
123
        if (CCV_IS_TENSOR_VIEW(tensor))
62
10
        {
63
10
          ccv_nnc_tensor_view_t* tensor_view = (ccv_nnc_tensor_view_t*)tensor;
64
10
          tensor_view->data.u8 = data + tensor_view->off;
65
10
        } else
66
113
          tensor->data.u8 = data;
67
123
      }
68
340
    c = c->p;
69
340
  } while (c);
70
206
}
71
72
ccv_nnc_graph_exec_t ccv_nnc_graph_while(ccv_nnc_graph_t* const graph, uint32_t cmd, ccv_nnc_graph_t* const while_graph)
73
15
{
74
15
  assert(cmd == CCV_NNC_GRAPH_FORWARD || cmd == CCV_NNC_GRAPH_BACKWARD);
75
15
  ccv_nnc_graph_exec_t while_exec = ccv_nnc_graph_exec_new(graph, ccv_nnc_cmd(cmd, 0, CMD_GENERIC(), 0), ccv_nnc_no_hint, 0, 0, 0, 0);
76
15
  ccv_nnc_graph_exec_info_t* while_exec_info = (ccv_nnc_graph_exec_info_t*)ccv_array_get(graph->exec_info, while_exec.d);
77
15
  if (!graph->sub_graphs)
78
14
    graph->sub_graphs = ccv_array_new(sizeof(ccv_nnc_graph_t*), 1, 0);
79
15
  int i;
80
15
  if (while_graph->nest_execs)
81
0
  {
82
0
    // Copy wraps from sub graph to parent graph.
83
0
    if (!graph->nest_execs)
84
0
      graph->nest_execs = ccv_array_new(sizeof(ccv_nnc_graph_exec_t), while_graph->nest_execs->rnum, 0);
85
0
    for (i = 0; 
i < while_graph->nest_execs->rnum0
;
i++0
)
86
0
      
ccv_array_push(graph->nest_execs, 0
ccv_array_get0
(while_graph->nest_execs, i));
87
0
  }
88
15
  ccv_array_push(graph->sub_graphs, &while_graph);
89
15
  while_graph->p = graph;
90
15
  while_graph->p_idx = graph->sub_graphs->rnum;
91
15
  while_graph->exec_idx = while_exec.d + 1;
92
15
  while_exec_info->graph_ref = graph->sub_graphs->rnum;
93
15
  return while_exec;
94
15
}
95
96
ccv_nnc_graph_t* ccv_nnc_graph_from_graph_exec(const ccv_nnc_graph_t* const graph, ccv_nnc_graph_exec_t exec)
97
1
{
98
1
  assert(exec.graph == graph);
99
1
  assert(exec.d < graph->exec_info->rnum);
100
1
  assert(graph->sub_graphs);
101
1
  ccv_nnc_graph_exec_info_t* exec_info = (ccv_nnc_graph_exec_info_t*)ccv_array_get(graph->exec_info, exec.d);
102
1
  assert(exec_info->graph_ref);
103
1
  const int graph_ref = exec_info->graph_ref - 1;
104
1
  assert(graph_ref < graph->sub_graphs->rnum);
105
1
  ccv_nnc_graph_t* sub_graph = *(ccv_nnc_graph_t**)ccv_array_get(graph->sub_graphs, graph_ref);
106
1
  return sub_graph;
107
1
}
108
109
void ccv_nnc_graph_set_while_expr(ccv_nnc_graph_t* const while_graph, const ccv_nnc_graph_while_f while_expr, const void* const while_data, const ccv_nnc_graph_exec_t* const breakpoints, const int breakpoint_size)
110
15
{
111
15
  while_graph->while_data = while_data;
112
15
  while_graph->while_expr = while_expr;
113
15
  assert(breakpoint_size > 0);
114
15
  while_graph->breakpoint_size = breakpoint_size;
115
15
  while_graph->breakpoints = (ccv_nnc_graph_exec_t*)((while_graph->breakpoints) ? 
ccrealloc0
(while_graph->breakpoints, sizeof(ccv_nnc_graph_exec_t) * breakpoint_size)0
:
ccmalloc15
(sizeof(ccv_nnc_graph_exec_t) * breakpoint_size)15
);
116
15
  memcpy(while_graph->breakpoints, breakpoints, sizeof(ccv_nnc_graph_exec_t) * breakpoint_size);
117
15
}
118
119
static void _ccv_nnc_graph_unwrap(const ccv_nnc_graph_t* const graph, const int count)
120
108
{
121
108
  if (!graph->nest_execs)
122
24
    return;
123
84
  int i, j;
124
252
  for (i = 0; 
i < graph->nest_execs->rnum252
;
i++168
)
125
168
  {
126
168
    const ccv_nnc_graph_exec_t* const exec = (const ccv_nnc_graph_exec_t*)ccv_array_get(graph->nest_execs, i);
127
168
    const ccv_nnc_graph_t* const sub_graph = exec->graph;
128
168
    ccv_nnc_graph_exec_info_t* const exec_info = (ccv_nnc_graph_exec_info_t*)ccv_array_get(sub_graph->exec_info, exec->d);
129
624
    for (j = 0; 
j < exec_info->tensor_nest_size624
;
j++456
)
130
456
    {
131
456
      ccv_nnc_graph_tensor_nest_t* const tensor_nest = exec_info->tensor_nests[j];
132
456
      if (!tensor_nest)
133
204
        continue;
134
252
      ccv_nnc_tensor_t* tensor = tensor_nest->tensors[tensor_nest->index];
135
564
      while (
CCV_IS_TENSOR_MULTIVIEW564
(tensor) &&564
136
330
           (((ccv_nnc_tensor_multiview_t*)tensor)->anchor == (intptr_t)graph ||
137
30
          ((ccv_nnc_tensor_multiview_t*)tensor)->anchor == (intptr_t)graph->peer))
138
312
      {
139
312
        ccv_nnc_tensor_multiview_t* mv = (ccv_nnc_tensor_multiview_t*)tensor;
140
312
        const int off = mv->kind;
141
312
        const int mod = mv->repeat;
142
312
        tensor = 
CCV_NNC_MULTIVIEW_DATA312
(mv)[count >= off ?
((count - off) % mod) + off296
:
count16
]; // Unwrap.
143
312
        // If reached the root.
144
312
        if (
!312
CCV_IS_TENSOR_MULTIVIEW312
(tensor))
145
234
          tensor_nest->broadcast_required = 1; // Need to broadcast tensor updates.
146
312
        ++tensor_nest->index;
147
312
        tensor_nest->tensors[tensor_nest->index] = tensor;
148
312
        assert(tensor_nest->index < tensor_nest->count);
149
312
      }
150
252
    }
151
168
  }
152
84
}
153
154
static void _ccv_nnc_graph_rewrap(const ccv_nnc_graph_t* const graph) // Call this method at the end to roll the wrap_ptr back
155
108
{
156
108
  if (!graph->nest_execs)
157
24
    return;
158
84
  int i, j;
159
252
  for (i = 0; 
i < graph->nest_execs->rnum252
;
i++168
)
160
168
  {
161
168
    const ccv_nnc_graph_exec_t* const exec = (const ccv_nnc_graph_exec_t*)ccv_array_get(graph->nest_execs, i);
162
168
    const ccv_nnc_graph_t* const sub_graph = exec->graph;
163
168
    ccv_nnc_graph_exec_info_t* const exec_info = (ccv_nnc_graph_exec_info_t*)ccv_array_get(sub_graph->exec_info, exec->d);
164
624
    for (j = 0; 
j < exec_info->tensor_nest_size624
;
j++456
)
165
456
    {
166
456
      ccv_nnc_graph_tensor_nest_t* const tensor_nest = exec_info->tensor_nests[j];
167
456
      if (!tensor_nest)
168
204
        continue;
169
564
      
while (252
tensor_nest->index > 0 && 564
CCV_IS_TENSOR_MULTIVIEW402
(tensor_nest->tensors[tensor_nest->index - 1]) &&
170
402
          (((ccv_nnc_tensor_multiview_t*)tensor_nest->tensors[tensor_nest->index - 1])->anchor == (intptr_t)graph ||
171
102
           ((ccv_nnc_tensor_multiview_t*)tensor_nest->tensors[tensor_nest->index - 1])->anchor == (intptr_t)graph->peer))
172
312
        --tensor_nest->index;
173
252
    }
174
168
  }
175
84
}
176
177
static void _ccv_nnc_graph_exec_unwrap_io(const ccv_nnc_graph_t* const graph, ccv_nnc_graph_exec_info_t* node)
178
140
{
179
140
  if (!node->tensor_nest_size)
180
0
    return;
181
140
  int i;
182
140
  ccv_nnc_graph_tensor_nest_t** const tensor_nests = node->tensor_nests;
183
506
  for (i = 0; 
i < node->tensor_nest_size506
;
i++366
)
184
366
    
if (366
tensor_nests[i]366
)
185
206
    {
186
206
      assert(tensor_nests[i]->index > 0);
187
206
      ccv_nnc_tensor_multiview_t* mv = (ccv_nnc_tensor_multiview_t*)(tensor_nests[i]->tensors[tensor_nests[i]->index - 1]);
188
206
      assert(CCV_IS_TENSOR_MULTIVIEW(mv));
189
206
      assert(mv->anchor == (intptr_t)graph || mv->anchor == (intptr_t)graph->peer);
190
206
      // Only now set the mv->it, because now this node is about to get executed.
191
206
      mv->it = tensor_nests[i]->tensors[tensor_nests[i]->index];
192
206
      if (tensor_nests[i]->broadcast_required)
193
206
      {
194
206
        // Now broadcast the final pointer.
195
206
        ccv_nnc_tensor_multiview_broadcast(mv);
196
206
        tensor_nests[i]->broadcast_required = 0; // Reset, no need to broadcast.
197
206
      }
198
206
    }
199
345
  for (i = 0; 
i < node->input_size345
;
i++205
)
200
205
    
if (205
tensor_nests[i]205
)
201
75
      node->inputs[i] = tensor_nests[i]->tensors[tensor_nests[i]->index];
202
140
  const int d = node->input_size;
203
225
  for (i = 0; 
i < node->output_size225
;
i++85
)
204
85
    
if (85
tensor_nests[d + i]85
)
205
55
      node->outputs[i] = tensor_nests[d + i]->tensors[tensor_nests[d + i]->index];
206
140
}
207
208
static int _ccv_nnc_graph_while_run(ccv_nnc_graph_t* const graph, const int cmd, ccv_nnc_tensor_t* const* const inputs, const int input_size, ccv_nnc_tensor_t* const* const outputs, const int output_size, ccv_nnc_tensor_tape_t* const tensor_tape, const int flags, const ccv_nnc_graph_exec_t* const sources, const int source_size, const ccv_nnc_graph_exec_t* const destinations, const int destination_size)
209
30
{
210
30
  int i, j;
211
60
  for (i = 0; 
i < source_size60
;
i++30
)
212
30
    
if (30
sources[i].graph != graph30
)
213
0
      return CCV_NNC_EXEC_INVALID;
214
60
  
for (i = 0; 30
i < destination_size60
;
i++30
)
215
30
    
if (30
destinations[i].graph != graph30
)
216
0
      return CCV_NNC_EXEC_INVALID;
217
30
#define visitor(node, idx, d, ...) \
218
237
  
do 237
{ \237
219
237
    /* Broadcast the updates to all subscribed references for input / output, even though at this
220
237
     * time output is not written yet, propagate pointer change is still valid. */ \
221
237
    if (node->tensor_nest_size) \
222
140
      _ccv_nnc_graph_exec_unwrap_io(graph, node); \
223
237
    ccv_nnc_tensor_t** inputs = node->inputs; \
224
237
    ccv_nnc_tensor_t** outputs = inputs + node->input_size; \
225
237
    if (tensor_tape) \
226
24
      ccv_nnc_tensor_tape_io(tensor_tape, graph, inputs, node->input_size, outputs, node->output_size); \
227
237
    if (
node->cmd.cmd == CCV_NNC_GRAPH_FORWARD || 237
node->cmd.cmd == CCV_NNC_GRAPH_BACKWARD220
) \
228
18
    { \
229
18
      ccv_nnc_graph_t* sub_graph = *(ccv_nnc_graph_t**)ccv_array_get(graph->sub_graphs, node->graph_ref - 1); \
230
18
      _ccv_nnc_graph_while_run(sub_graph, node->cmd.cmd, inputs, node->input_size, outputs, node->output_size, tensor_tape, flags, (ccv_nnc_graph_exec_t*)
ccv_array_get18
(sub_graph->sources, 0), sub_graph->sources->rnum, (ccv_nnc_graph_exec_t*)
ccv_array_get18
(sub_graph->destinations, 0), sub_graph->destinations->rnum); \
231
219
    } else { \
232
219
      PRINT(CCV_CLI_VERBOSE, "%s [%d, %d]: [%d] -> [%d]\n", ccv_nnc_cmd_name(node->cmd.cmd), idx, d, node->input_size, node->output_size); \
233
486
      for (i = 0; 
i < node->input_size486
;
i++267
) \
234
267
        PRINT(CCV_CLI_VERBOSE, "|-> %d. %p (%p)\n", i + 1, inputs[i], (inputs[i] ? inputs[i]->data.u8 : 0)); \
235
335
      for (i = 0; 
i < node->output_size335
;
i++116
) \
236
116
        PRINT(CCV_CLI_VERBOSE, "|<- %d. %p (%p)\n", i + 1, outputs[i], (outputs[i] ? outputs[i]->data.u8 : 0)); \
237
219
      ccv_nnc_cmd_exec(node->cmd, node->hint, flags, inputs, node->input_size, outputs, node->output_size, 0); \
238
219
    } \
239
237
  } while (0)
240
30
  
if (30
graph->while_expr30
)
241
18
  {
242
18
    uint64_t count = 0;
243
18
    ccv_nnc_tensor_t count_tensor = ccv_nnc_tensor(&count, ONE_CPU_TENSOR(1, 1, 1), 0);
244
18
    ccv_nnc_tensor_t* special_tensors[] = { &count_tensor };
245
18
    // This is a forward while loop. Backward while loop will just consult its peering part.
246
18
    if (cmd == CCV_NNC_GRAPH_FORWARD)
247
17
    {
248
17
      ccv_array_t* follows = ccv_array_new(sizeof(ccv_nnc_graph_exec_t), graph->breakpoint_size, 0);
249
34
      for (i = 0; 
i < graph->breakpoint_size34
;
i++17
)
250
17
      {
251
17
        const ccv_nnc_graph_exec_info_t* const exec_info = (const ccv_nnc_graph_exec_info_t*)ccv_array_get(graph->exec_info, graph->breakpoints->d);
252
17
        if (exec_info->outgoings)
253
34
          
for (j = 0; 17
j < exec_info->outgoings->rnum34
;
j++17
)
254
17
          {
255
17
            const ccv_nnc_graph_exec_t exec = {
256
17
              .d = *(int*)ccv_array_get(exec_info->outgoings, j),
257
17
              .graph = graph,
258
17
            };
259
17
            ccv_array_push(follows, &exec);
260
17
          }
261
17
      }
262
85
      for (;; ++count)
263
102
      {
264
102
        graph->while_count = count;
265
102
        if (tensor_tape)
266
6
          ccv_nnc_tensor_tape_set_while_count(tensor_tape, graph, count);
267
102
        _ccv_nnc_graph_unwrap(graph, count);
268
102
        
CCV_NNC_GRAPH_VISIT102
(graph, (ccv_nnc_graph_exec_info_t*)ccv_array_get(graph->exec_info, 0), graph->exec_info->rnum, sources, source_size, graph->breakpoints, graph->breakpoint_size,
visitor102
);
269
102
        // Reached breakpoints, now check the breakpoint, if not met, break out.
270
102
        if (!graph->while_expr(special_tensors, 1, inputs, input_size, outputs, output_size, graph->while_data))
271
17
        {
272
17
          _ccv_nnc_graph_rewrap(graph);
273
17
          break;
274
17
        }
275
85
        
if (85
follows->rnum > 085
)
276
105
          
CCV_NNC_GRAPH_VISIT85
(graph, (ccv_nnc_graph_exec_info_t*)ccv_array_get(graph->exec_info, 0), graph->exec_info->rnum, (ccv_nnc_graph_exec_t*)ccv_array_get(follows, 0), follows->rnum, destinations, destination_size,
visitor105
);
277
85
        _ccv_nnc_graph_rewrap(graph);
278
85
      }
279
17
      ccv_array_free(follows);
280
1
    } else {
281
1
      // For backward graph, no need to evaluate the while expr.
282
1
      assert(cmd == CCV_NNC_GRAPH_BACKWARD);
283
1
      assert(graph->peer);
284
1
      assert(tensor_tape);
285
1
      graph->while_count = count = ccv_nnc_tensor_tape_while_count(tensor_tape, graph);
286
1
      _ccv_nnc_graph_unwrap(graph, count);
287
1
      
CCV_NNC_GRAPH_VISIT1
(graph, (ccv_nnc_graph_exec_info_t*)ccv_array_get(graph->exec_info, 0), graph->exec_info->rnum, graph->breakpoints, graph->breakpoint_size, destinations, destination_size,
visitor1
);
288
1
      _ccv_nnc_graph_rewrap(graph);
289
1
      if (count > 0)
290
5
        
do 1
{5
291
5
          graph->while_count = --count;
292
5
          _ccv_nnc_graph_unwrap(graph, count);
293
10
          
CCV_NNC_GRAPH_VISIT5
(graph, (ccv_nnc_graph_exec_info_t*)ccv_array_get(graph->exec_info, 0), graph->exec_info->rnum, sources, source_size, destinations, destination_size,
visitor10
);
294
5
          _ccv_nnc_graph_rewrap(graph);
295
5
        } while (count > 0);
296
1
    }
297
12
  } else {
298
12
    graph->while_count = 0;
299
19
    
CCV_NNC_GRAPH_VISIT12
(graph, (ccv_nnc_graph_exec_info_t*)ccv_array_get(graph->exec_info, 0), graph->exec_info->rnum, sources, source_size, destinations, destination_size,
visitor19
);
300
12
  }
301
30
  return CCV_NNC_EXEC_SUCCESS;
302
30
}
303
304
int ccv_nnc_graph_while_run(ccv_nnc_graph_t* const graph, ccv_nnc_tensor_tape_t* const tensor_tape, const int flags, const ccv_nnc_graph_exec_t* const sources, const int source_size, const ccv_nnc_graph_exec_t* const destinations, const int destination_size)
305
12
{
306
12
  return _ccv_nnc_graph_while_run(graph, 0, 0, 0, 0, 0, tensor_tape, flags, sources, source_size, destinations, destination_size);
307
12
}