Coverage Report

Created: 2026-05-04 15:30

next uncovered line (L), next uncovered region (R), next uncovered branch (B)
/home/liu/actions-runner/_work/ccv/ccv/test/int/nnc/parallel.tests.c
Line
Count
Source
1
#include "case.h"
2
#include "ccv_case.h"
3
#include "ccv_nnc_case.h"
4
#include <ccv.h>
5
#include <nnc/ccv_nnc.h>
6
#include <nnc/ccv_nnc_easy.h>
7
#include <3rdparty/dsfmt/dSFMT.h>
8
9
TEST_SETUP()
10
{
11
  ccv_nnc_init();
12
}
13
14
TEST_CASE("cnnp send to devices and all-to-all")
15
1
{
16
1
  GUARD_ELSE_RETURN(ccv_nnc_cmd_ok(CCV_NNC_COMM_ALL_TO_ALL_FORWARD, CCV_NNC_BACKEND_GPU_NCCL));
17
1
  const int device_count = ccv_nnc_device_count(CCV_STREAM_CONTEXT_GPU);
18
1
  GUARD_ELSE_RETURN(device_count > 1);
19
1
  const int chunk = 5;
20
1
  const ccv_cnnp_model_io_t x = ccv_cnnp_input();
21
1
  ccv_cnnp_model_io_t chunks = ccv_cnnp_model_apply(ccv_cnnp_chunk(device_count, 0, "chunk"), MODEL_IO_LIST(x));
22
1
  ccv_cnnp_model_io_t sent[device_count];
23
1
  int i, j, k;
24
5
  for (i = 0; i < device_count; 
i++4
)
25
4
  {
26
4
    ccv_cnnp_model_io_t chunk_i = ccv_cnnp_model_apply(ccv_cnnp_extract(i, 0), &chunks, 1);
27
4
    sent[i] = ccv_cnnp_model_apply(ccv_cnnp_send(i, 0), &chunk_i, 1);
28
4
  }
29
1
  ccv_cnnp_model_io_t exchanged = ccv_cnnp_model_apply(ccv_cnnp_all_to_all(device_count, 1, "all_to_all"), sent, device_count);
30
1
  ccv_cnnp_model_t* const model = ccv_cnnp_model_new(MODEL_IO_LIST(x), &exchanged, 1, 0, "send_all_to_all");
31
1
  ccv_nnc_tensor_param_t input_params = GPU_TENSOR_NHWC(000, 32F, device_count, device_count * chunk);
32
1
  ccv_cnnp_model_compile(model, TENSOR_PARAM_LIST(input_params), CMD_NOOP(), CMD_NOOP());
33
1
  ccv_nnc_tensor_param_t output_params[device_count];
34
1
  ccv_cnnp_model_tensor_auto(model, output_params, device_count);
35
1
  ccv_nnc_tensor_t* const h_input = ccv_nnc_tensor_new(0, CPU_TENSOR_NHWC(32F, device_count, device_count * chunk), 0);
36
5
  for (i = 0; i < device_count; 
i++4
)
37
84
    
for (j = 0; 4
j < device_count * chunk; 
j++80
)
38
80
      h_input->data.f32[i * device_count * chunk + j] = (float)(i * 1000 + j);
39
1
  ccv_nnc_tensor_t* const d_input = ccv_nnc_tensor_new(0, input_params, 0);
40
1
  ccv_nnc_cmd_exec(CMD_DATA_TRANSFER_FORWARD(), ccv_nnc_no_hint, 0, TENSOR_LIST(h_input), TENSOR_LIST(d_input), 0);
41
1
  ccv_nnc_tensor_t* outputs[device_count];
42
5
  for (i = 0; i < device_count; 
i++4
)
43
4
  {
44
4
    REQUIRE_EQ(CCV_TENSOR_GET_DEVICE_ID(output_params[i].type), i, "output device should match all-to-all rank");
45
4
    outputs[i] = ccv_nnc_tensor_new(0, output_params[i], 0);
46
4
  }
47
1
  ccv_cnnp_model_evaluate(model, (ccv_cnnp_evaluate_param_t){}, TENSOR_LIST(d_input), outputs, device_count, 0, 0);
48
5
  for (j = 0; j < device_count; 
j++4
)
49
4
  {
50
4
    ccv_nnc_tensor_t* const h_output = ccv_nnc_tensor_new(0, CPU_TENSOR_NHWC(32F, 1, device_count * chunk), 0);
51
4
    ccv_nnc_cmd_exec(CMD_DATA_TRANSFER_FORWARD(), ccv_nnc_no_hint, 0, TENSOR_LIST(outputs[j]), TENSOR_LIST(h_output), 0);
52
20
    for (i = 0; i < device_count; 
i++16
)
53
96
      
for (k = 0; 16
k < chunk; 
k++80
)
54
80
      {
55
80
        const float expected = h_input->data.f32[i * device_count * chunk + j * chunk + k];
56
80
        REQUIRE_EQ_WITH_TOLERANCE(h_output->data.f32[i * chunk + k], expected, 1e-5, "all-to-all output should match expected exchange");
57
80
      }
58
4
    ccv_nnc_tensor_free(h_output);
59
4
  }
60
5
  
for (i = 0; 1
i < device_count; 
i++4
)
61
4
    ccv_nnc_tensor_free(outputs[i]);
62
1
  ccv_nnc_tensor_free(d_input);
63
1
  ccv_nnc_tensor_free(h_input);
64
1
  ccv_cnnp_model_free(model);
65
1
}
66
67
TEST_CASE("cnnp replicated dense feeds all-to-all")
68
1
{
69
1
  GUARD_ELSE_RETURN(ccv_nnc_cmd_ok(CCV_NNC_GEMM_FORWARD, CCV_NNC_BACKEND_GPU_CUBLAS));
70
1
  GUARD_ELSE_RETURN(ccv_nnc_cmd_ok(CCV_NNC_COMM_ALL_TO_ALL_FORWARD, CCV_NNC_BACKEND_GPU_NCCL));
71
1
  const int device_count = ccv_nnc_device_count(CCV_STREAM_CONTEXT_GPU);
72
1
  GUARD_ELSE_RETURN(device_count > 1);
73
1
  const int rank_count = device_count;
74
1
  const int rows_per_rank = 2;
75
1
  const int input_dim = 4;
76
1
  const int output_chunk = 3;
77
1
  const int output_dim = rank_count * output_chunk;
78
1
  const ccv_cnnp_model_io_t x = ccv_cnnp_input();
79
1
  ccv_cnnp_model_io_t chunks = ccv_cnnp_model_apply(ccv_cnnp_chunk(rank_count, 0, "chunk"), MODEL_IO_LIST(x));
80
1
  ccv_cnnp_model_io_t sent[rank_count];
81
1
  int i, j, k, r;
82
5
  for (i = 0; i < rank_count; 
i++4
)
83
4
  {
84
4
    ccv_cnnp_model_io_t chunk_i = ccv_cnnp_model_apply(ccv_cnnp_extract(i, 0), &chunks, 1);
85
4
    sent[i] = ccv_cnnp_model_apply(ccv_cnnp_send(i, 0), &chunk_i, 1);
86
4
  }
87
1
  ccv_cnnp_model_t* const dense = ccv_cnnp_dense(output_dim, 1, 0, 1, "dense");
88
1
  ccv_cnnp_model_io_t dense_outputs = ccv_cnnp_model_apply(ccv_cnnp_replicated(dense, rank_count, 1, "replicated_dense"), sent, rank_count);
89
1
  ccv_cnnp_model_io_t exchanged = ccv_cnnp_model_apply(ccv_cnnp_all_to_all(rank_count, 1, "all_to_all"), MODEL_IO_LIST(dense_outputs));
90
1
  ccv_cnnp_model_t* const model = ccv_cnnp_model_new(MODEL_IO_LIST(x), &exchanged, 1, 0, "replicated_dense_all_to_all");
91
1
  ccv_nnc_tensor_param_t input_params = GPU_TENSOR_NHWC(000, 32F, rank_count * rows_per_rank, input_dim);
92
1
  ccv_cnnp_model_compile(model, TENSOR_PARAM_LIST(input_params), CMD_NOOP(), CMD_NOOP());
93
1
  ccv_nnc_tensor_t* const h_input = ccv_nnc_tensor_new(0, CPU_TENSOR_NHWC(32F, rank_count * rows_per_rank, input_dim), 0);
94
9
  for (i = 0; i < rank_count * rows_per_rank; 
i++8
)
95
40
    
for (j = 0; 8
j < input_dim; 
j++32
)
96
32
      h_input->data.f32[i * input_dim + j] = (float)(i * 10 + j + 1);
97
1
  ccv_nnc_tensor_t* const h_weight = ccv_nnc_tensor_new(0, CPU_TENSOR_NHWC(32F, output_dim, input_dim), 0);
98
13
  for (i = 0; i < output_dim; 
i++12
)
99
60
    
for (j = 0; 12
j < input_dim; 
j++48
)
100
48
      h_weight->data.f32[i * input_dim + j] = (float)((i + 1) * 0.25 + (j + 1) * 0.125);
101
1
  ccv_cnnp_model_set_parameter(model, ccv_cnnp_model_parameters(dense, ALL_PARAMETERS, 0), h_weight);
102
1
  ccv_nnc_tensor_t* const d_input = ccv_nnc_tensor_new(0, input_params, 0);
103
1
  ccv_nnc_cmd_exec(CMD_DATA_TRANSFER_FORWARD(), ccv_nnc_no_hint, 0, TENSOR_LIST(h_input), TENSOR_LIST(d_input), 0);
104
1
  ccv_nnc_tensor_param_t output_params[rank_count];
105
1
  ccv_cnnp_model_tensor_auto(model, output_params, rank_count);
106
1
  ccv_nnc_tensor_t* outputs[rank_count];
107
5
  for (i = 0; i < rank_count; 
i++4
)
108
4
  {
109
4
    REQUIRE_EQ(CCV_TENSOR_GET_DEVICE_ID(output_params[i].type), i, "output device should match all-to-all rank");
110
4
    outputs[i] = ccv_nnc_tensor_new(0, output_params[i], 0);
111
4
  }
112
1
  ccv_cnnp_model_evaluate(model, (ccv_cnnp_evaluate_param_t){}, TENSOR_LIST(d_input), outputs, rank_count, 0, 0);
113
5
  for (j = 0; j < rank_count; 
j++4
)
114
4
  {
115
4
    ccv_nnc_tensor_t* const h_output = ccv_nnc_tensor_new(0, CPU_TENSOR_NHWC(32F, rows_per_rank, output_dim), 0);
116
4
    ccv_nnc_cmd_exec(CMD_DATA_TRANSFER_FORWARD(), ccv_nnc_no_hint, 0, TENSOR_LIST(outputs[j]), TENSOR_LIST(h_output), 0);
117
12
    for (r = 0; r < rows_per_rank; 
r++8
)
118
40
      
for (i = 0; 8
i < rank_count; 
i++32
)
119
128
        
for (k = 0; 32
k < output_chunk; 
k++96
)
120
96
        {
121
96
          float expected = 0;
122
96
          const int input_row = i * rows_per_rank + r;
123
96
          const int output_channel = j * output_chunk + k;
124
96
          int l;
125
480
          for (l = 0; l < input_dim; 
l++384
)
126
384
            expected += h_input->data.f32[input_row * input_dim + l] * h_weight->data.f32[output_channel * input_dim + l];
127
96
          REQUIRE_EQ_WITH_TOLERANCE(h_output->data.f32[r * output_dim + i * output_chunk + k], expected, 1e-4, "replicated dense all-to-all output should match CPU reference");
128
96
        }
129
4
    ccv_nnc_tensor_free(h_output);
130
4
  }
131
5
  
for (i = 0; 1
i < rank_count; 
i++4
)
132
4
    ccv_nnc_tensor_free(outputs[i]);
133
1
  ccv_nnc_tensor_free(d_input);
134
1
  ccv_nnc_tensor_free(h_weight);
135
1
  ccv_nnc_tensor_free(h_input);
136
1
  ccv_cnnp_model_free(model);
137
1
}
138
139
TEST_CASE("cnnp replicated embedding matches CPU reference")
140
1
{
141
1
  GUARD_ELSE_RETURN(ccv_nnc_cmd_ok(CCV_NNC_INDEX_SELECT_FORWARD, CCV_NNC_BACKEND_GPU_REF));
142
1
  const int device_count = ccv_nnc_device_count(CCV_STREAM_CONTEXT_GPU);
143
1
  GUARD_ELSE_RETURN(device_count > 1);
144
1
  const int rank_count = device_count;
145
1
  const int rows_per_rank = 3;
146
1
  const int vocab_size = 11;
147
1
  const int embed_size = 5;
148
1
  const ccv_cnnp_model_io_t x = ccv_cnnp_input();
149
1
  ccv_cnnp_model_io_t chunks = ccv_cnnp_model_apply(ccv_cnnp_chunk(rank_count, 0, "chunk"), MODEL_IO_LIST(x));
150
1
  ccv_cnnp_model_io_t sent[rank_count];
151
1
  int i, j, r;
152
5
  for (i = 0; i < rank_count; 
i++4
)
153
4
  {
154
4
    ccv_cnnp_model_io_t chunk_i = ccv_cnnp_model_apply(ccv_cnnp_extract(i, 0), &chunks, 1);
155
4
    sent[i] = ccv_cnnp_model_apply(ccv_cnnp_send(i, 0), &chunk_i, 1);
156
4
  }
157
1
  ccv_cnnp_model_t* const embedding = ccv_cnnp_embedding(CCV_32F, vocab_size, embed_size, 1, "embedding");
158
1
  ccv_cnnp_model_io_t embedding_outputs = ccv_cnnp_model_apply(ccv_cnnp_replicated(embedding, rank_count, 1, "replicated_embedding"), sent, rank_count);
159
1
  ccv_cnnp_model_t* const model = ccv_cnnp_model_new(MODEL_IO_LIST(x), MODEL_IO_LIST(embedding_outputs), 0, "replicated_embedding");
160
1
  ccv_nnc_tensor_param_t input_params = GPU_TENSOR_NHWC(000, 32S, rank_count * rows_per_rank);
161
1
  ccv_cnnp_model_compile(model, TENSOR_PARAM_LIST(input_params), CMD_NOOP(), CMD_NOOP());
162
1
  ccv_nnc_tensor_t* const h_input = ccv_nnc_tensor_new(0, CPU_TENSOR_NHWC(32S, rank_count * rows_per_rank), 0);
163
13
  for (i = 0; i < rank_count * rows_per_rank; 
i++12
)
164
12
    h_input->data.i32[i] = (i * 3 + 1) % vocab_size;
165
1
  ccv_nnc_tensor_t* const h_vocab = ccv_nnc_tensor_new(0, CPU_TENSOR_NHWC(32F, vocab_size, embed_size), 0);
166
12
  for (i = 0; i < vocab_size; 
i++11
)
167
66
    
for (j = 0; 11
j < embed_size; 
j++55
)
168
55
      h_vocab->data.f32[i * embed_size + j] = (float)(i * 0.5 + j * 0.25 + 1);
169
1
  ccv_cnnp_model_set_parameter(model, ccv_cnnp_model_parameters(embedding, ALL_PARAMETERS, 0), h_vocab);
170
1
  ccv_nnc_tensor_t* const d_input = ccv_nnc_tensor_new(0, input_params, 0);
171
1
  ccv_nnc_cmd_exec(CMD_DATA_TRANSFER_FORWARD(), ccv_nnc_no_hint, 0, TENSOR_LIST(h_input), TENSOR_LIST(d_input), 0);
172
1
  ccv_nnc_tensor_param_t output_params[rank_count];
173
1
  ccv_cnnp_model_tensor_auto(model, output_params, rank_count);
174
1
  ccv_nnc_tensor_t* outputs[rank_count];
175
5
  for (i = 0; i < rank_count; 
i++4
)
176
4
  {
177
4
    REQUIRE_EQ(CCV_TENSOR_GET_DEVICE_ID(output_params[i].type), i, "output device should match replicated rank");
178
4
    outputs[i] = ccv_nnc_tensor_new(0, output_params[i], 0);
179
4
  }
180
1
  ccv_cnnp_model_evaluate(model, (ccv_cnnp_evaluate_param_t){}, TENSOR_LIST(d_input), outputs, rank_count, 0, 0);
181
5
  for (i = 0; i < rank_count; 
i++4
)
182
4
  {
183
4
    ccv_nnc_tensor_t* const h_output = ccv_nnc_tensor_new(0, CPU_TENSOR_NHWC(32F, rows_per_rank, embed_size), 0);
184
4
    ccv_nnc_cmd_exec(CMD_DATA_TRANSFER_FORWARD(), ccv_nnc_no_hint, 0, TENSOR_LIST(outputs[i]), TENSOR_LIST(h_output), 0);
185
16
    for (r = 0; r < rows_per_rank; 
r++12
)
186
12
    {
187
12
      const int token = h_input->data.i32[i * rows_per_rank + r];
188
72
      for (j = 0; j < embed_size; 
j++60
)
189
60
        REQUIRE_EQ_WITH_TOLERANCE(h_output->data.f32[r * embed_size + j], h_vocab->data.f32[token * embed_size + j], 1e-5, "replicated embedding output should match CPU reference");
190
12
    }
191
4
    ccv_nnc_tensor_free(h_output);
192
4
  }
193
5
  
for (i = 0; 1
i < rank_count; 
i++4
)
194
4
    ccv_nnc_tensor_free(outputs[i]);
195
1
  ccv_nnc_tensor_free(d_input);
196
1
  ccv_nnc_tensor_free(h_vocab);
197
1
  ccv_nnc_tensor_free(h_input);
198
1
  ccv_cnnp_model_free(model);
199
1
}
200
201
TEST_CASE("cnnp replicated layer norm matches CPU reference")
202
1
{
203
1
  GUARD_ELSE_RETURN(ccv_nnc_cmd_ok(CCV_NNC_LAYER_NORM_FORWARD, CCV_NNC_BACKEND_GPU_CUDNN));
204
1
  const int device_count = ccv_nnc_device_count(CCV_STREAM_CONTEXT_GPU);
205
1
  GUARD_ELSE_RETURN(device_count > 1);
206
1
  const int rank_count = device_count;
207
1
  const int rows_per_rank = 2;
208
1
  const int cols = 6;
209
1
  const float epsilon = 1e-5;
210
1
  const int axis[] = {1};
211
1
  const ccv_cnnp_model_io_t x = ccv_cnnp_input();
212
1
  ccv_cnnp_model_io_t chunks = ccv_cnnp_model_apply(ccv_cnnp_chunk(rank_count, 0, "chunk"), MODEL_IO_LIST(x));
213
1
  ccv_cnnp_model_io_t sent[rank_count];
214
1
  int i, j, r;
215
5
  for (i = 0; i < rank_count; 
i++4
)
216
4
  {
217
4
    ccv_cnnp_model_io_t chunk_i = ccv_cnnp_model_apply(ccv_cnnp_extract(i, 0), &chunks, 1);
218
4
    sent[i] = ccv_cnnp_model_apply(ccv_cnnp_send(i, 0), &chunk_i, 1);
219
4
  }
220
1
  ccv_cnnp_model_t* const layer_norm = ccv_cnnp_layer_norm(epsilon, axis, 1, 1, 1, "layer_norm");
221
1
  ccv_cnnp_model_io_t normalized = ccv_cnnp_model_apply(ccv_cnnp_replicated(layer_norm, rank_count, 1, "replicated_layer_norm"), sent, rank_count);
222
1
  ccv_cnnp_model_t* const model = ccv_cnnp_model_new(MODEL_IO_LIST(x), MODEL_IO_LIST(normalized), 0, "replicated_layer_norm");
223
1
  ccv_nnc_tensor_param_t input_params = GPU_TENSOR_NHWC(000, 32F, rank_count * rows_per_rank, cols);
224
1
  ccv_cnnp_model_compile(model, TENSOR_PARAM_LIST(input_params), CMD_NOOP(), CMD_NOOP());
225
1
  ccv_nnc_tensor_t* const h_input = ccv_nnc_tensor_new(0, CPU_TENSOR_NHWC(32F, rank_count * rows_per_rank, cols), 0);
226
9
  for (i = 0; i < rank_count * rows_per_rank; 
i++8
)
227
56
    
for (j = 0; 8
j < cols; 
j++48
)
228
48
      h_input->data.f32[i * cols + j] = (float)((i + 1) * 0.75 + (j + 1) * 0.5);
229
1
  ccv_nnc_tensor_t* const h_scale = ccv_nnc_tensor_new(0, CPU_TENSOR_NHWC(32F, 1, cols), 0);
230
1
  ccv_nnc_tensor_t* const h_bias = ccv_nnc_tensor_new(0, CPU_TENSOR_NHWC(32F, 1, cols), 0);
231
7
  for (j = 0; j < cols; 
j++6
)
232
6
  {
233
6
    h_scale->data.f32[j] = 0.5f + j * 0.125f;
234
6
    h_bias->data.f32[j] = -0.75f + j * 0.2f;
235
6
  }
236
1
  ccv_cnnp_model_set_parameter(model, ccv_cnnp_model_parameters(layer_norm, ALL_PARAMETERS, 0), h_scale);
237
1
  ccv_cnnp_model_set_parameter(model, ccv_cnnp_model_parameters(layer_norm, ALL_PARAMETERS, 1), h_bias);
238
1
  ccv_nnc_tensor_t* const d_input = ccv_nnc_tensor_new(0, input_params, 0);
239
1
  ccv_nnc_cmd_exec(CMD_DATA_TRANSFER_FORWARD(), ccv_nnc_no_hint, 0, TENSOR_LIST(h_input), TENSOR_LIST(d_input), 0);
240
1
  ccv_nnc_tensor_param_t output_params[rank_count];
241
1
  ccv_cnnp_model_tensor_auto(model, output_params, rank_count);
242
1
  ccv_nnc_tensor_t* outputs[rank_count];
243
5
  for (i = 0; i < rank_count; 
i++4
)
244
4
  {
245
4
    REQUIRE_EQ(CCV_TENSOR_GET_DEVICE_ID(output_params[i].type), i, "output device should match replicated rank");
246
4
    outputs[i] = ccv_nnc_tensor_new(0, output_params[i], 0);
247
4
  }
248
1
  ccv_cnnp_model_evaluate(model, (ccv_cnnp_evaluate_param_t){}, TENSOR_LIST(d_input), outputs, rank_count, 0, 0);
249
5
  for (i = 0; i < rank_count; 
i++4
)
250
4
  {
251
4
    ccv_nnc_tensor_t* const h_output = ccv_nnc_tensor_new(0, CPU_TENSOR_NHWC(32F, rows_per_rank, cols), 0);
252
4
    ccv_nnc_cmd_exec(CMD_DATA_TRANSFER_FORWARD(), ccv_nnc_no_hint, 0, TENSOR_LIST(outputs[i]), TENSOR_LIST(h_output), 0);
253
12
    for (r = 0; r < rows_per_rank; 
r++8
)
254
8
    {
255
8
      const int row = i * rows_per_rank + r;
256
8
      float mean = 0;
257
56
      for (j = 0; j < cols; 
j++48
)
258
48
        mean += h_input->data.f32[row * cols + j];
259
8
      mean /= cols;
260
8
      float variance = 0;
261
56
      for (j = 0; j < cols; 
j++48
)
262
48
      {
263
48
        const float centered = h_input->data.f32[row * cols + j] - mean;
264
48
        variance += centered * centered;
265
48
      }
266
8
      variance /= cols;
267
8
      const float inv_std = 1.0 / sqrtf(variance + epsilon);
268
56
      for (j = 0; j < cols; 
j++48
)
269
48
      {
270
48
        const float centered = h_input->data.f32[row * cols + j] - mean;
271
48
        const float expected = centered * inv_std * h_scale->data.f32[j] + h_bias->data.f32[j];
272
48
        REQUIRE_EQ_WITH_TOLERANCE(h_output->data.f32[r * cols + j], expected, 1e-4, "replicated layer norm output should match CPU reference");
273
48
      }
274
8
    }
275
4
    ccv_nnc_tensor_free(h_output);
276
4
  }
277
5
  
for (i = 0; 1
i < rank_count; 
i++4
)
278
4
    ccv_nnc_tensor_free(outputs[i]);
279
1
  ccv_nnc_tensor_free(d_input);
280
1
  ccv_nnc_tensor_free(h_bias);
281
1
  ccv_nnc_tensor_free(h_scale);
282
1
  ccv_nnc_tensor_free(h_input);
283
1
  ccv_cnnp_model_free(model);
284
1
}
285
286
TEST_CASE("schedule symbolic graph to data parallel with broadcast and reduce")
287
1
{
288
1
  GUARD_ELSE_RETURN(ccv_nnc_cmd_ok(CCV_NNC_CONVOLUTION_FORWARD, CCV_NNC_BACKEND_GPU_CUDNN));
289
1
  GUARD_ELSE_RETURN(ccv_nnc_device_count(CCV_STREAM_CONTEXT_GPU) >= 2);
290
1
  ccv_nnc_tensor_t* updated[4];
291
1
  ccv_nnc_tensor_t* cpu_inputs[2];
292
1
  ccv_nnc_tensor_t* cpu_fits[2];
293
1
  ccv_nnc_tensor_t* w1_tensor = ccv_nnc_tensor_new(0, CPU_TENSOR_NCHW(32F, 8, 3, 5, 5), 0);
294
1
  ccv_nnc_tensor_t* w3_tensor = ccv_nnc_tensor_new(0, CPU_TENSOR_NCHW(32F, 8, 8, 5, 5), 0);
295
1
  {
296
1
    ccv_nnc_symbolic_graph_t* const symbolic_graph = ccv_nnc_symbolic_graph_new();
297
1
    const ccv_nnc_tensor_symbol_t x = ccv_nnc_tensor_symbol_new(symbolic_graph, GPU_TENSOR_NCHW(000, 32F, 16, 3, 32, 32), 0);
298
1
    const ccv_nnc_tensor_symbol_t w1 = ccv_nnc_tensor_symbol_new(symbolic_graph, GPU_TENSOR_NCHW(000, 32F, 8, 3, 5, 5), 0);
299
1
    const ccv_nnc_tensor_symbol_t bias1 = ccv_nnc_tensor_symbol_new(symbolic_graph, GPU_TENSOR_NCHW(000, 32F, 8), 0);
300
1
    const ccv_nnc_tensor_symbol_t y1 = ccv_nnc_tensor_symbol_new(symbolic_graph, GPU_TENSOR_NCHW(000, 32F, 16, 8, 32, 32), 0);
301
1
    const ccv_nnc_graph_exec_symbol_t conv1 = ccv_nnc_graph_exec_symbol_new(symbolic_graph, CMD_CONVOLUTION_FORWARD(1, 8, 5, 5, 32), TENSOR_SYMBOL_LIST(x, w1, bias1), TENSOR_SYMBOL_LIST(y1), "conv1");
302
1
    ccv_nnc_graph_exec_symbol_set_hint(symbolic_graph, conv1, HINT((1, 1), (2, 2)));
303
1
    const ccv_nnc_tensor_symbol_t y2 = ccv_nnc_tensor_symbol_new(symbolic_graph, GPU_TENSOR_NCHW(000, 32F, 16, 8, 16, 16), 0);
304
1
    const ccv_nnc_graph_exec_symbol_t avg2 = ccv_nnc_graph_exec_symbol_new(symbolic_graph, CMD_AVERAGE_POOL_FORWARD(2, 2), TENSOR_SYMBOL_LIST(y1), TENSOR_SYMBOL_LIST(y2), "avg2");
305
1
    ccv_nnc_graph_exec_symbol_set_hint(symbolic_graph, avg2, HINT((2, 2)));
306
1
    const ccv_nnc_tensor_symbol_t w3 = ccv_nnc_tensor_symbol_new(symbolic_graph, GPU_TENSOR_NCHW(000, 32F, 8, 8, 5, 5), 0);
307
1
    const ccv_nnc_tensor_symbol_t bias3 = ccv_nnc_tensor_symbol_new(symbolic_graph, GPU_TENSOR_NCHW(000, 32F, 8), 0);
308
1
    const ccv_nnc_tensor_symbol_t y3 = ccv_nnc_tensor_symbol_new(symbolic_graph, GPU_TENSOR_NCHW(000, 32F, 16, 8, 8, 8), 0);
309
1
    const ccv_nnc_graph_exec_symbol_t conv3 = ccv_nnc_graph_exec_symbol_new(symbolic_graph, CMD_CONVOLUTION_FORWARD(1, 8, 5, 5, 32), TENSOR_SYMBOL_LIST(y2, w3, bias3), TENSOR_SYMBOL_LIST(y3), "conv3");
310
1
    ccv_nnc_graph_exec_symbol_set_hint(symbolic_graph, conv3, HINT((2, 2), (2, 2)));
311
1
    const ccv_nnc_tensor_symbol_t y4 = ccv_nnc_tensor_symbol_new(symbolic_graph, GPU_TENSOR_NCHW(000, 32F, 16, 8, 1, 1), 0);
312
1
    ccv_nnc_graph_exec_symbol_new(symbolic_graph, CMD_AVERAGE_POOL_FORWARD(8, 8), TENSOR_SYMBOL_LIST(y3), TENSOR_SYMBOL_LIST(y4), "avg4");
313
1
    const ccv_nnc_tensor_symbol_t y4a = ccv_nnc_tensor_symbol_alias_new(symbolic_graph, y4, ccv_nnc_no_ofs, DIM_ALLOC(8, 1, 1, 1), GPU_TENSOR_NCHW(000, 32F, 16, 8), 0);
314
1
    const ccv_nnc_tensor_symbol_t label = ccv_nnc_tensor_symbol_new(symbolic_graph, GPU_TENSOR_NCHW(000, 32F, 16), "label");
315
1
    const ccv_nnc_tensor_symbol_t y5 = ccv_nnc_tensor_symbol_new(symbolic_graph, GPU_TENSOR_NCHW(000, 32F, 16, 8), "y5");
316
1
    const ccv_nnc_tensor_symbol_t loss = ccv_nnc_tensor_symbol_new(symbolic_graph, GPU_TENSOR_NCHW(000, 32F, 16), "loss");
317
1
    ccv_nnc_graph_exec_symbol_new(symbolic_graph, CMD_SOFTMAX_CROSSENTROPY_FORWARD(), TENSOR_SYMBOL_LIST(y4a, label), TENSOR_SYMBOL_LIST(loss, y5), "softmax crossentropy");
318
1
    ccv_nnc_graph_exec_symbol_autogen(symbolic_graph, 0, 0, CCV_NNC_AUTOGEN_ALL_EXECS | CCV_NNC_AUTOGEN_SOURCES_AND_DESTINATIONS);
319
1
    ccv_nnc_tensor_symbol_t updated_params[4];
320
1
    ccv_nnc_tensor_symbol_t gradients[4];
321
1
    const int saved_aux_size = ccv_nnc_minimizer_saved_aux_size(CMD_SGD_FORWARD(0, 0.001, 1, 0.99, 0.9, 0.9));
322
1
    ccv_nnc_tensor_symbol_map_t saved_aux[saved_aux_size * 4];
323
1
    ccv_nnc_graph_exec_symbol_t updated_execs[4];
324
1
    ccv_nnc_symbolic_graph_minimize(symbolic_graph, CMD_SGD_FORWARD(0, 0.001, 1, 0.99, 0.9, 0.9), TENSOR_SYMBOL_LIST(loss), TENSOR_SYMBOL_LIST(w1, bias1, w3, bias3), 0, 0, SYMBOLIC_GRAPH_SOURCES(symbolic_graph), SYMBOLIC_GRAPH_DESTINATIONS(symbolic_graph), gradients, updated_params, saved_aux, updated_execs);
325
1
    const ccv_nnc_tensor_symbol_t dloss = ccv_nnc_tensor_symbol_for_backward(symbolic_graph, loss);
326
1
    ccv_nnc_graph_exec_symbol_new(symbolic_graph, CMD_SET_FORWARD(1), TENSOR_SYMBOL_LIST(), TENSOR_SYMBOL_LIST(dloss), "set 1");
327
1
    int i;
328
5
    for (i = 0; i < saved_aux_size * 4; 
i++4
)
329
4
      ccv_nnc_graph_exec_symbol_new(symbolic_graph, CMD_SET_FORWARD(0), TENSOR_SYMBOL_LIST(), TENSOR_SYMBOL_LIST(saved_aux[i].source), "set 0");
330
1
    ccv_nnc_graph_exec_symbol_autogen(symbolic_graph, 0, 0, CCV_NNC_AUTOGEN_ALL_EXECS | CCV_NNC_AUTOGEN_SOURCES_AND_DESTINATIONS);
331
1
    ccv_nnc_symbolic_graph_data_parallel(symbolic_graph, 2, TENSOR_SYMBOL_LIST(w1, bias1, w3, bias3), 0, 0, 0, gradients, 4, 0, CCV_NNC_PARALLEL_REDUCE_OP_SUM, SYMBOLIC_GRAPH_SOURCES(symbolic_graph), updated_execs, 4);
332
1
    ccv_nnc_graph_exec_symbol_autogen(symbolic_graph, 0, 0, CCV_NNC_AUTOGEN_SOURCES_AND_DESTINATIONS);
333
1
    SYMBOLIC_GRAPH_GEN(symbolic_graph, CCV_NNC_LONG_DOT_GRAPH);
334
1
    ccv_nnc_graph_t* graph;
335
1
    ccv_nnc_tensor_arena_t* tensor_arena;
336
1
    ccv_nnc_graph_exec_arena_t* graph_exec_arena;
337
1
    ccv_nnc_symbolic_graph_compile(symbolic_graph, ccv_nnc_default_compile_params,
338
1
      0, 0,
339
1
      updated_params, 4,
340
1
      SYMBOLIC_GRAPH_SOURCES(symbolic_graph), SYMBOLIC_GRAPH_DESTINATIONS(symbolic_graph),
341
1
      &graph, &tensor_arena, &graph_exec_arena);
342
1
    ccv_nnc_graph_set_default_static_schedule(graph, CCV_STREAM_CONTEXT_GPU, 0);
343
1
    GRAPH_GEN(graph, CCV_NNC_LONG_DOT_GRAPH);
344
1
    cpu_inputs[0] = ccv_nnc_tensor_new(0, CPU_TENSOR_NCHW(32F, 16, 3, 32, 32), 0);
345
1
    cpu_inputs[1] = ccv_nnc_tensor_new(0, CPU_TENSOR_NCHW(32F, 16, 3, 32, 32), 0);
346
1
    dsfmt_t dsfmt;
347
1
    dsfmt_init_gen_rand(&dsfmt, 0);
348
49.1k
    for (i = 0; i < 16 * 3 * 32 * 32; 
i++49.1k
)
349
49.1k
      cpu_inputs[0]->data.f32[i] = dsfmt_genrand_open_close(&dsfmt);
350
49.1k
    for (i = 0; i < 16 * 3 * 32 * 32; 
i++49.1k
)
351
49.1k
      cpu_inputs[1]->data.f32[i] = dsfmt_genrand_open_close(&dsfmt);
352
1
    cpu_fits[0] = ccv_nnc_tensor_new(0, CPU_TENSOR_NCHW(32F, 16), 0);
353
1
    cpu_fits[1] = ccv_nnc_tensor_new(0, CPU_TENSOR_NCHW(32F, 16), 0);
354
17
    for (i = 0; i < 16; 
i++16
)
355
16
      cpu_fits[0]->data.f32[i] = cpu_fits[1]->data.f32[i] = (int)(dsfmt_genrand_open_close(&dsfmt) * 7.4); // Between 0 to 7.
356
1
    ccv_nnc_cmd_exec(CMD_DATA_TRANSFER_FORWARD(), ccv_nnc_no_hint, 0, TENSOR_LIST(cpu_inputs[0], cpu_inputs[1]), TENSOR_LIST(ccv_nnc_tensor_from_symbol(tensor_arena, x), ccv_nnc_tensor_from_symbol(tensor_arena, ccv_nnc_tensor_symbol_copy(symbolic_graph, x, 1))), 0);
357
1
    ccv_nnc_cmd_exec(CMD_DATA_TRANSFER_FORWARD(), ccv_nnc_no_hint, 0, TENSOR_LIST(cpu_fits[0], cpu_fits[1]), TENSOR_LIST(ccv_nnc_tensor_from_symbol(tensor_arena, label), ccv_nnc_tensor_from_symbol(tensor_arena, ccv_nnc_tensor_symbol_copy(symbolic_graph, label, 1))), 0);
358
1
    ccv_nnc_cmd_exec(CMD_SET_FORWARD(0), ccv_nnc_no_hint, 0, TENSOR_LIST(), TENSOR_LIST(ccv_nnc_tensor_from_symbol(tensor_arena, bias1), ccv_nnc_tensor_from_symbol(tensor_arena, bias3)), 0);
359
601
    for (i = 0; i < 8 * 3 * 5 * 5; 
i++600
)
360
600
      w1_tensor->data.f32[i] = dsfmt_genrand_open_close(&dsfmt);
361
1.60k
    for (i = 0; i < 8 * 8 * 5 * 5; 
i++1.60k
)
362
1.60k
      w3_tensor->data.f32[i] = dsfmt_genrand_open_close(&dsfmt);
363
1
    ccv_nnc_cmd_exec(CMD_DATA_TRANSFER_FORWARD(), ccv_nnc_no_hint, 0, TENSOR_LIST(w1_tensor, w3_tensor), TENSOR_LIST(ccv_nnc_tensor_from_symbol(tensor_arena, w1), ccv_nnc_tensor_from_symbol(tensor_arena, w3)), 0);
364
1
    ccv_nnc_graph_run(graph, 0, TRAVERSE_FULL, 0, ccv_nnc_graph_default_stream(graph));
365
1
    ccv_nnc_stream_context_wait(ccv_nnc_graph_default_stream(graph));
366
1
    updated[0] = ccv_nnc_tensor_new(0, CPU_TENSOR_NCHW(32F, 8, 3, 5, 5), 0);
367
1
    updated[1] = ccv_nnc_tensor_new(0, CPU_TENSOR_NCHW(32F, 8), 0);
368
1
    updated[2] = ccv_nnc_tensor_new(0, CPU_TENSOR_NCHW(32F, 8, 8, 5, 5), 0);
369
1
    updated[3] = ccv_nnc_tensor_new(0, CPU_TENSOR_NCHW(32F, 8), 0);
370
1
    ccv_nnc_cmd_exec(CMD_DATA_TRANSFER_FORWARD(), ccv_nnc_no_hint, 0, TENSOR_LIST(ccv_nnc_tensor_from_symbol(tensor_arena, updated_params[0]), ccv_nnc_tensor_from_symbol(tensor_arena, updated_params[1]), ccv_nnc_tensor_from_symbol(tensor_arena, updated_params[2]), ccv_nnc_tensor_from_symbol(tensor_arena, updated_params[3])), updated, 4, 0);
371
1
    ccv_nnc_symbolic_graph_free(symbolic_graph);
372
1
    ccv_nnc_graph_free(graph);
373
1
    ccv_nnc_tensor_arena_free(tensor_arena);
374
1
    ccv_nnc_graph_exec_arena_free(graph_exec_arena);
375
1
  }
376
  // Now, doing exactly the same, but with no parallel.
377
1
  {
378
1
    ccv_nnc_symbolic_graph_t* const symbolic_graph = ccv_nnc_symbolic_graph_new();
379
1
    const ccv_nnc_tensor_symbol_t x = ccv_nnc_tensor_symbol_new(symbolic_graph, GPU_TENSOR_NCHW(000, 32F, 32, 3, 32, 32), 0);
380
1
    const ccv_nnc_tensor_symbol_t w1 = ccv_nnc_tensor_symbol_new(symbolic_graph, GPU_TENSOR_NCHW(000, 32F, 8, 3, 5, 5), 0);
381
1
    const ccv_nnc_tensor_symbol_t bias1 = ccv_nnc_tensor_symbol_new(symbolic_graph, GPU_TENSOR_NCHW(000, 32F, 8), 0);
382
1
    const ccv_nnc_tensor_symbol_t y1 = ccv_nnc_tensor_symbol_new(symbolic_graph, GPU_TENSOR_NCHW(000, 32F, 32, 8, 32, 32), 0);
383
1
    const ccv_nnc_graph_exec_symbol_t conv1 = ccv_nnc_graph_exec_symbol_new(symbolic_graph, CMD_CONVOLUTION_FORWARD(1, 8, 5, 5, 32), TENSOR_SYMBOL_LIST(x, w1, bias1), TENSOR_SYMBOL_LIST(y1), "conv1");
384
1
    ccv_nnc_graph_exec_symbol_set_hint(symbolic_graph, conv1, HINT((1, 1), (2, 2)));
385
1
    const ccv_nnc_tensor_symbol_t y2 = ccv_nnc_tensor_symbol_new(symbolic_graph, GPU_TENSOR_NCHW(000, 32F, 32, 8, 16, 16), 0);
386
1
    const ccv_nnc_graph_exec_symbol_t avg2 = ccv_nnc_graph_exec_symbol_new(symbolic_graph, CMD_AVERAGE_POOL_FORWARD(2, 2), TENSOR_SYMBOL_LIST(y1), TENSOR_SYMBOL_LIST(y2), "avg2");
387
1
    ccv_nnc_graph_exec_symbol_set_hint(symbolic_graph, avg2, HINT((2, 2)));
388
1
    const ccv_nnc_tensor_symbol_t w3 = ccv_nnc_tensor_symbol_new(symbolic_graph, GPU_TENSOR_NCHW(000, 32F, 8, 8, 5, 5), 0);
389
1
    const ccv_nnc_tensor_symbol_t bias3 = ccv_nnc_tensor_symbol_new(symbolic_graph, GPU_TENSOR_NCHW(000, 32F, 8), 0);
390
1
    const ccv_nnc_tensor_symbol_t y3 = ccv_nnc_tensor_symbol_new(symbolic_graph, GPU_TENSOR_NCHW(000, 32F, 32, 8, 8, 8), 0);
391
1
    const ccv_nnc_graph_exec_symbol_t conv3 = ccv_nnc_graph_exec_symbol_new(symbolic_graph, CMD_CONVOLUTION_FORWARD(1, 8, 5, 5, 32), TENSOR_SYMBOL_LIST(y2, w3, bias3), TENSOR_SYMBOL_LIST(y3), "conv3");
392
1
    ccv_nnc_graph_exec_symbol_set_hint(symbolic_graph, conv3, HINT((2, 2), (2, 2)));
393
1
    const ccv_nnc_tensor_symbol_t y4 = ccv_nnc_tensor_symbol_new(symbolic_graph, GPU_TENSOR_NCHW(000, 32F, 32, 8, 1, 1), 0);
394
1
    ccv_nnc_graph_exec_symbol_new(symbolic_graph, CMD_AVERAGE_POOL_FORWARD(8, 8), TENSOR_SYMBOL_LIST(y3), TENSOR_SYMBOL_LIST(y4), "avg4");
395
1
    const ccv_nnc_tensor_symbol_t label = ccv_nnc_tensor_symbol_new(symbolic_graph, GPU_TENSOR_NCHW(000, 32F, 32), "label");
396
1
    const ccv_nnc_tensor_symbol_t y5 = ccv_nnc_tensor_symbol_new(symbolic_graph, GPU_TENSOR_NCHW(000, 32F, 32, 8), "y5");
397
1
    const ccv_nnc_tensor_symbol_t loss = ccv_nnc_tensor_symbol_new(symbolic_graph, GPU_TENSOR_NCHW(000, 32F, 32), "loss");
398
1
    ccv_nnc_graph_exec_symbol_new(symbolic_graph, CMD_SOFTMAX_CROSSENTROPY_FORWARD(), TENSOR_SYMBOL_LIST(y4, label), TENSOR_SYMBOL_LIST(loss, y5), "softmax crossentropy");
399
1
    ccv_nnc_graph_exec_symbol_autogen(symbolic_graph, 0, 0, CCV_NNC_AUTOGEN_ALL_EXECS | CCV_NNC_AUTOGEN_SOURCES_AND_DESTINATIONS);
400
1
    ccv_nnc_tensor_symbol_t updated_params[4];
401
1
    ccv_nnc_tensor_symbol_t gradients[4];
402
1
    const int saved_aux_size = ccv_nnc_minimizer_saved_aux_size(CMD_SGD_FORWARD(0, 0.001, 1, 0.99, 0.9, 0.9));
403
1
    ccv_nnc_tensor_symbol_map_t saved_aux[saved_aux_size * 4];
404
1
    ccv_nnc_graph_exec_symbol_t updated_execs[4];
405
1
    ccv_nnc_symbolic_graph_minimize(symbolic_graph, CMD_SGD_FORWARD(0, 0.001, 1, 0.99, 0.9, 0.9), TENSOR_SYMBOL_LIST(loss), TENSOR_SYMBOL_LIST(w1, bias1, w3, bias3), 0, 0, SYMBOLIC_GRAPH_SOURCES(symbolic_graph), SYMBOLIC_GRAPH_DESTINATIONS(symbolic_graph), gradients, updated_params, saved_aux, updated_execs);
406
1
    const ccv_nnc_tensor_symbol_t dloss = ccv_nnc_tensor_symbol_for_backward(symbolic_graph, loss);
407
1
    ccv_nnc_graph_exec_symbol_new(symbolic_graph, CMD_SET_FORWARD(1), TENSOR_SYMBOL_LIST(), TENSOR_SYMBOL_LIST(dloss), "set 1");
408
1
    int i;
409
5
    for (i = 0; i < saved_aux_size * 4; 
i++4
)
410
4
      ccv_nnc_graph_exec_symbol_new(symbolic_graph, CMD_SET_FORWARD(0), TENSOR_SYMBOL_LIST(), TENSOR_SYMBOL_LIST(saved_aux[i].source), "set 0");
411
1
    ccv_nnc_graph_exec_symbol_autogen(symbolic_graph, 0, 0, CCV_NNC_AUTOGEN_SOURCES_AND_DESTINATIONS);
412
1
    ccv_nnc_graph_t* graph;
413
1
    ccv_nnc_tensor_arena_t* tensor_arena;
414
1
    ccv_nnc_graph_exec_arena_t* graph_exec_arena;
415
1
    ccv_nnc_symbolic_graph_compile(symbolic_graph, ccv_nnc_default_compile_params,
416
1
      0, 0,
417
1
      updated_params, 4,
418
1
      SYMBOLIC_GRAPH_SOURCES(symbolic_graph), SYMBOLIC_GRAPH_DESTINATIONS(symbolic_graph),
419
1
      &graph, &tensor_arena, &graph_exec_arena);
420
1
    ccv_nnc_tensor_t* cpu_input = ccv_nnc_tensor_new(0, CPU_TENSOR_NCHW(32F, 32, 3, 32, 32), 0);
421
1
    memcpy(cpu_input->data.f32, cpu_inputs[0]->data.f32, sizeof(float) * 16 * 3 * 32 * 32);
422
1
    memcpy(cpu_input->data.f32 + 16 * 3 * 32 * 32, cpu_inputs[1]->data.f32, sizeof(float) * 16 * 3 * 32 * 32);
423
1
    ccv_nnc_tensor_t* cpu_fit = ccv_nnc_tensor_new(0, CPU_TENSOR_NCHW(32F, 32), 0);
424
1
    memcpy(cpu_fit->data.f32, cpu_fits[0]->data.f32, sizeof(float) * 16);
425
1
    memcpy(cpu_fit->data.f32 + 16, cpu_fits[1]->data.f32, sizeof(float) * 16);
426
1
    ccv_nnc_cmd_exec(CMD_DATA_TRANSFER_FORWARD(), ccv_nnc_no_hint, 0, TENSOR_LIST(cpu_input), TENSOR_LIST(ccv_nnc_tensor_from_symbol(tensor_arena, x)), 0);
427
1
    ccv_nnc_cmd_exec(CMD_DATA_TRANSFER_FORWARD(), ccv_nnc_no_hint, 0, TENSOR_LIST(cpu_fit), TENSOR_LIST(ccv_nnc_tensor_from_symbol(tensor_arena, label)), 0);
428
1
    ccv_nnc_cmd_exec(CMD_SET_FORWARD(0), ccv_nnc_no_hint, 0, TENSOR_LIST(), TENSOR_LIST(ccv_nnc_tensor_from_symbol(tensor_arena, bias1), ccv_nnc_tensor_from_symbol(tensor_arena, bias3)), 0);
429
1
    ccv_nnc_cmd_exec(CMD_DATA_TRANSFER_FORWARD(), ccv_nnc_no_hint, 0, TENSOR_LIST(w1_tensor, w3_tensor), TENSOR_LIST(ccv_nnc_tensor_from_symbol(tensor_arena, w1), ccv_nnc_tensor_from_symbol(tensor_arena, w3)), 0);
430
1
    ccv_nnc_graph_run(graph, 0, TRAVERSE_FULL, 0, 0);
431
1
    ccv_nnc_tensor_t* np_updated[4];
432
1
    np_updated[0] = ccv_nnc_tensor_new(0, CPU_TENSOR_NCHW(32F, 8, 3, 5, 5), 0);
433
1
    np_updated[1] = ccv_nnc_tensor_new(0, CPU_TENSOR_NCHW(32F, 8), 0);
434
1
    np_updated[2] = ccv_nnc_tensor_new(0, CPU_TENSOR_NCHW(32F, 8, 8, 5, 5), 0);
435
1
    np_updated[3] = ccv_nnc_tensor_new(0, CPU_TENSOR_NCHW(32F, 8), 0);
436
1
    ccv_nnc_cmd_exec(CMD_DATA_TRANSFER_FORWARD(), ccv_nnc_no_hint, 0, TENSOR_LIST(ccv_nnc_tensor_from_symbol(tensor_arena, updated_params[0]), ccv_nnc_tensor_from_symbol(tensor_arena, updated_params[1]), ccv_nnc_tensor_from_symbol(tensor_arena, updated_params[2]), ccv_nnc_tensor_from_symbol(tensor_arena, updated_params[3])), np_updated, 4, 0);
437
1
    ccv_nnc_symbolic_graph_free(symbolic_graph);
438
1
    ccv_nnc_graph_free(graph);
439
1
    ccv_nnc_tensor_arena_free(tensor_arena);
440
1
    ccv_nnc_graph_exec_arena_free(graph_exec_arena);
441
1
    REQUIRE_ARRAY_EQ_WITH_TOLERANCE(float, np_updated[0]->data.f32, updated[0]->data.f32, 8 * 3 * 5 * 5, 1e-4, "updated params should be equal");
442
1
    REQUIRE_ARRAY_EQ_WITH_TOLERANCE(float, np_updated[1]->data.f32, updated[1]->data.f32, 8, 1e-5, "updated params should be equal");
443
1
    REQUIRE_ARRAY_EQ_WITH_TOLERANCE(float, np_updated[2]->data.f32, updated[2]->data.f32, 8 * 8 * 5 * 5, 1e-4, "updated params should be equal");
444
1
    REQUIRE_ARRAY_EQ_WITH_TOLERANCE(float, np_updated[3]->data.f32, updated[3]->data.f32, 8, 1e-4, "updated params should be equal");
445
1
    ccv_nnc_tensor_free(cpu_input);
446
1
    ccv_nnc_tensor_free(cpu_fit);
447
1
    ccv_nnc_tensor_free(np_updated[0]);
448
1
    ccv_nnc_tensor_free(np_updated[1]);
449
1
    ccv_nnc_tensor_free(np_updated[2]);
450
1
    ccv_nnc_tensor_free(np_updated[3]);
451
1
  }
452
0
  ccv_nnc_tensor_free(updated[0]);
453
1
  ccv_nnc_tensor_free(updated[1]);
454
1
  ccv_nnc_tensor_free(updated[2]);
455
1
  ccv_nnc_tensor_free(updated[3]);
456
1
  ccv_nnc_tensor_free(cpu_inputs[0]);
457
1
  ccv_nnc_tensor_free(cpu_inputs[1]);
458
1
  ccv_nnc_tensor_free(cpu_fits[0]);
459
1
  ccv_nnc_tensor_free(cpu_fits[1]);
460
1
  ccv_nnc_tensor_free(w1_tensor);
461
1
  ccv_nnc_tensor_free(w3_tensor);
462
1
}
463
464
TEST_CASE("schedule symbolic graph to data parallel with allreduce")
465
1
{
466
1
  GUARD_ELSE_RETURN(ccv_nnc_cmd_ok(CCV_NNC_CONVOLUTION_FORWARD, CCV_NNC_BACKEND_GPU_CUDNN));
467
1
  GUARD_ELSE_RETURN(ccv_nnc_cmd_ok(CCV_NNC_COMM_ALLREDUCE_FORWARD, CCV_NNC_BACKEND_GPU_NCCL));
468
1
  GUARD_ELSE_RETURN(ccv_nnc_device_count(CCV_STREAM_CONTEXT_GPU) >= 2);
469
1
  ccv_nnc_tensor_t* updated[4];
470
1
  ccv_nnc_tensor_t* cpu_inputs[2];
471
1
  ccv_nnc_tensor_t* cpu_fits[2];
472
1
  ccv_nnc_tensor_t* w1_tensor = ccv_nnc_tensor_new(0, CPU_TENSOR_NCHW(32F, 8, 3, 5, 5), 0);
473
1
  ccv_nnc_tensor_t* w3_tensor = ccv_nnc_tensor_new(0, CPU_TENSOR_NCHW(32F, 8, 8, 5, 5), 0);
474
1
  {
475
1
    ccv_nnc_symbolic_graph_t* const symbolic_graph = ccv_nnc_symbolic_graph_new();
476
1
    const ccv_nnc_tensor_symbol_t x = ccv_nnc_tensor_symbol_new(symbolic_graph, GPU_TENSOR_NCHW(000, 32F, 16, 3, 32, 32), 0);
477
1
    const ccv_nnc_tensor_symbol_t w1 = ccv_nnc_tensor_symbol_new(symbolic_graph, GPU_TENSOR_NCHW(000, 32F, 8, 3, 5, 5), 0);
478
1
    const ccv_nnc_tensor_symbol_t bias1 = ccv_nnc_tensor_symbol_new(symbolic_graph, GPU_TENSOR_NCHW(000, 32F, 8), 0);
479
1
    const ccv_nnc_tensor_symbol_t y1 = ccv_nnc_tensor_symbol_new(symbolic_graph, GPU_TENSOR_NCHW(000, 32F, 16, 8, 32, 32), 0);
480
1
    const ccv_nnc_graph_exec_symbol_t conv1 = ccv_nnc_graph_exec_symbol_new(symbolic_graph, CMD_CONVOLUTION_FORWARD(1, 8, 5, 5, 32), TENSOR_SYMBOL_LIST(x, w1, bias1), TENSOR_SYMBOL_LIST(y1), "conv1");
481
1
    ccv_nnc_graph_exec_symbol_set_hint(symbolic_graph, conv1, HINT((1, 1), (2, 2)));
482
1
    const ccv_nnc_tensor_symbol_t y2 = ccv_nnc_tensor_symbol_new(symbolic_graph, GPU_TENSOR_NCHW(000, 32F, 16, 8, 16, 16), 0);
483
1
    const ccv_nnc_graph_exec_symbol_t avg2 = ccv_nnc_graph_exec_symbol_new(symbolic_graph, CMD_AVERAGE_POOL_FORWARD(2, 2), TENSOR_SYMBOL_LIST(y1), TENSOR_SYMBOL_LIST(y2), "avg2");
484
1
    ccv_nnc_graph_exec_symbol_set_hint(symbolic_graph, avg2, HINT((2, 2)));
485
1
    const ccv_nnc_tensor_symbol_t w3 = ccv_nnc_tensor_symbol_new(symbolic_graph, GPU_TENSOR_NCHW(000, 32F, 8, 8, 5, 5), 0);
486
1
    const ccv_nnc_tensor_symbol_t bias3 = ccv_nnc_tensor_symbol_new(symbolic_graph, GPU_TENSOR_NCHW(000, 32F, 8), 0);
487
1
    const ccv_nnc_tensor_symbol_t y3 = ccv_nnc_tensor_symbol_new(symbolic_graph, GPU_TENSOR_NCHW(000, 32F, 16, 8, 8, 8), 0);
488
1
    const ccv_nnc_graph_exec_symbol_t conv3 = ccv_nnc_graph_exec_symbol_new(symbolic_graph, CMD_CONVOLUTION_FORWARD(1, 8, 5, 5, 32), TENSOR_SYMBOL_LIST(y2, w3, bias3), TENSOR_SYMBOL_LIST(y3), "conv3");
489
1
    ccv_nnc_graph_exec_symbol_set_hint(symbolic_graph, conv3, HINT((2, 2), (2, 2)));
490
1
    const ccv_nnc_tensor_symbol_t y4 = ccv_nnc_tensor_symbol_new(symbolic_graph, GPU_TENSOR_NCHW(000, 32F, 16, 8, 1, 1), 0);
491
1
    ccv_nnc_graph_exec_symbol_new(symbolic_graph, CMD_AVERAGE_POOL_FORWARD(8, 8), TENSOR_SYMBOL_LIST(y3), TENSOR_SYMBOL_LIST(y4), "avg4");
492
1
    const ccv_nnc_tensor_symbol_t y4a = ccv_nnc_tensor_symbol_alias_new(symbolic_graph, y4, ccv_nnc_no_ofs, DIM_ALLOC(8, 1, 1, 1), GPU_TENSOR_NCHW(000, 32F, 16, 8), 0);
493
1
    const ccv_nnc_tensor_symbol_t label = ccv_nnc_tensor_symbol_new(symbolic_graph, GPU_TENSOR_NCHW(000, 32F, 16), "label");
494
1
    const ccv_nnc_tensor_symbol_t y5 = ccv_nnc_tensor_symbol_new(symbolic_graph, GPU_TENSOR_NCHW(000, 32F, 16, 8), "y5");
495
1
    const ccv_nnc_tensor_symbol_t loss = ccv_nnc_tensor_symbol_new(symbolic_graph, GPU_TENSOR_NCHW(000, 32F, 16), "loss");
496
1
    ccv_nnc_graph_exec_symbol_new(symbolic_graph, CMD_SOFTMAX_CROSSENTROPY_FORWARD(), TENSOR_SYMBOL_LIST(y4a, label), TENSOR_SYMBOL_LIST(loss, y5), "softmax crossentropy");
497
1
    ccv_nnc_graph_exec_symbol_autogen(symbolic_graph, 0, 0, CCV_NNC_AUTOGEN_ALL_EXECS | CCV_NNC_AUTOGEN_SOURCES_AND_DESTINATIONS);
498
1
    ccv_nnc_tensor_symbol_t updated_params[4];
499
1
    ccv_nnc_tensor_symbol_t gradients[4];
500
1
    const int saved_aux_size = ccv_nnc_minimizer_saved_aux_size(CMD_SGD_FORWARD(0, 0.001, 1, 0.99, 0.9, 0.9));
501
1
    ccv_nnc_tensor_symbol_map_t saved_aux[saved_aux_size * 4];
502
1
    ccv_nnc_graph_exec_symbol_t updated_execs[4];
503
1
    ccv_nnc_symbolic_graph_minimize(symbolic_graph, CMD_SGD_FORWARD(0, 0.001, 1, 0.99, 0.9, 0.9), TENSOR_SYMBOL_LIST(loss), TENSOR_SYMBOL_LIST(w1, bias1, w3, bias3), 0, 0, SYMBOLIC_GRAPH_SOURCES(symbolic_graph), SYMBOLIC_GRAPH_DESTINATIONS(symbolic_graph), gradients, updated_params, saved_aux, updated_execs);
504
1
    const ccv_nnc_tensor_symbol_t dloss = ccv_nnc_tensor_symbol_for_backward(symbolic_graph, loss);
505
1
    ccv_nnc_graph_exec_symbol_new(symbolic_graph, CMD_SET_FORWARD(1), TENSOR_SYMBOL_LIST(), TENSOR_SYMBOL_LIST(dloss), "set 1");
506
1
    int i;
507
5
    for (i = 0; i < saved_aux_size * 4; 
i++4
)
508
4
      ccv_nnc_graph_exec_symbol_new(symbolic_graph, CMD_SET_FORWARD(0), TENSOR_SYMBOL_LIST(), TENSOR_SYMBOL_LIST(saved_aux[i].source), "set 0");
509
1
    ccv_nnc_graph_exec_symbol_autogen(symbolic_graph, 0, 0, CCV_NNC_AUTOGEN_ALL_EXECS | CCV_NNC_AUTOGEN_SOURCES_AND_DESTINATIONS);
510
1
    ccv_nnc_symbolic_graph_data_parallel(symbolic_graph, 2, TENSOR_SYMBOL_LIST(w1, bias1, w3, bias3), gradients, 4, 0, 0, 0, 0, CCV_NNC_PARALLEL_REDUCE_OP_SUM, SYMBOLIC_GRAPH_SOURCES(symbolic_graph), updated_execs, 4);
511
1
    ccv_nnc_graph_exec_symbol_autogen(symbolic_graph, 0, 0, CCV_NNC_AUTOGEN_SOURCES_AND_DESTINATIONS);
512
1
    SYMBOLIC_GRAPH_GEN(symbolic_graph, CCV_NNC_LONG_DOT_GRAPH);
513
1
    ccv_nnc_graph_t* graph;
514
1
    ccv_nnc_tensor_arena_t* tensor_arena;
515
1
    ccv_nnc_graph_exec_arena_t* graph_exec_arena;
516
1
    ccv_nnc_symbolic_graph_compile(symbolic_graph, ccv_nnc_default_compile_params,
517
1
      0, 0,
518
1
      updated_params, 4,
519
1
      SYMBOLIC_GRAPH_SOURCES(symbolic_graph), SYMBOLIC_GRAPH_DESTINATIONS(symbolic_graph),
520
1
      &graph, &tensor_arena, &graph_exec_arena);
521
1
    ccv_nnc_graph_set_default_static_schedule(graph, CCV_STREAM_CONTEXT_GPU, 0);
522
1
    GRAPH_GEN(graph, CCV_NNC_LONG_DOT_GRAPH);
523
1
    cpu_inputs[0] = ccv_nnc_tensor_new(0, CPU_TENSOR_NCHW(32F, 16, 3, 32, 32), 0);
524
1
    cpu_inputs[1] = ccv_nnc_tensor_new(0, CPU_TENSOR_NCHW(32F, 16, 3, 32, 32), 0);
525
1
    dsfmt_t dsfmt;
526
1
    dsfmt_init_gen_rand(&dsfmt, 0);
527
49.1k
    for (i = 0; i < 16 * 3 * 32 * 32; 
i++49.1k
)
528
49.1k
      cpu_inputs[0]->data.f32[i] = dsfmt_genrand_open_close(&dsfmt);
529
49.1k
    for (i = 0; i < 16 * 3 * 32 * 32; 
i++49.1k
)
530
49.1k
      cpu_inputs[1]->data.f32[i] = dsfmt_genrand_open_close(&dsfmt);
531
1
    cpu_fits[0] = ccv_nnc_tensor_new(0, CPU_TENSOR_NCHW(32F, 16), 0);
532
1
    cpu_fits[1] = ccv_nnc_tensor_new(0, CPU_TENSOR_NCHW(32F, 16), 0);
533
17
    for (i = 0; i < 16; 
i++16
)
534
16
      cpu_fits[0]->data.f32[i] = cpu_fits[1]->data.f32[i] = (int)(dsfmt_genrand_open_close(&dsfmt) * 7.4); // Between 0 to 7.
535
1
    ccv_nnc_cmd_exec(CMD_DATA_TRANSFER_FORWARD(), ccv_nnc_no_hint, 0, TENSOR_LIST(cpu_inputs[0], cpu_inputs[1]), TENSOR_LIST(ccv_nnc_tensor_from_symbol(tensor_arena, x), ccv_nnc_tensor_from_symbol(tensor_arena, ccv_nnc_tensor_symbol_copy(symbolic_graph, x, 1))), 0);
536
1
    ccv_nnc_cmd_exec(CMD_DATA_TRANSFER_FORWARD(), ccv_nnc_no_hint, 0, TENSOR_LIST(cpu_fits[0], cpu_fits[1]), TENSOR_LIST(ccv_nnc_tensor_from_symbol(tensor_arena, label), ccv_nnc_tensor_from_symbol(tensor_arena, ccv_nnc_tensor_symbol_copy(symbolic_graph, label, 1))), 0);
537
1
    ccv_nnc_cmd_exec(CMD_SET_FORWARD(0), ccv_nnc_no_hint, 0, TENSOR_LIST(), TENSOR_LIST(ccv_nnc_tensor_from_symbol(tensor_arena, bias1), ccv_nnc_tensor_from_symbol(tensor_arena, bias3)), 0);
538
601
    for (i = 0; i < 8 * 3 * 5 * 5; 
i++600
)
539
600
      w1_tensor->data.f32[i] = dsfmt_genrand_open_close(&dsfmt);
540
1.60k
    for (i = 0; i < 8 * 8 * 5 * 5; 
i++1.60k
)
541
1.60k
      w3_tensor->data.f32[i] = dsfmt_genrand_open_close(&dsfmt);
542
1
    ccv_nnc_cmd_exec(CMD_DATA_TRANSFER_FORWARD(), ccv_nnc_no_hint, 0, TENSOR_LIST(w1_tensor, w3_tensor), TENSOR_LIST(ccv_nnc_tensor_from_symbol(tensor_arena, w1), ccv_nnc_tensor_from_symbol(tensor_arena, w3)), 0);
543
1
    ccv_nnc_graph_run(graph, 0, TRAVERSE_FULL, 0, ccv_nnc_graph_default_stream(graph));
544
1
    ccv_nnc_stream_context_wait(ccv_nnc_graph_default_stream(graph));
545
1
    updated[0] = ccv_nnc_tensor_new(0, CPU_TENSOR_NCHW(32F, 8, 3, 5, 5), 0);
546
1
    updated[1] = ccv_nnc_tensor_new(0, CPU_TENSOR_NCHW(32F, 8), 0);
547
1
    updated[2] = ccv_nnc_tensor_new(0, CPU_TENSOR_NCHW(32F, 8, 8, 5, 5), 0);
548
1
    updated[3] = ccv_nnc_tensor_new(0, CPU_TENSOR_NCHW(32F, 8), 0);
549
1
    ccv_nnc_cmd_exec(CMD_DATA_TRANSFER_FORWARD(), ccv_nnc_no_hint, 0, TENSOR_LIST(ccv_nnc_tensor_from_symbol(tensor_arena, updated_params[0]), ccv_nnc_tensor_from_symbol(tensor_arena, updated_params[1]), ccv_nnc_tensor_from_symbol(tensor_arena, updated_params[2]), ccv_nnc_tensor_from_symbol(tensor_arena, updated_params[3])), updated, 4, 0);
550
1
    ccv_nnc_symbolic_graph_free(symbolic_graph);
551
1
    ccv_nnc_graph_free(graph);
552
1
    ccv_nnc_tensor_arena_free(tensor_arena);
553
1
    ccv_nnc_graph_exec_arena_free(graph_exec_arena);
554
1
  }
555
  // Now, doing exactly the same, but with no parallel.
556
1
  {
557
1
    ccv_nnc_symbolic_graph_t* const symbolic_graph = ccv_nnc_symbolic_graph_new();
558
1
    const ccv_nnc_tensor_symbol_t x = ccv_nnc_tensor_symbol_new(symbolic_graph, GPU_TENSOR_NCHW(000, 32F, 32, 3, 32, 32), 0);
559
1
    const ccv_nnc_tensor_symbol_t w1 = ccv_nnc_tensor_symbol_new(symbolic_graph, GPU_TENSOR_NCHW(000, 32F, 8, 3, 5, 5), 0);
560
1
    const ccv_nnc_tensor_symbol_t bias1 = ccv_nnc_tensor_symbol_new(symbolic_graph, GPU_TENSOR_NCHW(000, 32F, 8), 0);
561
1
    const ccv_nnc_tensor_symbol_t y1 = ccv_nnc_tensor_symbol_new(symbolic_graph, GPU_TENSOR_NCHW(000, 32F, 32, 8, 32, 32), 0);
562
1
    const ccv_nnc_graph_exec_symbol_t conv1 = ccv_nnc_graph_exec_symbol_new(symbolic_graph, CMD_CONVOLUTION_FORWARD(1, 8, 5, 5, 32), TENSOR_SYMBOL_LIST(x, w1, bias1), TENSOR_SYMBOL_LIST(y1), "conv1");
563
1
    ccv_nnc_graph_exec_symbol_set_hint(symbolic_graph, conv1, HINT((1, 1), (2, 2)));
564
1
    const ccv_nnc_tensor_symbol_t y2 = ccv_nnc_tensor_symbol_new(symbolic_graph, GPU_TENSOR_NCHW(000, 32F, 32, 8, 16, 16), 0);
565
1
    const ccv_nnc_graph_exec_symbol_t avg2 = ccv_nnc_graph_exec_symbol_new(symbolic_graph, CMD_AVERAGE_POOL_FORWARD(2, 2), TENSOR_SYMBOL_LIST(y1), TENSOR_SYMBOL_LIST(y2), "avg2");
566
1
    ccv_nnc_graph_exec_symbol_set_hint(symbolic_graph, avg2, HINT((2, 2)));
567
1
    const ccv_nnc_tensor_symbol_t w3 = ccv_nnc_tensor_symbol_new(symbolic_graph, GPU_TENSOR_NCHW(000, 32F, 8, 8, 5, 5), 0);
568
1
    const ccv_nnc_tensor_symbol_t bias3 = ccv_nnc_tensor_symbol_new(symbolic_graph, GPU_TENSOR_NCHW(000, 32F, 8), 0);
569
1
    const ccv_nnc_tensor_symbol_t y3 = ccv_nnc_tensor_symbol_new(symbolic_graph, GPU_TENSOR_NCHW(000, 32F, 32, 8, 8, 8), 0);
570
1
    const ccv_nnc_graph_exec_symbol_t conv3 = ccv_nnc_graph_exec_symbol_new(symbolic_graph, CMD_CONVOLUTION_FORWARD(1, 8, 5, 5, 32), TENSOR_SYMBOL_LIST(y2, w3, bias3), TENSOR_SYMBOL_LIST(y3), "conv3");
571
1
    ccv_nnc_graph_exec_symbol_set_hint(symbolic_graph, conv3, HINT((2, 2), (2, 2)));
572
1
    const ccv_nnc_tensor_symbol_t y4 = ccv_nnc_tensor_symbol_new(symbolic_graph, GPU_TENSOR_NCHW(000, 32F, 32, 8, 1, 1), 0);
573
1
    ccv_nnc_graph_exec_symbol_new(symbolic_graph, CMD_AVERAGE_POOL_FORWARD(8, 8), TENSOR_SYMBOL_LIST(y3), TENSOR_SYMBOL_LIST(y4), "avg4");
574
1
    const ccv_nnc_tensor_symbol_t label = ccv_nnc_tensor_symbol_new(symbolic_graph, GPU_TENSOR_NCHW(000, 32F, 32), "label");
575
1
    const ccv_nnc_tensor_symbol_t y5 = ccv_nnc_tensor_symbol_new(symbolic_graph, GPU_TENSOR_NCHW(000, 32F, 32, 8), "y5");
576
1
    const ccv_nnc_tensor_symbol_t loss = ccv_nnc_tensor_symbol_new(symbolic_graph, GPU_TENSOR_NCHW(000, 32F, 32), "loss");
577
1
    ccv_nnc_graph_exec_symbol_new(symbolic_graph, CMD_SOFTMAX_CROSSENTROPY_FORWARD(), TENSOR_SYMBOL_LIST(y4, label), TENSOR_SYMBOL_LIST(loss, y5), "softmax crossentropy");
578
1
    ccv_nnc_graph_exec_symbol_autogen(symbolic_graph, 0, 0, CCV_NNC_AUTOGEN_ALL_EXECS | CCV_NNC_AUTOGEN_SOURCES_AND_DESTINATIONS);
579
1
    ccv_nnc_tensor_symbol_t updated_params[4];
580
1
    ccv_nnc_tensor_symbol_t gradients[4];
581
1
    const int saved_aux_size = ccv_nnc_minimizer_saved_aux_size(CMD_SGD_FORWARD(0, 0.001, 1, 0.99, 0.9, 0.9));
582
1
    ccv_nnc_tensor_symbol_map_t saved_aux[saved_aux_size * 4];
583
1
    ccv_nnc_graph_exec_symbol_t updated_execs[4];
584
1
    ccv_nnc_symbolic_graph_minimize(symbolic_graph, CMD_SGD_FORWARD(0, 0.001, 1, 0.99, 0.9, 0.9), TENSOR_SYMBOL_LIST(loss), TENSOR_SYMBOL_LIST(w1, bias1, w3, bias3), 0, 0, SYMBOLIC_GRAPH_SOURCES(symbolic_graph), SYMBOLIC_GRAPH_DESTINATIONS(symbolic_graph), gradients, updated_params, saved_aux, updated_execs);
585
1
    const ccv_nnc_tensor_symbol_t dloss = ccv_nnc_tensor_symbol_for_backward(symbolic_graph, loss);
586
1
    ccv_nnc_graph_exec_symbol_new(symbolic_graph, CMD_SET_FORWARD(1), TENSOR_SYMBOL_LIST(), TENSOR_SYMBOL_LIST(dloss), "set 1");
587
1
    int i;
588
5
    for (i = 0; i < saved_aux_size * 4; 
i++4
)
589
4
      ccv_nnc_graph_exec_symbol_new(symbolic_graph, CMD_SET_FORWARD(0), TENSOR_SYMBOL_LIST(), TENSOR_SYMBOL_LIST(saved_aux[i].source), "set 0");
590
1
    ccv_nnc_graph_exec_symbol_autogen(symbolic_graph, 0, 0, CCV_NNC_AUTOGEN_SOURCES_AND_DESTINATIONS);
591
1
    ccv_nnc_graph_t* graph;
592
1
    ccv_nnc_tensor_arena_t* tensor_arena;
593
1
    ccv_nnc_graph_exec_arena_t* graph_exec_arena;
594
1
    ccv_nnc_symbolic_graph_compile(symbolic_graph, ccv_nnc_default_compile_params,
595
1
      0, 0,
596
1
      updated_params, 4,
597
1
      SYMBOLIC_GRAPH_SOURCES(symbolic_graph), SYMBOLIC_GRAPH_DESTINATIONS(symbolic_graph),
598
1
      &graph, &tensor_arena, &graph_exec_arena);
599
1
    ccv_nnc_tensor_t* cpu_input = ccv_nnc_tensor_new(0, CPU_TENSOR_NCHW(32F, 32, 3, 32, 32), 0);
600
1
    memcpy(cpu_input->data.f32, cpu_inputs[0]->data.f32, sizeof(float) * 16 * 3 * 32 * 32);
601
1
    memcpy(cpu_input->data.f32 + 16 * 3 * 32 * 32, cpu_inputs[1]->data.f32, sizeof(float) * 16 * 3 * 32 * 32);
602
1
    ccv_nnc_tensor_t* cpu_fit = ccv_nnc_tensor_new(0, CPU_TENSOR_NCHW(32F, 32), 0);
603
1
    memcpy(cpu_fit->data.f32, cpu_fits[0]->data.f32, sizeof(float) * 16);
604
1
    memcpy(cpu_fit->data.f32 + 16, cpu_fits[1]->data.f32, sizeof(float) * 16);
605
1
    ccv_nnc_cmd_exec(CMD_DATA_TRANSFER_FORWARD(), ccv_nnc_no_hint, 0, TENSOR_LIST(cpu_input), TENSOR_LIST(ccv_nnc_tensor_from_symbol(tensor_arena, x)), 0);
606
1
    ccv_nnc_cmd_exec(CMD_DATA_TRANSFER_FORWARD(), ccv_nnc_no_hint, 0, TENSOR_LIST(cpu_fit), TENSOR_LIST(ccv_nnc_tensor_from_symbol(tensor_arena, label)), 0);
607
1
    ccv_nnc_cmd_exec(CMD_SET_FORWARD(0), ccv_nnc_no_hint, 0, TENSOR_LIST(), TENSOR_LIST(ccv_nnc_tensor_from_symbol(tensor_arena, bias1), ccv_nnc_tensor_from_symbol(tensor_arena, bias3)), 0);
608
1
    ccv_nnc_cmd_exec(CMD_DATA_TRANSFER_FORWARD(), ccv_nnc_no_hint, 0, TENSOR_LIST(w1_tensor, w3_tensor), TENSOR_LIST(ccv_nnc_tensor_from_symbol(tensor_arena, w1), ccv_nnc_tensor_from_symbol(tensor_arena, w3)), 0);
609
1
    ccv_nnc_graph_run(graph, 0, TRAVERSE_FULL, 0, 0);
610
1
    ccv_nnc_tensor_t* np_updated[4];
611
1
    np_updated[0] = ccv_nnc_tensor_new(0, CPU_TENSOR_NCHW(32F, 8, 3, 5, 5), 0);
612
1
    np_updated[1] = ccv_nnc_tensor_new(0, CPU_TENSOR_NCHW(32F, 8), 0);
613
1
    np_updated[2] = ccv_nnc_tensor_new(0, CPU_TENSOR_NCHW(32F, 8, 8, 5, 5), 0);
614
1
    np_updated[3] = ccv_nnc_tensor_new(0, CPU_TENSOR_NCHW(32F, 8), 0);
615
1
    ccv_nnc_cmd_exec(CMD_DATA_TRANSFER_FORWARD(), ccv_nnc_no_hint, 0, TENSOR_LIST(ccv_nnc_tensor_from_symbol(tensor_arena, updated_params[0]), ccv_nnc_tensor_from_symbol(tensor_arena, updated_params[1]), ccv_nnc_tensor_from_symbol(tensor_arena, updated_params[2]), ccv_nnc_tensor_from_symbol(tensor_arena, updated_params[3])), np_updated, 4, 0);
616
1
    ccv_nnc_symbolic_graph_free(symbolic_graph);
617
1
    ccv_nnc_graph_free(graph);
618
1
    ccv_nnc_tensor_arena_free(tensor_arena);
619
1
    ccv_nnc_graph_exec_arena_free(graph_exec_arena);
620
1
    REQUIRE_ARRAY_EQ_WITH_TOLERANCE(float, np_updated[0]->data.f32, updated[0]->data.f32, 8 * 3 * 5 * 5, 1e-4, "updated params should be equal");
621
1
    REQUIRE_ARRAY_EQ_WITH_TOLERANCE(float, np_updated[1]->data.f32, updated[1]->data.f32, 8, 1e-5, "updated params should be equal");
622
1
    REQUIRE_ARRAY_EQ_WITH_TOLERANCE(float, np_updated[2]->data.f32, updated[2]->data.f32, 8 * 8 * 5 * 5, 1e-4, "updated params should be equal");
623
1
    REQUIRE_ARRAY_EQ_WITH_TOLERANCE(float, np_updated[3]->data.f32, updated[3]->data.f32, 8, 1e-4, "updated params should be equal");
624
1
    ccv_nnc_tensor_free(cpu_input);
625
1
    ccv_nnc_tensor_free(cpu_fit);
626
1
    ccv_nnc_tensor_free(np_updated[0]);
627
1
    ccv_nnc_tensor_free(np_updated[1]);
628
1
    ccv_nnc_tensor_free(np_updated[2]);
629
1
    ccv_nnc_tensor_free(np_updated[3]);
630
1
  }
631
0
  ccv_nnc_tensor_free(updated[0]);
632
1
  ccv_nnc_tensor_free(updated[1]);
633
1
  ccv_nnc_tensor_free(updated[2]);
634
1
  ccv_nnc_tensor_free(updated[3]);
635
1
  ccv_nnc_tensor_free(cpu_inputs[0]);
636
1
  ccv_nnc_tensor_free(cpu_inputs[1]);
637
1
  ccv_nnc_tensor_free(cpu_fits[0]);
638
1
  ccv_nnc_tensor_free(cpu_fits[1]);
639
1
  ccv_nnc_tensor_free(w1_tensor);
640
1
  ccv_nnc_tensor_free(w3_tensor);
641
1
}
642
643
#include "case_main.h"