Coverage Report

Created: 2019-07-03 22:50

/home/liu/buildslave/linux-x64-runtests/build/test/unit/nnc/cnnp.core.tests.c
Line
Count
Source
1
#include "case.h"
2
#include "ccv_case.h"
3
#include "ccv_nnc_case.h"
4
#include <ccv.h>
5
#include <nnc/ccv_nnc.h>
6
#include <nnc/ccv_nnc_easy.h>
7
#include "3rdparty/dsfmt/dSFMT.h"
8
9
TEST_SETUP()
10
{
11
  ccv_nnc_init();
12
}
13
14
static ccv_cnnp_model_t* simple_cifar_10(void)
15
2
{
16
2
  return ccv_cnnp_sequential_new(MODEL_LIST(
17
2
    ccv_cnnp_convolution(1, 32, DIM_ALLOC(5, 5), (ccv_cnnp_param_t){
18
2
      .activation = CCV_CNNP_ACTIVATION_RELU,
19
2
      .hint = HINT((1, 1), (2, 2)),
20
2
    }),
21
2
    ccv_cnnp_max_pool(DIM_ALLOC(3, 3), (ccv_cnnp_param_t){
22
2
      .hint = HINT((2, 2), (0, 0)),
23
2
    }),
24
2
    ccv_cnnp_convolution(1, 32, DIM_ALLOC(5, 5), (ccv_cnnp_param_t){
25
2
      .activation = CCV_CNNP_ACTIVATION_RELU,
26
2
      .hint = HINT((1, 1), (2, 2)),
27
2
    }),
28
2
    ccv_cnnp_average_pool(DIM_ALLOC(3, 3), (ccv_cnnp_param_t){
29
2
      .hint = HINT((2, 2), (0, 0)),
30
2
    }),
31
2
    ccv_cnnp_convolution(1, 64, DIM_ALLOC(5, 5), (ccv_cnnp_param_t){
32
2
      .activation = CCV_CNNP_ACTIVATION_RELU,
33
2
      .hint = HINT((1, 1), (2, 2)),
34
2
    }),
35
2
    ccv_cnnp_average_pool(DIM_ALLOC(3, 3), (ccv_cnnp_param_t){
36
2
      .hint = HINT((2, 2), (0, 0)),
37
2
    }),
38
2
    ccv_cnnp_flatten(),
39
2
    ccv_cnnp_dense(256, (ccv_cnnp_param_t){
40
2
      .activation = CCV_CNNP_ACTIVATION_RELU,
41
2
    }),
42
2
    ccv_cnnp_dense(10, (ccv_cnnp_param_t){
43
2
      .activation = CCV_CNNP_ACTIVATION_SOFTMAX,
44
2
    })
45
2
  ));
46
2
}
47
48
TEST_CASE("compile simple cifar-10 model")
49
1
{
50
1
  ccv_cnnp_model_t* const sequential = simple_cifar_10();
51
1
  const ccv_nnc_tensor_param_t input = CPU_TENSOR_NHWC(32F, 1, 31, 31, 3);
52
1
  ccv_cnnp_model_compile(sequential, &input, 1, CMD_SGD_FORWARD(1, 0.001, 1, 0.99, 0.9, 0), CMD_CATEGORICAL_CROSSENTROPY_FORWARD());
53
1
  ccv_nnc_tensor_t* const input_tensor = ccv_nnc_tensor_new(0, CPU_TENSOR_NHWC(32F, 1, 31, 31, 3), 0);
54
1
  dsfmt_t dsfmt;
55
1
  int i;
56
1
  dsfmt_init_gen_rand(&dsfmt, 1);
57
2.88k
  for (i = 0; i < 31 * 31 * 3; 
i++2.88k
)
58
2.88k
    input_tensor->data.f32[i] = dsfmt_genrand_open_close(&dsfmt) * 2 - 1;
59
1
  ccv_nnc_tensor_t* const output_tensor = ccv_nnc_tensor_new(0, CPU_TENSOR_NHWC(32F, 1, 10), 0);
60
1
  memset(output_tensor->data.f32, 0, sizeof(float) * 10);
61
1
  ccv_cnnp_model_evaluate(sequential, (ccv_cnnp_evaluate_param_t){
62
1
    .is_test = 1
63
1
  }, TENSOR_LIST(input_tensor), TENSOR_LIST(output_tensor), 0);
64
1
  int t = 0;
65
1
  float max = output_tensor->data.f32[0];
66
10
  for (i = 1; i < 10; 
i++9
)
67
9
    if (output_tensor->data.f32[i] > max)
68
1
      max = output_tensor->data.f32[i], t = i;
69
1
  const int target = (t + 1) % 10;
70
1
  REQUIRE_NOT_EQ(target, t, "should not fit");
71
1
  // Doing training.
72
1
  ccv_nnc_tensor_t* const fit_tensor = ccv_nnc_tensor_new(0, CPU_TENSOR_NHWC(32F, 1), 0);
73
1
  fit_tensor->data.f32[0] = target;
74
101
  for (i = 0; i < 100; 
i++100
)
75
100
    ccv_cnnp_model_fit(sequential, TENSOR_LIST(input_tensor), TENSOR_LIST(fit_tensor), TENSOR_LIST(output_tensor), 0);
76
1
  memset(output_tensor->data.f32, 0, sizeof(float) * 10);
77
1
  // After training, it should fit.
78
1
  ccv_cnnp_model_evaluate(sequential, (ccv_cnnp_evaluate_param_t){
79
1
    .is_test = 1
80
1
  }, TENSOR_LIST(input_tensor), TENSOR_LIST(output_tensor), 0);
81
1
  t = 0;
82
1
  max = output_tensor->data.f32[0];
83
10
  for (i = 1; i < 10; 
i++9
)
84
9
    if (output_tensor->data.f32[i] > max)
85
4
      max = output_tensor->data.f32[i], t = i;
86
1
  REQUIRE_EQ(target, t, "should fit");
87
1
  remove("/tmp/compile_simple_cifar_10_model.checkpoint");
88
1
  ccv_cnnp_model_checkpoint(sequential, "/tmp/compile_simple_cifar_10_model.checkpoint", 0);
89
1
  CNNP_MODEL_GEN(sequential, CCV_NNC_LONG_DOT_GRAPH);
90
1
  ccv_cnnp_model_free(sequential);
91
1
  ccv_cnnp_model_t* const sequential2 = simple_cifar_10();
92
1
  ccv_cnnp_model_compile(sequential2, &input, 1, CMD_SGD_FORWARD(1, 0.001, 1, 0.99, 0.9, 0), CMD_CATEGORICAL_CROSSENTROPY_FORWARD());
93
1
  // Load from the checkpoint file.
94
1
  ccv_cnnp_model_checkpoint(sequential2, "/tmp/compile_simple_cifar_10_model.checkpoint", 0);
95
1
  remove("/tmp/compile_simple_cifar_10_model.checkpoint");
96
1
  memset(output_tensor->data.f32, 0, sizeof(float) * 10);
97
1
  ccv_cnnp_model_evaluate(sequential2, (ccv_cnnp_evaluate_param_t){
98
1
    .is_test = 1
99
1
  }, TENSOR_LIST(input_tensor), TENSOR_LIST(output_tensor), 0);
100
1
  t = 0;
101
1
  max = output_tensor->data.f32[0];
102
10
  for (i = 1; i < 10; 
i++9
)
103
9
    if (output_tensor->data.f32[i] > max)
104
4
      max = output_tensor->data.f32[i], t = i;
105
1
  REQUIRE_EQ(target, t, "should fit");
106
1
  ccv_cnnp_model_free(sequential2);
107
1
  ccv_nnc_tensor_free(input_tensor);
108
1
  ccv_nnc_tensor_free(fit_tensor);
109
1
  ccv_nnc_tensor_free(output_tensor);
110
1
}
111
112
TEST_CASE("inception layer for model")
113
1
{
114
1
  const ccv_cnnp_model_io_t x = ccv_cnnp_input();
115
1
  ccv_cnnp_model_io_t tower_1 = ccv_cnnp_model_apply(ccv_cnnp_convolution(1, 64, DIM_ALLOC(1, 1), (ccv_cnnp_param_t){
116
1
    .activation = CCV_CNNP_ACTIVATION_RELU,
117
1
    .hint = HINT((1, 1), (0, 0)),
118
1
  }), MODEL_IO_LIST(x));
119
1
  tower_1 = ccv_cnnp_model_apply(ccv_cnnp_convolution(1, 64, DIM_ALLOC(3, 3), (ccv_cnnp_param_t){
120
1
    .activation = CCV_CNNP_ACTIVATION_RELU,
121
1
    .hint = HINT((1, 1), (1, 1)),
122
1
  }), MODEL_IO_LIST(tower_1));
123
1
124
1
  ccv_cnnp_model_io_t tower_2 = ccv_cnnp_model_apply(ccv_cnnp_convolution(1, 64, DIM_ALLOC(1, 1), (ccv_cnnp_param_t){
125
1
    .activation = CCV_CNNP_ACTIVATION_RELU,
126
1
    .hint = HINT((1, 1), (0, 0)),
127
1
  }), MODEL_IO_LIST(x));
128
1
  tower_2 = ccv_cnnp_model_apply(ccv_cnnp_convolution(1, 64, DIM_ALLOC(5, 5), (ccv_cnnp_param_t){
129
1
    .activation = CCV_CNNP_ACTIVATION_RELU,
130
1
    .hint = HINT((1, 1), (2, 2)),
131
1
  }), MODEL_IO_LIST(tower_2));
132
1
133
1
  ccv_cnnp_model_io_t tower_3 = ccv_cnnp_model_apply(ccv_cnnp_max_pool(DIM_ALLOC(3, 3), (ccv_cnnp_param_t){
134
1
    .hint = HINT((1, 1), (1, 1)),
135
1
  }), MODEL_IO_LIST(x));
136
1
  tower_3 = ccv_cnnp_model_apply(ccv_cnnp_convolution(1, 64, DIM_ALLOC(1, 1), (ccv_cnnp_param_t){
137
1
    .activation = CCV_CNNP_ACTIVATION_RELU,
138
1
    .hint = HINT((1, 1), (0, 0)),
139
1
  }), MODEL_IO_LIST(tower_3));
140
1
141
1
  ccv_cnnp_model_io_t output = ccv_cnnp_model_apply(ccv_cnnp_add(), MODEL_IO_LIST(tower_1, tower_2, tower_3));
142
1
143
1
  ccv_cnnp_model_t* inception = ccv_cnnp_model_new(MODEL_IO_LIST(x), MODEL_IO_LIST(output));
144
1
  const ccv_nnc_tensor_param_t input = GPU_TENSOR_NCHW(000, 32F, 1, 3, 256, 256);
145
1
  ccv_cnnp_model_compile(inception, &input, 1, CMD_SGD_FORWARD(1, 0.001, 1, 0.99, 0.9, 0), CMD_CATEGORICAL_CROSSENTROPY_FORWARD());
146
1
  CNNP_MODEL_GEN(inception, CCV_NNC_LONG_DOT_GRAPH);
147
1
  ccv_cnnp_model_free(inception);
148
1
}
149
150
TEST_CASE("functional model's IO can represent multiple outputs")
151
1
{
152
1
  ccv_cnnp_model_io_t input0 = ccv_cnnp_input();
153
1
  ccv_cnnp_model_io_t input1 = ccv_cnnp_input();
154
1
  ccv_cnnp_model_io_t output0 = ccv_cnnp_model_apply(ccv_cnnp_convolution(1, 64, DIM_ALLOC(1, 1), (ccv_cnnp_param_t){
155
1
    .activation = CCV_CNNP_ACTIVATION_RELU,
156
1
    .hint = HINT((1, 1), (0, 0)),
157
1
  }), MODEL_IO_LIST(input0));
158
1
  ccv_cnnp_model_io_t output1 = ccv_cnnp_model_apply(ccv_cnnp_convolution(1, 64, DIM_ALLOC(3, 3), (ccv_cnnp_param_t){
159
1
    .activation = CCV_CNNP_ACTIVATION_RELU,
160
1
    .hint = HINT((1, 1), (1, 1)),
161
1
  }), MODEL_IO_LIST(input1));
162
1
  ccv_cnnp_model_t* model0 = ccv_cnnp_model_new(MODEL_IO_LIST(input0, input1), MODEL_IO_LIST(output0, output1));
163
1
  input0 = ccv_cnnp_input();
164
1
  input1 = ccv_cnnp_input();
165
1
  output0 = ccv_cnnp_model_apply(model0, MODEL_IO_LIST(input0, input1));
166
1
  ccv_cnnp_model_io_t input2 = ccv_cnnp_input();
167
1
  output1 = ccv_cnnp_model_apply(ccv_cnnp_convolution(1, 64, DIM_ALLOC(5, 5), (ccv_cnnp_param_t){
168
1
    .activation = CCV_CNNP_ACTIVATION_RELU,
169
1
    .hint = HINT((1, 1), (2, 2)),
170
1
  }), MODEL_IO_LIST(input2));
171
1
  ccv_cnnp_model_t* interim = ccv_cnnp_model_new(MODEL_IO_LIST(input0, input1, input2), MODEL_IO_LIST(output0, output1));
172
1
  input0 = ccv_cnnp_input();
173
1
  input1 = ccv_cnnp_input();
174
1
  input2 = ccv_cnnp_input();
175
1
  output0 = ccv_cnnp_model_apply(interim, MODEL_IO_LIST(input0, input1, input2));
176
1
  output0 = ccv_cnnp_model_apply(ccv_cnnp_add(), MODEL_IO_LIST(output0));
177
1
  ccv_cnnp_model_t* final = ccv_cnnp_model_new(MODEL_IO_LIST(input0, input1, input2), MODEL_IO_LIST(output0));
178
1
  const ccv_nnc_tensor_param_t a0 = GPU_TENSOR_NCHW(000, 32F, 1, 3, 256, 256);
179
1
  const ccv_nnc_tensor_param_t a1 = GPU_TENSOR_NCHW(000, 32F, 1, 3, 256, 256);
180
1
  const ccv_nnc_tensor_param_t a2 = GPU_TENSOR_NCHW(000, 32F, 1, 3, 256, 256);
181
1
  ccv_cnnp_model_compile(final, TENSOR_PARAM_LIST(a0, a1, a2), CMD_SGD_FORWARD(1, 0.001, 1, 0.99, 0.9, 0), CMD_CATEGORICAL_CROSSENTROPY_FORWARD());
182
1
  CNNP_MODEL_GEN(final, CCV_NNC_LONG_DOT_GRAPH);
183
1
  ccv_cnnp_model_free(final);
184
1
}
185
186
TEST_CASE("make sure reuse model enables share weights")
187
1
{
188
1
  ccv_cnnp_model_io_t input0 = ccv_cnnp_input();
189
1
  ccv_cnnp_model_io_t input1 = ccv_cnnp_input();
190
1
  ccv_cnnp_model_t* const dense = ccv_cnnp_dense(1, (ccv_cnnp_param_t){});
191
1
  ccv_cnnp_model_io_t output0 = ccv_cnnp_model_apply(dense, MODEL_IO_LIST(input0));
192
1
  ccv_cnnp_model_io_t output1 = ccv_cnnp_model_apply(dense, MODEL_IO_LIST(input1));
193
1
  ccv_cnnp_model_io_t final_output = ccv_cnnp_model_apply(ccv_cnnp_add(), MODEL_IO_LIST(output0, output1));
194
1
  ccv_cnnp_model_t* const final = ccv_cnnp_model_new(MODEL_IO_LIST(input0, input1), MODEL_IO_LIST(final_output));
195
1
  ccv_nnc_tensor_param_t a0 = CPU_TENSOR_NCHW(32F, 1, 1);
196
1
  ccv_nnc_tensor_param_t a1 = CPU_TENSOR_NCHW(32F, 1, 1);
197
1
  ccv_cnnp_model_compile(final, TENSOR_PARAM_LIST(a0, a1), CMD_SGD_FORWARD(1, 0.001, 1, 0.99, 0.9, 0), CMD_CATEGORICAL_CROSSENTROPY_FORWARD());
198
1
  CNNP_MODEL_GEN(final, CCV_NNC_LONG_DOT_GRAPH);
199
1
  ccv_cnnp_model_free(final);
200
1
}
201
202
TEST_CASE("train model with share weights and L2 loss")
203
1
{
204
1
  ccv_cnnp_model_io_t input0 = ccv_cnnp_input();
205
1
  ccv_cnnp_model_io_t input1 = ccv_cnnp_input();
206
1
  ccv_cnnp_model_t* const dense = ccv_cnnp_dense(1, (ccv_cnnp_param_t){});
207
1
  ccv_cnnp_model_io_t output0 = ccv_cnnp_model_apply(dense, MODEL_IO_LIST(input0));
208
1
  ccv_cnnp_model_io_t output1 = ccv_cnnp_model_apply(dense, MODEL_IO_LIST(input1));
209
1
  ccv_cnnp_model_io_t fit0 = ccv_cnnp_input();
210
1
  ccv_cnnp_model_io_t fit1 = ccv_cnnp_input();
211
1
  // Because we don't have L2 loss function available yet, manually create L2 loss.
212
1
  ccv_cnnp_model_io_t diff0 = ccv_cnnp_model_apply(
213
1
    ccv_cnnp_cmd_exec(CMD_ADD_FORWARD(1, -1), ccv_nnc_no_hint, 0,
214
1
      MODEL_CMD_TENSOR_MAP(KV(CCV_CNNP_IO), KV(CCV_CNNP_IO)),
215
1
      MODEL_CMD_TENSOR_LIST(CCV_CNNP_IO)),
216
1
    MODEL_IO_LIST(output0, fit0));
217
1
  ccv_cnnp_model_io_t sqr0 = ccv_cnnp_model_apply(
218
1
    ccv_cnnp_cmd_exec(CMD_EWPROD_FORWARD(), ccv_nnc_no_hint, 0,
219
1
      MODEL_CMD_TENSOR_MAP(KV(CCV_CNNP_IO), KV(CCV_CNNP_IO)),
220
1
      MODEL_CMD_TENSOR_LIST(CCV_CNNP_IO)),
221
1
    MODEL_IO_LIST(diff0, diff0));
222
1
  ccv_cnnp_model_io_t diff1 = ccv_cnnp_model_apply(
223
1
    ccv_cnnp_cmd_exec(CMD_ADD_FORWARD(1, -1), ccv_nnc_no_hint, 0,
224
1
      MODEL_CMD_TENSOR_MAP(KV(CCV_CNNP_IO), KV(CCV_CNNP_IO)),
225
1
      MODEL_CMD_TENSOR_LIST(CCV_CNNP_IO)),
226
1
    MODEL_IO_LIST(output1, fit1));
227
1
  ccv_cnnp_model_io_t sqr1 = ccv_cnnp_model_apply(
228
1
    ccv_cnnp_cmd_exec(CMD_EWPROD_FORWARD(), ccv_nnc_no_hint, 0,
229
1
      MODEL_CMD_TENSOR_MAP(KV(CCV_CNNP_IO), KV(CCV_CNNP_IO)),
230
1
      MODEL_CMD_TENSOR_LIST(CCV_CNNP_IO)),
231
1
    MODEL_IO_LIST(diff1, diff1));
232
1
  ccv_cnnp_model_io_t final_output = ccv_cnnp_model_apply(ccv_cnnp_add(), MODEL_IO_LIST(sqr0, sqr1));
233
1
  ccv_cnnp_model_t* const final = ccv_cnnp_model_new(MODEL_IO_LIST(input0, input1, fit0, fit1), MODEL_IO_LIST(final_output));
234
1
  ccv_nnc_tensor_param_t a0 = CPU_TENSOR_NCHW(32F, 1, 1);
235
1
  ccv_nnc_tensor_param_t a1 = CPU_TENSOR_NCHW(32F, 1, 1);
236
1
  ccv_nnc_tensor_param_t b0 = CPU_TENSOR_NCHW(32F, 1, 1);
237
1
  ccv_nnc_tensor_param_t b1 = CPU_TENSOR_NCHW(32F, 1, 1);
238
1
  ccv_cnnp_model_compile(final, TENSOR_PARAM_LIST(a0, a1, b0, b1), CMD_SGD_FORWARD(0, 0.1, 1, 0.1, 0, 0), CMD_NOOP());
239
1
  CNNP_MODEL_GEN(final, CCV_NNC_LONG_DOT_GRAPH);
240
1
  ccv_nnc_tensor_t* a0_tensor = ccv_nnc_tensor_new(0, a0, 0);
241
1
  ccv_nnc_tensor_t* a1_tensor = ccv_nnc_tensor_new(0, a1, 0);
242
1
  ccv_nnc_tensor_t* b0_tensor = ccv_nnc_tensor_new(0, b0, 0);
243
1
  ccv_nnc_tensor_t* b1_tensor = ccv_nnc_tensor_new(0, b1, 0);
244
1
  ccv_nnc_tensor_t* o0_tensor = ccv_nnc_tensor_new(0, CPU_TENSOR_NCHW(32F, 1), 0);
245
1
  a0_tensor->data.f32[0] = 1;
246
1
  a1_tensor->data.f32[0] = 3;
247
1
  b0_tensor->data.f32[0] = 2;
248
1
  b1_tensor->data.f32[0] = 3;
249
1
  int i;
250
11
  for (i = 0; i < 10; 
i++10
)
251
10
    ccv_cnnp_model_fit(final, TENSOR_LIST(a0_tensor, a1_tensor, b0_tensor, b1_tensor), 0, 0, TENSOR_LIST(o0_tensor), 0);
252
1
  ccv_cnnp_model_set_minimizer(final, CMD_SGD_FORWARD(0, 0.01, 1, 0.01, 0, 0), 0, 0);
253
101
  for (i = 0; i < 100; 
i++100
)
254
100
    ccv_cnnp_model_fit(final, TENSOR_LIST(a0_tensor, a1_tensor, b0_tensor, b1_tensor), 0, 0, TENSOR_LIST(o0_tensor), 0);
255
1
  ccv_cnnp_model_set_minimizer(final, CMD_SGD_FORWARD(0, 0.001, 1, 0.001, 0, 0), 0, 0);
256
1.00k
  for (i = 0; i < 1000; 
i++1.00k
)
257
1.00k
    ccv_cnnp_model_fit(final, TENSOR_LIST(a0_tensor, a1_tensor, b0_tensor, b1_tensor), 0, 0, TENSOR_LIST(o0_tensor), 0);
258
1
  a0_tensor->data.f32[0] = 2;
259
1
  a1_tensor->data.f32[0] = 2; // The final result should be 4.
260
1
  b0_tensor->data.f32[0] = 2; // diff is 0.5
261
1
  b1_tensor->data.f32[0] = 3; // diff is 0.5, and 0.5^2 + 0.5^2 = 0.5.
262
1
  ccv_cnnp_model_evaluate(final, (ccv_cnnp_evaluate_param_t){
263
1
    .is_test = 1
264
1
  }, TENSOR_LIST(a0_tensor, a1_tensor, b0_tensor, b1_tensor), TENSOR_LIST(o0_tensor), 0);
265
1
  REQUIRE_EQ_WITH_TOLERANCE(o0_tensor->data.f32[0], 0.5, 2 * 1e-2, "We should linear regressed this.");
266
1
  ccv_nnc_tensor_free(a0_tensor);
267
1
  ccv_nnc_tensor_free(a1_tensor);
268
1
  ccv_nnc_tensor_free(b0_tensor);
269
1
  ccv_nnc_tensor_free(b1_tensor);
270
1
  ccv_nnc_tensor_free(o0_tensor);
271
1
  ccv_cnnp_model_free(final);
272
1
}
273
274
static ccv_cnnp_model_t* simple_cifar_10_no_softmax(void)
275
2
{
276
2
  return ccv_cnnp_sequential_new(MODEL_LIST(
277
2
    ccv_cnnp_convolution(1, 32, DIM_ALLOC(5, 5), (ccv_cnnp_param_t){
278
2
      .activation = CCV_CNNP_ACTIVATION_RELU,
279
2
      .hint = HINT((1, 1), (2, 2)),
280
2
    }),
281
2
    ccv_cnnp_max_pool(DIM_ALLOC(3, 3), (ccv_cnnp_param_t){
282
2
      .hint = HINT((2, 2), (0, 0)),
283
2
    }),
284
2
    ccv_cnnp_convolution(1, 32, DIM_ALLOC(5, 5), (ccv_cnnp_param_t){
285
2
      .activation = CCV_CNNP_ACTIVATION_RELU,
286
2
      .hint = HINT((1, 1), (2, 2)),
287
2
    }),
288
2
    ccv_cnnp_average_pool(DIM_ALLOC(3, 3), (ccv_cnnp_param_t){
289
2
      .hint = HINT((2, 2), (0, 0)),
290
2
    }),
291
2
    ccv_cnnp_convolution(1, 64, DIM_ALLOC(5, 5), (ccv_cnnp_param_t){
292
2
      .activation = CCV_CNNP_ACTIVATION_RELU,
293
2
      .hint = HINT((1, 1), (2, 2)),
294
2
    }),
295
2
    ccv_cnnp_average_pool(DIM_ALLOC(3, 3), (ccv_cnnp_param_t){
296
2
      .hint = HINT((2, 2), (0, 0)),
297
2
    }),
298
2
    ccv_cnnp_flatten(),
299
2
    ccv_cnnp_dense(256, (ccv_cnnp_param_t){
300
2
      .activation = CCV_CNNP_ACTIVATION_RELU,
301
2
    }),
302
2
    ccv_cnnp_dense(10, (ccv_cnnp_param_t){
303
2
      .activation = CCV_CNNP_ACTIVATION_NONE,
304
2
    })
305
2
  ));
306
2
}
307
308
TEST_CASE("evaluate cifar-10 model in multi-stage mode")
309
1
{
310
1
  ccv_cnnp_model_t* const sequential = simple_cifar_10_no_softmax();
311
1
  const ccv_nnc_tensor_param_t input = CPU_TENSOR_NHWC(32F, 1, 31, 31, 3);
312
1
  ccv_cnnp_model_compile(sequential, &input, 1, CMD_SGD_FORWARD(0, 0.001, 1, 0.99, 0.9, 0.9), CMD_NOOP());
313
1
  ccv_nnc_tensor_t* const input_tensor = ccv_nnc_tensor_new(0, CPU_TENSOR_NHWC(32F, 1, 31, 31, 3), 0);
314
1
  dsfmt_t dsfmt;
315
1
  int i;
316
1
  dsfmt_init_gen_rand(&dsfmt, 1);
317
2.88k
  for (i = 0; i < 31 * 31 * 3; 
i++2.88k
)
318
2.88k
    input_tensor->data.f32[i] = dsfmt_genrand_open_close(&dsfmt) * 2 - 1;
319
1
  ccv_nnc_tensor_t* const output_tensor = ccv_nnc_tensor_new(0, CPU_TENSOR_NHWC(32F, 1, 10), 0);
320
1
  memset(output_tensor->data.f32, 0, sizeof(float) * 10);
321
1
  ccv_cnnp_model_evaluate(sequential, (ccv_cnnp_evaluate_param_t){
322
1
    .is_test = 1
323
1
  }, TENSOR_LIST(input_tensor), TENSOR_LIST(output_tensor), 0);
324
1
  int t = 0;
325
1
  float max = output_tensor->data.f32[0];
326
10
  for (i = 1; i < 10; 
i++9
)
327
9
    if (output_tensor->data.f32[i] > max)
328
1
      max = output_tensor->data.f32[i], t = i;
329
1
  const int target = (t + 1) % 10;
330
1
  REQUIRE_NOT_EQ(target, t, "should not fit");
331
1
  // Doing training.
332
1
  ccv_nnc_tensor_t* const fit_tensor = ccv_nnc_tensor_new(0, CPU_TENSOR_NHWC(32F, 1), 0);
333
1
  fit_tensor->data.f32[0] = target;
334
1
  ccv_nnc_tensor_t* const softmax_tensor = ccv_nnc_tensor_new(0, CPU_TENSOR_NHWC(32F, 1, 10), 0);
335
1
  ccv_nnc_tensor_t* const loss_tensor = ccv_nnc_tensor_new(0, CPU_TENSOR_NHWC(32F, 1), 0);
336
1
  ccv_nnc_tensor_t* const ingrad_tensor = ccv_nnc_tensor_new(0, CPU_TENSOR_NHWC(32F, 1, 10), 0);
337
101
  for (i = 0; i < 100; 
i++100
)
338
100
  {
339
100
    ccv_cnnp_model_evaluate(sequential, (ccv_cnnp_evaluate_param_t){
340
100
      .requires_grad = 1
341
100
    }, TENSOR_LIST(input_tensor), TENSOR_LIST(output_tensor), 0);
342
100
    ccv_nnc_cmd_exec(CMD_SOFTMAX_CROSSENTROPY_FORWARD(), ccv_nnc_no_hint, 0, TENSOR_LIST(output_tensor, fit_tensor), TENSOR_LIST(loss_tensor, softmax_tensor), 0);
343
100
    ccv_nnc_cmd_exec(CMD_SOFTMAX_CROSSENTROPY_BACKWARD(), ccv_nnc_no_hint, 0, TENSOR_LIST(0, 0, output_tensor, fit_tensor, loss_tensor, softmax_tensor), TENSOR_LIST(ingrad_tensor), 0);
344
100
    ccv_cnnp_model_backward(sequential, TENSOR_LIST(ingrad_tensor), 0, 0, 0);
345
100
    ccv_cnnp_model_apply_gradients(sequential, 0);
346
100
  }
347
1
  memset(output_tensor->data.f32, 0, sizeof(float) * 10);
348
1
  // After training, it should fit.
349
1
  ccv_cnnp_model_evaluate(sequential, (ccv_cnnp_evaluate_param_t){
350
1
    .is_test = 1
351
1
  }, TENSOR_LIST(input_tensor), TENSOR_LIST(output_tensor), 0);
352
1
  t = 0;
353
1
  max = output_tensor->data.f32[0];
354
10
  for (i = 1; i < 10; 
i++9
)
355
9
    if (output_tensor->data.f32[i] > max)
356
3
      max = output_tensor->data.f32[i], t = i;
357
1
  REQUIRE_EQ(target, t, "should fit");
358
1
  ccv_nnc_tensor_free(ingrad_tensor);
359
1
  ccv_nnc_tensor_free(fit_tensor);
360
1
  ccv_nnc_tensor_free(softmax_tensor);
361
1
  ccv_nnc_tensor_free(loss_tensor);
362
1
  ccv_nnc_tensor_free(input_tensor);
363
1
  ccv_nnc_tensor_free(output_tensor);
364
1
  ccv_cnnp_model_free(sequential);
365
1
}
366
367
TEST_CASE("evaluate cifar-10 model in multi-stage mode with gradient accumulated")
368
1
{
369
1
  ccv_cnnp_model_t* const sequential = simple_cifar_10_no_softmax();
370
1
  const ccv_nnc_tensor_param_t input = CPU_TENSOR_NHWC(32F, 1, 31, 31, 3);
371
1
  ccv_cnnp_model_compile(sequential, &input, 1, CMD_SGD_FORWARD(0, 0.00033, 1, 0.99, 0.9, 0.9), CMD_NOOP());
372
1
  ccv_nnc_tensor_t* const input_tensor = ccv_nnc_tensor_new(0, CPU_TENSOR_NHWC(32F, 1, 31, 31, 3), 0);
373
1
  dsfmt_t dsfmt;
374
1
  int i;
375
1
  dsfmt_init_gen_rand(&dsfmt, 1);
376
2.88k
  for (i = 0; i < 31 * 31 * 3; 
i++2.88k
)
377
2.88k
    input_tensor->data.f32[i] = dsfmt_genrand_open_close(&dsfmt) * 2 - 1;
378
1
  ccv_nnc_tensor_t* const output_tensor = ccv_nnc_tensor_new(0, CPU_TENSOR_NHWC(32F, 1, 10), 0);
379
1
  memset(output_tensor->data.f32, 0, sizeof(float) * 10);
380
1
  ccv_cnnp_model_evaluate(sequential, (ccv_cnnp_evaluate_param_t){
381
1
    .is_test = 1
382
1
  }, TENSOR_LIST(input_tensor), TENSOR_LIST(output_tensor), 0);
383
1
  int t = 0;
384
1
  float max = output_tensor->data.f32[0];
385
10
  for (i = 1; i < 10; 
i++9
)
386
9
    if (output_tensor->data.f32[i] > max)
387
1
      max = output_tensor->data.f32[i], t = i;
388
1
  const int target = (t + 1) % 10;
389
1
  REQUIRE_NOT_EQ(target, t, "should not fit");
390
1
  // Doing training.
391
1
  ccv_nnc_tensor_t* const fit_tensor = ccv_nnc_tensor_new(0, CPU_TENSOR_NHWC(32F, 1), 0);
392
1
  fit_tensor->data.f32[0] = target;
393
1
  ccv_nnc_tensor_t* const softmax_tensor = ccv_nnc_tensor_new(0, CPU_TENSOR_NHWC(32F, 1, 10), 0);
394
1
  ccv_nnc_tensor_t* const loss_tensor = ccv_nnc_tensor_new(0, CPU_TENSOR_NHWC(32F, 1), 0);
395
1
  ccv_nnc_tensor_t* const ingrad_tensor = ccv_nnc_tensor_new(0, CPU_TENSOR_NHWC(32F, 1, 10), 0);
396
101
  for (i = 0; i < 100; 
i++100
)
397
100
  {
398
100
    ccv_cnnp_model_evaluate(sequential, (ccv_cnnp_evaluate_param_t){
399
100
      .requires_grad = 1
400
100
    }, TENSOR_LIST(input_tensor), TENSOR_LIST(output_tensor), 0);
401
100
    ccv_nnc_cmd_exec(CMD_SOFTMAX_CROSSENTROPY_FORWARD(), ccv_nnc_no_hint, 0, TENSOR_LIST(output_tensor, fit_tensor), TENSOR_LIST(loss_tensor, softmax_tensor), 0);
402
100
    ccv_nnc_cmd_exec(CMD_SOFTMAX_CROSSENTROPY_BACKWARD(), ccv_nnc_no_hint, 0, TENSOR_LIST(0, 0, output_tensor, fit_tensor, loss_tensor, softmax_tensor), TENSOR_LIST(ingrad_tensor), 0);
403
100
    ccv_cnnp_model_backward(sequential, TENSOR_LIST(ingrad_tensor), 0, 0, 0);
404
100
    // Backward again to accumulate gradient.
405
100
    if (i % 2 == 0)
406
50
    {
407
50
      ccv_cnnp_model_backward(sequential, TENSOR_LIST(ingrad_tensor), 0, 0, 0);
408
50
      // Backward again to accumulate gradient.
409
50
      if (i % 3 == 0)
410
17
        ccv_cnnp_model_backward(sequential, TENSOR_LIST(ingrad_tensor), 0, 0, 0);
411
50
    }
412
100
    ccv_cnnp_model_apply_gradients(sequential, 0);
413
100
  }
414
1
  memset(output_tensor->data.f32, 0, sizeof(float) * 10);
415
1
  // After training, it should fit.
416
1
  ccv_cnnp_model_evaluate(sequential, (ccv_cnnp_evaluate_param_t){
417
1
    .is_test = 1
418
1
  }, TENSOR_LIST(input_tensor), TENSOR_LIST(output_tensor), 0);
419
1
  t = 0;
420
1
  max = output_tensor->data.f32[0];
421
10
  for (i = 1; i < 10; 
i++9
)
422
9
    if (output_tensor->data.f32[i] > max)
423
4
      max = output_tensor->data.f32[i], t = i;
424
1
  REQUIRE_EQ(target, t, "should fit");
425
1
  ccv_nnc_tensor_free(ingrad_tensor);
426
1
  ccv_nnc_tensor_free(fit_tensor);
427
1
  ccv_nnc_tensor_free(softmax_tensor);
428
1
  ccv_nnc_tensor_free(loss_tensor);
429
1
  ccv_nnc_tensor_free(input_tensor);
430
1
  ccv_nnc_tensor_free(output_tensor);
431
1
  ccv_cnnp_model_free(sequential);
432
1
}
433
434
TEST_CASE("train model with share weights and L2 loss and check out gradients")
435
1
{
436
1
  ccv_cnnp_model_io_t input0 = ccv_cnnp_input();
437
1
  ccv_cnnp_model_io_t input1 = ccv_cnnp_input();
438
1
  ccv_cnnp_model_t* const dense = ccv_cnnp_dense(1, (ccv_cnnp_param_t){});
439
1
  ccv_cnnp_model_io_t output0 = ccv_cnnp_model_apply(dense, MODEL_IO_LIST(input0));
440
1
  ccv_cnnp_model_io_t output1 = ccv_cnnp_model_apply(dense, MODEL_IO_LIST(input1));
441
1
  ccv_cnnp_model_io_t fit0 = ccv_cnnp_input();
442
1
  ccv_cnnp_model_io_t fit1 = ccv_cnnp_input();
443
1
  // Because we don't have L2 loss function available yet, manually create L2 loss.
444
1
  ccv_cnnp_model_io_t diff0 = ccv_cnnp_model_apply(
445
1
    ccv_cnnp_cmd_exec(CMD_ADD_FORWARD(1, -1), ccv_nnc_no_hint, 0,
446
1
      MODEL_CMD_TENSOR_MAP(KV(CCV_CNNP_IO), KV(CCV_CNNP_IO)),
447
1
      MODEL_CMD_TENSOR_LIST(CCV_CNNP_IO)),
448
1
    MODEL_IO_LIST(output0, fit0));
449
1
  ccv_cnnp_model_io_t sqr0 = ccv_cnnp_model_apply(
450
1
    ccv_cnnp_cmd_exec(CMD_EWPROD_FORWARD(), ccv_nnc_no_hint, 0,
451
1
      MODEL_CMD_TENSOR_MAP(KV(CCV_CNNP_IO), KV(CCV_CNNP_IO)),
452
1
      MODEL_CMD_TENSOR_LIST(CCV_CNNP_IO)),
453
1
    MODEL_IO_LIST(diff0, diff0));
454
1
  ccv_cnnp_model_io_t diff1 = ccv_cnnp_model_apply(
455
1
    ccv_cnnp_cmd_exec(CMD_ADD_FORWARD(1, -1), ccv_nnc_no_hint, 0,
456
1
      MODEL_CMD_TENSOR_MAP(KV(CCV_CNNP_IO), KV(CCV_CNNP_IO)),
457
1
      MODEL_CMD_TENSOR_LIST(CCV_CNNP_IO)),
458
1
    MODEL_IO_LIST(output1, fit1));
459
1
  ccv_cnnp_model_io_t sqr1 = ccv_cnnp_model_apply(
460
1
    ccv_cnnp_cmd_exec(CMD_EWPROD_FORWARD(), ccv_nnc_no_hint, 0,
461
1
      MODEL_CMD_TENSOR_MAP(KV(CCV_CNNP_IO), KV(CCV_CNNP_IO)),
462
1
      MODEL_CMD_TENSOR_LIST(CCV_CNNP_IO)),
463
1
    MODEL_IO_LIST(diff1, diff1));
464
1
  ccv_cnnp_model_io_t final_output = ccv_cnnp_model_apply(ccv_cnnp_add(), MODEL_IO_LIST(sqr0, sqr1));
465
1
  ccv_cnnp_model_t* const final = ccv_cnnp_model_new(MODEL_IO_LIST(input0, input1, fit0, fit1), MODEL_IO_LIST(final_output));
466
1
  ccv_nnc_tensor_param_t a0 = CPU_TENSOR_NCHW(32F, 1, 1);
467
1
  ccv_nnc_tensor_param_t a1 = CPU_TENSOR_NCHW(32F, 1, 1);
468
1
  ccv_nnc_tensor_param_t b0 = CPU_TENSOR_NCHW(32F, 1, 1);
469
1
  ccv_nnc_tensor_param_t b1 = CPU_TENSOR_NCHW(32F, 1, 1);
470
1
  ccv_cnnp_model_compile(final, TENSOR_PARAM_LIST(a0, a1, b0, b1), CMD_SGD_FORWARD(0, 0.1, 1, 0.1, 0, 0), CMD_NOOP());
471
1
  CNNP_MODEL_GEN(final, CCV_NNC_LONG_DOT_GRAPH);
472
1
  ccv_nnc_tensor_t* a0_tensor = ccv_nnc_tensor_new(0, a0, 0);
473
1
  ccv_nnc_tensor_t* a1_tensor = ccv_nnc_tensor_new(0, a1, 0);
474
1
  ccv_nnc_tensor_t* b0_tensor = ccv_nnc_tensor_new(0, b0, 0);
475
1
  ccv_nnc_tensor_t* b1_tensor = ccv_nnc_tensor_new(0, b1, 0);
476
1
  ccv_nnc_tensor_t* o0_tensor = ccv_nnc_tensor_new(0, CPU_TENSOR_NCHW(32F, 1), 0);
477
1
  // It should fit to 1*0.5+1.5=2, 3*0.5+1.5=3
478
1
  a0_tensor->data.f32[0] = 1;
479
1
  a1_tensor->data.f32[0] = 3;
480
1
  b0_tensor->data.f32[0] = 2;
481
1
  b1_tensor->data.f32[0] = 3;
482
1
  int i;
483
11
  for (i = 0; i < 10; 
i++10
)
484
10
    ccv_cnnp_model_fit(final, TENSOR_LIST(a0_tensor, a1_tensor, b0_tensor, b1_tensor), 0, 0, TENSOR_LIST(o0_tensor), 0);
485
1
  ccv_cnnp_model_set_minimizer(final, CMD_SGD_FORWARD(0, 0.01, 1, 0.01, 0, 0), 0, 0);
486
101
  for (i = 0; i < 100; 
i++100
)
487
100
    ccv_cnnp_model_fit(final, TENSOR_LIST(a0_tensor, a1_tensor, b0_tensor, b1_tensor), 0, 0, TENSOR_LIST(o0_tensor), 0);
488
1
  ccv_cnnp_model_set_minimizer(final, CMD_SGD_FORWARD(0, 0.001, 1, 0.001, 0, 0), 0, 0);
489
1.00k
  for (i = 0; i < 1000; 
i++1.00k
)
490
1.00k
    ccv_cnnp_model_fit(final, TENSOR_LIST(a0_tensor, a1_tensor, b0_tensor, b1_tensor), 0, 0, TENSOR_LIST(o0_tensor), 0);
491
1
  a0_tensor->data.f32[0] = 2;
492
1
  a1_tensor->data.f32[0] = 2; // The final result should be 4.
493
1
  b0_tensor->data.f32[0] = 2; // diff is 0.5
494
1
  b1_tensor->data.f32[0] = 3; // diff is 0.5, and 0.5^2 + 0.5^2 = 0.5.
495
1
  ccv_cnnp_model_evaluate(final, (ccv_cnnp_evaluate_param_t){
496
1
    .is_test = 1
497
1
  }, TENSOR_LIST(a0_tensor, a1_tensor, b0_tensor, b1_tensor), TENSOR_LIST(o0_tensor), 0);
498
1
  REQUIRE_EQ_WITH_TOLERANCE(o0_tensor->data.f32[0], 0.5, 2 * 1e-2, "We should linear regressed this.");
499
1
  // Figure out the actual weight and bias term in the model.
500
1
  a0_tensor->data.f32[0] = 0;
501
1
  a1_tensor->data.f32[0] = 0;
502
1
  b0_tensor->data.f32[0] = 0;
503
1
  b1_tensor->data.f32[0] = 0;
504
1
  // The output will be 2*bias^2
505
1
  ccv_cnnp_model_evaluate(final, (ccv_cnnp_evaluate_param_t){
506
1
    .is_test = 1
507
1
  }, TENSOR_LIST(a0_tensor, a1_tensor, b0_tensor, b1_tensor), TENSOR_LIST(o0_tensor), 0);
508
1
  const float bias = sqrtf(o0_tensor->data.f32[0] * 0.5);
509
1
  a0_tensor->data.f32[0] = 1;
510
1
  a1_tensor->data.f32[0] = 1;
511
1
  b0_tensor->data.f32[0] = 0;
512
1
  b1_tensor->data.f32[0] = 0;
513
1
  // The output will be 2*(w+bias)^2
514
1
  ccv_cnnp_model_evaluate(final, (ccv_cnnp_evaluate_param_t){
515
1
    .is_test = 1
516
1
  }, TENSOR_LIST(a0_tensor, a1_tensor, b0_tensor, b1_tensor), TENSOR_LIST(o0_tensor), 0);
517
1
  const float w = sqrt(o0_tensor->data.f32[0] * 0.5) - bias;
518
1
  // Compute the out gradient to verify.
519
1
  a0_tensor->data.f32[0] = 2;
520
1
  a1_tensor->data.f32[0] = 2; // The final result should be 4.
521
1
  b0_tensor->data.f32[0] = 2; // diff is 0.5
522
1
  b1_tensor->data.f32[0] = 3; // diff is 0.5, and 0.5^2 + 0.5^2 = 0.5.
523
1
  ccv_cnnp_model_evaluate(final, (ccv_cnnp_evaluate_param_t){
524
1
    .requires_grad = 1,
525
1
    .enable_outgrad = 1,
526
1
  }, TENSOR_LIST(a0_tensor, a1_tensor, b0_tensor, b1_tensor), TENSOR_LIST(o0_tensor), 0);
527
1
  // Note that I have to use new tensors and have to keep these tensors around since they were binded to the model when evaluate.
528
1
  ccv_nnc_tensor_t* da0_tensor = ccv_nnc_tensor_new(0, a0, 0);
529
1
  ccv_nnc_tensor_t* da1_tensor = ccv_nnc_tensor_new(0, a1, 0);
530
1
  ccv_nnc_tensor_t* db0_tensor = ccv_nnc_tensor_new(0, b0, 0);
531
1
  ccv_nnc_tensor_t* db1_tensor = ccv_nnc_tensor_new(0, b1, 0);
532
1
  ccv_nnc_tensor_t* do0_tensor = ccv_nnc_tensor_new(0, CPU_TENSOR_NCHW(32F, 1), 0);
533
1
  do0_tensor->data.f32[0] = 1;
534
1
  ccv_cnnp_model_backward(final, TENSOR_LIST(do0_tensor), TENSOR_LIST(da0_tensor, da1_tensor, db0_tensor, db1_tensor), 0);
535
1
  REQUIRE_EQ_WITH_TOLERANCE(da0_tensor->data.f32[0], 2 * w * (w * 2 + bias - 2), 1e-5, "da0=2*w*(w*a0+bias-b0), thus, 0.5");
536
1
  REQUIRE_EQ_WITH_TOLERANCE(da1_tensor->data.f32[0], 2 * w * (w * 2 + bias - 3), 1e-5, "da1=2*w*(w*a1+bias-b1), thus, -0.5");
537
1
  REQUIRE_EQ_WITH_TOLERANCE(db0_tensor->data.f32[0], -2 * (w * 2 + bias - 2), 1e-5, "db0=-2*(w*a0+bias-b0), thus, -1");
538
1
  REQUIRE_EQ_WITH_TOLERANCE(db1_tensor->data.f32[0], -2 * (w * 2 + bias - 3), 1e-5, "db1=-2*(w*a1+bias-b1), thus, 1");
539
1
  ccv_nnc_tensor_free(a0_tensor);
540
1
  ccv_nnc_tensor_free(a1_tensor);
541
1
  ccv_nnc_tensor_free(b0_tensor);
542
1
  ccv_nnc_tensor_free(b1_tensor);
543
1
  ccv_nnc_tensor_free(o0_tensor);
544
1
  ccv_nnc_tensor_free(da0_tensor);
545
1
  ccv_nnc_tensor_free(da1_tensor);
546
1
  ccv_nnc_tensor_free(db0_tensor);
547
1
  ccv_nnc_tensor_free(db1_tensor);
548
1
  ccv_nnc_tensor_free(do0_tensor);
549
1
  ccv_cnnp_model_free(final);
550
1
}
551
552
#include "case_main.h"