Coverage Report

Created: 2024-06-21 10:32

/home/liu/actions-runner/_work/ccv/ccv/test/unit/convnet.tests.c
Line
Count
Source
1
#include "ccv.h"
2
#include "case.h"
3
#include "ccv_case.h"
4
#include "3rdparty/dsfmt/dSFMT.h"
5
6
TEST_CASE("convolutional network of 11x11 on 225x225 with uniform weights")
7
1
{
8
1
  ccv_convnet_layer_param_t params = {
9
1
    .type = CCV_CONVNET_CONVOLUTIONAL,
10
1
    .bias = 0,
11
1
    .glorot = sqrtf(2),
12
1
    .input = {
13
1
      .matrix = {
14
1
        .rows = 225,
15
1
        .cols = 225,
16
1
        .channels = 3,
17
1
        .partition = 1,
18
1
      },
19
1
    },
20
1
    .output = {
21
1
      .convolutional = {
22
1
        .count = 4,
23
1
        .strides = 4,
24
1
        .border = 1,
25
1
        .rows = 11,
26
1
        .cols = 11,
27
1
        .channels = 3,
28
1
        .partition = 1,
29
1
      },
30
1
    },
31
1
  };
32
1
  ccv_convnet_t* convnet = ccv_convnet_new(0, ccv_size(225, 225), &params, 1);
33
1
  int i, x, y;
34
1.45k
  for (i = 0; i < 11 * 11 * 3 * 4; 
i++1.45k
)
35
1.45k
    convnet->layers[0].w[i] = 1;
36
1
  ccv_dense_matrix_t* a = ccv_dense_matrix_new(225, 225, CCV_32F | CCV_C3, 0, 0);
37
151k
  for (i = 0; i < 225 * 225 * 3; 
i++151k
)
38
151k
    a->data.f32[i] = 1;
39
1
  ccv_dense_matrix_t* b = 0;
40
1
  ccv_convnet_encode(convnet, &a, &b, 1);
41
1
  ccv_matrix_free(a);
42
1
  REQUIRE(b->rows == 55 && b->cols == 55, "11x11 convolves on 225x255 with strides 4 should produce 55x55 matrix");
43
1
  ccv_dense_matrix_t* c = ccv_dense_matrix_new(55, 55, CCV_32F | 4, 0, 0);
44
56
  for (y = 0; y < 55; 
y++55
)
45
3.08k
    
for (x = 0; 55
x < 55; 
x++3.02k
)
46
15.1k
      
for (i = 0; 3.02k
i < 4; 
i++12.1k
)
47
12.1k
      c->data.f32[(y * 55 + x) * 4 + i] = ((x == 0 && 
y == 0220
) || 
(12.0k
x == 012.0k
 && 
y == 54216
) || 
(12.0k
x == 5412.0k
 && 
y == 0220
) || 
(12.0k
x == 5412.0k
 && 
y == 54216
)) ? 
30016
 : 
(12.0k
(12.0k
x == 012.0k
 || 
y == 011.8k
 || 
x == 5411.6k
 || 
y == 5411.4k
) ? 
330848
 : 
36311.2k
);
48
1
  REQUIRE_MATRIX_EQ(b, c, "55x55 matrix should be exactly a matrix fill 363, with 300 on the corner and 330 on the border");
49
1
  ccv_matrix_free(b);
50
1
  ccv_matrix_free(c);
51
1
  ccv_convnet_free(convnet);
52
1
}
53
54
TEST_CASE("convolutional network of 5x5 on 27x27 with uniform weights")
55
1
{
56
1
  ccv_convnet_layer_param_t params = {
57
1
    .type = CCV_CONVNET_CONVOLUTIONAL,
58
1
    .bias = 0,
59
1
    .glorot = sqrtf(2),
60
1
    .input = {
61
1
      .matrix = {
62
1
        .rows = 27,
63
1
        .cols = 27,
64
1
        .channels = 1,
65
1
        .partition = 1,
66
1
      },
67
1
    },
68
1
    .output = {
69
1
      .convolutional = {
70
1
        .count = 4,
71
1
        .strides = 1,
72
1
        .border = 2,
73
1
        .rows = 5,
74
1
        .cols = 5,
75
1
        .channels = 1,
76
1
        .partition = 1,
77
1
      },
78
1
    },
79
1
  };
80
1
  ccv_convnet_t* convnet = ccv_convnet_new(0, ccv_size(27, 27), &params, 1);
81
1
  int i, x, y;
82
101
  for (i = 0; i < 5 * 5 * 4; 
i++100
)
83
100
    convnet->layers->w[i] = 1;
84
1
  ccv_dense_matrix_t* a = ccv_dense_matrix_new(27, 27, CCV_32F | CCV_C1, 0, 0);
85
730
  for (i = 0; i < 27 * 27; 
i++729
)
86
729
    a->data.f32[i] = 1;
87
1
  ccv_dense_matrix_t* b = 0;
88
1
  ccv_convnet_encode(convnet, &a, &b, 1);
89
1
  REQUIRE(b->rows == 27 && b->cols == 27, "5x5 convolves on 27x27 with border 2 should produce 27x27 matrix");
90
1
  ccv_matrix_free(a);
91
1
  ccv_dense_matrix_t* c = ccv_dense_matrix_new(27, 27, CCV_32F | 4, 0, 0);
92
28
  for (y = 0; y < 27; 
y++27
)
93
756
    
for (x = 0; 27
x < 27; 
x++729
)
94
3.64k
      
for (i = 0; 729
i < 4; 
i++2.91k
)
95
2.91k
      {
96
2.91k
        if ((x == 0 && 
y == 0108
) || 
(2.91k
x == 02.91k
 && 
y == 26104
) || 
(2.90k
x == 262.90k
 && 
y == 0108
) || 
(2.90k
x == 262.90k
 && 
y == 26104
))
97
16
          c->data.f32[(y * 27 + x) * 4 + i] = 9;
98
2.90k
        else if ((x == 0 && 
y == 1100
) || 
(2.89k
x == 02.89k
 && 
y == 2596
) || 
(2.89k
x == 12.89k
 && 
y == 0108
) || 
(2.88k
x == 12.88k
 && 
y == 26104
) || 
(2.88k
x == 252.88k
 && 
y == 0108
) || 
(2.88k
x == 252.88k
 && 
y == 26104
) || 
(2.87k
x == 262.87k
 && 
y == 1100
) || 
(2.87k
x == 262.87k
 && 
y == 2596
))
99
32
          c->data.f32[(y * 27 + x) * 4 + i] = 12;
100
2.86k
        else if (x == 0 || 
y == 02.77k
 || 
x == 262.68k
 || 
y == 262.59k
)
101
368
          c->data.f32[(y * 27 + x) * 4 + i] = 15;
102
2.50k
        else if ((x == 1 && 
y == 1100
) || 
(2.49k
x == 12.49k
 && 
y == 2596
) || 
(2.49k
x == 252.49k
 && 
y == 1100
) || 
(2.48k
x == 252.48k
 && 
y == 2596
))
103
16
          c->data.f32[(y * 27 + x) * 4 + i] = 16;
104
2.48k
        else if (x == 1 || 
y == 12.39k
 || 
x == 252.30k
 || 
y == 252.20k
)
105
368
          c->data.f32[(y * 27 + x) * 4 + i] = 20;
106
2.11k
        else
107
2.11k
          c->data.f32[(y * 27 + x) * 4 + i] = 25;
108
2.91k
      }
109
1
  REQUIRE_MATRIX_EQ(b, c, "27x27 matrix should be exactly a matrix fill 25, with 9, 16 on the corner and 12, 15, 20 on the border");
110
1
  ccv_matrix_free(b);
111
1
  ccv_matrix_free(c);
112
1
  ccv_convnet_free(convnet);
113
1
}
114
115
TEST_CASE("convolutional network of 11x11 on 225x225 with non-uniform weights")
116
1
{
117
1
  ccv_convnet_layer_param_t params = {
118
1
    .type = CCV_CONVNET_CONVOLUTIONAL,
119
1
    .bias = 0,
120
1
    .glorot = sqrtf(2),
121
1
    .input = {
122
1
      .matrix = {
123
1
        .rows = 225,
124
1
        .cols = 225,
125
1
        .channels = 1,
126
1
        .partition = 1,
127
1
      },
128
1
    },
129
1
    .output = {
130
1
      .convolutional = {
131
1
        .count = 4,
132
1
        .strides = 4,
133
1
        .border = 1,
134
1
        .rows = 11,
135
1
        .cols = 11,
136
1
        .channels = 1,
137
1
        .partition = 1,
138
1
      },
139
1
    },
140
1
  };
141
1
  ccv_convnet_t* convnet = ccv_convnet_new(0, ccv_size(225, 225), &params, 1);
142
1
  int i, x, y;
143
5
  for (x = 0; x < 4; 
x++4
)
144
488
    
for (i = 0; 4
i < 11 * 11; 
i++484
)
145
484
      convnet->layers[0].w[x * 11 * 11 + i] = i + 1;
146
1
  ccv_dense_matrix_t* a = ccv_dense_matrix_new(225, 225, CCV_32F | CCV_C1, 0, 0);
147
50.6k
  for (i = 0; i < 225 * 225; 
i++50.6k
)
148
50.6k
    a->data.f32[i] = i + 1;
149
1
  ccv_dense_matrix_t* b = 0;
150
1
  ccv_convnet_encode(convnet, &a, &b, 1);
151
1
  ccv_matrix_free(a);
152
1
  REQUIRE(b->rows == 55 && b->cols == 55, "11x11 convolves on 225x255 with strides 4 should produce 55x55 matrix");
153
1
  ccv_dense_matrix_t* c = ccv_dense_matrix_new(55, 55, CCV_32F | 4, 0, 0);
154
1
  float sum = 0;
155
  // first column
156
11
  for (y = 0; y < 10; 
y++10
)
157
110
    
for (x = 0; 10
x < 10; 
x++100
)
158
100
      sum += ((y + 1) * 11 + x + 2) * (y * 225 + x + 1);
159
5
  for (i = 0; i < 4; 
i++4
)
160
4
    c->data.f32[i] = sum;
161
1
  sum = 0;
162
11
  for (y = 0; y < 10; 
y++10
)
163
120
    
for (x = 0; 10
x < 11; 
x++110
)
164
110
      sum += ((y + 1) * 11 + x + 1) * (y * 225 + (x + 3) + 1);
165
54
  for (x = 1; x < 54; 
x++53
)
166
265
    
for (i = 0; 53
i < 4; 
i++212
)
167
212
      c->data.f32[x * 4 + i] = sum + (x - 1) * 4 * (11 * 11 + 12) * 11 * 10 / 2;
168
1
  sum = 0;
169
11
  for (y = 0; y < 10; 
y++10
)
170
110
    
for (x = 0; 10
x < 10; 
x++100
)
171
100
      sum += ((y + 1) * 11 + x + 1) * (y * 225 + (x + 215) + 1);
172
5
  for (i = 0; i < 4; 
i++4
)
173
4
    c->data.f32[54 * 4 + i] = sum;
174
  // last column
175
1
  sum = 0;
176
11
  for (y = 0; y < 10; 
y++10
)
177
110
    
for (x = 0; 10
x < 10; 
x++100
)
178
100
      sum += (y * 11 + x + 2) * ((y + 215) * 225 + x + 1);
179
5
  for (i = 0; i < 4; 
i++4
)
180
4
    c->data.f32[55 * 54 * 4 + i] = sum;
181
1
  sum = 0;
182
11
  for (y = 0; y < 10; 
y++10
)
183
120
    
for (x = 0; 10
x < 11; 
x++110
)
184
110
      sum += (y * 11 + x + 1) * ((y + 215) * 225 + (x + 3) + 1);
185
54
  for (x = 1; x < 54; 
x++53
)
186
265
    
for (i = 0; 53
i < 4; 
i++212
)
187
212
      c->data.f32[(55 * 54 + x) * 4 + i] = sum + (x - 1) * 4 * (10 * 11 + 1) * 11 * 10 / 2;
188
1
  sum = 0;
189
11
  for (y = 0; y < 10; 
y++10
)
190
110
    
for (x = 0; 10
x < 10; 
x++100
)
191
100
      sum += (y * 11 + x + 1) * ((y + 215) * 225 + (x + 215) + 1);
192
5
  for (i = 0; i < 4; 
i++4
)
193
4
    c->data.f32[(55 * 54 + 54) * 4 + i] = sum;
194
1
  float border[] = {
195
1
    0, 0
196
1
  };
197
12
  for (y = 0; y < 11; 
y++11
)
198
121
    
for (x = 0; 11
x < 10; 
x++110
)
199
110
      border[0] += (y * 11 + x + 2) * ((y + 3) * 225 + x + 1);
200
12
  for (y = 0; y < 11; 
y++11
)
201
121
    
for (x = 0; 11
x < 10; 
x++110
)
202
110
      border[1] += (y * 11 + x + 1) * ((y + 3) * 225 + (x + 215) + 1);
203
1
  sum = 0;
204
12
  for (y = 0; y < 11; 
y++11
)
205
132
    
for (x = 0; 11
x < 11; 
x++121
)
206
121
      sum += (y * 11 + x + 1) * ((y + 3) * 225 + (x + 3) + 1);
207
54
  for (y = 1; y < 54; 
y++53
)
208
53
  {
209
265
    for (i = 0; i < 4; 
i++212
)
210
212
      c->data.f32[y * 55 * 4 + i] = border[0];
211
2.86k
    for (x = 1; x < 54; 
x++2.80k
)
212
14.0k
      
for (i = 0; 2.80k
i < 4; 
i++11.2k
)
213
11.2k
        c->data.f32[(y * 55 + x) * 4 + i] = sum + (x - 1) * 4 * (11 * 11 + 1) * 11 * 11 / 2;
214
265
    for (i = 0; i < 4; 
i++212
)
215
212
      c->data.f32[(y * 55 + 54) * 4 + i] = border[1];
216
53
    sum += 225 * 4 * (11 * 11 + 1) * 11 * 11 / 2;
217
53
    border[0] += 225 * 4 * ((11 * 11 + 1) * 11 * 11 / 2 - (10 * 11 + 1 + 1) * 11 / 2);
218
53
    border[1] += 225 * 4 * ((11 * 11 + 1) * 11 * 11 / 2 - (11 * 11 + 11) * 11 / 2);
219
53
  }
220
  // regularize the output so it is within the tolerance
221
12.1k
  for (i = 0; i < 55 * 55 * 4; 
i++12.1k
)
222
12.1k
    c->data.f32[i] = c->data.f32[i] * 1e-7, b->data.f32[i] = b->data.f32[i] * 1e-7;
223
1
  REQUIRE_MATRIX_EQ(b, c, "55x55 matrix should be exactly the same");
224
1
  ccv_matrix_free(b);
225
1
  ccv_matrix_free(c);
226
1
  ccv_convnet_free(convnet);
227
1
}
228
229
TEST_CASE("convolutional network of 5x5 on 27x27 with non-uniform weights")
230
1
{
231
1
  ccv_convnet_layer_param_t params = {
232
1
    .type = CCV_CONVNET_CONVOLUTIONAL,
233
1
    .bias = 0,
234
1
    .glorot = sqrtf(2),
235
1
    .input = {
236
1
      .matrix = {
237
1
        .rows = 27,
238
1
        .cols = 27,
239
1
        .channels = 1,
240
1
        .partition = 1,
241
1
      },
242
1
    },
243
1
    .output = {
244
1
      .convolutional = {
245
1
        .count = 4,
246
1
        .strides = 1,
247
1
        .border = 2,
248
1
        .rows = 5,
249
1
        .cols = 5,
250
1
        .channels = 1,
251
1
        .partition = 1,
252
1
      },
253
1
    },
254
1
  };
255
1
  ccv_convnet_t* convnet = ccv_convnet_new(0, ccv_size(27, 27), &params, 1);
256
1
  int i, x, y;
257
5
  for (x = 0; x < 4; 
x++4
)
258
104
    
for (i = 0; 4
i < 5 * 5; 
i++100
)
259
100
      convnet->layers->w[x * 5 * 5 + i] = i + 1;
260
1
  ccv_dense_matrix_t* a = ccv_dense_matrix_new(27, 27, CCV_32F | CCV_C1, 0, 0);
261
730
  for (i = 0; i < 27 * 27; 
i++729
)
262
729
    a->data.f32[i] = i + 1;
263
1
  ccv_dense_matrix_t* b = 0;
264
1
  ccv_convnet_encode(convnet, &a, &b, 1);
265
1
  REQUIRE(b->rows == 27 && b->cols == 27, "5x5 convolves on 27x27 with border 2 should produce 27x27 matrix");
266
1
  ccv_matrix_free(a);
267
1
  ccv_dense_matrix_t* c = ccv_dense_matrix_new(27, 27, CCV_32F | 4, 0, 0);
268
  // the first column
269
1
  float sum = 0;
270
4
  for (y = 0; y < 3; 
y++3
)
271
12
    
for (x = 0; 3
x < 3; 
x++9
)
272
9
      sum += ((y + 2) * 5 + x + 3) * (y * 27 + x + 1);
273
5
  for (i = 0; i < 4; 
i++4
)
274
4
    c->data.f32[i] = sum;
275
1
  sum = 0;
276
4
  for (y = 0; y < 3; 
y++3
)
277
15
    
for (x = 0; 3
x < 4; 
x++12
)
278
12
      sum += ((y + 2) * 5 + x + 2) * (y * 27 + x + 1);
279
5
  for (i = 0; i < 4; 
i++4
)
280
4
    c->data.f32[4 + i] = sum;
281
1
  sum = 0;
282
4
  for (y = 0; y < 3; 
y++3
)
283
18
    
for (x = 0; 3
x < 5; 
x++15
)
284
15
      sum += ((y + 2) * 5 + x + 1) * (y * 27 + x + 1);
285
24
  for (x = 2; x < 25; 
x++23
)
286
115
    
for (i = 0; 23
i < 4; 
i++92
)
287
92
      c->data.f32[x * 4 + i] = sum + (x - 2) * 36 * 15 / 2;
288
1
  sum = 0;
289
4
  for (y = 0; y < 3; 
y++3
)
290
15
    
for (x = 0; 3
x < 4; 
x++12
)
291
12
      sum += ((y + 2) * 5 + x + 1) * (y * 27 + x + 24);
292
5
  for (i = 0; i < 4; 
i++4
)
293
4
    c->data.f32[25 * 4 + i] = sum;
294
1
  sum = 0;
295
4
  for (y = 0; y < 3; 
y++3
)
296
12
    
for (x = 0; 3
x < 3; 
x++9
)
297
9
      sum += ((y + 2) * 5 + x + 1) * (y * 27 + x + 25);
298
5
  for (i = 0; i < 4; 
i++4
)
299
4
    c->data.f32[26 * 4 + i] = sum;
300
  // the second column
301
1
  sum = 0;
302
5
  for (y = 0; y < 4; 
y++4
)
303
16
    
for (x = 0; 4
x < 3; 
x++12
)
304
12
      sum += ((y + 1) * 5 + x + 3) * (y * 27 + x + 1);
305
5
  for (i = 0; i < 4; 
i++4
)
306
4
    c->data.f32[27 * 4 + i] = sum;
307
1
  sum = 0;
308
5
  for (y = 0; y < 4; 
y++4
)
309
20
    
for (x = 0; 4
x < 4; 
x++16
)
310
16
      sum += ((y + 1) * 5 + x + 2) * (y * 27 + x + 1);
311
5
  for (i = 0; i < 4; 
i++4
)
312
4
    c->data.f32[28 * 4 + i] = sum;
313
1
  sum = 0;
314
5
  for (y = 0; y < 4; 
y++4
)
315
24
    
for (x = 0; 4
x < 5; 
x++20
)
316
20
      sum += ((y + 1) * 5 + x + 1) * (y * 27 + x + 1);
317
24
  for (x = 2; x < 25; 
x++23
)
318
115
    
for (i = 0; 23
i < 4; 
i++92
)
319
92
      c->data.f32[(27 + x) * 4 + i] = sum + (x - 2) * 31 * 20 / 2;
320
1
  sum = 0;
321
5
  for (y = 0; y < 4; 
y++4
)
322
20
    
for (x = 0; 4
x < 4; 
x++16
)
323
16
      sum += ((y + 1) * 5 + x + 1) * (y * 27 + x + 24);
324
5
  for (i = 0; i < 4; 
i++4
)
325
4
    c->data.f32[52 * 4 + i] = sum;
326
1
  sum = 0;
327
5
  for (y = 0; y < 4; 
y++4
)
328
16
    
for (x = 0; 4
x < 3; 
x++12
)
329
12
      sum += ((y + 1) * 5 + x + 1) * (y * 27 + x + 25);
330
5
  for (i = 0; i < 4; 
i++4
)
331
4
    c->data.f32[53 * 4 + i] = sum;
332
1
  sum = 0;
333
  // the last 2nd column
334
5
  for (y = 0; y < 4; 
y++4
)
335
16
    
for (x = 0; 4
x < 3; 
x++12
)
336
12
      sum += (y * 5 + x + 3) * ((y + 23) * 27 + x + 1);
337
5
  for (i = 0; i < 4; 
i++4
)
338
4
    c->data.f32[27 * 25 * 4 + i] = sum;
339
1
  sum = 0;
340
5
  for (y = 0; y < 4; 
y++4
)
341
20
    
for (x = 0; 4
x < 4; 
x++16
)
342
16
      sum += (y * 5 + x + 2) * ((y + 23) * 27 + x + 1);
343
5
  for (i = 0; i < 4; 
i++4
)
344
4
    c->data.f32[(27 * 25 + 1) * 4 + i] = sum;
345
1
  sum = 0;
346
5
  for (y = 0; y < 4; 
y++4
)
347
24
    
for (x = 0; 4
x < 5; 
x++20
)
348
20
      sum += (y * 5 + x + 1) * ((y + 23) * 27 + x + 1);
349
24
  for (x = 2; x < 25; 
x++23
)
350
115
    
for (i = 0; 23
i < 4; 
i++92
)
351
92
      c->data.f32[(27 * 25 + x) * 4 + i] = sum + (x - 2) * 21 * 20 / 2;
352
1
  sum = 0;
353
5
  for (y = 0; y < 4; 
y++4
)
354
20
    
for (x = 0; 4
x < 4; 
x++16
)
355
16
      sum += (y * 5 + x + 1) * ((y + 23) * 27 + x + 24);
356
5
  for (i = 0; i < 4; 
i++4
)
357
4
    c->data.f32[(27 * 25 + 25) * 4 + i] = sum;
358
1
  sum = 0;
359
5
  for (y = 0; y < 4; 
y++4
)
360
16
    
for (x = 0; 4
x < 3; 
x++12
)
361
12
      sum += (y * 5 + x + 1) * ((y + 23) * 27 + x + 25);
362
5
  for (i = 0; i < 4; 
i++4
)
363
4
    c->data.f32[(27 * 25 + 26) * 4 + i] = sum;
364
  // the last column
365
1
  sum = 0;
366
4
  for (y = 0; y < 3; 
y++3
)
367
12
    
for (x = 0; 3
x < 3; 
x++9
)
368
9
      sum += (y * 5 + x + 3) * ((y + 24) * 27 + x + 1);
369
5
  for (i = 0; i < 4; 
i++4
)
370
4
    c->data.f32[27 * 26 * 4 + i] = sum;
371
1
  sum = 0;
372
4
  for (y = 0; y < 3; 
y++3
)
373
15
    
for (x = 0; 3
x < 4; 
x++12
)
374
12
      sum += (y * 5 + x + 2) * ((y + 24) * 27 + x + 1);
375
5
  for (i = 0; i < 4; 
i++4
)
376
4
    c->data.f32[(27 * 26 + 1) * 4 + i] = sum;
377
1
  sum = 0;
378
4
  for (y = 0; y < 3; 
y++3
)
379
18
    
for (x = 0; 3
x < 5; 
x++15
)
380
15
      sum += (y * 5 + x + 1) * ((y + 24) * 27 + x + 1);
381
24
  for (x = 2; x < 25; 
x++23
)
382
115
    
for (i = 0; 23
i < 4; 
i++92
)
383
92
      c->data.f32[(27 * 26 + x) * 4 + i] = sum + (x - 2) * 16 * 15 / 2;
384
1
  sum = 0;
385
4
  for (y = 0; y < 3; 
y++3
)
386
15
    
for (x = 0; 3
x < 4; 
x++12
)
387
12
      sum += (y * 5 + x + 1) * ((y + 24) * 27 + x + 24);
388
5
  for (i = 0; i < 4; 
i++4
)
389
4
    c->data.f32[(27 * 26 + 25) * 4 + i] = sum;
390
1
  sum = 0;
391
4
  for (y = 0; y < 3; 
y++3
)
392
12
    
for (x = 0; 3
x < 3; 
x++9
)
393
9
      sum += (y * 5 + x + 1) * ((y + 24) * 27 + x + 25);
394
5
  for (i = 0; i < 4; 
i++4
)
395
4
    c->data.f32[(27 * 26 + 26) * 4 + i] = sum;
396
1
  float border[] = {
397
1
    0, 0, 0, 0
398
1
  };
399
6
  for (y = 0; y < 5; 
y++5
)
400
20
    
for (x = 0; 5
x < 3; 
x++15
)
401
15
      border[0] += (y * 5 + x + 3) * (y * 27 + x + 1);
402
6
  for (y = 0; y < 5; 
y++5
)
403
25
    
for (x = 0; 5
x < 4; 
x++20
)
404
20
      border[1] += (y * 5 + x + 2) * (y * 27 + x + 1);
405
6
  for (y = 0; y < 5; 
y++5
)
406
25
    
for (x = 0; 5
x < 4; 
x++20
)
407
20
      border[2] += (y * 5 + x + 1) * (y * 27 + x + 24);
408
6
  for (y = 0; y < 5; 
y++5
)
409
20
    
for (x = 0; 5
x < 3; 
x++15
)
410
15
      border[3] += (y * 5 + x + 1) * (y * 27 + x + 25);
411
1
  sum = 0;
412
6
  for (y = 0; y < 5; 
y++5
)
413
30
    
for (x = 0; 5
x < 5; 
x++25
)
414
25
      sum += (y * 5 + x + 1) * (y * 27 + x + 1);
415
24
  for (y = 2; y < 25; 
y++23
)
416
23
  {
417
115
    for (i = 0; i < 4; 
i++92
)
418
92
    {
419
92
      c->data.f32[y * 27 * 4 + i] = border[0] + (y - 2) * 27 * (3 + 4 + 5 + 8 + 9 + 10 + 13 + 14 + 15 + 18 + 19 + 20 + 23 + 24 + 25);
420
92
      c->data.f32[(y * 27 + 1) * 4 + i] = border[1] + (y - 2) * 27 * (2 + 3 + 4 + 5 + 7 + 8 + 9 + 10 + 12 + 13 + 14 + 15 + 17 + 18 + 19 + 20 + 22 + 23 + 24 + 25);
421
2.20k
      for (x = 2; x < 25; 
x++2.11k
)
422
2.11k
        c->data.f32[(y * 27 + x) * 4 + i] = sum + ((y - 2) * 27 + x - 2) * 26 * 25 / 2;
423
92
      c->data.f32[(y * 27 + 25) * 4 + i] = border[2] + (y - 2) * 27 * (1 + 2 + 3 + 4 + 6 + 7 + 8 + 9 + 11 + 12 + 13 + 14 + 16 + 17 + 18 + 19 + 21 + 22 + 23 + 24);
424
92
      c->data.f32[(y * 27 + 26) * 4 + i] = border[3] + (y - 2) * 27 * (1 + 2 + 3 + 6 + 7 + 8 + 11 + 12 + 13 + 16 + 17 + 18 + 21 + 22 + 23);
425
92
    }
426
23
  }
427
1
  REQUIRE_MATRIX_EQ(b, c, "27x27 matrix should be exactly the same");
428
1
  ccv_matrix_free(b);
429
1
  ccv_matrix_free(c);
430
1
  ccv_convnet_free(convnet);
431
1
}
432
433
TEST_CASE("convolutional network of 5x5x4 on 27x27x8 partitioned by 2")
434
1
{
435
1
  ccv_convnet_layer_param_t params = {
436
1
    .type = CCV_CONVNET_CONVOLUTIONAL,
437
1
    .bias = 0,
438
1
    .glorot = sqrtf(2),
439
1
    .input = {
440
1
      .matrix = {
441
1
        .rows = 27,
442
1
        .cols = 27,
443
1
        .channels = 4,
444
1
        .partition = 2,
445
1
      },
446
1
    },
447
1
    .output = {
448
1
      .convolutional = {
449
1
        .count = 8,
450
1
        .strides = 1,
451
1
        .border = 2,
452
1
        .rows = 5,
453
1
        .cols = 5,
454
1
        .channels = 4,
455
1
        .partition = 2,
456
1
      },
457
1
    },
458
1
  };
459
1
  ccv_convnet_t* convnet = ccv_convnet_new(0, ccv_size(27, 27), &params, 1);
460
1
  int i, k;
461
401
  for (i = 0; i < convnet->layers->wnum; 
i++400
)
462
400
    convnet->layers->w[i] = i;
463
9
  for (i = 0; i < convnet->layers->net.convolutional.count; 
i++8
)
464
8
    convnet->layers->bias[i] = i + 1;
465
1
  ccv_dense_matrix_t* a = ccv_dense_matrix_new(27, 27, CCV_32F | 4, 0, 0);
466
2.91k
  for (i = 0; i < 27 * 27 * 4; 
i++2.91k
)
467
2.91k
    a->data.f32[i] = 20 - i;
468
1
  ccv_dense_matrix_t* b = 0;
469
1
  ccv_convnet_encode(convnet, &a, &b, 1);
470
1
  ccv_convnet_layer_param_t partitioned_params = {
471
1
    .type = CCV_CONVNET_CONVOLUTIONAL,
472
1
    .bias = 0,
473
1
    .glorot = sqrtf(2),
474
1
    .input = {
475
1
      .matrix = {
476
1
        .rows = 27,
477
1
        .cols = 27,
478
1
        .channels = 2,
479
1
        .partition = 1,
480
1
      },
481
1
    },
482
1
    .output = {
483
1
      .convolutional = {
484
1
        .count = 4,
485
1
        .strides = 1,
486
1
        .border = 2,
487
1
        .rows = 5,
488
1
        .cols = 5,
489
1
        .channels = 2,
490
1
        .partition = 1,
491
1
      },
492
1
    },
493
1
  };
494
1
  ccv_convnet_t* partitioned_convnet = ccv_convnet_new(0, ccv_size(27, 27), &partitioned_params, 1);
495
1
  memcpy(partitioned_convnet->layers->w, convnet->layers->w, sizeof(float) * (convnet->layers->wnum / 2));
496
1
  memcpy(partitioned_convnet->layers->bias, convnet->layers->bias, sizeof(float) * (convnet->layers->net.convolutional.count / 2));
497
1
  ccv_dense_matrix_t* aa = ccv_dense_matrix_new(27, 27, CCV_32F | 2, 0, 0);
498
730
  for (i = 0; i < 27 * 27; 
i++729
)
499
2.18k
    
for (k = 0; 729
k < 2; 
k++1.45k
)
500
1.45k
      aa->data.f32[i * 2 + k] = a->data.f32[i * 4 + k];
501
1
  ccv_dense_matrix_t* bb = ccv_dense_matrix_new(27, 27, CCV_32F | 8, 0, 0);
502
1
  ccv_dense_matrix_t* cc = 0;
503
1
  ccv_convnet_encode(partitioned_convnet, &aa, &cc, 1);
504
730
  for (i = 0; i < 27 * 27; 
i++729
)
505
3.64k
    
for (k = 0; 729
k < 4; 
k++2.91k
)
506
2.91k
      bb->data.f32[i * 8 + k] = cc->data.f32[i * 4 + k];
507
1
  memcpy(partitioned_convnet->layers->w, convnet->layers->w + (convnet->layers->wnum / 2), sizeof(float) * (convnet->layers->wnum / 2));
508
1
  memcpy(partitioned_convnet->layers->bias, convnet->layers->bias + (convnet->layers->net.convolutional.count / 2), sizeof(float) * (convnet->layers->net.convolutional.count / 2));
509
730
  for (i = 0; i < 27 * 27; 
i++729
)
510
2.18k
    
for (k = 0; 729
k < 2; 
k++1.45k
)
511
1.45k
      aa->data.f32[i * 2 + k] = a->data.f32[i * 4 + 2 + k];
512
1
  ccv_convnet_encode(partitioned_convnet, &aa, &cc, 1);
513
730
  for (i = 0; i < 27 * 27; 
i++729
)
514
3.64k
    
for (k = 0; 729
k < 4; 
k++2.91k
)
515
2.91k
      bb->data.f32[i * 8 + 4 + k] = cc->data.f32[i * 4 + k];
516
1
  REQUIRE_MATRIX_EQ(b, bb, "27x27x8 matrix computed from convnet with partition and partitioned convnet should be exactly the same");
517
1
  ccv_matrix_free(a);
518
1
  ccv_matrix_free(b);
519
1
  ccv_matrix_free(aa);
520
1
  ccv_matrix_free(bb);
521
1
  ccv_matrix_free(cc);
522
1
  ccv_convnet_free(convnet);
523
1
  ccv_convnet_free(partitioned_convnet);
524
1
}
525
526
TEST_CASE("full connect network from 13x13x128 to 2048")
527
1
{
528
1
  ccv_convnet_layer_param_t params = {
529
1
    .type = CCV_CONVNET_FULL_CONNECT,
530
1
    .bias = 0,
531
1
    .glorot = sqrtf(2),
532
1
    .input = {
533
1
      .matrix = {
534
1
        .rows = 13,
535
1
        .cols = 13,
536
1
        .channels = 128,
537
1
        .partition = 1,
538
1
      },
539
1
      .node = {
540
1
        .count = 13 * 13 * 128,
541
1
      },
542
1
    },
543
1
    .output = {
544
1
      .full_connect = {
545
1
        .relu = 0,
546
1
        .count = 2048,
547
1
      },
548
1
    },
549
1
  };
550
1
  ccv_convnet_t* convnet = ccv_convnet_new(0, ccv_size(13, 13), &params, 1);
551
1
  int i;
552
44.3M
  for (i = 0; i < 13 * 13 * 128 * 2048; 
i++44.3M
)
553
44.3M
    convnet->layers->w[i] = 1;
554
1
  ccv_dense_matrix_t* a = ccv_dense_matrix_new(13, 13, CCV_32F | 128, 0, 0);
555
21.6k
  for (i = 0; i < 13 * 13 * 128; 
i++21.6k
)
556
21.6k
    a->data.f32[i] = 1;
557
1
  ccv_dense_matrix_t* b = 0;
558
1
  ccv_convnet_encode(convnet, &a, &b, 1);
559
1
  ccv_matrix_free(a);
560
1
  REQUIRE(b->rows == 2048 && b->cols == 1, "full connect network output should be 2048 neurons");
561
1
  ccv_dense_matrix_t* c = ccv_dense_matrix_new(2048, 1, CCV_32F | CCV_C1, 0, 0);
562
2.04k
  for (i = 0; i < 2048; 
i++2.04k
)
563
2.04k
    c->data.f32[i] = 13 * 13 * 128;
564
1
  REQUIRE_MATRIX_EQ(b, c, "full connect network output should be exactly 13 * 13 * 128");
565
1
  ccv_matrix_free(b);
566
1
  ccv_matrix_free(c);
567
1
  ccv_convnet_free(convnet);
568
1
}
569
570
TEST_CASE("maximum pool network of 55x55 with window of 3x3 and stride of 2")
571
1
{
572
1
  ccv_convnet_layer_param_t params = {
573
1
    .type = CCV_CONVNET_MAX_POOL,
574
1
    .bias = 0,
575
1
    .glorot = sqrtf(2),
576
1
    .input = {
577
1
      .matrix = {
578
1
        .rows = 55,
579
1
        .cols = 55,
580
1
        .channels = 1,
581
1
        .partition = 1,
582
1
      },
583
1
    },
584
1
    .output = {
585
1
      .pool = {
586
1
        .size = 3,
587
1
        .strides = 2,
588
1
        .border = 0,
589
1
      },
590
1
    },
591
1
  };
592
1
  ccv_convnet_t* convnet = ccv_convnet_new(0, ccv_size(55, 55), &params, 1);
593
1
  ccv_dense_matrix_t* a = ccv_dense_matrix_new(55, 55, CCV_32F | CCV_C1, 0, 0);
594
1
  int i, x, y;
595
3.02k
  for (i = 0; i < 55 * 55; 
i++3.02k
)
596
3.02k
    a->data.f32[i] = i + 1;
597
1
  ccv_dense_matrix_t* b = 0;
598
1
  ccv_convnet_encode(convnet, &a, &b, 1);
599
1
  ccv_matrix_free(a);
600
1
  ccv_dense_matrix_t* c = ccv_dense_matrix_new(27, 27, CCV_32F | CCV_C1, 0, 0);
601
28
  for (y = 0; y < 27; 
y++27
)
602
756
    
for (x = 0; 27
x < 27; 
x++729
)
603
729
      c->data.f32[y * 27 + x] = 113 + y * 110 + x * 2;
604
1
  REQUIRE_MATRIX_EQ(b, c, "max pool network output should be exactly the same");
605
1
  ccv_matrix_free(b);
606
1
  ccv_matrix_free(c);
607
1
  ccv_convnet_free(convnet);
608
1
}
609
610
TEST_CASE("maximum pool network of 57x57 with window of 3x3 and stride of 3")
611
1
{
612
1
  ccv_convnet_layer_param_t params = {
613
1
    .type = CCV_CONVNET_MAX_POOL,
614
1
    .bias = 0,
615
1
    .glorot = sqrtf(2),
616
1
    .input = {
617
1
      .matrix = {
618
1
        .rows = 57,
619
1
        .cols = 57,
620
1
        .channels = 1,
621
1
        .partition = 1,
622
1
      },
623
1
    },
624
1
    .output = {
625
1
      .pool = {
626
1
        .size = 3,
627
1
        .strides = 3,
628
1
        .border = 0,
629
1
      },
630
1
    },
631
1
  };
632
1
  ccv_convnet_t* convnet = ccv_convnet_new(0, ccv_size(57, 57), &params, 1);
633
1
  ccv_dense_matrix_t* a = ccv_dense_matrix_new(57, 57, CCV_32F | CCV_C1, 0, 0);
634
1
  int i, x, y;
635
3.25k
  for (i = 0; i < 57 * 57; 
i++3.24k
)
636
3.24k
    a->data.f32[i] = i + 1;
637
1
  ccv_dense_matrix_t* b = 0;
638
1
  ccv_convnet_encode(convnet, &a, &b, 1);
639
1
  ccv_matrix_free(a);
640
1
  ccv_dense_matrix_t* c = ccv_dense_matrix_new(19, 19, CCV_32F | CCV_C1, 0, 0);
641
20
  for (y = 0; y < 19; 
y++19
)
642
380
    
for (x = 0; 19
x < 19; 
x++361
)
643
361
      c->data.f32[y * 19 + x] = 117 + y * 171 + x * 3;
644
1
  REQUIRE_MATRIX_EQ(b, c, "max pool network output should be exactly the same");
645
1
  ccv_matrix_free(b);
646
1
  ccv_matrix_free(c);
647
1
  ccv_convnet_free(convnet);
648
1
}
649
650
TEST_CASE("maximum pool network of 54x54 with window of 2x2 and stride of 2")
651
1
{
652
1
  ccv_convnet_layer_param_t params = {
653
1
    .type = CCV_CONVNET_MAX_POOL,
654
1
    .bias = 0,
655
1
    .glorot = sqrtf(2),
656
1
    .input = {
657
1
      .matrix = {
658
1
        .rows = 54,
659
1
        .cols = 54,
660
1
        .channels = 1,
661
1
        .partition = 1,
662
1
      },
663
1
    },
664
1
    .output = {
665
1
      .pool = {
666
1
        .size = 2,
667
1
        .strides = 2,
668
1
        .border = 0,
669
1
      },
670
1
    },
671
1
  };
672
1
  ccv_convnet_t* convnet = ccv_convnet_new(0, ccv_size(54, 54), &params, 1);
673
1
  ccv_dense_matrix_t* a = ccv_dense_matrix_new(54, 54, CCV_32F | CCV_C1, 0, 0);
674
1
  int i, x, y;
675
2.91k
  for (i = 0; i < 54 * 54; 
i++2.91k
)
676
2.91k
    a->data.f32[i] = i + 1;
677
1
  ccv_dense_matrix_t* b = 0;
678
1
  ccv_convnet_encode(convnet, &a, &b, 1);
679
1
  ccv_matrix_free(a);
680
1
  ccv_dense_matrix_t* c = ccv_dense_matrix_new(27, 27, CCV_32F | CCV_C1, 0, 0);
681
28
  for (y = 0; y < 27; 
y++27
)
682
756
    
for (x = 0; 27
x < 27; 
x++729
)
683
729
      c->data.f32[y * 27 + x] = 56 + y * 108 + x * 2;
684
1
  REQUIRE_MATRIX_EQ(b, c, "max pool network output should be exactly the same");
685
1
  ccv_matrix_free(b);
686
1
  ccv_matrix_free(c);
687
1
  ccv_convnet_free(convnet);
688
1
}
689
690
TEST_CASE("average pool network of 55x55 with window of 3x3 and stride of 2")
691
1
{
692
1
  ccv_convnet_layer_param_t params = {
693
1
    .type = CCV_CONVNET_AVERAGE_POOL,
694
1
    .bias = 0,
695
1
    .glorot = sqrtf(2),
696
1
    .input = {
697
1
      .matrix = {
698
1
        .rows = 55,
699
1
        .cols = 55,
700
1
        .channels = 1,
701
1
        .partition = 1,
702
1
      },
703
1
    },
704
1
    .output = {
705
1
      .pool = {
706
1
        .size = 3,
707
1
        .strides = 2,
708
1
        .border = 0,
709
1
      },
710
1
    },
711
1
  };
712
1
  ccv_convnet_t* convnet = ccv_convnet_new(0, ccv_size(55, 55), &params, 1);
713
1
  ccv_dense_matrix_t* a = ccv_dense_matrix_new(55, 55, CCV_32F | CCV_C1, 0, 0);
714
1
  int i, x, y;
715
3.02k
  for (i = 0; i < 55 * 55; 
i++3.02k
)
716
3.02k
    a->data.f32[i] = i + 1;
717
1
  ccv_dense_matrix_t* b = 0;
718
1
  ccv_convnet_encode(convnet, &a, &b, 1);
719
1
  ccv_matrix_free(a);
720
1
  ccv_dense_matrix_t* c = ccv_dense_matrix_new(27, 27, CCV_32F | CCV_C1, 0, 0);
721
28
  for (y = 0; y < 27; 
y++27
)
722
756
    
for (x = 0; 27
x < 27; 
x++729
)
723
729
      c->data.f32[y * 27 + x] = 57 + y * 110 + x * 2;
724
1
  REQUIRE_MATRIX_EQ(b, c, "average pool network output should be exactly the same");
725
1
  ccv_matrix_free(b);
726
1
  ccv_matrix_free(c);
727
1
  ccv_convnet_free(convnet);
728
1
}
729
730
TEST_CASE("average pool network of 57x57 with window of 3x3 and stride of 3")
731
1
{
732
1
  ccv_convnet_layer_param_t params = {
733
1
    .type = CCV_CONVNET_AVERAGE_POOL,
734
1
    .bias = 0,
735
1
    .glorot = sqrtf(2),
736
1
    .input = {
737
1
      .matrix = {
738
1
        .rows = 57,
739
1
        .cols = 57,
740
1
        .channels = 1,
741
1
        .partition = 1,
742
1
      },
743
1
    },
744
1
    .output = {
745
1
      .pool = {
746
1
        .size = 3,
747
1
        .strides = 3,
748
1
        .border = 0,
749
1
      },
750
1
    },
751
1
  };
752
1
  ccv_convnet_t* convnet = ccv_convnet_new(0, ccv_size(57, 57), &params, 1);
753
1
  ccv_dense_matrix_t* a = ccv_dense_matrix_new(57, 57, CCV_32F | CCV_C1, 0, 0);
754
1
  int i, x, y;
755
3.25k
  for (i = 0; i < 57 * 57; 
i++3.24k
)
756
3.24k
    a->data.f32[i] = i + 1;
757
1
  ccv_dense_matrix_t* b = 0;
758
1
  ccv_convnet_encode(convnet, &a, &b, 1);
759
1
  ccv_matrix_free(a);
760
1
  ccv_dense_matrix_t* c = ccv_dense_matrix_new(19, 19, CCV_32F | CCV_C1, 0, 0);
761
20
  for (y = 0; y < 19; 
y++19
)
762
380
    
for (x = 0; 19
x < 19; 
x++361
)
763
361
      c->data.f32[y * 19 + x] = 59 + y * 171 + x * 3;
764
1
  REQUIRE_MATRIX_EQ(b, c, "average pool network output should be exactly the same");
765
1
  ccv_matrix_free(b);
766
1
  ccv_matrix_free(c);
767
1
  ccv_convnet_free(convnet);
768
1
}
769
770
TEST_CASE("average pool network of 54x54 with window of 2x2 and stride of 2")
771
1
{
772
1
  ccv_convnet_layer_param_t params = {
773
1
    .type = CCV_CONVNET_AVERAGE_POOL,
774
1
    .bias = 0,
775
1
    .glorot = sqrtf(2),
776
1
    .input = {
777
1
      .matrix = {
778
1
        .rows = 54,
779
1
        .cols = 54,
780
1
        .channels = 1,
781
1
        .partition = 1,
782
1
      },
783
1
    },
784
1
    .output = {
785
1
      .pool = {
786
1
        .size = 2,
787
1
        .strides = 2,
788
1
        .border = 0,
789
1
      },
790
1
    },
791
1
  };
792
1
  ccv_convnet_t* convnet = ccv_convnet_new(0, ccv_size(54, 54), &params, 1);
793
1
  ccv_dense_matrix_t* a = ccv_dense_matrix_new(54, 54, CCV_32F | CCV_C1, 0, 0);
794
1
  int i, x, y;
795
2.91k
  for (i = 0; i < 54 * 54; 
i++2.91k
)
796
2.91k
    a->data.f32[i] = i + 1;
797
1
  ccv_dense_matrix_t* b = 0;
798
1
  ccv_convnet_encode(convnet, &a, &b, 1);
799
1
  ccv_matrix_free(a);
800
1
  ccv_dense_matrix_t* c = ccv_dense_matrix_new(27, 27, CCV_32F | CCV_C1, 0, 0);
801
28
  for (y = 0; y < 27; 
y++27
)
802
756
    
for (x = 0; 27
x < 27; 
x++729
)
803
729
      c->data.f32[y * 27 + x] = 28.5 + y * 108 + x * 2;
804
1
  REQUIRE_MATRIX_EQ(b, c, "average pool network output should be exactly the same");
805
1
  ccv_matrix_free(b);
806
1
  ccv_matrix_free(c);
807
1
  ccv_convnet_free(convnet);
808
1
}
809
810
TEST_CASE("local response normalization with partitioned by 2")
811
1
{
812
1
  ccv_convnet_layer_param_t params = {
813
1
    .type = CCV_CONVNET_LOCAL_RESPONSE_NORM,
814
1
    .input = {
815
1
      .matrix = {
816
1
        .rows = 27,
817
1
        .cols = 27,
818
1
        .channels = 10,
819
1
        .partition = 2,
820
1
      },
821
1
    },
822
1
    .output = {
823
1
      .rnorm = {
824
1
        .size = 3,
825
1
        .kappa = 2,
826
1
        .alpha = 1e-4,
827
1
        .beta = 0.75,
828
1
      },
829
1
    },
830
1
  };
831
1
  ccv_convnet_t* convnet = ccv_convnet_new(0, ccv_size(27, 27), &params, 1);
832
1
  int i, k;
833
1
  ccv_dense_matrix_t* a = ccv_dense_matrix_new(27, 27, CCV_32F | 10, 0, 0);
834
7.29k
  for (i = 0; i < 27 * 27 * 10; 
i++7.29k
)
835
7.29k
    a->data.f32[i] = i;
836
1
  ccv_dense_matrix_t* b = 0;
837
1
  ccv_convnet_encode(convnet, &a, &b, 1);
838
1
  ccv_convnet_layer_param_t partitioned_params = {
839
1
    .type = CCV_CONVNET_LOCAL_RESPONSE_NORM,
840
1
    .input = {
841
1
      .matrix = {
842
1
        .rows = 27,
843
1
        .cols = 27,
844
1
        .channels = 5,
845
1
        .partition = 1,
846
1
      },
847
1
    },
848
1
    .output = {
849
1
      .rnorm = {
850
1
        .size = 3,
851
1
        .kappa = 2,
852
1
        .alpha = 1e-4,
853
1
        .beta = 0.75,
854
1
      },
855
1
    },
856
1
  };
857
1
  ccv_convnet_t* partitioned_convnet = ccv_convnet_new(0, ccv_size(27, 27), &partitioned_params, 1);
858
1
  ccv_dense_matrix_t* aa = ccv_dense_matrix_new(27, 27, CCV_32F | 5, 0, 0);
859
730
  for (i = 0; i < 27 * 27; 
i++729
)
860
4.37k
    
for (k = 0; 729
k < 5; 
k++3.64k
)
861
3.64k
      aa->data.f32[i * 5 + k] = a->data.f32[i * 10 + k];
862
1
  ccv_dense_matrix_t* bb = ccv_dense_matrix_new(27, 27, CCV_32F | 10, 0, 0);
863
1
  ccv_dense_matrix_t* cc = 0;
864
1
  ccv_convnet_encode(partitioned_convnet, &aa, &cc, 1);
865
730
  for (i = 0; i < 27 * 27; 
i++729
)
866
4.37k
    
for (k = 0; 729
k < 5; 
k++3.64k
)
867
3.64k
      bb->data.f32[i * 10 + k] = cc->data.f32[i * 5 + k];
868
730
  for (i = 0; i < 27 * 27; 
i++729
)
869
4.37k
    
for (k = 0; 729
k < 5; 
k++3.64k
)
870
3.64k
      aa->data.f32[i * 5 + k] = a->data.f32[i * 10 + 5 + k];
871
1
  ccv_convnet_encode(partitioned_convnet, &aa, &cc, 1);
872
730
  for (i = 0; i < 27 * 27; 
i++729
)
873
4.37k
    
for (k = 0; 729
k < 5; 
k++3.64k
)
874
3.64k
      bb->data.f32[i * 10 + 5 + k] = cc->data.f32[i * 5 + k];
875
1
  REQUIRE_MATRIX_EQ(b, bb, "27x27x10 matrix computed from convnet with partition and partitioned convnet should be exactly the same");
876
1
  ccv_matrix_free(a);
877
1
  ccv_matrix_free(b);
878
1
  ccv_matrix_free(aa);
879
1
  ccv_matrix_free(bb);
880
1
  ccv_matrix_free(cc);
881
1
  ccv_convnet_free(convnet);
882
1
  ccv_convnet_free(partitioned_convnet);
883
1
}
884
885
// we probably won't cover all static functions in this test, disable annoying warnings
886
#pragma GCC diagnostic ignored "-Wunused-function"
887
// so that we can test static functions, note that CASE_TESTS is defined in case.h, which will disable all extern functions
888
#include "ccv_convnet.c"
889
890
#ifdef HAVE_GSL
891
TEST_CASE("full connect network backward propagate")
892
1
{
893
1
  ccv_convnet_layer_param_t params = {
894
1
    .type = CCV_CONVNET_FULL_CONNECT,
895
1
    .bias = 0,
896
1
    .glorot = sqrtf(2),
897
1
    .input = {
898
1
      .matrix = {
899
1
        .rows = 3,
900
1
        .cols = 3,
901
1
        .channels = 64,
902
1
        .partition = 1,
903
1
      },
904
1
      .node = {
905
1
        .count = 3 * 3 * 64,
906
1
      },
907
1
    },
908
1
    .output = {
909
1
      .full_connect = {
910
1
        .relu = 0,
911
1
        .count = 10,
912
1
      },
913
1
    },
914
1
  };
915
1
  ccv_convnet_t *convnet = ccv_convnet_new(0, ccv_size(3, 3), &params, 1);
916
1
  int i, j;
917
5.76k
  for (i = 0; i < 3 * 3 * 64 * 10; 
i++5.76k
)
918
5.76k
    convnet->layers[0].w[i] = 2;
919
1
  ccv_convnet_t* update_params = _ccv_convnet_update_new(convnet);
920
1
  _ccv_convnet_update_zero(update_params);
921
1
  ccv_dense_matrix_t* x = ccv_dense_matrix_new(3, 3, CCV_32F | 64, 0, 0);
922
577
  for (i = 0; i < 3 * 3 * 64; 
i++576
)
923
576
    x->data.f32[i] = 1;
924
1
  ccv_dense_matrix_t* y = 0;
925
1
  ccv_convnet_encode(convnet, &x, &y, 1);
926
1
  REQUIRE(y->rows == 10 && y->cols == 1 && CCV_GET_CHANNEL(y->type) == 1, "y should be a 10-dimensional vector");
927
1
  ccv_dense_matrix_t* loss = ccv_dense_matrix_new(10, 1, CCV_32F | CCV_C1, 0, 0);
928
1
  loss->data.f32[0] = 18;
929
10
  for (i = 1; i < 10; 
i++9
)
930
9
    loss->data.f32[i] = -1;
931
1
  ccv_dense_matrix_t* b = 0;
932
1
  _ccv_convnet_full_connect_backward_propagate(convnet->layers, loss, y, x, &b, update_params->layers);
933
1
  ccv_matrix_free(y);
934
1
  ccv_matrix_free(x);
935
1
  ccv_matrix_free(loss);
936
1
  ccv_dense_matrix_t* db = ccv_dense_matrix_new(3, 3, CCV_32F | 64, 0, 0);
937
577
  for (i = 0; i < 3 * 3 * 64; 
i++576
)
938
576
    db->data.f32[i] = 18;
939
1
  REQUIRE_MATRIX_EQ(b, db, "propagated error doesn't match the expected value");
940
1
  ccv_matrix_free(db);
941
1
  ccv_matrix_free(b);
942
1
  float* dw = (float*)ccmalloc(sizeof(float) * 10 * 3 * 3 * 64);
943
577
  for (j = 0; j < 3 * 3 * 64; 
j++576
)
944
576
    dw[j] = 18;
945
10
  for (i = 1; i < 10; 
i++9
)
946
5.19k
    
for (j = 0; 9
j < 3 * 3 * 64; 
j++5.18k
)
947
5.18k
      dw[i * 3 * 3 * 64 + j] = -1;
948
1
  REQUIRE_ARRAY_EQ_WITH_TOLERANCE(float, dw, update_params->layers[0].w, 10 * 3 * 3 * 64, 1e-4, "weight gradient doesn't match the expected value");
949
1
  ccfree(dw);
950
1
  float* dbias = (float*)ccmalloc(sizeof(float) * 10);
951
1
  dbias[0] = 18;
952
10
  for (i = 1; i < 10; 
i++9
)
953
9
    dbias[i] = -1;
954
1
  REQUIRE_ARRAY_EQ_WITH_TOLERANCE(float, dbias, update_params->layers[0].bias, 10, 1e-4, "bias gradient doesn't match the expected value");
955
1
  ccfree(dbias);
956
1
  ccv_convnet_free(update_params);
957
1
  ccv_convnet_free(convnet);
958
1
}
959
960
TEST_CASE("convolutional network backward propagate")
961
1
{
962
1
  ccv_convnet_layer_param_t params = {
963
1
    .type = CCV_CONVNET_CONVOLUTIONAL,
964
1
    .bias = 0,
965
1
    .glorot = sqrtf(2),
966
1
    .input = {
967
1
      .matrix = {
968
1
        .rows = 31,
969
1
        .cols = 31,
970
1
        .channels = 3,
971
1
        .partition = 1,
972
1
      },
973
1
    },
974
1
    .output = {
975
1
      .convolutional = {
976
1
        .rows = 5,
977
1
        .cols = 5,
978
1
        .channels = 3,
979
1
        .border = 2,
980
1
        .strides = 1,
981
1
        .count = 32,
982
1
        .partition = 1,
983
1
      },
984
1
    },
985
1
  };
986
1
  ccv_convnet_t* convnet = ccv_convnet_new(0, ccv_size(31, 31), &params, 1);
987
1
  int i, j, k;
988
2.40k
  for (i = 0; i < 5 * 5 * 3 * 32; 
i++2.40k
)
989
2.40k
    convnet->layers[0].w[i] = 2;
990
1
  ccv_convnet_t* update_params = _ccv_convnet_update_new(convnet);
991
1
  _ccv_convnet_update_zero(update_params);
992
1
  ccv_dense_matrix_t* x = ccv_dense_matrix_new(31, 31, CCV_32F | CCV_C3, 0, 0);
993
2.88k
  for (i = 0; i < 31 * 31 * 3; 
i++2.88k
)
994
2.88k
    x->data.f32[i] = 1;
995
1
  ccv_dense_matrix_t* y = 0;
996
1
  ccv_convnet_encode(convnet, &x, &y, 1);
997
1
  REQUIRE(y->rows == 31 && y->cols == 31 && CCV_GET_CHANNEL(y->type) == 32, "convnet should return a 31x31x32 matrix");
998
1
  ccv_dense_matrix_t* loss = ccv_dense_matrix_new(y->rows, y->cols, CCV_32F | CCV_GET_CHANNEL(y->type), 0, 0);
999
30.7k
  for (i = 0; i < 31 * 31 * 32; 
i++30.7k
)
1000
30.7k
    loss->data.f32[i] = 1;
1001
1
  ccv_dense_matrix_t* d = 0;
1002
1
  _ccv_convnet_convolutional_backward_propagate(convnet->layers, loss, y, x, &d, update_params->layers);
1003
1
  ccv_matrix_free(loss);
1004
1
  ccv_matrix_free(y);
1005
1
  ccv_matrix_free(x);
1006
1
  ccv_dense_matrix_t* dd = ccv_dense_matrix_new(31, 31, CCV_32F | CCV_C3, 0, 0);
1007
32
  for (i = 0; i < 31; 
i++31
)
1008
992
    
for (j = 0; 31
j < 31; 
j++961
)
1009
961
      dd->data.f32[(i * 31 + j) * 3] =
1010
961
      dd->data.f32[(i * 31 + j) * 3 + 1] =
1011
961
      dd->data.f32[(i * 31 + j) * 3 + 2] = 32 * 2 * (5 + ccv_min(i - 2, 0) + ccv_min(28 - i, 0)) * (5 + ccv_min(j - 2, 0) + ccv_min(28 - j, 0));
1012
1
  REQUIRE_MATRIX_EQ(d, dd, "propagated error doesn't match the expected value");
1013
1
  ccv_matrix_free(d);
1014
1
  ccv_matrix_free(dd);
1015
1
  float* dw = (float*)ccmalloc(sizeof(float) * 5 * 5 * 3 * 32);
1016
33
  for (k = 0; k < 32; 
k++32
)
1017
192
    
for (i = 0; 32
i < 5; 
i++160
)
1018
960
      
for (j = 0; 160
j < 5; 
j++800
)
1019
800
        dw[k * 5 * 5 * 3 + (i * 5 + j) * 3] =
1020
800
        dw[k * 5 * 5 * 3 + (i * 5 + j) * 3 + 1] =
1021
800
        dw[k * 5 * 5 * 3 + (i * 5 + j) * 3 + 2] = (31 - abs(i - 2)) * (31 - abs(j - 2));
1022
1
  REQUIRE_ARRAY_EQ_WITH_TOLERANCE(float, dw, update_params->layers[0].w, 5 * 5 * 3 * 32, 1e-4, "weight gradient doesn't match the expected value");
1023
1
  ccfree(dw);
1024
1
  float* dbias = (float*)ccmalloc(sizeof(float) * 32);
1025
33
  for (i = 0; i < 32; 
i++32
)
1026
32
    dbias[i] = 31 * 31;
1027
1
  REQUIRE_ARRAY_EQ_WITH_TOLERANCE(float, dbias, update_params->layers[0].bias, 32, 1e-4, "bias gradient doesn't match the expected value");
1028
1
  ccfree(dbias);
1029
1
  ccv_convnet_free(update_params);
1030
1
  ccv_convnet_free(convnet);
1031
1
}
1032
1033
TEST_CASE("convolutional network backward propagate with partitioned by 2")
1034
1
{
1035
1
  ccv_convnet_layer_param_t params = {
1036
1
    .type = CCV_CONVNET_CONVOLUTIONAL,
1037
1
    .bias = 0,
1038
1
    .glorot = sqrtf(2),
1039
1
    .input = {
1040
1
      .matrix = {
1041
1
        .rows = 31,
1042
1
        .cols = 31,
1043
1
        .channels = 4,
1044
1
        .partition = 2,
1045
1
      },
1046
1
    },
1047
1
    .output = {
1048
1
      .convolutional = {
1049
1
        .rows = 5,
1050
1
        .cols = 5,
1051
1
        .channels = 4,
1052
1
        .border = 2,
1053
1
        .strides = 1,
1054
1
        .count = 8,
1055
1
        .partition = 2,
1056
1
      },
1057
1
    },
1058
1
  };
1059
1
  ccv_convnet_t* convnet = ccv_convnet_new(0, ccv_size(31, 31), &params, 1);
1060
1
  int i, k;
1061
401
  for (i = 0; i < convnet->layers->wnum; 
i++400
)
1062
400
    convnet->layers->w[i] = i * 1e-2;
1063
9
  for (i = 0; i < convnet->layers->net.convolutional.count; 
i++8
)
1064
8
    convnet->layers->bias[i] = i;
1065
1
  ccv_dense_matrix_t* a = ccv_dense_matrix_new(31, 31, CCV_32F | 4, 0, 0);
1066
3.84k
  for (i = 0; i < 31 * 31 * 4; 
i++3.84k
)
1067
3.84k
    a->data.f32[i] = 2000 - i;
1068
1
  ccv_dense_matrix_t* b = 0;
1069
1
  ccv_convnet_encode(convnet, &a, &b, 1);
1070
1
  ccv_dense_matrix_t* loss = ccv_dense_matrix_new(b->rows, b->cols, CCV_32F | CCV_GET_CHANNEL(b->type), 0, 0);
1071
7.68k
  for (i = 0; i < 31 * 31 * 8; 
i++7.68k
)
1072
7.68k
    loss->data.f32[i] = 1;
1073
1
  ccv_dense_matrix_t* d = 0;
1074
1
  ccv_convnet_t* update_params = _ccv_convnet_update_new(convnet);
1075
1
  _ccv_convnet_update_zero(update_params);
1076
1
  _ccv_convnet_convolutional_backward_propagate(convnet->layers, loss, b, a, &d, update_params->layers);
1077
1
  ccv_matrix_free(loss);
1078
1
  ccv_convnet_layer_param_t partitioned_params = {
1079
1
    .type = CCV_CONVNET_CONVOLUTIONAL,
1080
1
    .bias = 0,
1081
1
    .glorot = sqrtf(2),
1082
1
    .input = {
1083
1
      .matrix = {
1084
1
        .rows = 31,
1085
1
        .cols = 31,
1086
1
        .channels = 2,
1087
1
        .partition = 1,
1088
1
      },
1089
1
    },
1090
1
    .output = {
1091
1
      .convolutional = {
1092
1
        .rows = 5,
1093
1
        .cols = 5,
1094
1
        .channels = 2,
1095
1
        .border = 2,
1096
1
        .strides = 1,
1097
1
        .count = 4,
1098
1
        .partition = 1,
1099
1
      },
1100
1
    },
1101
1
  };
1102
1
  ccv_convnet_t* partitioned_convnet = ccv_convnet_new(0, ccv_size(31, 31), &partitioned_params, 1);
1103
1
  ccv_dense_matrix_t* aa = ccv_dense_matrix_new(31, 31, CCV_32F | 2, 0, 0);
1104
  // first partition
1105
962
  for (i = 0; i < 31 * 31; 
i++961
)
1106
2.88k
    
for (k = 0; 961
k < 2; 
k++1.92k
)
1107
1.92k
      aa->data.f32[i * 2 + k] = a->data.f32[i * 4 + k];
1108
1
  memcpy(partitioned_convnet->layers->w, convnet->layers->w, sizeof(float) * (convnet->layers->wnum / 2));
1109
1
  memcpy(partitioned_convnet->layers->bias, convnet->layers->bias, sizeof(float) * (convnet->layers->net.convolutional.count / 2));
1110
1
  ccv_dense_matrix_t* bb = 0;
1111
1
  ccv_convnet_encode(partitioned_convnet, &aa, &bb, 1);
1112
1
  ccv_dense_matrix_t* bbb = ccv_dense_matrix_new(31, 31, CCV_32F | 8, 0, 0);
1113
962
  for (i = 0; i < 31 * 31; 
i++961
)
1114
4.80k
    
for (k = 0; 961
k < 4; 
k++3.84k
)
1115
3.84k
      bbb->data.f32[i * 8 + k] = bb->data.f32[i * 4 + k];
1116
1
  loss = ccv_dense_matrix_new(31, 31, CCV_32F | CCV_GET_CHANNEL(bb->type), 0, 0);
1117
3.84k
  for (i = 0; i < 31 * 31 * 4; 
i++3.84k
)
1118
3.84k
    loss->data.f32[i] = 1;
1119
1
  ccv_dense_matrix_t* dd = 0;
1120
1
  ccv_convnet_t* partitioned_update_params = _ccv_convnet_update_new(convnet);
1121
1
  _ccv_convnet_update_zero(partitioned_update_params);
1122
1
  _ccv_convnet_convolutional_backward_propagate(partitioned_convnet->layers, loss, bb, aa, &dd, partitioned_update_params->layers);
1123
1
  ccv_dense_matrix_t* ddd = ccv_dense_matrix_new(31, 31, CCV_32F | 4, 0, 0);
1124
1
  float* ww = (float*)ccmalloc(sizeof(float) * (convnet->layers->wnum + convnet->layers->net.convolutional.count));
1125
1
  float* bbias = ww + convnet->layers->wnum;
1126
1
  memcpy(ww, partitioned_update_params->layers->w, sizeof(float) * (convnet->layers->wnum / 2));
1127
1
  memcpy(bbias, partitioned_update_params->layers->bias, sizeof(float) * (convnet->layers->net.convolutional.count / 2));
1128
962
  for (i = 0; i < 31 * 31; 
i++961
)
1129
2.88k
    
for (k = 0; 961
k < 2; 
k++1.92k
)
1130
1.92k
      ddd->data.f32[i * 4 + k] = dd->data.f32[i * 2 + k];
1131
  // second partition
1132
962
  for (i = 0; i < 31 * 31; 
i++961
)
1133
2.88k
    
for (k = 0; 961
k < 2; 
k++1.92k
)
1134
1.92k
      aa->data.f32[i * 2 + k] = a->data.f32[i * 4 + 2 + k];
1135
1
  memcpy(partitioned_convnet->layers->w, convnet->layers->w + (convnet->layers->wnum / 2), sizeof(float) * (convnet->layers->wnum / 2));
1136
1
  memcpy(partitioned_convnet->layers->bias, convnet->layers->bias + (convnet->layers->net.convolutional.count / 2), sizeof(float) * (convnet->layers->net.convolutional.count / 2));
1137
1
  ccv_convnet_compact(partitioned_convnet); // because it is reused, we need to clear intermediate data
1138
1
  ccv_convnet_encode(partitioned_convnet, &aa, &bb, 1);
1139
962
  for (i = 0; i < 31 * 31; 
i++961
)
1140
4.80k
    
for (k = 0; 961
k < 4; 
k++3.84k
)
1141
3.84k
      bbb->data.f32[i * 8 + 4 + k] = bb->data.f32[i * 4 + k];
1142
1
  REQUIRE_MATRIX_EQ(b, bbb, "forward pass doesn't match the expected value");
1143
1
  _ccv_convnet_update_zero(partitioned_update_params);
1144
1
  _ccv_convnet_convolutional_backward_propagate(partitioned_convnet->layers, loss, bb, aa, &dd, partitioned_update_params->layers);
1145
1
  memcpy(ww + (convnet->layers->wnum / 2), partitioned_update_params->layers->w, sizeof(float) * (convnet->layers->wnum / 2));
1146
1
  memcpy(bbias + (convnet->layers->net.convolutional.count / 2), partitioned_update_params->layers->bias, sizeof(float) * (convnet->layers->net.convolutional.count / 2));
1147
962
  for (i = 0; i < 31 * 31; 
i++961
)
1148
2.88k
    
for (k = 0; 961
k < 2; 
k++1.92k
)
1149
1.92k
      ddd->data.f32[i * 4 + 2 + k] = dd->data.f32[i * 2 + k];
1150
1
  REQUIRE_MATRIX_EQ(d, ddd, "propagated error doesn't match the expected value");
1151
1
  REQUIRE_ARRAY_EQ_WITH_TOLERANCE(float, ww, update_params->layers[0].w, convnet->layers->wnum, 1e-4, "weight gradient doesn't match the expected value");
1152
1
  REQUIRE_ARRAY_EQ_WITH_TOLERANCE(float, bbias, update_params->layers[0].bias, convnet->layers->net.convolutional.count, 1e-4, "bias gradient doesn't match the expected value");
1153
1
  ccfree(ww);
1154
1
  ccv_matrix_free(loss);
1155
1
  ccv_matrix_free(ddd);
1156
1
  ccv_matrix_free(dd);
1157
1
  ccv_matrix_free(bbb);
1158
1
  ccv_matrix_free(bb);
1159
1
  ccv_matrix_free(aa);
1160
1
  ccv_matrix_free(d);
1161
1
  ccv_matrix_free(b);
1162
1
  ccv_matrix_free(a);
1163
1
  ccv_convnet_free(convnet);
1164
1
  ccv_convnet_free(update_params);
1165
1
  ccv_convnet_free(partitioned_convnet);
1166
1
  ccv_convnet_free(partitioned_update_params);
1167
1
}
1168
1169
TEST_CASE("local response normalization backward propagate with partitioned by 2")
1170
1
{
1171
1
  ccv_convnet_layer_param_t params = {
1172
1
    .type = CCV_CONVNET_LOCAL_RESPONSE_NORM,
1173
1
    .input = {
1174
1
      .matrix = {
1175
1
        .rows = 27,
1176
1
        .cols = 27,
1177
1
        .channels = 6,
1178
1
        .partition = 2,
1179
1
      },
1180
1
    },
1181
1
    .output = {
1182
1
      .rnorm = {
1183
1
        .size = 3,
1184
1
        .kappa = 2,
1185
1
        .alpha = 1e-4,
1186
1
        .beta = 0.75,
1187
1
      },
1188
1
    },
1189
1
  };
1190
1
  ccv_convnet_t* convnet = ccv_convnet_new(0, ccv_size(27, 27), &params, 1);
1191
1
  int i, k;
1192
1
  ccv_dense_matrix_t* a = ccv_dense_matrix_new(27, 27, CCV_32F | 6, 0, 0);
1193
4.37k
  for (i = 0; i < 27 * 27 * 6; 
i++4.37k
)
1194
4.37k
    a->data.f32[i] = i;
1195
1
  ccv_dense_matrix_t* b = 0;
1196
1
  ccv_convnet_encode(convnet, &a, &b, 1);
1197
1
  ccv_dense_matrix_t* d = 0;
1198
1
  ccv_dense_matrix_t* loss = ccv_dense_matrix_new(27, 27, CCV_32F | 6, 0, 0);
1199
4.37k
  for (i = 0; i < 27 * 27 * 6; 
i++4.37k
)
1200
4.37k
    loss->data.f32[i] = 1;
1201
1
  _ccv_convnet_rnorm_backward_propagate(convnet->layers, loss, b, a, convnet->denoms[0], &d);
1202
1
  ccv_convnet_layer_param_t partitioned_params = {
1203
1
    .type = CCV_CONVNET_LOCAL_RESPONSE_NORM,
1204
1
    .input = {
1205
1
      .matrix = {
1206
1
        .rows = 27,
1207
1
        .cols = 27,
1208
1
        .channels = 3,
1209
1
        .partition = 1,
1210
1
      },
1211
1
    },
1212
1
    .output = {
1213
1
      .rnorm = {
1214
1
        .size = 3,
1215
1
        .kappa = 2,
1216
1
        .alpha = 1e-4,
1217
1
        .beta = 0.75,
1218
1
      },
1219
1
    },
1220
1
  };
1221
1
  ccv_convnet_t* partitioned_convnet = ccv_convnet_new(0, ccv_size(27, 27), &partitioned_params, 1);
1222
1
  ccv_dense_matrix_t* aa = ccv_dense_matrix_new(27, 27, CCV_32F | 3, 0, 0);
1223
  // first partition
1224
730
  for (i = 0; i < 27 * 27; 
i++729
)
1225
2.91k
    
for (k = 0; 729
k < 3; 
k++2.18k
)
1226
2.18k
      aa->data.f32[i * 3 + k] = a->data.f32[i * 6 + k];
1227
1
  ccv_dense_matrix_t* bb = 0;
1228
1
  ccv_convnet_encode(partitioned_convnet, &aa, &bb, 1);
1229
1
  ccv_matrix_free(loss);
1230
1
  loss = ccv_dense_matrix_new(27, 27, CCV_32F | 3, 0, 0);
1231
2.18k
  for (i = 0; i < 27 * 27 * 3; 
i++2.18k
)
1232
2.18k
    loss->data.f32[i] = 1;
1233
1
  ccv_dense_matrix_t* dd = 0;
1234
1
  _ccv_convnet_rnorm_backward_propagate(partitioned_convnet->layers, loss, bb, aa, partitioned_convnet->denoms[0], &dd);
1235
1
  ccv_dense_matrix_t* ddd = ccv_dense_matrix_new(27, 27, CCV_32F | 6, 0, 0);
1236
730
  for (i = 0; i < 27 * 27; 
i++729
)
1237
2.91k
    
for (k = 0; 729
k < 3; 
k++2.18k
)
1238
2.18k
      ddd->data.f32[i * 6 + k] = dd->data.f32[i * 3 + k];
1239
  // second partition
1240
730
  for (i = 0; i < 27 * 27; 
i++729
)
1241
2.91k
    
for (k = 0; 729
k < 3; 
k++2.18k
)
1242
2.18k
      aa->data.f32[i * 3 + k] = a->data.f32[i * 6 + 3 + k];
1243
1
  ccv_convnet_encode(partitioned_convnet, &aa, &bb, 1);
1244
1
  _ccv_convnet_rnorm_backward_propagate(partitioned_convnet->layers, loss, bb, aa, partitioned_convnet->denoms[0], &dd);
1245
730
  for (i = 0; i < 27 * 27; 
i++729
)
1246
2.91k
    
for (k = 0; 729
k < 3; 
k++2.18k
)
1247
2.18k
      ddd->data.f32[i * 6 + 3 + k] = dd->data.f32[i * 3 + k];
1248
1
  REQUIRE_MATRIX_EQ(d, ddd, "27x27x6 error local response normalization backward propagated from convnet with partition and partitioned convnet should be exactly the same");
1249
1
  ccv_matrix_free(a);
1250
1
  ccv_matrix_free(b);
1251
1
  ccv_matrix_free(d);
1252
1
  ccv_matrix_free(aa);
1253
1
  ccv_matrix_free(bb);
1254
1
  ccv_matrix_free(dd);
1255
1
  ccv_matrix_free(ddd);
1256
1
  ccv_matrix_free(loss);
1257
1
  ccv_convnet_free(convnet);
1258
1
  ccv_convnet_free(partitioned_convnet);
1259
1
}
1260
1261
// five-stencil constants
1262
static float fs[4] = { 1, -8, 8, -1 };
1263
static float fsh[4] = { -2, -1, 1, 2 };
1264
1265
static float dsfmt_genrand_gaussian(dsfmt_t* dsfmt, float sigma)
1266
2.29k
{
1267
2.29k
  double rand1 = dsfmt_genrand_open_close(dsfmt);
1268
2.29k
  rand1 = -2 * log(rand1);
1269
2.29k
  double rand2 = dsfmt_genrand_open_close(dsfmt) * CCV_PI * 2;
1270
2.29k
  return (float)(sqrt(sigma * rand1) * cos(rand2));
1271
2.29k
}
1272
1273
TEST_CASE("numerical gradient versus analytical gradient for full connect network")
1274
1
{
1275
1
  ccv_convnet_layer_param_t params = {
1276
1
    .type = CCV_CONVNET_FULL_CONNECT,
1277
1
    .bias = 0,
1278
1
    .glorot = sqrtf(2),
1279
1
    .input = {
1280
1
      .matrix = {
1281
1
        .rows = 3,
1282
1
        .cols = 3,
1283
1
        .channels = 8,
1284
1
        .partition = 1,
1285
1
      },
1286
1
      .node = {
1287
1
        .count = 3 * 3 * 8,
1288
1
      },
1289
1
    },
1290
1
    .output = {
1291
1
      .full_connect = {
1292
1
        .relu = 0,
1293
1
        .count = 10,
1294
1
      },
1295
1
    },
1296
1
  };
1297
1
  ccv_convnet_t *convnet = ccv_convnet_new(0, ccv_size(3, 3), &params, 1);
1298
1
  dsfmt_t dsfmt;
1299
1
  dsfmt_init_gen_rand(&dsfmt, 0);
1300
1
  int i, j, k;
1301
721
  for (i = 0; i < convnet->layers->wnum; 
i++720
)
1302
720
    convnet->layers->w[i] = dsfmt_genrand_gaussian(&dsfmt, 0.01);
1303
1
  ccv_convnet_t* update_params = _ccv_convnet_update_new(convnet);
1304
1
  _ccv_convnet_update_zero(update_params);
1305
1
  ccv_dense_matrix_t* x = ccv_dense_matrix_new(3, 3, CCV_32F | 8, 0, 0);
1306
73
  for (i = 0; i < 3 * 3 * 8; 
i++72
)
1307
72
    x->data.f32[i] = i;
1308
1
  ccv_dense_matrix_t* y = 0;
1309
1
  ccv_convnet_encode(convnet, &x, &y, 1);
1310
1
  REQUIRE(y->rows == 10 && y->cols == 1 && CCV_GET_CHANNEL(y->type) == 1, "y should be a 10-dimensional vector");
1311
1
  _ccv_convnet_compute_softmax(y, &y, 0);
1312
1
  ccv_dense_matrix_t* dloss = ccv_dense_matrix_new(10, 1, CCV_32F | CCV_C1, 0, 0);;
1313
11
  for (i = 0; i < 10; 
i++10
)
1314
10
    dloss->data.f32[i] = y->data.f32[i] - (i == 2);
1315
1
  float* dw = (float*)ccmalloc(sizeof(float) * 3 * 3 * 8 * 10);
1316
1
  static const float eps = 0.0001;
1317
11
  for (i = 0; i < 10; 
i++10
)
1318
730
    
for (j = 0; 10
j < 3 * 3 * 8; 
j++720
)
1319
720
    {
1320
720
      dw[j + i * 3 * 3 * 8] = 0;
1321
3.60k
      for (k = 0; k < 4; 
k++2.88k
)
1322
2.88k
      {
1323
2.88k
        float w = convnet->layers->w[j + i * 3 * 3 * 8];
1324
2.88k
        convnet->layers->w[j + i * 3 * 3 * 8] += fsh[k] * eps;
1325
2.88k
        ccv_dense_matrix_t* z = 0;
1326
2.88k
        ccv_convnet_encode(convnet, &x, &z, 1);
1327
2.88k
        _ccv_convnet_compute_softmax(z, &z, 0);
1328
2.88k
        dw[j + i * 3 * 3 * 8] += -logf(z->data.f32[2]) * fs[k];
1329
2.88k
        ccv_matrix_free(z);
1330
2.88k
        convnet->layers->w[j + i * 3 * 3 * 8] = w;
1331
2.88k
      }
1332
720
      dw[j + i * 3 * 3 * 8] *= 1.0 / (12 * eps);
1333
720
    }
1334
1
  float* dbias = (float*)ccmalloc(sizeof(float) * 10);
1335
11
  for (i = 0; i < 10; 
i++10
)
1336
10
  {
1337
10
    dbias[i] = 0;
1338
50
    for (k = 0; k < 4; 
k++40
)
1339
40
    {
1340
40
      float bias = convnet->layers->bias[i];
1341
40
      convnet->layers->bias[i] += fsh[k] * eps;
1342
40
      ccv_dense_matrix_t* z = 0;
1343
40
      ccv_convnet_encode(convnet, &x, &z, 1);
1344
40
      _ccv_convnet_compute_softmax(z, &z, 0);
1345
40
      dbias[i] += -logf(z->data.f32[2]) * fs[k];
1346
40
      ccv_matrix_free(z);
1347
40
      convnet->layers->bias[i] = bias;
1348
40
    }
1349
10
    dbias[i] *= 1.0 / (12 * eps);
1350
10
  }
1351
1
  ccv_dense_matrix_t* b = 0;
1352
1
  _ccv_convnet_full_connect_backward_propagate(convnet->layers, dloss, y, x, &b, update_params->layers);
1353
1
  ccv_matrix_free(y);
1354
1
  ccv_matrix_free(x);
1355
1
  ccv_matrix_free(dloss);
1356
1
  ccv_matrix_free(b);
1357
1
  REQUIRE_ARRAY_EQ_WITHIN_ANGLE_AND_MAGNITUDE(float, dw, update_params->layers[0].w, 3 * 3 * 8 * 10, 30, 2e-1, "weight gradient from analytical method doesn't match the one from numerical method");
1358
1
  ccfree(dw);
1359
1
  REQUIRE_ARRAY_EQ_WITHIN_ANGLE_AND_MAGNITUDE(float, dbias, update_params->layers[0].bias, 10, 30, 2e-1, "bias gradient from analytical method doesn't match the one from numerical method");
1360
1
  ccfree(dbias);
1361
1
  ccv_convnet_free(update_params);
1362
1
  ccv_convnet_free(convnet);
1363
1
}
1364
1365
TEST_CASE("numerical gradient versus analytical gradient for convolutional network")
1366
1
{
1367
1
  ccv_convnet_layer_param_t params = {
1368
1
    .type = CCV_CONVNET_CONVOLUTIONAL,
1369
1
    .bias = 0,
1370
1
    .glorot = sqrtf(2),
1371
1
    .input = {
1372
1
      .matrix = {
1373
1
        .rows = 31,
1374
1
        .cols = 31,
1375
1
        .channels = 3,
1376
1
        .partition = 1,
1377
1
      },
1378
1
    },
1379
1
    .output = {
1380
1
      .convolutional = {
1381
1
        .rows = 5,
1382
1
        .cols = 5,
1383
1
        .channels = 3,
1384
1
        .border = 2,
1385
1
        .strides = 1,
1386
1
        .count = 4,
1387
1
        .partition = 1,
1388
1
      },
1389
1
    },
1390
1
  };
1391
1
  ccv_convnet_t* convnet = ccv_convnet_new(0, ccv_size(31, 31), &params, 1);
1392
1
  dsfmt_t dsfmt;
1393
1
  dsfmt_init_gen_rand(&dsfmt, 1);
1394
1
  int i, k;
1395
301
  for (i = 0; i < convnet->layers->wnum; 
i++300
)
1396
300
    convnet->layers->w[i] = dsfmt_genrand_gaussian(&dsfmt, 0.0001);
1397
1
  ccv_convnet_t* update_params = _ccv_convnet_update_new(convnet);
1398
1
  _ccv_convnet_update_zero(update_params);
1399
1
  ccv_dense_matrix_t* x = ccv_dense_matrix_new(31, 31, CCV_32F | CCV_C3, 0, 0);
1400
2.88k
  for (i = 0; i < 31 * 31 * 3; 
i++2.88k
)
1401
2.88k
    x->data.f32[i] = i;
1402
1
  ccv_dense_matrix_t* y = 0;
1403
1
  ccv_convnet_encode(convnet, &x, &y, 1);
1404
1
  REQUIRE(y->rows == 31 && y->cols == 31 && CCV_GET_CHANNEL(y->type) == 4, "convnet should return a 31x31x4 matrix");
1405
1
  ccv_dense_matrix_t* softmax = 0;
1406
1
  _ccv_convnet_compute_softmax(y, &softmax, 0);
1407
1
  ccv_dense_matrix_t* dloss = ccv_dense_matrix_new(y->rows, y->cols, CCV_32F | CCV_GET_CHANNEL(y->type), 0, 0);
1408
3.84k
  for (i = 0; i < 31 * 31 * 4; 
i++3.84k
)
1409
3.84k
    dloss->data.f32[i] = softmax->data.f32[i] - (i == 24);
1410
1
  static const float eps = 0.000005;
1411
1
  float* dw = (float*)ccmalloc(sizeof(float) * 5 * 5 * 3 * 4); 
1412
301
  for (i = 0; i < 5 * 5 * 3 * 4; 
i++300
)
1413
300
  {
1414
300
    dw[i] = 0;
1415
1.50k
    for (k = 0; k < 4; 
k++1.20k
)
1416
1.20k
    {
1417
1.20k
      float w = convnet->layers->w[i];
1418
1.20k
      convnet->layers->w[i] += fsh[k] * eps;
1419
1.20k
      ccv_dense_matrix_t* z = 0;
1420
1.20k
      ccv_convnet_compact(convnet);
1421
1.20k
      ccv_convnet_encode(convnet, &x, &z, 1);
1422
1.20k
      _ccv_convnet_compute_softmax(z, &z, 0);
1423
1.20k
      dw[i] += -logf(z->data.f32[24]) * fs[k];
1424
1.20k
      ccv_matrix_free(z);
1425
1.20k
      convnet->layers->w[i] = w;
1426
1.20k
    }
1427
300
    dw[i] *= 1.0 / (12 * eps);
1428
300
  }
1429
1
  float* dbias = (float*)ccmalloc(sizeof(float) * 4);
1430
5
  for (i = 0; i < 4; 
i++4
)
1431
4
  {
1432
4
    dbias[i] = 0;
1433
20
    for (k = 0; k < 4; 
k++16
)
1434
16
    {
1435
16
      float bias = convnet->layers->bias[i];
1436
16
      convnet->layers->bias[i] += fsh[k] * eps;
1437
16
      ccv_dense_matrix_t* z = 0;
1438
16
      ccv_convnet_compact(convnet);
1439
16
      ccv_convnet_encode(convnet, &x, &z, 1);
1440
16
      _ccv_convnet_compute_softmax(z, &z, 0);
1441
16
      dbias[i] += -logf(z->data.f32[24]) * fs[k];
1442
16
      ccv_matrix_free(z);
1443
16
      convnet->layers->bias[i] = bias;
1444
16
    }
1445
4
    dbias[i] *= 1.0 / (12 * eps);
1446
4
  }
1447
1
  ccv_dense_matrix_t* d = 0;
1448
1
  _ccv_convnet_convolutional_backward_propagate(convnet->layers, dloss, y, x, &d, update_params->layers);
1449
1
  ccv_matrix_free(softmax);
1450
1
  ccv_matrix_free(dloss);
1451
1
  ccv_matrix_free(y);
1452
1
  ccv_matrix_free(x);
1453
1
  ccv_matrix_free(d);
1454
1
  REQUIRE_ARRAY_EQ_WITHIN_ANGLE_AND_MAGNITUDE(float, dw, update_params->layers[0].w, 5 * 5 * 3 * 4, 30, 2e-1, "weight gradient from analytical method doesn't match the one from numerical method");
1455
1
  ccfree(dw);
1456
1
  REQUIRE_ARRAY_EQ_WITHIN_ANGLE_AND_MAGNITUDE(float, dbias, update_params->layers[0].bias, 4, 30, 2e-1, "bias gradient from analytical method doesn't match the one from numerical method");
1457
1
  ccfree(dbias);
1458
1
  ccv_convnet_free(update_params);
1459
1
  ccv_convnet_free(convnet);
1460
1
}
1461
1462
TEST_CASE("numerical gradient versus analytical gradient for full connect network over convolutional network")
1463
1
{
1464
1
  ccv_convnet_layer_param_t params[] = {
1465
1
    {
1466
1
      .type = CCV_CONVNET_CONVOLUTIONAL,
1467
1
      .bias = 0,
1468
1
      .glorot = sqrtf(2),
1469
1
      .input = {
1470
1
        .matrix = {
1471
1
          .rows = 5,
1472
1
          .cols = 5,
1473
1
          .channels = 2,
1474
1
          .partition = 1,
1475
1
        },
1476
1
      },
1477
1
      .output = {
1478
1
        .convolutional = {
1479
1
          .rows = 3,
1480
1
          .cols = 3,
1481
1
          .channels = 2,
1482
1
          .border = 1,
1483
1
          .strides = 1,
1484
1
          .count = 4,
1485
1
          .partition = 1,
1486
1
        },
1487
1
      },
1488
1
    },
1489
1
    {
1490
1
      .type = CCV_CONVNET_FULL_CONNECT,
1491
1
      .bias = 0,
1492
1
      .glorot = sqrtf(2),
1493
1
      .input = {
1494
1
        .matrix = {
1495
1
          .rows = 5,
1496
1
          .cols = 5,
1497
1
          .channels = 4,
1498
1
          .partition = 1,
1499
1
        },
1500
1
        .node = {
1501
1
          .count = 5 * 5 * 4,
1502
1
        },
1503
1
      },
1504
1
      .output = {
1505
1
        .full_connect = {
1506
1
          .relu = 0,
1507
1
          .count = 10,
1508
1
        },
1509
1
      },
1510
1
    },
1511
1
  };
1512
1
  ccv_convnet_t* convnet = ccv_convnet_new(0, ccv_size(5, 5), params, 2);
1513
1
  dsfmt_t dsfmt;
1514
1
  dsfmt_init_gen_rand(&dsfmt, 2);
1515
1
  int i, k;
1516
73
  for (i = 0; i < convnet->layers[0].wnum; 
i++72
)
1517
72
    convnet->layers[0].w[i] = dsfmt_genrand_gaussian(&dsfmt, 0.001);
1518
1.00k
  for (i = 0; i < convnet->layers[1].wnum; 
i++1.00k
)
1519
1.00k
    convnet->layers[1].w[i] = dsfmt_genrand_gaussian(&dsfmt, 0.01);
1520
1
  ccv_convnet_t* update_params = _ccv_convnet_update_new(convnet);
1521
1
  _ccv_convnet_update_zero(update_params);
1522
1
  ccv_dense_matrix_t* x = ccv_dense_matrix_new(5, 5, CCV_32F | CCV_C2, 0, 0);
1523
51
  for (i = 0; i < 5 * 5 * 2; 
i++50
)
1524
50
    x->data.f32[i] = 0.2;
1525
1
  ccv_dense_matrix_t* y = 0;
1526
1
  ccv_convnet_encode(convnet, &x, &y, 1);
1527
1
  REQUIRE(y->rows == 10 && y->cols == 1 && CCV_GET_CHANNEL(y->type) == 1, "y should be a 10-dimensional vector");
1528
1
  _ccv_convnet_compute_softmax(y, &y, 0);
1529
1
  ccv_dense_matrix_t* dloss = ccv_dense_matrix_new(10, 1, CCV_32F | CCV_C1, 0, 0);;
1530
11
  for (i = 0; i < 10; 
i++10
)
1531
10
    dloss->data.f32[i] = y->data.f32[i] - (i == 2);
1532
1
  _ccv_convnet_propagate_loss(convnet, x, dloss, update_params);
1533
1
  ccv_matrix_free(dloss);
1534
1
  static const float eps = 0.0001;
1535
1
  float* dw = (float*)ccmalloc(sizeof(float) * 3 * 3 * 2 * 4); 
1536
73
  for (i = 0; i < 3 * 3 * 2 * 4; 
i++72
)
1537
72
  {
1538
72
    dw[i] = 0;
1539
360
    for (k = 0; k < 4; 
k++288
)
1540
288
    {
1541
288
      float w = convnet->layers->w[i];
1542
288
      convnet->layers->w[i] += fsh[k] * eps;
1543
288
      ccv_dense_matrix_t* z = 0;
1544
288
      ccv_convnet_compact(convnet);
1545
288
      ccv_convnet_encode(convnet, &x, &z, 1);
1546
288
      _ccv_convnet_compute_softmax(z, &z, 0);
1547
288
      dw[i] += -logf(z->data.f32[2]) * fs[k];
1548
288
      ccv_matrix_free(z);
1549
288
      convnet->layers->w[i] = w;
1550
288
    }
1551
72
    dw[i] *= 1.0 / (12 * eps);
1552
72
  }
1553
1
  float* dbias = (float*)ccmalloc(sizeof(float) * 4);
1554
5
  for (i = 0; i < 4; 
i++4
)
1555
4
  {
1556
4
    dbias[i] = 0;
1557
20
    for (k = 0; k < 4; 
k++16
)
1558
16
    {
1559
16
      float bias = convnet->layers->bias[i];
1560
16
      convnet->layers->bias[i] += fsh[k] * eps;
1561
16
      ccv_dense_matrix_t* z = 0;
1562
16
      ccv_convnet_compact(convnet);
1563
16
      ccv_convnet_encode(convnet, &x, &z, 1);
1564
16
      _ccv_convnet_compute_softmax(z, &z, 0);
1565
16
      dbias[i] += -logf(z->data.f32[2]) * fs[k];
1566
16
      ccv_matrix_free(z);
1567
16
      convnet->layers->bias[i] = bias;
1568
16
    }
1569
4
    dbias[i] *= 1.0 / (12 * eps);
1570
4
  }
1571
1
  ccv_matrix_free(y);
1572
1
  ccv_matrix_free(x);
1573
1
  REQUIRE_ARRAY_EQ_WITHIN_ANGLE_AND_MAGNITUDE(float, dw, update_params->layers[0].w, 3 * 3 * 2 * 4, 30, 2e-1, "weight gradient from analytical method doesn't match the one from numerical method");
1574
1
  ccfree(dw);
1575
1
  REQUIRE_ARRAY_EQ_WITHIN_ANGLE_AND_MAGNITUDE(float, dbias, update_params->layers[0].bias, 4, 30, 2e-1, "bias gradient from analytical method doesn't match the one from numerical method");
1576
1
  ccfree(dbias);
1577
1
  ccv_convnet_free(update_params);
1578
1
  ccv_convnet_free(convnet);
1579
1
}
1580
1581
TEST_CASE("numerical gradient versus analytical gradient for local response normalization over convolutional network")
1582
1
{
1583
1
  ccv_convnet_layer_param_t params[] = {
1584
1
    {
1585
1
      .type = CCV_CONVNET_CONVOLUTIONAL,
1586
1
      .bias = 0,
1587
1
      .glorot = sqrtf(2),
1588
1
      .input = {
1589
1
        .matrix = {
1590
1
          .rows = 31,
1591
1
          .cols = 31,
1592
1
          .channels = 2,
1593
1
          .partition = 1,
1594
1
        },
1595
1
      },
1596
1
      .output = {
1597
1
        .convolutional = {
1598
1
          .rows = 5,
1599
1
          .cols = 5,
1600
1
          .channels = 2,
1601
1
          .border = 2,
1602
1
          .strides = 1,
1603
1
          .count = 4,
1604
1
          .partition = 1,
1605
1
        },
1606
1
      },
1607
1
    },
1608
1
    {
1609
1
      .type = CCV_CONVNET_LOCAL_RESPONSE_NORM,
1610
1
      .input = {
1611
1
        .matrix = {
1612
1
          .rows = 31,
1613
1
          .cols = 31,
1614
1
          .channels = 4,
1615
1
          .partition = 1,
1616
1
        },
1617
1
      },
1618
1
      .output = {
1619
1
        .rnorm = {
1620
1
          .size = 3,
1621
1
          .kappa = 2,
1622
1
          .alpha = 0.00005,
1623
1
          .beta = 0.75,
1624
1
        },
1625
1
      },
1626
1
    },
1627
1
  };
1628
1
  ccv_convnet_t* convnet = ccv_convnet_new(0, ccv_size(31, 31), params, 2);
1629
1
  dsfmt_t dsfmt;
1630
1
  dsfmt_init_gen_rand(&dsfmt, 3);
1631
1
  int i, k;
1632
201
  for (i = 0; i < convnet->layers->wnum; 
i++200
)
1633
200
    convnet->layers->w[i] = dsfmt_genrand_gaussian(&dsfmt, 0.001);
1634
1
  ccv_convnet_t* update_params = _ccv_convnet_update_new(convnet);
1635
1
  _ccv_convnet_update_zero(update_params);
1636
1
  ccv_dense_matrix_t* x = ccv_dense_matrix_new(31, 31, CCV_32F | CCV_C2, 0, 0);
1637
1.92k
  for (i = 0; i < 31 * 31 * 2; 
i++1.92k
)
1638
1.92k
    x->data.f32[i] = i;
1639
1
  ccv_dense_matrix_t* y = 0;
1640
1
  ccv_convnet_encode(convnet, &x, &y, 1);
1641
1
  REQUIRE(y->rows == 31 && y->cols == 31 && CCV_GET_CHANNEL(y->type) == 4, "convnet should return a 31x31x4 matrix");
1642
1
  ccv_dense_matrix_t* softmax = 0;
1643
1
  _ccv_convnet_compute_softmax(y, &softmax, 0);
1644
1
  ccv_dense_matrix_t* dloss = ccv_dense_matrix_new(y->rows, y->cols, CCV_32F | CCV_GET_CHANNEL(y->type), 0, 0);
1645
3.84k
  for (i = 0; i < 31 * 31 * 4; 
i++3.84k
)
1646
3.84k
    dloss->data.f32[i] = softmax->data.f32[i] - (i == 24);
1647
1
  ccv_dense_matrix_t* d = 0;
1648
1
  _ccv_convnet_rnorm_backward_propagate(convnet->layers + 1, dloss, y, convnet->acts[0], convnet->denoms[1], update_params->acts);
1649
1
  _ccv_convnet_convolutional_backward_propagate(convnet->layers, update_params->acts[0], convnet->acts[0], x, &d, update_params->layers);
1650
1
  static const float eps = 0.000001;
1651
1
  float* dw = (float*)ccmalloc(sizeof(float) * 5 * 5 * 2 * 4); 
1652
201
  for (i = 0; i < 5 * 5 * 2 * 4; 
i++200
)
1653
200
  {
1654
200
    dw[i] = 0;
1655
1.00k
    for (k = 0; k < 4; 
k++800
)
1656
800
    {
1657
800
      float w = convnet->layers->w[i];
1658
800
      convnet->layers->w[i] += fsh[k] * eps;
1659
800
      ccv_dense_matrix_t* z = 0;
1660
800
      ccv_convnet_compact(convnet);
1661
800
      ccv_convnet_encode(convnet, &x, &z, 1);
1662
800
      _ccv_convnet_compute_softmax(z, &z, 0);
1663
800
      dw[i] += -logf(z->data.f32[24]) * fs[k];
1664
800
      ccv_matrix_free(z);
1665
800
      convnet->layers->w[i] = w;
1666
800
    }
1667
200
    dw[i] *= 1.0 / (12 * eps);
1668
200
  }
1669
1
  float* dbias = (float*)ccmalloc(sizeof(float) * 4);
1670
1
  static const float beps = 0.0001;
1671
5
  for (i = 0; i < 4; 
i++4
)
1672
4
  {
1673
4
    dbias[i] = 0;
1674
20
    for (k = 0; k < 4; 
k++16
)
1675
16
    {
1676
16
      float bias = convnet->layers->bias[i];
1677
16
      convnet->layers->bias[i] += fsh[k] * beps;
1678
16
      ccv_dense_matrix_t* z = 0;
1679
16
      ccv_convnet_compact(convnet);
1680
16
      ccv_convnet_encode(convnet, &x, &z, 1);
1681
16
      _ccv_convnet_compute_softmax(z, &z, 0);
1682
16
      dbias[i] += -logf(z->data.f32[24]) * fs[k];
1683
16
      ccv_matrix_free(z);
1684
16
      convnet->layers->bias[i] = bias;
1685
16
    }
1686
4
    dbias[i] *= 1.0 / (12 * beps);
1687
4
  }
1688
1
  ccv_matrix_free(softmax);
1689
1
  ccv_matrix_free(dloss);
1690
1
  ccv_matrix_free(y);
1691
1
  ccv_matrix_free(x);
1692
1
  ccv_matrix_free(d);
1693
1
  REQUIRE_ARRAY_EQ_WITHIN_ANGLE_AND_MAGNITUDE(float, dw, update_params->layers[0].w, 5 * 5 * 2 * 4, 30, 2e-1, "weight gradient from analytical method doesn't match the one from numerical method");
1694
1
  ccfree(dw);
1695
1
  REQUIRE_ARRAY_EQ_WITHIN_ANGLE_AND_MAGNITUDE(float, dbias, update_params->layers[0].bias, 4, 30, 2e-1, "bias gradient from analytical method doesn't match the one from numerical method");
1696
1
  ccfree(dbias);
1697
1
  ccv_convnet_free(update_params);
1698
1
  ccv_convnet_free(convnet);
1699
1
}
1700
1701
TEST_CASE("max pool network backward propagate")
1702
1
{
1703
1
  ccv_convnet_layer_param_t params = {
1704
1
    .type = CCV_CONVNET_MAX_POOL,
1705
1
    .input = {
1706
1
      .matrix = {
1707
1
        .rows = 31,
1708
1
        .cols = 31,
1709
1
        .channels = 2,
1710
1
        .partition = 1,
1711
1
      },
1712
1
    },
1713
1
    .output = {
1714
1
      .pool = {
1715
1
        .size = 3,
1716
1
        .strides = 2,
1717
1
        .border = 0,
1718
1
      },
1719
1
    },
1720
1
  };
1721
1
  ccv_convnet_t* convnet = ccv_convnet_new(0, ccv_size(31, 31), &params, 1);
1722
1
  ccv_dense_matrix_t* x = ccv_dense_matrix_new(31, 31, CCV_32F | CCV_C2, 0, 0);
1723
1
  int i, j, k;
1724
1.92k
  for (i = 0; i < 31 * 31 * 2; 
i++1.92k
)
1725
1.92k
    x->data.f32[i] = i;
1726
1
  ccv_dense_matrix_t* y = 0;
1727
1
  ccv_convnet_encode(convnet, &x, &y, 1);
1728
1
  ccv_dense_matrix_t* loss = ccv_dense_matrix_new(15, 15, CCV_32F | CCV_C2, 0, 0);
1729
451
  for (i = 0; i < 15 * 15 * 2; 
i++450
)
1730
450
    loss->data.f32[i] = i + 1;
1731
1
  ccv_dense_matrix_t* b = 0;
1732
1
  _ccv_convnet_max_pool_backward_propagate(convnet->layers, loss, y, x, &b);
1733
1
  ccv_matrix_free(loss);
1734
1
  ccv_matrix_free(x);
1735
1
  ccv_matrix_free(y);
1736
1
  ccv_dense_matrix_t* db = ccv_dense_matrix_new(31, 31, CCV_32F | CCV_C2, 0, 0);
1737
1
  ccv_zero(db);
1738
16
  for (i = 0; i < 15; 
i++15
)
1739
240
    
for (j = 0; 15
j < 15; 
j++225
)
1740
675
      
for (k = 0; 225
k < 2; 
k++450
)
1741
450
        db->data.f32[(j * 2 + 2 + (i * 2 + 2) * 31) * 2 + k] = (i * 15 + j) * 2 + 1 + k;
1742
1
  REQUIRE_MATRIX_EQ(b, db, "propagated error doesn't match the expected value");
1743
1
  ccv_matrix_free(db);
1744
1
  ccv_matrix_free(b);
1745
1
  ccv_convnet_free(convnet);
1746
1
}
1747
1748
TEST_CASE("average pool network backward propagate")
1749
1
{
1750
1
  ccv_convnet_layer_param_t params = {
1751
1
    .type = CCV_CONVNET_AVERAGE_POOL,
1752
1
    .input = {
1753
1
      .matrix = {
1754
1
        .rows = 31,
1755
1
        .cols = 31,
1756
1
        .channels = 2,
1757
1
        .partition = 1,
1758
1
      },
1759
1
    },
1760
1
    .output = {
1761
1
      .pool = {
1762
1
        .size = 3,
1763
1
        .strides = 2,
1764
1
        .border = 0,
1765
1
      },
1766
1
    },
1767
1
  };
1768
1
  ccv_convnet_t* convnet = ccv_convnet_new(0, ccv_size(31, 31), &params, 1);
1769
1
  ccv_dense_matrix_t* x = ccv_dense_matrix_new(31, 31, CCV_32F | CCV_C2, 0, 0);
1770
1
  int i, j, k;
1771
1.92k
  for (i = 0; i < 31 * 31 * 2; 
i++1.92k
)
1772
1.92k
    x->data.f32[i] = i;
1773
1
  ccv_dense_matrix_t* loss = ccv_dense_matrix_new(15, 15, CCV_32F | CCV_C2, 0, 0);
1774
451
  for (i = 0; i < 15 * 15 * 2; 
i++450
)
1775
450
    loss->data.f32[i] = i + 1;
1776
1
  ccv_dense_matrix_t* b = 0;
1777
1
  _ccv_convnet_average_pool_backward_propagate(convnet->layers, loss, x, &b);
1778
1
  ccv_matrix_free(x);
1779
1
  ccv_matrix_free(loss);
1780
1
  ccv_dense_matrix_t* db = ccv_dense_matrix_new(31, 31, CCV_32F | CCV_C2, 0, 0);
1781
1
  float inv_size = 1.0 / (3 * 3);
1782
32
  for (i = 0; i < 31; 
i++31
)
1783
992
    
for (j = 0; 31
j < 31; 
j++961
)
1784
2.88k
      
for (k = 0; 961
k < 2; 
k++1.92k
)
1785
1.92k
      {
1786
1.92k
        int x, y;
1787
1.92k
        db->data.f32[(i * 31 + j) * 2 + k] = 0;
1788
4.77k
        for (y = (i - 1) / 2; y <= i / 2; 
y++2.85k
)
1789
7.08k
          
for (x = (j - 1) / 2; 2.85k
x <= j / 2; 
x++4.23k
)
1790
4.23k
            if (x >= 0 && x < 15 && 
y >= 04.14k
 && 
y < 154.14k
)
1791
4.05k
              db->data.f32[(i * 31 + j) * 2 + k] += ((y * 15 + x) * 2 + k + 1) * inv_size;
1792
1.92k
      }
1793
1
  REQUIRE_MATRIX_EQ(b, db, "propagated error doesn't match the expected value");
1794
1
  ccv_matrix_free(db);
1795
1
  ccv_matrix_free(b);
1796
1
  ccv_convnet_free(convnet);
1797
1
}
1798
#endif
1799
1800
#include "case_main.h"