## Coverage Report

#### Created: 2026-05-04 15:30

next uncovered line (L), next uncovered region (R), next uncovered branch (B)

    /home/liu/actions-runner/_work/ccv/ccv/test/unit/convnet.tests.c

    Line
    Count
    Source
    1
    #include "ccv.h"
    2
    #include "case.h"
    3
    #include "ccv_case.h"
    4
    #include "3rdparty/dsfmt/dSFMT.h"
    5
    6
    TEST_CASE("convolutional network of 11x11 on 225x225 with uniform weights")
    7
    1
    {
    8
    1
      ccv_convnet_layer_param_t params = {
    9
    1
        .type = CCV_CONVNET_CONVOLUTIONAL,
    10
    1
        .bias = 0,
    11
    1
        .glorot = sqrtf(2),
    12
    1
        .input = {
    13
    1
          .matrix = {
    14
    1
            .rows = 225,
    15
    1
            .cols = 225,
    16
    1
            .channels = 3,
    17
    1
            .partition = 1,
    18
    1
          },
    19
    1
        },
    20
    1
        .output = {
    21
    1
          .convolutional = {
    22
    1
            .count = 4,
    23
    1
            .strides = 4,
    24
    1
            .border = 1,
    25
    1
            .rows = 11,
    26
    1
            .cols = 11,
    27
    1
            .channels = 3,
    28
    1
            .partition = 1,
    29
    1
          },
    30
    1
        },
    31
    1
      };
    32
    1
      ccv_convnet_t* convnet = ccv_convnet_new(0, ccv_size(225, 225), &params, 1);
    33
    1
      int i, x, y;
    34
    1.45k
      for (i = 0; i < 11 * 11 * 3 * 4; i++1.45k)
    35
    1.45k
        convnet->layers[0].w[i] = 1;
    36
    1
      ccv_dense_matrix_t* a = ccv_dense_matrix_new(225, 225, CCV_32F | CCV_C3, 0, 0);
    37
    151k
      for (i = 0; i < 225 * 225 * 3; i++151k)
    38
    151k
        a->data.f32[i] = 1;
    39
    1
      ccv_dense_matrix_t* b = 0;
    40
    1
      ccv_convnet_encode(convnet, &a, &b, 1);
    41
    1
      ccv_matrix_free(a);
    42
    1
      REQUIRE(b->rows == 55 && b->cols == 55, "11x11 convolves on 225x255 with strides 4 should produce 55x55 matrix");
    43
    1
      ccv_dense_matrix_t* c = ccv_dense_matrix_new(55, 55, CCV_32F | 4, 0, 0);
    44
    56
      for (y = 0; y < 55; y++55)
    45
    3.08k
        for (x = 0; 55x < 55; x++3.02k)
    46
    15.1k
          for (i = 0; 3.02ki < 4; i++12.1k)
    47
    12.1k
          c->data.f32[(y * 55 + x) * 4 + i] = ((x == 0 && y == 0220) || (12.0kx == 012.0k && y == 54216) || (12.0kx == 5412.0k && y == 0220) || (12.0kx == 5412.0k && y == 54216)) ? 30016 : (12.0k(12.0kx == 012.0k || y == 011.8k || x == 5411.6k || y == 5411.4k) ? 330848 : 36311.2k);
    48
    1
      REQUIRE_MATRIX_EQ(b, c, "55x55 matrix should be exactly a matrix fill 363, with 300 on the corner and 330 on the border");
    49
    1
      ccv_matrix_free(b);
    50
    1
      ccv_matrix_free(c);
    51
    1
      ccv_convnet_free(convnet);
    52
    1
    }
    53
    54
    TEST_CASE("convolutional network of 5x5 on 27x27 with uniform weights")
    55
    1
    {
    56
    1
      ccv_convnet_layer_param_t params = {
    57
    1
        .type = CCV_CONVNET_CONVOLUTIONAL,
    58
    1
        .bias = 0,
    59
    1
        .glorot = sqrtf(2),
    60
    1
        .input = {
    61
    1
          .matrix = {
    62
    1
            .rows = 27,
    63
    1
            .cols = 27,
    64
    1
            .channels = 1,
    65
    1
            .partition = 1,
    66
    1
          },
    67
    1
        },
    68
    1
        .output = {
    69
    1
          .convolutional = {
    70
    1
            .count = 4,
    71
    1
            .strides = 1,
    72
    1
            .border = 2,
    73
    1
            .rows = 5,
    74
    1
            .cols = 5,
    75
    1
            .channels = 1,
    76
    1
            .partition = 1,
    77
    1
          },
    78
    1
        },
    79
    1
      };
    80
    1
      ccv_convnet_t* convnet = ccv_convnet_new(0, ccv_size(27, 27), &params, 1);
    81
    1
      int i, x, y;
    82
    101
      for (i = 0; i < 5 * 5 * 4; i++100)
    83
    100
        convnet->layers->w[i] = 1;
    84
    1
      ccv_dense_matrix_t* a = ccv_dense_matrix_new(27, 27, CCV_32F | CCV_C1, 0, 0);
    85
    730
      for (i = 0; i < 27 * 27; i++729)
    86
    729
        a->data.f32[i] = 1;
    87
    1
      ccv_dense_matrix_t* b = 0;
    88
    1
      ccv_convnet_encode(convnet, &a, &b, 1);
    89
    1
      REQUIRE(b->rows == 27 && b->cols == 27, "5x5 convolves on 27x27 with border 2 should produce 27x27 matrix");
    90
    1
      ccv_matrix_free(a);
    91
    1
      ccv_dense_matrix_t* c = ccv_dense_matrix_new(27, 27, CCV_32F | 4, 0, 0);
    92
    28
      for (y = 0; y < 27; y++27)
    93
    756
        for (x = 0; 27x < 27; x++729)
    94
    3.64k
          for (i = 0; 729i < 4; i++2.91k)
    95
    2.91k
          {
    96
    2.91k
            if ((x == 0 && y == 0108) || (2.91kx == 02.91k && y == 26104) || (2.90kx == 262.90k && y == 0108) || (2.90kx == 262.90k && y == 26104))
    97
    16
              c->data.f32[(y * 27 + x) * 4 + i] = 9;
    98
    2.90k
            else if ((x == 0 && y == 1100) || (2.89kx == 02.89k && y == 2596) || (2.89kx == 12.89k && y == 0108) || (2.88kx == 12.88k && y == 26104) || (2.88kx == 252.88k && y == 0108) || (2.88kx == 252.88k && y == 26104) || (2.87kx == 262.87k && y == 1100) || (2.87kx == 262.87k && y == 2596))
    99
    32
              c->data.f32[(y * 27 + x) * 4 + i] = 12;
    100
    2.86k
            else if (x == 0 || y == 02.77k || x == 262.68k || y == 262.59k)
    101
    368
              c->data.f32[(y * 27 + x) * 4 + i] = 15;
    102
    2.50k
            else if ((x == 1 && y == 1100) || (2.49kx == 12.49k && y == 2596) || (2.49kx == 252.49k && y == 1100) || (2.48kx == 252.48k && y == 2596))
    103
    16
              c->data.f32[(y * 27 + x) * 4 + i] = 16;
    104
    2.48k
            else if (x == 1 || y == 12.39k || x == 252.30k || y == 252.20k)
    105
    368
              c->data.f32[(y * 27 + x) * 4 + i] = 20;
    106
    2.11k
            else
    107
    2.11k
              c->data.f32[(y * 27 + x) * 4 + i] = 25;
    108
    2.91k
          }
    109
    1
      REQUIRE_MATRIX_EQ(b, c, "27x27 matrix should be exactly a matrix fill 25, with 9, 16 on the corner and 12, 15, 20 on the border");
    110
    1
      ccv_matrix_free(b);
    111
    1
      ccv_matrix_free(c);
    112
    1
      ccv_convnet_free(convnet);
    113
    1
    }
    114
    115
    TEST_CASE("convolutional network of 11x11 on 225x225 with non-uniform weights")
    116
    1
    {
    117
    1
      ccv_convnet_layer_param_t params = {
    118
    1
        .type = CCV_CONVNET_CONVOLUTIONAL,
    119
    1
        .bias = 0,
    120
    1
        .glorot = sqrtf(2),
    121
    1
        .input = {
    122
    1
          .matrix = {
    123
    1
            .rows = 225,
    124
    1
            .cols = 225,
    125
    1
            .channels = 1,
    126
    1
            .partition = 1,
    127
    1
          },
    128
    1
        },
    129
    1
        .output = {
    130
    1
          .convolutional = {
    131
    1
            .count = 4,
    132
    1
            .strides = 4,
    133
    1
            .border = 1,
    134
    1
            .rows = 11,
    135
    1
            .cols = 11,
    136
    1
            .channels = 1,
    137
    1
            .partition = 1,
    138
    1
          },
    139
    1
        },
    140
    1
      };
    141
    1
      ccv_convnet_t* convnet = ccv_convnet_new(0, ccv_size(225, 225), &params, 1);
    142
    1
      int i, x, y;
    143
    5
      for (x = 0; x < 4; x++4)
    144
    488
        for (i = 0; 4i < 11 * 11; i++484)
    145
    484
          convnet->layers[0].w[x * 11 * 11 + i] = i + 1;
    146
    1
      ccv_dense_matrix_t* a = ccv_dense_matrix_new(225, 225, CCV_32F | CCV_C1, 0, 0);
    147
    50.6k
      for (i = 0; i < 225 * 225; i++50.6k)
    148
    50.6k
        a->data.f32[i] = i + 1;
    149
    1
      ccv_dense_matrix_t* b = 0;
    150
    1
      ccv_convnet_encode(convnet, &a, &b, 1);
    151
    1
      ccv_matrix_free(a);
    152
    1
      REQUIRE(b->rows == 55 && b->cols == 55, "11x11 convolves on 225x255 with strides 4 should produce 55x55 matrix");
    153
    1
      ccv_dense_matrix_t* c = ccv_dense_matrix_new(55, 55, CCV_32F | 4, 0, 0);
    154
    1
      float sum = 0;
    155
      // first column
    156
    11
      for (y = 0; y < 10; y++10)
    157
    110
        for (x = 0; 10x < 10; x++100)
    158
    100
          sum += ((y + 1) * 11 + x + 2) * (y * 225 + x + 1);
    159
    5
      for (i = 0; i < 4; i++4)
    160
    4
        c->data.f32[i] = sum;
    161
    1
      sum = 0;
    162
    11
      for (y = 0; y < 10; y++10)
    163
    120
        for (x = 0; 10x < 11; x++110)
    164
    110
          sum += ((y + 1) * 11 + x + 1) * (y * 225 + (x + 3) + 1);
    165
    54
      for (x = 1; x < 54; x++53)
    166
    265
        for (i = 0; 53i < 4; i++212)
    167
    212
          c->data.f32[x * 4 + i] = sum + (x - 1) * 4 * (11 * 11 + 12) * 11 * 10 / 2;
    168
    1
      sum = 0;
    169
    11
      for (y = 0; y < 10; y++10)
    170
    110
        for (x = 0; 10x < 10; x++100)
    171
    100
          sum += ((y + 1) * 11 + x + 1) * (y * 225 + (x + 215) + 1);
    172
    5
      for (i = 0; i < 4; i++4)
    173
    4
        c->data.f32[54 * 4 + i] = sum;
    174
      // last column
    175
    1
      sum = 0;
    176
    11
      for (y = 0; y < 10; y++10)
    177
    110
        for (x = 0; 10x < 10; x++100)
    178
    100
          sum += (y * 11 + x + 2) * ((y + 215) * 225 + x + 1);
    179
    5
      for (i = 0; i < 4; i++4)
    180
    4
        c->data.f32[55 * 54 * 4 + i] = sum;
    181
    1
      sum = 0;
    182
    11
      for (y = 0; y < 10; y++10)
    183
    120
        for (x = 0; 10x < 11; x++110)
    184
    110
          sum += (y * 11 + x + 1) * ((y + 215) * 225 + (x + 3) + 1);
    185
    54
      for (x = 1; x < 54; x++53)
    186
    265
        for (i = 0; 53i < 4; i++212)
    187
    212
          c->data.f32[(55 * 54 + x) * 4 + i] = sum + (x - 1) * 4 * (10 * 11 + 1) * 11 * 10 / 2;
    188
    1
      sum = 0;
    189
    11
      for (y = 0; y < 10; y++10)
    190
    110
        for (x = 0; 10x < 10; x++100)
    191
    100
          sum += (y * 11 + x + 1) * ((y + 215) * 225 + (x + 215) + 1);
    192
    5
      for (i = 0; i < 4; i++4)
    193
    4
        c->data.f32[(55 * 54 + 54) * 4 + i] = sum;
    194
    1
      float border[] = {
    195
    1
        0, 0
    196
    1
      };
    197
    12
      for (y = 0; y < 11; y++11)
    198
    121
        for (x = 0; 11x < 10; x++110)
    199
    110
          border[0] += (y * 11 + x + 2) * ((y + 3) * 225 + x + 1);
    200
    12
      for (y = 0; y < 11; y++11)
    201
    121
        for (x = 0; 11x < 10; x++110)
    202
    110
          border[1] += (y * 11 + x + 1) * ((y + 3) * 225 + (x + 215) + 1);
    203
    1
      sum = 0;
    204
    12
      for (y = 0; y < 11; y++11)
    205
    132
        for (x = 0; 11x < 11; x++121)
    206
    121
          sum += (y * 11 + x + 1) * ((y + 3) * 225 + (x + 3) + 1);
    207
    54
      for (y = 1; y < 54; y++53)
    208
    53
      {
    209
    265
        for (i = 0; i < 4; i++212)
    210
    212
          c->data.f32[y * 55 * 4 + i] = border[0];
    211
    2.86k
        for (x = 1; x < 54; x++2.80k)
    212
    14.0k
          for (i = 0; 2.80ki < 4; i++11.2k)
    213
    11.2k
            c->data.f32[(y * 55 + x) * 4 + i] = sum + (x - 1) * 4 * (11 * 11 + 1) * 11 * 11 / 2;
    214
    265
        for (i = 0; i < 4; i++212)
    215
    212
          c->data.f32[(y * 55 + 54) * 4 + i] = border[1];
    216
    53
        sum += 225 * 4 * (11 * 11 + 1) * 11 * 11 / 2;
    217
    53
        border[0] += 225 * 4 * ((11 * 11 + 1) * 11 * 11 / 2 - (10 * 11 + 1 + 1) * 11 / 2);
    218
    53
        border[1] += 225 * 4 * ((11 * 11 + 1) * 11 * 11 / 2 - (11 * 11 + 11) * 11 / 2);
    219
    53
      }
    220
      // regularize the output so it is within the tolerance
    221
    12.1k
      for (i = 0; i < 55 * 55 * 4; i++12.1k)
    222
    12.1k
        c->data.f32[i] = c->data.f32[i] * 1e-7, b->data.f32[i] = b->data.f32[i] * 1e-7;
    223
    1
      REQUIRE_MATRIX_EQ(b, c, "55x55 matrix should be exactly the same");
    224
    1
      ccv_matrix_free(b);
    225
    1
      ccv_matrix_free(c);
    226
    1
      ccv_convnet_free(convnet);
    227
    1
    }
    228
    229
    TEST_CASE("convolutional network of 5x5 on 27x27 with non-uniform weights")
    230
    1
    {
    231
    1
      ccv_convnet_layer_param_t params = {
    232
    1
        .type = CCV_CONVNET_CONVOLUTIONAL,
    233
    1
        .bias = 0,
    234
    1
        .glorot = sqrtf(2),
    235
    1
        .input = {
    236
    1
          .matrix = {
    237
    1
            .rows = 27,
    238
    1
            .cols = 27,
    239
    1
            .channels = 1,
    240
    1
            .partition = 1,
    241
    1
          },
    242
    1
        },
    243
    1
        .output = {
    244
    1
          .convolutional = {
    245
    1
            .count = 4,
    246
    1
            .strides = 1,
    247
    1
            .border = 2,
    248
    1
            .rows = 5,
    249
    1
            .cols = 5,
    250
    1
            .channels = 1,
    251
    1
            .partition = 1,
    252
    1
          },
    253
    1
        },
    254
    1
      };
    255
    1
      ccv_convnet_t* convnet = ccv_convnet_new(0, ccv_size(27, 27), &params, 1);
    256
    1
      int i, x, y;
    257
    5
      for (x = 0; x < 4; x++4)
    258
    104
        for (i = 0; 4i < 5 * 5; i++100)
    259
    100
          convnet->layers->w[x * 5 * 5 + i] = i + 1;
    260
    1
      ccv_dense_matrix_t* a = ccv_dense_matrix_new(27, 27, CCV_32F | CCV_C1, 0, 0);
    261
    730
      for (i = 0; i < 27 * 27; i++729)
    262
    729
        a->data.f32[i] = i + 1;
    263
    1
      ccv_dense_matrix_t* b = 0;
    264
    1
      ccv_convnet_encode(convnet, &a, &b, 1);
    265
    1
      REQUIRE(b->rows == 27 && b->cols == 27, "5x5 convolves on 27x27 with border 2 should produce 27x27 matrix");
    266
    1
      ccv_matrix_free(a);
    267
    1
      ccv_dense_matrix_t* c = ccv_dense_matrix_new(27, 27, CCV_32F | 4, 0, 0);
    268
      // the first column
    269
    1
      float sum = 0;
    270
    4
      for (y = 0; y < 3; y++3)
    271
    12
        for (x = 0; 3x < 3; x++9)
    272
    9
          sum += ((y + 2) * 5 + x + 3) * (y * 27 + x + 1);
    273
    5
      for (i = 0; i < 4; i++4)
    274
    4
        c->data.f32[i] = sum;
    275
    1
      sum = 0;
    276
    4
      for (y = 0; y < 3; y++3)
    277
    15
        for (x = 0; 3x < 4; x++12)
    278
    12
          sum += ((y + 2) * 5 + x + 2) * (y * 27 + x + 1);
    279
    5
      for (i = 0; i < 4; i++4)
    280
    4
        c->data.f32[4 + i] = sum;
    281
    1
      sum = 0;
    282
    4
      for (y = 0; y < 3; y++3)
    283
    18
        for (x = 0; 3x < 5; x++15)
    284
    15
          sum += ((y + 2) * 5 + x + 1) * (y * 27 + x + 1);
    285
    24
      for (x = 2; x < 25; x++23)
    286
    115
        for (i = 0; 23i < 4; i++92)
    287
    92
          c->data.f32[x * 4 + i] = sum + (x - 2) * 36 * 15 / 2;
    288
    1
      sum = 0;
    289
    4
      for (y = 0; y < 3; y++3)
    290
    15
        for (x = 0; 3x < 4; x++12)
    291
    12
          sum += ((y + 2) * 5 + x + 1) * (y * 27 + x + 24);
    292
    5
      for (i = 0; i < 4; i++4)
    293
    4
        c->data.f32[25 * 4 + i] = sum;
    294
    1
      sum = 0;
    295
    4
      for (y = 0; y < 3; y++3)
    296
    12
        for (x = 0; 3x < 3; x++9)
    297
    9
          sum += ((y + 2) * 5 + x + 1) * (y * 27 + x + 25);
    298
    5
      for (i = 0; i < 4; i++4)
    299
    4
        c->data.f32[26 * 4 + i] = sum;
    300
      // the second column
    301
    1
      sum = 0;
    302
    5
      for (y = 0; y < 4; y++4)
    303
    16
        for (x = 0; 4x < 3; x++12)
    304
    12
          sum += ((y + 1) * 5 + x + 3) * (y * 27 + x + 1);
    305
    5
      for (i = 0; i < 4; i++4)
    306
    4
        c->data.f32[27 * 4 + i] = sum;
    307
    1
      sum = 0;
    308
    5
      for (y = 0; y < 4; y++4)
    309
    20
        for (x = 0; 4x < 4; x++16)
    310
    16
          sum += ((y + 1) * 5 + x + 2) * (y * 27 + x + 1);
    311
    5
      for (i = 0; i < 4; i++4)
    312
    4
        c->data.f32[28 * 4 + i] = sum;
    313
    1
      sum = 0;
    314
    5
      for (y = 0; y < 4; y++4)
    315
    24
        for (x = 0; 4x < 5; x++20)
    316
    20
          sum += ((y + 1) * 5 + x + 1) * (y * 27 + x + 1);
    317
    24
      for (x = 2; x < 25; x++23)
    318
    115
        for (i = 0; 23i < 4; i++92)
    319
    92
          c->data.f32[(27 + x) * 4 + i] = sum + (x - 2) * 31 * 20 / 2;
    320
    1
      sum = 0;
    321
    5
      for (y = 0; y < 4; y++4)
    322
    20
        for (x = 0; 4x < 4; x++16)
    323
    16
          sum += ((y + 1) * 5 + x + 1) * (y * 27 + x + 24);
    324
    5
      for (i = 0; i < 4; i++4)
    325
    4
        c->data.f32[52 * 4 + i] = sum;
    326
    1
      sum = 0;
    327
    5
      for (y = 0; y < 4; y++4)
    328
    16
        for (x = 0; 4x < 3; x++12)
    329
    12
          sum += ((y + 1) * 5 + x + 1) * (y * 27 + x + 25);
    330
    5
      for (i = 0; i < 4; i++4)
    331
    4
        c->data.f32[53 * 4 + i] = sum;
    332
    1
      sum = 0;
    333
      // the last 2nd column
    334
    5
      for (y = 0; y < 4; y++4)
    335
    16
        for (x = 0; 4x < 3; x++12)
    336
    12
          sum += (y * 5 + x + 3) * ((y + 23) * 27 + x + 1);
    337
    5
      for (i = 0; i < 4; i++4)
    338
    4
        c->data.f32[27 * 25 * 4 + i] = sum;
    339
    1
      sum = 0;
    340
    5
      for (y = 0; y < 4; y++4)
    341
    20
        for (x = 0; 4x < 4; x++16)
    342
    16
          sum += (y * 5 + x + 2) * ((y + 23) * 27 + x + 1);
    343
    5
      for (i = 0; i < 4; i++4)
    344
    4
        c->data.f32[(27 * 25 + 1) * 4 + i] = sum;
    345
    1
      sum = 0;
    346
    5
      for (y = 0; y < 4; y++4)
    347
    24
        for (x = 0; 4x < 5; x++20)
    348
    20
          sum += (y * 5 + x + 1) * ((y + 23) * 27 + x + 1);
    349
    24
      for (x = 2; x < 25; x++23)
    350
    115
        for (i = 0; 23i < 4; i++92)
    351
    92
          c->data.f32[(27 * 25 + x) * 4 + i] = sum + (x - 2) * 21 * 20 / 2;
    352
    1
      sum = 0;
    353
    5
      for (y = 0; y < 4; y++4)
    354
    20
        for (x = 0; 4x < 4; x++16)
    355
    16
          sum += (y * 5 + x + 1) * ((y + 23) * 27 + x + 24);
    356
    5
      for (i = 0; i < 4; i++4)
    357
    4
        c->data.f32[(27 * 25 + 25) * 4 + i] = sum;
    358
    1
      sum = 0;
    359
    5
      for (y = 0; y < 4; y++4)
    360
    16
        for (x = 0; 4x < 3; x++12)
    361
    12
          sum += (y * 5 + x + 1) * ((y + 23) * 27 + x + 25);
    362
    5
      for (i = 0; i < 4; i++4)
    363
    4
        c->data.f32[(27 * 25 + 26) * 4 + i] = sum;
    364
      // the last column
    365
    1
      sum = 0;
    366
    4
      for (y = 0; y < 3; y++3)
    367
    12
        for (x = 0; 3x < 3; x++9)
    368
    9
          sum += (y * 5 + x + 3) * ((y + 24) * 27 + x + 1);
    369
    5
      for (i = 0; i < 4; i++4)
    370
    4
        c->data.f32[27 * 26 * 4 + i] = sum;
    371
    1
      sum = 0;
    372
    4
      for (y = 0; y < 3; y++3)
    373
    15
        for (x = 0; 3x < 4; x++12)
    374
    12
          sum += (y * 5 + x + 2) * ((y + 24) * 27 + x + 1);
    375
    5
      for (i = 0; i < 4; i++4)
    376
    4
        c->data.f32[(27 * 26 + 1) * 4 + i] = sum;
    377
    1
      sum = 0;
    378
    4
      for (y = 0; y < 3; y++3)
    379
    18
        for (x = 0; 3x < 5; x++15)
    380
    15
          sum += (y * 5 + x + 1) * ((y + 24) * 27 + x + 1);
    381
    24
      for (x = 2; x < 25; x++23)
    382
    115
        for (i = 0; 23i < 4; i++92)
    383
    92
          c->data.f32[(27 * 26 + x) * 4 + i] = sum + (x - 2) * 16 * 15 / 2;
    384
    1
      sum = 0;
    385
    4
      for (y = 0; y < 3; y++3)
    386
    15
        for (x = 0; 3x < 4; x++12)
    387
    12
          sum += (y * 5 + x + 1) * ((y + 24) * 27 + x + 24);
    388
    5
      for (i = 0; i < 4; i++4)
    389
    4
        c->data.f32[(27 * 26 + 25) * 4 + i] = sum;
    390
    1
      sum = 0;
    391
    4
      for (y = 0; y < 3; y++3)
    392
    12
        for (x = 0; 3x < 3; x++9)
    393
    9
          sum += (y * 5 + x + 1) * ((y + 24) * 27 + x + 25);
    394
    5
      for (i = 0; i < 4; i++4)
    395
    4
        c->data.f32[(27 * 26 + 26) * 4 + i] = sum;
    396
    1
      float border[] = {
    397
    1
        0, 0, 0, 0
    398
    1
      };
    399
    6
      for (y = 0; y < 5; y++5)
    400
    20
        for (x = 0; 5x < 3; x++15)
    401
    15
          border[0] += (y * 5 + x + 3) * (y * 27 + x + 1);
    402
    6
      for (y = 0; y < 5; y++5)
    403
    25
        for (x = 0; 5x < 4; x++20)
    404
    20
          border[1] += (y * 5 + x + 2) * (y * 27 + x + 1);
    405
    6
      for (y = 0; y < 5; y++5)
    406
    25
        for (x = 0; 5x < 4; x++20)
    407
    20
          border[2] += (y * 5 + x + 1) * (y * 27 + x + 24);
    408
    6
      for (y = 0; y < 5; y++5)
    409
    20
        for (x = 0; 5x < 3; x++15)
    410
    15
          border[3] += (y * 5 + x + 1) * (y * 27 + x + 25);
    411
    1
      sum = 0;
    412
    6
      for (y = 0; y < 5; y++5)
    413
    30
        for (x = 0; 5x < 5; x++25)
    414
    25
          sum += (y * 5 + x + 1) * (y * 27 + x + 1);
    415
    24
      for (y = 2; y < 25; y++23)
    416
    23
      {
    417
    115
        for (i = 0; i < 4; i++92)
    418
    92
        {
    419
    92
          c->data.f32[y * 27 * 4 + i] = border[0] + (y - 2) * 27 * (3 + 4 + 5 + 8 + 9 + 10 + 13 + 14 + 15 + 18 + 19 + 20 + 23 + 24 + 25);
    420
    92
          c->data.f32[(y * 27 + 1) * 4 + i] = border[1] + (y - 2) * 27 * (2 + 3 + 4 + 5 + 7 + 8 + 9 + 10 + 12 + 13 + 14 + 15 + 17 + 18 + 19 + 20 + 22 + 23 + 24 + 25);
    421
    2.20k
          for (x = 2; x < 25; x++2.11k)
    422
    2.11k
            c->data.f32[(y * 27 + x) * 4 + i] = sum + ((y - 2) * 27 + x - 2) * 26 * 25 / 2;
    423
    92
          c->data.f32[(y * 27 + 25) * 4 + i] = border[2] + (y - 2) * 27 * (1 + 2 + 3 + 4 + 6 + 7 + 8 + 9 + 11 + 12 + 13 + 14 + 16 + 17 + 18 + 19 + 21 + 22 + 23 + 24);
    424
    92
          c->data.f32[(y * 27 + 26) * 4 + i] = border[3] + (y - 2) * 27 * (1 + 2 + 3 + 6 + 7 + 8 + 11 + 12 + 13 + 16 + 17 + 18 + 21 + 22 + 23);
    425
    92
        }
    426
    23
      }
    427
    1
      REQUIRE_MATRIX_EQ(b, c, "27x27 matrix should be exactly the same");
    428
    1
      ccv_matrix_free(b);
    429
    1
      ccv_matrix_free(c);
    430
    1
      ccv_convnet_free(convnet);
    431
    1
    }
    432
    433
    TEST_CASE("convolutional network of 5x5x4 on 27x27x8 partitioned by 2")
    434
    1
    {
    435
    1
      ccv_convnet_layer_param_t params = {
    436
    1
        .type = CCV_CONVNET_CONVOLUTIONAL,
    437
    1
        .bias = 0,
    438
    1
        .glorot = sqrtf(2),
    439
    1
        .input = {
    440
    1
          .matrix = {
    441
    1
            .rows = 27,
    442
    1
            .cols = 27,
    443
    1
            .channels = 4,
    444
    1
            .partition = 2,
    445
    1
          },
    446
    1
        },
    447
    1
        .output = {
    448
    1
          .convolutional = {
    449
    1
            .count = 8,
    450
    1
            .strides = 1,
    451
    1
            .border = 2,
    452
    1
            .rows = 5,
    453
    1
            .cols = 5,
    454
    1
            .channels = 4,
    455
    1
            .partition = 2,
    456
    1
          },
    457
    1
        },
    458
    1
      };
    459
    1
      ccv_convnet_t* convnet = ccv_convnet_new(0, ccv_size(27, 27), &params, 1);
    460
    1
      int i, k;
    461
    401
      for (i = 0; i < convnet->layers->wnum; i++400)
    462
    400
        convnet->layers->w[i] = i;
    463
    9
      for (i = 0; i < convnet->layers->net.convolutional.count; i++8)
    464
    8
        convnet->layers->bias[i] = i + 1;
    465
    1
      ccv_dense_matrix_t* a = ccv_dense_matrix_new(27, 27, CCV_32F | 4, 0, 0);
    466
    2.91k
      for (i = 0; i < 27 * 27 * 4; i++2.91k)
    467
    2.91k
        a->data.f32[i] = 20 - i;
    468
    1
      ccv_dense_matrix_t* b = 0;
    469
    1
      ccv_convnet_encode(convnet, &a, &b, 1);
    470
    1
      ccv_convnet_layer_param_t partitioned_params = {
    471
    1
        .type = CCV_CONVNET_CONVOLUTIONAL,
    472
    1
        .bias = 0,
    473
    1
        .glorot = sqrtf(2),
    474
    1
        .input = {
    475
    1
          .matrix = {
    476
    1
            .rows = 27,
    477
    1
            .cols = 27,
    478
    1
            .channels = 2,
    479
    1
            .partition = 1,
    480
    1
          },
    481
    1
        },
    482
    1
        .output = {
    483
    1
          .convolutional = {
    484
    1
            .count = 4,
    485
    1
            .strides = 1,
    486
    1
            .border = 2,
    487
    1
            .rows = 5,
    488
    1
            .cols = 5,
    489
    1
            .channels = 2,
    490
    1
            .partition = 1,
    491
    1
          },
    492
    1
        },
    493
    1
      };
    494
    1
      ccv_convnet_t* partitioned_convnet = ccv_convnet_new(0, ccv_size(27, 27), &partitioned_params, 1);
    495
    1
      memcpy(partitioned_convnet->layers->w, convnet->layers->w, sizeof(float) * (convnet->layers->wnum / 2));
    496
    1
      memcpy(partitioned_convnet->layers->bias, convnet->layers->bias, sizeof(float) * (convnet->layers->net.convolutional.count / 2));
    497
    1
      ccv_dense_matrix_t* aa = ccv_dense_matrix_new(27, 27, CCV_32F | 2, 0, 0);
    498
    730
      for (i = 0; i < 27 * 27; i++729)
    499
    2.18k
        for (k = 0; 729k < 2; k++1.45k)
    500
    1.45k
          aa->data.f32[i * 2 + k] = a->data.f32[i * 4 + k];
    501
    1
      ccv_dense_matrix_t* bb = ccv_dense_matrix_new(27, 27, CCV_32F | 8, 0, 0);
    502
    1
      ccv_dense_matrix_t* cc = 0;
    503
    1
      ccv_convnet_encode(partitioned_convnet, &aa, &cc, 1);
    504
    730
      for (i = 0; i < 27 * 27; i++729)
    505
    3.64k
        for (k = 0; 729k < 4; k++2.91k)
    506
    2.91k
          bb->data.f32[i * 8 + k] = cc->data.f32[i * 4 + k];
    507
    1
      memcpy(partitioned_convnet->layers->w, convnet->layers->w + (convnet->layers->wnum / 2), sizeof(float) * (convnet->layers->wnum / 2));
    508
    1
      memcpy(partitioned_convnet->layers->bias, convnet->layers->bias + (convnet->layers->net.convolutional.count / 2), sizeof(float) * (convnet->layers->net.convolutional.count / 2));
    509
    730
      for (i = 0; i < 27 * 27; i++729)
    510
    2.18k
        for (k = 0; 729k < 2; k++1.45k)
    511
    1.45k
          aa->data.f32[i * 2 + k] = a->data.f32[i * 4 + 2 + k];
    512
    1
      ccv_convnet_encode(partitioned_convnet, &aa, &cc, 1);
    513
    730
      for (i = 0; i < 27 * 27; i++729)
    514
    3.64k
        for (k = 0; 729k < 4; k++2.91k)
    515
    2.91k
          bb->data.f32[i * 8 + 4 + k] = cc->data.f32[i * 4 + k];
    516
    1
      REQUIRE_MATRIX_EQ(b, bb, "27x27x8 matrix computed from convnet with partition and partitioned convnet should be exactly the same");
    517
    1
      ccv_matrix_free(a);
    518
    1
      ccv_matrix_free(b);
    519
    1
      ccv_matrix_free(aa);
    520
    1
      ccv_matrix_free(bb);
    521
    1
      ccv_matrix_free(cc);
    522
    1
      ccv_convnet_free(convnet);
    523
    1
      ccv_convnet_free(partitioned_convnet);
    524
    1
    }
    525
    526
    TEST_CASE("full connect network from 13x13x128 to 2048")
    527
    1
    {
    528
    1
      ccv_convnet_layer_param_t params = {
    529
    1
        .type = CCV_CONVNET_FULL_CONNECT,
    530
    1
        .bias = 0,
    531
    1
        .glorot = sqrtf(2),
    532
    1
        .input = {
    533
    1
          .matrix = {
    534
    1
            .rows = 13,
    535
    1
            .cols = 13,
    536
    1
            .channels = 128,
    537
    1
            .partition = 1,
    538
    1
          },
    539
    1
          .node = {
    540
    1
            .count = 13 * 13 * 128,
    541
    1
          },
    542
    1
        },
    543
    1
        .output = {
    544
    1
          .full_connect = {
    545
    1
            .relu = 0,
    546
    1
            .count = 2048,
    547
    1
          },
    548
    1
        },
    549
    1
      };
    550
    1
      ccv_convnet_t* convnet = ccv_convnet_new(0, ccv_size(13, 13), &params, 1);
    551
    1
      int i;
    552
    44.3M
      for (i = 0; i < 13 * 13 * 128 * 2048; i++44.3M)
    553
    44.3M
        convnet->layers->w[i] = 1;
    554
    1
      ccv_dense_matrix_t* a = ccv_dense_matrix_new(13, 13, CCV_32F | 128, 0, 0);
    555
    21.6k
      for (i = 0; i < 13 * 13 * 128; i++21.6k)
    556
    21.6k
        a->data.f32[i] = 1;
    557
    1
      ccv_dense_matrix_t* b = 0;
    558
    1
      ccv_convnet_encode(convnet, &a, &b, 1);
    559
    1
      ccv_matrix_free(a);
    560
    1
      REQUIRE(b->rows == 2048 && b->cols == 1, "full connect network output should be 2048 neurons");
    561
    1
      ccv_dense_matrix_t* c = ccv_dense_matrix_new(2048, 1, CCV_32F | CCV_C1, 0, 0);
    562
    2.04k
      for (i = 0; i < 2048; i++2.04k)
    563
    2.04k
        c->data.f32[i] = 13 * 13 * 128;
    564
    1
      REQUIRE_MATRIX_EQ(b, c, "full connect network output should be exactly 13 * 13 * 128");
    565
    1
      ccv_matrix_free(b);
    566
    1
      ccv_matrix_free(c);
    567
    1
      ccv_convnet_free(convnet);
    568
    1
    }
    569
    570
    TEST_CASE("maximum pool network of 55x55 with window of 3x3 and stride of 2")
    571
    1
    {
    572
    1
      ccv_convnet_layer_param_t params = {
    573
    1
        .type = CCV_CONVNET_MAX_POOL,
    574
    1
        .bias = 0,
    575
    1
        .glorot = sqrtf(2),
    576
    1
        .input = {
    577
    1
          .matrix = {
    578
    1
            .rows = 55,
    579
    1
            .cols = 55,
    580
    1
            .channels = 1,
    581
    1
            .partition = 1,
    582
    1
          },
    583
    1
        },
    584
    1
        .output = {
    585
    1
          .pool = {
    586
    1
            .size = 3,
    587
    1
            .strides = 2,
    588
    1
            .border = 0,
    589
    1
          },
    590
    1
        },
    591
    1
      };
    592
    1
      ccv_convnet_t* convnet = ccv_convnet_new(0, ccv_size(55, 55), &params, 1);
    593
    1
      ccv_dense_matrix_t* a = ccv_dense_matrix_new(55, 55, CCV_32F | CCV_C1, 0, 0);
    594
    1
      int i, x, y;
    595
    3.02k
      for (i = 0; i < 55 * 55; i++3.02k)
    596
    3.02k
        a->data.f32[i] = i + 1;
    597
    1
      ccv_dense_matrix_t* b = 0;
    598
    1
      ccv_convnet_encode(convnet, &a, &b, 1);
    599
    1
      ccv_matrix_free(a);
    600
    1
      ccv_dense_matrix_t* c = ccv_dense_matrix_new(27, 27, CCV_32F | CCV_C1, 0, 0);
    601
    28
      for (y = 0; y < 27; y++27)
    602
    756
        for (x = 0; 27x < 27; x++729)
    603
    729
          c->data.f32[y * 27 + x] = 113 + y * 110 + x * 2;
    604
    1
      REQUIRE_MATRIX_EQ(b, c, "max pool network output should be exactly the same");
    605
    1
      ccv_matrix_free(b);
    606
    1
      ccv_matrix_free(c);
    607
    1
      ccv_convnet_free(convnet);
    608
    1
    }
    609
    610
    TEST_CASE("maximum pool network of 57x57 with window of 3x3 and stride of 3")
    611
    1
    {
    612
    1
      ccv_convnet_layer_param_t params = {
    613
    1
        .type = CCV_CONVNET_MAX_POOL,
    614
    1
        .bias = 0,
    615
    1
        .glorot = sqrtf(2),
    616
    1
        .input = {
    617
    1
          .matrix = {
    618
    1
            .rows = 57,
    619
    1
            .cols = 57,
    620
    1
            .channels = 1,
    621
    1
            .partition = 1,
    622
    1
          },
    623
    1
        },
    624
    1
        .output = {
    625
    1
          .pool = {
    626
    1
            .size = 3,
    627
    1
            .strides = 3,
    628
    1
            .border = 0,
    629
    1
          },
    630
    1
        },
    631
    1
      };
    632
    1
      ccv_convnet_t* convnet = ccv_convnet_new(0, ccv_size(57, 57), &params, 1);
    633
    1
      ccv_dense_matrix_t* a = ccv_dense_matrix_new(57, 57, CCV_32F | CCV_C1, 0, 0);
    634
    1
      int i, x, y;
    635
    3.25k
      for (i = 0; i < 57 * 57; i++3.24k)
    636
    3.24k
        a->data.f32[i] = i + 1;
    637
    1
      ccv_dense_matrix_t* b = 0;
    638
    1
      ccv_convnet_encode(convnet, &a, &b, 1);
    639
    1
      ccv_matrix_free(a);
    640
    1
      ccv_dense_matrix_t* c = ccv_dense_matrix_new(19, 19, CCV_32F | CCV_C1, 0, 0);
    641
    20
      for (y = 0; y < 19; y++19)
    642
    380
        for (x = 0; 19x < 19; x++361)
    643
    361
          c->data.f32[y * 19 + x] = 117 + y * 171 + x * 3;
    644
    1
      REQUIRE_MATRIX_EQ(b, c, "max pool network output should be exactly the same");
    645
    1
      ccv_matrix_free(b);
    646
    1
      ccv_matrix_free(c);
    647
    1
      ccv_convnet_free(convnet);
    648
    1
    }
    649
    650
    TEST_CASE("maximum pool network of 54x54 with window of 2x2 and stride of 2")
    651
    1
    {
    652
    1
      ccv_convnet_layer_param_t params = {
    653
    1
        .type = CCV_CONVNET_MAX_POOL,
    654
    1
        .bias = 0,
    655
    1
        .glorot = sqrtf(2),
    656
    1
        .input = {
    657
    1
          .matrix = {
    658
    1
            .rows = 54,
    659
    1
            .cols = 54,
    660
    1
            .channels = 1,
    661
    1
            .partition = 1,
    662
    1
          },
    663
    1
        },
    664
    1
        .output = {
    665
    1
          .pool = {
    666
    1
            .size = 2,
    667
    1
            .strides = 2,
    668
    1
            .border = 0,
    669
    1
          },
    670
    1
        },
    671
    1
      };
    672
    1
      ccv_convnet_t* convnet = ccv_convnet_new(0, ccv_size(54, 54), &params, 1);
    673
    1
      ccv_dense_matrix_t* a = ccv_dense_matrix_new(54, 54, CCV_32F | CCV_C1, 0, 0);
    674
    1
      int i, x, y;
    675
    2.91k
      for (i = 0; i < 54 * 54; i++2.91k)
    676
    2.91k
        a->data.f32[i] = i + 1;
    677
    1
      ccv_dense_matrix_t* b = 0;
    678
    1
      ccv_convnet_encode(convnet, &a, &b, 1);
    679
    1
      ccv_matrix_free(a);
    680
    1
      ccv_dense_matrix_t* c = ccv_dense_matrix_new(27, 27, CCV_32F | CCV_C1, 0, 0);
    681
    28
      for (y = 0; y < 27; y++27)
    682
    756
        for (x = 0; 27x < 27; x++729)
    683
    729
          c->data.f32[y * 27 + x] = 56 + y * 108 + x * 2;
    684
    1
      REQUIRE_MATRIX_EQ(b, c, "max pool network output should be exactly the same");
    685
    1
      ccv_matrix_free(b);
    686
    1
      ccv_matrix_free(c);
    687
    1
      ccv_convnet_free(convnet);
    688
    1
    }
    689
    690
    TEST_CASE("average pool network of 55x55 with window of 3x3 and stride of 2")
    691
    1
    {
    692
    1
      ccv_convnet_layer_param_t params = {
    693
    1
        .type = CCV_CONVNET_AVERAGE_POOL,
    694
    1
        .bias = 0,
    695
    1
        .glorot = sqrtf(2),
    696
    1
        .input = {
    697
    1
          .matrix = {
    698
    1
            .rows = 55,
    699
    1
            .cols = 55,
    700
    1
            .channels = 1,
    701
    1
            .partition = 1,
    702
    1
          },
    703
    1
        },
    704
    1
        .output = {
    705
    1
          .pool = {
    706
    1
            .size = 3,
    707
    1
            .strides = 2,
    708
    1
            .border = 0,
    709
    1
          },
    710
    1
        },
    711
    1
      };
    712
    1
      ccv_convnet_t* convnet = ccv_convnet_new(0, ccv_size(55, 55), &params, 1);
    713
    1
      ccv_dense_matrix_t* a = ccv_dense_matrix_new(55, 55, CCV_32F | CCV_C1, 0, 0);
    714
    1
      int i, x, y;
    715
    3.02k
      for (i = 0; i < 55 * 55; i++3.02k)
    716
    3.02k
        a->data.f32[i] = i + 1;
    717
    1
      ccv_dense_matrix_t* b = 0;
    718
    1
      ccv_convnet_encode(convnet, &a, &b, 1);
    719
    1
      ccv_matrix_free(a);
    720
    1
      ccv_dense_matrix_t* c = ccv_dense_matrix_new(27, 27, CCV_32F | CCV_C1, 0, 0);
    721
    28
      for (y = 0; y < 27; y++27)
    722
    756
        for (x = 0; 27x < 27; x++729)
    723
    729
          c->data.f32[y * 27 + x] = 57 + y * 110 + x * 2;
    724
    1
      REQUIRE_MATRIX_EQ(b, c, "average pool network output should be exactly the same");
    725
    1
      ccv_matrix_free(b);
    726
    1
      ccv_matrix_free(c);
    727
    1
      ccv_convnet_free(convnet);
    728
    1
    }
    729
    730
    TEST_CASE("average pool network of 57x57 with window of 3x3 and stride of 3")
    731
    1
    {
    732
    1
      ccv_convnet_layer_param_t params = {
    733
    1
        .type = CCV_CONVNET_AVERAGE_POOL,
    734
    1
        .bias = 0,
    735
    1
        .glorot = sqrtf(2),
    736
    1
        .input = {
    737
    1
          .matrix = {
    738
    1
            .rows = 57,
    739
    1
            .cols = 57,
    740
    1
            .channels = 1,
    741
    1
            .partition = 1,
    742
    1
          },
    743
    1
        },
    744
    1
        .output = {
    745
    1
          .pool = {
    746
    1
            .size = 3,
    747
    1
            .strides = 3,
    748
    1
            .border = 0,
    749
    1
          },
    750
    1
        },
    751
    1
      };
    752
    1
      ccv_convnet_t* convnet = ccv_convnet_new(0, ccv_size(57, 57), &params, 1);
    753
    1
      ccv_dense_matrix_t* a = ccv_dense_matrix_new(57, 57, CCV_32F | CCV_C1, 0, 0);
    754
    1
      int i, x, y;
    755
    3.25k
      for (i = 0; i < 57 * 57; i++3.24k)
    756
    3.24k
        a->data.f32[i] = i + 1;
    757
    1
      ccv_dense_matrix_t* b = 0;
    758
    1
      ccv_convnet_encode(convnet, &a, &b, 1);
    759
    1
      ccv_matrix_free(a);
    760
    1
      ccv_dense_matrix_t* c = ccv_dense_matrix_new(19, 19, CCV_32F | CCV_C1, 0, 0);
    761
    20
      for (y = 0; y < 19; y++19)
    762
    380
        for (x = 0; 19x < 19; x++361)
    763
    361
          c->data.f32[y * 19 + x] = 59 + y * 171 + x * 3;
    764
    1
      REQUIRE_MATRIX_EQ(b, c, "average pool network output should be exactly the same");
    765
    1
      ccv_matrix_free(b);
    766
    1
      ccv_matrix_free(c);
    767
    1
      ccv_convnet_free(convnet);
    768
    1
    }
    769
    770
    TEST_CASE("average pool network of 54x54 with window of 2x2 and stride of 2")
    771
    1
    {
    772
    1
      ccv_convnet_layer_param_t params = {
    773
    1
        .type = CCV_CONVNET_AVERAGE_POOL,
    774
    1
        .bias = 0,
    775
    1
        .glorot = sqrtf(2),
    776
    1
        .input = {
    777
    1
          .matrix = {
    778
    1
            .rows = 54,
    779
    1
            .cols = 54,
    780
    1
            .channels = 1,
    781
    1
            .partition = 1,
    782
    1
          },
    783
    1
        },
    784
    1
        .output = {
    785
    1
          .pool = {
    786
    1
            .size = 2,
    787
    1
            .strides = 2,
    788
    1
            .border = 0,
    789
    1
          },
    790
    1
        },
    791
    1
      };
    792
    1
      ccv_convnet_t* convnet = ccv_convnet_new(0, ccv_size(54, 54), &params, 1);
    793
    1
      ccv_dense_matrix_t* a = ccv_dense_matrix_new(54, 54, CCV_32F | CCV_C1, 0, 0);
    794
    1
      int i, x, y;
    795
    2.91k
      for (i = 0; i < 54 * 54; i++2.91k)
    796
    2.91k
        a->data.f32[i] = i + 1;
    797
    1
      ccv_dense_matrix_t* b = 0;
    798
    1
      ccv_convnet_encode(convnet, &a, &b, 1);
    799
    1
      ccv_matrix_free(a);
    800
    1
      ccv_dense_matrix_t* c = ccv_dense_matrix_new(27, 27, CCV_32F | CCV_C1, 0, 0);
    801
    28
      for (y = 0; y < 27; y++27)
    802
    756
        for (x = 0; 27x < 27; x++729)
    803
    729
          c->data.f32[y * 27 + x] = 28.5 + y * 108 + x * 2;
    804
    1
      REQUIRE_MATRIX_EQ(b, c, "average pool network output should be exactly the same");
    805
    1
      ccv_matrix_free(b);
    806
    1
      ccv_matrix_free(c);
    807
    1
      ccv_convnet_free(convnet);
    808
    1
    }
    809
    810
    TEST_CASE("local response normalization with partitioned by 2")
    811
    1
    {
    812
    1
      ccv_convnet_layer_param_t params = {
    813
    1
        .type = CCV_CONVNET_LOCAL_RESPONSE_NORM,
    814
    1
        .input = {
    815
    1
          .matrix = {
    816
    1
            .rows = 27,
    817
    1
            .cols = 27,
    818
    1
            .channels = 10,
    819
    1
            .partition = 2,
    820
    1
          },
    821
    1
        },
    822
    1
        .output = {
    823
    1
          .rnorm = {
    824
    1
            .size = 3,
    825
    1
            .kappa = 2,
    826
    1
            .alpha = 1e-4,
    827
    1
            .beta = 0.75,
    828
    1
          },
    829
    1
        },
    830
    1
      };
    831
    1
      ccv_convnet_t* convnet = ccv_convnet_new(0, ccv_size(27, 27), &params, 1);
    832
    1
      int i, k;
    833
    1
      ccv_dense_matrix_t* a = ccv_dense_matrix_new(27, 27, CCV_32F | 10, 0, 0);
    834
    7.29k
      for (i = 0; i < 27 * 27 * 10; i++7.29k)
    835
    7.29k
        a->data.f32[i] = i;
    836
    1
      ccv_dense_matrix_t* b = 0;
    837
    1
      ccv_convnet_encode(convnet, &a, &b, 1);
    838
    1
      ccv_convnet_layer_param_t partitioned_params = {
    839
    1
        .type = CCV_CONVNET_LOCAL_RESPONSE_NORM,
    840
    1
        .input = {
    841
    1
          .matrix = {
    842
    1
            .rows = 27,
    843
    1
            .cols = 27,
    844
    1
            .channels = 5,
    845
    1
            .partition = 1,
    846
    1
          },
    847
    1
        },
    848
    1
        .output = {
    849
    1
          .rnorm = {
    850
    1
            .size = 3,
    851
    1
            .kappa = 2,
    852
    1
            .alpha = 1e-4,
    853
    1
            .beta = 0.75,
    854
    1
          },
    855
    1
        },
    856
    1
      };
    857
    1
      ccv_convnet_t* partitioned_convnet = ccv_convnet_new(0, ccv_size(27, 27), &partitioned_params, 1);
    858
    1
      ccv_dense_matrix_t* aa = ccv_dense_matrix_new(27, 27, CCV_32F | 5, 0, 0);
    859
    730
      for (i = 0; i < 27 * 27; i++729)
    860
    4.37k
        for (k = 0; 729k < 5; k++3.64k)
    861
    3.64k
          aa->data.f32[i * 5 + k] = a->data.f32[i * 10 + k];
    862
    1
      ccv_dense_matrix_t* bb = ccv_dense_matrix_new(27, 27, CCV_32F | 10, 0, 0);
    863
    1
      ccv_dense_matrix_t* cc = 0;
    864
    1
      ccv_convnet_encode(partitioned_convnet, &aa, &cc, 1);
    865
    730
      for (i = 0; i < 27 * 27; i++729)
    866
    4.37k
        for (k = 0; 729k < 5; k++3.64k)
    867
    3.64k
          bb->data.f32[i * 10 + k] = cc->data.f32[i * 5 + k];
    868
    730
      for (i = 0; i < 27 * 27; i++729)
    869
    4.37k
        for (k = 0; 729k < 5; k++3.64k)
    870
    3.64k
          aa->data.f32[i * 5 + k] = a->data.f32[i * 10 + 5 + k];
    871
    1
      ccv_convnet_encode(partitioned_convnet, &aa, &cc, 1);
    872
    730
      for (i = 0; i < 27 * 27; i++729)
    873
    4.37k
        for (k = 0; 729k < 5; k++3.64k)
    874
    3.64k
          bb->data.f32[i * 10 + 5 + k] = cc->data.f32[i * 5 + k];
    875
    1
      REQUIRE_MATRIX_EQ(b, bb, "27x27x10 matrix computed from convnet with partition and partitioned convnet should be exactly the same");
    876
    1
      ccv_matrix_free(a);
    877
    1
      ccv_matrix_free(b);
    878
    1
      ccv_matrix_free(aa);
    879
    1
      ccv_matrix_free(bb);
    880
    1
      ccv_matrix_free(cc);
    881
    1
      ccv_convnet_free(convnet);
    882
    1
      ccv_convnet_free(partitioned_convnet);
    883
    1
    }
    884
    885
    // we probably won't cover all static functions in this test, disable annoying warnings
    886
    #pragma GCC diagnostic ignored "-Wunused-function"
    887
    // so that we can test static functions, note that CASE_TESTS is defined in case.h, which will disable all extern functions
    888
    #include "ccv_convnet.c"
    889
    890
    #ifdef HAVE_GSL
    891
    TEST_CASE("full connect network backward propagate")
    892
    1
    {
    893
    1
      ccv_convnet_layer_param_t params = {
    894
    1
        .type = CCV_CONVNET_FULL_CONNECT,
    895
    1
        .bias = 0,
    896
    1
        .glorot = sqrtf(2),
    897
    1
        .input = {
    898
    1
          .matrix = {
    899
    1
            .rows = 3,
    900
    1
            .cols = 3,
    901
    1
            .channels = 64,
    902
    1
            .partition = 1,
    903
    1
          },
    904
    1
          .node = {
    905
    1
            .count = 3 * 3 * 64,
    906
    1
          },
    907
    1
        },
    908
    1
        .output = {
    909
    1
          .full_connect = {
    910
    1
            .relu = 0,
    911
    1
            .count = 10,
    912
    1
          },
    913
    1
        },
    914
    1
      };
    915
    1
      ccv_convnet_t *convnet = ccv_convnet_new(0, ccv_size(3, 3), &params, 1);
    916
    1
      int i, j;
    917
    5.76k
      for (i = 0; i < 3 * 3 * 64 * 10; i++5.76k)
    918
    5.76k
        convnet->layers[0].w[i] = 2;
    919
    1
      ccv_convnet_t* update_params = _ccv_convnet_update_new(convnet);
    920
    1
      _ccv_convnet_update_zero(update_params);
    921
    1
      ccv_dense_matrix_t* x = ccv_dense_matrix_new(3, 3, CCV_32F | 64, 0, 0);
    922
    577
      for (i = 0; i < 3 * 3 * 64; i++576)
    923
    576
        x->data.f32[i] = 1;
    924
    1
      ccv_dense_matrix_t* y = 0;
    925
    1
      ccv_convnet_encode(convnet, &x, &y, 1);
    926
    1
      REQUIRE(y->rows == 10 && y->cols == 1 && CCV_GET_CHANNEL(y->type) == 1, "y should be a 10-dimensional vector");
    927
    1
      ccv_dense_matrix_t* loss = ccv_dense_matrix_new(10, 1, CCV_32F | CCV_C1, 0, 0);
    928
    1
      loss->data.f32[0] = 18;
    929
    10
      for (i = 1; i < 10; i++9)
    930
    9
        loss->data.f32[i] = -1;
    931
    1
      ccv_dense_matrix_t* b = 0;
    932
    1
      _ccv_convnet_full_connect_backward_propagate(convnet->layers, loss, y, x, &b, update_params->layers);
    933
    1
      ccv_matrix_free(y);
    934
    1
      ccv_matrix_free(x);
    935
    1
      ccv_matrix_free(loss);
    936
    1
      ccv_dense_matrix_t* db = ccv_dense_matrix_new(3, 3, CCV_32F | 64, 0, 0);
    937
    577
      for (i = 0; i < 3 * 3 * 64; i++576)
    938
    576
        db->data.f32[i] = 18;
    939
    1
      REQUIRE_MATRIX_EQ(b, db, "propagated error doesn't match the expected value");
    940
    1
      ccv_matrix_free(db);
    941
    1
      ccv_matrix_free(b);
    942
    1
      float* dw = (float*)ccmalloc(sizeof(float) * 10 * 3 * 3 * 64);
    943
    577
      for (j = 0; j < 3 * 3 * 64; j++576)
    944
    576
        dw[j] = 18;
    945
    10
      for (i = 1; i < 10; i++9)
    946
    5.19k
        for (j = 0; 9j < 3 * 3 * 64; j++5.18k)
    947
    5.18k
          dw[i * 3 * 3 * 64 + j] = -1;
    948
    1
      REQUIRE_ARRAY_EQ_WITH_TOLERANCE(float, dw, update_params->layers[0].w, 10 * 3 * 3 * 64, 1e-4, "weight gradient doesn't match the expected value");
    949
    1
      ccfree(dw);
    950
    1
      float* dbias = (float*)ccmalloc(sizeof(float) * 10);
    951
    1
      dbias[0] = 18;
    952
    10
      for (i = 1; i < 10; i++9)
    953
    9
        dbias[i] = -1;
    954
    1
      REQUIRE_ARRAY_EQ_WITH_TOLERANCE(float, dbias, update_params->layers[0].bias, 10, 1e-4, "bias gradient doesn't match the expected value");
    955
    1
      ccfree(dbias);
    956
    1
      ccv_convnet_free(update_params);
    957
    1
      ccv_convnet_free(convnet);
    958
    1
    }
    959
    960
    TEST_CASE("convolutional network backward propagate")
    961
    1
    {
    962
    1
      ccv_convnet_layer_param_t params = {
    963
    1
        .type = CCV_CONVNET_CONVOLUTIONAL,
    964
    1
        .bias = 0,
    965
    1
        .glorot = sqrtf(2),
    966
    1
        .input = {
    967
    1
          .matrix = {
    968
    1
            .rows = 31,
    969
    1
            .cols = 31,
    970
    1
            .channels = 3,
    971
    1
            .partition = 1,
    972
    1
          },
    973
    1
        },
    974
    1
        .output = {
    975
    1
          .convolutional = {
    976
    1
            .rows = 5,
    977
    1
            .cols = 5,
    978
    1
            .channels = 3,
    979
    1
            .border = 2,
    980
    1
            .strides = 1,
    981
    1
            .count = 32,
    982
    1
            .partition = 1,
    983
    1
          },
    984
    1
        },
    985
    1
      };
    986
    1
      ccv_convnet_t* convnet = ccv_convnet_new(0, ccv_size(31, 31), &params, 1);
    987
    1
      int i, j, k;
    988
    2.40k
      for (i = 0; i < 5 * 5 * 3 * 32; i++2.40k)
    989
    2.40k
        convnet->layers[0].w[i] = 2;
    990
    1
      ccv_convnet_t* update_params = _ccv_convnet_update_new(convnet);
    991
    1
      _ccv_convnet_update_zero(update_params);
    992
    1
      ccv_dense_matrix_t* x = ccv_dense_matrix_new(31, 31, CCV_32F | CCV_C3, 0, 0);
    993
    2.88k
      for (i = 0; i < 31 * 31 * 3; i++2.88k)
    994
    2.88k
        x->data.f32[i] = 1;
    995
    1
      ccv_dense_matrix_t* y = 0;
    996
    1
      ccv_convnet_encode(convnet, &x, &y, 1);
    997
    1
      REQUIRE(y->rows == 31 && y->cols == 31 && CCV_GET_CHANNEL(y->type) == 32, "convnet should return a 31x31x32 matrix");
    998
    1
      ccv_dense_matrix_t* loss = ccv_dense_matrix_new(y->rows, y->cols, CCV_32F | CCV_GET_CHANNEL(y->type), 0, 0);
    999
    30.7k
      for (i = 0; i < 31 * 31 * 32; i++30.7k)
    1000
    30.7k
        loss->data.f32[i] = 1;
    1001
    1
      ccv_dense_matrix_t* d = 0;
    1002
    1
      _ccv_convnet_convolutional_backward_propagate(convnet->layers, loss, y, x, &d, update_params->layers);
    1003
    1
      ccv_matrix_free(loss);
    1004
    1
      ccv_matrix_free(y);
    1005
    1
      ccv_matrix_free(x);
    1006
    1
      ccv_dense_matrix_t* dd = ccv_dense_matrix_new(31, 31, CCV_32F | CCV_C3, 0, 0);
    1007
    32
      for (i = 0; i < 31; i++31)
    1008
    992
        for (j = 0; 31j < 31; j++961)
    1009
    961
          dd->data.f32[(i * 31 + j) * 3] =
    1010
    961
          dd->data.f32[(i * 31 + j) * 3 + 1] =
    1011
    961
          dd->data.f32[(i * 31 + j) * 3 + 2] = 32 * 2 * (5 + ccv_min(i - 2, 0) + ccv_min(28 - i, 0)) * (5 + ccv_min(j - 2, 0) + ccv_min(28 - j, 0));
    1012
    1
      REQUIRE_MATRIX_EQ(d, dd, "propagated error doesn't match the expected value");
    1013
    1
      ccv_matrix_free(d);
    1014
    1
      ccv_matrix_free(dd);
    1015
    1
      float* dw = (float*)ccmalloc(sizeof(float) * 5 * 5 * 3 * 32);
    1016
    33
      for (k = 0; k < 32; k++32)
    1017
    192
        for (i = 0; 32i < 5; i++160)
    1018
    960
          for (j = 0; 160j < 5; j++800)
    1019
    800
            dw[k * 5 * 5 * 3 + (i * 5 + j) * 3] =
    1020
    800
            dw[k * 5 * 5 * 3 + (i * 5 + j) * 3 + 1] =
    1021
    800
            dw[k * 5 * 5 * 3 + (i * 5 + j) * 3 + 2] = (31 - abs(i - 2)) * (31 - abs(j - 2));
    1022
    1
      REQUIRE_ARRAY_EQ_WITH_TOLERANCE(float, dw, update_params->layers[0].w, 5 * 5 * 3 * 32, 1e-4, "weight gradient doesn't match the expected value");
    1023
    1
      ccfree(dw);
    1024
    1
      float* dbias = (float*)ccmalloc(sizeof(float) * 32);
    1025
    33
      for (i = 0; i < 32; i++32)
    1026
    32
        dbias[i] = 31 * 31;
    1027
    1
      REQUIRE_ARRAY_EQ_WITH_TOLERANCE(float, dbias, update_params->layers[0].bias, 32, 1e-4, "bias gradient doesn't match the expected value");
    1028
    1
      ccfree(dbias);
    1029
    1
      ccv_convnet_free(update_params);
    1030
    1
      ccv_convnet_free(convnet);
    1031
    1
    }
    1032
    1033
    TEST_CASE("convolutional network backward propagate with partitioned by 2")
    1034
    1
    {
    1035
    1
      ccv_convnet_layer_param_t params = {
    1036
    1
        .type = CCV_CONVNET_CONVOLUTIONAL,
    1037
    1
        .bias = 0,
    1038
    1
        .glorot = sqrtf(2),
    1039
    1
        .input = {
    1040
    1
          .matrix = {
    1041
    1
            .rows = 31,
    1042
    1
            .cols = 31,
    1043
    1
            .channels = 4,
    1044
    1
            .partition = 2,
    1045
    1
          },
    1046
    1
        },
    1047
    1
        .output = {
    1048
    1
          .convolutional = {
    1049
    1
            .rows = 5,
    1050
    1
            .cols = 5,
    1051
    1
            .channels = 4,
    1052
    1
            .border = 2,
    1053
    1
            .strides = 1,
    1054
    1
            .count = 8,
    1055
    1
            .partition = 2,
    1056
    1
          },
    1057
    1
        },
    1058
    1
      };
    1059
    1
      ccv_convnet_t* convnet = ccv_convnet_new(0, ccv_size(31, 31), &params, 1);
    1060
    1
      int i, k;
    1061
    401
      for (i = 0; i < convnet->layers->wnum; i++400)
    1062
    400
        convnet->layers->w[i] = i * 1e-2;
    1063
    9
      for (i = 0; i < convnet->layers->net.convolutional.count; i++8)
    1064
    8
        convnet->layers->bias[i] = i;
    1065
    1
      ccv_dense_matrix_t* a = ccv_dense_matrix_new(31, 31, CCV_32F | 4, 0, 0);
    1066
    3.84k
      for (i = 0; i < 31 * 31 * 4; i++3.84k)
    1067
    3.84k
        a->data.f32[i] = 2000 - i;
    1068
    1
      ccv_dense_matrix_t* b = 0;
    1069
    1
      ccv_convnet_encode(convnet, &a, &b, 1);
    1070
    1
      ccv_dense_matrix_t* loss = ccv_dense_matrix_new(b->rows, b->cols, CCV_32F | CCV_GET_CHANNEL(b->type), 0, 0);
    1071
    7.68k
      for (i = 0; i < 31 * 31 * 8; i++7.68k)
    1072
    7.68k
        loss->data.f32[i] = 1;
    1073
    1
      ccv_dense_matrix_t* d = 0;
    1074
    1
      ccv_convnet_t* update_params = _ccv_convnet_update_new(convnet);
    1075
    1
      _ccv_convnet_update_zero(update_params);
    1076
    1
      _ccv_convnet_convolutional_backward_propagate(convnet->layers, loss, b, a, &d, update_params->layers);
    1077
    1
      ccv_matrix_free(loss);
    1078
    1
      ccv_convnet_layer_param_t partitioned_params = {
    1079
    1
        .type = CCV_CONVNET_CONVOLUTIONAL,
    1080
    1
        .bias = 0,
    1081
    1
        .glorot = sqrtf(2),
    1082
    1
        .input = {
    1083
    1
          .matrix = {
    1084
    1
            .rows = 31,
    1085
    1
            .cols = 31,
    1086
    1
            .channels = 2,
    1087
    1
            .partition = 1,
    1088
    1
          },
    1089
    1
        },
    1090
    1
        .output = {
    1091
    1
          .convolutional = {
    1092
    1
            .rows = 5,
    1093
    1
            .cols = 5,
    1094
    1
            .channels = 2,
    1095
    1
            .border = 2,
    1096
    1
            .strides = 1,
    1097
    1
            .count = 4,
    1098
    1
            .partition = 1,
    1099
    1
          },
    1100
    1
        },
    1101
    1
      };
    1102
    1
      ccv_convnet_t* partitioned_convnet = ccv_convnet_new(0, ccv_size(31, 31), &partitioned_params, 1);
    1103
    1
      ccv_dense_matrix_t* aa = ccv_dense_matrix_new(31, 31, CCV_32F | 2, 0, 0);
    1104
      // first partition
    1105
    962
      for (i = 0; i < 31 * 31; i++961)
    1106
    2.88k
        for (k = 0; 961k < 2; k++1.92k)
    1107
    1.92k
          aa->data.f32[i * 2 + k] = a->data.f32[i * 4 + k];
    1108
    1
      memcpy(partitioned_convnet->layers->w, convnet->layers->w, sizeof(float) * (convnet->layers->wnum / 2));
    1109
    1
      memcpy(partitioned_convnet->layers->bias, convnet->layers->bias, sizeof(float) * (convnet->layers->net.convolutional.count / 2));
    1110
    1
      ccv_dense_matrix_t* bb = 0;
    1111
    1
      ccv_convnet_encode(partitioned_convnet, &aa, &bb, 1);
    1112
    1
      ccv_dense_matrix_t* bbb = ccv_dense_matrix_new(31, 31, CCV_32F | 8, 0, 0);
    1113
    962
      for (i = 0; i < 31 * 31; i++961)
    1114
    4.80k
        for (k = 0; 961k < 4; k++3.84k)
    1115
    3.84k
          bbb->data.f32[i * 8 + k] = bb->data.f32[i * 4 + k];
    1116
    1
      loss = ccv_dense_matrix_new(31, 31, CCV_32F | CCV_GET_CHANNEL(bb->type), 0, 0);
    1117
    3.84k
      for (i = 0; i < 31 * 31 * 4; i++3.84k)
    1118
    3.84k
        loss->data.f32[i] = 1;
    1119
    1
      ccv_dense_matrix_t* dd = 0;
    1120
    1
      ccv_convnet_t* partitioned_update_params = _ccv_convnet_update_new(convnet);
    1121
    1
      _ccv_convnet_update_zero(partitioned_update_params);
    1122
    1
      _ccv_convnet_convolutional_backward_propagate(partitioned_convnet->layers, loss, bb, aa, &dd, partitioned_update_params->layers);
    1123
    1
      ccv_dense_matrix_t* ddd = ccv_dense_matrix_new(31, 31, CCV_32F | 4, 0, 0);
    1124
    1
      float* ww = (float*)ccmalloc(sizeof(float) * (convnet->layers->wnum + convnet->layers->net.convolutional.count));
    1125
    1
      float* bbias = ww + convnet->layers->wnum;
    1126
    1
      memcpy(ww, partitioned_update_params->layers->w, sizeof(float) * (convnet->layers->wnum / 2));
    1127
    1
      memcpy(bbias, partitioned_update_params->layers->bias, sizeof(float) * (convnet->layers->net.convolutional.count / 2));
    1128
    962
      for (i = 0; i < 31 * 31; i++961)
    1129
    2.88k
        for (k = 0; 961k < 2; k++1.92k)
    1130
    1.92k
          ddd->data.f32[i * 4 + k] = dd->data.f32[i * 2 + k];
    1131
      // second partition
    1132
    962
      for (i = 0; i < 31 * 31; i++961)
    1133
    2.88k
        for (k = 0; 961k < 2; k++1.92k)
    1134
    1.92k
          aa->data.f32[i * 2 + k] = a->data.f32[i * 4 + 2 + k];
    1135
    1
      memcpy(partitioned_convnet->layers->w, convnet->layers->w + (convnet->layers->wnum / 2), sizeof(float) * (convnet->layers->wnum / 2));
    1136
    1
      memcpy(partitioned_convnet->layers->bias, convnet->layers->bias + (convnet->layers->net.convolutional.count / 2), sizeof(float) * (convnet->layers->net.convolutional.count / 2));
    1137
    1
      ccv_convnet_compact(partitioned_convnet); // because it is reused, we need to clear intermediate data
    1138
    1
      ccv_convnet_encode(partitioned_convnet, &aa, &bb, 1);
    1139
    962
      for (i = 0; i < 31 * 31; i++961)
    1140
    4.80k
        for (k = 0; 961k < 4; k++3.84k)
    1141
    3.84k
          bbb->data.f32[i * 8 + 4 + k] = bb->data.f32[i * 4 + k];
    1142
    1
      REQUIRE_MATRIX_EQ(b, bbb, "forward pass doesn't match the expected value");
    1143
    1
      _ccv_convnet_update_zero(partitioned_update_params);
    1144
    1
      _ccv_convnet_convolutional_backward_propagate(partitioned_convnet->layers, loss, bb, aa, &dd, partitioned_update_params->layers);
    1145
    1
      memcpy(ww + (convnet->layers->wnum / 2), partitioned_update_params->layers->w, sizeof(float) * (convnet->layers->wnum / 2));
    1146
    1
      memcpy(bbias + (convnet->layers->net.convolutional.count / 2), partitioned_update_params->layers->bias, sizeof(float) * (convnet->layers->net.convolutional.count / 2));
    1147
    962
      for (i = 0; i < 31 * 31; i++961)
    1148
    2.88k
        for (k = 0; 961k < 2; k++1.92k)
    1149
    1.92k
          ddd->data.f32[i * 4 + 2 + k] = dd->data.f32[i * 2 + k];
    1150
    1
      REQUIRE_MATRIX_EQ(d, ddd, "propagated error doesn't match the expected value");
    1151
    1
      REQUIRE_ARRAY_EQ_WITH_TOLERANCE(float, ww, update_params->layers[0].w, convnet->layers->wnum, 1e-4, "weight gradient doesn't match the expected value");
    1152
    1
      REQUIRE_ARRAY_EQ_WITH_TOLERANCE(float, bbias, update_params->layers[0].bias, convnet->layers->net.convolutional.count, 1e-4, "bias gradient doesn't match the expected value");
    1153
    1
      ccfree(ww);
    1154
    1
      ccv_matrix_free(loss);
    1155
    1
      ccv_matrix_free(ddd);
    1156
    1
      ccv_matrix_free(dd);
    1157
    1
      ccv_matrix_free(bbb);
    1158
    1
      ccv_matrix_free(bb);
    1159
    1
      ccv_matrix_free(aa);
    1160
    1
      ccv_matrix_free(d);
    1161
    1
      ccv_matrix_free(b);
    1162
    1
      ccv_matrix_free(a);
    1163
    1
      ccv_convnet_free(convnet);
    1164
    1
      ccv_convnet_free(update_params);
    1165
    1
      ccv_convnet_free(partitioned_convnet);
    1166
    1
      ccv_convnet_free(partitioned_update_params);
    1167
    1
    }
    1168
    1169
    TEST_CASE("local response normalization backward propagate with partitioned by 2")
    1170
    1
    {
    1171
    1
      ccv_convnet_layer_param_t params = {
    1172
    1
        .type = CCV_CONVNET_LOCAL_RESPONSE_NORM,
    1173
    1
        .input = {
    1174
    1
          .matrix = {
    1175
    1
            .rows = 27,
    1176
    1
            .cols = 27,
    1177
    1
            .channels = 6,
    1178
    1
            .partition = 2,
    1179
    1
          },
    1180
    1
        },
    1181
    1
        .output = {
    1182
    1
          .rnorm = {
    1183
    1
            .size = 3,
    1184
    1
            .kappa = 2,
    1185
    1
            .alpha = 1e-4,
    1186
    1
            .beta = 0.75,
    1187
    1
          },
    1188
    1
        },
    1189
    1
      };
    1190
    1
      ccv_convnet_t* convnet = ccv_convnet_new(0, ccv_size(27, 27), &params, 1);
    1191
    1
      int i, k;
    1192
    1
      ccv_dense_matrix_t* a = ccv_dense_matrix_new(27, 27, CCV_32F | 6, 0, 0);
    1193
    4.37k
      for (i = 0; i < 27 * 27 * 6; i++4.37k)
    1194
    4.37k
        a->data.f32[i] = i;
    1195
    1
      ccv_dense_matrix_t* b = 0;
    1196
    1
      ccv_convnet_encode(convnet, &a, &b, 1);
    1197
    1
      ccv_dense_matrix_t* d = 0;
    1198
    1
      ccv_dense_matrix_t* loss = ccv_dense_matrix_new(27, 27, CCV_32F | 6, 0, 0);
    1199
    4.37k
      for (i = 0; i < 27 * 27 * 6; i++4.37k)
    1200
    4.37k
        loss->data.f32[i] = 1;
    1201
    1
      _ccv_convnet_rnorm_backward_propagate(convnet->layers, loss, b, a, convnet->denoms[0], &d);
    1202
    1
      ccv_convnet_layer_param_t partitioned_params = {
    1203
    1
        .type = CCV_CONVNET_LOCAL_RESPONSE_NORM,
    1204
    1
        .input = {
    1205
    1
          .matrix = {
    1206
    1
            .rows = 27,
    1207
    1
            .cols = 27,
    1208
    1
            .channels = 3,
    1209
    1
            .partition = 1,
    1210
    1
          },
    1211
    1
        },
    1212
    1
        .output = {
    1213
    1
          .rnorm = {
    1214
    1
            .size = 3,
    1215
    1
            .kappa = 2,
    1216
    1
            .alpha = 1e-4,
    1217
    1
            .beta = 0.75,
    1218
    1
          },
    1219
    1
        },
    1220
    1
      };
    1221
    1
      ccv_convnet_t* partitioned_convnet = ccv_convnet_new(0, ccv_size(27, 27), &partitioned_params, 1);
    1222
    1
      ccv_dense_matrix_t* aa = ccv_dense_matrix_new(27, 27, CCV_32F | 3, 0, 0);
    1223
      // first partition
    1224
    730
      for (i = 0; i < 27 * 27; i++729)
    1225
    2.91k
        for (k = 0; 729k < 3; k++2.18k)
    1226
    2.18k
          aa->data.f32[i * 3 + k] = a->data.f32[i * 6 + k];
    1227
    1
      ccv_dense_matrix_t* bb = 0;
    1228
    1
      ccv_convnet_encode(partitioned_convnet, &aa, &bb, 1);
    1229
    1
      ccv_matrix_free(loss);
    1230
    1
      loss = ccv_dense_matrix_new(27, 27, CCV_32F | 3, 0, 0);
    1231
    2.18k
      for (i = 0; i < 27 * 27 * 3; i++2.18k)
    1232
    2.18k
        loss->data.f32[i] = 1;
    1233
    1
      ccv_dense_matrix_t* dd = 0;
    1234
    1
      _ccv_convnet_rnorm_backward_propagate(partitioned_convnet->layers, loss, bb, aa, partitioned_convnet->denoms[0], &dd);
    1235
    1
      ccv_dense_matrix_t* ddd = ccv_dense_matrix_new(27, 27, CCV_32F | 6, 0, 0);
    1236
    730
      for (i = 0; i < 27 * 27; i++729)
    1237
    2.91k
        for (k = 0; 729k < 3; k++2.18k)
    1238
    2.18k
          ddd->data.f32[i * 6 + k] = dd->data.f32[i * 3 + k];
    1239
      // second partition
    1240
    730
      for (i = 0; i < 27 * 27; i++729)
    1241
    2.91k
        for (k = 0; 729k < 3; k++2.18k)
    1242
    2.18k
          aa->data.f32[i * 3 + k] = a->data.f32[i * 6 + 3 + k];
    1243
    1
      ccv_convnet_encode(partitioned_convnet, &aa, &bb, 1);
    1244
    1
      _ccv_convnet_rnorm_backward_propagate(partitioned_convnet->layers, loss, bb, aa, partitioned_convnet->denoms[0], &dd);
    1245
    730
      for (i = 0; i < 27 * 27; i++729)
    1246
    2.91k
        for (k = 0; 729k < 3; k++2.18k)
    1247
    2.18k
          ddd->data.f32[i * 6 + 3 + k] = dd->data.f32[i * 3 + k];
    1248
    1
      REQUIRE_MATRIX_EQ(d, ddd, "27x27x6 error local response normalization backward propagated from convnet with partition and partitioned convnet should be exactly the same");
    1249
    1
      ccv_matrix_free(a);
    1250
    1
      ccv_matrix_free(b);
    1251
    1
      ccv_matrix_free(d);
    1252
    1
      ccv_matrix_free(aa);
    1253
    1
      ccv_matrix_free(bb);
    1254
    1
      ccv_matrix_free(dd);
    1255
    1
      ccv_matrix_free(ddd);
    1256
    1
      ccv_matrix_free(loss);
    1257
    1
      ccv_convnet_free(convnet);
    1258
    1
      ccv_convnet_free(partitioned_convnet);
    1259
    1
    }
    1260
    1261
    // five-stencil constants
    1262
    static float fs[4] = { 1, -8, 8, -1 };
    1263
    static float fsh[4] = { -2, -1, 1, 2 };
    1264
    1265
    static float dsfmt_genrand_gaussian(dsfmt_t* dsfmt, float sigma)
    1266
    2.29k
    {
    1267
    2.29k
      double rand1 = dsfmt_genrand_open_close(dsfmt);
    1268
    2.29k
      rand1 = -2 * log(rand1);
    1269
    2.29k
      double rand2 = dsfmt_genrand_open_close(dsfmt) * CCV_PI * 2;
    1270
    2.29k
      return (float)(sqrt(sigma * rand1) * cos(rand2));
    1271
    2.29k
    }
    1272
    1273
    TEST_CASE("numerical gradient versus analytical gradient for full connect network")
    1274
    1
    {
    1275
    1
      ccv_convnet_layer_param_t params = {
    1276
    1
        .type = CCV_CONVNET_FULL_CONNECT,
    1277
    1
        .bias = 0,
    1278
    1
        .glorot = sqrtf(2),
    1279
    1
        .input = {
    1280
    1
          .matrix = {
    1281
    1
            .rows = 3,
    1282
    1
            .cols = 3,
    1283
    1
            .channels = 8,
    1284
    1
            .partition = 1,
    1285
    1
          },
    1286
    1
          .node = {
    1287
    1
            .count = 3 * 3 * 8,
    1288
    1
          },
    1289
    1
        },
    1290
    1
        .output = {
    1291
    1
          .full_connect = {
    1292
    1
            .relu = 0,
    1293
    1
            .count = 10,
    1294
    1
          },
    1295
    1
        },
    1296
    1
      };
    1297
    1
      ccv_convnet_t *convnet = ccv_convnet_new(0, ccv_size(3, 3), &params, 1);
    1298
    1
      dsfmt_t dsfmt;
    1299
    1
      dsfmt_init_gen_rand(&dsfmt, 0);
    1300
    1
      int i, j, k;
    1301
    721
      for (i = 0; i < convnet->layers->wnum; i++720)
    1302
    720
        convnet->layers->w[i] = dsfmt_genrand_gaussian(&dsfmt, 0.01);
    1303
    1
      ccv_convnet_t* update_params = _ccv_convnet_update_new(convnet);
    1304
    1
      _ccv_convnet_update_zero(update_params);
    1305
    1
      ccv_dense_matrix_t* x = ccv_dense_matrix_new(3, 3, CCV_32F | 8, 0, 0);
    1306
    73
      for (i = 0; i < 3 * 3 * 8; i++72)
    1307
    72
        x->data.f32[i] = i;
    1308
    1
      ccv_dense_matrix_t* y = 0;
    1309
    1
      ccv_convnet_encode(convnet, &x, &y, 1);
    1310
    1
      REQUIRE(y->rows == 10 && y->cols == 1 && CCV_GET_CHANNEL(y->type) == 1, "y should be a 10-dimensional vector");
    1311
    1
      _ccv_convnet_compute_softmax(y, &y, 0);
    1312
    1
      ccv_dense_matrix_t* dloss = ccv_dense_matrix_new(10, 1, CCV_32F | CCV_C1, 0, 0);;
    1313
    11
      for (i = 0; i < 10; i++10)
    1314
    10
        dloss->data.f32[i] = y->data.f32[i] - (i == 2);
    1315
    1
      float* dw = (float*)ccmalloc(sizeof(float) * 3 * 3 * 8 * 10);
    1316
    1
      static const float eps = 0.0001;
    1317
    11
      for (i = 0; i < 10; i++10)
    1318
    730
        for (j = 0; 10j < 3 * 3 * 8; j++720)
    1319
    720
        {
    1320
    720
          dw[j + i * 3 * 3 * 8] = 0;
    1321
    3.60k
          for (k = 0; k < 4; k++2.88k)
    1322
    2.88k
          {
    1323
    2.88k
            float w = convnet->layers->w[j + i * 3 * 3 * 8];
    1324
    2.88k
            convnet->layers->w[j + i * 3 * 3 * 8] += fsh[k] * eps;
    1325
    2.88k
            ccv_dense_matrix_t* z = 0;
    1326
    2.88k
            ccv_convnet_encode(convnet, &x, &z, 1);
    1327
    2.88k
            _ccv_convnet_compute_softmax(z, &z, 0);
    1328
    2.88k
            dw[j + i * 3 * 3 * 8] += -logf(z->data.f32[2]) * fs[k];
    1329
    2.88k
            ccv_matrix_free(z);
    1330
    2.88k
            convnet->layers->w[j + i * 3 * 3 * 8] = w;
    1331
    2.88k
          }
    1332
    720
          dw[j + i * 3 * 3 * 8] *= 1.0 / (12 * eps);
    1333
    720
        }
    1334
    1
      float* dbias = (float*)ccmalloc(sizeof(float) * 10);
    1335
    11
      for (i = 0; i < 10; i++10)
    1336
    10
      {
    1337
    10
        dbias[i] = 0;
    1338
    50
        for (k = 0; k < 4; k++40)
    1339
    40
        {
    1340
    40
          float bias = convnet->layers->bias[i];
    1341
    40
          convnet->layers->bias[i] += fsh[k] * eps;
    1342
    40
          ccv_dense_matrix_t* z = 0;
    1343
    40
          ccv_convnet_encode(convnet, &x, &z, 1);
    1344
    40
          _ccv_convnet_compute_softmax(z, &z, 0);
    1345
    40
          dbias[i] += -logf(z->data.f32[2]) * fs[k];
    1346
    40
          ccv_matrix_free(z);
    1347
    40
          convnet->layers->bias[i] = bias;
    1348
    40
        }
    1349
    10
        dbias[i] *= 1.0 / (12 * eps);
    1350
    10
      }
    1351
    1
      ccv_dense_matrix_t* b = 0;
    1352
    1
      _ccv_convnet_full_connect_backward_propagate(convnet->layers, dloss, y, x, &b, update_params->layers);
    1353
    1
      ccv_matrix_free(y);
    1354
    1
      ccv_matrix_free(x);
    1355
    1
      ccv_matrix_free(dloss);
    1356
    1
      ccv_matrix_free(b);
    1357
    1
      REQUIRE_ARRAY_EQ_WITHIN_ANGLE_AND_MAGNITUDE(float, dw, update_params->layers[0].w, 3 * 3 * 8 * 10, 30, 2e-1, "weight gradient from analytical method doesn't match the one from numerical method");
    1358
    1
      ccfree(dw);
    1359
    1
      REQUIRE_ARRAY_EQ_WITHIN_ANGLE_AND_MAGNITUDE(float, dbias, update_params->layers[0].bias, 10, 30, 2e-1, "bias gradient from analytical method doesn't match the one from numerical method");
    1360
    1
      ccfree(dbias);
    1361
    1
      ccv_convnet_free(update_params);
    1362
    1
      ccv_convnet_free(convnet);
    1363
    1
    }
    1364
    1365
    TEST_CASE("numerical gradient versus analytical gradient for convolutional network")
    1366
    1
    {
    1367
    1
      ccv_convnet_layer_param_t params = {
    1368
    1
        .type = CCV_CONVNET_CONVOLUTIONAL,
    1369
    1
        .bias = 0,
    1370
    1
        .glorot = sqrtf(2),
    1371
    1
        .input = {
    1372
    1
          .matrix = {
    1373
    1
            .rows = 31,
    1374
    1
            .cols = 31,
    1375
    1
            .channels = 3,
    1376
    1
            .partition = 1,
    1377
    1
          },
    1378
    1
        },
    1379
    1
        .output = {
    1380
    1
          .convolutional = {
    1381
    1
            .rows = 5,
    1382
    1
            .cols = 5,
    1383
    1
            .channels = 3,
    1384
    1
            .border = 2,
    1385
    1
            .strides = 1,
    1386
    1
            .count = 4,
    1387
    1
            .partition = 1,
    1388
    1
          },
    1389
    1
        },
    1390
    1
      };
    1391
    1
      ccv_convnet_t* convnet = ccv_convnet_new(0, ccv_size(31, 31), &params, 1);
    1392
    1
      dsfmt_t dsfmt;
    1393
    1
      dsfmt_init_gen_rand(&dsfmt, 1);
    1394
    1
      int i, k;
    1395
    301
      for (i = 0; i < convnet->layers->wnum; i++300)
    1396
    300
        convnet->layers->w[i] = dsfmt_genrand_gaussian(&dsfmt, 0.0001);
    1397
    1
      ccv_convnet_t* update_params = _ccv_convnet_update_new(convnet);
    1398
    1
      _ccv_convnet_update_zero(update_params);
    1399
    1
      ccv_dense_matrix_t* x = ccv_dense_matrix_new(31, 31, CCV_32F | CCV_C3, 0, 0);
    1400
    2.88k
      for (i = 0; i < 31 * 31 * 3; i++2.88k)
    1401
    2.88k
        x->data.f32[i] = i;
    1402
    1
      ccv_dense_matrix_t* y = 0;
    1403
    1
      ccv_convnet_encode(convnet, &x, &y, 1);
    1404
    1
      REQUIRE(y->rows == 31 && y->cols == 31 && CCV_GET_CHANNEL(y->type) == 4, "convnet should return a 31x31x4 matrix");
    1405
    1
      ccv_dense_matrix_t* softmax = 0;
    1406
    1
      _ccv_convnet_compute_softmax(y, &softmax, 0);
    1407
    1
      ccv_dense_matrix_t* dloss = ccv_dense_matrix_new(y->rows, y->cols, CCV_32F | CCV_GET_CHANNEL(y->type), 0, 0);
    1408
    3.84k
      for (i = 0; i < 31 * 31 * 4; i++3.84k)
    1409
    3.84k
        dloss->data.f32[i] = softmax->data.f32[i] - (i == 24);
    1410
    1
      static const float eps = 0.000005;
    1411
    1
      float* dw = (float*)ccmalloc(sizeof(float) * 5 * 5 * 3 * 4);
    1412
    301
      for (i = 0; i < 5 * 5 * 3 * 4; i++300)
    1413
    300
      {
    1414
    300
        dw[i] = 0;
    1415
    1.50k
        for (k = 0; k < 4; k++1.20k)
    1416
    1.20k
        {
    1417
    1.20k
          float w = convnet->layers->w[i];
    1418
    1.20k
          convnet->layers->w[i] += fsh[k] * eps;
    1419
    1.20k
          ccv_dense_matrix_t* z = 0;
    1420
    1.20k
          ccv_convnet_compact(convnet);
    1421
    1.20k
          ccv_convnet_encode(convnet, &x, &z, 1);
    1422
    1.20k
          _ccv_convnet_compute_softmax(z, &z, 0);
    1423
    1.20k
          dw[i] += -logf(z->data.f32[24]) * fs[k];
    1424
    1.20k
          ccv_matrix_free(z);
    1425
    1.20k
          convnet->layers->w[i] = w;
    1426
    1.20k
        }
    1427
    300
        dw[i] *= 1.0 / (12 * eps);
    1428
    300
      }
    1429
    1
      float* dbias = (float*)ccmalloc(sizeof(float) * 4);
    1430
    5
      for (i = 0; i < 4; i++4)
    1431
    4
      {
    1432
    4
        dbias[i] = 0;
    1433
    20
        for (k = 0; k < 4; k++16)
    1434
    16
        {
    1435
    16
          float bias = convnet->layers->bias[i];
    1436
    16
          convnet->layers->bias[i] += fsh[k] * eps;
    1437
    16
          ccv_dense_matrix_t* z = 0;
    1438
    16
          ccv_convnet_compact(convnet);
    1439
    16
          ccv_convnet_encode(convnet, &x, &z, 1);
    1440
    16
          _ccv_convnet_compute_softmax(z, &z, 0);
    1441
    16
          dbias[i] += -logf(z->data.f32[24]) * fs[k];
    1442
    16
          ccv_matrix_free(z);
    1443
    16
          convnet->layers->bias[i] = bias;
    1444
    16
        }
    1445
    4
        dbias[i] *= 1.0 / (12 * eps);
    1446
    4
      }
    1447
    1
      ccv_dense_matrix_t* d = 0;
    1448
    1
      _ccv_convnet_convolutional_backward_propagate(convnet->layers, dloss, y, x, &d, update_params->layers);
    1449
    1
      ccv_matrix_free(softmax);
    1450
    1
      ccv_matrix_free(dloss);
    1451
    1
      ccv_matrix_free(y);
    1452
    1
      ccv_matrix_free(x);
    1453
    1
      ccv_matrix_free(d);
    1454
    1
      REQUIRE_ARRAY_EQ_WITHIN_ANGLE_AND_MAGNITUDE(float, dw, update_params->layers[0].w, 5 * 5 * 3 * 4, 30, 2e-1, "weight gradient from analytical method doesn't match the one from numerical method");
    1455
    1
      ccfree(dw);
    1456
    1
      REQUIRE_ARRAY_EQ_WITHIN_ANGLE_AND_MAGNITUDE(float, dbias, update_params->layers[0].bias, 4, 30, 2e-1, "bias gradient from analytical method doesn't match the one from numerical method");
    1457
    1
      ccfree(dbias);
    1458
    1
      ccv_convnet_free(update_params);
    1459
    1
      ccv_convnet_free(convnet);
    1460
    1
    }
    1461
    1462
    TEST_CASE("numerical gradient versus analytical gradient for full connect network over convolutional network")
    1463
    1
    {
    1464
    1
      ccv_convnet_layer_param_t params[] = {
    1465
    1
        {
    1466
    1
          .type = CCV_CONVNET_CONVOLUTIONAL,
    1467
    1
          .bias = 0,
    1468
    1
          .glorot = sqrtf(2),
    1469
    1
          .input = {
    1470
    1
            .matrix = {
    1471
    1
              .rows = 5,
    1472
    1
              .cols = 5,
    1473
    1
              .channels = 2,
    1474
    1
              .partition = 1,
    1475
    1
            },
    1476
    1
          },
    1477
    1
          .output = {
    1478
    1
            .convolutional = {
    1479
    1
              .rows = 3,
    1480
    1
              .cols = 3,
    1481
    1
              .channels = 2,
    1482
    1
              .border = 1,
    1483
    1
              .strides = 1,
    1484
    1
              .count = 4,
    1485
    1
              .partition = 1,
    1486
    1
            },
    1487
    1
          },
    1488
    1
        },
    1489
    1
        {
    1490
    1
          .type = CCV_CONVNET_FULL_CONNECT,
    1491
    1
          .bias = 0,
    1492
    1
          .glorot = sqrtf(2),
    1493
    1
          .input = {
    1494
    1
            .matrix = {
    1495
    1
              .rows = 5,
    1496
    1
              .cols = 5,
    1497
    1
              .channels = 4,
    1498
    1
              .partition = 1,
    1499
    1
            },
    1500
    1
            .node = {
    1501
    1
              .count = 5 * 5 * 4,
    1502
    1
            },
    1503
    1
          },
    1504
    1
          .output = {
    1505
    1
            .full_connect = {
    1506
    1
              .relu = 0,
    1507
    1
              .count = 10,
    1508
    1
            },
    1509
    1
          },
    1510
    1
        },
    1511
    1
      };
    1512
    1
      ccv_convnet_t* convnet = ccv_convnet_new(0, ccv_size(5, 5), params, 2);
    1513
    1
      dsfmt_t dsfmt;
    1514
    1
      dsfmt_init_gen_rand(&dsfmt, 2);
    1515
    1
      int i, k;
    1516
    73
      for (i = 0; i < convnet->layers[0].wnum; i++72)
    1517
    72
        convnet->layers[0].w[i] = dsfmt_genrand_gaussian(&dsfmt, 0.001);
    1518
    1.00k
      for (i = 0; i < convnet->layers[1].wnum; i++1.00k)
    1519
    1.00k
        convnet->layers[1].w[i] = dsfmt_genrand_gaussian(&dsfmt, 0.01);
    1520
    1
      ccv_convnet_t* update_params = _ccv_convnet_update_new(convnet);
    1521
    1
      _ccv_convnet_update_zero(update_params);
    1522
    1
      ccv_dense_matrix_t* x = ccv_dense_matrix_new(5, 5, CCV_32F | CCV_C2, 0, 0);
    1523
    51
      for (i = 0; i < 5 * 5 * 2; i++50)
    1524
    50
        x->data.f32[i] = 0.2;
    1525
    1
      ccv_dense_matrix_t* y = 0;
    1526
    1
      ccv_convnet_encode(convnet, &x, &y, 1);
    1527
    1
      REQUIRE(y->rows == 10 && y->cols == 1 && CCV_GET_CHANNEL(y->type) == 1, "y should be a 10-dimensional vector");
    1528
    1
      _ccv_convnet_compute_softmax(y, &y, 0);
    1529
    1
      ccv_dense_matrix_t* dloss = ccv_dense_matrix_new(10, 1, CCV_32F | CCV_C1, 0, 0);;
    1530
    11
      for (i = 0; i < 10; i++10)
    1531
    10
        dloss->data.f32[i] = y->data.f32[i] - (i == 2);
    1532
    1
      _ccv_convnet_propagate_loss(convnet, x, dloss, update_params);
    1533
    1
      ccv_matrix_free(dloss);
    1534
    1
      static const float eps = 0.0001;
    1535
    1
      float* dw = (float*)ccmalloc(sizeof(float) * 3 * 3 * 2 * 4);
    1536
    73
      for (i = 0; i < 3 * 3 * 2 * 4; i++72)
    1537
    72
      {
    1538
    72
        dw[i] = 0;
    1539
    360
        for (k = 0; k < 4; k++288)
    1540
    288
        {
    1541
    288
          float w = convnet->layers->w[i];
    1542
    288
          convnet->layers->w[i] += fsh[k] * eps;
    1543
    288
          ccv_dense_matrix_t* z = 0;
    1544
    288
          ccv_convnet_compact(convnet);
    1545
    288
          ccv_convnet_encode(convnet, &x, &z, 1);
    1546
    288
          _ccv_convnet_compute_softmax(z, &z, 0);
    1547
    288
          dw[i] += -logf(z->data.f32[2]) * fs[k];
    1548
    288
          ccv_matrix_free(z);
    1549
    288
          convnet->layers->w[i] = w;
    1550
    288
        }
    1551
    72
        dw[i] *= 1.0 / (12 * eps);
    1552
    72
      }
    1553
    1
      float* dbias = (float*)ccmalloc(sizeof(float) * 4);
    1554
    5
      for (i = 0; i < 4; i++4)
    1555
    4
      {
    1556
    4
        dbias[i] = 0;
    1557
    20
        for (k = 0; k < 4; k++16)
    1558
    16
        {
    1559
    16
          float bias = convnet->layers->bias[i];
    1560
    16
          convnet->layers->bias[i] += fsh[k] * eps;
    1561
    16
          ccv_dense_matrix_t* z = 0;
    1562
    16
          ccv_convnet_compact(convnet);
    1563
    16
          ccv_convnet_encode(convnet, &x, &z, 1);
    1564
    16
          _ccv_convnet_compute_softmax(z, &z, 0);
    1565
    16
          dbias[i] += -logf(z->data.f32[2]) * fs[k];
    1566
    16
          ccv_matrix_free(z);
    1567
    16
          convnet->layers->bias[i] = bias;
    1568
    16
        }
    1569
    4
        dbias[i] *= 1.0 / (12 * eps);
    1570
    4
      }
    1571
    1
      ccv_matrix_free(y);
    1572
    1
      ccv_matrix_free(x);
    1573
    1
      REQUIRE_ARRAY_EQ_WITHIN_ANGLE_AND_MAGNITUDE(float, dw, update_params->layers[0].w, 3 * 3 * 2 * 4, 30, 2e-1, "weight gradient from analytical method doesn't match the one from numerical method");
    1574
    1
      ccfree(dw);
    1575
    1
      REQUIRE_ARRAY_EQ_WITHIN_ANGLE_AND_MAGNITUDE(float, dbias, update_params->layers[0].bias, 4, 30, 2e-1, "bias gradient from analytical method doesn't match the one from numerical method");
    1576
    1
      ccfree(dbias);
    1577
    1
      ccv_convnet_free(update_params);
    1578
    1
      ccv_convnet_free(convnet);
    1579
    1
    }
    1580
    1581
    TEST_CASE("numerical gradient versus analytical gradient for local response normalization over convolutional network")
    1582
    1
    {
    1583
    1
      ccv_convnet_layer_param_t params[] = {
    1584
    1
        {
    1585
    1
          .type = CCV_CONVNET_CONVOLUTIONAL,
    1586
    1
          .bias = 0,
    1587
    1
          .glorot = sqrtf(2),
    1588
    1
          .input = {
    1589
    1
            .matrix = {
    1590
    1
              .rows = 31,
    1591
    1
              .cols = 31,
    1592
    1
              .channels = 2,
    1593
    1
              .partition = 1,
    1594
    1
            },
    1595
    1
          },
    1596
    1
          .output = {
    1597
    1
            .convolutional = {
    1598
    1
              .rows = 5,
    1599
    1
              .cols = 5,
    1600
    1
              .channels = 2,
    1601
    1
              .border = 2,
    1602
    1
              .strides = 1,
    1603
    1
              .count = 4,
    1604
    1
              .partition = 1,
    1605
    1
            },
    1606
    1
          },
    1607
    1
        },
    1608
    1
        {
    1609
    1
          .type = CCV_CONVNET_LOCAL_RESPONSE_NORM,
    1610
    1
          .input = {
    1611
    1
            .matrix = {
    1612
    1
              .rows = 31,
    1613
    1
              .cols = 31,
    1614
    1
              .channels = 4,
    1615
    1
              .partition = 1,
    1616
    1
            },
    1617
    1
          },
    1618
    1
          .output = {
    1619
    1
            .rnorm = {
    1620
    1
              .size = 3,
    1621
    1
              .kappa = 2,
    1622
    1
              .alpha = 0.00005,
    1623
    1
              .beta = 0.75,
    1624
    1
            },
    1625
    1
          },
    1626
    1
        },
    1627
    1
      };
    1628
    1
      ccv_convnet_t* convnet = ccv_convnet_new(0, ccv_size(31, 31), params, 2);
    1629
    1
      dsfmt_t dsfmt;
    1630
    1
      dsfmt_init_gen_rand(&dsfmt, 3);
    1631
    1
      int i, k;
    1632
    201
      for (i = 0; i < convnet->layers->wnum; i++200)
    1633
    200
        convnet->layers->w[i] = dsfmt_genrand_gaussian(&dsfmt, 0.001);
    1634
    1
      ccv_convnet_t* update_params = _ccv_convnet_update_new(convnet);
    1635
    1
      _ccv_convnet_update_zero(update_params);
    1636
    1
      ccv_dense_matrix_t* x = ccv_dense_matrix_new(31, 31, CCV_32F | CCV_C2, 0, 0);
    1637
    1.92k
      for (i = 0; i < 31 * 31 * 2; i++1.92k)
    1638
    1.92k
        x->data.f32[i] = i;
    1639
    1
      ccv_dense_matrix_t* y = 0;
    1640
    1
      ccv_convnet_encode(convnet, &x, &y, 1);
    1641
    1
      REQUIRE(y->rows == 31 && y->cols == 31 && CCV_GET_CHANNEL(y->type) == 4, "convnet should return a 31x31x4 matrix");
    1642
    1
      ccv_dense_matrix_t* softmax = 0;
    1643
    1
      _ccv_convnet_compute_softmax(y, &softmax, 0);
    1644
    1
      ccv_dense_matrix_t* dloss = ccv_dense_matrix_new(y->rows, y->cols, CCV_32F | CCV_GET_CHANNEL(y->type), 0, 0);
    1645
    3.84k
      for (i = 0; i < 31 * 31 * 4; i++3.84k)
    1646
    3.84k
        dloss->data.f32[i] = softmax->data.f32[i] - (i == 24);
    1647
    1
      ccv_dense_matrix_t* d = 0;
    1648
    1
      _ccv_convnet_rnorm_backward_propagate(convnet->layers + 1, dloss, y, convnet->acts[0], convnet->denoms[1], update_params->acts);
    1649
    1
      _ccv_convnet_convolutional_backward_propagate(convnet->layers, update_params->acts[0], convnet->acts[0], x, &d, update_params->layers);
    1650
    1
      static const float eps = 0.000001;
    1651
    1
      float* dw = (float*)ccmalloc(sizeof(float) * 5 * 5 * 2 * 4);
    1652
    201
      for (i = 0; i < 5 * 5 * 2 * 4; i++200)
    1653
    200
      {
    1654
    200
        dw[i] = 0;
    1655
    1.00k
        for (k = 0; k < 4; k++800)
    1656
    800
        {
    1657
    800
          float w = convnet->layers->w[i];
    1658
    800
          convnet->layers->w[i] += fsh[k] * eps;
    1659
    800
          ccv_dense_matrix_t* z = 0;
    1660
    800
          ccv_convnet_compact(convnet);
    1661
    800
          ccv_convnet_encode(convnet, &x, &z, 1);
    1662
    800
          _ccv_convnet_compute_softmax(z, &z, 0);
    1663
    800
          dw[i] += -logf(z->data.f32[24]) * fs[k];
    1664
    800
          ccv_matrix_free(z);
    1665
    800
          convnet->layers->w[i] = w;
    1666
    800
        }
    1667
    200
        dw[i] *= 1.0 / (12 * eps);
    1668
    200
      }
    1669
    1
      float* dbias = (float*)ccmalloc(sizeof(float) * 4);
    1670
    1
      static const float beps = 0.0001;
    1671
    5
      for (i = 0; i < 4; i++4)
    1672
    4
      {
    1673
    4
        dbias[i] = 0;
    1674
    20
        for (k = 0; k < 4; k++16)
    1675
    16
        {
    1676
    16
          float bias = convnet->layers->bias[i];
    1677
    16
          convnet->layers->bias[i] += fsh[k] * beps;
    1678
    16
          ccv_dense_matrix_t* z = 0;
    1679
    16
          ccv_convnet_compact(convnet);
    1680
    16
          ccv_convnet_encode(convnet, &x, &z, 1);
    1681
    16
          _ccv_convnet_compute_softmax(z, &z, 0);
    1682
    16
          dbias[i] += -logf(z->data.f32[24]) * fs[k];
    1683
    16
          ccv_matrix_free(z);
    1684
    16
          convnet->layers->bias[i] = bias;
    1685
    16
        }
    1686
    4
        dbias[i] *= 1.0 / (12 * beps);
    1687
    4
      }
    1688
    1
      ccv_matrix_free(softmax);
    1689
    1
      ccv_matrix_free(dloss);
    1690
    1
      ccv_matrix_free(y);
    1691
    1
      ccv_matrix_free(x);
    1692
    1
      ccv_matrix_free(d);
    1693
    1
      REQUIRE_ARRAY_EQ_WITHIN_ANGLE_AND_MAGNITUDE(float, dw, update_params->layers[0].w, 5 * 5 * 2 * 4, 30, 2e-1, "weight gradient from analytical method doesn't match the one from numerical method");
    1694
    1
      ccfree(dw);
    1695
    1
      REQUIRE_ARRAY_EQ_WITHIN_ANGLE_AND_MAGNITUDE(float, dbias, update_params->layers[0].bias, 4, 30, 2e-1, "bias gradient from analytical method doesn't match the one from numerical method");
    1696
    1
      ccfree(dbias);
    1697
    1
      ccv_convnet_free(update_params);
    1698
    1
      ccv_convnet_free(convnet);
    1699
    1
    }
    1700
    1701
    TEST_CASE("max pool network backward propagate")
    1702
    1
    {
    1703
    1
      ccv_convnet_layer_param_t params = {
    1704
    1
        .type = CCV_CONVNET_MAX_POOL,
    1705
    1
        .input = {
    1706
    1
          .matrix = {
    1707
    1
            .rows = 31,
    1708
    1
            .cols = 31,
    1709
    1
            .channels = 2,
    1710
    1
            .partition = 1,
    1711
    1
          },
    1712
    1
        },
    1713
    1
        .output = {
    1714
    1
          .pool = {
    1715
    1
            .size = 3,
    1716
    1
            .strides = 2,
    1717
    1
            .border = 0,
    1718
    1
          },
    1719
    1
        },
    1720
    1
      };
    1721
    1
      ccv_convnet_t* convnet = ccv_convnet_new(0, ccv_size(31, 31), &params, 1);
    1722
    1
      ccv_dense_matrix_t* x = ccv_dense_matrix_new(31, 31, CCV_32F | CCV_C2, 0, 0);
    1723
    1
      int i, j, k;
    1724
    1.92k
      for (i = 0; i < 31 * 31 * 2; i++1.92k)
    1725
    1.92k
        x->data.f32[i] = i;
    1726
    1
      ccv_dense_matrix_t* y = 0;
    1727
    1
      ccv_convnet_encode(convnet, &x, &y, 1);
    1728
    1
      ccv_dense_matrix_t* loss = ccv_dense_matrix_new(15, 15, CCV_32F | CCV_C2, 0, 0);
    1729
    451
      for (i = 0; i < 15 * 15 * 2; i++450)
    1730
    450
        loss->data.f32[i] = i + 1;
    1731
    1
      ccv_dense_matrix_t* b = 0;
    1732
    1
      _ccv_convnet_max_pool_backward_propagate(convnet->layers, loss, y, x, &b);
    1733
    1
      ccv_matrix_free(loss);
    1734
    1
      ccv_matrix_free(x);
    1735
    1
      ccv_matrix_free(y);
    1736
    1
      ccv_dense_matrix_t* db = ccv_dense_matrix_new(31, 31, CCV_32F | CCV_C2, 0, 0);
    1737
    1
      ccv_zero(db);
    1738
    16
      for (i = 0; i < 15; i++15)
    1739
    240
        for (j = 0; 15j < 15; j++225)
    1740
    675
          for (k = 0; 225k < 2; k++450)
    1741
    450
            db->data.f32[(j * 2 + 2 + (i * 2 + 2) * 31) * 2 + k] = (i * 15 + j) * 2 + 1 + k;
    1742
    1
      REQUIRE_MATRIX_EQ(b, db, "propagated error doesn't match the expected value");
    1743
    1
      ccv_matrix_free(db);
    1744
    1
      ccv_matrix_free(b);
    1745
    1
      ccv_convnet_free(convnet);
    1746
    1
    }
    1747
    1748
    TEST_CASE("average pool network backward propagate")
    1749
    1
    {
    1750
    1
      ccv_convnet_layer_param_t params = {
    1751
    1
        .type = CCV_CONVNET_AVERAGE_POOL,
    1752
    1
        .input = {
    1753
    1
          .matrix = {
    1754
    1
            .rows = 31,
    1755
    1
            .cols = 31,
    1756
    1
            .channels = 2,
    1757
    1
            .partition = 1,
    1758
    1
          },
    1759
    1
        },
    1760
    1
        .output = {
    1761
    1
          .pool = {
    1762
    1
            .size = 3,
    1763
    1
            .strides = 2,
    1764
    1
            .border = 0,
    1765
    1
          },
    1766
    1
        },
    1767
    1
      };
    1768
    1
      ccv_convnet_t* convnet = ccv_convnet_new(0, ccv_size(31, 31), &params, 1);
    1769
    1
      ccv_dense_matrix_t* x = ccv_dense_matrix_new(31, 31, CCV_32F | CCV_C2, 0, 0);
    1770
    1
      int i, j, k;
    1771
    1.92k
      for (i = 0; i < 31 * 31 * 2; i++1.92k)
    1772
    1.92k
        x->data.f32[i] = i;
    1773
    1
      ccv_dense_matrix_t* loss = ccv_dense_matrix_new(15, 15, CCV_32F | CCV_C2, 0, 0);
    1774
    451
      for (i = 0; i < 15 * 15 * 2; i++450)
    1775
    450
        loss->data.f32[i] = i + 1;
    1776
    1
      ccv_dense_matrix_t* b = 0;
    1777
    1
      _ccv_convnet_average_pool_backward_propagate(convnet->layers, loss, x, &b);
    1778
    1
      ccv_matrix_free(x);
    1779
    1
      ccv_matrix_free(loss);
    1780
    1
      ccv_dense_matrix_t* db = ccv_dense_matrix_new(31, 31, CCV_32F | CCV_C2, 0, 0);
    1781
    1
      float inv_size = 1.0 / (3 * 3);
    1782
    32
      for (i = 0; i < 31; i++31)
    1783
    992
        for (j = 0; 31j < 31; j++961)
    1784
    2.88k
          for (k = 0; 961k < 2; k++1.92k)
    1785
    1.92k
          {
    1786
    1.92k
            int x, y;
    1787
    1.92k
            db->data.f32[(i * 31 + j) * 2 + k] = 0;
    1788
    4.77k
            for (y = (i - 1) / 2; y <= i / 2; y++2.85k)
    1789
    7.08k
              for (x = (j - 1) / 2; 2.85kx <= j / 2; x++4.23k)
    1790
    4.23k
                if (x >= 0 && x < 15 && y >= 04.14k && y < 154.14k)
    1791
    4.05k
                  db->data.f32[(i * 31 + j) * 2 + k] += ((y * 15 + x) * 2 + k + 1) * inv_size;
    1792
    1.92k
          }
    1793
    1
      REQUIRE_MATRIX_EQ(b, db, "propagated error doesn't match the expected value");
    1794
    1
      ccv_matrix_free(db);
    1795
    1
      ccv_matrix_free(b);
    1796
    1
      ccv_convnet_free(convnet);
    1797
    1
    }
    1798
    #endif
    1799
    1800
    #include "case_main.h"