## Coverage Report

#### Created: 2026-05-04 15:30

next uncovered line (L), next uncovered region (R), next uncovered branch (B)

    /home/liu/actions-runner/_work/ccv/ccv/test/int/nnc/parallel.tests.c

    Line
    Count
    Source
    1
    #include "case.h"
    2
    #include "ccv_case.h"
    3
    #include "ccv_nnc_case.h"
    4
    #include <ccv.h>
    5
    #include <nnc/ccv_nnc.h>
    6
    #include <nnc/ccv_nnc_easy.h>
    7
    #include <3rdparty/dsfmt/dSFMT.h>
    8
    9
    TEST_SETUP()
    10
    {
    11
      ccv_nnc_init();
    12
    }
    13
    14
    TEST_CASE("cnnp send to devices and all-to-all")
    15
    1
    {
    16
    1
      GUARD_ELSE_RETURN(ccv_nnc_cmd_ok(CCV_NNC_COMM_ALL_TO_ALL_FORWARD, CCV_NNC_BACKEND_GPU_NCCL));
    17
    1
      const int device_count = ccv_nnc_device_count(CCV_STREAM_CONTEXT_GPU);
    18
    1
      GUARD_ELSE_RETURN(device_count > 1);
    19
    1
      const int chunk = 5;
    20
    1
      const ccv_cnnp_model_io_t x = ccv_cnnp_input();
    21
    1
      ccv_cnnp_model_io_t chunks = ccv_cnnp_model_apply(ccv_cnnp_chunk(device_count, 0, "chunk"), MODEL_IO_LIST(x));
    22
    1
      ccv_cnnp_model_io_t sent[device_count];
    23
    1
      int i, j, k;
    24
    5
      for (i = 0; i < device_count; i++4)
    25
    4
      {
    26
    4
        ccv_cnnp_model_io_t chunk_i = ccv_cnnp_model_apply(ccv_cnnp_extract(i, 0), &chunks, 1);
    27
    4
        sent[i] = ccv_cnnp_model_apply(ccv_cnnp_send(i, 0), &chunk_i, 1);
    28
    4
      }
    29
    1
      ccv_cnnp_model_io_t exchanged = ccv_cnnp_model_apply(ccv_cnnp_all_to_all(device_count, 1, "all_to_all"), sent, device_count);
    30
    1
      ccv_cnnp_model_t* const model = ccv_cnnp_model_new(MODEL_IO_LIST(x), &exchanged, 1, 0, "send_all_to_all");
    31
    1
      ccv_nnc_tensor_param_t input_params = GPU_TENSOR_NHWC(000, 32F, device_count, device_count * chunk);
    32
    1
      ccv_cnnp_model_compile(model, TENSOR_PARAM_LIST(input_params), CMD_NOOP(), CMD_NOOP());
    33
    1
      ccv_nnc_tensor_param_t output_params[device_count];
    34
    1
      ccv_cnnp_model_tensor_auto(model, output_params, device_count);
    35
    1
      ccv_nnc_tensor_t* const h_input = ccv_nnc_tensor_new(0, CPU_TENSOR_NHWC(32F, device_count, device_count * chunk), 0);
    36
    5
      for (i = 0; i < device_count; i++4)
    37
    84
        for (j = 0; 4j < device_count * chunk; j++80)
    38
    80
          h_input->data.f32[i * device_count * chunk + j] = (float)(i * 1000 + j);
    39
    1
      ccv_nnc_tensor_t* const d_input = ccv_nnc_tensor_new(0, input_params, 0);
    40
    1
      ccv_nnc_cmd_exec(CMD_DATA_TRANSFER_FORWARD(), ccv_nnc_no_hint, 0, TENSOR_LIST(h_input), TENSOR_LIST(d_input), 0);
    41
    1
      ccv_nnc_tensor_t* outputs[device_count];
    42
    5
      for (i = 0; i < device_count; i++4)
    43
    4
      {
    44
    4
        REQUIRE_EQ(CCV_TENSOR_GET_DEVICE_ID(output_params[i].type), i, "output device should match all-to-all rank");
    45
    4
        outputs[i] = ccv_nnc_tensor_new(0, output_params[i], 0);
    46
    4
      }
    47
    1
      ccv_cnnp_model_evaluate(model, (ccv_cnnp_evaluate_param_t){}, TENSOR_LIST(d_input), outputs, device_count, 0, 0);
    48
    5
      for (j = 0; j < device_count; j++4)
    49
    4
      {
    50
    4
        ccv_nnc_tensor_t* const h_output = ccv_nnc_tensor_new(0, CPU_TENSOR_NHWC(32F, 1, device_count * chunk), 0);
    51
    4
        ccv_nnc_cmd_exec(CMD_DATA_TRANSFER_FORWARD(), ccv_nnc_no_hint, 0, TENSOR_LIST(outputs[j]), TENSOR_LIST(h_output), 0);
    52
    20
        for (i = 0; i < device_count; i++16)
    53
    96
          for (k = 0; 16k < chunk; k++80)
    54
    80
          {
    55
    80
            const float expected = h_input->data.f32[i * device_count * chunk + j * chunk + k];
    56
    80
            REQUIRE_EQ_WITH_TOLERANCE(h_output->data.f32[i * chunk + k], expected, 1e-5, "all-to-all output should match expected exchange");
    57
    80
          }
    58
    4
        ccv_nnc_tensor_free(h_output);
    59
    4
      }
    60
    5
      for (i = 0; 1i < device_count; i++4)
    61
    4
        ccv_nnc_tensor_free(outputs[i]);
    62
    1
      ccv_nnc_tensor_free(d_input);
    63
    1
      ccv_nnc_tensor_free(h_input);
    64
    1
      ccv_cnnp_model_free(model);
    65
    1
    }
    66
    67
    TEST_CASE("cnnp replicated dense feeds all-to-all")
    68
    1
    {
    69
    1
      GUARD_ELSE_RETURN(ccv_nnc_cmd_ok(CCV_NNC_GEMM_FORWARD, CCV_NNC_BACKEND_GPU_CUBLAS));
    70
    1
      GUARD_ELSE_RETURN(ccv_nnc_cmd_ok(CCV_NNC_COMM_ALL_TO_ALL_FORWARD, CCV_NNC_BACKEND_GPU_NCCL));
    71
    1
      const int device_count = ccv_nnc_device_count(CCV_STREAM_CONTEXT_GPU);
    72
    1
      GUARD_ELSE_RETURN(device_count > 1);
    73
    1
      const int rank_count = device_count;
    74
    1
      const int rows_per_rank = 2;
    75
    1
      const int input_dim = 4;
    76
    1
      const int output_chunk = 3;
    77
    1
      const int output_dim = rank_count * output_chunk;
    78
    1
      const ccv_cnnp_model_io_t x = ccv_cnnp_input();
    79
    1
      ccv_cnnp_model_io_t chunks = ccv_cnnp_model_apply(ccv_cnnp_chunk(rank_count, 0, "chunk"), MODEL_IO_LIST(x));
    80
    1
      ccv_cnnp_model_io_t sent[rank_count];
    81
    1
      int i, j, k, r;
    82
    5
      for (i = 0; i < rank_count; i++4)
    83
    4
      {
    84
    4
        ccv_cnnp_model_io_t chunk_i = ccv_cnnp_model_apply(ccv_cnnp_extract(i, 0), &chunks, 1);
    85
    4
        sent[i] = ccv_cnnp_model_apply(ccv_cnnp_send(i, 0), &chunk_i, 1);
    86
    4
      }
    87
    1
      ccv_cnnp_model_t* const dense = ccv_cnnp_dense(output_dim, 1, 0, 1, "dense");
    88
    1
      ccv_cnnp_model_io_t dense_outputs = ccv_cnnp_model_apply(ccv_cnnp_replicated(dense, rank_count, 1, "replicated_dense"), sent, rank_count);
    89
    1
      ccv_cnnp_model_io_t exchanged = ccv_cnnp_model_apply(ccv_cnnp_all_to_all(rank_count, 1, "all_to_all"), MODEL_IO_LIST(dense_outputs));
    90
    1
      ccv_cnnp_model_t* const model = ccv_cnnp_model_new(MODEL_IO_LIST(x), &exchanged, 1, 0, "replicated_dense_all_to_all");
    91
    1
      ccv_nnc_tensor_param_t input_params = GPU_TENSOR_NHWC(000, 32F, rank_count * rows_per_rank, input_dim);
    92
    1
      ccv_cnnp_model_compile(model, TENSOR_PARAM_LIST(input_params), CMD_NOOP(), CMD_NOOP());
    93
    1
      ccv_nnc_tensor_t* const h_input = ccv_nnc_tensor_new(0, CPU_TENSOR_NHWC(32F, rank_count * rows_per_rank, input_dim), 0);
    94
    9
      for (i = 0; i < rank_count * rows_per_rank; i++8)
    95
    40
        for (j = 0; 8j < input_dim; j++32)
    96
    32
          h_input->data.f32[i * input_dim + j] = (float)(i * 10 + j + 1);
    97
    1
      ccv_nnc_tensor_t* const h_weight = ccv_nnc_tensor_new(0, CPU_TENSOR_NHWC(32F, output_dim, input_dim), 0);
    98
    13
      for (i = 0; i < output_dim; i++12)
    99
    60
        for (j = 0; 12j < input_dim; j++48)
    100
    48
          h_weight->data.f32[i * input_dim + j] = (float)((i + 1) * 0.25 + (j + 1) * 0.125);
    101
    1
      ccv_cnnp_model_set_parameter(model, ccv_cnnp_model_parameters(dense, ALL_PARAMETERS, 0), h_weight);
    102
    1
      ccv_nnc_tensor_t* const d_input = ccv_nnc_tensor_new(0, input_params, 0);
    103
    1
      ccv_nnc_cmd_exec(CMD_DATA_TRANSFER_FORWARD(), ccv_nnc_no_hint, 0, TENSOR_LIST(h_input), TENSOR_LIST(d_input), 0);
    104
    1
      ccv_nnc_tensor_param_t output_params[rank_count];
    105
    1
      ccv_cnnp_model_tensor_auto(model, output_params, rank_count);
    106
    1
      ccv_nnc_tensor_t* outputs[rank_count];
    107
    5
      for (i = 0; i < rank_count; i++4)
    108
    4
      {
    109
    4
        REQUIRE_EQ(CCV_TENSOR_GET_DEVICE_ID(output_params[i].type), i, "output device should match all-to-all rank");
    110
    4
        outputs[i] = ccv_nnc_tensor_new(0, output_params[i], 0);
    111
    4
      }
    112
    1
      ccv_cnnp_model_evaluate(model, (ccv_cnnp_evaluate_param_t){}, TENSOR_LIST(d_input), outputs, rank_count, 0, 0);
    113
    5
      for (j = 0; j < rank_count; j++4)
    114
    4
      {
    115
    4
        ccv_nnc_tensor_t* const h_output = ccv_nnc_tensor_new(0, CPU_TENSOR_NHWC(32F, rows_per_rank, output_dim), 0);
    116
    4
        ccv_nnc_cmd_exec(CMD_DATA_TRANSFER_FORWARD(), ccv_nnc_no_hint, 0, TENSOR_LIST(outputs[j]), TENSOR_LIST(h_output), 0);
    117
    12
        for (r = 0; r < rows_per_rank; r++8)
    118
    40
          for (i = 0; 8i < rank_count; i++32)
    119
    128
            for (k = 0; 32k < output_chunk; k++96)
    120
    96
            {
    121
    96
              float expected = 0;
    122
    96
              const int input_row = i * rows_per_rank + r;
    123
    96
              const int output_channel = j * output_chunk + k;
    124
    96
              int l;
    125
    480
              for (l = 0; l < input_dim; l++384)
    126
    384
                expected += h_input->data.f32[input_row * input_dim + l] * h_weight->data.f32[output_channel * input_dim + l];
    127
    96
              REQUIRE_EQ_WITH_TOLERANCE(h_output->data.f32[r * output_dim + i * output_chunk + k], expected, 1e-4, "replicated dense all-to-all output should match CPU reference");
    128
    96
            }
    129
    4
        ccv_nnc_tensor_free(h_output);
    130
    4
      }
    131
    5
      for (i = 0; 1i < rank_count; i++4)
    132
    4
        ccv_nnc_tensor_free(outputs[i]);
    133
    1
      ccv_nnc_tensor_free(d_input);
    134
    1
      ccv_nnc_tensor_free(h_weight);
    135
    1
      ccv_nnc_tensor_free(h_input);
    136
    1
      ccv_cnnp_model_free(model);
    137
    1
    }
    138
    139
    TEST_CASE("cnnp replicated embedding matches CPU reference")
    140
    1
    {
    141
    1
      GUARD_ELSE_RETURN(ccv_nnc_cmd_ok(CCV_NNC_INDEX_SELECT_FORWARD, CCV_NNC_BACKEND_GPU_REF));
    142
    1
      const int device_count = ccv_nnc_device_count(CCV_STREAM_CONTEXT_GPU);
    143
    1
      GUARD_ELSE_RETURN(device_count > 1);
    144
    1
      const int rank_count = device_count;
    145
    1
      const int rows_per_rank = 3;
    146
    1
      const int vocab_size = 11;
    147
    1
      const int embed_size = 5;
    148
    1
      const ccv_cnnp_model_io_t x = ccv_cnnp_input();
    149
    1
      ccv_cnnp_model_io_t chunks = ccv_cnnp_model_apply(ccv_cnnp_chunk(rank_count, 0, "chunk"), MODEL_IO_LIST(x));
    150
    1
      ccv_cnnp_model_io_t sent[rank_count];
    151
    1
      int i, j, r;
    152
    5
      for (i = 0; i < rank_count; i++4)
    153
    4
      {
    154
    4
        ccv_cnnp_model_io_t chunk_i = ccv_cnnp_model_apply(ccv_cnnp_extract(i, 0), &chunks, 1);
    155
    4
        sent[i] = ccv_cnnp_model_apply(ccv_cnnp_send(i, 0), &chunk_i, 1);
    156
    4
      }
    157
    1
      ccv_cnnp_model_t* const embedding = ccv_cnnp_embedding(CCV_32F, vocab_size, embed_size, 1, "embedding");
    158
    1
      ccv_cnnp_model_io_t embedding_outputs = ccv_cnnp_model_apply(ccv_cnnp_replicated(embedding, rank_count, 1, "replicated_embedding"), sent, rank_count);
    159
    1
      ccv_cnnp_model_t* const model = ccv_cnnp_model_new(MODEL_IO_LIST(x), MODEL_IO_LIST(embedding_outputs), 0, "replicated_embedding");
    160
    1
      ccv_nnc_tensor_param_t input_params = GPU_TENSOR_NHWC(000, 32S, rank_count * rows_per_rank);
    161
    1
      ccv_cnnp_model_compile(model, TENSOR_PARAM_LIST(input_params), CMD_NOOP(), CMD_NOOP());
    162
    1
      ccv_nnc_tensor_t* const h_input = ccv_nnc_tensor_new(0, CPU_TENSOR_NHWC(32S, rank_count * rows_per_rank), 0);
    163
    13
      for (i = 0; i < rank_count * rows_per_rank; i++12)
    164
    12
        h_input->data.i32[i] = (i * 3 + 1) % vocab_size;
    165
    1
      ccv_nnc_tensor_t* const h_vocab = ccv_nnc_tensor_new(0, CPU_TENSOR_NHWC(32F, vocab_size, embed_size), 0);
    166
    12
      for (i = 0; i < vocab_size; i++11)
    167
    66
        for (j = 0; 11j < embed_size; j++55)
    168
    55
          h_vocab->data.f32[i * embed_size + j] = (float)(i * 0.5 + j * 0.25 + 1);
    169
    1
      ccv_cnnp_model_set_parameter(model, ccv_cnnp_model_parameters(embedding, ALL_PARAMETERS, 0), h_vocab);
    170
    1
      ccv_nnc_tensor_t* const d_input = ccv_nnc_tensor_new(0, input_params, 0);
    171
    1
      ccv_nnc_cmd_exec(CMD_DATA_TRANSFER_FORWARD(), ccv_nnc_no_hint, 0, TENSOR_LIST(h_input), TENSOR_LIST(d_input), 0);
    172
    1
      ccv_nnc_tensor_param_t output_params[rank_count];
    173
    1
      ccv_cnnp_model_tensor_auto(model, output_params, rank_count);
    174
    1
      ccv_nnc_tensor_t* outputs[rank_count];
    175
    5
      for (i = 0; i < rank_count; i++4)
    176
    4
      {
    177
    4
        REQUIRE_EQ(CCV_TENSOR_GET_DEVICE_ID(output_params[i].type), i, "output device should match replicated rank");
    178
    4
        outputs[i] = ccv_nnc_tensor_new(0, output_params[i], 0);
    179
    4
      }
    180
    1
      ccv_cnnp_model_evaluate(model, (ccv_cnnp_evaluate_param_t){}, TENSOR_LIST(d_input), outputs, rank_count, 0, 0);
    181
    5
      for (i = 0; i < rank_count; i++4)
    182
    4
      {
    183
    4
        ccv_nnc_tensor_t* const h_output = ccv_nnc_tensor_new(0, CPU_TENSOR_NHWC(32F, rows_per_rank, embed_size), 0);
    184
    4
        ccv_nnc_cmd_exec(CMD_DATA_TRANSFER_FORWARD(), ccv_nnc_no_hint, 0, TENSOR_LIST(outputs[i]), TENSOR_LIST(h_output), 0);
    185
    16
        for (r = 0; r < rows_per_rank; r++12)
    186
    12
        {
    187
    12
          const int token = h_input->data.i32[i * rows_per_rank + r];
    188
    72
          for (j = 0; j < embed_size; j++60)
    189
    60
            REQUIRE_EQ_WITH_TOLERANCE(h_output->data.f32[r * embed_size + j], h_vocab->data.f32[token * embed_size + j], 1e-5, "replicated embedding output should match CPU reference");
    190
    12
        }
    191
    4
        ccv_nnc_tensor_free(h_output);
    192
    4
      }
    193
    5
      for (i = 0; 1i < rank_count; i++4)
    194
    4
        ccv_nnc_tensor_free(outputs[i]);
    195
    1
      ccv_nnc_tensor_free(d_input);
    196
    1
      ccv_nnc_tensor_free(h_vocab);
    197
    1
      ccv_nnc_tensor_free(h_input);
    198
    1
      ccv_cnnp_model_free(model);
    199
    1
    }
    200
    201
    TEST_CASE("cnnp replicated layer norm matches CPU reference")
    202
    1
    {
    203
    1
      GUARD_ELSE_RETURN(ccv_nnc_cmd_ok(CCV_NNC_LAYER_NORM_FORWARD, CCV_NNC_BACKEND_GPU_CUDNN));
    204
    1
      const int device_count = ccv_nnc_device_count(CCV_STREAM_CONTEXT_GPU);
    205
    1
      GUARD_ELSE_RETURN(device_count > 1);
    206
    1
      const int rank_count = device_count;
    207
    1
      const int rows_per_rank = 2;
    208
    1
      const int cols = 6;
    209
    1
      const float epsilon = 1e-5;
    210
    1
      const int axis[] = {1};
    211
    1
      const ccv_cnnp_model_io_t x = ccv_cnnp_input();
    212
    1
      ccv_cnnp_model_io_t chunks = ccv_cnnp_model_apply(ccv_cnnp_chunk(rank_count, 0, "chunk"), MODEL_IO_LIST(x));
    213
    1
      ccv_cnnp_model_io_t sent[rank_count];
    214
    1
      int i, j, r;
    215
    5
      for (i = 0; i < rank_count; i++4)
    216
    4
      {
    217
    4
        ccv_cnnp_model_io_t chunk_i = ccv_cnnp_model_apply(ccv_cnnp_extract(i, 0), &chunks, 1);
    218
    4
        sent[i] = ccv_cnnp_model_apply(ccv_cnnp_send(i, 0), &chunk_i, 1);
    219
    4
      }
    220
    1
      ccv_cnnp_model_t* const layer_norm = ccv_cnnp_layer_norm(epsilon, axis, 1, 1, 1, "layer_norm");
    221
    1
      ccv_cnnp_model_io_t normalized = ccv_cnnp_model_apply(ccv_cnnp_replicated(layer_norm, rank_count, 1, "replicated_layer_norm"), sent, rank_count);
    222
    1
      ccv_cnnp_model_t* const model = ccv_cnnp_model_new(MODEL_IO_LIST(x), MODEL_IO_LIST(normalized), 0, "replicated_layer_norm");
    223
    1
      ccv_nnc_tensor_param_t input_params = GPU_TENSOR_NHWC(000, 32F, rank_count * rows_per_rank, cols);
    224
    1
      ccv_cnnp_model_compile(model, TENSOR_PARAM_LIST(input_params), CMD_NOOP(), CMD_NOOP());
    225
    1
      ccv_nnc_tensor_t* const h_input = ccv_nnc_tensor_new(0, CPU_TENSOR_NHWC(32F, rank_count * rows_per_rank, cols), 0);
    226
    9
      for (i = 0; i < rank_count * rows_per_rank; i++8)
    227
    56
        for (j = 0; 8j < cols; j++48)
    228
    48
          h_input->data.f32[i * cols + j] = (float)((i + 1) * 0.75 + (j + 1) * 0.5);
    229
    1
      ccv_nnc_tensor_t* const h_scale = ccv_nnc_tensor_new(0, CPU_TENSOR_NHWC(32F, 1, cols), 0);
    230
    1
      ccv_nnc_tensor_t* const h_bias = ccv_nnc_tensor_new(0, CPU_TENSOR_NHWC(32F, 1, cols), 0);
    231
    7
      for (j = 0; j < cols; j++6)
    232
    6
      {
    233
    6
        h_scale->data.f32[j] = 0.5f + j * 0.125f;
    234
    6
        h_bias->data.f32[j] = -0.75f + j * 0.2f;
    235
    6
      }
    236
    1
      ccv_cnnp_model_set_parameter(model, ccv_cnnp_model_parameters(layer_norm, ALL_PARAMETERS, 0), h_scale);
    237
    1
      ccv_cnnp_model_set_parameter(model, ccv_cnnp_model_parameters(layer_norm, ALL_PARAMETERS, 1), h_bias);
    238
    1
      ccv_nnc_tensor_t* const d_input = ccv_nnc_tensor_new(0, input_params, 0);
    239
    1
      ccv_nnc_cmd_exec(CMD_DATA_TRANSFER_FORWARD(), ccv_nnc_no_hint, 0, TENSOR_LIST(h_input), TENSOR_LIST(d_input), 0);
    240
    1
      ccv_nnc_tensor_param_t output_params[rank_count];
    241
    1
      ccv_cnnp_model_tensor_auto(model, output_params, rank_count);
    242
    1
      ccv_nnc_tensor_t* outputs[rank_count];
    243
    5
      for (i = 0; i < rank_count; i++4)
    244
    4
      {
    245
    4
        REQUIRE_EQ(CCV_TENSOR_GET_DEVICE_ID(output_params[i].type), i, "output device should match replicated rank");
    246
    4
        outputs[i] = ccv_nnc_tensor_new(0, output_params[i], 0);
    247
    4
      }
    248
    1
      ccv_cnnp_model_evaluate(model, (ccv_cnnp_evaluate_param_t){}, TENSOR_LIST(d_input), outputs, rank_count, 0, 0);
    249
    5
      for (i = 0; i < rank_count; i++4)
    250
    4
      {
    251
    4
        ccv_nnc_tensor_t* const h_output = ccv_nnc_tensor_new(0, CPU_TENSOR_NHWC(32F, rows_per_rank, cols), 0);
    252
    4
        ccv_nnc_cmd_exec(CMD_DATA_TRANSFER_FORWARD(), ccv_nnc_no_hint, 0, TENSOR_LIST(outputs[i]), TENSOR_LIST(h_output), 0);
    253
    12
        for (r = 0; r < rows_per_rank; r++8)
    254
    8
        {
    255
    8
          const int row = i * rows_per_rank + r;
    256
    8
          float mean = 0;
    257
    56
          for (j = 0; j < cols; j++48)
    258
    48
            mean += h_input->data.f32[row * cols + j];
    259
    8
          mean /= cols;
    260
    8
          float variance = 0;
    261
    56
          for (j = 0; j < cols; j++48)
    262
    48
          {
    263
    48
            const float centered = h_input->data.f32[row * cols + j] - mean;
    264
    48
            variance += centered * centered;
    265
    48
          }
    266
    8
          variance /= cols;
    267
    8
          const float inv_std = 1.0 / sqrtf(variance + epsilon);
    268
    56
          for (j = 0; j < cols; j++48)
    269
    48
          {
    270
    48
            const float centered = h_input->data.f32[row * cols + j] - mean;
    271
    48
            const float expected = centered * inv_std * h_scale->data.f32[j] + h_bias->data.f32[j];
    272
    48
            REQUIRE_EQ_WITH_TOLERANCE(h_output->data.f32[r * cols + j], expected, 1e-4, "replicated layer norm output should match CPU reference");
    273
    48
          }
    274
    8
        }
    275
    4
        ccv_nnc_tensor_free(h_output);
    276
    4
      }
    277
    5
      for (i = 0; 1i < rank_count; i++4)
    278
    4
        ccv_nnc_tensor_free(outputs[i]);
    279
    1
      ccv_nnc_tensor_free(d_input);
    280
    1
      ccv_nnc_tensor_free(h_bias);
    281
    1
      ccv_nnc_tensor_free(h_scale);
    282
    1
      ccv_nnc_tensor_free(h_input);
    283
    1
      ccv_cnnp_model_free(model);
    284
    1
    }
    285
    286
    TEST_CASE("schedule symbolic graph to data parallel with broadcast and reduce")
    287
    1
    {
    288
    1
      GUARD_ELSE_RETURN(ccv_nnc_cmd_ok(CCV_NNC_CONVOLUTION_FORWARD, CCV_NNC_BACKEND_GPU_CUDNN));
    289
    1
      GUARD_ELSE_RETURN(ccv_nnc_device_count(CCV_STREAM_CONTEXT_GPU) >= 2);
    290
    1
      ccv_nnc_tensor_t* updated[4];
    291
    1
      ccv_nnc_tensor_t* cpu_inputs[2];
    292
    1
      ccv_nnc_tensor_t* cpu_fits[2];
    293
    1
      ccv_nnc_tensor_t* w1_tensor = ccv_nnc_tensor_new(0, CPU_TENSOR_NCHW(32F, 8, 3, 5, 5), 0);
    294
    1
      ccv_nnc_tensor_t* w3_tensor = ccv_nnc_tensor_new(0, CPU_TENSOR_NCHW(32F, 8, 8, 5, 5), 0);
    295
    1
      {
    296
    1
        ccv_nnc_symbolic_graph_t* const symbolic_graph = ccv_nnc_symbolic_graph_new();
    297
    1
        const ccv_nnc_tensor_symbol_t x = ccv_nnc_tensor_symbol_new(symbolic_graph, GPU_TENSOR_NCHW(000, 32F, 16, 3, 32, 32), 0);
    298
    1
        const ccv_nnc_tensor_symbol_t w1 = ccv_nnc_tensor_symbol_new(symbolic_graph, GPU_TENSOR_NCHW(000, 32F, 8, 3, 5, 5), 0);
    299
    1
        const ccv_nnc_tensor_symbol_t bias1 = ccv_nnc_tensor_symbol_new(symbolic_graph, GPU_TENSOR_NCHW(000, 32F, 8), 0);
    300
    1
        const ccv_nnc_tensor_symbol_t y1 = ccv_nnc_tensor_symbol_new(symbolic_graph, GPU_TENSOR_NCHW(000, 32F, 16, 8, 32, 32), 0);
    301
    1
        const ccv_nnc_graph_exec_symbol_t conv1 = ccv_nnc_graph_exec_symbol_new(symbolic_graph, CMD_CONVOLUTION_FORWARD(1, 8, 5, 5, 32), TENSOR_SYMBOL_LIST(x, w1, bias1), TENSOR_SYMBOL_LIST(y1), "conv1");
    302
    1
        ccv_nnc_graph_exec_symbol_set_hint(symbolic_graph, conv1, HINT((1, 1), (2, 2)));
    303
    1
        const ccv_nnc_tensor_symbol_t y2 = ccv_nnc_tensor_symbol_new(symbolic_graph, GPU_TENSOR_NCHW(000, 32F, 16, 8, 16, 16), 0);
    304
    1
        const ccv_nnc_graph_exec_symbol_t avg2 = ccv_nnc_graph_exec_symbol_new(symbolic_graph, CMD_AVERAGE_POOL_FORWARD(2, 2), TENSOR_SYMBOL_LIST(y1), TENSOR_SYMBOL_LIST(y2), "avg2");
    305
    1
        ccv_nnc_graph_exec_symbol_set_hint(symbolic_graph, avg2, HINT((2, 2)));
    306
    1
        const ccv_nnc_tensor_symbol_t w3 = ccv_nnc_tensor_symbol_new(symbolic_graph, GPU_TENSOR_NCHW(000, 32F, 8, 8, 5, 5), 0);
    307
    1
        const ccv_nnc_tensor_symbol_t bias3 = ccv_nnc_tensor_symbol_new(symbolic_graph, GPU_TENSOR_NCHW(000, 32F, 8), 0);
    308
    1
        const ccv_nnc_tensor_symbol_t y3 = ccv_nnc_tensor_symbol_new(symbolic_graph, GPU_TENSOR_NCHW(000, 32F, 16, 8, 8, 8), 0);
    309
    1
        const ccv_nnc_graph_exec_symbol_t conv3 = ccv_nnc_graph_exec_symbol_new(symbolic_graph, CMD_CONVOLUTION_FORWARD(1, 8, 5, 5, 32), TENSOR_SYMBOL_LIST(y2, w3, bias3), TENSOR_SYMBOL_LIST(y3), "conv3");
    310
    1
        ccv_nnc_graph_exec_symbol_set_hint(symbolic_graph, conv3, HINT((2, 2), (2, 2)));
    311
    1
        const ccv_nnc_tensor_symbol_t y4 = ccv_nnc_tensor_symbol_new(symbolic_graph, GPU_TENSOR_NCHW(000, 32F, 16, 8, 1, 1), 0);
    312
    1
        ccv_nnc_graph_exec_symbol_new(symbolic_graph, CMD_AVERAGE_POOL_FORWARD(8, 8), TENSOR_SYMBOL_LIST(y3), TENSOR_SYMBOL_LIST(y4), "avg4");
    313
    1
        const ccv_nnc_tensor_symbol_t y4a = ccv_nnc_tensor_symbol_alias_new(symbolic_graph, y4, ccv_nnc_no_ofs, DIM_ALLOC(8, 1, 1, 1), GPU_TENSOR_NCHW(000, 32F, 16, 8), 0);
    314
    1
        const ccv_nnc_tensor_symbol_t label = ccv_nnc_tensor_symbol_new(symbolic_graph, GPU_TENSOR_NCHW(000, 32F, 16), "label");
    315
    1
        const ccv_nnc_tensor_symbol_t y5 = ccv_nnc_tensor_symbol_new(symbolic_graph, GPU_TENSOR_NCHW(000, 32F, 16, 8), "y5");
    316
    1
        const ccv_nnc_tensor_symbol_t loss = ccv_nnc_tensor_symbol_new(symbolic_graph, GPU_TENSOR_NCHW(000, 32F, 16), "loss");
    317
    1
        ccv_nnc_graph_exec_symbol_new(symbolic_graph, CMD_SOFTMAX_CROSSENTROPY_FORWARD(), TENSOR_SYMBOL_LIST(y4a, label), TENSOR_SYMBOL_LIST(loss, y5), "softmax crossentropy");
    318
    1
        ccv_nnc_graph_exec_symbol_autogen(symbolic_graph, 0, 0, CCV_NNC_AUTOGEN_ALL_EXECS | CCV_NNC_AUTOGEN_SOURCES_AND_DESTINATIONS);
    319
    1
        ccv_nnc_tensor_symbol_t updated_params[4];
    320
    1
        ccv_nnc_tensor_symbol_t gradients[4];
    321
    1
        const int saved_aux_size = ccv_nnc_minimizer_saved_aux_size(CMD_SGD_FORWARD(0, 0.001, 1, 0.99, 0.9, 0.9));
    322
    1
        ccv_nnc_tensor_symbol_map_t saved_aux[saved_aux_size * 4];
    323
    1
        ccv_nnc_graph_exec_symbol_t updated_execs[4];
    324
    1
        ccv_nnc_symbolic_graph_minimize(symbolic_graph, CMD_SGD_FORWARD(0, 0.001, 1, 0.99, 0.9, 0.9), TENSOR_SYMBOL_LIST(loss), TENSOR_SYMBOL_LIST(w1, bias1, w3, bias3), 0, 0, SYMBOLIC_GRAPH_SOURCES(symbolic_graph), SYMBOLIC_GRAPH_DESTINATIONS(symbolic_graph), gradients, updated_params, saved_aux, updated_execs);
    325
    1
        const ccv_nnc_tensor_symbol_t dloss = ccv_nnc_tensor_symbol_for_backward(symbolic_graph, loss);
    326
    1
        ccv_nnc_graph_exec_symbol_new(symbolic_graph, CMD_SET_FORWARD(1), TENSOR_SYMBOL_LIST(), TENSOR_SYMBOL_LIST(dloss), "set 1");
    327
    1
        int i;
    328
    5
        for (i = 0; i < saved_aux_size * 4; i++4)
    329
    4
          ccv_nnc_graph_exec_symbol_new(symbolic_graph, CMD_SET_FORWARD(0), TENSOR_SYMBOL_LIST(), TENSOR_SYMBOL_LIST(saved_aux[i].source), "set 0");
    330
    1
        ccv_nnc_graph_exec_symbol_autogen(symbolic_graph, 0, 0, CCV_NNC_AUTOGEN_ALL_EXECS | CCV_NNC_AUTOGEN_SOURCES_AND_DESTINATIONS);
    331
    1
        ccv_nnc_symbolic_graph_data_parallel(symbolic_graph, 2, TENSOR_SYMBOL_LIST(w1, bias1, w3, bias3), 0, 0, 0, gradients, 4, 0, CCV_NNC_PARALLEL_REDUCE_OP_SUM, SYMBOLIC_GRAPH_SOURCES(symbolic_graph), updated_execs, 4);
    332
    1
        ccv_nnc_graph_exec_symbol_autogen(symbolic_graph, 0, 0, CCV_NNC_AUTOGEN_SOURCES_AND_DESTINATIONS);
    333
    1
        SYMBOLIC_GRAPH_GEN(symbolic_graph, CCV_NNC_LONG_DOT_GRAPH);
    334
    1
        ccv_nnc_graph_t* graph;
    335
    1
        ccv_nnc_tensor_arena_t* tensor_arena;
    336
    1
        ccv_nnc_graph_exec_arena_t* graph_exec_arena;
    337
    1
        ccv_nnc_symbolic_graph_compile(symbolic_graph, ccv_nnc_default_compile_params,
    338
    1
          0, 0,
    339
    1
          updated_params, 4,
    340
    1
          SYMBOLIC_GRAPH_SOURCES(symbolic_graph), SYMBOLIC_GRAPH_DESTINATIONS(symbolic_graph),
    341
    1
          &graph, &tensor_arena, &graph_exec_arena);
    342
    1
        ccv_nnc_graph_set_default_static_schedule(graph, CCV_STREAM_CONTEXT_GPU, 0);
    343
    1
        GRAPH_GEN(graph, CCV_NNC_LONG_DOT_GRAPH);
    344
    1
        cpu_inputs[0] = ccv_nnc_tensor_new(0, CPU_TENSOR_NCHW(32F, 16, 3, 32, 32), 0);
    345
    1
        cpu_inputs[1] = ccv_nnc_tensor_new(0, CPU_TENSOR_NCHW(32F, 16, 3, 32, 32), 0);
    346
    1
        dsfmt_t dsfmt;
    347
    1
        dsfmt_init_gen_rand(&dsfmt, 0);
    348
    49.1k
        for (i = 0; i < 16 * 3 * 32 * 32; i++49.1k)
    349
    49.1k
          cpu_inputs[0]->data.f32[i] = dsfmt_genrand_open_close(&dsfmt);
    350
    49.1k
        for (i = 0; i < 16 * 3 * 32 * 32; i++49.1k)
    351
    49.1k
          cpu_inputs[1]->data.f32[i] = dsfmt_genrand_open_close(&dsfmt);
    352
    1
        cpu_fits[0] = ccv_nnc_tensor_new(0, CPU_TENSOR_NCHW(32F, 16), 0);
    353
    1
        cpu_fits[1] = ccv_nnc_tensor_new(0, CPU_TENSOR_NCHW(32F, 16), 0);
    354
    17
        for (i = 0; i < 16; i++16)
    355
    16
          cpu_fits[0]->data.f32[i] = cpu_fits[1]->data.f32[i] = (int)(dsfmt_genrand_open_close(&dsfmt) * 7.4); // Between 0 to 7.
    356
    1
        ccv_nnc_cmd_exec(CMD_DATA_TRANSFER_FORWARD(), ccv_nnc_no_hint, 0, TENSOR_LIST(cpu_inputs[0], cpu_inputs[1]), TENSOR_LIST(ccv_nnc_tensor_from_symbol(tensor_arena, x), ccv_nnc_tensor_from_symbol(tensor_arena, ccv_nnc_tensor_symbol_copy(symbolic_graph, x, 1))), 0);
    357
    1
        ccv_nnc_cmd_exec(CMD_DATA_TRANSFER_FORWARD(), ccv_nnc_no_hint, 0, TENSOR_LIST(cpu_fits[0], cpu_fits[1]), TENSOR_LIST(ccv_nnc_tensor_from_symbol(tensor_arena, label), ccv_nnc_tensor_from_symbol(tensor_arena, ccv_nnc_tensor_symbol_copy(symbolic_graph, label, 1))), 0);
    358
    1
        ccv_nnc_cmd_exec(CMD_SET_FORWARD(0), ccv_nnc_no_hint, 0, TENSOR_LIST(), TENSOR_LIST(ccv_nnc_tensor_from_symbol(tensor_arena, bias1), ccv_nnc_tensor_from_symbol(tensor_arena, bias3)), 0);
    359
    601
        for (i = 0; i < 8 * 3 * 5 * 5; i++600)
    360
    600
          w1_tensor->data.f32[i] = dsfmt_genrand_open_close(&dsfmt);
    361
    1.60k
        for (i = 0; i < 8 * 8 * 5 * 5; i++1.60k)
    362
    1.60k
          w3_tensor->data.f32[i] = dsfmt_genrand_open_close(&dsfmt);
    363
    1
        ccv_nnc_cmd_exec(CMD_DATA_TRANSFER_FORWARD(), ccv_nnc_no_hint, 0, TENSOR_LIST(w1_tensor, w3_tensor), TENSOR_LIST(ccv_nnc_tensor_from_symbol(tensor_arena, w1), ccv_nnc_tensor_from_symbol(tensor_arena, w3)), 0);
    364
    1
        ccv_nnc_graph_run(graph, 0, TRAVERSE_FULL, 0, ccv_nnc_graph_default_stream(graph));
    365
    1
        ccv_nnc_stream_context_wait(ccv_nnc_graph_default_stream(graph));
    366
    1
        updated[0] = ccv_nnc_tensor_new(0, CPU_TENSOR_NCHW(32F, 8, 3, 5, 5), 0);
    367
    1
        updated[1] = ccv_nnc_tensor_new(0, CPU_TENSOR_NCHW(32F, 8), 0);
    368
    1
        updated[2] = ccv_nnc_tensor_new(0, CPU_TENSOR_NCHW(32F, 8, 8, 5, 5), 0);
    369
    1
        updated[3] = ccv_nnc_tensor_new(0, CPU_TENSOR_NCHW(32F, 8), 0);
    370
    1
        ccv_nnc_cmd_exec(CMD_DATA_TRANSFER_FORWARD(), ccv_nnc_no_hint, 0, TENSOR_LIST(ccv_nnc_tensor_from_symbol(tensor_arena, updated_params[0]), ccv_nnc_tensor_from_symbol(tensor_arena, updated_params[1]), ccv_nnc_tensor_from_symbol(tensor_arena, updated_params[2]), ccv_nnc_tensor_from_symbol(tensor_arena, updated_params[3])), updated, 4, 0);
    371
    1
        ccv_nnc_symbolic_graph_free(symbolic_graph);
    372
    1
        ccv_nnc_graph_free(graph);
    373
    1
        ccv_nnc_tensor_arena_free(tensor_arena);
    374
    1
        ccv_nnc_graph_exec_arena_free(graph_exec_arena);
    375
    1
      }
    376
      // Now, doing exactly the same, but with no parallel.
    377
    1
      {
    378
    1
        ccv_nnc_symbolic_graph_t* const symbolic_graph = ccv_nnc_symbolic_graph_new();
    379
    1
        const ccv_nnc_tensor_symbol_t x = ccv_nnc_tensor_symbol_new(symbolic_graph, GPU_TENSOR_NCHW(000, 32F, 32, 3, 32, 32), 0);
    380
    1
        const ccv_nnc_tensor_symbol_t w1 = ccv_nnc_tensor_symbol_new(symbolic_graph, GPU_TENSOR_NCHW(000, 32F, 8, 3, 5, 5), 0);
    381
    1
        const ccv_nnc_tensor_symbol_t bias1 = ccv_nnc_tensor_symbol_new(symbolic_graph, GPU_TENSOR_NCHW(000, 32F, 8), 0);
    382
    1
        const ccv_nnc_tensor_symbol_t y1 = ccv_nnc_tensor_symbol_new(symbolic_graph, GPU_TENSOR_NCHW(000, 32F, 32, 8, 32, 32), 0);
    383
    1
        const ccv_nnc_graph_exec_symbol_t conv1 = ccv_nnc_graph_exec_symbol_new(symbolic_graph, CMD_CONVOLUTION_FORWARD(1, 8, 5, 5, 32), TENSOR_SYMBOL_LIST(x, w1, bias1), TENSOR_SYMBOL_LIST(y1), "conv1");
    384
    1
        ccv_nnc_graph_exec_symbol_set_hint(symbolic_graph, conv1, HINT((1, 1), (2, 2)));
    385
    1
        const ccv_nnc_tensor_symbol_t y2 = ccv_nnc_tensor_symbol_new(symbolic_graph, GPU_TENSOR_NCHW(000, 32F, 32, 8, 16, 16), 0);
    386
    1
        const ccv_nnc_graph_exec_symbol_t avg2 = ccv_nnc_graph_exec_symbol_new(symbolic_graph, CMD_AVERAGE_POOL_FORWARD(2, 2), TENSOR_SYMBOL_LIST(y1), TENSOR_SYMBOL_LIST(y2), "avg2");
    387
    1
        ccv_nnc_graph_exec_symbol_set_hint(symbolic_graph, avg2, HINT((2, 2)));
    388
    1
        const ccv_nnc_tensor_symbol_t w3 = ccv_nnc_tensor_symbol_new(symbolic_graph, GPU_TENSOR_NCHW(000, 32F, 8, 8, 5, 5), 0);
    389
    1
        const ccv_nnc_tensor_symbol_t bias3 = ccv_nnc_tensor_symbol_new(symbolic_graph, GPU_TENSOR_NCHW(000, 32F, 8), 0);
    390
    1
        const ccv_nnc_tensor_symbol_t y3 = ccv_nnc_tensor_symbol_new(symbolic_graph, GPU_TENSOR_NCHW(000, 32F, 32, 8, 8, 8), 0);
    391
    1
        const ccv_nnc_graph_exec_symbol_t conv3 = ccv_nnc_graph_exec_symbol_new(symbolic_graph, CMD_CONVOLUTION_FORWARD(1, 8, 5, 5, 32), TENSOR_SYMBOL_LIST(y2, w3, bias3), TENSOR_SYMBOL_LIST(y3), "conv3");
    392
    1
        ccv_nnc_graph_exec_symbol_set_hint(symbolic_graph, conv3, HINT((2, 2), (2, 2)));
    393
    1
        const ccv_nnc_tensor_symbol_t y4 = ccv_nnc_tensor_symbol_new(symbolic_graph, GPU_TENSOR_NCHW(000, 32F, 32, 8, 1, 1), 0);
    394
    1
        ccv_nnc_graph_exec_symbol_new(symbolic_graph, CMD_AVERAGE_POOL_FORWARD(8, 8), TENSOR_SYMBOL_LIST(y3), TENSOR_SYMBOL_LIST(y4), "avg4");
    395
    1
        const ccv_nnc_tensor_symbol_t label = ccv_nnc_tensor_symbol_new(symbolic_graph, GPU_TENSOR_NCHW(000, 32F, 32), "label");
    396
    1
        const ccv_nnc_tensor_symbol_t y5 = ccv_nnc_tensor_symbol_new(symbolic_graph, GPU_TENSOR_NCHW(000, 32F, 32, 8), "y5");
    397
    1
        const ccv_nnc_tensor_symbol_t loss = ccv_nnc_tensor_symbol_new(symbolic_graph, GPU_TENSOR_NCHW(000, 32F, 32), "loss");
    398
    1
        ccv_nnc_graph_exec_symbol_new(symbolic_graph, CMD_SOFTMAX_CROSSENTROPY_FORWARD(), TENSOR_SYMBOL_LIST(y4, label), TENSOR_SYMBOL_LIST(loss, y5), "softmax crossentropy");
    399
    1
        ccv_nnc_graph_exec_symbol_autogen(symbolic_graph, 0, 0, CCV_NNC_AUTOGEN_ALL_EXECS | CCV_NNC_AUTOGEN_SOURCES_AND_DESTINATIONS);
    400
    1
        ccv_nnc_tensor_symbol_t updated_params[4];
    401
    1
        ccv_nnc_tensor_symbol_t gradients[4];
    402
    1
        const int saved_aux_size = ccv_nnc_minimizer_saved_aux_size(CMD_SGD_FORWARD(0, 0.001, 1, 0.99, 0.9, 0.9));
    403
    1
        ccv_nnc_tensor_symbol_map_t saved_aux[saved_aux_size * 4];
    404
    1
        ccv_nnc_graph_exec_symbol_t updated_execs[4];
    405
    1
        ccv_nnc_symbolic_graph_minimize(symbolic_graph, CMD_SGD_FORWARD(0, 0.001, 1, 0.99, 0.9, 0.9), TENSOR_SYMBOL_LIST(loss), TENSOR_SYMBOL_LIST(w1, bias1, w3, bias3), 0, 0, SYMBOLIC_GRAPH_SOURCES(symbolic_graph), SYMBOLIC_GRAPH_DESTINATIONS(symbolic_graph), gradients, updated_params, saved_aux, updated_execs);
    406
    1
        const ccv_nnc_tensor_symbol_t dloss = ccv_nnc_tensor_symbol_for_backward(symbolic_graph, loss);
    407
    1
        ccv_nnc_graph_exec_symbol_new(symbolic_graph, CMD_SET_FORWARD(1), TENSOR_SYMBOL_LIST(), TENSOR_SYMBOL_LIST(dloss), "set 1");
    408
    1
        int i;
    409
    5
        for (i = 0; i < saved_aux_size * 4; i++4)
    410
    4
          ccv_nnc_graph_exec_symbol_new(symbolic_graph, CMD_SET_FORWARD(0), TENSOR_SYMBOL_LIST(), TENSOR_SYMBOL_LIST(saved_aux[i].source), "set 0");
    411
    1
        ccv_nnc_graph_exec_symbol_autogen(symbolic_graph, 0, 0, CCV_NNC_AUTOGEN_SOURCES_AND_DESTINATIONS);
    412
    1
        ccv_nnc_graph_t* graph;
    413
    1
        ccv_nnc_tensor_arena_t* tensor_arena;
    414
    1
        ccv_nnc_graph_exec_arena_t* graph_exec_arena;
    415
    1
        ccv_nnc_symbolic_graph_compile(symbolic_graph, ccv_nnc_default_compile_params,
    416
    1
          0, 0,
    417
    1
          updated_params, 4,
    418
    1
          SYMBOLIC_GRAPH_SOURCES(symbolic_graph), SYMBOLIC_GRAPH_DESTINATIONS(symbolic_graph),
    419
    1
          &graph, &tensor_arena, &graph_exec_arena);
    420
    1
        ccv_nnc_tensor_t* cpu_input = ccv_nnc_tensor_new(0, CPU_TENSOR_NCHW(32F, 32, 3, 32, 32), 0);
    421
    1
        memcpy(cpu_input->data.f32, cpu_inputs[0]->data.f32, sizeof(float) * 16 * 3 * 32 * 32);
    422
    1
        memcpy(cpu_input->data.f32 + 16 * 3 * 32 * 32, cpu_inputs[1]->data.f32, sizeof(float) * 16 * 3 * 32 * 32);
    423
    1
        ccv_nnc_tensor_t* cpu_fit = ccv_nnc_tensor_new(0, CPU_TENSOR_NCHW(32F, 32), 0);
    424
    1
        memcpy(cpu_fit->data.f32, cpu_fits[0]->data.f32, sizeof(float) * 16);
    425
    1
        memcpy(cpu_fit->data.f32 + 16, cpu_fits[1]->data.f32, sizeof(float) * 16);
    426
    1
        ccv_nnc_cmd_exec(CMD_DATA_TRANSFER_FORWARD(), ccv_nnc_no_hint, 0, TENSOR_LIST(cpu_input), TENSOR_LIST(ccv_nnc_tensor_from_symbol(tensor_arena, x)), 0);
    427
    1
        ccv_nnc_cmd_exec(CMD_DATA_TRANSFER_FORWARD(), ccv_nnc_no_hint, 0, TENSOR_LIST(cpu_fit), TENSOR_LIST(ccv_nnc_tensor_from_symbol(tensor_arena, label)), 0);
    428
    1
        ccv_nnc_cmd_exec(CMD_SET_FORWARD(0), ccv_nnc_no_hint, 0, TENSOR_LIST(), TENSOR_LIST(ccv_nnc_tensor_from_symbol(tensor_arena, bias1), ccv_nnc_tensor_from_symbol(tensor_arena, bias3)), 0);
    429
    1
        ccv_nnc_cmd_exec(CMD_DATA_TRANSFER_FORWARD(), ccv_nnc_no_hint, 0, TENSOR_LIST(w1_tensor, w3_tensor), TENSOR_LIST(ccv_nnc_tensor_from_symbol(tensor_arena, w1), ccv_nnc_tensor_from_symbol(tensor_arena, w3)), 0);
    430
    1
        ccv_nnc_graph_run(graph, 0, TRAVERSE_FULL, 0, 0);
    431
    1
        ccv_nnc_tensor_t* np_updated[4];
    432
    1
        np_updated[0] = ccv_nnc_tensor_new(0, CPU_TENSOR_NCHW(32F, 8, 3, 5, 5), 0);
    433
    1
        np_updated[1] = ccv_nnc_tensor_new(0, CPU_TENSOR_NCHW(32F, 8), 0);
    434
    1
        np_updated[2] = ccv_nnc_tensor_new(0, CPU_TENSOR_NCHW(32F, 8, 8, 5, 5), 0);
    435
    1
        np_updated[3] = ccv_nnc_tensor_new(0, CPU_TENSOR_NCHW(32F, 8), 0);
    436
    1
        ccv_nnc_cmd_exec(CMD_DATA_TRANSFER_FORWARD(), ccv_nnc_no_hint, 0, TENSOR_LIST(ccv_nnc_tensor_from_symbol(tensor_arena, updated_params[0]), ccv_nnc_tensor_from_symbol(tensor_arena, updated_params[1]), ccv_nnc_tensor_from_symbol(tensor_arena, updated_params[2]), ccv_nnc_tensor_from_symbol(tensor_arena, updated_params[3])), np_updated, 4, 0);
    437
    1
        ccv_nnc_symbolic_graph_free(symbolic_graph);
    438
    1
        ccv_nnc_graph_free(graph);
    439
    1
        ccv_nnc_tensor_arena_free(tensor_arena);
    440
    1
        ccv_nnc_graph_exec_arena_free(graph_exec_arena);
    441
    1
        REQUIRE_ARRAY_EQ_WITH_TOLERANCE(float, np_updated[0]->data.f32, updated[0]->data.f32, 8 * 3 * 5 * 5, 1e-4, "updated params should be equal");
    442
    1
        REQUIRE_ARRAY_EQ_WITH_TOLERANCE(float, np_updated[1]->data.f32, updated[1]->data.f32, 8, 1e-5, "updated params should be equal");
    443
    1
        REQUIRE_ARRAY_EQ_WITH_TOLERANCE(float, np_updated[2]->data.f32, updated[2]->data.f32, 8 * 8 * 5 * 5, 1e-4, "updated params should be equal");
    444
    1
        REQUIRE_ARRAY_EQ_WITH_TOLERANCE(float, np_updated[3]->data.f32, updated[3]->data.f32, 8, 1e-4, "updated params should be equal");
    445
    1
        ccv_nnc_tensor_free(cpu_input);
    446
    1
        ccv_nnc_tensor_free(cpu_fit);
    447
    1
        ccv_nnc_tensor_free(np_updated[0]);
    448
    1
        ccv_nnc_tensor_free(np_updated[1]);
    449
    1
        ccv_nnc_tensor_free(np_updated[2]);
    450
    1
        ccv_nnc_tensor_free(np_updated[3]);
    451
    1
      }
    452
    0
      ccv_nnc_tensor_free(updated[0]);
    453
    1
      ccv_nnc_tensor_free(updated[1]);
    454
    1
      ccv_nnc_tensor_free(updated[2]);
    455
    1
      ccv_nnc_tensor_free(updated[3]);
    456
    1
      ccv_nnc_tensor_free(cpu_inputs[0]);
    457
    1
      ccv_nnc_tensor_free(cpu_inputs[1]);
    458
    1
      ccv_nnc_tensor_free(cpu_fits[0]);
    459
    1
      ccv_nnc_tensor_free(cpu_fits[1]);
    460
    1
      ccv_nnc_tensor_free(w1_tensor);
    461
    1
      ccv_nnc_tensor_free(w3_tensor);
    462
    1
    }
    463
    464
    TEST_CASE("schedule symbolic graph to data parallel with allreduce")
    465
    1
    {
    466
    1
      GUARD_ELSE_RETURN(ccv_nnc_cmd_ok(CCV_NNC_CONVOLUTION_FORWARD, CCV_NNC_BACKEND_GPU_CUDNN));
    467
    1
      GUARD_ELSE_RETURN(ccv_nnc_cmd_ok(CCV_NNC_COMM_ALLREDUCE_FORWARD, CCV_NNC_BACKEND_GPU_NCCL));
    468
    1
      GUARD_ELSE_RETURN(ccv_nnc_device_count(CCV_STREAM_CONTEXT_GPU) >= 2);
    469
    1
      ccv_nnc_tensor_t* updated[4];
    470
    1
      ccv_nnc_tensor_t* cpu_inputs[2];
    471
    1
      ccv_nnc_tensor_t* cpu_fits[2];
    472
    1
      ccv_nnc_tensor_t* w1_tensor = ccv_nnc_tensor_new(0, CPU_TENSOR_NCHW(32F, 8, 3, 5, 5), 0);
    473
    1
      ccv_nnc_tensor_t* w3_tensor = ccv_nnc_tensor_new(0, CPU_TENSOR_NCHW(32F, 8, 8, 5, 5), 0);
    474
    1
      {
    475
    1
        ccv_nnc_symbolic_graph_t* const symbolic_graph = ccv_nnc_symbolic_graph_new();
    476
    1
        const ccv_nnc_tensor_symbol_t x = ccv_nnc_tensor_symbol_new(symbolic_graph, GPU_TENSOR_NCHW(000, 32F, 16, 3, 32, 32), 0);
    477
    1
        const ccv_nnc_tensor_symbol_t w1 = ccv_nnc_tensor_symbol_new(symbolic_graph, GPU_TENSOR_NCHW(000, 32F, 8, 3, 5, 5), 0);
    478
    1
        const ccv_nnc_tensor_symbol_t bias1 = ccv_nnc_tensor_symbol_new(symbolic_graph, GPU_TENSOR_NCHW(000, 32F, 8), 0);
    479
    1
        const ccv_nnc_tensor_symbol_t y1 = ccv_nnc_tensor_symbol_new(symbolic_graph, GPU_TENSOR_NCHW(000, 32F, 16, 8, 32, 32), 0);
    480
    1
        const ccv_nnc_graph_exec_symbol_t conv1 = ccv_nnc_graph_exec_symbol_new(symbolic_graph, CMD_CONVOLUTION_FORWARD(1, 8, 5, 5, 32), TENSOR_SYMBOL_LIST(x, w1, bias1), TENSOR_SYMBOL_LIST(y1), "conv1");
    481
    1
        ccv_nnc_graph_exec_symbol_set_hint(symbolic_graph, conv1, HINT((1, 1), (2, 2)));
    482
    1
        const ccv_nnc_tensor_symbol_t y2 = ccv_nnc_tensor_symbol_new(symbolic_graph, GPU_TENSOR_NCHW(000, 32F, 16, 8, 16, 16), 0);
    483
    1
        const ccv_nnc_graph_exec_symbol_t avg2 = ccv_nnc_graph_exec_symbol_new(symbolic_graph, CMD_AVERAGE_POOL_FORWARD(2, 2), TENSOR_SYMBOL_LIST(y1), TENSOR_SYMBOL_LIST(y2), "avg2");
    484
    1
        ccv_nnc_graph_exec_symbol_set_hint(symbolic_graph, avg2, HINT((2, 2)));
    485
    1
        const ccv_nnc_tensor_symbol_t w3 = ccv_nnc_tensor_symbol_new(symbolic_graph, GPU_TENSOR_NCHW(000, 32F, 8, 8, 5, 5), 0);
    486
    1
        const ccv_nnc_tensor_symbol_t bias3 = ccv_nnc_tensor_symbol_new(symbolic_graph, GPU_TENSOR_NCHW(000, 32F, 8), 0);
    487
    1
        const ccv_nnc_tensor_symbol_t y3 = ccv_nnc_tensor_symbol_new(symbolic_graph, GPU_TENSOR_NCHW(000, 32F, 16, 8, 8, 8), 0);
    488
    1
        const ccv_nnc_graph_exec_symbol_t conv3 = ccv_nnc_graph_exec_symbol_new(symbolic_graph, CMD_CONVOLUTION_FORWARD(1, 8, 5, 5, 32), TENSOR_SYMBOL_LIST(y2, w3, bias3), TENSOR_SYMBOL_LIST(y3), "conv3");
    489
    1
        ccv_nnc_graph_exec_symbol_set_hint(symbolic_graph, conv3, HINT((2, 2), (2, 2)));
    490
    1
        const ccv_nnc_tensor_symbol_t y4 = ccv_nnc_tensor_symbol_new(symbolic_graph, GPU_TENSOR_NCHW(000, 32F, 16, 8, 1, 1), 0);
    491
    1
        ccv_nnc_graph_exec_symbol_new(symbolic_graph, CMD_AVERAGE_POOL_FORWARD(8, 8), TENSOR_SYMBOL_LIST(y3), TENSOR_SYMBOL_LIST(y4), "avg4");
    492
    1
        const ccv_nnc_tensor_symbol_t y4a = ccv_nnc_tensor_symbol_alias_new(symbolic_graph, y4, ccv_nnc_no_ofs, DIM_ALLOC(8, 1, 1, 1), GPU_TENSOR_NCHW(000, 32F, 16, 8), 0);
    493
    1
        const ccv_nnc_tensor_symbol_t label = ccv_nnc_tensor_symbol_new(symbolic_graph, GPU_TENSOR_NCHW(000, 32F, 16), "label");
    494
    1
        const ccv_nnc_tensor_symbol_t y5 = ccv_nnc_tensor_symbol_new(symbolic_graph, GPU_TENSOR_NCHW(000, 32F, 16, 8), "y5");
    495
    1
        const ccv_nnc_tensor_symbol_t loss = ccv_nnc_tensor_symbol_new(symbolic_graph, GPU_TENSOR_NCHW(000, 32F, 16), "loss");
    496
    1
        ccv_nnc_graph_exec_symbol_new(symbolic_graph, CMD_SOFTMAX_CROSSENTROPY_FORWARD(), TENSOR_SYMBOL_LIST(y4a, label), TENSOR_SYMBOL_LIST(loss, y5), "softmax crossentropy");
    497
    1
        ccv_nnc_graph_exec_symbol_autogen(symbolic_graph, 0, 0, CCV_NNC_AUTOGEN_ALL_EXECS | CCV_NNC_AUTOGEN_SOURCES_AND_DESTINATIONS);
    498
    1
        ccv_nnc_tensor_symbol_t updated_params[4];
    499
    1
        ccv_nnc_tensor_symbol_t gradients[4];
    500
    1
        const int saved_aux_size = ccv_nnc_minimizer_saved_aux_size(CMD_SGD_FORWARD(0, 0.001, 1, 0.99, 0.9, 0.9));
    501
    1
        ccv_nnc_tensor_symbol_map_t saved_aux[saved_aux_size * 4];
    502
    1
        ccv_nnc_graph_exec_symbol_t updated_execs[4];
    503
    1
        ccv_nnc_symbolic_graph_minimize(symbolic_graph, CMD_SGD_FORWARD(0, 0.001, 1, 0.99, 0.9, 0.9), TENSOR_SYMBOL_LIST(loss), TENSOR_SYMBOL_LIST(w1, bias1, w3, bias3), 0, 0, SYMBOLIC_GRAPH_SOURCES(symbolic_graph), SYMBOLIC_GRAPH_DESTINATIONS(symbolic_graph), gradients, updated_params, saved_aux, updated_execs);
    504
    1
        const ccv_nnc_tensor_symbol_t dloss = ccv_nnc_tensor_symbol_for_backward(symbolic_graph, loss);
    505
    1
        ccv_nnc_graph_exec_symbol_new(symbolic_graph, CMD_SET_FORWARD(1), TENSOR_SYMBOL_LIST(), TENSOR_SYMBOL_LIST(dloss), "set 1");
    506
    1
        int i;
    507
    5
        for (i = 0; i < saved_aux_size * 4; i++4)
    508
    4
          ccv_nnc_graph_exec_symbol_new(symbolic_graph, CMD_SET_FORWARD(0), TENSOR_SYMBOL_LIST(), TENSOR_SYMBOL_LIST(saved_aux[i].source), "set 0");
    509
    1
        ccv_nnc_graph_exec_symbol_autogen(symbolic_graph, 0, 0, CCV_NNC_AUTOGEN_ALL_EXECS | CCV_NNC_AUTOGEN_SOURCES_AND_DESTINATIONS);
    510
    1
        ccv_nnc_symbolic_graph_data_parallel(symbolic_graph, 2, TENSOR_SYMBOL_LIST(w1, bias1, w3, bias3), gradients, 4, 0, 0, 0, 0, CCV_NNC_PARALLEL_REDUCE_OP_SUM, SYMBOLIC_GRAPH_SOURCES(symbolic_graph), updated_execs, 4);
    511
    1
        ccv_nnc_graph_exec_symbol_autogen(symbolic_graph, 0, 0, CCV_NNC_AUTOGEN_SOURCES_AND_DESTINATIONS);
    512
    1
        SYMBOLIC_GRAPH_GEN(symbolic_graph, CCV_NNC_LONG_DOT_GRAPH);
    513
    1
        ccv_nnc_graph_t* graph;
    514
    1
        ccv_nnc_tensor_arena_t* tensor_arena;
    515
    1
        ccv_nnc_graph_exec_arena_t* graph_exec_arena;
    516
    1
        ccv_nnc_symbolic_graph_compile(symbolic_graph, ccv_nnc_default_compile_params,
    517
    1
          0, 0,
    518
    1
          updated_params, 4,
    519
    1
          SYMBOLIC_GRAPH_SOURCES(symbolic_graph), SYMBOLIC_GRAPH_DESTINATIONS(symbolic_graph),
    520
    1
          &graph, &tensor_arena, &graph_exec_arena);
    521
    1
        ccv_nnc_graph_set_default_static_schedule(graph, CCV_STREAM_CONTEXT_GPU, 0);
    522
    1
        GRAPH_GEN(graph, CCV_NNC_LONG_DOT_GRAPH);
    523
    1
        cpu_inputs[0] = ccv_nnc_tensor_new(0, CPU_TENSOR_NCHW(32F, 16, 3, 32, 32), 0);
    524
    1
        cpu_inputs[1] = ccv_nnc_tensor_new(0, CPU_TENSOR_NCHW(32F, 16, 3, 32, 32), 0);
    525
    1
        dsfmt_t dsfmt;
    526
    1
        dsfmt_init_gen_rand(&dsfmt, 0);
    527
    49.1k
        for (i = 0; i < 16 * 3 * 32 * 32; i++49.1k)
    528
    49.1k
          cpu_inputs[0]->data.f32[i] = dsfmt_genrand_open_close(&dsfmt);
    529
    49.1k
        for (i = 0; i < 16 * 3 * 32 * 32; i++49.1k)
    530
    49.1k
          cpu_inputs[1]->data.f32[i] = dsfmt_genrand_open_close(&dsfmt);
    531
    1
        cpu_fits[0] = ccv_nnc_tensor_new(0, CPU_TENSOR_NCHW(32F, 16), 0);
    532
    1
        cpu_fits[1] = ccv_nnc_tensor_new(0, CPU_TENSOR_NCHW(32F, 16), 0);
    533
    17
        for (i = 0; i < 16; i++16)
    534
    16
          cpu_fits[0]->data.f32[i] = cpu_fits[1]->data.f32[i] = (int)(dsfmt_genrand_open_close(&dsfmt) * 7.4); // Between 0 to 7.
    535
    1
        ccv_nnc_cmd_exec(CMD_DATA_TRANSFER_FORWARD(), ccv_nnc_no_hint, 0, TENSOR_LIST(cpu_inputs[0], cpu_inputs[1]), TENSOR_LIST(ccv_nnc_tensor_from_symbol(tensor_arena, x), ccv_nnc_tensor_from_symbol(tensor_arena, ccv_nnc_tensor_symbol_copy(symbolic_graph, x, 1))), 0);
    536
    1
        ccv_nnc_cmd_exec(CMD_DATA_TRANSFER_FORWARD(), ccv_nnc_no_hint, 0, TENSOR_LIST(cpu_fits[0], cpu_fits[1]), TENSOR_LIST(ccv_nnc_tensor_from_symbol(tensor_arena, label), ccv_nnc_tensor_from_symbol(tensor_arena, ccv_nnc_tensor_symbol_copy(symbolic_graph, label, 1))), 0);
    537
    1
        ccv_nnc_cmd_exec(CMD_SET_FORWARD(0), ccv_nnc_no_hint, 0, TENSOR_LIST(), TENSOR_LIST(ccv_nnc_tensor_from_symbol(tensor_arena, bias1), ccv_nnc_tensor_from_symbol(tensor_arena, bias3)), 0);
    538
    601
        for (i = 0; i < 8 * 3 * 5 * 5; i++600)
    539
    600
          w1_tensor->data.f32[i] = dsfmt_genrand_open_close(&dsfmt);
    540
    1.60k
        for (i = 0; i < 8 * 8 * 5 * 5; i++1.60k)
    541
    1.60k
          w3_tensor->data.f32[i] = dsfmt_genrand_open_close(&dsfmt);
    542
    1
        ccv_nnc_cmd_exec(CMD_DATA_TRANSFER_FORWARD(), ccv_nnc_no_hint, 0, TENSOR_LIST(w1_tensor, w3_tensor), TENSOR_LIST(ccv_nnc_tensor_from_symbol(tensor_arena, w1), ccv_nnc_tensor_from_symbol(tensor_arena, w3)), 0);
    543
    1
        ccv_nnc_graph_run(graph, 0, TRAVERSE_FULL, 0, ccv_nnc_graph_default_stream(graph));
    544
    1
        ccv_nnc_stream_context_wait(ccv_nnc_graph_default_stream(graph));
    545
    1
        updated[0] = ccv_nnc_tensor_new(0, CPU_TENSOR_NCHW(32F, 8, 3, 5, 5), 0);
    546
    1
        updated[1] = ccv_nnc_tensor_new(0, CPU_TENSOR_NCHW(32F, 8), 0);
    547
    1
        updated[2] = ccv_nnc_tensor_new(0, CPU_TENSOR_NCHW(32F, 8, 8, 5, 5), 0);
    548
    1
        updated[3] = ccv_nnc_tensor_new(0, CPU_TENSOR_NCHW(32F, 8), 0);
    549
    1
        ccv_nnc_cmd_exec(CMD_DATA_TRANSFER_FORWARD(), ccv_nnc_no_hint, 0, TENSOR_LIST(ccv_nnc_tensor_from_symbol(tensor_arena, updated_params[0]), ccv_nnc_tensor_from_symbol(tensor_arena, updated_params[1]), ccv_nnc_tensor_from_symbol(tensor_arena, updated_params[2]), ccv_nnc_tensor_from_symbol(tensor_arena, updated_params[3])), updated, 4, 0);
    550
    1
        ccv_nnc_symbolic_graph_free(symbolic_graph);
    551
    1
        ccv_nnc_graph_free(graph);
    552
    1
        ccv_nnc_tensor_arena_free(tensor_arena);
    553
    1
        ccv_nnc_graph_exec_arena_free(graph_exec_arena);
    554
    1
      }
    555
      // Now, doing exactly the same, but with no parallel.
    556
    1
      {
    557
    1
        ccv_nnc_symbolic_graph_t* const symbolic_graph = ccv_nnc_symbolic_graph_new();
    558
    1
        const ccv_nnc_tensor_symbol_t x = ccv_nnc_tensor_symbol_new(symbolic_graph, GPU_TENSOR_NCHW(000, 32F, 32, 3, 32, 32), 0);
    559
    1
        const ccv_nnc_tensor_symbol_t w1 = ccv_nnc_tensor_symbol_new(symbolic_graph, GPU_TENSOR_NCHW(000, 32F, 8, 3, 5, 5), 0);
    560
    1
        const ccv_nnc_tensor_symbol_t bias1 = ccv_nnc_tensor_symbol_new(symbolic_graph, GPU_TENSOR_NCHW(000, 32F, 8), 0);
    561
    1
        const ccv_nnc_tensor_symbol_t y1 = ccv_nnc_tensor_symbol_new(symbolic_graph, GPU_TENSOR_NCHW(000, 32F, 32, 8, 32, 32), 0);
    562
    1
        const ccv_nnc_graph_exec_symbol_t conv1 = ccv_nnc_graph_exec_symbol_new(symbolic_graph, CMD_CONVOLUTION_FORWARD(1, 8, 5, 5, 32), TENSOR_SYMBOL_LIST(x, w1, bias1), TENSOR_SYMBOL_LIST(y1), "conv1");
    563
    1
        ccv_nnc_graph_exec_symbol_set_hint(symbolic_graph, conv1, HINT((1, 1), (2, 2)));
    564
    1
        const ccv_nnc_tensor_symbol_t y2 = ccv_nnc_tensor_symbol_new(symbolic_graph, GPU_TENSOR_NCHW(000, 32F, 32, 8, 16, 16), 0);
    565
    1
        const ccv_nnc_graph_exec_symbol_t avg2 = ccv_nnc_graph_exec_symbol_new(symbolic_graph, CMD_AVERAGE_POOL_FORWARD(2, 2), TENSOR_SYMBOL_LIST(y1), TENSOR_SYMBOL_LIST(y2), "avg2");
    566
    1
        ccv_nnc_graph_exec_symbol_set_hint(symbolic_graph, avg2, HINT((2, 2)));
    567
    1
        const ccv_nnc_tensor_symbol_t w3 = ccv_nnc_tensor_symbol_new(symbolic_graph, GPU_TENSOR_NCHW(000, 32F, 8, 8, 5, 5), 0);
    568
    1
        const ccv_nnc_tensor_symbol_t bias3 = ccv_nnc_tensor_symbol_new(symbolic_graph, GPU_TENSOR_NCHW(000, 32F, 8), 0);
    569
    1
        const ccv_nnc_tensor_symbol_t y3 = ccv_nnc_tensor_symbol_new(symbolic_graph, GPU_TENSOR_NCHW(000, 32F, 32, 8, 8, 8), 0);
    570
    1
        const ccv_nnc_graph_exec_symbol_t conv3 = ccv_nnc_graph_exec_symbol_new(symbolic_graph, CMD_CONVOLUTION_FORWARD(1, 8, 5, 5, 32), TENSOR_SYMBOL_LIST(y2, w3, bias3), TENSOR_SYMBOL_LIST(y3), "conv3");
    571
    1
        ccv_nnc_graph_exec_symbol_set_hint(symbolic_graph, conv3, HINT((2, 2), (2, 2)));
    572
    1
        const ccv_nnc_tensor_symbol_t y4 = ccv_nnc_tensor_symbol_new(symbolic_graph, GPU_TENSOR_NCHW(000, 32F, 32, 8, 1, 1), 0);
    573
    1
        ccv_nnc_graph_exec_symbol_new(symbolic_graph, CMD_AVERAGE_POOL_FORWARD(8, 8), TENSOR_SYMBOL_LIST(y3), TENSOR_SYMBOL_LIST(y4), "avg4");
    574
    1
        const ccv_nnc_tensor_symbol_t label = ccv_nnc_tensor_symbol_new(symbolic_graph, GPU_TENSOR_NCHW(000, 32F, 32), "label");
    575
    1
        const ccv_nnc_tensor_symbol_t y5 = ccv_nnc_tensor_symbol_new(symbolic_graph, GPU_TENSOR_NCHW(000, 32F, 32, 8), "y5");
    576
    1
        const ccv_nnc_tensor_symbol_t loss = ccv_nnc_tensor_symbol_new(symbolic_graph, GPU_TENSOR_NCHW(000, 32F, 32), "loss");
    577
    1
        ccv_nnc_graph_exec_symbol_new(symbolic_graph, CMD_SOFTMAX_CROSSENTROPY_FORWARD(), TENSOR_SYMBOL_LIST(y4, label), TENSOR_SYMBOL_LIST(loss, y5), "softmax crossentropy");
    578
    1
        ccv_nnc_graph_exec_symbol_autogen(symbolic_graph, 0, 0, CCV_NNC_AUTOGEN_ALL_EXECS | CCV_NNC_AUTOGEN_SOURCES_AND_DESTINATIONS);
    579
    1
        ccv_nnc_tensor_symbol_t updated_params[4];
    580
    1
        ccv_nnc_tensor_symbol_t gradients[4];
    581
    1
        const int saved_aux_size = ccv_nnc_minimizer_saved_aux_size(CMD_SGD_FORWARD(0, 0.001, 1, 0.99, 0.9, 0.9));
    582
    1
        ccv_nnc_tensor_symbol_map_t saved_aux[saved_aux_size * 4];
    583
    1
        ccv_nnc_graph_exec_symbol_t updated_execs[4];
    584
    1
        ccv_nnc_symbolic_graph_minimize(symbolic_graph, CMD_SGD_FORWARD(0, 0.001, 1, 0.99, 0.9, 0.9), TENSOR_SYMBOL_LIST(loss), TENSOR_SYMBOL_LIST(w1, bias1, w3, bias3), 0, 0, SYMBOLIC_GRAPH_SOURCES(symbolic_graph), SYMBOLIC_GRAPH_DESTINATIONS(symbolic_graph), gradients, updated_params, saved_aux, updated_execs);
    585
    1
        const ccv_nnc_tensor_symbol_t dloss = ccv_nnc_tensor_symbol_for_backward(symbolic_graph, loss);
    586
    1
        ccv_nnc_graph_exec_symbol_new(symbolic_graph, CMD_SET_FORWARD(1), TENSOR_SYMBOL_LIST(), TENSOR_SYMBOL_LIST(dloss), "set 1");
    587
    1
        int i;
    588
    5
        for (i = 0; i < saved_aux_size * 4; i++4)
    589
    4
          ccv_nnc_graph_exec_symbol_new(symbolic_graph, CMD_SET_FORWARD(0), TENSOR_SYMBOL_LIST(), TENSOR_SYMBOL_LIST(saved_aux[i].source), "set 0");
    590
    1
        ccv_nnc_graph_exec_symbol_autogen(symbolic_graph, 0, 0, CCV_NNC_AUTOGEN_SOURCES_AND_DESTINATIONS);
    591
    1
        ccv_nnc_graph_t* graph;
    592
    1
        ccv_nnc_tensor_arena_t* tensor_arena;
    593
    1
        ccv_nnc_graph_exec_arena_t* graph_exec_arena;
    594
    1
        ccv_nnc_symbolic_graph_compile(symbolic_graph, ccv_nnc_default_compile_params,
    595
    1
          0, 0,
    596
    1
          updated_params, 4,
    597
    1
          SYMBOLIC_GRAPH_SOURCES(symbolic_graph), SYMBOLIC_GRAPH_DESTINATIONS(symbolic_graph),
    598
    1
          &graph, &tensor_arena, &graph_exec_arena);
    599
    1
        ccv_nnc_tensor_t* cpu_input = ccv_nnc_tensor_new(0, CPU_TENSOR_NCHW(32F, 32, 3, 32, 32), 0);
    600
    1
        memcpy(cpu_input->data.f32, cpu_inputs[0]->data.f32, sizeof(float) * 16 * 3 * 32 * 32);
    601
    1
        memcpy(cpu_input->data.f32 + 16 * 3 * 32 * 32, cpu_inputs[1]->data.f32, sizeof(float) * 16 * 3 * 32 * 32);
    602
    1
        ccv_nnc_tensor_t* cpu_fit = ccv_nnc_tensor_new(0, CPU_TENSOR_NCHW(32F, 32), 0);
    603
    1
        memcpy(cpu_fit->data.f32, cpu_fits[0]->data.f32, sizeof(float) * 16);
    604
    1
        memcpy(cpu_fit->data.f32 + 16, cpu_fits[1]->data.f32, sizeof(float) * 16);
    605
    1
        ccv_nnc_cmd_exec(CMD_DATA_TRANSFER_FORWARD(), ccv_nnc_no_hint, 0, TENSOR_LIST(cpu_input), TENSOR_LIST(ccv_nnc_tensor_from_symbol(tensor_arena, x)), 0);
    606
    1
        ccv_nnc_cmd_exec(CMD_DATA_TRANSFER_FORWARD(), ccv_nnc_no_hint, 0, TENSOR_LIST(cpu_fit), TENSOR_LIST(ccv_nnc_tensor_from_symbol(tensor_arena, label)), 0);
    607
    1
        ccv_nnc_cmd_exec(CMD_SET_FORWARD(0), ccv_nnc_no_hint, 0, TENSOR_LIST(), TENSOR_LIST(ccv_nnc_tensor_from_symbol(tensor_arena, bias1), ccv_nnc_tensor_from_symbol(tensor_arena, bias3)), 0);
    608
    1
        ccv_nnc_cmd_exec(CMD_DATA_TRANSFER_FORWARD(), ccv_nnc_no_hint, 0, TENSOR_LIST(w1_tensor, w3_tensor), TENSOR_LIST(ccv_nnc_tensor_from_symbol(tensor_arena, w1), ccv_nnc_tensor_from_symbol(tensor_arena, w3)), 0);
    609
    1
        ccv_nnc_graph_run(graph, 0, TRAVERSE_FULL, 0, 0);
    610
    1
        ccv_nnc_tensor_t* np_updated[4];
    611
    1
        np_updated[0] = ccv_nnc_tensor_new(0, CPU_TENSOR_NCHW(32F, 8, 3, 5, 5), 0);
    612
    1
        np_updated[1] = ccv_nnc_tensor_new(0, CPU_TENSOR_NCHW(32F, 8), 0);
    613
    1
        np_updated[2] = ccv_nnc_tensor_new(0, CPU_TENSOR_NCHW(32F, 8, 8, 5, 5), 0);
    614
    1
        np_updated[3] = ccv_nnc_tensor_new(0, CPU_TENSOR_NCHW(32F, 8), 0);
    615
    1
        ccv_nnc_cmd_exec(CMD_DATA_TRANSFER_FORWARD(), ccv_nnc_no_hint, 0, TENSOR_LIST(ccv_nnc_tensor_from_symbol(tensor_arena, updated_params[0]), ccv_nnc_tensor_from_symbol(tensor_arena, updated_params[1]), ccv_nnc_tensor_from_symbol(tensor_arena, updated_params[2]), ccv_nnc_tensor_from_symbol(tensor_arena, updated_params[3])), np_updated, 4, 0);
    616
    1
        ccv_nnc_symbolic_graph_free(symbolic_graph);
    617
    1
        ccv_nnc_graph_free(graph);
    618
    1
        ccv_nnc_tensor_arena_free(tensor_arena);
    619
    1
        ccv_nnc_graph_exec_arena_free(graph_exec_arena);
    620
    1
        REQUIRE_ARRAY_EQ_WITH_TOLERANCE(float, np_updated[0]->data.f32, updated[0]->data.f32, 8 * 3 * 5 * 5, 1e-4, "updated params should be equal");
    621
    1
        REQUIRE_ARRAY_EQ_WITH_TOLERANCE(float, np_updated[1]->data.f32, updated[1]->data.f32, 8, 1e-5, "updated params should be equal");
    622
    1
        REQUIRE_ARRAY_EQ_WITH_TOLERANCE(float, np_updated[2]->data.f32, updated[2]->data.f32, 8 * 8 * 5 * 5, 1e-4, "updated params should be equal");
    623
    1
        REQUIRE_ARRAY_EQ_WITH_TOLERANCE(float, np_updated[3]->data.f32, updated[3]->data.f32, 8, 1e-4, "updated params should be equal");
    624
    1
        ccv_nnc_tensor_free(cpu_input);
    625
    1
        ccv_nnc_tensor_free(cpu_fit);
    626
    1
        ccv_nnc_tensor_free(np_updated[0]);
    627
    1
        ccv_nnc_tensor_free(np_updated[1]);
    628
    1
        ccv_nnc_tensor_free(np_updated[2]);
    629
    1
        ccv_nnc_tensor_free(np_updated[3]);
    630
    1
      }
    631
    0
      ccv_nnc_tensor_free(updated[0]);
    632
    1
      ccv_nnc_tensor_free(updated[1]);
    633
    1
      ccv_nnc_tensor_free(updated[2]);
    634
    1
      ccv_nnc_tensor_free(updated[3]);
    635
    1
      ccv_nnc_tensor_free(cpu_inputs[0]);
    636
    1
      ccv_nnc_tensor_free(cpu_inputs[1]);
    637
    1
      ccv_nnc_tensor_free(cpu_fits[0]);
    638
    1
      ccv_nnc_tensor_free(cpu_fits[1]);
    639
    1
      ccv_nnc_tensor_free(w1_tensor);
    640
    1
      ccv_nnc_tensor_free(w3_tensor);
    641
    1
    }
    642
    643
    #include "case_main.h"