## Coverage Report

#### Created: 2026-05-04 15:30

next uncovered line (L), next uncovered region (R), next uncovered branch (B)

    /home/liu/actions-runner/_work/ccv/ccv/test/int/nnc/mpsdnn.tests.c

    Line
    Count
    Source
    1
    #include "case.h"
    2
    #include "ccv_case.h"
    3
    #include "ccv_nnc_case.h"
    4
    #include <ccv.h>
    5
    #include <nnc/ccv_nnc.h>
    6
    #include <nnc/ccv_nnc_easy.h>
    7
    #include <3rdparty/dsfmt/dSFMT.h>
    8
    #include <nnc/ccv_nnc_internal.h>
    9
    #include <math.h>
    10
    11
    TEST_SETUP()
    12
    {
    13
      ccv_nnc_init();
    14
    }
    15
    16
    0
    #define INPUT_DIM (3)
    17
    0
    #define OUTPUT_DIM (96)
    18
    19
    0
    #define INPUT_SIZE (224)
    20
    0
    #define OUTPUT_SIZE (112)
    21
    22
    0
    #define KERNEL_SIZE (7)
    23
    24
    #define BATCH_SIZE (16)
    25
    26
    0
    #define LN_DIM (10)
    27
    0
    #define GN_C_DIM (16)
    28
    #define GN_RC_DIM (4)
    29
    30
    TEST_CASE("mps forward convolution")
    31
    1
    {
    32
    1
      GUARD_ELSE_RETURN(ccv_nnc_cmd_ok(CCV_NNC_CONVOLUTION_FORWARD, CCV_NNC_BACKEND_MPS));
    33
    0
      ccv_nnc_tensor_t* a = ccv_nnc_tensor_new(0, CPU_TENSOR_NHWC(32F, BATCH_SIZE, INPUT_SIZE, INPUT_SIZE, INPUT_DIM), 0);
    34
    0
      ccv_nnc_tensor_t* b = ccv_nnc_tensor_new(0, CPU_TENSOR_NHWC(32F, BATCH_SIZE, OUTPUT_SIZE, OUTPUT_SIZE, OUTPUT_DIM), 0);
    35
    0
      ccv_nnc_cmd_t cmd = CMD_CONVOLUTION_FORWARD(1, OUTPUT_DIM, KERNEL_SIZE, KERNEL_SIZE, INPUT_DIM);
    36
    0
      cmd.backend = CCV_NNC_BACKEND_CPU_REF;
    37
    0
      assert(cmd.backend >= 0);
    38
    0
      ccv_nnc_hint_t hint = ccv_nnc_hint_auto(cmd.info, a->info, b->info);
    39
    0
      assert(ccv_nnc_hint_verify(hint, cmd.info, a->info, b->info) == 0);
    40
    0
      ccv_nnc_tensor_t* w = ccv_nnc_tensor_new(0, CPU_TENSOR_NHWC(32F, OUTPUT_DIM, KERNEL_SIZE, KERNEL_SIZE, INPUT_DIM), 0);
    41
    0
      ccv_nnc_tensor_t* bias = ccv_nnc_tensor_new(0, CPU_TENSOR_NHWC(32F, OUTPUT_DIM), 0);
    42
      // configure the inlets.
    43
    0
      dsfmt_t dsfmt;
    44
    0
      dsfmt_init_gen_rand(&dsfmt, 0);
    45
    0
      int i;
    46
    0
      for (i = 0; i < INPUT_DIM * KERNEL_SIZE * KERNEL_SIZE * OUTPUT_DIM; i++)
    47
    0
        w->data.f32[i] = dsfmt_genrand_open_close(&dsfmt) / (INPUT_DIM * KERNEL_SIZE * KERNEL_SIZE);
    48
    0
      for (i = 0; i < INPUT_SIZE * INPUT_SIZE * INPUT_DIM * ccv_max(1, BATCH_SIZE); i++)
    49
    0
        a->data.f32[i] = dsfmt_genrand_open_close(&dsfmt);
    50
    0
      for (i = 0; i < OUTPUT_DIM; i++)
    51
    0
        bias->data.f32[i] = (float)i / OUTPUT_DIM;
    52
      // Copy generated matrix values over to GPU.
    53
    0
      ccv_nnc_tensor_t* ga = ccv_nnc_tensor_new(0, GPU_TENSOR_NHWC(000, 32F, BATCH_SIZE, INPUT_SIZE, INPUT_SIZE, INPUT_DIM), 0);
    54
    0
      ccv_nnc_tensor_t* gw = ccv_nnc_tensor_new(0, GPU_TENSOR_NHWC(000, 32F, OUTPUT_DIM, KERNEL_SIZE, KERNEL_SIZE, INPUT_DIM), 0);
    55
    0
      ccv_nnc_tensor_t* gwo = ccv_nnc_tensor_new(0, GPU_TENSOR_NCHW(000, 32F, OUTPUT_DIM, INPUT_DIM, KERNEL_SIZE, KERNEL_SIZE), 0);
    56
    0
      ccv_nnc_tensor_t* gbias = ccv_nnc_tensor_new(0, GPU_TENSOR_NHWC(000, 32F, OUTPUT_DIM), 0);
    57
    0
      ccv_nnc_cmd_t move = CMD_DATA_TRANSFER_FORWARD();
    58
    0
      move.backend = CCV_NNC_BACKEND_MPS;
    59
    0
      assert(move.backend >= 0);
    60
    0
      ccv_nnc_cmd_exec(move, ccv_nnc_no_hint, 0, TENSOR_LIST(a, w, bias), TENSOR_LIST(ga, gw, gbias), 0);
    61
    0
      ccv_nnc_cmd_exec(cmd, hint, 0, TENSOR_LIST(a, w, bias), TENSOR_LIST(b), 0);
    62
    0
      ccv_nnc_tensor_t* gc = ccv_nnc_tensor_new(0, GPU_TENSOR_NHWC(000, 32F, BATCH_SIZE, OUTPUT_SIZE, OUTPUT_SIZE, OUTPUT_DIM), 0);
    63
    64
    0
      ccv_nnc_cmd_t transform = CMD_FORMAT_TRANSFORM_FORWARD();
    65
    0
      transform.backend = CCV_NNC_BACKEND_MPS;
    66
    0
      assert(transform.backend >= 0);
    67
    0
      ccv_nnc_stream_context_t* stream_context = ccv_nnc_stream_context_new(CCV_STREAM_CONTEXT_GPU);
    68
    0
      ccv_nnc_cmd_exec(transform, ccv_nnc_no_hint, 0, TENSOR_LIST(gw), TENSOR_LIST(gwo), stream_context);
    69
    0
      ccv_nnc_stream_context_wait(stream_context);
    70
    0
      ccv_nnc_tensor_free(gw);
    71
    72
    0
      cmd.backend = CCV_NNC_BACKEND_MPS;
    73
    0
      assert(cmd.backend >= 0);
    74
    0
      cmd.algorithm = -1;
    75
    0
      cmd = ccv_nnc_cmd_autotune(cmd, 1 * 1024 * 1024 * 1024, hint, 0, TENSOR_LIST(ga, gwo, gbias), TENSOR_LIST(gc), stream_context);
    76
    0
      assert(CCV_NNC_EXEC_SUCCESS == ccv_nnc_cmd_exec(cmd, hint, 0, TENSOR_LIST(ga, gwo, gbias), TENSOR_LIST(gc), stream_context));
    77
    0
      ccv_nnc_stream_context_wait(stream_context);
    78
    0
      ccv_nnc_stream_context_free(stream_context);
    79
    0
      ccv_nnc_tensor_t* c = ccv_nnc_tensor_new(0, CPU_TENSOR_NHWC(32F, BATCH_SIZE, OUTPUT_SIZE, OUTPUT_SIZE, OUTPUT_DIM), 0);
    80
    0
      ccv_nnc_cmd_exec(move, ccv_nnc_no_hint, 0, TENSOR_LIST(gc), TENSOR_LIST(c), 0);
    81
    0
      REQUIRE_ARRAY_EQ_WITH_TOLERANCE(float, b->data.f32, c->data.f32, OUTPUT_DIM * OUTPUT_SIZE * OUTPUT_SIZE, 5e-4, "output from mps should match from CPU");
    82
    0
      ccv_nnc_tensor_free(c);
    83
    0
      ccv_nnc_tensor_free(gc);
    84
    0
      ccv_nnc_tensor_free(bias);
    85
    0
      ccv_nnc_tensor_free(w);
    86
    0
      ccv_nnc_tensor_free(b);
    87
    0
      ccv_nnc_tensor_free(a);
    88
    0
      ccv_nnc_tensor_free(gbias);
    89
    0
      ccv_nnc_tensor_free(gwo);
    90
    0
      ccv_nnc_tensor_free(ga);
    91
    0
    }
    92
    93
    TEST_CASE("mps forward convolution in nchw format")
    94
    1
    {
    95
    1
      GUARD_ELSE_RETURN(ccv_nnc_cmd_ok(CCV_NNC_CONVOLUTION_FORWARD, CCV_NNC_BACKEND_MPS));
    96
    0
      ccv_nnc_tensor_t* a = ccv_nnc_tensor_new(0, CPU_TENSOR_NCHW(32F, BATCH_SIZE, INPUT_DIM, INPUT_SIZE, INPUT_SIZE), 0);
    97
    0
      ccv_nnc_tensor_t* b = ccv_nnc_tensor_new(0, CPU_TENSOR_NCHW(32F, BATCH_SIZE, OUTPUT_DIM, OUTPUT_SIZE, OUTPUT_SIZE), 0);
    98
    0
      ccv_nnc_cmd_t cmd = CMD_CONVOLUTION_FORWARD(1, OUTPUT_DIM, KERNEL_SIZE, KERNEL_SIZE, INPUT_DIM);
    99
    0
      cmd.backend = CCV_NNC_BACKEND_CPU_REF;
    100
    0
      assert(cmd.backend >= 0);
    101
    0
      ccv_nnc_hint_t hint = ccv_nnc_hint_auto(cmd.info, a->info, b->info);
    102
    0
      assert(ccv_nnc_hint_verify(hint, cmd.info, a->info, b->info) == 0);
    103
    0
      ccv_nnc_tensor_t* w = ccv_nnc_tensor_new(0, CPU_TENSOR_NHWC(32F, OUTPUT_DIM, INPUT_DIM, KERNEL_SIZE, KERNEL_SIZE), 0);
    104
    0
      ccv_nnc_tensor_t* bias = ccv_nnc_tensor_new(0, CPU_TENSOR_NHWC(32F, OUTPUT_DIM), 0);
    105
      // configure the inlets.
    106
    0
      dsfmt_t dsfmt;
    107
    0
      dsfmt_init_gen_rand(&dsfmt, 0);
    108
    0
      int i;
    109
    0
      for (i = 0; i < INPUT_DIM * KERNEL_SIZE * KERNEL_SIZE * OUTPUT_DIM; i++)
    110
    0
        w->data.f32[i] = dsfmt_genrand_open_close(&dsfmt) / (INPUT_DIM * KERNEL_SIZE * KERNEL_SIZE);
    111
    0
      for (i = 0; i < INPUT_SIZE * INPUT_SIZE * INPUT_DIM * ccv_max(1, BATCH_SIZE); i++)
    112
    0
        a->data.f32[i] = dsfmt_genrand_open_close(&dsfmt);
    113
    0
      for (i = 0; i < OUTPUT_DIM; i++)
    114
    0
        bias->data.f32[i] = (float)i / OUTPUT_DIM;
    115
      // Copy generated matrix values over to GPU.
    116
    0
      ccv_nnc_tensor_t* ga = ccv_nnc_tensor_new(0, GPU_TENSOR_NCHW(000, 32F, BATCH_SIZE, INPUT_DIM, INPUT_SIZE, INPUT_SIZE), 0);
    117
    0
      ccv_nnc_tensor_t* gw = ccv_nnc_tensor_new(0, GPU_TENSOR_NCHW(000, 32F, OUTPUT_DIM, INPUT_DIM, KERNEL_SIZE, KERNEL_SIZE), 0);
    118
    0
      ccv_nnc_tensor_t* gbias = ccv_nnc_tensor_new(0, GPU_TENSOR_NCHW(000, 32F, OUTPUT_DIM), 0);
    119
    0
      ccv_nnc_cmd_t move = CMD_DATA_TRANSFER_FORWARD();
    120
    0
      move.backend = CCV_NNC_BACKEND_MPS;
    121
    0
      assert(move.backend >= 0);
    122
    0
      ccv_nnc_cmd_exec(move, ccv_nnc_no_hint, 0, TENSOR_LIST(a, w, bias), TENSOR_LIST(ga, gw, gbias), 0);
    123
    0
      ccv_nnc_cmd_exec(cmd, hint, 0, TENSOR_LIST(a, w, bias), TENSOR_LIST(b), 0);
    124
    0
      ccv_nnc_tensor_t* gc = ccv_nnc_tensor_new(0, GPU_TENSOR_NCHW(000, 32F, BATCH_SIZE, OUTPUT_DIM, OUTPUT_SIZE, OUTPUT_SIZE), 0);
    125
    126
    0
      ccv_nnc_cmd_t transform = CMD_FORMAT_TRANSFORM_FORWARD();
    127
    0
      transform.backend = CCV_NNC_BACKEND_MPS;
    128
    0
      assert(transform.backend >= 0);
    129
    0
      cmd.backend = CCV_NNC_BACKEND_MPS;
    130
    0
      assert(cmd.backend >= 0);
    131
    0
      cmd.algorithm = -1;
    132
    0
      cmd = ccv_nnc_cmd_autotune(cmd, 1 * 1024 * 1024 * 1024, hint, 0, TENSOR_LIST(ga, gw, gbias), TENSOR_LIST(gc), 0);
    133
    0
      assert(CCV_NNC_EXEC_SUCCESS == ccv_nnc_cmd_exec(cmd, hint, 0, TENSOR_LIST(ga, gw, gbias), TENSOR_LIST(gc), 0));
    134
    0
      ccv_nnc_tensor_t* c = ccv_nnc_tensor_new(0, CPU_TENSOR_NCHW(32F, BATCH_SIZE, OUTPUT_DIM, OUTPUT_SIZE, OUTPUT_SIZE), 0);
    135
    0
      ccv_nnc_cmd_exec(move, ccv_nnc_no_hint, 0, TENSOR_LIST(gc), TENSOR_LIST(c), 0);
    136
    0
      REQUIRE_ARRAY_EQ_WITH_TOLERANCE(float, b->data.f32, c->data.f32, OUTPUT_DIM * OUTPUT_SIZE * OUTPUT_SIZE, 1e-3, "output from mps should match from CPU");
    137
    0
      ccv_nnc_tensor_free(c);
    138
    0
      ccv_nnc_tensor_free(gc);
    139
    0
      ccv_nnc_tensor_free(bias);
    140
    0
      ccv_nnc_tensor_free(w);
    141
    0
      ccv_nnc_tensor_free(b);
    142
    0
      ccv_nnc_tensor_free(a);
    143
    0
      ccv_nnc_tensor_free(gbias);
    144
    0
      ccv_nnc_tensor_free(gw);
    145
    0
      ccv_nnc_tensor_free(ga);
    146
    0
    }
    147
    148
    TEST_CASE("mps forward grouped convolution falls back with mfa enabled")
    149
    1
    {
    150
    1
      GUARD_ELSE_RETURN(ccv_nnc_cmd_ok(CCV_NNC_CONVOLUTION_FORWARD, CCV_NNC_BACKEND_MPS));
    151
    0
      const int groups = 2;
    152
    0
      const int input_dim = 4;
    153
    0
      const int output_dim = 6;
    154
    0
      const int batch_size = 2;
    155
    0
      const int input_size = 8;
    156
    0
      const int kernel_size = 3;
    157
    0
      const int output_size = input_size - kernel_size + 1;
    158
    0
      ccv_nnc_tensor_t* a = ccv_nnc_tensor_new(0, CPU_TENSOR_NCHW(32F, batch_size, input_dim, input_size, input_size), 0);
    159
    0
      ccv_nnc_tensor_t* b = ccv_nnc_tensor_new(0, CPU_TENSOR_NCHW(32F, batch_size, output_dim, output_size, output_size), 0);
    160
    0
      ccv_nnc_tensor_t* w = ccv_nnc_tensor_new(0, CPU_TENSOR_NCHW(32F, output_dim, input_dim / groups, kernel_size, kernel_size), 0);
    161
    0
      ccv_nnc_cmd_t cmd = CMD_CONVOLUTION_FORWARD(groups, output_dim, kernel_size, kernel_size, input_dim / groups);
    162
    0
      ccv_nnc_hint_t hint = HINT((1, 1), (0, 0));
    163
    0
      dsfmt_t dsfmt;
    164
    0
      dsfmt_init_gen_rand(&dsfmt, 0);
    165
    0
      int i;
    166
    0
      for (i = 0; i < batch_size * input_dim * input_size * input_size; i++)
    167
    0
        a->data.f32[i] = dsfmt_genrand_open_close(&dsfmt);
    168
    0
      for (i = 0; i < output_dim * (input_dim / groups) * kernel_size * kernel_size; i++)
    169
    0
        w->data.f32[i] = dsfmt_genrand_open_close(&dsfmt) / (input_dim / groups * kernel_size * kernel_size);
    170
    0
      ccv_nnc_cmd_t cpu_cmd = cmd;
    171
    0
      cpu_cmd.backend = CCV_NNC_BACKEND_CPU_REF;
    172
    0
      assert(cpu_cmd.backend >= 0);
    173
    0
      ccv_nnc_cmd_exec(cpu_cmd, hint, 0, TENSOR_LIST(a, w), TENSOR_LIST(b), 0);
    174
    0
      ccv_nnc_tensor_t* ga = ccv_nnc_tensor_new(0, GPU_TENSOR_NCHW(000, 32F, batch_size, input_dim, input_size, input_size), 0);
    175
    0
      ccv_nnc_tensor_t* gw = ccv_nnc_tensor_new(0, GPU_TENSOR_NCHW(000, 32F, output_dim, input_dim / groups, kernel_size, kernel_size), 0);
    176
    0
      ccv_nnc_tensor_t* gc = ccv_nnc_tensor_new(0, GPU_TENSOR_NCHW(000, 32F, batch_size, output_dim, output_size, output_size), 0);
    177
    0
      ccv_nnc_cmd_t move = CMD_DATA_TRANSFER_FORWARD();
    178
    0
      move.backend = CCV_NNC_BACKEND_MPS;
    179
    0
      assert(move.backend >= 0);
    180
    0
      ccv_nnc_cmd_exec(move, ccv_nnc_no_hint, 0, TENSOR_LIST(a, w), TENSOR_LIST(ga, gw), 0);
    181
    0
      ccv_nnc_cmd_t mps_cmd = cmd;
    182
    0
      mps_cmd.backend = CCV_NNC_BACKEND_MPS;
    183
    0
      assert(mps_cmd.backend >= 0);
    184
    0
      assert(CCV_NNC_EXEC_SUCCESS == ccv_nnc_cmd_exec(mps_cmd, hint, 0, TENSOR_LIST(ga, gw), TENSOR_LIST(gc), 0));
    185
    0
      ccv_nnc_tensor_t* c = ccv_nnc_tensor_new(0, CPU_TENSOR_NCHW(32F, batch_size, output_dim, output_size, output_size), 0);
    186
    0
      ccv_nnc_cmd_exec(move, ccv_nnc_no_hint, 0, TENSOR_LIST(gc), TENSOR_LIST(c), 0);
    187
    0
      REQUIRE_ARRAY_EQ_WITH_TOLERANCE(float, b->data.f32, c->data.f32, batch_size * output_dim * output_size * output_size, 1e-5, "grouped convolution output from mps should match from CPU");
    188
    0
      ccv_nnc_tensor_free(c);
    189
    0
      ccv_nnc_tensor_free(gc);
    190
    0
      ccv_nnc_tensor_free(gw);
    191
    0
      ccv_nnc_tensor_free(ga);
    192
    0
      ccv_nnc_tensor_free(w);
    193
    0
      ccv_nnc_tensor_free(b);
    194
    0
      ccv_nnc_tensor_free(a);
    195
    0
    }
    196
    197
    TEST_CASE("mps forward convolution with 1x1 kernel")
    198
    1
    {
    199
    1
      GUARD_ELSE_RETURN(ccv_nnc_cmd_ok(CCV_NNC_CONVOLUTION_FORWARD, CCV_NNC_BACKEND_MPS));
    200
    0
      ccv_nnc_tensor_t* a = ccv_nnc_tensor_new(0, CPU_TENSOR_NHWC(32F, BATCH_SIZE, OUTPUT_SIZE, OUTPUT_SIZE, INPUT_DIM), 0);
    201
    0
      ccv_nnc_tensor_t* b = ccv_nnc_tensor_new(0, CPU_TENSOR_NHWC(32F, BATCH_SIZE, OUTPUT_SIZE, OUTPUT_SIZE, OUTPUT_DIM), 0);
    202
    0
      ccv_nnc_cmd_t cmd = CMD_CONVOLUTION_FORWARD(1, OUTPUT_DIM, 1, 1, INPUT_DIM);
    203
    0
      cmd.backend = CCV_NNC_BACKEND_CPU_REF;
    204
    0
      assert(cmd.backend >= 0);
    205
    0
      ccv_nnc_hint_t hint = ccv_nnc_hint_auto(cmd.info, a->info, b->info);
    206
    0
      assert(ccv_nnc_hint_verify(hint, cmd.info, a->info, b->info) == 0);
    207
    0
      ccv_nnc_tensor_t* w = ccv_nnc_tensor_new(0, CPU_TENSOR_NHWC(32F, OUTPUT_DIM, 1, 1, INPUT_DIM), 0);
    208
    0
      ccv_nnc_tensor_t* bias = ccv_nnc_tensor_new(0, CPU_TENSOR_NHWC(32F, OUTPUT_DIM), 0);
    209
      // configure the inlets.
    210
    0
      dsfmt_t dsfmt;
    211
    0
      dsfmt_init_gen_rand(&dsfmt, 0);
    212
    0
      int i;
    213
    0
      for (i = 0; i < INPUT_DIM * 1 * 1 * OUTPUT_DIM; i++)
    214
    0
        w->data.f32[i] = dsfmt_genrand_open_close(&dsfmt) / (INPUT_DIM * 1 * 1);
    215
    0
      for (i = 0; i < OUTPUT_SIZE * OUTPUT_SIZE * INPUT_DIM * ccv_max(1, BATCH_SIZE); i++)
    216
    0
        a->data.f32[i] = dsfmt_genrand_open_close(&dsfmt);
    217
    0
      for (i = 0; i < OUTPUT_DIM; i++)
    218
    0
        bias->data.f32[i] = (float)i / OUTPUT_DIM;
    219
      // Copy generated matrix values over to GPU.
    220
    0
      ccv_nnc_tensor_t* ga = ccv_nnc_tensor_new(0, GPU_TENSOR_NHWC(000, 32F, BATCH_SIZE, OUTPUT_SIZE, OUTPUT_SIZE, INPUT_DIM), 0);
    221
    0
      ccv_nnc_tensor_t* gw = ccv_nnc_tensor_new(0, GPU_TENSOR_NHWC(000, 32F, OUTPUT_DIM, 1, 1, INPUT_DIM), 0);
    222
    0
      ccv_nnc_tensor_t* gwo = ccv_nnc_tensor_new(0, GPU_TENSOR_NCHW(000, 32F, OUTPUT_DIM, INPUT_DIM, 1, 1), 0);
    223
    0
      ccv_nnc_tensor_t* gbias = ccv_nnc_tensor_new(0, GPU_TENSOR_NHWC(000, 32F, OUTPUT_DIM), 0);
    224
    0
      ccv_nnc_cmd_t move = CMD_DATA_TRANSFER_FORWARD();
    225
    0
      move.backend = CCV_NNC_BACKEND_MPS;
    226
    0
      assert(move.backend >= 0);
    227
    0
      ccv_nnc_cmd_exec(move, ccv_nnc_no_hint, 0, TENSOR_LIST(a, w, bias), TENSOR_LIST(ga, gw, gbias), 0);
    228
    0
      ccv_nnc_cmd_exec(cmd, hint, 0, TENSOR_LIST(a, w, bias), TENSOR_LIST(b), 0);
    229
    0
      ccv_nnc_tensor_t* gc = ccv_nnc_tensor_new(0, GPU_TENSOR_NHWC(000, 32F, BATCH_SIZE, OUTPUT_SIZE, OUTPUT_SIZE, OUTPUT_DIM), 0);
    230
    231
    0
      ccv_nnc_cmd_t transform = CMD_FORMAT_TRANSFORM_FORWARD();
    232
    0
      transform.backend = CCV_NNC_BACKEND_MPS;
    233
    0
      assert(transform.backend >= 0);
    234
    0
      ccv_nnc_stream_context_t* stream_context = ccv_nnc_stream_context_new(CCV_STREAM_CONTEXT_GPU);
    235
    0
      ccv_nnc_cmd_exec(transform, ccv_nnc_no_hint, 0, TENSOR_LIST(gw), TENSOR_LIST(gwo), stream_context);
    236
    0
      ccv_nnc_stream_context_wait(stream_context);
    237
    0
      ccv_nnc_tensor_free(gw);
    238
    239
    0
      cmd.backend = CCV_NNC_BACKEND_MPS;
    240
    0
      assert(cmd.backend >= 0);
    241
    0
      cmd.algorithm = -1;
    242
    0
      cmd = ccv_nnc_cmd_autotune(cmd, 1 * 1024 * 1024 * 1024, hint, 0, TENSOR_LIST(ga, gwo, gbias), TENSOR_LIST(gc), stream_context);
    243
    0
      assert(CCV_NNC_EXEC_SUCCESS == ccv_nnc_cmd_exec(cmd, hint, 0, TENSOR_LIST(ga, gwo, gbias), TENSOR_LIST(gc), stream_context));
    244
    0
      ccv_nnc_stream_context_wait(stream_context);
    245
    0
      ccv_nnc_stream_context_free(stream_context);
    246
    0
      ccv_nnc_tensor_t* c = ccv_nnc_tensor_new(0, CPU_TENSOR_NHWC(32F, BATCH_SIZE, OUTPUT_SIZE, OUTPUT_SIZE, OUTPUT_DIM), 0);
    247
    0
      ccv_nnc_cmd_exec(move, ccv_nnc_no_hint, 0, TENSOR_LIST(gc), TENSOR_LIST(c), 0);
    248
    0
      REQUIRE_ARRAY_EQ_WITH_TOLERANCE(float, b->data.f32, c->data.f32, OUTPUT_DIM * OUTPUT_SIZE * OUTPUT_SIZE, 1e-3, "output from mps should match from CPU");
    249
    0
      ccv_nnc_tensor_free(c);
    250
    0
      ccv_nnc_tensor_free(gc);
    251
    0
      ccv_nnc_tensor_free(bias);
    252
    0
      ccv_nnc_tensor_free(w);
    253
    0
      ccv_nnc_tensor_free(b);
    254
    0
      ccv_nnc_tensor_free(a);
    255
    0
      ccv_nnc_tensor_free(gbias);
    256
    0
      ccv_nnc_tensor_free(gwo);
    257
    0
      ccv_nnc_tensor_free(ga);
    258
    0
    }
    259
    260
    TEST_CASE("mps forward convolution in nchw format with 1x1 kernel")
    261
    1
    {
    262
    1
      GUARD_ELSE_RETURN(ccv_nnc_cmd_ok(CCV_NNC_CONVOLUTION_FORWARD, CCV_NNC_BACKEND_MPS));
    263
    0
      ccv_nnc_tensor_t* a = ccv_nnc_tensor_new(0, CPU_TENSOR_NCHW(32F, BATCH_SIZE, INPUT_DIM, OUTPUT_SIZE, OUTPUT_SIZE), 0);
    264
    0
      ccv_nnc_tensor_t* b = ccv_nnc_tensor_new(0, CPU_TENSOR_NCHW(32F, BATCH_SIZE, OUTPUT_DIM, OUTPUT_SIZE, OUTPUT_SIZE), 0);
    265
    0
      ccv_nnc_cmd_t cmd = CMD_CONVOLUTION_FORWARD(1, OUTPUT_DIM, 1, 1, INPUT_DIM);
    266
    0
      cmd.backend = CCV_NNC_BACKEND_CPU_REF;
    267
    0
      assert(cmd.backend >= 0);
    268
    0
      ccv_nnc_hint_t hint = ccv_nnc_hint_auto(cmd.info, a->info, b->info);
    269
    0
      assert(ccv_nnc_hint_verify(hint, cmd.info, a->info, b->info) == 0);
    270
    0
      ccv_nnc_tensor_t* w = ccv_nnc_tensor_new(0, CPU_TENSOR_NHWC(32F, OUTPUT_DIM, INPUT_DIM, 1, 1), 0);
    271
    0
      ccv_nnc_tensor_t* bias = ccv_nnc_tensor_new(0, CPU_TENSOR_NHWC(32F, OUTPUT_DIM), 0);
    272
      // configure the inlets.
    273
    0
      dsfmt_t dsfmt;
    274
    0
      dsfmt_init_gen_rand(&dsfmt, 0);
    275
    0
      int i;
    276
    0
      for (i = 0; i < INPUT_DIM * 1 * 1 * OUTPUT_DIM; i++)
    277
    0
        w->data.f32[i] = dsfmt_genrand_open_close(&dsfmt) / (INPUT_DIM * 1 * 1);
    278
    0
      for (i = 0; i < OUTPUT_SIZE * OUTPUT_SIZE * INPUT_DIM * ccv_max(1, BATCH_SIZE); i++)
    279
    0
        a->data.f32[i] = dsfmt_genrand_open_close(&dsfmt);
    280
    0
      for (i = 0; i < OUTPUT_DIM; i++)
    281
    0
        bias->data.f32[i] = (float)i / OUTPUT_DIM;
    282
      // Copy generated matrix values over to GPU.
    283
    0
      ccv_nnc_tensor_t* ga = ccv_nnc_tensor_new(0, GPU_TENSOR_NCHW(000, 32F, BATCH_SIZE, INPUT_DIM, OUTPUT_SIZE, OUTPUT_SIZE), 0);
    284
    0
      ccv_nnc_tensor_t* gw = ccv_nnc_tensor_new(0, GPU_TENSOR_NCHW(000, 32F, OUTPUT_DIM, INPUT_DIM, 1, 1), 0);
    285
    0
      ccv_nnc_tensor_t* gbias = ccv_nnc_tensor_new(0, GPU_TENSOR_NCHW(000, 32F, OUTPUT_DIM), 0);
    286
    0
      ccv_nnc_cmd_t move = CMD_DATA_TRANSFER_FORWARD();
    287
    0
      move.backend = CCV_NNC_BACKEND_MPS;
    288
    0
      assert(move.backend >= 0);
    289
    0
      ccv_nnc_cmd_exec(move, ccv_nnc_no_hint, 0, TENSOR_LIST(a, w, bias), TENSOR_LIST(ga, gw, gbias), 0);
    290
    0
      ccv_nnc_cmd_exec(cmd, hint, 0, TENSOR_LIST(a, w, bias), TENSOR_LIST(b), 0);
    291
    0
      ccv_nnc_tensor_t* gc = ccv_nnc_tensor_new(0, GPU_TENSOR_NCHW(000, 32F, BATCH_SIZE, OUTPUT_DIM, OUTPUT_SIZE, OUTPUT_SIZE), 0);
    292
    293
    0
      ccv_nnc_cmd_t transform = CMD_FORMAT_TRANSFORM_FORWARD();
    294
    0
      transform.backend = CCV_NNC_BACKEND_MPS;
    295
    0
      assert(transform.backend >= 0);
    296
    0
      cmd.backend = CCV_NNC_BACKEND_MPS;
    297
    0
      assert(cmd.backend >= 0);
    298
    0
      cmd.algorithm = -1;
    299
    0
      cmd = ccv_nnc_cmd_autotune(cmd, 1 * 1024 * 1024 * 1024, hint, 0, TENSOR_LIST(ga, gw, gbias), TENSOR_LIST(gc), 0);
    300
    0
      assert(CCV_NNC_EXEC_SUCCESS == ccv_nnc_cmd_exec(cmd, hint, 0, TENSOR_LIST(ga, gw, gbias), TENSOR_LIST(gc), 0));
    301
    0
      ccv_nnc_tensor_t* c = ccv_nnc_tensor_new(0, CPU_TENSOR_NCHW(32F, BATCH_SIZE, OUTPUT_DIM, OUTPUT_SIZE, OUTPUT_SIZE), 0);
    302
    0
      ccv_nnc_cmd_exec(move, ccv_nnc_no_hint, 0, TENSOR_LIST(gc), TENSOR_LIST(c), 0);
    303
    0
      REQUIRE_ARRAY_EQ_WITH_TOLERANCE(float, b->data.f32, c->data.f32, OUTPUT_DIM * OUTPUT_SIZE * OUTPUT_SIZE, 1e-3, "output from mps should match from CPU");
    304
    0
      ccv_nnc_tensor_free(c);
    305
    0
      ccv_nnc_tensor_free(gc);
    306
    0
      ccv_nnc_tensor_free(bias);
    307
    0
      ccv_nnc_tensor_free(w);
    308
    0
      ccv_nnc_tensor_free(b);
    309
    0
      ccv_nnc_tensor_free(a);
    310
    0
      ccv_nnc_tensor_free(gbias);
    311
    0
      ccv_nnc_tensor_free(gw);
    312
    0
      ccv_nnc_tensor_free(ga);
    313
    0
    }
    314
    315
    TEST_CASE("mps forward convolution in nchw format with row-wise 8i weight")
    316
    1
    {
    317
    1
      GUARD_ELSE_RETURN(ccv_nnc_cmd_ok(CCV_NNC_CONVOLUTION_FORWARD, CCV_NNC_BACKEND_MPS));
    318
    0
      const int batch_size = 2;
    319
    0
      const int input_dim = 8;
    320
    0
      const int output_dim = 12;
    321
    0
      const int spatial = 9;
    322
    0
      ccv_nnc_tensor_t* a = ccv_nnc_tensor_new(0, CPU_TENSOR_NCHW(32F, batch_size, input_dim, spatial, spatial), 0);
    323
    0
      ccv_nnc_tensor_t* b = ccv_nnc_tensor_new(0, CPU_TENSOR_NCHW(32F, batch_size, output_dim, spatial, spatial), 0);
    324
    0
      ccv_nnc_cmd_t cmd = CMD_CONVOLUTION_FORWARD(1, output_dim, 1, 1, input_dim);
    325
    0
      cmd.backend = CCV_NNC_BACKEND_CPU_REF;
    326
    0
      assert(cmd.backend >= 0);
    327
    0
      ccv_nnc_hint_t hint = ccv_nnc_hint_auto(cmd.info, a->info, b->info);
    328
    0
      assert(ccv_nnc_hint_verify(hint, cmd.info, a->info, b->info) == 0);
    329
    0
      ccv_nnc_tensor_t* w = ccv_nnc_tensor_new(0, CPU_TENSOR_NCHW(32F, output_dim, input_dim, 1, 1), 0);
    330
    0
      ccv_nnc_tensor_t* wq = ccv_nnc_tensor_new(0, ccv_nnc_tensor_8i_rowwise(CPU_TENSOR_NCHW(32F, output_dim, input_dim, 1, 1)), 0);
    331
    0
      ccv_nnc_tensor_t* bias = ccv_nnc_tensor_new(0, CPU_TENSOR_NHWC(32F, output_dim), 0);
    332
    0
      dsfmt_t dsfmt;
    333
    0
      dsfmt_init_gen_rand(&dsfmt, 0);
    334
    0
      int i;
    335
    0
      for (i = 0; i < batch_size * input_dim * spatial * spatial; i++)
    336
    0
        a->data.f32[i] = dsfmt_genrand_open_close(&dsfmt);
    337
    0
      for (i = 0; i < output_dim * input_dim; i++)
    338
    0
        w->data.f32[i] = dsfmt_genrand_open_close(&dsfmt) - 0.5f;
    339
    0
      for (i = 0; i < output_dim; i++)
    340
    0
        bias->data.f32[i] = (float)i / output_dim;
    341
    0
      const size_t qsize = ccv_nnc_quantize_8i_rowwise(w->data.f32, CCV_32F, CCV_TENSOR_CPU_MEMORY, output_dim * input_dim, 1, wq->data.u8, ccv_nnc_tensor_data_size_without_padding(wq->info));
    342
    0
      REQUIRE_EQ(qsize, ccv_nnc_tensor_data_size_without_padding(wq->info), "row-wise 8i convolution weight should fit the tensor exactly");
    343
    0
      ccv_nnc_cmd_exec(cmd, hint, 0, TENSOR_LIST(a, w, bias), TENSOR_LIST(b), 0);
    344
    0
      ccv_nnc_tensor_t* ga = ccv_nnc_tensor_new(0, GPU_TENSOR_NCHW(000, 32F, batch_size, input_dim, spatial, spatial), 0);
    345
    0
      ccv_nnc_tensor_t* gwq = ccv_nnc_tensor_new(0, ccv_nnc_tensor_8i_rowwise(GPU_TENSOR_NCHW(000, 32F, output_dim, input_dim, 1, 1)), 0);
    346
    0
      ccv_nnc_tensor_t* gbias = ccv_nnc_tensor_new(0, GPU_TENSOR_NCHW(000, 32F, output_dim), 0);
    347
    0
      ccv_nnc_tensor_t* gc = ccv_nnc_tensor_new(0, GPU_TENSOR_NCHW(000, 32F, batch_size, output_dim, spatial, spatial), 0);
    348
    0
      ccv_nnc_cmd_t move = CMD_DATA_TRANSFER_FORWARD();
    349
    0
      move.backend = CCV_NNC_BACKEND_MPS;
    350
    0
      assert(move.backend >= 0);
    351
    0
      ccv_nnc_cmd_exec(move, ccv_nnc_no_hint, 0, TENSOR_LIST(a, wq, bias), TENSOR_LIST(ga, gwq, gbias), 0);
    352
    0
      cmd.backend = CCV_NNC_BACKEND_MPS;
    353
    0
      assert(cmd.backend >= 0);
    354
    0
      cmd.algorithm = -1;
    355
    0
      cmd = ccv_nnc_cmd_autotune(cmd, 1 * 1024 * 1024 * 1024, hint, 0, TENSOR_LIST(ga, gwq, gbias), TENSOR_LIST(gc), 0);
    356
    0
      assert(CCV_NNC_EXEC_SUCCESS == ccv_nnc_cmd_exec(cmd, hint, 0, TENSOR_LIST(ga, gwq, gbias), TENSOR_LIST(gc), 0));
    357
    0
      ccv_nnc_tensor_t* c = ccv_nnc_tensor_new(0, CPU_TENSOR_NCHW(32F, batch_size, output_dim, spatial, spatial), 0);
    358
    0
      ccv_nnc_cmd_exec(move, ccv_nnc_no_hint, 0, TENSOR_LIST(gc), TENSOR_LIST(c), 0);
    359
    0
      REQUIRE_ARRAY_EQ_WITH_TOLERANCE(float, b->data.f32, c->data.f32, batch_size * output_dim * spatial * spatial, 2e-3, "output from mps should match CPU when row-wise 8i weights are dequantized to dense scratch");
    360
    0
      ccv_nnc_tensor_free(c);
    361
    0
      ccv_nnc_tensor_free(gc);
    362
    0
      ccv_nnc_tensor_free(gbias);
    363
    0
      ccv_nnc_tensor_free(gwq);
    364
    0
      ccv_nnc_tensor_free(ga);
    365
    0
      ccv_nnc_tensor_free(bias);
    366
    0
      ccv_nnc_tensor_free(wq);
    367
    0
      ccv_nnc_tensor_free(w);
    368
    0
      ccv_nnc_tensor_free(b);
    369
    0
      ccv_nnc_tensor_free(a);
    370
    0
    }
    371
    372
    TEST_CASE("mps forward convolution in nchw format with 1x1 kernel and no bias")
    373
    1
    {
    374
    1
      GUARD_ELSE_RETURN(ccv_nnc_cmd_ok(CCV_NNC_CONVOLUTION_FORWARD, CCV_NNC_BACKEND_MPS));
    375
    0
      ccv_nnc_tensor_t* a = ccv_nnc_tensor_new(0, CPU_TENSOR_NCHW(32F, BATCH_SIZE, INPUT_DIM, OUTPUT_SIZE, OUTPUT_SIZE), 0);
    376
    0
      ccv_nnc_tensor_t* b = ccv_nnc_tensor_new(0, CPU_TENSOR_NCHW(32F, BATCH_SIZE, OUTPUT_DIM, OUTPUT_SIZE, OUTPUT_SIZE), 0);
    377
    0
      ccv_nnc_cmd_t cmd = CMD_CONVOLUTION_FORWARD(1, OUTPUT_DIM, 1, 1, INPUT_DIM);
    378
    0
      cmd.backend = CCV_NNC_BACKEND_CPU_REF;
    379
    0
      assert(cmd.backend >= 0);
    380
    0
      ccv_nnc_hint_t hint = ccv_nnc_hint_auto(cmd.info, a->info, b->info);
    381
    0
      assert(ccv_nnc_hint_verify(hint, cmd.info, a->info, b->info) == 0);
    382
    0
      ccv_nnc_tensor_t* w = ccv_nnc_tensor_new(0, CPU_TENSOR_NHWC(32F, OUTPUT_DIM, INPUT_DIM, 1, 1), 0);
    383
    0
      dsfmt_t dsfmt;
    384
    0
      dsfmt_init_gen_rand(&dsfmt, 1);
    385
    0
      int i;
    386
    0
      for (i = 0; i < INPUT_DIM * 1 * 1 * OUTPUT_DIM; i++)
    387
    0
        w->data.f32[i] = dsfmt_genrand_open_close(&dsfmt) / (INPUT_DIM * 1 * 1);
    388
    0
      for (i = 0; i < OUTPUT_SIZE * OUTPUT_SIZE * INPUT_DIM * ccv_max(1, BATCH_SIZE); i++)
    389
    0
        a->data.f32[i] = dsfmt_genrand_open_close(&dsfmt);
    390
    0
      ccv_nnc_tensor_t* ga = ccv_nnc_tensor_new(0, GPU_TENSOR_NCHW(000, 32F, BATCH_SIZE, INPUT_DIM, OUTPUT_SIZE, OUTPUT_SIZE), 0);
    391
    0
      ccv_nnc_tensor_t* gw = ccv_nnc_tensor_new(0, GPU_TENSOR_NCHW(000, 32F, OUTPUT_DIM, INPUT_DIM, 1, 1), 0);
    392
    0
      ccv_nnc_cmd_t move = CMD_DATA_TRANSFER_FORWARD();
    393
    0
      move.backend = CCV_NNC_BACKEND_MPS;
    394
    0
      assert(move.backend >= 0);
    395
    0
      ccv_nnc_cmd_exec(move, ccv_nnc_no_hint, 0, TENSOR_LIST(a, w), TENSOR_LIST(ga, gw), 0);
    396
    0
      ccv_nnc_cmd_exec(cmd, hint, 0, TENSOR_LIST(a, w), TENSOR_LIST(b), 0);
    397
    0
      ccv_nnc_tensor_t* gc = ccv_nnc_tensor_new(0, GPU_TENSOR_NCHW(000, 32F, BATCH_SIZE, OUTPUT_DIM, OUTPUT_SIZE, OUTPUT_SIZE), 0);
    398
    399
    0
      cmd.backend = CCV_NNC_BACKEND_MPS;
    400
    0
      assert(cmd.backend >= 0);
    401
    0
      cmd.algorithm = -1;
    402
    0
      cmd = ccv_nnc_cmd_autotune(cmd, 1 * 1024 * 1024 * 1024, hint, 0, TENSOR_LIST(ga, gw), TENSOR_LIST(gc), 0);
    403
    0
      assert(CCV_NNC_EXEC_SUCCESS == ccv_nnc_cmd_exec(cmd, hint, 0, TENSOR_LIST(ga, gw), TENSOR_LIST(gc), 0));
    404
    0
      ccv_nnc_tensor_t* c = ccv_nnc_tensor_new(0, CPU_TENSOR_NCHW(32F, BATCH_SIZE, OUTPUT_DIM, OUTPUT_SIZE, OUTPUT_SIZE), 0);
    405
    0
      ccv_nnc_cmd_exec(move, ccv_nnc_no_hint, 0, TENSOR_LIST(gc), TENSOR_LIST(c), 0);
    406
    0
      REQUIRE_ARRAY_EQ_WITH_TOLERANCE(float, b->data.f32, c->data.f32, OUTPUT_DIM * OUTPUT_SIZE * OUTPUT_SIZE, 1e-3, "output from mps should match from CPU");
    407
    0
      ccv_nnc_tensor_free(c);
    408
    0
      ccv_nnc_tensor_free(gc);
    409
    0
      ccv_nnc_tensor_free(w);
    410
    0
      ccv_nnc_tensor_free(b);
    411
    0
      ccv_nnc_tensor_free(a);
    412
    0
      ccv_nnc_tensor_free(gw);
    413
    0
      ccv_nnc_tensor_free(ga);
    414
    0
    }
    415
    416
    TEST_CASE("mps forward convolution in nchw format with 1x1 kernel on edge tiles")
    417
    1
    {
    418
    1
      GUARD_ELSE_RETURN(ccv_nnc_cmd_ok(CCV_NNC_CONVOLUTION_FORWARD, CCV_NNC_BACKEND_MPS));
    419
    0
      const int batch_size = 2;
    420
    0
      const int input_dim = 17;
    421
    0
      const int output_dim = 77;
    422
    0
      const int output_h = 17;
    423
    0
      const int output_w = 19;
    424
    0
      ccv_nnc_tensor_t* a = ccv_nnc_tensor_new(0, CPU_TENSOR_NCHW(32F, batch_size, input_dim, output_h, output_w), 0);
    425
    0
      ccv_nnc_tensor_t* b = ccv_nnc_tensor_new(0, CPU_TENSOR_NCHW(32F, batch_size, output_dim, output_h, output_w), 0);
    426
    0
      ccv_nnc_cmd_t cmd = CMD_CONVOLUTION_FORWARD(1, output_dim, 1, 1, input_dim);
    427
    0
      cmd.backend = CCV_NNC_BACKEND_CPU_REF;
    428
    0
      assert(cmd.backend >= 0);
    429
    0
      ccv_nnc_hint_t hint = ccv_nnc_hint_auto(cmd.info, a->info, b->info);
    430
    0
      assert(ccv_nnc_hint_verify(hint, cmd.info, a->info, b->info) == 0);
    431
    0
      ccv_nnc_tensor_t* w = ccv_nnc_tensor_new(0, CPU_TENSOR_NHWC(32F, output_dim, input_dim, 1, 1), 0);
    432
    0
      ccv_nnc_tensor_t* bias = ccv_nnc_tensor_new(0, CPU_TENSOR_NHWC(32F, output_dim), 0);
    433
    0
      dsfmt_t dsfmt;
    434
    0
      dsfmt_init_gen_rand(&dsfmt, 2);
    435
    0
      int i;
    436
    0
      for (i = 0; i < input_dim * output_dim; i++)
    437
    0
        w->data.f32[i] = dsfmt_genrand_open_close(&dsfmt) / input_dim;
    438
    0
      for (i = 0; i < batch_size * input_dim * output_h * output_w; i++)
    439
    0
        a->data.f32[i] = dsfmt_genrand_open_close(&dsfmt);
    440
    0
      for (i = 0; i < output_dim; i++)
    441
    0
        bias->data.f32[i] = (float)i / output_dim;
    442
    0
      ccv_nnc_tensor_t* ga = ccv_nnc_tensor_new(0, GPU_TENSOR_NCHW(000, 32F, batch_size, input_dim, output_h, output_w), 0);
    443
    0
      ccv_nnc_tensor_t* gw = ccv_nnc_tensor_new(0, GPU_TENSOR_NCHW(000, 32F, output_dim, input_dim, 1, 1), 0);
    444
    0
      ccv_nnc_tensor_t* gbias = ccv_nnc_tensor_new(0, GPU_TENSOR_NCHW(000, 32F, output_dim), 0);
    445
    0
      ccv_nnc_cmd_t move = CMD_DATA_TRANSFER_FORWARD();
    446
    0
      move.backend = CCV_NNC_BACKEND_MPS;
    447
    0
      assert(move.backend >= 0);
    448
    0
      ccv_nnc_cmd_exec(move, ccv_nnc_no_hint, 0, TENSOR_LIST(a, w, bias), TENSOR_LIST(ga, gw, gbias), 0);
    449
    0
      ccv_nnc_cmd_exec(cmd, hint, 0, TENSOR_LIST(a, w, bias), TENSOR_LIST(b), 0);
    450
    0
      ccv_nnc_tensor_t* gc = ccv_nnc_tensor_new(0, GPU_TENSOR_NCHW(000, 32F, batch_size, output_dim, output_h, output_w), 0);
    451
    452
    0
      cmd.backend = CCV_NNC_BACKEND_MPS;
    453
    0
      assert(cmd.backend >= 0);
    454
    0
      cmd.algorithm = -1;
    455
    0
      cmd = ccv_nnc_cmd_autotune(cmd, 1 * 1024 * 1024 * 1024, hint, 0, TENSOR_LIST(ga, gw, gbias), TENSOR_LIST(gc), 0);
    456
    0
      assert(CCV_NNC_EXEC_SUCCESS == ccv_nnc_cmd_exec(cmd, hint, 0, TENSOR_LIST(ga, gw, gbias), TENSOR_LIST(gc), 0));
    457
    0
      ccv_nnc_tensor_t* c = ccv_nnc_tensor_new(0, CPU_TENSOR_NCHW(32F, batch_size, output_dim, output_h, output_w), 0);
    458
    0
      ccv_nnc_cmd_exec(move, ccv_nnc_no_hint, 0, TENSOR_LIST(gc), TENSOR_LIST(c), 0);
    459
    0
      REQUIRE_ARRAY_EQ_WITH_TOLERANCE(float, b->data.f32, c->data.f32, batch_size * output_dim * output_h * output_w, 1e-3, "output from mps should match from CPU");
    460
    0
      ccv_nnc_tensor_free(c);
    461
    0
      ccv_nnc_tensor_free(gc);
    462
    0
      ccv_nnc_tensor_free(bias);
    463
    0
      ccv_nnc_tensor_free(w);
    464
    0
      ccv_nnc_tensor_free(b);
    465
    0
      ccv_nnc_tensor_free(a);
    466
    0
      ccv_nnc_tensor_free(gbias);
    467
    0
      ccv_nnc_tensor_free(gw);
    468
    0
      ccv_nnc_tensor_free(ga);
    469
    0
    }
    470
    471
    TEST_CASE("mps forward convolution in nchw format with 1x1 kernel on edge tiles and no bias")
    472
    1
    {
    473
    1
      GUARD_ELSE_RETURN(ccv_nnc_cmd_ok(CCV_NNC_CONVOLUTION_FORWARD, CCV_NNC_BACKEND_MPS));
    474
    0
      const int batch_size = 2;
    475
    0
      const int input_dim = 17;
    476
    0
      const int output_dim = 77;
    477
    0
      const int output_h = 17;
    478
    0
      const int output_w = 19;
    479
    0
      ccv_nnc_tensor_t* a = ccv_nnc_tensor_new(0, CPU_TENSOR_NCHW(32F, batch_size, input_dim, output_h, output_w), 0);
    480
    0
      ccv_nnc_tensor_t* b = ccv_nnc_tensor_new(0, CPU_TENSOR_NCHW(32F, batch_size, output_dim, output_h, output_w), 0);
    481
    0
      ccv_nnc_cmd_t cmd = CMD_CONVOLUTION_FORWARD(1, output_dim, 1, 1, input_dim);
    482
    0
      cmd.backend = CCV_NNC_BACKEND_CPU_REF;
    483
    0
      assert(cmd.backend >= 0);
    484
    0
      ccv_nnc_hint_t hint = ccv_nnc_hint_auto(cmd.info, a->info, b->info);
    485
    0
      assert(ccv_nnc_hint_verify(hint, cmd.info, a->info, b->info) == 0);
    486
    0
      ccv_nnc_tensor_t* w = ccv_nnc_tensor_new(0, CPU_TENSOR_NHWC(32F, output_dim, input_dim, 1, 1), 0);
    487
    0
      dsfmt_t dsfmt;
    488
    0
      dsfmt_init_gen_rand(&dsfmt, 3);
    489
    0
      int i;
    490
    0
      for (i = 0; i < input_dim * output_dim; i++)
    491
    0
        w->data.f32[i] = dsfmt_genrand_open_close(&dsfmt) / input_dim;
    492
    0
      for (i = 0; i < batch_size * input_dim * output_h * output_w; i++)
    493
    0
        a->data.f32[i] = dsfmt_genrand_open_close(&dsfmt);
    494
    0
      ccv_nnc_tensor_t* ga = ccv_nnc_tensor_new(0, GPU_TENSOR_NCHW(000, 32F, batch_size, input_dim, output_h, output_w), 0);
    495
    0
      ccv_nnc_tensor_t* gw = ccv_nnc_tensor_new(0, GPU_TENSOR_NCHW(000, 32F, output_dim, input_dim, 1, 1), 0);
    496
    0
      ccv_nnc_cmd_t move = CMD_DATA_TRANSFER_FORWARD();
    497
    0
      move.backend = CCV_NNC_BACKEND_MPS;
    498
    0
      assert(move.backend >= 0);
    499
    0
      ccv_nnc_cmd_exec(move, ccv_nnc_no_hint, 0, TENSOR_LIST(a, w), TENSOR_LIST(ga, gw), 0);
    500
    0
      ccv_nnc_cmd_exec(cmd, hint, 0, TENSOR_LIST(a, w), TENSOR_LIST(b), 0);
    501
    0
      ccv_nnc_tensor_t* gc = ccv_nnc_tensor_new(0, GPU_TENSOR_NCHW(000, 32F, batch_size, output_dim, output_h, output_w), 0);
    502
    503
    0
      cmd.backend = CCV_NNC_BACKEND_MPS;
    504
    0
      assert(cmd.backend >= 0);
    505
    0
      cmd.algorithm = -1;
    506
    0
      cmd = ccv_nnc_cmd_autotune(cmd, 1 * 1024 * 1024 * 1024, hint, 0, TENSOR_LIST(ga, gw), TENSOR_LIST(gc), 0);
    507
    0
      assert(CCV_NNC_EXEC_SUCCESS == ccv_nnc_cmd_exec(cmd, hint, 0, TENSOR_LIST(ga, gw), TENSOR_LIST(gc), 0));
    508
    0
      ccv_nnc_tensor_t* c = ccv_nnc_tensor_new(0, CPU_TENSOR_NCHW(32F, batch_size, output_dim, output_h, output_w), 0);
    509
    0
      ccv_nnc_cmd_exec(move, ccv_nnc_no_hint, 0, TENSOR_LIST(gc), TENSOR_LIST(c), 0);
    510
    0
      REQUIRE_ARRAY_EQ_WITH_TOLERANCE(float, b->data.f32, c->data.f32, batch_size * output_dim * output_h * output_w, 1e-3, "output from mps should match from CPU");
    511
    0
      ccv_nnc_tensor_free(c);
    512
    0
      ccv_nnc_tensor_free(gc);
    513
    0
      ccv_nnc_tensor_free(w);
    514
    0
      ccv_nnc_tensor_free(b);
    515
    0
      ccv_nnc_tensor_free(a);
    516
    0
      ccv_nnc_tensor_free(gw);
    517
    0
      ccv_nnc_tensor_free(ga);
    518
    0
    }
    519
    520
    TEST_CASE("mps forward convolution in half precision")
    521
    1
    {
    522
    1
      GUARD_ELSE_RETURN(ccv_nnc_cmd_ok(CCV_NNC_CONVOLUTION_FORWARD, CCV_NNC_BACKEND_MPS));
    523
    0
      ccv_nnc_tensor_t* a = ccv_nnc_tensor_new(0, CPU_TENSOR_NHWC(32F, BATCH_SIZE, INPUT_SIZE, INPUT_SIZE, INPUT_DIM), 0);
    524
    0
      ccv_nnc_tensor_t* b = ccv_nnc_tensor_new(0, CPU_TENSOR_NHWC(32F, BATCH_SIZE, OUTPUT_SIZE, OUTPUT_SIZE, OUTPUT_DIM), 0);
    525
    0
      ccv_nnc_cmd_t cmd = CMD_CONVOLUTION_FORWARD(1, OUTPUT_DIM, KERNEL_SIZE, KERNEL_SIZE, INPUT_DIM);
    526
    0
      cmd.backend = CCV_NNC_BACKEND_CPU_REF;
    527
    0
      assert(cmd.backend >= 0);
    528
    0
      ccv_nnc_hint_t hint = ccv_nnc_hint_auto(cmd.info, a->info, b->info);
    529
    0
      assert(ccv_nnc_hint_verify(hint, cmd.info, a->info, b->info) == 0);
    530
    0
      ccv_nnc_tensor_t* w = ccv_nnc_tensor_new(0, CPU_TENSOR_NHWC(32F, OUTPUT_DIM, KERNEL_SIZE, KERNEL_SIZE, INPUT_DIM), 0);
    531
    0
      ccv_nnc_tensor_t* bias = ccv_nnc_tensor_new(0, CPU_TENSOR_NHWC(32F, OUTPUT_DIM), 0);
    532
      // configure the inlets.
    533
    0
      dsfmt_t dsfmt;
    534
    0
      dsfmt_init_gen_rand(&dsfmt, 0);
    535
    0
      int i;
    536
    0
      for (i = 0; i < INPUT_DIM * KERNEL_SIZE * KERNEL_SIZE * OUTPUT_DIM; i++)
    537
    0
        w->data.f32[i] = dsfmt_genrand_open_close(&dsfmt) / (INPUT_DIM * KERNEL_SIZE * KERNEL_SIZE);
    538
    0
      for (i = 0; i < INPUT_SIZE * INPUT_SIZE * INPUT_DIM * ccv_max(1, BATCH_SIZE); i++)
    539
    0
        a->data.f32[i] = dsfmt_genrand_open_close(&dsfmt);
    540
    0
      for (i = 0; i < OUTPUT_DIM; i++)
    541
    0
        bias->data.f32[i] = (float)i / OUTPUT_DIM;
    542
    0
      ccv_nnc_tensor_t* a1 = ccv_nnc_tensor_new(0, CPU_TENSOR_NHWC(16F, BATCH_SIZE, INPUT_SIZE, INPUT_SIZE, INPUT_DIM), 0);
    543
    0
      ccv_nnc_tensor_t* w1 = ccv_nnc_tensor_new(0, CPU_TENSOR_NHWC(16F, OUTPUT_DIM, KERNEL_SIZE, KERNEL_SIZE, INPUT_DIM), 0);
    544
    0
      ccv_nnc_tensor_t* bias1 = ccv_nnc_tensor_new(0, CPU_TENSOR_NHWC(16F, OUTPUT_DIM), 0);
    545
    0
      ccv_nnc_cmd_exec(CMD_DATATYPE_CONVERSION_FORWARD(), ccv_nnc_no_hint, 0, TENSOR_LIST(a, w, bias), TENSOR_LIST(a1, w1, bias1), 0);
    546
      // Copy generated matrix values over to GPU.
    547
    0
      ccv_nnc_tensor_t* ga = ccv_nnc_tensor_new(0, GPU_TENSOR_NHWC(000, 16F, BATCH_SIZE, INPUT_SIZE, INPUT_SIZE, INPUT_DIM), 0);
    548
    0
      ccv_nnc_tensor_t* gw = ccv_nnc_tensor_new(0, GPU_TENSOR_NHWC(000, 16F, OUTPUT_DIM, KERNEL_SIZE, KERNEL_SIZE, INPUT_DIM), 0);
    549
    0
      ccv_nnc_tensor_t* gwo = ccv_nnc_tensor_new(0, GPU_TENSOR_NCHW(000, 16F, OUTPUT_DIM, INPUT_DIM, KERNEL_SIZE, KERNEL_SIZE), 0);
    550
    0
      ccv_nnc_tensor_t* gbias = ccv_nnc_tensor_new(0, GPU_TENSOR_NHWC(000, 16F, OUTPUT_DIM), 0);
    551
    0
      ccv_nnc_cmd_exec(CMD_DATA_TRANSFER_FORWARD(), ccv_nnc_no_hint, 0, TENSOR_LIST(a1, w1, bias1), TENSOR_LIST(ga, gw, gbias), 0);
    552
    0
      ccv_nnc_cmd_exec(cmd, hint, 0, TENSOR_LIST(a, w, bias), TENSOR_LIST(b), 0);
    553
    0
      ccv_nnc_tensor_t* gc = ccv_nnc_tensor_new(0, GPU_TENSOR_NHWC(000, 16F, BATCH_SIZE, OUTPUT_SIZE, OUTPUT_SIZE, OUTPUT_DIM), 0);
    554
    555
    0
      ccv_nnc_cmd_t transform = CMD_FORMAT_TRANSFORM_FORWARD();
    556
    0
      transform.backend = CCV_NNC_BACKEND_MPS;
    557
    0
      assert(transform.backend >= 0);
    558
    0
      ccv_nnc_stream_context_t* stream_context = ccv_nnc_stream_context_new(CCV_STREAM_CONTEXT_GPU);
    559
    0
      ccv_nnc_cmd_exec(transform, ccv_nnc_no_hint, 0, TENSOR_LIST(gw), TENSOR_LIST(gwo), stream_context);
    560
    0
      ccv_nnc_stream_context_wait(stream_context);
    561
    0
      ccv_nnc_tensor_free(gw);
    562
    563
    0
      cmd.backend = CCV_NNC_BACKEND_MPS;
    564
    0
      assert(cmd.backend >= 0);
    565
    0
      cmd.algorithm = -1;
    566
    0
      cmd = ccv_nnc_cmd_autotune(cmd, 1 * 1024 * 1024 * 1024, hint, 0, TENSOR_LIST(ga, gwo, gbias), TENSOR_LIST(gc), stream_context);
    567
    0
      assert(CCV_NNC_EXEC_SUCCESS == ccv_nnc_cmd_exec(cmd, hint, 0, TENSOR_LIST(ga, gwo, gbias), TENSOR_LIST(gc), stream_context));
    568
    0
      ccv_nnc_stream_context_wait(stream_context);
    569
    0
      ccv_nnc_stream_context_free(stream_context);
    570
    0
      ccv_nnc_tensor_t* c1 = ccv_nnc_tensor_new(0, CPU_TENSOR_NHWC(16F, BATCH_SIZE, OUTPUT_SIZE, OUTPUT_SIZE, OUTPUT_DIM), 0);
    571
    0
      ccv_nnc_cmd_exec(CMD_DATA_TRANSFER_FORWARD(), ccv_nnc_no_hint, 0, TENSOR_LIST(gc), TENSOR_LIST(c1), 0);
    572
    0
      ccv_nnc_tensor_t* c = ccv_nnc_tensor_new(0, CPU_TENSOR_NHWC(32F, BATCH_SIZE, OUTPUT_SIZE, OUTPUT_SIZE, OUTPUT_DIM), 0);
    573
    0
      ccv_nnc_cmd_exec(CMD_DATATYPE_CONVERSION_FORWARD(), ccv_nnc_no_hint, 0, TENSOR_LIST(c1), TENSOR_LIST(c), 0);
    574
    0
      REQUIRE_ARRAY_EQ_WITH_TOLERANCE(float, b->data.f32, c->data.f32, OUTPUT_DIM * OUTPUT_SIZE * OUTPUT_SIZE, 5e-3, "output from mps should match from CPU");
    575
    0
      ccv_nnc_tensor_free(c);
    576
    0
      ccv_nnc_tensor_free(gc);
    577
    0
      ccv_nnc_tensor_free(bias);
    578
    0
      ccv_nnc_tensor_free(w);
    579
    0
      ccv_nnc_tensor_free(b);
    580
    0
      ccv_nnc_tensor_free(a);
    581
    0
      ccv_nnc_tensor_free(c1);
    582
    0
      ccv_nnc_tensor_free(bias1);
    583
    0
      ccv_nnc_tensor_free(w1);
    584
    0
      ccv_nnc_tensor_free(a1);
    585
    0
      ccv_nnc_tensor_free(gbias);
    586
    0
      ccv_nnc_tensor_free(gwo);
    587
    0
      ccv_nnc_tensor_free(ga);
    588
    0
    }
    589
    590
    TEST_CASE("mps forward convolution in bfloat precision")
    591
    1
    {
    592
    1
      GUARD_ELSE_RETURN(ccv_nnc_cmd_ok(CCV_NNC_CONVOLUTION_FORWARD, CCV_NNC_BACKEND_MPS));
    593
    0
      const int batch_size = 2;
    594
    0
      const int input_size = 17;
    595
    0
      const int kernel_size = 3;
    596
    0
      const int output_size = 15;
    597
    0
      const int input_dim = 8;
    598
    0
      const int output_dim = 16;
    599
    0
      ccv_nnc_tensor_t* a = ccv_nnc_tensor_new(0, CPU_TENSOR_NHWC(32F, batch_size, input_size, input_size, input_dim), 0);
    600
    0
      ccv_nnc_tensor_t* b = ccv_nnc_tensor_new(0, CPU_TENSOR_NHWC(32F, batch_size, output_size, output_size, output_dim), 0);
    601
    0
      ccv_nnc_cmd_t cmd = CMD_CONVOLUTION_FORWARD(1, output_dim, kernel_size, kernel_size, input_dim);
    602
    0
      cmd.backend = CCV_NNC_BACKEND_CPU_REF;
    603
    0
      assert(cmd.backend >= 0);
    604
    0
      ccv_nnc_hint_t hint = ccv_nnc_hint_auto(cmd.info, a->info, b->info);
    605
    0
      assert(ccv_nnc_hint_verify(hint, cmd.info, a->info, b->info) == 0);
    606
    0
      ccv_nnc_tensor_t* w = ccv_nnc_tensor_new(0, CPU_TENSOR_NHWC(32F, output_dim, kernel_size, kernel_size, input_dim), 0);
    607
    0
      ccv_nnc_tensor_t* bias = ccv_nnc_tensor_new(0, CPU_TENSOR_NHWC(32F, output_dim), 0);
    608
    0
      dsfmt_t dsfmt;
    609
    0
      dsfmt_init_gen_rand(&dsfmt, 10);
    610
    0
      int i;
    611
    0
      for (i = 0; i < input_dim * kernel_size * kernel_size * output_dim; i++)
    612
    0
        w->data.f32[i] = (dsfmt_genrand_open_close(&dsfmt) - 0.5) / (input_dim * kernel_size * kernel_size);
    613
    0
      for (i = 0; i < input_size * input_size * input_dim * batch_size; i++)
    614
    0
        a->data.f32[i] = dsfmt_genrand_open_close(&dsfmt) - 0.5;
    615
    0
      for (i = 0; i < output_dim; i++)
    616
    0
        bias->data.f32[i] = (dsfmt_genrand_open_close(&dsfmt) - 0.5) * 0.1;
    617
    0
      const int input_count = batch_size * input_size * input_size * input_dim;
    618
    0
      const int weight_count = output_dim * kernel_size * kernel_size * input_dim;
    619
    0
      const int output_count = batch_size * output_size * output_size * output_dim;
    620
    0
      ccv_nnc_tensor_t* a16bf = ccv_nnc_tensor_new(0, CPU_TENSOR_NHWC(16BF, batch_size, input_size, input_size, input_dim), 0);
    621
    0
      ccv_nnc_tensor_t* w16bf = ccv_nnc_tensor_new(0, CPU_TENSOR_NHWC(16BF, output_dim, kernel_size, kernel_size, input_dim), 0);
    622
    0
      ccv_nnc_tensor_t* bias16bf = ccv_nnc_tensor_new(0, CPU_TENSOR_NHWC(16BF, output_dim), 0);
    623
    0
      ccv_float_to_bfloat(a->data.f32, (uint16_t*)a16bf->data.f16, input_count);
    624
    0
      ccv_float_to_bfloat(w->data.f32, (uint16_t*)w16bf->data.f16, weight_count);
    625
    0
      ccv_float_to_bfloat(bias->data.f32, (uint16_t*)bias16bf->data.f16, output_dim);
    626
    0
      ccv_bfloat_to_float((uint16_t*)a16bf->data.f16, a->data.f32, input_count);
    627
    0
      ccv_bfloat_to_float((uint16_t*)w16bf->data.f16, w->data.f32, weight_count);
    628
    0
      ccv_bfloat_to_float((uint16_t*)bias16bf->data.f16, bias->data.f32, output_dim);
    629
    0
      ccv_nnc_cmd_exec(cmd, hint, 0, TENSOR_LIST(a, w, bias), TENSOR_LIST(b), 0);
    630
    0
      ccv_nnc_tensor_t* b16bf = ccv_nnc_tensor_new(0, CPU_TENSOR_NHWC(16BF, batch_size, output_size, output_size, output_dim), 0);
    631
    0
      ccv_float_to_bfloat(b->data.f32, (uint16_t*)b16bf->data.f16, output_count);
    632
    0
      ccv_bfloat_to_float((uint16_t*)b16bf->data.f16, b->data.f32, output_count);
    633
    0
      ccv_nnc_tensor_t* ga = ccv_nnc_tensor_new(0, GPU_TENSOR_NHWC(000, 16BF, batch_size, input_size, input_size, input_dim), 0);
    634
    0
      ccv_nnc_tensor_t* gw = ccv_nnc_tensor_new(0, GPU_TENSOR_NHWC(000, 16BF, output_dim, kernel_size, kernel_size, input_dim), 0);
    635
    0
      ccv_nnc_tensor_t* gwo = ccv_nnc_tensor_new(0, GPU_TENSOR_NCHW(000, 16BF, output_dim, input_dim, kernel_size, kernel_size), 0);
    636
    0
      ccv_nnc_tensor_t* gbias = ccv_nnc_tensor_new(0, GPU_TENSOR_NHWC(000, 16BF, output_dim), 0);
    637
    0
      ccv_nnc_tensor_t* gc = ccv_nnc_tensor_new(0, GPU_TENSOR_NHWC(000, 16BF, batch_size, output_size, output_size, output_dim), 0);
    638
    0
      ccv_nnc_cmd_t move = CMD_DATA_TRANSFER_FORWARD();
    639
    0
      move.backend = CCV_NNC_BACKEND_MPS;
    640
    0
      assert(move.backend >= 0);
    641
    0
      ccv_nnc_cmd_exec(move, ccv_nnc_no_hint, 0, TENSOR_LIST(a16bf, w16bf, bias16bf), TENSOR_LIST(ga, gw, gbias), 0);
    642
    0
      ccv_nnc_cmd_t transform = CMD_FORMAT_TRANSFORM_FORWARD();
    643
    0
      transform.backend = CCV_NNC_BACKEND_MPS;
    644
    0
      assert(transform.backend >= 0);
    645
    0
      ccv_nnc_stream_context_t* stream_context = ccv_nnc_stream_context_new(CCV_STREAM_CONTEXT_GPU);
    646
    0
      ccv_nnc_cmd_exec(transform, ccv_nnc_no_hint, 0, TENSOR_LIST(gw), TENSOR_LIST(gwo), stream_context);
    647
    0
      ccv_nnc_stream_context_wait(stream_context);
    648
    0
      ccv_nnc_tensor_free(gw);
    649
    0
      cmd.backend = CCV_NNC_BACKEND_MPS;
    650
    0
      assert(cmd.backend >= 0);
    651
    0
      cmd.algorithm = -1;
    652
    0
      cmd = ccv_nnc_cmd_autotune(cmd, 1 * 1024 * 1024 * 1024, hint, 0, TENSOR_LIST(ga, gwo, gbias), TENSOR_LIST(gc), stream_context);
    653
    0
      assert(CCV_NNC_EXEC_SUCCESS == ccv_nnc_cmd_exec(cmd, hint, 0, TENSOR_LIST(ga, gwo, gbias), TENSOR_LIST(gc), stream_context));
    654
    0
      ccv_nnc_stream_context_wait(stream_context);
    655
    0
      ccv_nnc_stream_context_free(stream_context);
    656
    0
      ccv_nnc_tensor_t* c16bf = ccv_nnc_tensor_new(0, CPU_TENSOR_NHWC(16BF, batch_size, output_size, output_size, output_dim), 0);
    657
    0
      ccv_nnc_tensor_t* c = ccv_nnc_tensor_new(0, CPU_TENSOR_NHWC(32F, batch_size, output_size, output_size, output_dim), 0);
    658
    0
      ccv_nnc_cmd_exec(move, ccv_nnc_no_hint, 0, TENSOR_LIST(gc), TENSOR_LIST(c16bf), 0);
    659
    0
      ccv_bfloat_to_float((uint16_t*)c16bf->data.f16, c->data.f32, output_count);
    660
    0
      REQUIRE_ARRAY_EQ_WITH_TOLERANCE(float, b->data.f32, c->data.f32, output_count, 5e-2, "bfloat output from mps should match CPU reference rounded to bfloat");
    661
    0
      ccv_nnc_tensor_free(c);
    662
    0
      ccv_nnc_tensor_free(c16bf);
    663
    0
      ccv_nnc_tensor_free(gc);
    664
    0
      ccv_nnc_tensor_free(gbias);
    665
    0
      ccv_nnc_tensor_free(gwo);
    666
    0
      ccv_nnc_tensor_free(ga);
    667
    0
      ccv_nnc_tensor_free(b16bf);
    668
    0
      ccv_nnc_tensor_free(bias16bf);
    669
    0
      ccv_nnc_tensor_free(w16bf);
    670
    0
      ccv_nnc_tensor_free(a16bf);
    671
    0
      ccv_nnc_tensor_free(bias);
    672
    0
      ccv_nnc_tensor_free(w);
    673
    0
      ccv_nnc_tensor_free(b);
    674
    0
      ccv_nnc_tensor_free(a);
    675
    0
    }
    676
    677
    TEST_CASE("mps forward convolution in bfloat precision with 1x1 kernel")
    678
    1
    {
    679
    1
      GUARD_ELSE_RETURN(ccv_nnc_cmd_ok(CCV_NNC_CONVOLUTION_FORWARD, CCV_NNC_BACKEND_MPS));
    680
    0
      const int batch_size = 2;
    681
    0
      const int input_size = 17;
    682
    0
      const int output_size = 17;
    683
    0
      const int input_dim = 16;
    684
    0
      const int output_dim = 32;
    685
    0
      ccv_nnc_tensor_t* a = ccv_nnc_tensor_new(0, CPU_TENSOR_NHWC(32F, batch_size, input_size, input_size, input_dim), 0);
    686
    0
      ccv_nnc_tensor_t* b = ccv_nnc_tensor_new(0, CPU_TENSOR_NHWC(32F, batch_size, output_size, output_size, output_dim), 0);
    687
    0
      ccv_nnc_cmd_t cmd = CMD_CONVOLUTION_FORWARD(1, output_dim, 1, 1, input_dim);
    688
    0
      cmd.backend = CCV_NNC_BACKEND_CPU_REF;
    689
    0
      assert(cmd.backend >= 0);
    690
    0
      ccv_nnc_hint_t hint = ccv_nnc_hint_auto(cmd.info, a->info, b->info);
    691
    0
      assert(ccv_nnc_hint_verify(hint, cmd.info, a->info, b->info) == 0);
    692
    0
      ccv_nnc_tensor_t* w = ccv_nnc_tensor_new(0, CPU_TENSOR_NHWC(32F, output_dim, 1, 1, input_dim), 0);
    693
    0
      ccv_nnc_tensor_t* bias = ccv_nnc_tensor_new(0, CPU_TENSOR_NHWC(32F, output_dim), 0);
    694
    0
      dsfmt_t dsfmt;
    695
    0
      dsfmt_init_gen_rand(&dsfmt, 11);
    696
    0
      int i;
    697
    0
      for (i = 0; i < input_dim * output_dim; i++)
    698
    0
        w->data.f32[i] = (dsfmt_genrand_open_close(&dsfmt) - 0.5) / input_dim;
    699
    0
      for (i = 0; i < input_size * input_size * input_dim * batch_size; i++)
    700
    0
        a->data.f32[i] = dsfmt_genrand_open_close(&dsfmt) - 0.5;
    701
    0
      for (i = 0; i < output_dim; i++)
    702
    0
        bias->data.f32[i] = (dsfmt_genrand_open_close(&dsfmt) - 0.5) * 0.1;
    703
    0
      const int input_count = batch_size * input_size * input_size * input_dim;
    704
    0
      const int weight_count = output_dim * input_dim;
    705
    0
      const int output_count = batch_size * output_size * output_size * output_dim;
    706
    0
      ccv_nnc_tensor_t* a16bf = ccv_nnc_tensor_new(0, CPU_TENSOR_NHWC(16BF, batch_size, input_size, input_size, input_dim), 0);
    707
    0
      ccv_nnc_tensor_t* w16bf = ccv_nnc_tensor_new(0, CPU_TENSOR_NHWC(16BF, output_dim, 1, 1, input_dim), 0);
    708
    0
      ccv_nnc_tensor_t* bias16bf = ccv_nnc_tensor_new(0, CPU_TENSOR_NHWC(16BF, output_dim), 0);
    709
    0
      ccv_float_to_bfloat(a->data.f32, (uint16_t*)a16bf->data.f16, input_count);
    710
    0
      ccv_float_to_bfloat(w->data.f32, (uint16_t*)w16bf->data.f16, weight_count);
    711
    0
      ccv_float_to_bfloat(bias->data.f32, (uint16_t*)bias16bf->data.f16, output_dim);
    712
    0
      ccv_bfloat_to_float((uint16_t*)a16bf->data.f16, a->data.f32, input_count);
    713
    0
      ccv_bfloat_to_float((uint16_t*)w16bf->data.f16, w->data.f32, weight_count);
    714
    0
      ccv_bfloat_to_float((uint16_t*)bias16bf->data.f16, bias->data.f32, output_dim);
    715
    0
      ccv_nnc_cmd_exec(cmd, hint, 0, TENSOR_LIST(a, w, bias), TENSOR_LIST(b), 0);
    716
    0
      ccv_nnc_tensor_t* b16bf = ccv_nnc_tensor_new(0, CPU_TENSOR_NHWC(16BF, batch_size, output_size, output_size, output_dim), 0);
    717
    0
      ccv_float_to_bfloat(b->data.f32, (uint16_t*)b16bf->data.f16, output_count);
    718
    0
      ccv_bfloat_to_float((uint16_t*)b16bf->data.f16, b->data.f32, output_count);
    719
    0
      ccv_nnc_tensor_t* ga = ccv_nnc_tensor_new(0, GPU_TENSOR_NHWC(000, 16BF, batch_size, input_size, input_size, input_dim), 0);
    720
    0
      ccv_nnc_tensor_t* gw = ccv_nnc_tensor_new(0, GPU_TENSOR_NHWC(000, 16BF, output_dim, 1, 1, input_dim), 0);
    721
    0
      ccv_nnc_tensor_t* gwo = ccv_nnc_tensor_new(0, GPU_TENSOR_NCHW(000, 16BF, output_dim, input_dim, 1, 1), 0);
    722
    0
      ccv_nnc_tensor_t* gbias = ccv_nnc_tensor_new(0, GPU_TENSOR_NHWC(000, 16BF, output_dim), 0);
    723
    0
      ccv_nnc_tensor_t* gc = ccv_nnc_tensor_new(0, GPU_TENSOR_NHWC(000, 16BF, batch_size, output_size, output_size, output_dim), 0);
    724
    0
      ccv_nnc_cmd_t move = CMD_DATA_TRANSFER_FORWARD();
    725
    0
      move.backend = CCV_NNC_BACKEND_MPS;
    726
    0
      assert(move.backend >= 0);
    727
    0
      ccv_nnc_cmd_exec(move, ccv_nnc_no_hint, 0, TENSOR_LIST(a16bf, w16bf, bias16bf), TENSOR_LIST(ga, gw, gbias), 0);
    728
    0
      ccv_nnc_cmd_t transform = CMD_FORMAT_TRANSFORM_FORWARD();
    729
    0
      transform.backend = CCV_NNC_BACKEND_MPS;
    730
    0
      assert(transform.backend >= 0);
    731
    0
      ccv_nnc_stream_context_t* stream_context = ccv_nnc_stream_context_new(CCV_STREAM_CONTEXT_GPU);
    732
    0
      ccv_nnc_cmd_exec(transform, ccv_nnc_no_hint, 0, TENSOR_LIST(gw), TENSOR_LIST(gwo), stream_context);
    733
    0
      ccv_nnc_stream_context_wait(stream_context);
    734
    0
      ccv_nnc_tensor_free(gw);
    735
    0
      cmd.backend = CCV_NNC_BACKEND_MPS;
    736
    0
      assert(cmd.backend >= 0);
    737
    0
      cmd.algorithm = -1;
    738
    0
      cmd = ccv_nnc_cmd_autotune(cmd, 1 * 1024 * 1024 * 1024, hint, 0, TENSOR_LIST(ga, gwo, gbias), TENSOR_LIST(gc), stream_context);
    739
    0
      assert(CCV_NNC_EXEC_SUCCESS == ccv_nnc_cmd_exec(cmd, hint, 0, TENSOR_LIST(ga, gwo, gbias), TENSOR_LIST(gc), stream_context));
    740
    0
      ccv_nnc_stream_context_wait(stream_context);
    741
    0
      ccv_nnc_stream_context_free(stream_context);
    742
    0
      ccv_nnc_tensor_t* c16bf = ccv_nnc_tensor_new(0, CPU_TENSOR_NHWC(16BF, batch_size, output_size, output_size, output_dim), 0);
    743
    0
      ccv_nnc_tensor_t* c = ccv_nnc_tensor_new(0, CPU_TENSOR_NHWC(32F, batch_size, output_size, output_size, output_dim), 0);
    744
    0
      ccv_nnc_cmd_exec(move, ccv_nnc_no_hint, 0, TENSOR_LIST(gc), TENSOR_LIST(c16bf), 0);
    745
    0
      ccv_bfloat_to_float((uint16_t*)c16bf->data.f16, c->data.f32, output_count);
    746
    0
      REQUIRE_ARRAY_EQ_WITH_TOLERANCE(float, b->data.f32, c->data.f32, output_count, 5e-2, "bfloat output from mps 1x1 convolution should match CPU reference rounded to bfloat");
    747
    0
      ccv_nnc_tensor_free(c);
    748
    0
      ccv_nnc_tensor_free(c16bf);
    749
    0
      ccv_nnc_tensor_free(gc);
    750
    0
      ccv_nnc_tensor_free(gbias);
    751
    0
      ccv_nnc_tensor_free(gwo);
    752
    0
      ccv_nnc_tensor_free(ga);
    753
    0
      ccv_nnc_tensor_free(b16bf);
    754
    0
      ccv_nnc_tensor_free(bias16bf);
    755
    0
      ccv_nnc_tensor_free(w16bf);
    756
    0
      ccv_nnc_tensor_free(a16bf);
    757
    0
      ccv_nnc_tensor_free(bias);
    758
    0
      ccv_nnc_tensor_free(w);
    759
    0
      ccv_nnc_tensor_free(b);
    760
    0
      ccv_nnc_tensor_free(a);
    761
    0
    }
    762
    763
    TEST_CASE("mps forward convolution with dilation 2, 3")
    764
    1
    {
    765
    1
      GUARD_ELSE_RETURN(ccv_nnc_cmd_ok(CCV_NNC_CONVOLUTION_FORWARD, CCV_NNC_BACKEND_MPS));
    766
    0
      ccv_nnc_tensor_t* a = ccv_nnc_tensor_new(0, CPU_TENSOR_NHWC(32F, BATCH_SIZE, INPUT_SIZE, INPUT_SIZE, INPUT_DIM), 0);
    767
    0
      ccv_nnc_tensor_t* b = ccv_nnc_tensor_new(0, CPU_TENSOR_NHWC(32F, BATCH_SIZE, OUTPUT_SIZE, OUTPUT_SIZE, OUTPUT_DIM), 0);
    768
    0
      ccv_nnc_cmd_t cmd = CMD_CONVOLUTION_FORWARD(1, OUTPUT_DIM, KERNEL_SIZE, KERNEL_SIZE, INPUT_DIM);
    769
    0
      cmd.info.convolution.dilation[0] = 2;
    770
    0
      cmd.info.convolution.dilation[1] = 3;
    771
    0
      cmd.backend = CCV_NNC_BACKEND_CPU_REF;
    772
    0
      assert(cmd.backend >= 0);
    773
    0
      ccv_nnc_cmd_param_t modified_cmd = cmd.info;
    774
    0
      modified_cmd.size.dim[0] = (cmd.info.size.dim[0] - 1) * ccv_max(cmd.info.convolution.dilation[0], 1) + 1;
    775
    0
      modified_cmd.size.dim[1] = (cmd.info.size.dim[1] - 1) * ccv_max(cmd.info.convolution.dilation[1], 1) + 1;
    776
    0
      ccv_nnc_hint_t hint = ccv_nnc_hint_auto(modified_cmd, a->info, b->info);
    777
    0
      assert(ccv_nnc_hint_verify(hint, modified_cmd, a->info, b->info) == 0);
    778
    0
      ccv_nnc_tensor_t* w = ccv_nnc_tensor_new(0, CPU_TENSOR_NHWC(32F, OUTPUT_DIM, KERNEL_SIZE, KERNEL_SIZE, INPUT_DIM), 0);
    779
    0
      ccv_nnc_tensor_t* bias = ccv_nnc_tensor_new(0, CPU_TENSOR_NHWC(32F, OUTPUT_DIM), 0);
    780
      // configure the inlets.
    781
    0
      dsfmt_t dsfmt;
    782
    0
      dsfmt_init_gen_rand(&dsfmt, 0);
    783
    0
      int i;
    784
    0
      for (i = 0; i < INPUT_DIM * KERNEL_SIZE * KERNEL_SIZE * OUTPUT_DIM; i++)
    785
    0
        w->data.f32[i] = dsfmt_genrand_open_close(&dsfmt) / (INPUT_DIM * KERNEL_SIZE * KERNEL_SIZE);
    786
    0
      for (i = 0; i < INPUT_SIZE * INPUT_SIZE * INPUT_DIM * ccv_max(1, BATCH_SIZE); i++)
    787
    0
        a->data.f32[i] = dsfmt_genrand_open_close(&dsfmt);
    788
    0
      for (i = 0; i < OUTPUT_DIM; i++)
    789
    0
        bias->data.f32[i] = (float)i / OUTPUT_DIM;
    790
      // Copy generated matrix values over to GPU.
    791
    0
      ccv_nnc_tensor_t* ga = ccv_nnc_tensor_new(0, GPU_TENSOR_NHWC(000, 32F, BATCH_SIZE, INPUT_SIZE, INPUT_SIZE, INPUT_DIM), 0);
    792
    0
      ccv_nnc_tensor_t* gw = ccv_nnc_tensor_new(0, GPU_TENSOR_NHWC(000, 32F, OUTPUT_DIM, KERNEL_SIZE, KERNEL_SIZE, INPUT_DIM), 0);
    793
    0
      ccv_nnc_tensor_t* gwo = ccv_nnc_tensor_new(0, GPU_TENSOR_NCHW(000, 32F, OUTPUT_DIM, INPUT_DIM, KERNEL_SIZE, KERNEL_SIZE), 0);
    794
    0
      ccv_nnc_tensor_t* gbias = ccv_nnc_tensor_new(0, GPU_TENSOR_NHWC(000, 32F, OUTPUT_DIM), 0);
    795
    0
      ccv_nnc_cmd_t move = CMD_DATA_TRANSFER_FORWARD();
    796
    0
      move.backend = CCV_NNC_BACKEND_MPS;
    797
    0
      assert(move.backend >= 0);
    798
    0
      ccv_nnc_cmd_exec(move, ccv_nnc_no_hint, 0, TENSOR_LIST(a, w, bias), TENSOR_LIST(ga, gw, gbias), 0);
    799
    0
      ccv_nnc_cmd_exec(cmd, hint, 0, TENSOR_LIST(a, w, bias), TENSOR_LIST(b), 0);
    800
    0
      ccv_nnc_tensor_t* gc = ccv_nnc_tensor_new(0, GPU_TENSOR_NHWC(000, 32F, BATCH_SIZE, OUTPUT_SIZE, OUTPUT_SIZE, OUTPUT_DIM), 0);
    801
    802
    0
      ccv_nnc_cmd_t transform = CMD_FORMAT_TRANSFORM_FORWARD();
    803
    0
      transform.backend = CCV_NNC_BACKEND_MPS;
    804
    0
      assert(transform.backend >= 0);
    805
    0
      ccv_nnc_stream_context_t* stream_context = ccv_nnc_stream_context_new(CCV_STREAM_CONTEXT_GPU);
    806
    0
      ccv_nnc_cmd_exec(transform, ccv_nnc_no_hint, 0, TENSOR_LIST(gw), TENSOR_LIST(gwo), stream_context);
    807
    0
      ccv_nnc_stream_context_wait(stream_context);
    808
    0
      ccv_nnc_tensor_free(gw);
    809
    810
    0
      cmd.backend = CCV_NNC_BACKEND_MPS;
    811
    0
      assert(cmd.backend >= 0);
    812
    0
      cmd.algorithm = -1;
    813
    0
      cmd = ccv_nnc_cmd_autotune(cmd, 1 * 1024 * 1024 * 1024, hint, 0, TENSOR_LIST(ga, gwo, gbias), TENSOR_LIST(gc), stream_context);
    814
    0
      assert(CCV_NNC_EXEC_SUCCESS == ccv_nnc_cmd_exec(cmd, hint, 0, TENSOR_LIST(ga, gwo, gbias), TENSOR_LIST(gc), stream_context));
    815
    0
      ccv_nnc_stream_context_wait(stream_context);
    816
    0
      ccv_nnc_stream_context_free(stream_context);
    817
    0
      ccv_nnc_tensor_t* c = ccv_nnc_tensor_new(0, CPU_TENSOR_NHWC(32F, BATCH_SIZE, OUTPUT_SIZE, OUTPUT_SIZE, OUTPUT_DIM), 0);
    818
    0
      ccv_nnc_cmd_exec(move, ccv_nnc_no_hint, 0, TENSOR_LIST(gc), TENSOR_LIST(c), 0);
    819
    0
      REQUIRE_ARRAY_EQ_WITH_TOLERANCE(float, b->data.f32, c->data.f32, OUTPUT_DIM * OUTPUT_SIZE * OUTPUT_SIZE, 1e-5, "output from mps should match from CPU");
    820
    0
      ccv_nnc_tensor_free(c);
    821
    0
      ccv_nnc_tensor_free(gc);
    822
    0
      ccv_nnc_tensor_free(bias);
    823
    0
      ccv_nnc_tensor_free(w);
    824
    0
      ccv_nnc_tensor_free(b);
    825
    0
      ccv_nnc_tensor_free(a);
    826
    0
      ccv_nnc_tensor_free(gbias);
    827
    0
      ccv_nnc_tensor_free(gwo);
    828
    0
      ccv_nnc_tensor_free(ga);
    829
    0
    }
    830
    831
    TEST_CASE("mps forward convolution 3d")
    832
    1
    {
    833
    1
      GUARD_ELSE_RETURN(ccv_nnc_cmd_ok(CCV_NNC_CONVOLUTION_FORWARD, CCV_NNC_BACKEND_MPS));
    834
    0
      ccv_nnc_tensor_t* a = ccv_nnc_tensor_new(0, CPU_TENSOR_NHWC(32F, BATCH_SIZE, 5, INPUT_SIZE, INPUT_SIZE, INPUT_DIM), 0);
    835
    0
      ccv_nnc_tensor_t* b = ccv_nnc_tensor_new(0, CPU_TENSOR_NHWC(32F, BATCH_SIZE, 3, OUTPUT_SIZE, OUTPUT_SIZE, OUTPUT_DIM), 0);
    836
    0
      ccv_nnc_cmd_t cmd = CMD_CONVOLUTION_FORWARD(1, OUTPUT_DIM, 3, KERNEL_SIZE, KERNEL_SIZE, INPUT_DIM);
    837
    0
      ccv_nnc_hint_t hint = ccv_nnc_hint_auto(cmd.info, a->info, b->info);
    838
    0
      hint.stride.dim[0] = 2;
    839
    0
      hint.border.begin[0] = 1;
    840
    0
      hint.border.end[0] = 1;
    841
    0
      assert(ccv_nnc_hint_verify(hint, cmd.info, a->info, b->info) == 0);
    842
    0
      ccv_nnc_tensor_t* w = ccv_nnc_tensor_new(0, CPU_TENSOR_NHWC(32F, OUTPUT_DIM, 3, KERNEL_SIZE, KERNEL_SIZE, INPUT_DIM), 0);
    843
    0
      ccv_nnc_tensor_t* bias = ccv_nnc_tensor_new(0, CPU_TENSOR_NHWC(32F, OUTPUT_DIM), 0);
    844
      // configure the inlets.
    845
    0
      dsfmt_t dsfmt;
    846
    0
      dsfmt_init_gen_rand(&dsfmt, 0);
    847
    0
      int i;
    848
    0
      for (i = 0; i < INPUT_DIM * 3 * KERNEL_SIZE * KERNEL_SIZE * OUTPUT_DIM; i++)
    849
    0
        w->data.f32[i] = dsfmt_genrand_open_close(&dsfmt) / (INPUT_DIM * KERNEL_SIZE * KERNEL_SIZE);
    850
    0
      for (i = 0; i < 5 * INPUT_SIZE * INPUT_SIZE * INPUT_DIM * ccv_max(1, BATCH_SIZE); i++)
    851
    0
        a->data.f32[i] = dsfmt_genrand_open_close(&dsfmt);
    852
    0
      for (i = 0; i < OUTPUT_DIM; i++)
    853
    0
        bias->data.f32[i] = (float)i / OUTPUT_DIM;
    854
      // Copy generated matrix values over to GPU.
    855
    0
      ccv_nnc_tensor_t* ga = ccv_nnc_tensor_new(0, GPU_TENSOR_NHWC(000, 32F, BATCH_SIZE, 5, INPUT_SIZE, INPUT_SIZE, INPUT_DIM), 0);
    856
    0
      ccv_nnc_tensor_t* gw = ccv_nnc_tensor_new(0, GPU_TENSOR_NHWC(000, 32F, OUTPUT_DIM, 3, KERNEL_SIZE, KERNEL_SIZE, INPUT_DIM), 0);
    857
    0
      ccv_nnc_tensor_t* gwo = ccv_nnc_tensor_new(0, GPU_TENSOR_NCHW(000, 32F, OUTPUT_DIM, INPUT_DIM, 3, KERNEL_SIZE, KERNEL_SIZE), 0);
    858
    0
      ccv_nnc_tensor_t* gbias = ccv_nnc_tensor_new(0, GPU_TENSOR_NHWC(000, 32F, OUTPUT_DIM), 0);
    859
    0
      ccv_nnc_cmd_t move = CMD_DATA_TRANSFER_FORWARD();
    860
    0
      move.backend = CCV_NNC_BACKEND_MPS;
    861
    0
      assert(move.backend >= 0);
    862
    0
      ccv_nnc_cmd_exec(move, ccv_nnc_no_hint, 0, TENSOR_LIST(a, w, bias), TENSOR_LIST(ga, gw, gbias), 0);
    863
    0
      ccv_nnc_tensor_t* gc = ccv_nnc_tensor_new(0, GPU_TENSOR_NHWC(000, 32F, BATCH_SIZE, 3, OUTPUT_SIZE, OUTPUT_SIZE, OUTPUT_DIM), 0);
    864
    0
      ccv_nnc_cmd_t transform = CMD_FORMAT_TRANSFORM_FORWARD();
    865
    0
      transform.backend = CCV_NNC_BACKEND_MPS;
    866
    0
      assert(transform.backend >= 0);
    867
    0
      ccv_nnc_stream_context_t* stream_context = ccv_nnc_stream_context_new(CCV_STREAM_CONTEXT_GPU);
    868
    0
      ccv_nnc_cmd_exec(transform, ccv_nnc_no_hint, 0, TENSOR_LIST(gw), TENSOR_LIST(gwo), stream_context);
    869
    0
      ccv_nnc_stream_context_wait(stream_context);
    870
    0
      ccv_nnc_tensor_free(gw);
    871
    0
      cmd.backend = CCV_NNC_BACKEND_MPS;
    872
    0
      assert(cmd.backend >= 0);
    873
    0
      cmd.algorithm = -1;
    874
    0
      cmd = ccv_nnc_cmd_autotune(cmd, 1 * 1024 * 1024 * 1024, hint, 0, TENSOR_LIST(ga, gwo, gbias), TENSOR_LIST(gc), stream_context);
    875
    0
      assert(CCV_NNC_EXEC_SUCCESS == ccv_nnc_cmd_exec(cmd, hint, 0, TENSOR_LIST(ga, gwo, gbias), TENSOR_LIST(gc), stream_context));
    876
    0
      ccv_nnc_stream_context_wait(stream_context);
    877
    0
      ccv_nnc_stream_context_free(stream_context);
    878
    0
      ccv_nnc_tensor_t* c = ccv_nnc_tensor_new(0, CPU_TENSOR_NHWC(32F, BATCH_SIZE, 3, OUTPUT_SIZE, OUTPUT_SIZE, OUTPUT_DIM), 0);
    879
    0
      ccv_nnc_cmd_exec(move, ccv_nnc_no_hint, 0, TENSOR_LIST(gc), TENSOR_LIST(c), 0);
    880
    0
      cmd.backend = CCV_NNC_BACKEND_CPU_REF;
    881
    0
      assert(cmd.backend >= 0);
    882
    0
      ccv_nnc_cmd_exec(cmd, hint, 0, TENSOR_LIST(a, w, bias), TENSOR_LIST(b), 0);
    883
    0
      REQUIRE_ARRAY_EQ_WITH_TOLERANCE(float, b->data.f32, c->data.f32, BATCH_SIZE * 3 * OUTPUT_DIM * OUTPUT_SIZE * OUTPUT_SIZE, 1e-4, "output from mps should match from CPU");
    884
    0
      ccv_nnc_tensor_free(c);
    885
    0
      ccv_nnc_tensor_free(gc);
    886
    0
      ccv_nnc_tensor_free(bias);
    887
    0
      ccv_nnc_tensor_free(w);
    888
    0
      ccv_nnc_tensor_free(b);
    889
    0
      ccv_nnc_tensor_free(a);
    890
    0
      ccv_nnc_tensor_free(gbias);
    891
    0
      ccv_nnc_tensor_free(gwo);
    892
    0
      ccv_nnc_tensor_free(ga);
    893
    0
    }
    894
    895
    TEST_CASE("mps forward convolution 3d via mfa conv3d")
    896
    1
    {
    897
    1
      GUARD_ELSE_RETURN(ccv_nnc_cmd_ok(CCV_NNC_CONVOLUTION_FORWARD, CCV_NNC_BACKEND_MPS));
    898
    0
      const int batch_size = 2;
    899
    0
      const int input_channels = 16;
    900
    0
      const int output_channels = 32;
    901
    0
      const int input_depth = 5;
    902
    0
      const int input_height = 10;
    903
    0
      const int input_width = 10;
    904
    0
      const int kernel_depth = 3;
    905
    0
      const int kernel_height = 3;
    906
    0
      const int kernel_width = 3;
    907
    0
      const int output_depth = 3;
    908
    0
      const int output_height = 8;
    909
    0
      const int output_width = 8;
    910
    0
      ccv_nnc_tensor_t* a = ccv_nnc_tensor_new(0, CPU_TENSOR_NHWC(32F, batch_size, input_depth, input_height, input_width, input_channels), 0);
    911
    0
      ccv_nnc_tensor_t* b = ccv_nnc_tensor_new(0, CPU_TENSOR_NHWC(32F, batch_size, output_depth, output_height, output_width, output_channels), 0);
    912
    0
      ccv_nnc_cmd_t cmd = CMD_CONVOLUTION_FORWARD(1, output_channels, kernel_depth, kernel_height, kernel_width, input_channels);
    913
    0
      ccv_nnc_hint_t hint = ccv_nnc_hint_auto(cmd.info, a->info, b->info);
    914
    0
      assert(ccv_nnc_hint_verify(hint, cmd.info, a->info, b->info) == 0);
    915
    0
      ccv_nnc_tensor_t* w = ccv_nnc_tensor_new(0, CPU_TENSOR_NHWC(32F, output_channels, kernel_depth, kernel_height, kernel_width, input_channels), 0);
    916
    0
      dsfmt_t dsfmt;
    917
    0
      dsfmt_init_gen_rand(&dsfmt, 1);
    918
    0
      int i;
    919
    0
      for (i = 0; i < batch_size * input_depth * input_height * input_width * input_channels; i++)
    920
    0
        a->data.f32[i] = dsfmt_genrand_open_close(&dsfmt) - 0.5;
    921
    0
      for (i = 0; i < output_channels * kernel_depth * kernel_height * kernel_width * input_channels; i++)
    922
    0
        w->data.f32[i] = (dsfmt_genrand_open_close(&dsfmt) - 0.5) * 0.1;
    923
    0
      ccv_nnc_tensor_t* ga = ccv_nnc_tensor_new(0, GPU_TENSOR_NHWC(000, 32F, batch_size, input_depth, input_height, input_width, input_channels), 0);
    924
    0
      ccv_nnc_tensor_t* gw = ccv_nnc_tensor_new(0, GPU_TENSOR_NHWC(000, 32F, output_channels, kernel_depth, kernel_height, kernel_width, input_channels), 0);
    925
    0
      ccv_nnc_tensor_t* gwo = ccv_nnc_tensor_new(0, GPU_TENSOR_NCHW(000, 32F, output_channels, input_channels, kernel_depth, kernel_height, kernel_width), 0);
    926
    0
      ccv_nnc_tensor_t* gc = ccv_nnc_tensor_new(0, GPU_TENSOR_NHWC(000, 32F, batch_size, output_depth, output_height, output_width, output_channels), 0);
    927
    0
      ccv_nnc_cmd_t move = CMD_DATA_TRANSFER_FORWARD();
    928
    0
      move.backend = CCV_NNC_BACKEND_MPS;
    929
    0
      assert(move.backend >= 0);
    930
    0
      ccv_nnc_cmd_exec(move, ccv_nnc_no_hint, 0, TENSOR_LIST(a, w), TENSOR_LIST(ga, gw), 0);
    931
    0
      ccv_nnc_cmd_t transform = CMD_FORMAT_TRANSFORM_FORWARD();
    932
    0
      transform.backend = CCV_NNC_BACKEND_MPS;
    933
    0
      assert(transform.backend >= 0);
    934
    0
      ccv_nnc_stream_context_t* stream_context = ccv_nnc_stream_context_new(CCV_STREAM_CONTEXT_GPU);
    935
    0
      ccv_nnc_cmd_exec(transform, ccv_nnc_no_hint, 0, TENSOR_LIST(gw), TENSOR_LIST(gwo), stream_context);
    936
    0
      ccv_nnc_stream_context_wait(stream_context);
    937
    0
      ccv_nnc_tensor_free(gw);
    938
    0
      cmd.backend = CCV_NNC_BACKEND_MPS;
    939
    0
      assert(cmd.backend >= 0);
    940
    0
      cmd.algorithm = -1;
    941
    0
      cmd = ccv_nnc_cmd_autotune(cmd, 1 * 1024 * 1024 * 1024, hint, 0, TENSOR_LIST(ga, gwo), TENSOR_LIST(gc), stream_context);
    942
    0
      assert(CCV_NNC_EXEC_SUCCESS == ccv_nnc_cmd_exec(cmd, hint, 0, TENSOR_LIST(ga, gwo), TENSOR_LIST(gc), stream_context));
    943
    0
      ccv_nnc_stream_context_wait(stream_context);
    944
    0
      ccv_nnc_stream_context_free(stream_context);
    945
    0
      ccv_nnc_tensor_t* c = ccv_nnc_tensor_new(0, CPU_TENSOR_NHWC(32F, batch_size, output_depth, output_height, output_width, output_channels), 0);
    946
    0
      ccv_nnc_cmd_exec(move, ccv_nnc_no_hint, 0, TENSOR_LIST(gc), TENSOR_LIST(c), 0);
    947
    0
      cmd.backend = CCV_NNC_BACKEND_CPU_REF;
    948
    0
      assert(cmd.backend >= 0);
    949
    0
      ccv_nnc_cmd_exec(cmd, hint, 0, TENSOR_LIST(a, w), TENSOR_LIST(b), 0);
    950
    0
      REQUIRE_ARRAY_EQ_WITH_TOLERANCE(float, b->data.f32, c->data.f32, batch_size * output_depth * output_height * output_width * output_channels, 1e-4, "output from mps should match from CPU");
    951
    0
      ccv_nnc_tensor_free(c);
    952
    0
      ccv_nnc_tensor_free(gc);
    953
    0
      ccv_nnc_tensor_free(w);
    954
    0
      ccv_nnc_tensor_free(b);
    955
    0
      ccv_nnc_tensor_free(a);
    956
    0
      ccv_nnc_tensor_free(gwo);
    957
    0
      ccv_nnc_tensor_free(ga);
    958
    0
    }
    959
    960
    TEST_CASE("mps forward convolution 3d via mfa conv3d with bias")
    961
    1
    {
    962
    1
      GUARD_ELSE_RETURN(ccv_nnc_cmd_ok(CCV_NNC_CONVOLUTION_FORWARD, CCV_NNC_BACKEND_MPS));
    963
    0
      const int batch_size = 2;
    964
    0
      const int input_channels = 16;
    965
    0
      const int output_channels = 32;
    966
    0
      const int input_depth = 5;
    967
    0
      const int input_height = 10;
    968
    0
      const int input_width = 10;
    969
    0
      const int kernel_depth = 3;
    970
    0
      const int kernel_height = 3;
    971
    0
      const int kernel_width = 3;
    972
    0
      const int output_depth = 3;
    973
    0
      const int output_height = 8;
    974
    0
      const int output_width = 8;
    975
    0
      ccv_nnc_tensor_t* a = ccv_nnc_tensor_new(0, CPU_TENSOR_NHWC(32F, batch_size, input_depth, input_height, input_width, input_channels), 0);
    976
    0
      ccv_nnc_tensor_t* b = ccv_nnc_tensor_new(0, CPU_TENSOR_NHWC(32F, batch_size, output_depth, output_height, output_width, output_channels), 0);
    977
    0
      ccv_nnc_cmd_t cmd = CMD_CONVOLUTION_FORWARD(1, output_channels, kernel_depth, kernel_height, kernel_width, input_channels);
    978
    0
      ccv_nnc_hint_t hint = ccv_nnc_hint_auto(cmd.info, a->info, b->info);
    979
    0
      assert(ccv_nnc_hint_verify(hint, cmd.info, a->info, b->info) == 0);
    980
    0
      ccv_nnc_tensor_t* w = ccv_nnc_tensor_new(0, CPU_TENSOR_NHWC(32F, output_channels, kernel_depth, kernel_height, kernel_width, input_channels), 0);
    981
    0
      ccv_nnc_tensor_t* bias = ccv_nnc_tensor_new(0, CPU_TENSOR_NHWC(32F, output_channels), 0);
    982
    0
      dsfmt_t dsfmt;
    983
    0
      dsfmt_init_gen_rand(&dsfmt, 2);
    984
    0
      int i;
    985
    0
      for (i = 0; i < batch_size * input_depth * input_height * input_width * input_channels; i++)
    986
    0
        a->data.f32[i] = dsfmt_genrand_open_close(&dsfmt) - 0.5;
    987
    0
      for (i = 0; i < output_channels * kernel_depth * kernel_height * kernel_width * input_channels; i++)
    988
    0
        w->data.f32[i] = (dsfmt_genrand_open_close(&dsfmt) - 0.5) * 0.1;
    989
    0
      for (i = 0; i < output_channels; i++)
    990
    0
        bias->data.f32[i] = (dsfmt_genrand_open_close(&dsfmt) - 0.5) * 0.1;
    991
    0
      ccv_nnc_tensor_t* ga = ccv_nnc_tensor_new(0, GPU_TENSOR_NHWC(000, 32F, batch_size, input_depth, input_height, input_width, input_channels), 0);
    992
    0
      ccv_nnc_tensor_t* gw = ccv_nnc_tensor_new(0, GPU_TENSOR_NHWC(000, 32F, output_channels, kernel_depth, kernel_height, kernel_width, input_channels), 0);
    993
    0
      ccv_nnc_tensor_t* gwo = ccv_nnc_tensor_new(0, GPU_TENSOR_NCHW(000, 32F, output_channels, input_channels, kernel_depth, kernel_height, kernel_width), 0);
    994
    0
      ccv_nnc_tensor_t* gbias = ccv_nnc_tensor_new(0, GPU_TENSOR_NHWC(000, 32F, output_channels), 0);
    995
    0
      ccv_nnc_tensor_t* gc = ccv_nnc_tensor_new(0, GPU_TENSOR_NHWC(000, 32F, batch_size, output_depth, output_height, output_width, output_channels), 0);
    996
    0
      ccv_nnc_cmd_t move = CMD_DATA_TRANSFER_FORWARD();
    997
    0
      move.backend = CCV_NNC_BACKEND_MPS;
    998
    0
      assert(move.backend >= 0);
    999
    0
      ccv_nnc_cmd_exec(move, ccv_nnc_no_hint, 0, TENSOR_LIST(a, w, bias), TENSOR_LIST(ga, gw, gbias), 0);
    1000
    0
      ccv_nnc_cmd_t transform = CMD_FORMAT_TRANSFORM_FORWARD();
    1001
    0
      transform.backend = CCV_NNC_BACKEND_MPS;
    1002
    0
      assert(transform.backend >= 0);
    1003
    0
      ccv_nnc_stream_context_t* stream_context = ccv_nnc_stream_context_new(CCV_STREAM_CONTEXT_GPU);
    1004
    0
      ccv_nnc_cmd_exec(transform, ccv_nnc_no_hint, 0, TENSOR_LIST(gw), TENSOR_LIST(gwo), stream_context);
    1005
    0
      ccv_nnc_stream_context_wait(stream_context);
    1006
    0
      ccv_nnc_tensor_free(gw);
    1007
    0
      cmd.backend = CCV_NNC_BACKEND_MPS;
    1008
    0
      assert(cmd.backend >= 0);
    1009
    0
      cmd.algorithm = -1;
    1010
    0
      cmd = ccv_nnc_cmd_autotune(cmd, 1 * 1024 * 1024 * 1024, hint, 0, TENSOR_LIST(ga, gwo, gbias), TENSOR_LIST(gc), stream_context);
    1011
    0
      assert(CCV_NNC_EXEC_SUCCESS == ccv_nnc_cmd_exec(cmd, hint, 0, TENSOR_LIST(ga, gwo, gbias), TENSOR_LIST(gc), stream_context));
    1012
    0
      ccv_nnc_stream_context_wait(stream_context);
    1013
    0
      ccv_nnc_stream_context_free(stream_context);
    1014
    0
      ccv_nnc_tensor_t* c = ccv_nnc_tensor_new(0, CPU_TENSOR_NHWC(32F, batch_size, output_depth, output_height, output_width, output_channels), 0);
    1015
    0
      ccv_nnc_cmd_exec(move, ccv_nnc_no_hint, 0, TENSOR_LIST(gc), TENSOR_LIST(c), 0);
    1016
    0
      cmd.backend = CCV_NNC_BACKEND_CPU_REF;
    1017
    0
      assert(cmd.backend >= 0);
    1018
    0
      ccv_nnc_cmd_exec(cmd, hint, 0, TENSOR_LIST(a, w, bias), TENSOR_LIST(b), 0);
    1019
    0
      REQUIRE_ARRAY_EQ_WITH_TOLERANCE(float, b->data.f32, c->data.f32, batch_size * output_depth * output_height * output_width * output_channels, 1e-4, "output from mps should match from CPU");
    1020
    0
      ccv_nnc_tensor_free(c);
    1021
    0
      ccv_nnc_tensor_free(gc);
    1022
    0
      ccv_nnc_tensor_free(bias);
    1023
    0
      ccv_nnc_tensor_free(w);
    1024
    0
      ccv_nnc_tensor_free(b);
    1025
    0
      ccv_nnc_tensor_free(a);
    1026
    0
      ccv_nnc_tensor_free(gbias);
    1027
    0
      ccv_nnc_tensor_free(gwo);
    1028
    0
      ccv_nnc_tensor_free(ga);
    1029
    0
    }
    1030
    1031
    TEST_CASE("mps forward convolution 3d via mfa conv3d with no padding on 17x17 spatial dimensions")
    1032
    1
    {
    1033
    1
      GUARD_ELSE_RETURN(ccv_nnc_cmd_ok(CCV_NNC_CONVOLUTION_FORWARD, CCV_NNC_BACKEND_MPS));
    1034
    0
      const int batch_size = 2;
    1035
    0
      const int input_channels = 16;
    1036
    0
      const int output_channels = 32;
    1037
    0
      const int input_depth = 5;
    1038
    0
      const int input_height = 17;
    1039
    0
      const int input_width = 17;
    1040
    0
      const int kernel_depth = 3;
    1041
    0
      const int kernel_height = 3;
    1042
    0
      const int kernel_width = 3;
    1043
    0
      const int output_depth = 3;
    1044
    0
      const int output_height = 15;
    1045
    0
      const int output_width = 15;
    1046
    0
      ccv_nnc_tensor_t* a = ccv_nnc_tensor_new(0, CPU_TENSOR_NHWC(32F, batch_size, input_depth, input_height, input_width, input_channels), 0);
    1047
    0
      ccv_nnc_tensor_t* b = ccv_nnc_tensor_new(0, CPU_TENSOR_NHWC(32F, batch_size, output_depth, output_height, output_width, output_channels), 0);
    1048
    0
      ccv_nnc_cmd_t cmd = CMD_CONVOLUTION_FORWARD(1, output_channels, kernel_depth, kernel_height, kernel_width, input_channels);
    1049
    0
      ccv_nnc_hint_t hint = ccv_nnc_hint_auto(cmd.info, a->info, b->info);
    1050
    0
      assert(ccv_nnc_hint_verify(hint, cmd.info, a->info, b->info) == 0);
    1051
    0
      ccv_nnc_tensor_t* w = ccv_nnc_tensor_new(0, CPU_TENSOR_NHWC(32F, output_channels, kernel_depth, kernel_height, kernel_width, input_channels), 0);
    1052
    0
      dsfmt_t dsfmt;
    1053
    0
      dsfmt_init_gen_rand(&dsfmt, 21);
    1054
    0
      int i;
    1055
    0
      for (i = 0; i < batch_size * input_depth * input_height * input_width * input_channels; i++)
    1056
    0
        a->data.f32[i] = dsfmt_genrand_open_close(&dsfmt) - 0.5;
    1057
    0
      for (i = 0; i < output_channels * kernel_depth * kernel_height * kernel_width * input_channels; i++)
    1058
    0
        w->data.f32[i] = (dsfmt_genrand_open_close(&dsfmt) - 0.5) * 0.1;
    1059
    0
      ccv_nnc_tensor_t* ga = ccv_nnc_tensor_new(0, GPU_TENSOR_NHWC(000, 32F, batch_size, input_depth, input_height, input_width, input_channels), 0);
    1060
    0
      ccv_nnc_tensor_t* gw = ccv_nnc_tensor_new(0, GPU_TENSOR_NHWC(000, 32F, output_channels, kernel_depth, kernel_height, kernel_width, input_channels), 0);
    1061
    0
      ccv_nnc_tensor_t* gwo = ccv_nnc_tensor_new(0, GPU_TENSOR_NCHW(000, 32F, output_channels, input_channels, kernel_depth, kernel_height, kernel_width), 0);
    1062
    0
      ccv_nnc_tensor_t* gc = ccv_nnc_tensor_new(0, GPU_TENSOR_NHWC(000, 32F, batch_size, output_depth, output_height, output_width, output_channels), 0);
    1063
    0
      ccv_nnc_cmd_t move = CMD_DATA_TRANSFER_FORWARD();
    1064
    0
      move.backend = CCV_NNC_BACKEND_MPS;
    1065
    0
      assert(move.backend >= 0);
    1066
    0
      ccv_nnc_cmd_exec(move, ccv_nnc_no_hint, 0, TENSOR_LIST(a, w), TENSOR_LIST(ga, gw), 0);
    1067
    0
      ccv_nnc_cmd_t transform = CMD_FORMAT_TRANSFORM_FORWARD();
    1068
    0
      transform.backend = CCV_NNC_BACKEND_MPS;
    1069
    0
      assert(transform.backend >= 0);
    1070
    0
      ccv_nnc_stream_context_t* stream_context = ccv_nnc_stream_context_new(CCV_STREAM_CONTEXT_GPU);
    1071
    0
      ccv_nnc_cmd_exec(transform, ccv_nnc_no_hint, 0, TENSOR_LIST(gw), TENSOR_LIST(gwo), stream_context);
    1072
    0
      ccv_nnc_stream_context_wait(stream_context);
    1073
    0
      ccv_nnc_tensor_free(gw);
    1074
    0
      cmd.backend = CCV_NNC_BACKEND_MPS;
    1075
    0
      assert(cmd.backend >= 0);
    1076
    0
      cmd.algorithm = -1;
    1077
    0
      cmd = ccv_nnc_cmd_autotune(cmd, 1 * 1024 * 1024 * 1024, hint, 0, TENSOR_LIST(ga, gwo), TENSOR_LIST(gc), stream_context);
    1078
    0
      assert(CCV_NNC_EXEC_SUCCESS == ccv_nnc_cmd_exec(cmd, hint, 0, TENSOR_LIST(ga, gwo), TENSOR_LIST(gc), stream_context));
    1079
    0
      ccv_nnc_stream_context_wait(stream_context);
    1080
    0
      ccv_nnc_stream_context_free(stream_context);
    1081
    0
      ccv_nnc_tensor_t* c = ccv_nnc_tensor_new(0, CPU_TENSOR_NHWC(32F, batch_size, output_depth, output_height, output_width, output_channels), 0);
    1082
    0
      ccv_nnc_cmd_exec(move, ccv_nnc_no_hint, 0, TENSOR_LIST(gc), TENSOR_LIST(c), 0);
    1083
    0
      cmd.backend = CCV_NNC_BACKEND_CPU_REF;
    1084
    0
      assert(cmd.backend >= 0);
    1085
    0
      ccv_nnc_cmd_exec(cmd, hint, 0, TENSOR_LIST(a, w), TENSOR_LIST(b), 0);
    1086
    0
      REQUIRE_ARRAY_EQ_WITH_TOLERANCE(float, b->data.f32, c->data.f32, batch_size * output_depth * output_height * output_width * output_channels, 1e-4, "output from mps should match from CPU");
    1087
    0
      ccv_nnc_tensor_free(c);
    1088
    0
      ccv_nnc_tensor_free(gc);
    1089
    0
      ccv_nnc_tensor_free(w);
    1090
    0
      ccv_nnc_tensor_free(b);
    1091
    0
      ccv_nnc_tensor_free(a);
    1092
    0
      ccv_nnc_tensor_free(gwo);
    1093
    0
      ccv_nnc_tensor_free(ga);
    1094
    0
    }
    1095
    1096
    TEST_CASE("mps forward convolution 3d via mfa conv3d with no padding on 65x65 spatial dimensions")
    1097
    1
    {
    1098
    1
      GUARD_ELSE_RETURN(ccv_nnc_cmd_ok(CCV_NNC_CONVOLUTION_FORWARD, CCV_NNC_BACKEND_MPS));
    1099
    0
      const int batch_size = 2;
    1100
    0
      const int input_channels = 16;
    1101
    0
      const int output_channels = 32;
    1102
    0
      const int input_depth = 5;
    1103
    0
      const int input_height = 65;
    1104
    0
      const int input_width = 65;
    1105
    0
      const int kernel_depth = 3;
    1106
    0
      const int kernel_height = 3;
    1107
    0
      const int kernel_width = 3;
    1108
    0
      const int output_depth = 3;
    1109
    0
      const int output_height = 63;
    1110
    0
      const int output_width = 63;
    1111
    0
      ccv_nnc_tensor_t* a = ccv_nnc_tensor_new(0, CPU_TENSOR_NHWC(32F, batch_size, input_depth, input_height, input_width, input_channels), 0);
    1112
    0
      ccv_nnc_tensor_t* b = ccv_nnc_tensor_new(0, CPU_TENSOR_NHWC(32F, batch_size, output_depth, output_height, output_width, output_channels), 0);
    1113
    0
      ccv_nnc_cmd_t cmd = CMD_CONVOLUTION_FORWARD(1, output_channels, kernel_depth, kernel_height, kernel_width, input_channels);
    1114
    0
      ccv_nnc_hint_t hint = ccv_nnc_hint_auto(cmd.info, a->info, b->info);
    1115
    0
      assert(ccv_nnc_hint_verify(hint, cmd.info, a->info, b->info) == 0);
    1116
    0
      ccv_nnc_tensor_t* w = ccv_nnc_tensor_new(0, CPU_TENSOR_NHWC(32F, output_channels, kernel_depth, kernel_height, kernel_width, input_channels), 0);
    1117
    0
      dsfmt_t dsfmt;
    1118
    0
      dsfmt_init_gen_rand(&dsfmt, 22);
    1119
    0
      int i;
    1120
    0
      for (i = 0; i < batch_size * input_depth * input_height * input_width * input_channels; i++)
    1121
    0
        a->data.f32[i] = dsfmt_genrand_open_close(&dsfmt) - 0.5;
    1122
    0
      for (i = 0; i < output_channels * kernel_depth * kernel_height * kernel_width * input_channels; i++)
    1123
    0
        w->data.f32[i] = (dsfmt_genrand_open_close(&dsfmt) - 0.5) * 0.1;
    1124
    0
      ccv_nnc_tensor_t* ga = ccv_nnc_tensor_new(0, GPU_TENSOR_NHWC(000, 32F, batch_size, input_depth, input_height, input_width, input_channels), 0);
    1125
    0
      ccv_nnc_tensor_t* gw = ccv_nnc_tensor_new(0, GPU_TENSOR_NHWC(000, 32F, output_channels, kernel_depth, kernel_height, kernel_width, input_channels), 0);
    1126
    0
      ccv_nnc_tensor_t* gwo = ccv_nnc_tensor_new(0, GPU_TENSOR_NCHW(000, 32F, output_channels, input_channels, kernel_depth, kernel_height, kernel_width), 0);
    1127
    0
      ccv_nnc_tensor_t* gc = ccv_nnc_tensor_new(0, GPU_TENSOR_NHWC(000, 32F, batch_size, output_depth, output_height, output_width, output_channels), 0);
    1128
    0
      ccv_nnc_cmd_t move = CMD_DATA_TRANSFER_FORWARD();
    1129
    0
      move.backend = CCV_NNC_BACKEND_MPS;
    1130
    0
      assert(move.backend >= 0);
    1131
    0
      ccv_nnc_cmd_exec(move, ccv_nnc_no_hint, 0, TENSOR_LIST(a, w), TENSOR_LIST(ga, gw), 0);
    1132
    0
      ccv_nnc_cmd_t transform = CMD_FORMAT_TRANSFORM_FORWARD();
    1133
    0
      transform.backend = CCV_NNC_BACKEND_MPS;
    1134
    0
      assert(transform.backend >= 0);
    1135
    0
      ccv_nnc_stream_context_t* stream_context = ccv_nnc_stream_context_new(CCV_STREAM_CONTEXT_GPU);
    1136
    0
      ccv_nnc_cmd_exec(transform, ccv_nnc_no_hint, 0, TENSOR_LIST(gw), TENSOR_LIST(gwo), stream_context);
    1137
    0
      ccv_nnc_stream_context_wait(stream_context);
    1138
    0
      ccv_nnc_tensor_free(gw);
    1139
    0
      cmd.backend = CCV_NNC_BACKEND_MPS;
    1140
    0
      assert(cmd.backend >= 0);
    1141
    0
      cmd.algorithm = -1;
    1142
    0
      cmd = ccv_nnc_cmd_autotune(cmd, 1 * 1024 * 1024 * 1024, hint, 0, TENSOR_LIST(ga, gwo), TENSOR_LIST(gc), stream_context);
    1143
    0
      assert(CCV_NNC_EXEC_SUCCESS == ccv_nnc_cmd_exec(cmd, hint, 0, TENSOR_LIST(ga, gwo), TENSOR_LIST(gc), stream_context));
    1144
    0
      ccv_nnc_stream_context_wait(stream_context);
    1145
    0
      ccv_nnc_stream_context_free(stream_context);
    1146
    0
      ccv_nnc_tensor_t* c = ccv_nnc_tensor_new(0, CPU_TENSOR_NHWC(32F, batch_size, output_depth, output_height, output_width, output_channels), 0);
    1147
    0
      ccv_nnc_cmd_exec(move, ccv_nnc_no_hint, 0, TENSOR_LIST(gc), TENSOR_LIST(c), 0);
    1148
    0
      cmd.backend = CCV_NNC_BACKEND_CPU_REF;
    1149
    0
      assert(cmd.backend >= 0);
    1150
    0
      ccv_nnc_cmd_exec(cmd, hint, 0, TENSOR_LIST(a, w), TENSOR_LIST(b), 0);
    1151
    0
      REQUIRE_ARRAY_EQ_WITH_TOLERANCE(float, b->data.f32, c->data.f32, batch_size * output_depth * output_height * output_width * output_channels, 1e-4, "output from mps should match from CPU");
    1152
    0
      ccv_nnc_tensor_free(c);
    1153
    0
      ccv_nnc_tensor_free(gc);
    1154
    0
      ccv_nnc_tensor_free(w);
    1155
    0
      ccv_nnc_tensor_free(b);
    1156
    0
      ccv_nnc_tensor_free(a);
    1157
    0
      ccv_nnc_tensor_free(gwo);
    1158
    0
      ccv_nnc_tensor_free(ga);
    1159
    0
    }
    1160
    1161
    TEST_CASE("mps forward convolution 3d via mfa conv3d with no padding on 129x129 spatial dimensions")
    1162
    1
    {
    1163
    1
      GUARD_ELSE_RETURN(ccv_nnc_cmd_ok(CCV_NNC_CONVOLUTION_FORWARD, CCV_NNC_BACKEND_MPS));
    1164
    0
      const int batch_size = 2;
    1165
    0
      const int input_channels = 16;
    1166
    0
      const int output_channels = 32;
    1167
    0
      const int input_depth = 5;
    1168
    0
      const int input_height = 129;
    1169
    0
      const int input_width = 129;
    1170
    0
      const int kernel_depth = 3;
    1171
    0
      const int kernel_height = 3;
    1172
    0
      const int kernel_width = 3;
    1173
    0
      const int output_depth = 3;
    1174
    0
      const int output_height = 127;
    1175
    0
      const int output_width = 127;
    1176
    0
      ccv_nnc_tensor_t* a = ccv_nnc_tensor_new(0, CPU_TENSOR_NHWC(32F, batch_size, input_depth, input_height, input_width, input_channels), 0);
    1177
    0
      ccv_nnc_tensor_t* b = ccv_nnc_tensor_new(0, CPU_TENSOR_NHWC(32F, batch_size, output_depth, output_height, output_width, output_channels), 0);
    1178
    0
      ccv_nnc_cmd_t cmd = CMD_CONVOLUTION_FORWARD(1, output_channels, kernel_depth, kernel_height, kernel_width, input_channels);
    1179
    0
      ccv_nnc_hint_t hint = ccv_nnc_hint_auto(cmd.info, a->info, b->info);
    1180
    0
      assert(ccv_nnc_hint_verify(hint, cmd.info, a->info, b->info) == 0);
    1181
    0
      ccv_nnc_tensor_t* w = ccv_nnc_tensor_new(0, CPU_TENSOR_NHWC(32F, output_channels, kernel_depth, kernel_height, kernel_width, input_channels), 0);
    1182
    0
      dsfmt_t dsfmt;
    1183
    0
      dsfmt_init_gen_rand(&dsfmt, 23);
    1184
    0
      int i;
    1185
    0
      for (i = 0; i < batch_size * input_depth * input_height * input_width * input_channels; i++)
    1186
    0
        a->data.f32[i] = dsfmt_genrand_open_close(&dsfmt) - 0.5;
    1187
    0
      for (i = 0; i < output_channels * kernel_depth * kernel_height * kernel_width * input_channels; i++)
    1188
    0
        w->data.f32[i] = (dsfmt_genrand_open_close(&dsfmt) - 0.5) * 0.1;
    1189
    0
      ccv_nnc_tensor_t* ga = ccv_nnc_tensor_new(0, GPU_TENSOR_NHWC(000, 32F, batch_size, input_depth, input_height, input_width, input_channels), 0);
    1190
    0
      ccv_nnc_tensor_t* gw = ccv_nnc_tensor_new(0, GPU_TENSOR_NHWC(000, 32F, output_channels, kernel_depth, kernel_height, kernel_width, input_channels), 0);
    1191
    0
      ccv_nnc_tensor_t* gwo = ccv_nnc_tensor_new(0, GPU_TENSOR_NCHW(000, 32F, output_channels, input_channels, kernel_depth, kernel_height, kernel_width), 0);
    1192
    0
      ccv_nnc_tensor_t* gc = ccv_nnc_tensor_new(0, GPU_TENSOR_NHWC(000, 32F, batch_size, output_depth, output_height, output_width, output_channels), 0);
    1193
    0
      ccv_nnc_cmd_t move = CMD_DATA_TRANSFER_FORWARD();
    1194
    0
      move.backend = CCV_NNC_BACKEND_MPS;
    1195
    0
      assert(move.backend >= 0);
    1196
    0
      ccv_nnc_cmd_exec(move, ccv_nnc_no_hint, 0, TENSOR_LIST(a, w), TENSOR_LIST(ga, gw), 0);
    1197
    0
      ccv_nnc_cmd_t transform = CMD_FORMAT_TRANSFORM_FORWARD();
    1198
    0
      transform.backend = CCV_NNC_BACKEND_MPS;
    1199
    0
      assert(transform.backend >= 0);
    1200
    0
      ccv_nnc_stream_context_t* stream_context = ccv_nnc_stream_context_new(CCV_STREAM_CONTEXT_GPU);
    1201
    0
      ccv_nnc_cmd_exec(transform, ccv_nnc_no_hint, 0, TENSOR_LIST(gw), TENSOR_LIST(gwo), stream_context);
    1202
    0
      ccv_nnc_stream_context_wait(stream_context);
    1203
    0
      ccv_nnc_tensor_free(gw);
    1204
    0
      cmd.backend = CCV_NNC_BACKEND_MPS;
    1205
    0
      assert(cmd.backend >= 0);
    1206
    0
      cmd.algorithm = -1;
    1207
    0
      cmd = ccv_nnc_cmd_autotune(cmd, 1 * 1024 * 1024 * 1024, hint, 0, TENSOR_LIST(ga, gwo), TENSOR_LIST(gc), stream_context);
    1208
    0
      assert(CCV_NNC_EXEC_SUCCESS == ccv_nnc_cmd_exec(cmd, hint, 0, TENSOR_LIST(ga, gwo), TENSOR_LIST(gc), stream_context));
    1209
    0
      ccv_nnc_stream_context_wait(stream_context);
    1210
    0
      ccv_nnc_stream_context_free(stream_context);
    1211
    0
      ccv_nnc_tensor_t* c = ccv_nnc_tensor_new(0, CPU_TENSOR_NHWC(32F, batch_size, output_depth, output_height, output_width, output_channels), 0);
    1212
    0
      ccv_nnc_cmd_exec(move, ccv_nnc_no_hint, 0, TENSOR_LIST(gc), TENSOR_LIST(c), 0);
    1213
    0
      cmd.backend = CCV_NNC_BACKEND_CPU_REF;
    1214
    0
      assert(cmd.backend >= 0);
    1215
    0
      ccv_nnc_cmd_exec(cmd, hint, 0, TENSOR_LIST(a, w), TENSOR_LIST(b), 0);
    1216
    0
      REQUIRE_ARRAY_EQ_WITH_TOLERANCE(float, b->data.f32, c->data.f32, batch_size * output_depth * output_height * output_width * output_channels, 1e-4, "output from mps should match from CPU");
    1217
    0
      ccv_nnc_tensor_free(c);
    1218
    0
      ccv_nnc_tensor_free(gc);
    1219
    0
      ccv_nnc_tensor_free(w);
    1220
    0
      ccv_nnc_tensor_free(b);
    1221
    0
      ccv_nnc_tensor_free(a);
    1222
    0
      ccv_nnc_tensor_free(gwo);
    1223
    0
      ccv_nnc_tensor_free(ga);
    1224
    0
    }
    1225
    1226
    TEST_CASE("mps forward convolution 3d via mfa conv3d 5x5")
    1227
    1
    {
    1228
    1
      GUARD_ELSE_RETURN(ccv_nnc_cmd_ok(CCV_NNC_CONVOLUTION_FORWARD, CCV_NNC_BACKEND_MPS));
    1229
    0
      const int batch_size = 2;
    1230
    0
      const int input_channels = 16;
    1231
    0
      const int output_channels = 32;
    1232
    0
      const int input_depth = 5;
    1233
    0
      const int input_height = 12;
    1234
    0
      const int input_width = 12;
    1235
    0
      const int kernel_depth = 3;
    1236
    0
      const int kernel_height = 5;
    1237
    0
      const int kernel_width = 5;
    1238
    0
      const int output_depth = 3;
    1239
    0
      const int output_height = 8;
    1240
    0
      const int output_width = 8;
    1241
    0
      ccv_nnc_tensor_t* a = ccv_nnc_tensor_new(0, CPU_TENSOR_NHWC(32F, batch_size, input_depth, input_height, input_width, input_channels), 0);
    1242
    0
      ccv_nnc_tensor_t* b = ccv_nnc_tensor_new(0, CPU_TENSOR_NHWC(32F, batch_size, output_depth, output_height, output_width, output_channels), 0);
    1243
    0
      ccv_nnc_cmd_t cmd = CMD_CONVOLUTION_FORWARD(1, output_channels, kernel_depth, kernel_height, kernel_width, input_channels);
    1244
    0
      ccv_nnc_hint_t hint = ccv_nnc_hint_auto(cmd.info, a->info, b->info);
    1245
    0
      assert(ccv_nnc_hint_verify(hint, cmd.info, a->info, b->info) == 0);
    1246
    0
      ccv_nnc_tensor_t* w = ccv_nnc_tensor_new(0, CPU_TENSOR_NHWC(32F, output_channels, kernel_depth, kernel_height, kernel_width, input_channels), 0);
    1247
    0
      dsfmt_t dsfmt;
    1248
    0
      dsfmt_init_gen_rand(&dsfmt, 3);
    1249
    0
      int i;
    1250
    0
      for (i = 0; i < batch_size * input_depth * input_height * input_width * input_channels; i++)
    1251
    0
        a->data.f32[i] = dsfmt_genrand_open_close(&dsfmt) - 0.5;
    1252
    0
      for (i = 0; i < output_channels * kernel_depth * kernel_height * kernel_width * input_channels; i++)
    1253
    0
        w->data.f32[i] = (dsfmt_genrand_open_close(&dsfmt) - 0.5) * 0.1;
    1254
    0
      ccv_nnc_tensor_t* ga = ccv_nnc_tensor_new(0, GPU_TENSOR_NHWC(000, 32F, batch_size, input_depth, input_height, input_width, input_channels), 0);
    1255
    0
      ccv_nnc_tensor_t* gw = ccv_nnc_tensor_new(0, GPU_TENSOR_NHWC(000, 32F, output_channels, kernel_depth, kernel_height, kernel_width, input_channels), 0);
    1256
    0
      ccv_nnc_tensor_t* gwo = ccv_nnc_tensor_new(0, GPU_TENSOR_NCHW(000, 32F, output_channels, input_channels, kernel_depth, kernel_height, kernel_width), 0);
    1257
    0
      ccv_nnc_tensor_t* gc = ccv_nnc_tensor_new(0, GPU_TENSOR_NHWC(000, 32F, batch_size, output_depth, output_height, output_width, output_channels), 0);
    1258
    0
      ccv_nnc_cmd_t move = CMD_DATA_TRANSFER_FORWARD();
    1259
    0
      move.backend = CCV_NNC_BACKEND_MPS;
    1260
    0
      assert(move.backend >= 0);
    1261
    0
      ccv_nnc_cmd_exec(move, ccv_nnc_no_hint, 0, TENSOR_LIST(a, w), TENSOR_LIST(ga, gw), 0);
    1262
    0
      ccv_nnc_cmd_t transform = CMD_FORMAT_TRANSFORM_FORWARD();
    1263
    0
      transform.backend = CCV_NNC_BACKEND_MPS;
    1264
    0
      assert(transform.backend >= 0);
    1265
    0
      ccv_nnc_stream_context_t* stream_context = ccv_nnc_stream_context_new(CCV_STREAM_CONTEXT_GPU);
    1266
    0
      ccv_nnc_cmd_exec(transform, ccv_nnc_no_hint, 0, TENSOR_LIST(gw), TENSOR_LIST(gwo), stream_context);
    1267
    0
      ccv_nnc_stream_context_wait(stream_context);
    1268
    0
      ccv_nnc_tensor_free(gw);
    1269
    0
      cmd.backend = CCV_NNC_BACKEND_MPS;
    1270
    0
      assert(cmd.backend >= 0);
    1271
    0
      cmd.algorithm = -1;
    1272
    0
      cmd = ccv_nnc_cmd_autotune(cmd, 1 * 1024 * 1024 * 1024, hint, 0, TENSOR_LIST(ga, gwo), TENSOR_LIST(gc), stream_context);
    1273
    0
      assert(CCV_NNC_EXEC_SUCCESS == ccv_nnc_cmd_exec(cmd, hint, 0, TENSOR_LIST(ga, gwo), TENSOR_LIST(gc), stream_context));
    1274
    0
      ccv_nnc_stream_context_wait(stream_context);
    1275
    0
      ccv_nnc_stream_context_free(stream_context);
    1276
    0
      ccv_nnc_tensor_t* c = ccv_nnc_tensor_new(0, CPU_TENSOR_NHWC(32F, batch_size, output_depth, output_height, output_width, output_channels), 0);
    1277
    0
      ccv_nnc_cmd_exec(move, ccv_nnc_no_hint, 0, TENSOR_LIST(gc), TENSOR_LIST(c), 0);
    1278
    0
      cmd.backend = CCV_NNC_BACKEND_CPU_REF;
    1279
    0
      assert(cmd.backend >= 0);
    1280
    0
      ccv_nnc_cmd_exec(cmd, hint, 0, TENSOR_LIST(a, w), TENSOR_LIST(b), 0);
    1281
    0
      REQUIRE_ARRAY_EQ_WITH_TOLERANCE(float, b->data.f32, c->data.f32, batch_size * output_depth * output_height * output_width * output_channels, 1e-4, "output from mps should match from CPU");
    1282
    0
      ccv_nnc_tensor_free(c);
    1283
    0
      ccv_nnc_tensor_free(gc);
    1284
    0
      ccv_nnc_tensor_free(w);
    1285
    0
      ccv_nnc_tensor_free(b);
    1286
    0
      ccv_nnc_tensor_free(a);
    1287
    0
      ccv_nnc_tensor_free(gwo);
    1288
    0
      ccv_nnc_tensor_free(ga);
    1289
    0
    }
    1290
    1291
    TEST_CASE("mps forward convolution 3d via mfa conv3d 7x7")
    1292
    1
    {
    1293
    1
      GUARD_ELSE_RETURN(ccv_nnc_cmd_ok(CCV_NNC_CONVOLUTION_FORWARD, CCV_NNC_BACKEND_MPS));
    1294
    0
      const int batch_size = 2;
    1295
    0
      const int input_channels = 16;
    1296
    0
      const int output_channels = 32;
    1297
    0
      const int input_depth = 5;
    1298
    0
      const int input_height = 14;
    1299
    0
      const int input_width = 14;
    1300
    0
      const int kernel_depth = 3;
    1301
    0
      const int kernel_height = 7;
    1302
    0
      const int kernel_width = 7;
    1303
    0
      const int output_depth = 3;
    1304
    0
      const int output_height = 8;
    1305
    0
      const int output_width = 8;
    1306
    0
      ccv_nnc_tensor_t* a = ccv_nnc_tensor_new(0, CPU_TENSOR_NHWC(32F, batch_size, input_depth, input_height, input_width, input_channels), 0);
    1307
    0
      ccv_nnc_tensor_t* b = ccv_nnc_tensor_new(0, CPU_TENSOR_NHWC(32F, batch_size, output_depth, output_height, output_width, output_channels), 0);
    1308
    0
      ccv_nnc_cmd_t cmd = CMD_CONVOLUTION_FORWARD(1, output_channels, kernel_depth, kernel_height, kernel_width, input_channels);
    1309
    0
      ccv_nnc_hint_t hint = ccv_nnc_hint_auto(cmd.info, a->info, b->info);
    1310
    0
      assert(ccv_nnc_hint_verify(hint, cmd.info, a->info, b->info) == 0);
    1311
    0
      ccv_nnc_tensor_t* w = ccv_nnc_tensor_new(0, CPU_TENSOR_NHWC(32F, output_channels, kernel_depth, kernel_height, kernel_width, input_channels), 0);
    1312
    0
      dsfmt_t dsfmt;
    1313
    0
      dsfmt_init_gen_rand(&dsfmt, 4);
    1314
    0
      int i;
    1315
    0
      for (i = 0; i < batch_size * input_depth * input_height * input_width * input_channels; i++)
    1316
    0
        a->data.f32[i] = dsfmt_genrand_open_close(&dsfmt) - 0.5;
    1317
    0
      for (i = 0; i < output_channels * kernel_depth * kernel_height * kernel_width * input_channels; i++)
    1318
    0
        w->data.f32[i] = (dsfmt_genrand_open_close(&dsfmt) - 0.5) * 0.1;
    1319
    0
      ccv_nnc_tensor_t* ga = ccv_nnc_tensor_new(0, GPU_TENSOR_NHWC(000, 32F, batch_size, input_depth, input_height, input_width, input_channels), 0);
    1320
    0
      ccv_nnc_tensor_t* gw = ccv_nnc_tensor_new(0, GPU_TENSOR_NHWC(000, 32F, output_channels, kernel_depth, kernel_height, kernel_width, input_channels), 0);
    1321
    0
      ccv_nnc_tensor_t* gwo = ccv_nnc_tensor_new(0, GPU_TENSOR_NCHW(000, 32F, output_channels, input_channels, kernel_depth, kernel_height, kernel_width), 0);
    1322
    0
      ccv_nnc_tensor_t* gc = ccv_nnc_tensor_new(0, GPU_TENSOR_NHWC(000, 32F, batch_size, output_depth, output_height, output_width, output_channels), 0);
    1323
    0
      ccv_nnc_cmd_t move = CMD_DATA_TRANSFER_FORWARD();
    1324
    0
      move.backend = CCV_NNC_BACKEND_MPS;
    1325
    0
      assert(move.backend >= 0);
    1326
    0
      ccv_nnc_cmd_exec(move, ccv_nnc_no_hint, 0, TENSOR_LIST(a, w), TENSOR_LIST(ga, gw), 0);
    1327
    0
      ccv_nnc_cmd_t transform = CMD_FORMAT_TRANSFORM_FORWARD();
    1328
    0
      transform.backend = CCV_NNC_BACKEND_MPS;
    1329
    0
      assert(transform.backend >= 0);
    1330
    0
      ccv_nnc_stream_context_t* stream_context = ccv_nnc_stream_context_new(CCV_STREAM_CONTEXT_GPU);
    1331
    0
      ccv_nnc_cmd_exec(transform, ccv_nnc_no_hint, 0, TENSOR_LIST(gw), TENSOR_LIST(gwo), stream_context);
    1332
    0
      ccv_nnc_stream_context_wait(stream_context);
    1333
    0
      ccv_nnc_tensor_free(gw);
    1334
    0
      cmd.backend = CCV_NNC_BACKEND_MPS;
    1335
    0
      assert(cmd.backend >= 0);
    1336
    0
      cmd.algorithm = -1;
    1337
    0
      cmd = ccv_nnc_cmd_autotune(cmd, 1 * 1024 * 1024 * 1024, hint, 0, TENSOR_LIST(ga, gwo), TENSOR_LIST(gc), stream_context);
    1338
    0
      assert(CCV_NNC_EXEC_SUCCESS == ccv_nnc_cmd_exec(cmd, hint, 0, TENSOR_LIST(ga, gwo), TENSOR_LIST(gc), stream_context));
    1339
    0
      ccv_nnc_stream_context_wait(stream_context);
    1340
    0
      ccv_nnc_stream_context_free(stream_context);
    1341
    0
      ccv_nnc_tensor_t* c = ccv_nnc_tensor_new(0, CPU_TENSOR_NHWC(32F, batch_size, output_depth, output_height, output_width, output_channels), 0);
    1342
    0
      ccv_nnc_cmd_exec(move, ccv_nnc_no_hint, 0, TENSOR_LIST(gc), TENSOR_LIST(c), 0);
    1343
    0
      cmd.backend = CCV_NNC_BACKEND_CPU_REF;
    1344
    0
      assert(cmd.backend >= 0);
    1345
    0
      ccv_nnc_cmd_exec(cmd, hint, 0, TENSOR_LIST(a, w), TENSOR_LIST(b), 0);
    1346
    0
      REQUIRE_ARRAY_EQ_WITH_TOLERANCE(float, b->data.f32, c->data.f32, batch_size * output_depth * output_height * output_width * output_channels, 1e-4, "output from mps should match from CPU");
    1347
    0
      ccv_nnc_tensor_free(c);
    1348
    0
      ccv_nnc_tensor_free(gc);
    1349
    0
      ccv_nnc_tensor_free(w);
    1350
    0
      ccv_nnc_tensor_free(b);
    1351
    0
      ccv_nnc_tensor_free(a);
    1352
    0
      ccv_nnc_tensor_free(gwo);
    1353
    0
      ccv_nnc_tensor_free(ga);
    1354
    0
    }
    1355
    1356
    TEST_CASE("mps forward convolution 3d via mfa conv3d with padding")
    1357
    1
    {
    1358
    1
      GUARD_ELSE_RETURN(ccv_nnc_cmd_ok(CCV_NNC_CONVOLUTION_FORWARD, CCV_NNC_BACKEND_MPS));
    1359
    0
      const int batch_size = 2;
    1360
    0
      const int input_channels = 16;
    1361
    0
      const int output_channels = 32;
    1362
    0
      const int input_depth = 5;
    1363
    0
      const int input_height = 10;
    1364
    0
      const int input_width = 10;
    1365
    0
      const int kernel_depth = 3;
    1366
    0
      const int kernel_height = 3;
    1367
    0
      const int kernel_width = 3;
    1368
    0
      const int padding_top = 1;
    1369
    0
      const int padding_bottom = 1;
    1370
    0
      const int padding_left = 1;
    1371
    0
      const int padding_right = 1;
    1372
    0
      const int output_depth = input_depth - kernel_depth + 1;
    1373
    0
      const int output_height = input_height + padding_top + padding_bottom - kernel_height + 1;
    1374
    0
      const int output_width = input_width + padding_left + padding_right - kernel_width + 1;
    1375
    0
      ccv_nnc_tensor_t* a = ccv_nnc_tensor_new(0, CPU_TENSOR_NHWC(32F, batch_size, input_depth, input_height, input_width, input_channels), 0);
    1376
    0
      ccv_nnc_tensor_t* b = ccv_nnc_tensor_new(0, CPU_TENSOR_NHWC(32F, batch_size, output_depth, output_height, output_width, output_channels), 0);
    1377
    0
      ccv_nnc_cmd_t cmd = CMD_CONVOLUTION_FORWARD(1, output_channels, kernel_depth, kernel_height, kernel_width, input_channels);
    1378
    0
      ccv_nnc_hint_t hint = ccv_nnc_no_hint;
    1379
    0
      hint.stride.dim[0] = 1;
    1380
    0
      hint.stride.dim[1] = 1;
    1381
    0
      hint.stride.dim[2] = 1;
    1382
    0
      hint.border.begin[0] = 0;
    1383
    0
      hint.border.end[0] = 0;
    1384
    0
      hint.border.begin[1] = padding_top;
    1385
    0
      hint.border.end[1] = padding_bottom;
    1386
    0
      hint.border.begin[2] = padding_left;
    1387
    0
      hint.border.end[2] = padding_right;
    1388
    0
      assert(ccv_nnc_hint_verify(hint, cmd.info, a->info, b->info) == 0);
    1389
    0
      ccv_nnc_tensor_t* w = ccv_nnc_tensor_new(0, CPU_TENSOR_NHWC(32F, output_channels, kernel_depth, kernel_height, kernel_width, input_channels), 0);
    1390
    0
      dsfmt_t dsfmt;
    1391
    0
      dsfmt_init_gen_rand(&dsfmt, 5);
    1392
    0
      int i;
    1393
    0
      for (i = 0; i < batch_size * input_depth * input_height * input_width * input_channels; i++)
    1394
    0
        a->data.f32[i] = dsfmt_genrand_open_close(&dsfmt) - 0.5;
    1395
    0
      for (i = 0; i < output_channels * kernel_depth * kernel_height * kernel_width * input_channels; i++)
    1396
    0
        w->data.f32[i] = (dsfmt_genrand_open_close(&dsfmt) - 0.5) * 0.1;
    1397
    0
      ccv_nnc_tensor_t* ga = ccv_nnc_tensor_new(0, GPU_TENSOR_NHWC(000, 32F, batch_size, input_depth, input_height, input_width, input_channels), 0);
    1398
    0
      ccv_nnc_tensor_t* gw = ccv_nnc_tensor_new(0, GPU_TENSOR_NHWC(000, 32F, output_channels, kernel_depth, kernel_height, kernel_width, input_channels), 0);
    1399
    0
      ccv_nnc_tensor_t* gwo = ccv_nnc_tensor_new(0, GPU_TENSOR_NCHW(000, 32F, output_channels, input_channels, kernel_depth, kernel_height, kernel_width), 0);
    1400
    0
      ccv_nnc_tensor_t* gc = ccv_nnc_tensor_new(0, GPU_TENSOR_NHWC(000, 32F, batch_size, output_depth, output_height, output_width, output_channels), 0);
    1401
    0
      ccv_nnc_cmd_t move = CMD_DATA_TRANSFER_FORWARD();
    1402
    0
      move.backend = CCV_NNC_BACKEND_MPS;
    1403
    0
      assert(move.backend >= 0);
    1404
    0
      ccv_nnc_cmd_exec(move, ccv_nnc_no_hint, 0, TENSOR_LIST(a, w), TENSOR_LIST(ga, gw), 0);
    1405
    0
      ccv_nnc_cmd_t transform = CMD_FORMAT_TRANSFORM_FORWARD();
    1406
    0
      transform.backend = CCV_NNC_BACKEND_MPS;
    1407
    0
      assert(transform.backend >= 0);
    1408
    0
      ccv_nnc_stream_context_t* stream_context = ccv_nnc_stream_context_new(CCV_STREAM_CONTEXT_GPU);
    1409
    0
      ccv_nnc_cmd_exec(transform, ccv_nnc_no_hint, 0, TENSOR_LIST(gw), TENSOR_LIST(gwo), stream_context);
    1410
    0
      ccv_nnc_stream_context_wait(stream_context);
    1411
    0
      ccv_nnc_tensor_free(gw);
    1412
    0
      cmd.backend = CCV_NNC_BACKEND_MPS;
    1413
    0
      assert(cmd.backend >= 0);
    1414
    0
      cmd.algorithm = -1;
    1415
    0
      cmd = ccv_nnc_cmd_autotune(cmd, 1 * 1024 * 1024 * 1024, hint, 0, TENSOR_LIST(ga, gwo), TENSOR_LIST(gc), stream_context);
    1416
    0
      assert(CCV_NNC_EXEC_SUCCESS == ccv_nnc_cmd_exec(cmd, hint, 0, TENSOR_LIST(ga, gwo), TENSOR_LIST(gc), stream_context));
    1417
    0
      ccv_nnc_stream_context_wait(stream_context);
    1418
    0
      ccv_nnc_stream_context_free(stream_context);
    1419
    0
      ccv_nnc_tensor_t* c = ccv_nnc_tensor_new(0, CPU_TENSOR_NHWC(32F, batch_size, output_depth, output_height, output_width, output_channels), 0);
    1420
    0
      ccv_nnc_cmd_exec(move, ccv_nnc_no_hint, 0, TENSOR_LIST(gc), TENSOR_LIST(c), 0);
    1421
    0
      cmd.backend = CCV_NNC_BACKEND_CPU_REF;
    1422
    0
      assert(cmd.backend >= 0);
    1423
    0
      ccv_nnc_cmd_exec(cmd, hint, 0, TENSOR_LIST(a, w), TENSOR_LIST(b), 0);
    1424
    0
      REQUIRE_ARRAY_EQ_WITH_TOLERANCE(float, b->data.f32, c->data.f32, batch_size * output_depth * output_height * output_width * output_channels, 1e-4, "output from mps should match from CPU");
    1425
    0
      ccv_nnc_tensor_free(c);
    1426
    0
      ccv_nnc_tensor_free(gc);
    1427
    0
      ccv_nnc_tensor_free(w);
    1428
    0
      ccv_nnc_tensor_free(b);
    1429
    0
      ccv_nnc_tensor_free(a);
    1430
    0
      ccv_nnc_tensor_free(gwo);
    1431
    0
      ccv_nnc_tensor_free(ga);
    1432
    0
    }
    1433
    1434
    TEST_CASE("mps forward convolution 3d via mfa conv3d with asymmetric padding")
    1435
    1
    {
    1436
    1
      GUARD_ELSE_RETURN(ccv_nnc_cmd_ok(CCV_NNC_CONVOLUTION_FORWARD, CCV_NNC_BACKEND_MPS));
    1437
    0
      const int batch_size = 2;
    1438
    0
      const int input_channels = 16;
    1439
    0
      const int output_channels = 32;
    1440
    0
      const int input_depth = 5;
    1441
    0
      const int input_height = 10;
    1442
    0
      const int input_width = 11;
    1443
    0
      const int kernel_depth = 3;
    1444
    0
      const int kernel_height = 5;
    1445
    0
      const int kernel_width = 5;
    1446
    0
      const int padding_top = 1;
    1447
    0
      const int padding_bottom = 0;
    1448
    0
      const int padding_left = 2;
    1449
    0
      const int padding_right = 1;
    1450
    0
      const int output_depth = input_depth - kernel_depth + 1;
    1451
    0
      const int output_height = input_height + padding_top + padding_bottom - kernel_height + 1;
    1452
    0
      const int output_width = input_width + padding_left + padding_right - kernel_width + 1;
    1453
    0
      ccv_nnc_tensor_t* a = ccv_nnc_tensor_new(0, CPU_TENSOR_NHWC(32F, batch_size, input_depth, input_height, input_width, input_channels), 0);
    1454
    0
      ccv_nnc_tensor_t* b = ccv_nnc_tensor_new(0, CPU_TENSOR_NHWC(32F, batch_size, output_depth, output_height, output_width, output_channels), 0);
    1455
    0
      ccv_nnc_cmd_t cmd = CMD_CONVOLUTION_FORWARD(1, output_channels, kernel_depth, kernel_height, kernel_width, input_channels);
    1456
    0
      ccv_nnc_hint_t hint = ccv_nnc_no_hint;
    1457
    0
      hint.stride.dim[0] = 1;
    1458
    0
      hint.stride.dim[1] = 1;
    1459
    0
      hint.stride.dim[2] = 1;
    1460
    0
      hint.border.begin[0] = 0;
    1461
    0
      hint.border.end[0] = 0;
    1462
    0
      hint.border.begin[1] = padding_top;
    1463
    0
      hint.border.end[1] = padding_bottom;
    1464
    0
      hint.border.begin[2] = padding_left;
    1465
    0
      hint.border.end[2] = padding_right;
    1466
    0
      assert(ccv_nnc_hint_verify(hint, cmd.info, a->info, b->info) == 0);
    1467
    0
      ccv_nnc_tensor_t* w = ccv_nnc_tensor_new(0, CPU_TENSOR_NHWC(32F, output_channels, kernel_depth, kernel_height, kernel_width, input_channels), 0);
    1468
    0
      dsfmt_t dsfmt;
    1469
    0
      dsfmt_init_gen_rand(&dsfmt, 6);
    1470
    0
      int i;
    1471
    0
      for (i = 0; i < batch_size * input_depth * input_height * input_width * input_channels; i++)
    1472
    0
        a->data.f32[i] = dsfmt_genrand_open_close(&dsfmt) - 0.5;
    1473
    0
      for (i = 0; i < output_channels * kernel_depth * kernel_height * kernel_width * input_channels; i++)
    1474
    0
        w->data.f32[i] = (dsfmt_genrand_open_close(&dsfmt) - 0.5) * 0.1;
    1475
    0
      ccv_nnc_tensor_t* ga = ccv_nnc_tensor_new(0, GPU_TENSOR_NHWC(000, 32F, batch_size, input_depth, input_height, input_width, input_channels), 0);
    1476
    0
      ccv_nnc_tensor_t* gw = ccv_nnc_tensor_new(0, GPU_TENSOR_NHWC(000, 32F, output_channels, kernel_depth, kernel_height, kernel_width, input_channels), 0);
    1477
    0
      ccv_nnc_tensor_t* gwo = ccv_nnc_tensor_new(0, GPU_TENSOR_NCHW(000, 32F, output_channels, input_channels, kernel_depth, kernel_height, kernel_width), 0);
    1478
    0
      ccv_nnc_tensor_t* gc = ccv_nnc_tensor_new(0, GPU_TENSOR_NHWC(000, 32F, batch_size, output_depth, output_height, output_width, output_channels), 0);
    1479
    0
      ccv_nnc_cmd_t move = CMD_DATA_TRANSFER_FORWARD();
    1480
    0
      move.backend = CCV_NNC_BACKEND_MPS;
    1481
    0
      assert(move.backend >= 0);
    1482
    0
      ccv_nnc_cmd_exec(move, ccv_nnc_no_hint, 0, TENSOR_LIST(a, w), TENSOR_LIST(ga, gw), 0);
    1483
    0
      ccv_nnc_cmd_t transform = CMD_FORMAT_TRANSFORM_FORWARD();
    1484
    0
      transform.backend = CCV_NNC_BACKEND_MPS;
    1485
    0
      assert(transform.backend >= 0);
    1486
    0
      ccv_nnc_stream_context_t* stream_context = ccv_nnc_stream_context_new(CCV_STREAM_CONTEXT_GPU);
    1487
    0
      ccv_nnc_cmd_exec(transform, ccv_nnc_no_hint, 0, TENSOR_LIST(gw), TENSOR_LIST(gwo), stream_context);
    1488
    0
      ccv_nnc_stream_context_wait(stream_context);
    1489
    0
      ccv_nnc_tensor_free(gw);
    1490
    0
      cmd.backend = CCV_NNC_BACKEND_MPS;
    1491
    0
      assert(cmd.backend >= 0);
    1492
    0
      cmd.algorithm = -1;
    1493
    0
      cmd = ccv_nnc_cmd_autotune(cmd, 1 * 1024 * 1024 * 1024, hint, 0, TENSOR_LIST(ga, gwo), TENSOR_LIST(gc), stream_context);
    1494
    0
      assert(CCV_NNC_EXEC_SUCCESS == ccv_nnc_cmd_exec(cmd, hint, 0, TENSOR_LIST(ga, gwo), TENSOR_LIST(gc), stream_context));
    1495
    0
      ccv_nnc_stream_context_wait(stream_context);
    1496
    0
      ccv_nnc_stream_context_free(stream_context);
    1497
    0
      ccv_nnc_tensor_t* c = ccv_nnc_tensor_new(0, CPU_TENSOR_NHWC(32F, batch_size, output_depth, output_height, output_width, output_channels), 0);
    1498
    0
      ccv_nnc_cmd_exec(move, ccv_nnc_no_hint, 0, TENSOR_LIST(gc), TENSOR_LIST(c), 0);
    1499
    0
      cmd.backend = CCV_NNC_BACKEND_CPU_REF;
    1500
    0
      assert(cmd.backend >= 0);
    1501
    0
      ccv_nnc_cmd_exec(cmd, hint, 0, TENSOR_LIST(a, w), TENSOR_LIST(b), 0);
    1502
    0
      REQUIRE_ARRAY_EQ_WITH_TOLERANCE(float, b->data.f32, c->data.f32, batch_size * output_depth * output_height * output_width * output_channels, 1e-4, "output from mps should match from CPU");
    1503
    0
      ccv_nnc_tensor_free(c);
    1504
    0
      ccv_nnc_tensor_free(gc);
    1505
    0
      ccv_nnc_tensor_free(w);
    1506
    0
      ccv_nnc_tensor_free(b);
    1507
    0
      ccv_nnc_tensor_free(a);
    1508
    0
      ccv_nnc_tensor_free(gwo);
    1509
    0
      ccv_nnc_tensor_free(ga);
    1510
    0
    }
    1511
    1512
    TEST_CASE("mps forward convolution 3d via mfa conv3d with padding on small spatial dimensions")
    1513
    1
    {
    1514
    1
      GUARD_ELSE_RETURN(ccv_nnc_cmd_ok(CCV_NNC_CONVOLUTION_FORWARD, CCV_NNC_BACKEND_MPS));
    1515
    0
      const int batch_size = 2;
    1516
    0
      const int input_channels = 16;
    1517
    0
      const int output_channels = 32;
    1518
    0
      const int input_depth = 5;
    1519
    0
      const int input_height = 5;
    1520
    0
      const int input_width = 5;
    1521
    0
      const int kernel_depth = 3;
    1522
    0
      const int kernel_height = 3;
    1523
    0
      const int kernel_width = 3;
    1524
    0
      const int padding_top = 1;
    1525
    0
      const int padding_bottom = 1;
    1526
    0
      const int padding_left = 1;
    1527
    0
      const int padding_right = 1;
    1528
    0
      const int output_depth = input_depth - kernel_depth + 1;
    1529
    0
      const int output_height = input_height + padding_top + padding_bottom - kernel_height + 1;
    1530
    0
      const int output_width = input_width + padding_left + padding_right - kernel_width + 1;
    1531
    0
      ccv_nnc_tensor_t* a = ccv_nnc_tensor_new(0, CPU_TENSOR_NHWC(32F, batch_size, input_depth, input_height, input_width, input_channels), 0);
    1532
    0
      ccv_nnc_tensor_t* b = ccv_nnc_tensor_new(0, CPU_TENSOR_NHWC(32F, batch_size, output_depth, output_height, output_width, output_channels), 0);
    1533
    0
      ccv_nnc_cmd_t cmd = CMD_CONVOLUTION_FORWARD(1, output_channels, kernel_depth, kernel_height, kernel_width, input_channels);
    1534
    0
      ccv_nnc_hint_t hint = ccv_nnc_no_hint;
    1535
    0
      hint.stride.dim[0] = 1;
    1536
    0
      hint.stride.dim[1] = 1;
    1537
    0
      hint.stride.dim[2] = 1;
    1538
    0
      hint.border.begin[0] = 0;
    1539
    0
      hint.border.end[0] = 0;
    1540
    0
      hint.border.begin[1] = padding_top;
    1541
    0
      hint.border.end[1] = padding_bottom;
    1542
    0
      hint.border.begin[2] = padding_left;
    1543
    0
      hint.border.end[2] = padding_right;
    1544
    0
      assert(ccv_nnc_hint_verify(hint, cmd.info, a->info, b->info) == 0);
    1545
    0
      ccv_nnc_tensor_t* w = ccv_nnc_tensor_new(0, CPU_TENSOR_NHWC(32F, output_channels, kernel_depth, kernel_height, kernel_width, input_channels), 0);
    1546
    0
      dsfmt_t dsfmt;
    1547
    0
      dsfmt_init_gen_rand(&dsfmt, 7);
    1548
    0
      int i;
    1549
    0
      for (i = 0; i < batch_size * input_depth * input_height * input_width * input_channels; i++)
    1550
    0
        a->data.f32[i] = dsfmt_genrand_open_close(&dsfmt) - 0.5;
    1551
    0
      for (i = 0; i < output_channels * kernel_depth * kernel_height * kernel_width * input_channels; i++)
    1552
    0
        w->data.f32[i] = (dsfmt_genrand_open_close(&dsfmt) - 0.5) * 0.1;
    1553
    0
      ccv_nnc_tensor_t* ga = ccv_nnc_tensor_new(0, GPU_TENSOR_NHWC(000, 32F, batch_size, input_depth, input_height, input_width, input_channels), 0);
    1554
    0
      ccv_nnc_tensor_t* gw = ccv_nnc_tensor_new(0, GPU_TENSOR_NHWC(000, 32F, output_channels, kernel_depth, kernel_height, kernel_width, input_channels), 0);
    1555
    0
      ccv_nnc_tensor_t* gwo = ccv_nnc_tensor_new(0, GPU_TENSOR_NCHW(000, 32F, output_channels, input_channels, kernel_depth, kernel_height, kernel_width), 0);
    1556
    0
      ccv_nnc_tensor_t* gc = ccv_nnc_tensor_new(0, GPU_TENSOR_NHWC(000, 32F, batch_size, output_depth, output_height, output_width, output_channels), 0);
    1557
    0
      ccv_nnc_cmd_t move = CMD_DATA_TRANSFER_FORWARD();
    1558
    0
      move.backend = CCV_NNC_BACKEND_MPS;
    1559
    0
      assert(move.backend >= 0);
    1560
    0
      ccv_nnc_cmd_exec(move, ccv_nnc_no_hint, 0, TENSOR_LIST(a, w), TENSOR_LIST(ga, gw), 0);
    1561
    0
      ccv_nnc_cmd_t transform = CMD_FORMAT_TRANSFORM_FORWARD();
    1562
    0
      transform.backend = CCV_NNC_BACKEND_MPS;
    1563
    0
      assert(transform.backend >= 0);
    1564
    0
      ccv_nnc_stream_context_t* stream_context = ccv_nnc_stream_context_new(CCV_STREAM_CONTEXT_GPU);
    1565
    0
      ccv_nnc_cmd_exec(transform, ccv_nnc_no_hint, 0, TENSOR_LIST(gw), TENSOR_LIST(gwo), stream_context);
    1566
    0
      ccv_nnc_stream_context_wait(stream_context);
    1567
    0
      ccv_nnc_tensor_free(gw);
    1568
    0
      cmd.backend = CCV_NNC_BACKEND_MPS;
    1569
    0
      assert(cmd.backend >= 0);
    1570
    0
      cmd.algorithm = -1;
    1571
    0
      cmd = ccv_nnc_cmd_autotune(cmd, 1 * 1024 * 1024 * 1024, hint, 0, TENSOR_LIST(ga, gwo), TENSOR_LIST(gc), stream_context);
    1572
    0
      assert(CCV_NNC_EXEC_SUCCESS == ccv_nnc_cmd_exec(cmd, hint, 0, TENSOR_LIST(ga, gwo), TENSOR_LIST(gc), stream_context));
    1573
    0
      ccv_nnc_stream_context_wait(stream_context);
    1574
    0
      ccv_nnc_stream_context_free(stream_context);
    1575
    0
      ccv_nnc_tensor_t* c = ccv_nnc_tensor_new(0, CPU_TENSOR_NHWC(32F, batch_size, output_depth, output_height, output_width, output_channels), 0);
    1576
    0
      ccv_nnc_cmd_exec(move, ccv_nnc_no_hint, 0, TENSOR_LIST(gc), TENSOR_LIST(c), 0);
    1577
    0
      cmd.backend = CCV_NNC_BACKEND_CPU_REF;
    1578
    0
      assert(cmd.backend >= 0);
    1579
    0
      ccv_nnc_cmd_exec(cmd, hint, 0, TENSOR_LIST(a, w), TENSOR_LIST(b), 0);
    1580
    0
      REQUIRE_ARRAY_EQ_WITH_TOLERANCE(float, b->data.f32, c->data.f32, batch_size * output_depth * output_height * output_width * output_channels, 1e-4, "output from mps should match from CPU");
    1581
    0
      ccv_nnc_tensor_free(c);
    1582
    0
      ccv_nnc_tensor_free(gc);
    1583
    0
      ccv_nnc_tensor_free(w);
    1584
    0
      ccv_nnc_tensor_free(b);
    1585
    0
      ccv_nnc_tensor_free(a);
    1586
    0
      ccv_nnc_tensor_free(gwo);
    1587
    0
      ccv_nnc_tensor_free(ga);
    1588
    0
    }
    1589
    1590
    TEST_CASE("mps forward convolution 3d via mfa conv3d with padding and bias")
    1591
    1
    {
    1592
    1
      GUARD_ELSE_RETURN(ccv_nnc_cmd_ok(CCV_NNC_CONVOLUTION_FORWARD, CCV_NNC_BACKEND_MPS));
    1593
    0
      const int batch_size = 2;
    1594
    0
      const int input_channels = 16;
    1595
    0
      const int output_channels = 32;
    1596
    0
      const int input_depth = 5;
    1597
    0
      const int input_height = 9;
    1598
    0
      const int input_width = 9;
    1599
    0
      const int kernel_depth = 3;
    1600
    0
      const int kernel_height = 3;
    1601
    0
      const int kernel_width = 3;
    1602
    0
      const int padding_top = 1;
    1603
    0
      const int padding_bottom = 1;
    1604
    0
      const int padding_left = 1;
    1605
    0
      const int padding_right = 1;
    1606
    0
      const int output_depth = input_depth - kernel_depth + 1;
    1607
    0
      const int output_height = input_height + padding_top + padding_bottom - kernel_height + 1;
    1608
    0
      const int output_width = input_width + padding_left + padding_right - kernel_width + 1;
    1609
    0
      ccv_nnc_tensor_t* a = ccv_nnc_tensor_new(0, CPU_TENSOR_NHWC(32F, batch_size, input_depth, input_height, input_width, input_channels), 0);
    1610
    0
      ccv_nnc_tensor_t* b = ccv_nnc_tensor_new(0, CPU_TENSOR_NHWC(32F, batch_size, output_depth, output_height, output_width, output_channels), 0);
    1611
    0
      ccv_nnc_cmd_t cmd = CMD_CONVOLUTION_FORWARD(1, output_channels, kernel_depth, kernel_height, kernel_width, input_channels);
    1612
    0
      ccv_nnc_hint_t hint = ccv_nnc_no_hint;
    1613
    0
      hint.stride.dim[0] = 1;
    1614
    0
      hint.stride.dim[1] = 1;
    1615
    0
      hint.stride.dim[2] = 1;
    1616
    0
      hint.border.begin[0] = 0;
    1617
    0
      hint.border.end[0] = 0;
    1618
    0
      hint.border.begin[1] = padding_top;
    1619
    0
      hint.border.end[1] = padding_bottom;
    1620
    0
      hint.border.begin[2] = padding_left;
    1621
    0
      hint.border.end[2] = padding_right;
    1622
    0
      assert(ccv_nnc_hint_verify(hint, cmd.info, a->info, b->info) == 0);
    1623
    0
      ccv_nnc_tensor_t* w = ccv_nnc_tensor_new(0, CPU_TENSOR_NHWC(32F, output_channels, kernel_depth, kernel_height, kernel_width, input_channels), 0);
    1624
    0
      ccv_nnc_tensor_t* bias = ccv_nnc_tensor_new(0, CPU_TENSOR_NHWC(32F, output_channels), 0);
    1625
    0
      dsfmt_t dsfmt;
    1626
    0
      dsfmt_init_gen_rand(&dsfmt, 19);
    1627
    0
      int i;
    1628
    0
      for (i = 0; i < batch_size * input_depth * input_height * input_width * input_channels; i++)
    1629
    0
        a->data.f32[i] = dsfmt_genrand_open_close(&dsfmt) - 0.5;
    1630
    0
      for (i = 0; i < output_channels * kernel_depth * kernel_height * kernel_width * input_channels; i++)
    1631
    0
        w->data.f32[i] = (dsfmt_genrand_open_close(&dsfmt) - 0.5) * 0.1;
    1632
    0
      for (i = 0; i < output_channels; i++)
    1633
    0
        bias->data.f32[i] = (dsfmt_genrand_open_close(&dsfmt) - 0.5) * 0.1;
    1634
    0
      ccv_nnc_tensor_t* ga = ccv_nnc_tensor_new(0, GPU_TENSOR_NHWC(000, 32F, batch_size, input_depth, input_height, input_width, input_channels), 0);
    1635
    0
      ccv_nnc_tensor_t* gw = ccv_nnc_tensor_new(0, GPU_TENSOR_NHWC(000, 32F, output_channels, kernel_depth, kernel_height, kernel_width, input_channels), 0);
    1636
    0
      ccv_nnc_tensor_t* gwo = ccv_nnc_tensor_new(0, GPU_TENSOR_NCHW(000, 32F, output_channels, input_channels, kernel_depth, kernel_height, kernel_width), 0);
    1637
    0
      ccv_nnc_tensor_t* gbias = ccv_nnc_tensor_new(0, GPU_TENSOR_NHWC(000, 32F, output_channels), 0);
    1638
    0
      ccv_nnc_tensor_t* gc = ccv_nnc_tensor_new(0, GPU_TENSOR_NHWC(000, 32F, batch_size, output_depth, output_height, output_width, output_channels), 0);
    1639
    0
      ccv_nnc_cmd_t move = CMD_DATA_TRANSFER_FORWARD();
    1640
    0
      move.backend = CCV_NNC_BACKEND_MPS;
    1641
    0
      assert(move.backend >= 0);
    1642
    0
      ccv_nnc_cmd_exec(move, ccv_nnc_no_hint, 0, TENSOR_LIST(a, w, bias), TENSOR_LIST(ga, gw, gbias), 0);
    1643
    0
      ccv_nnc_cmd_t transform = CMD_FORMAT_TRANSFORM_FORWARD();
    1644
    0
      transform.backend = CCV_NNC_BACKEND_MPS;
    1645
    0
      assert(transform.backend >= 0);
    1646
    0
      ccv_nnc_stream_context_t* stream_context = ccv_nnc_stream_context_new(CCV_STREAM_CONTEXT_GPU);
    1647
    0
      ccv_nnc_cmd_exec(transform, ccv_nnc_no_hint, 0, TENSOR_LIST(gw), TENSOR_LIST(gwo), stream_context);
    1648
    0
      ccv_nnc_stream_context_wait(stream_context);
    1649
    0
      ccv_nnc_tensor_free(gw);
    1650
    0
      cmd.backend = CCV_NNC_BACKEND_MPS;
    1651
    0
      assert(cmd.backend >= 0);
    1652
    0
      cmd.algorithm = -1;
    1653
    0
      cmd = ccv_nnc_cmd_autotune(cmd, 1 * 1024 * 1024 * 1024, hint, 0, TENSOR_LIST(ga, gwo, gbias), TENSOR_LIST(gc), stream_context);
    1654
    0
      assert(CCV_NNC_EXEC_SUCCESS == ccv_nnc_cmd_exec(cmd, hint, 0, TENSOR_LIST(ga, gwo, gbias), TENSOR_LIST(gc), stream_context));
    1655
    0
      ccv_nnc_stream_context_wait(stream_context);
    1656
    0
      ccv_nnc_stream_context_free(stream_context);
    1657
    0
      ccv_nnc_tensor_t* c = ccv_nnc_tensor_new(0, CPU_TENSOR_NHWC(32F, batch_size, output_depth, output_height, output_width, output_channels), 0);
    1658
    0
      ccv_nnc_cmd_exec(move, ccv_nnc_no_hint, 0, TENSOR_LIST(gc), TENSOR_LIST(c), 0);
    1659
    0
      cmd.backend = CCV_NNC_BACKEND_CPU_REF;
    1660
    0
      assert(cmd.backend >= 0);
    1661
    0
      ccv_nnc_cmd_exec(cmd, hint, 0, TENSOR_LIST(a, w, bias), TENSOR_LIST(b), 0);
    1662
    0
      REQUIRE_ARRAY_EQ_WITH_TOLERANCE(float, b->data.f32, c->data.f32, batch_size * output_depth * output_height * output_width * output_channels, 1e-4, "output from mps should match from CPU");
    1663
    0
      ccv_nnc_tensor_free(c);
    1664
    0
      ccv_nnc_tensor_free(gc);
    1665
    0
      ccv_nnc_tensor_free(bias);
    1666
    0
      ccv_nnc_tensor_free(w);
    1667
    0
      ccv_nnc_tensor_free(b);
    1668
    0
      ccv_nnc_tensor_free(a);
    1669
    0
      ccv_nnc_tensor_free(gbias);
    1670
    0
      ccv_nnc_tensor_free(gwo);
    1671
    0
      ccv_nnc_tensor_free(ga);
    1672
    0
    }
    1673
    1674
    TEST_CASE("mps forward convolution 3d via mfa conv3d with one point padding on tile boundary and bias")
    1675
    1
    {
    1676
    1
      GUARD_ELSE_RETURN(ccv_nnc_cmd_ok(CCV_NNC_CONVOLUTION_FORWARD, CCV_NNC_BACKEND_MPS));
    1677
    0
      const int batch_size = 2;
    1678
    0
      const int input_channels = 16;
    1679
    0
      const int output_channels = 32;
    1680
    0
      const int input_depth = 5;
    1681
    0
      const int input_height = 8;
    1682
    0
      const int input_width = 8;
    1683
    0
      const int kernel_depth = 3;
    1684
    0
      const int kernel_height = 3;
    1685
    0
      const int kernel_width = 3;
    1686
    0
      const int padding_top = 1;
    1687
    0
      const int padding_bottom = 1;
    1688
    0
      const int padding_left = 1;
    1689
    0
      const int padding_right = 1;
    1690
    0
      const int output_depth = input_depth - kernel_depth + 1;
    1691
    0
      const int output_height = input_height + padding_top + padding_bottom - kernel_height + 1;
    1692
    0
      const int output_width = input_width + padding_left + padding_right - kernel_width + 1;
    1693
    0
      ccv_nnc_tensor_t* a = ccv_nnc_tensor_new(0, CPU_TENSOR_NHWC(32F, batch_size, input_depth, input_height, input_width, input_channels), 0);
    1694
    0
      ccv_nnc_tensor_t* b = ccv_nnc_tensor_new(0, CPU_TENSOR_NHWC(32F, batch_size, output_depth, output_height, output_width, output_channels), 0);
    1695
    0
      ccv_nnc_cmd_t cmd = CMD_CONVOLUTION_FORWARD(1, output_channels, kernel_depth, kernel_height, kernel_width, input_channels);
    1696
    0
      ccv_nnc_hint_t hint = ccv_nnc_no_hint;
    1697
    0
      hint.stride.dim[0] = 1;
    1698
    0
      hint.stride.dim[1] = 1;
    1699
    0
      hint.stride.dim[2] = 1;
    1700
    0
      hint.border.begin[0] = 0;
    1701
    0
      hint.border.end[0] = 0;
    1702
    0
      hint.border.begin[1] = padding_top;
    1703
    0
      hint.border.end[1] = padding_bottom;
    1704
    0
      hint.border.begin[2] = padding_left;
    1705
    0
      hint.border.end[2] = padding_right;
    1706
    0
      assert(ccv_nnc_hint_verify(hint, cmd.info, a->info, b->info) == 0);
    1707
    0
      ccv_nnc_tensor_t* w = ccv_nnc_tensor_new(0, CPU_TENSOR_NHWC(32F, output_channels, kernel_depth, kernel_height, kernel_width, input_channels), 0);
    1708
    0
      ccv_nnc_tensor_t* bias = ccv_nnc_tensor_new(0, CPU_TENSOR_NHWC(32F, output_channels), 0);
    1709
    0
      dsfmt_t dsfmt;
    1710
    0
      dsfmt_init_gen_rand(&dsfmt, 20);
    1711
    0
      int i;
    1712
    0
      for (i = 0; i < batch_size * input_depth * input_height * input_width * input_channels; i++)
    1713
    0
        a->data.f32[i] = dsfmt_genrand_open_close(&dsfmt) - 0.5;
    1714
    0
      for (i = 0; i < output_channels * kernel_depth * kernel_height * kernel_width * input_channels; i++)
    1715
    0
        w->data.f32[i] = (dsfmt_genrand_open_close(&dsfmt) - 0.5) * 0.1;
    1716
    0
      for (i = 0; i < output_channels; i++)
    1717
    0
        bias->data.f32[i] = (dsfmt_genrand_open_close(&dsfmt) - 0.5) * 0.1;
    1718
    0
      ccv_nnc_tensor_t* ga = ccv_nnc_tensor_new(0, GPU_TENSOR_NHWC(000, 32F, batch_size, input_depth, input_height, input_width, input_channels), 0);
    1719
    0
      ccv_nnc_tensor_t* gw = ccv_nnc_tensor_new(0, GPU_TENSOR_NHWC(000, 32F, output_channels, kernel_depth, kernel_height, kernel_width, input_channels), 0);
    1720
    0
      ccv_nnc_tensor_t* gwo = ccv_nnc_tensor_new(0, GPU_TENSOR_NCHW(000, 32F, output_channels, input_channels, kernel_depth, kernel_height, kernel_width), 0);
    1721
    0
      ccv_nnc_tensor_t* gbias = ccv_nnc_tensor_new(0, GPU_TENSOR_NHWC(000, 32F, output_channels), 0);
    1722
    0
      ccv_nnc_tensor_t* gc = ccv_nnc_tensor_new(0, GPU_TENSOR_NHWC(000, 32F, batch_size, output_depth, output_height, output_width, output_channels), 0);
    1723
    0
      ccv_nnc_cmd_t move = CMD_DATA_TRANSFER_FORWARD();
    1724
    0
      move.backend = CCV_NNC_BACKEND_MPS;
    1725
    0
      assert(move.backend >= 0);
    1726
    0
      ccv_nnc_cmd_exec(move, ccv_nnc_no_hint, 0, TENSOR_LIST(a, w, bias), TENSOR_LIST(ga, gw, gbias), 0);
    1727
    0
      ccv_nnc_cmd_t transform = CMD_FORMAT_TRANSFORM_FORWARD();
    1728
    0
      transform.backend = CCV_NNC_BACKEND_MPS;
    1729
    0
      assert(transform.backend >= 0);
    1730
    0
      ccv_nnc_stream_context_t* stream_context = ccv_nnc_stream_context_new(CCV_STREAM_CONTEXT_GPU);
    1731
    0
      ccv_nnc_cmd_exec(transform, ccv_nnc_no_hint, 0, TENSOR_LIST(gw), TENSOR_LIST(gwo), stream_context);
    1732
    0
      ccv_nnc_stream_context_wait(stream_context);
    1733
    0
      ccv_nnc_tensor_free(gw);
    1734
    0
      cmd.backend = CCV_NNC_BACKEND_MPS;
    1735
    0
      assert(cmd.backend >= 0);
    1736
    0
      cmd.algorithm = -1;
    1737
    0
      cmd = ccv_nnc_cmd_autotune(cmd, 1 * 1024 * 1024 * 1024, hint, 0, TENSOR_LIST(ga, gwo, gbias), TENSOR_LIST(gc), stream_context);
    1738
    0
      assert(CCV_NNC_EXEC_SUCCESS == ccv_nnc_cmd_exec(cmd, hint, 0, TENSOR_LIST(ga, gwo, gbias), TENSOR_LIST(gc), stream_context));
    1739
    0
      ccv_nnc_stream_context_wait(stream_context);
    1740
    0
      ccv_nnc_stream_context_free(stream_context);
    1741
    0
      ccv_nnc_tensor_t* c = ccv_nnc_tensor_new(0, CPU_TENSOR_NHWC(32F, batch_size, output_depth, output_height, output_width, output_channels), 0);
    1742
    0
      ccv_nnc_cmd_exec(move, ccv_nnc_no_hint, 0, TENSOR_LIST(gc), TENSOR_LIST(c), 0);
    1743
    0
      cmd.backend = CCV_NNC_BACKEND_CPU_REF;
    1744
    0
      assert(cmd.backend >= 0);
    1745
    0
      ccv_nnc_cmd_exec(cmd, hint, 0, TENSOR_LIST(a, w, bias), TENSOR_LIST(b), 0);
    1746
    0
      REQUIRE_ARRAY_EQ_WITH_TOLERANCE(float, b->data.f32, c->data.f32, batch_size * output_depth * output_height * output_width * output_channels, 1e-4, "output from mps should match from CPU");
    1747
    0
      ccv_nnc_tensor_free(c);
    1748
    0
      ccv_nnc_tensor_free(gc);
    1749
    0
      ccv_nnc_tensor_free(bias);
    1750
    0
      ccv_nnc_tensor_free(w);
    1751
    0
      ccv_nnc_tensor_free(b);
    1752
    0
      ccv_nnc_tensor_free(a);
    1753
    0
      ccv_nnc_tensor_free(gbias);
    1754
    0
      ccv_nnc_tensor_free(gwo);
    1755
    0
      ccv_nnc_tensor_free(ga);
    1756
    0
    }
    1757
    1758
    TEST_CASE("mps forward convolution 3d via mfa conv3d with one point padding on tile boundary")
    1759
    1
    {
    1760
    1
      GUARD_ELSE_RETURN(ccv_nnc_cmd_ok(CCV_NNC_CONVOLUTION_FORWARD, CCV_NNC_BACKEND_MPS));
    1761
    0
      const int batch_size = 2;
    1762
    0
      const int input_channels = 16;
    1763
    0
      const int output_channels = 32;
    1764
    0
      const int input_depth = 5;
    1765
    0
      const int input_height = 8;
    1766
    0
      const int input_width = 8;
    1767
    0
      const int kernel_depth = 3;
    1768
    0
      const int kernel_height = 3;
    1769
    0
      const int kernel_width = 3;
    1770
    0
      const int padding_top = 1;
    1771
    0
      const int padding_bottom = 1;
    1772
    0
      const int padding_left = 1;
    1773
    0
      const int padding_right = 1;
    1774
    0
      const int output_depth = input_depth - kernel_depth + 1;
    1775
    0
      const int output_height = input_height + padding_top + padding_bottom - kernel_height + 1;
    1776
    0
      const int output_width = input_width + padding_left + padding_right - kernel_width + 1;
    1777
    0
      ccv_nnc_tensor_t* a = ccv_nnc_tensor_new(0, CPU_TENSOR_NHWC(32F, batch_size, input_depth, input_height, input_width, input_channels), 0);
    1778
    0
      ccv_nnc_tensor_t* b = ccv_nnc_tensor_new(0, CPU_TENSOR_NHWC(32F, batch_size, output_depth, output_height, output_width, output_channels), 0);
    1779
    0
      ccv_nnc_cmd_t cmd = CMD_CONVOLUTION_FORWARD(1, output_channels, kernel_depth, kernel_height, kernel_width, input_channels);
    1780
    0
      ccv_nnc_hint_t hint = ccv_nnc_no_hint;
    1781
    0
      hint.stride.dim[0] = 1;
    1782
    0
      hint.stride.dim[1] = 1;
    1783
    0
      hint.stride.dim[2] = 1;
    1784
    0
      hint.border.begin[0] = 0;
    1785
    0
      hint.border.end[0] = 0;
    1786
    0
      hint.border.begin[1] = padding_top;
    1787
    0
      hint.border.end[1] = padding_bottom;
    1788
    0
      hint.border.begin[2] = padding_left;
    1789
    0
      hint.border.end[2] = padding_right;
    1790
    0
      assert(ccv_nnc_hint_verify(hint, cmd.info, a->info, b->info) == 0);
    1791
    0
      ccv_nnc_tensor_t* w = ccv_nnc_tensor_new(0, CPU_TENSOR_NHWC(32F, output_channels, kernel_depth, kernel_height, kernel_width, input_channels), 0);
    1792
    0
      dsfmt_t dsfmt;
    1793
    0
      dsfmt_init_gen_rand(&dsfmt, 9);
    1794
    0
      int i;
    1795
    0
      for (i = 0; i < batch_size * input_depth * input_height * input_width * input_channels; i++)
    1796
    0
        a->data.f32[i] = dsfmt_genrand_open_close(&dsfmt) - 0.5;
    1797
    0
      for (i = 0; i < output_channels * kernel_depth * kernel_height * kernel_width * input_channels; i++)
    1798
    0
        w->data.f32[i] = (dsfmt_genrand_open_close(&dsfmt) - 0.5) * 0.1;
    1799
    0
      ccv_nnc_tensor_t* ga = ccv_nnc_tensor_new(0, GPU_TENSOR_NHWC(000, 32F, batch_size, input_depth, input_height, input_width, input_channels), 0);
    1800
    0
      ccv_nnc_tensor_t* gw = ccv_nnc_tensor_new(0, GPU_TENSOR_NHWC(000, 32F, output_channels, kernel_depth, kernel_height, kernel_width, input_channels), 0);
    1801
    0
      ccv_nnc_tensor_t* gwo = ccv_nnc_tensor_new(0, GPU_TENSOR_NCHW(000, 32F, output_channels, input_channels, kernel_depth, kernel_height, kernel_width), 0);
    1802
    0
      ccv_nnc_tensor_t* gc = ccv_nnc_tensor_new(0, GPU_TENSOR_NHWC(000, 32F, batch_size, output_depth, output_height, output_width, output_channels), 0);
    1803
    0
      ccv_nnc_cmd_t move = CMD_DATA_TRANSFER_FORWARD();
    1804
    0
      move.backend = CCV_NNC_BACKEND_MPS;
    1805
    0
      assert(move.backend >= 0);
    1806
    0
      ccv_nnc_cmd_exec(move, ccv_nnc_no_hint, 0, TENSOR_LIST(a, w), TENSOR_LIST(ga, gw), 0);
    1807
    0
      ccv_nnc_cmd_t transform = CMD_FORMAT_TRANSFORM_FORWARD();
    1808
    0
      transform.backend = CCV_NNC_BACKEND_MPS;
    1809
    0
      assert(transform.backend >= 0);
    1810
    0
      ccv_nnc_stream_context_t* stream_context = ccv_nnc_stream_context_new(CCV_STREAM_CONTEXT_GPU);
    1811
    0
      ccv_nnc_cmd_exec(transform, ccv_nnc_no_hint, 0, TENSOR_LIST(gw), TENSOR_LIST(gwo), stream_context);
    1812
    0
      ccv_nnc_stream_context_wait(stream_context);
    1813
    0
      ccv_nnc_tensor_free(gw);
    1814
    0
      cmd.backend = CCV_NNC_BACKEND_MPS;
    1815
    0
      assert(cmd.backend >= 0);
    1816
    0
      cmd.algorithm = -1;
    1817
    0
      cmd = ccv_nnc_cmd_autotune(cmd, 1 * 1024 * 1024 * 1024, hint, 0, TENSOR_LIST(ga, gwo), TENSOR_LIST(gc), stream_context);
    1818
    0
      assert(CCV_NNC_EXEC_SUCCESS == ccv_nnc_cmd_exec(cmd, hint, 0, TENSOR_LIST(ga, gwo), TENSOR_LIST(gc), stream_context));
    1819
    0
      ccv_nnc_stream_context_wait(stream_context);
    1820
    0
      ccv_nnc_stream_context_free(stream_context);
    1821
    0
      ccv_nnc_tensor_t* c = ccv_nnc_tensor_new(0, CPU_TENSOR_NHWC(32F, batch_size, output_depth, output_height, output_width, output_channels), 0);
    1822
    0
      ccv_nnc_cmd_exec(move, ccv_nnc_no_hint, 0, TENSOR_LIST(gc), TENSOR_LIST(c), 0);
    1823
    0
      cmd.backend = CCV_NNC_BACKEND_CPU_REF;
    1824
    0
      assert(cmd.backend >= 0);
    1825
    0
      ccv_nnc_cmd_exec(cmd, hint, 0, TENSOR_LIST(a, w), TENSOR_LIST(b), 0);
    1826
    0
      REQUIRE_ARRAY_EQ_WITH_TOLERANCE(float, b->data.f32, c->data.f32, batch_size * output_depth * output_height * output_width * output_channels, 1e-4, "output from mps should match from CPU");
    1827
    0
      ccv_nnc_tensor_free(c);
    1828
    0
      ccv_nnc_tensor_free(gc);
    1829
    0
      ccv_nnc_tensor_free(w);
    1830
    0
      ccv_nnc_tensor_free(b);
    1831
    0
      ccv_nnc_tensor_free(a);
    1832
    0
      ccv_nnc_tensor_free(gwo);
    1833
    0
      ccv_nnc_tensor_free(ga);
    1834
    0
    }
    1835
    1836
    TEST_CASE("mps forward convolution 3d via mfa conv3d with one point padding on small rectangular spatial dimensions")
    1837
    1
    {
    1838
    1
      GUARD_ELSE_RETURN(ccv_nnc_cmd_ok(CCV_NNC_CONVOLUTION_FORWARD, CCV_NNC_BACKEND_MPS));
    1839
    0
      const int batch_size = 2;
    1840
    0
      const int input_channels = 16;
    1841
    0
      const int output_channels = 32;
    1842
    0
      const int input_depth = 5;
    1843
    0
      const int input_height = 6;
    1844
    0
      const int input_width = 7;
    1845
    0
      const int kernel_depth = 3;
    1846
    0
      const int kernel_height = 3;
    1847
    0
      const int kernel_width = 3;
    1848
    0
      const int padding_top = 1;
    1849
    0
      const int padding_bottom = 1;
    1850
    0
      const int padding_left = 1;
    1851
    0
      const int padding_right = 1;
    1852
    0
      const int output_depth = input_depth - kernel_depth + 1;
    1853
    0
      const int output_height = input_height + padding_top + padding_bottom - kernel_height + 1;
    1854
    0
      const int output_width = input_width + padding_left + padding_right - kernel_width + 1;
    1855
    0
      ccv_nnc_tensor_t* a = ccv_nnc_tensor_new(0, CPU_TENSOR_NHWC(32F, batch_size, input_depth, input_height, input_width, input_channels), 0);
    1856
    0
      ccv_nnc_tensor_t* b = ccv_nnc_tensor_new(0, CPU_TENSOR_NHWC(32F, batch_size, output_depth, output_height, output_width, output_channels), 0);
    1857
    0
      ccv_nnc_cmd_t cmd = CMD_CONVOLUTION_FORWARD(1, output_channels, kernel_depth, kernel_height, kernel_width, input_channels);
    1858
    0
      ccv_nnc_hint_t hint = ccv_nnc_no_hint;
    1859
    0
      hint.stride.dim[0] = 1;
    1860
    0
      hint.stride.dim[1] = 1;
    1861
    0
      hint.stride.dim[2] = 1;
    1862
    0
      hint.border.begin[0] = 0;
    1863
    0
      hint.border.end[0] = 0;
    1864
    0
      hint.border.begin[1] = padding_top;
    1865
    0
      hint.border.end[1] = padding_bottom;
    1866
    0
      hint.border.begin[2] = padding_left;
    1867
    0
      hint.border.end[2] = padding_right;
    1868
    0
      assert(ccv_nnc_hint_verify(hint, cmd.info, a->info, b->info) == 0);
    1869
    0
      ccv_nnc_tensor_t* w = ccv_nnc_tensor_new(0, CPU_TENSOR_NHWC(32F, output_channels, kernel_depth, kernel_height, kernel_width, input_channels), 0);
    1870
    0
      dsfmt_t dsfmt;
    1871
    0
      dsfmt_init_gen_rand(&dsfmt, 10);
    1872
    0
      int i;
    1873
    0
      for (i = 0; i < batch_size * input_depth * input_height * input_width * input_channels; i++)
    1874
    0
        a->data.f32[i] = dsfmt_genrand_open_close(&dsfmt) - 0.5;
    1875
    0
      for (i = 0; i < output_channels * kernel_depth * kernel_height * kernel_width * input_channels; i++)
    1876
    0
        w->data.f32[i] = (dsfmt_genrand_open_close(&dsfmt) - 0.5) * 0.1;
    1877
    0
      ccv_nnc_tensor_t* ga = ccv_nnc_tensor_new(0, GPU_TENSOR_NHWC(000, 32F, batch_size, input_depth, input_height, input_width, input_channels), 0);
    1878
    0
      ccv_nnc_tensor_t* gw = ccv_nnc_tensor_new(0, GPU_TENSOR_NHWC(000, 32F, output_channels, kernel_depth, kernel_height, kernel_width, input_channels), 0);
    1879
    0
      ccv_nnc_tensor_t* gwo = ccv_nnc_tensor_new(0, GPU_TENSOR_NCHW(000, 32F, output_channels, input_channels, kernel_depth, kernel_height, kernel_width), 0);
    1880
    0
      ccv_nnc_tensor_t* gc = ccv_nnc_tensor_new(0, GPU_TENSOR_NHWC(000, 32F, batch_size, output_depth, output_height, output_width, output_channels), 0);
    1881
    0
      ccv_nnc_cmd_t move = CMD_DATA_TRANSFER_FORWARD();
    1882
    0
      move.backend = CCV_NNC_BACKEND_MPS;
    1883
    0
      assert(move.backend >= 0);
    1884
    0
      ccv_nnc_cmd_exec(move, ccv_nnc_no_hint, 0, TENSOR_LIST(a, w), TENSOR_LIST(ga, gw), 0);
    1885
    0
      ccv_nnc_cmd_t transform = CMD_FORMAT_TRANSFORM_FORWARD();
    1886
    0
      transform.backend = CCV_NNC_BACKEND_MPS;
    1887
    0
      assert(transform.backend >= 0);
    1888
    0
      ccv_nnc_stream_context_t* stream_context = ccv_nnc_stream_context_new(CCV_STREAM_CONTEXT_GPU);
    1889
    0
      ccv_nnc_cmd_exec(transform, ccv_nnc_no_hint, 0, TENSOR_LIST(gw), TENSOR_LIST(gwo), stream_context);
    1890
    0
      ccv_nnc_stream_context_wait(stream_context);
    1891
    0
      ccv_nnc_tensor_free(gw);
    1892
    0
      cmd.backend = CCV_NNC_BACKEND_MPS;
    1893
    0
      assert(cmd.backend >= 0);
    1894
    0
      cmd.algorithm = -1;
    1895
    0
      cmd = ccv_nnc_cmd_autotune(cmd, 1 * 1024 * 1024 * 1024, hint, 0, TENSOR_LIST(ga, gwo), TENSOR_LIST(gc), stream_context);
    1896
    0
      assert(CCV_NNC_EXEC_SUCCESS == ccv_nnc_cmd_exec(cmd, hint, 0, TENSOR_LIST(ga, gwo), TENSOR_LIST(gc), stream_context));
    1897
    0
      ccv_nnc_stream_context_wait(stream_context);
    1898
    0
      ccv_nnc_stream_context_free(stream_context);
    1899
    0
      ccv_nnc_tensor_t* c = ccv_nnc_tensor_new(0, CPU_TENSOR_NHWC(32F, batch_size, output_depth, output_height, output_width, output_channels), 0);
    1900
    0
      ccv_nnc_cmd_exec(move, ccv_nnc_no_hint, 0, TENSOR_LIST(gc), TENSOR_LIST(c), 0);
    1901
    0
      cmd.backend = CCV_NNC_BACKEND_CPU_REF;
    1902
    0
      assert(cmd.backend >= 0);
    1903
    0
      ccv_nnc_cmd_exec(cmd, hint, 0, TENSOR_LIST(a, w), TENSOR_LIST(b), 0);
    1904
    0
      REQUIRE_ARRAY_EQ_WITH_TOLERANCE(float, b->data.f32, c->data.f32, batch_size * output_depth * output_height * output_width * output_channels, 1e-4, "output from mps should match from CPU");
    1905
    0
      ccv_nnc_tensor_free(c);
    1906
    0
      ccv_nnc_tensor_free(gc);
    1907
    0
      ccv_nnc_tensor_free(w);
    1908
    0
      ccv_nnc_tensor_free(b);
    1909
    0
      ccv_nnc_tensor_free(a);
    1910
    0
      ccv_nnc_tensor_free(gwo);
    1911
    0
      ccv_nnc_tensor_free(ga);
    1912
    0
    }
    1913
    1914
    TEST_CASE("mps forward convolution 3d via mfa conv3d with one point padding on 9x9 spatial dimensions")
    1915
    1
    {
    1916
    1
      GUARD_ELSE_RETURN(ccv_nnc_cmd_ok(CCV_NNC_CONVOLUTION_FORWARD, CCV_NNC_BACKEND_MPS));
    1917
    0
      const int batch_size = 2;
    1918
    0
      const int input_channels = 16;
    1919
    0
      const int output_channels = 32;
    1920
    0
      const int input_depth = 5;
    1921
    0
      const int input_height = 9;
    1922
    0
      const int input_width = 9;
    1923
    0
      const int kernel_depth = 3;
    1924
    0
      const int kernel_height = 3;
    1925
    0
      const int kernel_width = 3;
    1926
    0
      const int padding_top = 1;
    1927
    0
      const int padding_bottom = 1;
    1928
    0
      const int padding_left = 1;
    1929
    0
      const int padding_right = 1;
    1930
    0
      const int output_depth = input_depth - kernel_depth + 1;
    1931
    0
      const int output_height = input_height + padding_top + padding_bottom - kernel_height + 1;
    1932
    0
      const int output_width = input_width + padding_left + padding_right - kernel_width + 1;
    1933
    0
      ccv_nnc_tensor_t* a = ccv_nnc_tensor_new(0, CPU_TENSOR_NHWC(32F, batch_size, input_depth, input_height, input_width, input_channels), 0);
    1934
    0
      ccv_nnc_tensor_t* b = ccv_nnc_tensor_new(0, CPU_TENSOR_NHWC(32F, batch_size, output_depth, output_height, output_width, output_channels), 0);
    1935
    0
      ccv_nnc_cmd_t cmd = CMD_CONVOLUTION_FORWARD(1, output_channels, kernel_depth, kernel_height, kernel_width, input_channels);
    1936
    0
      ccv_nnc_hint_t hint = ccv_nnc_no_hint;
    1937
    0
      hint.stride.dim[0] = 1;
    1938
    0
      hint.stride.dim[1] = 1;
    1939
    0
      hint.stride.dim[2] = 1;
    1940
    0
      hint.border.begin[0] = 0;
    1941
    0
      hint.border.end[0] = 0;
    1942
    0
      hint.border.begin[1] = padding_top;
    1943
    0
      hint.border.end[1] = padding_bottom;
    1944
    0
      hint.border.begin[2] = padding_left;
    1945
    0
      hint.border.end[2] = padding_right;
    1946
    0
      assert(ccv_nnc_hint_verify(hint, cmd.info, a->info, b->info) == 0);
    1947
    0
      ccv_nnc_tensor_t* w = ccv_nnc_tensor_new(0, CPU_TENSOR_NHWC(32F, output_channels, kernel_depth, kernel_height, kernel_width, input_channels), 0);
    1948
    0
      dsfmt_t dsfmt;
    1949
    0
      dsfmt_init_gen_rand(&dsfmt, 11);
    1950
    0
      int i;
    1951
    0
      for (i = 0; i < batch_size * input_depth * input_height * input_width * input_channels; i++)
    1952
    0
        a->data.f32[i] = dsfmt_genrand_open_close(&dsfmt) - 0.5;
    1953
    0
      for (i = 0; i < output_channels * kernel_depth * kernel_height * kernel_width * input_channels; i++)
    1954
    0
        w->data.f32[i] = (dsfmt_genrand_open_close(&dsfmt) - 0.5) * 0.1;
    1955
    0
      ccv_nnc_tensor_t* ga = ccv_nnc_tensor_new(0, GPU_TENSOR_NHWC(000, 32F, batch_size, input_depth, input_height, input_width, input_channels), 0);
    1956
    0
      ccv_nnc_tensor_t* gw = ccv_nnc_tensor_new(0, GPU_TENSOR_NHWC(000, 32F, output_channels, kernel_depth, kernel_height, kernel_width, input_channels), 0);
    1957
    0
      ccv_nnc_tensor_t* gwo = ccv_nnc_tensor_new(0, GPU_TENSOR_NCHW(000, 32F, output_channels, input_channels, kernel_depth, kernel_height, kernel_width), 0);
    1958
    0
      ccv_nnc_tensor_t* gc = ccv_nnc_tensor_new(0, GPU_TENSOR_NHWC(000, 32F, batch_size, output_depth, output_height, output_width, output_channels), 0);
    1959
    0
      ccv_nnc_cmd_t move = CMD_DATA_TRANSFER_FORWARD();
    1960
    0
      move.backend = CCV_NNC_BACKEND_MPS;
    1961
    0
      assert(move.backend >= 0);
    1962
    0
      ccv_nnc_cmd_exec(move, ccv_nnc_no_hint, 0, TENSOR_LIST(a, w), TENSOR_LIST(ga, gw), 0);
    1963
    0
      ccv_nnc_cmd_t transform = CMD_FORMAT_TRANSFORM_FORWARD();
    1964
    0
      transform.backend = CCV_NNC_BACKEND_MPS;
    1965
    0
      assert(transform.backend >= 0);
    1966
    0
      ccv_nnc_stream_context_t* stream_context = ccv_nnc_stream_context_new(CCV_STREAM_CONTEXT_GPU);
    1967
    0
      ccv_nnc_cmd_exec(transform, ccv_nnc_no_hint, 0, TENSOR_LIST(gw), TENSOR_LIST(gwo), stream_context);
    1968
    0
      ccv_nnc_stream_context_wait(stream_context);
    1969
    0
      ccv_nnc_tensor_free(gw);
    1970
    0
      cmd.backend = CCV_NNC_BACKEND_MPS;
    1971
    0
      assert(cmd.backend >= 0);
    1972
    0
      cmd.algorithm = -1;
    1973
    0
      cmd = ccv_nnc_cmd_autotune(cmd, 1 * 1024 * 1024 * 1024, hint, 0, TENSOR_LIST(ga, gwo), TENSOR_LIST(gc), stream_context);
    1974
    0
      assert(CCV_NNC_EXEC_SUCCESS == ccv_nnc_cmd_exec(cmd, hint, 0, TENSOR_LIST(ga, gwo), TENSOR_LIST(gc), stream_context));
    1975
    0
      ccv_nnc_stream_context_wait(stream_context);
    1976
    0
      ccv_nnc_stream_context_free(stream_context);
    1977
    0
      ccv_nnc_tensor_t* c = ccv_nnc_tensor_new(0, CPU_TENSOR_NHWC(32F, batch_size, output_depth, output_height, output_width, output_channels), 0);
    1978
    0
      ccv_nnc_cmd_exec(move, ccv_nnc_no_hint, 0, TENSOR_LIST(gc), TENSOR_LIST(c), 0);
    1979
    0
      cmd.backend = CCV_NNC_BACKEND_CPU_REF;
    1980
    0
      assert(cmd.backend >= 0);
    1981
    0
      ccv_nnc_cmd_exec(cmd, hint, 0, TENSOR_LIST(a, w), TENSOR_LIST(b), 0);
    1982
    0
      REQUIRE_ARRAY_EQ_WITH_TOLERANCE(float, b->data.f32, c->data.f32, batch_size * output_depth * output_height * output_width * output_channels, 1e-4, "output from mps should match from CPU");
    1983
    0
      ccv_nnc_tensor_free(c);
    1984
    0
      ccv_nnc_tensor_free(gc);
    1985
    0
      ccv_nnc_tensor_free(w);
    1986
    0
      ccv_nnc_tensor_free(b);
    1987
    0
      ccv_nnc_tensor_free(a);
    1988
    0
      ccv_nnc_tensor_free(gwo);
    1989
    0
      ccv_nnc_tensor_free(ga);
    1990
    0
    }
    1991
    1992
    TEST_CASE("mps forward convolution 3d via mfa conv3d with one point padding on 8x9 spatial dimensions")
    1993
    1
    {
    1994
    1
      GUARD_ELSE_RETURN(ccv_nnc_cmd_ok(CCV_NNC_CONVOLUTION_FORWARD, CCV_NNC_BACKEND_MPS));
    1995
    0
      const int batch_size = 2;
    1996
    0
      const int input_channels = 16;
    1997
    0
      const int output_channels = 32;
    1998
    0
      const int input_depth = 5;
    1999
    0
      const int input_height = 8;
    2000
    0
      const int input_width = 9;
    2001
    0
      const int kernel_depth = 3;
    2002
    0
      const int kernel_height = 3;
    2003
    0
      const int kernel_width = 3;
    2004
    0
      const int padding_top = 1;
    2005
    0
      const int padding_bottom = 1;
    2006
    0
      const int padding_left = 1;
    2007
    0
      const int padding_right = 1;
    2008
    0
      const int output_depth = input_depth - kernel_depth + 1;
    2009
    0
      const int output_height = input_height + padding_top + padding_bottom - kernel_height + 1;
    2010
    0
      const int output_width = input_width + padding_left + padding_right - kernel_width + 1;
    2011
    0
      ccv_nnc_tensor_t* a = ccv_nnc_tensor_new(0, CPU_TENSOR_NHWC(32F, batch_size, input_depth, input_height, input_width, input_channels), 0);
    2012
    0
      ccv_nnc_tensor_t* b = ccv_nnc_tensor_new(0, CPU_TENSOR_NHWC(32F, batch_size, output_depth, output_height, output_width, output_channels), 0);
    2013
    0
      ccv_nnc_cmd_t cmd = CMD_CONVOLUTION_FORWARD(1, output_channels, kernel_depth, kernel_height, kernel_width, input_channels);
    2014
    0
      ccv_nnc_hint_t hint = ccv_nnc_no_hint;
    2015
    0
      hint.stride.dim[0] = 1;
    2016
    0
      hint.stride.dim[1] = 1;
    2017
    0
      hint.stride.dim[2] = 1;
    2018
    0
      hint.border.begin[0] = 0;
    2019
    0
      hint.border.end[0] = 0;
    2020
    0
      hint.border.begin[1] = padding_top;
    2021
    0
      hint.border.end[1] = padding_bottom;
    2022
    0
      hint.border.begin[2] = padding_left;
    2023
    0
      hint.border.end[2] = padding_right;
    2024
    0
      assert(ccv_nnc_hint_verify(hint, cmd.info, a->info, b->info) == 0);
    2025
    0
      ccv_nnc_tensor_t* w = ccv_nnc_tensor_new(0, CPU_TENSOR_NHWC(32F, output_channels, kernel_depth, kernel_height, kernel_width, input_channels), 0);
    2026
    0
      dsfmt_t dsfmt;
    2027
    0
      dsfmt_init_gen_rand(&dsfmt, 12);
    2028
    0
      int i;
    2029
    0
      for (i = 0; i < batch_size * input_depth * input_height * input_width * input_channels; i++)
    2030
    0
        a->data.f32[i] = dsfmt_genrand_open_close(&dsfmt) - 0.5;
    2031
    0
      for (i = 0; i < output_channels * kernel_depth * kernel_height * kernel_width * input_channels; i++)
    2032
    0
        w->data.f32[i] = (dsfmt_genrand_open_close(&dsfmt) - 0.5) * 0.1;
    2033
    0
      ccv_nnc_tensor_t* ga = ccv_nnc_tensor_new(0, GPU_TENSOR_NHWC(000, 32F, batch_size, input_depth, input_height, input_width, input_channels), 0);
    2034
    0
      ccv_nnc_tensor_t* gw = ccv_nnc_tensor_new(0, GPU_TENSOR_NHWC(000, 32F, output_channels, kernel_depth, kernel_height, kernel_width, input_channels), 0);
    2035
    0
      ccv_nnc_tensor_t* gwo = ccv_nnc_tensor_new(0, GPU_TENSOR_NCHW(000, 32F, output_channels, input_channels, kernel_depth, kernel_height, kernel_width), 0);
    2036
    0
      ccv_nnc_tensor_t* gc = ccv_nnc_tensor_new(0, GPU_TENSOR_NHWC(000, 32F, batch_size, output_depth, output_height, output_width, output_channels), 0);
    2037
    0
      ccv_nnc_cmd_t move = CMD_DATA_TRANSFER_FORWARD();
    2038
    0
      move.backend = CCV_NNC_BACKEND_MPS;
    2039
    0
      assert(move.backend >= 0);
    2040
    0
      ccv_nnc_cmd_exec(move, ccv_nnc_no_hint, 0, TENSOR_LIST(a, w), TENSOR_LIST(ga, gw), 0);
    2041
    0
      ccv_nnc_cmd_t transform = CMD_FORMAT_TRANSFORM_FORWARD();
    2042
    0
      transform.backend = CCV_NNC_BACKEND_MPS;
    2043
    0
      assert(transform.backend >= 0);
    2044
    0
      ccv_nnc_stream_context_t* stream_context = ccv_nnc_stream_context_new(CCV_STREAM_CONTEXT_GPU);
    2045
    0
      ccv_nnc_cmd_exec(transform, ccv_nnc_no_hint, 0, TENSOR_LIST(gw), TENSOR_LIST(gwo), stream_context);
    2046
    0
      ccv_nnc_stream_context_wait(stream_context);
    2047
    0
      ccv_nnc_tensor_free(gw);
    2048
    0
      cmd.backend = CCV_NNC_BACKEND_MPS;
    2049
    0
      assert(cmd.backend >= 0);
    2050
    0
      cmd.algorithm = -1;
    2051
    0
      cmd = ccv_nnc_cmd_autotune(cmd, 1 * 1024 * 1024 * 1024, hint, 0, TENSOR_LIST(ga, gwo), TENSOR_LIST(gc), stream_context);
    2052
    0
      assert(CCV_NNC_EXEC_SUCCESS == ccv_nnc_cmd_exec(cmd, hint, 0, TENSOR_LIST(ga, gwo), TENSOR_LIST(gc), stream_context));
    2053
    0
      ccv_nnc_stream_context_wait(stream_context);
    2054
    0
      ccv_nnc_stream_context_free(stream_context);
    2055
    0
      ccv_nnc_tensor_t* c = ccv_nnc_tensor_new(0, CPU_TENSOR_NHWC(32F, batch_size, output_depth, output_height, output_width, output_channels), 0);
    2056
    0
      ccv_nnc_cmd_exec(move, ccv_nnc_no_hint, 0, TENSOR_LIST(gc), TENSOR_LIST(c), 0);
    2057
    0
      cmd.backend = CCV_NNC_BACKEND_CPU_REF;
    2058
    0
      assert(cmd.backend >= 0);
    2059
    0
      ccv_nnc_cmd_exec(cmd, hint, 0, TENSOR_LIST(a, w), TENSOR_LIST(b), 0);
    2060
    0
      REQUIRE_ARRAY_EQ_WITH_TOLERANCE(float, b->data.f32, c->data.f32, batch_size * output_depth * output_height * output_width * output_channels, 1e-4, "output from mps should match from CPU");
    2061
    0
      ccv_nnc_tensor_free(c);
    2062
    0
      ccv_nnc_tensor_free(gc);
    2063
    0
      ccv_nnc_tensor_free(w);
    2064
    0
      ccv_nnc_tensor_free(b);
    2065
    0
      ccv_nnc_tensor_free(a);
    2066
    0
      ccv_nnc_tensor_free(gwo);
    2067
    0
      ccv_nnc_tensor_free(ga);
    2068
    0
    }
    2069
    2070
    TEST_CASE("mps forward convolution 3d via mfa conv3d with one point padding on 9x8 spatial dimensions")
    2071
    1
    {
    2072
    1
      GUARD_ELSE_RETURN(ccv_nnc_cmd_ok(CCV_NNC_CONVOLUTION_FORWARD, CCV_NNC_BACKEND_MPS));
    2073
    0
      const int batch_size = 2;
    2074
    0
      const int input_channels = 16;
    2075
    0
      const int output_channels = 32;
    2076
    0
      const int input_depth = 5;
    2077
    0
      const int input_height = 9;
    2078
    0
      const int input_width = 8;
    2079
    0
      const int kernel_depth = 3;
    2080
    0
      const int kernel_height = 3;
    2081
    0
      const int kernel_width = 3;
    2082
    0
      const int padding_top = 1;
    2083
    0
      const int padding_bottom = 1;
    2084
    0
      const int padding_left = 1;
    2085
    0
      const int padding_right = 1;
    2086
    0
      const int output_depth = input_depth - kernel_depth + 1;
    2087
    0
      const int output_height = input_height + padding_top + padding_bottom - kernel_height + 1;
    2088
    0
      const int output_width = input_width + padding_left + padding_right - kernel_width + 1;
    2089
    0
      ccv_nnc_tensor_t* a = ccv_nnc_tensor_new(0, CPU_TENSOR_NHWC(32F, batch_size, input_depth, input_height, input_width, input_channels), 0);
    2090
    0
      ccv_nnc_tensor_t* b = ccv_nnc_tensor_new(0, CPU_TENSOR_NHWC(32F, batch_size, output_depth, output_height, output_width, output_channels), 0);
    2091
    0
      ccv_nnc_cmd_t cmd = CMD_CONVOLUTION_FORWARD(1, output_channels, kernel_depth, kernel_height, kernel_width, input_channels);
    2092
    0
      ccv_nnc_hint_t hint = ccv_nnc_no_hint;
    2093
    0
      hint.stride.dim[0] = 1;
    2094
    0
      hint.stride.dim[1] = 1;
    2095
    0
      hint.stride.dim[2] = 1;
    2096
    0
      hint.border.begin[0] = 0;
    2097
    0
      hint.border.end[0] = 0;
    2098
    0
      hint.border.begin[1] = padding_top;
    2099
    0
      hint.border.end[1] = padding_bottom;
    2100
    0
      hint.border.begin[2] = padding_left;
    2101
    0
      hint.border.end[2] = padding_right;
    2102
    0
      assert(ccv_nnc_hint_verify(hint, cmd.info, a->info, b->info) == 0);
    2103
    0
      ccv_nnc_tensor_t* w = ccv_nnc_tensor_new(0, CPU_TENSOR_NHWC(32F, output_channels, kernel_depth, kernel_height, kernel_width, input_channels), 0);
    2104
    0
      dsfmt_t dsfmt;
    2105
    0
      dsfmt_init_gen_rand(&dsfmt, 13);
    2106
    0
      int i;
    2107
    0
      for (i = 0; i < batch_size * input_depth * input_height * input_width * input_channels; i++)
    2108
    0
        a->data.f32[i] = dsfmt_genrand_open_close(&dsfmt) - 0.5;
    2109
    0
      for (i = 0; i < output_channels * kernel_depth * kernel_height * kernel_width * input_channels; i++)
    2110
    0
        w->data.f32[i] = (dsfmt_genrand_open_close(&dsfmt) - 0.5) * 0.1;
    2111
    0
      ccv_nnc_tensor_t* ga = ccv_nnc_tensor_new(0, GPU_TENSOR_NHWC(000, 32F, batch_size, input_depth, input_height, input_width, input_channels), 0);
    2112
    0
      ccv_nnc_tensor_t* gw = ccv_nnc_tensor_new(0, GPU_TENSOR_NHWC(000, 32F, output_channels, kernel_depth, kernel_height, kernel_width, input_channels), 0);
    2113
    0
      ccv_nnc_tensor_t* gwo = ccv_nnc_tensor_new(0, GPU_TENSOR_NCHW(000, 32F, output_channels, input_channels, kernel_depth, kernel_height, kernel_width), 0);
    2114
    0
      ccv_nnc_tensor_t* gc = ccv_nnc_tensor_new(0, GPU_TENSOR_NHWC(000, 32F, batch_size, output_depth, output_height, output_width, output_channels), 0);
    2115
    0
      ccv_nnc_cmd_t move = CMD_DATA_TRANSFER_FORWARD();
    2116
    0
      move.backend = CCV_NNC_BACKEND_MPS;
    2117
    0
      assert(move.backend >= 0);
    2118
    0
      ccv_nnc_cmd_exec(move, ccv_nnc_no_hint, 0, TENSOR_LIST(a, w), TENSOR_LIST(ga, gw), 0);
    2119
    0
      ccv_nnc_cmd_t transform = CMD_FORMAT_TRANSFORM_FORWARD();
    2120
    0
      transform.backend = CCV_NNC_BACKEND_MPS;
    2121
    0
      assert(transform.backend >= 0);
    2122
    0
      ccv_nnc_stream_context_t* stream_context = ccv_nnc_stream_context_new(CCV_STREAM_CONTEXT_GPU);
    2123
    0
      ccv_nnc_cmd_exec(transform, ccv_nnc_no_hint, 0, TENSOR_LIST(gw), TENSOR_LIST(gwo), stream_context);
    2124
    0
      ccv_nnc_stream_context_wait(stream_context);
    2125
    0
      ccv_nnc_tensor_free(gw);
    2126
    0
      cmd.backend = CCV_NNC_BACKEND_MPS;
    2127
    0
      assert(cmd.backend >= 0);
    2128
    0
      cmd.algorithm = -1;
    2129
    0
      cmd = ccv_nnc_cmd_autotune(cmd, 1 * 1024 * 1024 * 1024, hint, 0, TENSOR_LIST(ga, gwo), TENSOR_LIST(gc), stream_context);
    2130
    0
      assert(CCV_NNC_EXEC_SUCCESS == ccv_nnc_cmd_exec(cmd, hint, 0, TENSOR_LIST(ga, gwo), TENSOR_LIST(gc), stream_context));
    2131
    0
      ccv_nnc_stream_context_wait(stream_context);
    2132
    0
      ccv_nnc_stream_context_free(stream_context);
    2133
    0
      ccv_nnc_tensor_t* c = ccv_nnc_tensor_new(0, CPU_TENSOR_NHWC(32F, batch_size, output_depth, output_height, output_width, output_channels), 0);
    2134
    0
      ccv_nnc_cmd_exec(move, ccv_nnc_no_hint, 0, TENSOR_LIST(gc), TENSOR_LIST(c), 0);
    2135
    0
      cmd.backend = CCV_NNC_BACKEND_CPU_REF;
    2136
    0
      assert(cmd.backend >= 0);
    2137
    0
      ccv_nnc_cmd_exec(cmd, hint, 0, TENSOR_LIST(a, w), TENSOR_LIST(b), 0);
    2138
    0
      REQUIRE_ARRAY_EQ_WITH_TOLERANCE(float, b->data.f32, c->data.f32, batch_size * output_depth * output_height * output_width * output_channels, 1e-4, "output from mps should match from CPU");
    2139
    0
      ccv_nnc_tensor_free(c);
    2140
    0
      ccv_nnc_tensor_free(gc);
    2141
    0
      ccv_nnc_tensor_free(w);
    2142
    0
      ccv_nnc_tensor_free(b);
    2143
    0
      ccv_nnc_tensor_free(a);
    2144
    0
      ccv_nnc_tensor_free(gwo);
    2145
    0
      ccv_nnc_tensor_free(ga);
    2146
    0
    }
    2147
    2148
    TEST_CASE("mps forward convolution 3d via mfa conv3d with one point padding on 17x19 spatial dimensions")
    2149
    1
    {
    2150
    1
      GUARD_ELSE_RETURN(ccv_nnc_cmd_ok(CCV_NNC_CONVOLUTION_FORWARD, CCV_NNC_BACKEND_MPS));
    2151
    0
      const int batch_size = 2;
    2152
    0
      const int input_channels = 16;
    2153
    0
      const int output_channels = 32;
    2154
    0
      const int input_depth = 5;
    2155
    0
      const int input_height = 17;
    2156
    0
      const int input_width = 19;
    2157
    0
      const int kernel_depth = 3;
    2158
    0
      const int kernel_height = 3;
    2159
    0
      const int kernel_width = 3;
    2160
    0
      const int padding_top = 1;
    2161
    0
      const int padding_bottom = 1;
    2162
    0
      const int padding_left = 1;
    2163
    0
      const int padding_right = 1;
    2164
    0
      const int output_depth = input_depth - kernel_depth + 1;
    2165
    0
      const int output_height = input_height + padding_top + padding_bottom - kernel_height + 1;
    2166
    0
      const int output_width = input_width + padding_left + padding_right - kernel_width + 1;
    2167
    0
      ccv_nnc_tensor_t* a = ccv_nnc_tensor_new(0, CPU_TENSOR_NHWC(32F, batch_size, input_depth, input_height, input_width, input_channels), 0);
    2168
    0
      ccv_nnc_tensor_t* b = ccv_nnc_tensor_new(0, CPU_TENSOR_NHWC(32F, batch_size, output_depth, output_height, output_width, output_channels), 0);
    2169
    0
      ccv_nnc_cmd_t cmd = CMD_CONVOLUTION_FORWARD(1, output_channels, kernel_depth, kernel_height, kernel_width, input_channels);
    2170
    0
      ccv_nnc_hint_t hint = ccv_nnc_no_hint;
    2171
    0
      hint.stride.dim[0] = 1;
    2172
    0
      hint.stride.dim[1] = 1;
    2173
    0
      hint.stride.dim[2] = 1;
    2174
    0
      hint.border.begin[0] = 0;
    2175
    0
      hint.border.end[0] = 0;
    2176
    0
      hint.border.begin[1] = padding_top;
    2177
    0
      hint.border.end[1] = padding_bottom;
    2178
    0
      hint.border.begin[2] = padding_left;
    2179
    0
      hint.border.end[2] = padding_right;
    2180
    0
      assert(ccv_nnc_hint_verify(hint, cmd.info, a->info, b->info) == 0);
    2181
    0
      ccv_nnc_tensor_t* w = ccv_nnc_tensor_new(0, CPU_TENSOR_NHWC(32F, output_channels, kernel_depth, kernel_height, kernel_width, input_channels), 0);
    2182
    0
      dsfmt_t dsfmt;
    2183
    0
      dsfmt_init_gen_rand(&dsfmt, 17);
    2184
    0
      int i;
    2185
    0
      for (i = 0; i < batch_size * input_depth * input_height * input_width * input_channels; i++)
    2186
    0
        a->data.f32[i] = dsfmt_genrand_open_close(&dsfmt) - 0.5;
    2187
    0
      for (i = 0; i < output_channels * kernel_depth * kernel_height * kernel_width * input_channels; i++)
    2188
    0
        w->data.f32[i] = (dsfmt_genrand_open_close(&dsfmt) - 0.5) * 0.1;
    2189
    0
      ccv_nnc_tensor_t* ga = ccv_nnc_tensor_new(0, GPU_TENSOR_NHWC(000, 32F, batch_size, input_depth, input_height, input_width, input_channels), 0);
    2190
    0
      ccv_nnc_tensor_t* gw = ccv_nnc_tensor_new(0, GPU_TENSOR_NHWC(000, 32F, output_channels, kernel_depth, kernel_height, kernel_width, input_channels), 0);
    2191
    0
      ccv_nnc_tensor_t* gwo = ccv_nnc_tensor_new(0, GPU_TENSOR_NCHW(000, 32F, output_channels, input_channels, kernel_depth, kernel_height, kernel_width), 0);
    2192
    0
      ccv_nnc_tensor_t* gc = ccv_nnc_tensor_new(0, GPU_TENSOR_NHWC(000, 32F, batch_size, output_depth, output_height, output_width, output_channels), 0);
    2193
    0
      ccv_nnc_cmd_t move = CMD_DATA_TRANSFER_FORWARD();
    2194
    0
      move.backend = CCV_NNC_BACKEND_MPS;
    2195
    0
      assert(move.backend >= 0);
    2196
    0
      ccv_nnc_cmd_exec(move, ccv_nnc_no_hint, 0, TENSOR_LIST(a, w), TENSOR_LIST(ga, gw), 0);
    2197
    0
      ccv_nnc_cmd_t transform = CMD_FORMAT_TRANSFORM_FORWARD();
    2198
    0
      transform.backend = CCV_NNC_BACKEND_MPS;
    2199
    0
      assert(transform.backend >= 0);
    2200
    0
      ccv_nnc_stream_context_t* stream_context = ccv_nnc_stream_context_new(CCV_STREAM_CONTEXT_GPU);
    2201
    0
      ccv_nnc_cmd_exec(transform, ccv_nnc_no_hint, 0, TENSOR_LIST(gw), TENSOR_LIST(gwo), stream_context);
    2202
    0
      ccv_nnc_stream_context_wait(stream_context);
    2203
    0
      ccv_nnc_tensor_free(gw);
    2204
    0
      cmd.backend = CCV_NNC_BACKEND_MPS;
    2205
    0
      assert(cmd.backend >= 0);
    2206
    0
      cmd.algorithm = -1;
    2207
    0
      cmd = ccv_nnc_cmd_autotune(cmd, 1 * 1024 * 1024 * 1024, hint, 0, TENSOR_LIST(ga, gwo), TENSOR_LIST(gc), stream_context);
    2208
    0
      assert(CCV_NNC_EXEC_SUCCESS == ccv_nnc_cmd_exec(cmd, hint, 0, TENSOR_LIST(ga, gwo), TENSOR_LIST(gc), stream_context));
    2209
    0
      ccv_nnc_stream_context_wait(stream_context);
    2210
    0
      ccv_nnc_stream_context_free(stream_context);
    2211
    0
      ccv_nnc_tensor_t* c = ccv_nnc_tensor_new(0, CPU_TENSOR_NHWC(32F, batch_size, output_depth, output_height, output_width, output_channels), 0);
    2212
    0
      ccv_nnc_cmd_exec(move, ccv_nnc_no_hint, 0, TENSOR_LIST(gc), TENSOR_LIST(c), 0);
    2213
    0
      cmd.backend = CCV_NNC_BACKEND_CPU_REF;
    2214
    0
      assert(cmd.backend >= 0);
    2215
    0
      ccv_nnc_cmd_exec(cmd, hint, 0, TENSOR_LIST(a, w), TENSOR_LIST(b), 0);
    2216
    0
      REQUIRE_ARRAY_EQ_WITH_TOLERANCE(float, b->data.f32, c->data.f32, batch_size * output_depth * output_height * output_width * output_channels, 1e-4, "output from mps should match from CPU");
    2217
    0
      ccv_nnc_tensor_free(c);
    2218
    0
      ccv_nnc_tensor_free(gc);
    2219
    0
      ccv_nnc_tensor_free(w);
    2220
    0
      ccv_nnc_tensor_free(b);
    2221
    0
      ccv_nnc_tensor_free(a);
    2222
    0
      ccv_nnc_tensor_free(gwo);
    2223
    0
      ccv_nnc_tensor_free(ga);
    2224
    0
    }
    2225
    2226
    TEST_CASE("mps forward convolution 3d via mfa conv3d with one point padding on 33x35 spatial dimensions")
    2227
    1
    {
    2228
    1
      GUARD_ELSE_RETURN(ccv_nnc_cmd_ok(CCV_NNC_CONVOLUTION_FORWARD, CCV_NNC_BACKEND_MPS));
    2229
    0
      const int batch_size = 2;
    2230
    0
      const int input_channels = 16;
    2231
    0
      const int output_channels = 32;
    2232
    0
      const int input_depth = 4;
    2233
    0
      const int input_height = 33;
    2234
    0
      const int input_width = 35;
    2235
    0
      const int kernel_depth = 3;
    2236
    0
      const int kernel_height = 3;
    2237
    0
      const int kernel_width = 3;
    2238
    0
      const int padding_top = 1;
    2239
    0
      const int padding_bottom = 1;
    2240
    0
      const int padding_left = 1;
    2241
    0
      const int padding_right = 1;
    2242
    0
      const int output_depth = input_depth - kernel_depth + 1;
    2243
    0
      const int output_height = input_height + padding_top + padding_bottom - kernel_height + 1;
    2244
    0
      const int output_width = input_width + padding_left + padding_right - kernel_width + 1;
    2245
    0
      ccv_nnc_tensor_t* a = ccv_nnc_tensor_new(0, CPU_TENSOR_NHWC(32F, batch_size, input_depth, input_height, input_width, input_channels), 0);
    2246
    0
      ccv_nnc_tensor_t* b = ccv_nnc_tensor_new(0, CPU_TENSOR_NHWC(32F, batch_size, output_depth, output_height, output_width, output_channels), 0);
    2247
    0
      ccv_nnc_cmd_t cmd = CMD_CONVOLUTION_FORWARD(1, output_channels, kernel_depth, kernel_height, kernel_width, input_channels);
    2248
    0
      ccv_nnc_hint_t hint = ccv_nnc_no_hint;
    2249
    0
      hint.stride.dim[0] = 1;
    2250
    0
      hint.stride.dim[1] = 1;
    2251
    0
      hint.stride.dim[2] = 1;
    2252
    0
      hint.border.begin[0] = 0;
    2253
    0
      hint.border.end[0] = 0;
    2254
    0
      hint.border.begin[1] = padding_top;
    2255
    0
      hint.border.end[1] = padding_bottom;
    2256
    0
      hint.border.begin[2] = padding_left;
    2257
    0
      hint.border.end[2] = padding_right;
    2258
    0
      assert(ccv_nnc_hint_verify(hint, cmd.info, a->info, b->info) == 0);
    2259
    0
      ccv_nnc_tensor_t* w = ccv_nnc_tensor_new(0, CPU_TENSOR_NHWC(32F, output_channels, kernel_depth, kernel_height, kernel_width, input_channels), 0);
    2260
    0
      dsfmt_t dsfmt;
    2261
    0
      dsfmt_init_gen_rand(&dsfmt, 18);
    2262
    0
      int i;
    2263
    0
      for (i = 0; i < batch_size * input_depth * input_height * input_width * input_channels; i++)
    2264
    0
        a->data.f32[i] = dsfmt_genrand_open_close(&dsfmt) - 0.5;
    2265
    0
      for (i = 0; i < output_channels * kernel_depth * kernel_height * kernel_width * input_channels; i++)
    2266
    0
        w->data.f32[i] = (dsfmt_genrand_open_close(&dsfmt) - 0.5) * 0.1;
    2267
    0
      ccv_nnc_tensor_t* ga = ccv_nnc_tensor_new(0, GPU_TENSOR_NHWC(000, 32F, batch_size, input_depth, input_height, input_width, input_channels), 0);
    2268
    0
      ccv_nnc_tensor_t* gw = ccv_nnc_tensor_new(0, GPU_TENSOR_NHWC(000, 32F, output_channels, kernel_depth, kernel_height, kernel_width, input_channels), 0);
    2269
    0
      ccv_nnc_tensor_t* gwo = ccv_nnc_tensor_new(0, GPU_TENSOR_NCHW(000, 32F, output_channels, input_channels, kernel_depth, kernel_height, kernel_width), 0);
    2270
    0
      ccv_nnc_tensor_t* gc = ccv_nnc_tensor_new(0, GPU_TENSOR_NHWC(000, 32F, batch_size, output_depth, output_height, output_width, output_channels), 0);
    2271
    0
      ccv_nnc_cmd_t move = CMD_DATA_TRANSFER_FORWARD();
    2272
    0
      move.backend = CCV_NNC_BACKEND_MPS;
    2273
    0
      assert(move.backend >= 0);
    2274
    0
      ccv_nnc_cmd_exec(move, ccv_nnc_no_hint, 0, TENSOR_LIST(a, w), TENSOR_LIST(ga, gw), 0);
    2275
    0
      ccv_nnc_cmd_t transform = CMD_FORMAT_TRANSFORM_FORWARD();
    2276
    0
      transform.backend = CCV_NNC_BACKEND_MPS;
    2277
    0
      assert(transform.backend >= 0);
    2278
    0
      ccv_nnc_stream_context_t* stream_context = ccv_nnc_stream_context_new(CCV_STREAM_CONTEXT_GPU);
    2279
    0
      ccv_nnc_cmd_exec(transform, ccv_nnc_no_hint, 0, TENSOR_LIST(gw), TENSOR_LIST(gwo), stream_context);
    2280
    0
      ccv_nnc_stream_context_wait(stream_context);
    2281
    0
      ccv_nnc_tensor_free(gw);
    2282
    0
      cmd.backend = CCV_NNC_BACKEND_MPS;
    2283
    0
      assert(cmd.backend >= 0);
    2284
    0
      cmd.algorithm = -1;
    2285
    0
      cmd = ccv_nnc_cmd_autotune(cmd, 1 * 1024 * 1024 * 1024, hint, 0, TENSOR_LIST(ga, gwo), TENSOR_LIST(gc), stream_context);
    2286
    0
      assert(CCV_NNC_EXEC_SUCCESS == ccv_nnc_cmd_exec(cmd, hint, 0, TENSOR_LIST(ga, gwo), TENSOR_LIST(gc), stream_context));
    2287
    0
      ccv_nnc_stream_context_wait(stream_context);
    2288
    0
      ccv_nnc_stream_context_free(stream_context);
    2289
    0
      ccv_nnc_tensor_t* c = ccv_nnc_tensor_new(0, CPU_TENSOR_NHWC(32F, batch_size, output_depth, output_height, output_width, output_channels), 0);
    2290
    0
      ccv_nnc_cmd_exec(move, ccv_nnc_no_hint, 0, TENSOR_LIST(gc), TENSOR_LIST(c), 0);
    2291
    0
      cmd.backend = CCV_NNC_BACKEND_CPU_REF;
    2292
    0
      assert(cmd.backend >= 0);
    2293
    0
      ccv_nnc_cmd_exec(cmd, hint, 0, TENSOR_LIST(a, w), TENSOR_LIST(b), 0);
    2294
    0
      REQUIRE_ARRAY_EQ_WITH_TOLERANCE(float, b->data.f32, c->data.f32, batch_size * output_depth * output_height * output_width * output_channels, 1e-4, "output from mps should match from CPU");
    2295
    0
      ccv_nnc_tensor_free(c);
    2296
    0
      ccv_nnc_tensor_free(gc);
    2297
    0
      ccv_nnc_tensor_free(w);
    2298
    0
      ccv_nnc_tensor_free(b);
    2299
    0
      ccv_nnc_tensor_free(a);
    2300
    0
      ccv_nnc_tensor_free(gwo);
    2301
    0
      ccv_nnc_tensor_free(ga);
    2302
    0
    }
    2303
    2304
    TEST_CASE("mps forward convolution 3d via mfa conv3d with one point padding on 9x65 spatial dimensions")
    2305
    1
    {
    2306
    1
      GUARD_ELSE_RETURN(ccv_nnc_cmd_ok(CCV_NNC_CONVOLUTION_FORWARD, CCV_NNC_BACKEND_MPS));
    2307
    0
      const int batch_size = 2;
    2308
    0
      const int input_channels = 16;
    2309
    0
      const int output_channels = 32;
    2310
    0
      const int input_depth = 4;
    2311
    0
      const int input_height = 9;
    2312
    0
      const int input_width = 65;
    2313
    0
      const int kernel_depth = 3;
    2314
    0
      const int kernel_height = 3;
    2315
    0
      const int kernel_width = 3;
    2316
    0
      const int padding_top = 1;
    2317
    0
      const int padding_bottom = 1;
    2318
    0
      const int padding_left = 1;
    2319
    0
      const int padding_right = 1;
    2320
    0
      const int output_depth = input_depth - kernel_depth + 1;
    2321
    0
      const int output_height = input_height + padding_top + padding_bottom - kernel_height + 1;
    2322
    0
      const int output_width = input_width + padding_left + padding_right - kernel_width + 1;
    2323
    0
      ccv_nnc_tensor_t* a = ccv_nnc_tensor_new(0, CPU_TENSOR_NHWC(32F, batch_size, input_depth, input_height, input_width, input_channels), 0);
    2324
    0
      ccv_nnc_tensor_t* b = ccv_nnc_tensor_new(0, CPU_TENSOR_NHWC(32F, batch_size, output_depth, output_height, output_width, output_channels), 0);
    2325
    0
      ccv_nnc_cmd_t cmd = CMD_CONVOLUTION_FORWARD(1, output_channels, kernel_depth, kernel_height, kernel_width, input_channels);
    2326
    0
      ccv_nnc_hint_t hint = ccv_nnc_no_hint;
    2327
    0
      hint.stride.dim[0] = 1;
    2328
    0
      hint.stride.dim[1] = 1;
    2329
    0
      hint.stride.dim[2] = 1;
    2330
    0
      hint.border.begin[0] = 0;
    2331
    0
      hint.border.end[0] = 0;
    2332
    0
      hint.border.begin[1] = padding_top;
    2333
    0
      hint.border.end[1] = padding_bottom;
    2334
    0
      hint.border.begin[2] = padding_left;
    2335
    0
      hint.border.end[2] = padding_right;
    2336
    0
      assert(ccv_nnc_hint_verify(hint, cmd.info, a->info, b->info) == 0);
    2337
    0
      ccv_nnc_tensor_t* w = ccv_nnc_tensor_new(0, CPU_TENSOR_NHWC(32F, output_channels, kernel_depth, kernel_height, kernel_width, input_channels), 0);
    2338
    0
      dsfmt_t dsfmt;
    2339
    0
      dsfmt_init_gen_rand(&dsfmt, 19);
    2340
    0
      int i;
    2341
    0
      for (i = 0; i < batch_size * input_depth * input_height * input_width * input_channels; i++)
    2342
    0
        a->data.f32[i] = dsfmt_genrand_open_close(&dsfmt) - 0.5;
    2343
    0
      for (i = 0; i < output_channels * kernel_depth * kernel_height * kernel_width * input_channels; i++)
    2344
    0
        w->data.f32[i] = (dsfmt_genrand_open_close(&dsfmt) - 0.5) * 0.1;
    2345
    0
      ccv_nnc_tensor_t* ga = ccv_nnc_tensor_new(0, GPU_TENSOR_NHWC(000, 32F, batch_size, input_depth, input_height, input_width, input_channels), 0);
    2346
    0
      ccv_nnc_tensor_t* gw = ccv_nnc_tensor_new(0, GPU_TENSOR_NHWC(000, 32F, output_channels, kernel_depth, kernel_height, kernel_width, input_channels), 0);
    2347
    0
      ccv_nnc_tensor_t* gwo = ccv_nnc_tensor_new(0, GPU_TENSOR_NCHW(000, 32F, output_channels, input_channels, kernel_depth, kernel_height, kernel_width), 0);
    2348
    0
      ccv_nnc_tensor_t* gc = ccv_nnc_tensor_new(0, GPU_TENSOR_NHWC(000, 32F, batch_size, output_depth, output_height, output_width, output_channels), 0);
    2349
    0
      ccv_nnc_cmd_t move = CMD_DATA_TRANSFER_FORWARD();
    2350
    0
      move.backend = CCV_NNC_BACKEND_MPS;
    2351
    0
      assert(move.backend >= 0);
    2352
    0
      ccv_nnc_cmd_exec(move, ccv_nnc_no_hint, 0, TENSOR_LIST(a, w), TENSOR_LIST(ga, gw), 0);
    2353
    0
      ccv_nnc_cmd_t transform = CMD_FORMAT_TRANSFORM_FORWARD();
    2354
    0
      transform.backend = CCV_NNC_BACKEND_MPS;
    2355
    0
      assert(transform.backend >= 0);
    2356
    0
      ccv_nnc_stream_context_t* stream_context = ccv_nnc_stream_context_new(CCV_STREAM_CONTEXT_GPU);
    2357
    0
      ccv_nnc_cmd_exec(transform, ccv_nnc_no_hint, 0, TENSOR_LIST(gw), TENSOR_LIST(gwo), stream_context);
    2358
    0
      ccv_nnc_stream_context_wait(stream_context);
    2359
    0
      ccv_nnc_tensor_free(gw);
    2360
    0
      cmd.backend = CCV_NNC_BACKEND_MPS;
    2361
    0
      assert(cmd.backend >= 0);
    2362
    0
      cmd.algorithm = -1;
    2363
    0
      cmd = ccv_nnc_cmd_autotune(cmd, 1 * 1024 * 1024 * 1024, hint, 0, TENSOR_LIST(ga, gwo), TENSOR_LIST(gc), stream_context);
    2364
    0
      assert(CCV_NNC_EXEC_SUCCESS == ccv_nnc_cmd_exec(cmd, hint, 0, TENSOR_LIST(ga, gwo), TENSOR_LIST(gc), stream_context));
    2365
    0
      ccv_nnc_stream_context_wait(stream_context);
    2366
    0
      ccv_nnc_stream_context_free(stream_context);
    2367
    0
      ccv_nnc_tensor_t* c = ccv_nnc_tensor_new(0, CPU_TENSOR_NHWC(32F, batch_size, output_depth, output_height, output_width, output_channels), 0);
    2368
    0
      ccv_nnc_cmd_exec(move, ccv_nnc_no_hint, 0, TENSOR_LIST(gc), TENSOR_LIST(c), 0);
    2369
    0
      cmd.backend = CCV_NNC_BACKEND_CPU_REF;
    2370
    0
      assert(cmd.backend >= 0);
    2371
    0
      ccv_nnc_cmd_exec(cmd, hint, 0, TENSOR_LIST(a, w), TENSOR_LIST(b), 0);
    2372
    0
      REQUIRE_ARRAY_EQ_WITH_TOLERANCE(float, b->data.f32, c->data.f32, batch_size * output_depth * output_height * output_width * output_channels, 1e-4, "output from mps should match from CPU");
    2373
    0
      ccv_nnc_tensor_free(c);
    2374
    0
      ccv_nnc_tensor_free(gc);
    2375
    0
      ccv_nnc_tensor_free(w);
    2376
    0
      ccv_nnc_tensor_free(b);
    2377
    0
      ccv_nnc_tensor_free(a);
    2378
    0
      ccv_nnc_tensor_free(gwo);
    2379
    0
      ccv_nnc_tensor_free(ga);
    2380
    0
    }
    2381
    2382
    TEST_CASE("mps forward convolution 3d via mfa conv3d with partial output channel tile and bias")
    2383
    1
    {
    2384
    1
      GUARD_ELSE_RETURN(ccv_nnc_cmd_ok(CCV_NNC_CONVOLUTION_FORWARD, CCV_NNC_BACKEND_MPS));
    2385
    0
      const int batch_size = 2;
    2386
    0
      const int input_channels = 16;
    2387
    0
      const int output_channels = 16;
    2388
    0
      const int input_depth = 4;
    2389
    0
      const int input_height = 33;
    2390
    0
      const int input_width = 35;
    2391
    0
      const int kernel_depth = 3;
    2392
    0
      const int kernel_height = 3;
    2393
    0
      const int kernel_width = 3;
    2394
    0
      const int padding_top = 1;
    2395
    0
      const int padding_bottom = 1;
    2396
    0
      const int padding_left = 1;
    2397
    0
      const int padding_right = 1;
    2398
    0
      const int output_depth = input_depth - kernel_depth + 1;
    2399
    0
      const int output_height = input_height + padding_top + padding_bottom - kernel_height + 1;
    2400
    0
      const int output_width = input_width + padding_left + padding_right - kernel_width + 1;
    2401
    0
      ccv_nnc_tensor_t* a = ccv_nnc_tensor_new(0, CPU_TENSOR_NHWC(32F, batch_size, input_depth, input_height, input_width, input_channels), 0);
    2402
    0
      ccv_nnc_tensor_t* b = ccv_nnc_tensor_new(0, CPU_TENSOR_NHWC(32F, batch_size, output_depth, output_height, output_width, output_channels), 0);
    2403
    0
      ccv_nnc_cmd_t cmd = CMD_CONVOLUTION_FORWARD(1, output_channels, kernel_depth, kernel_height, kernel_width, input_channels);
    2404
    0
      ccv_nnc_hint_t hint = ccv_nnc_no_hint;
    2405
    0
      hint.stride.dim[0] = 1;
    2406
    0
      hint.stride.dim[1] = 1;
    2407
    0
      hint.stride.dim[2] = 1;
    2408
    0
      hint.border.begin[0] = 0;
    2409
    0
      hint.border.end[0] = 0;
    2410
    0
      hint.border.begin[1] = padding_top;
    2411
    0
      hint.border.end[1] = padding_bottom;
    2412
    0
      hint.border.begin[2] = padding_left;
    2413
    0
      hint.border.end[2] = padding_right;
    2414
    0
      assert(ccv_nnc_hint_verify(hint, cmd.info, a->info, b->info) == 0);
    2415
    0
      ccv_nnc_tensor_t* w = ccv_nnc_tensor_new(0, CPU_TENSOR_NHWC(32F, output_channels, kernel_depth, kernel_height, kernel_width, input_channels), 0);
    2416
    0
      ccv_nnc_tensor_t* bias = ccv_nnc_tensor_new(0, CPU_TENSOR_NHWC(32F, output_channels), 0);
    2417
    0
      dsfmt_t dsfmt;
    2418
    0
      dsfmt_init_gen_rand(&dsfmt, 21);
    2419
    0
      int i;
    2420
    0
      for (i = 0; i < batch_size * input_depth * input_height * input_width * input_channels; i++)
    2421
    0
        a->data.f32[i] = dsfmt_genrand_open_close(&dsfmt) - 0.5;
    2422
    0
      for (i = 0; i < output_channels * kernel_depth * kernel_height * kernel_width * input_channels; i++)
    2423
    0
        w->data.f32[i] = (dsfmt_genrand_open_close(&dsfmt) - 0.5) * 0.1;
    2424
    0
      for (i = 0; i < output_channels; i++)
    2425
    0
        bias->data.f32[i] = (dsfmt_genrand_open_close(&dsfmt) - 0.5) * 0.1;
    2426
    0
      ccv_nnc_tensor_t* ga = ccv_nnc_tensor_new(0, GPU_TENSOR_NHWC(000, 32F, batch_size, input_depth, input_height, input_width, input_channels), 0);
    2427
    0
      ccv_nnc_tensor_t* gw = ccv_nnc_tensor_new(0, GPU_TENSOR_NHWC(000, 32F, output_channels, kernel_depth, kernel_height, kernel_width, input_channels), 0);
    2428
    0
      ccv_nnc_tensor_t* gwo = ccv_nnc_tensor_new(0, GPU_TENSOR_NCHW(000, 32F, output_channels, input_channels, kernel_depth, kernel_height, kernel_width), 0);
    2429
    0
      ccv_nnc_tensor_t* gbias = ccv_nnc_tensor_new(0, GPU_TENSOR_NHWC(000, 32F, output_channels), 0);
    2430
    0
      ccv_nnc_tensor_t* gc = ccv_nnc_tensor_new(0, GPU_TENSOR_NHWC(000, 32F, batch_size, output_depth, output_height, output_width, output_channels), 0);
    2431
    0
      ccv_nnc_cmd_t move = CMD_DATA_TRANSFER_FORWARD();
    2432
    0
      move.backend = CCV_NNC_BACKEND_MPS;
    2433
    0
      assert(move.backend >= 0);
    2434
    0
      ccv_nnc_cmd_exec(move, ccv_nnc_no_hint, 0, TENSOR_LIST(a, w, bias), TENSOR_LIST(ga, gw, gbias), 0);
    2435
    0
      ccv_nnc_cmd_t transform = CMD_FORMAT_TRANSFORM_FORWARD();
    2436
    0
      transform.backend = CCV_NNC_BACKEND_MPS;
    2437
    0
      assert(transform.backend >= 0);
    2438
    0
      ccv_nnc_stream_context_t* stream_context = ccv_nnc_stream_context_new(CCV_STREAM_CONTEXT_GPU);
    2439
    0
      ccv_nnc_cmd_exec(transform, ccv_nnc_no_hint, 0, TENSOR_LIST(gw), TENSOR_LIST(gwo), stream_context);
    2440
    0
      ccv_nnc_stream_context_wait(stream_context);
    2441
    0
      ccv_nnc_tensor_free(gw);
    2442
    0
      cmd.backend = CCV_NNC_BACKEND_MPS;
    2443
    0
      assert(cmd.backend >= 0);
    2444
    0
      cmd.algorithm = -1;
    2445
    0
      cmd = ccv_nnc_cmd_autotune(cmd, 1 * 1024 * 1024 * 1024, hint, 0, TENSOR_LIST(ga, gwo, gbias), TENSOR_LIST(gc), stream_context);
    2446
    0
      assert(CCV_NNC_EXEC_SUCCESS == ccv_nnc_cmd_exec(cmd, hint, 0, TENSOR_LIST(ga, gwo, gbias), TENSOR_LIST(gc), stream_context));
    2447
    0
      ccv_nnc_stream_context_wait(stream_context);
    2448
    0
      ccv_nnc_stream_context_free(stream_context);
    2449
    0
      ccv_nnc_tensor_t* c = ccv_nnc_tensor_new(0, CPU_TENSOR_NHWC(32F, batch_size, output_depth, output_height, output_width, output_channels), 0);
    2450
    0
      ccv_nnc_cmd_exec(move, ccv_nnc_no_hint, 0, TENSOR_LIST(gc), TENSOR_LIST(c), 0);
    2451
    0
      cmd.backend = CCV_NNC_BACKEND_CPU_REF;
    2452
    0
      assert(cmd.backend >= 0);
    2453
    0
      ccv_nnc_cmd_exec(cmd, hint, 0, TENSOR_LIST(a, w, bias), TENSOR_LIST(b), 0);
    2454
    0
      REQUIRE_ARRAY_EQ_WITH_TOLERANCE(float, b->data.f32, c->data.f32, batch_size * output_depth * output_height * output_width * output_channels, 1e-4, "output from mps should match from CPU");
    2455
    0
      ccv_nnc_tensor_free(c);
    2456
    0
      ccv_nnc_tensor_free(gc);
    2457
    0
      ccv_nnc_tensor_free(bias);
    2458
    0
      ccv_nnc_tensor_free(w);
    2459
    0
      ccv_nnc_tensor_free(b);
    2460
    0
      ccv_nnc_tensor_free(a);
    2461
    0
      ccv_nnc_tensor_free(gbias);
    2462
    0
      ccv_nnc_tensor_free(gwo);
    2463
    0
      ccv_nnc_tensor_free(ga);
    2464
    0
    }
    2465
    2466
    TEST_CASE("mps forward convolution 3d via mfa conv3d with 48 channels and partial output channel tile")
    2467
    1
    {
    2468
    1
      GUARD_ELSE_RETURN(ccv_nnc_cmd_ok(CCV_NNC_CONVOLUTION_FORWARD, CCV_NNC_BACKEND_MPS));
    2469
    0
      const int batch_size = 2;
    2470
    0
      const int input_channels = 48;
    2471
    0
      const int output_channels = 48;
    2472
    0
      const int input_depth = 4;
    2473
    0
      const int input_height = 33;
    2474
    0
      const int input_width = 35;
    2475
    0
      const int kernel_depth = 3;
    2476
    0
      const int kernel_height = 3;
    2477
    0
      const int kernel_width = 3;
    2478
    0
      const int padding_top = 1;
    2479
    0
      const int padding_bottom = 1;
    2480
    0
      const int padding_left = 1;
    2481
    0
      const int padding_right = 1;
    2482
    0
      const int output_depth = input_depth - kernel_depth + 1;
    2483
    0
      const int output_height = input_height + padding_top + padding_bottom - kernel_height + 1;
    2484
    0
      const int output_width = input_width + padding_left + padding_right - kernel_width + 1;
    2485
    0
      ccv_nnc_tensor_t* a = ccv_nnc_tensor_new(0, CPU_TENSOR_NHWC(32F, batch_size, input_depth, input_height, input_width, input_channels), 0);
    2486
    0
      ccv_nnc_tensor_t* b = ccv_nnc_tensor_new(0, CPU_TENSOR_NHWC(32F, batch_size, output_depth, output_height, output_width, output_channels), 0);
    2487
    0
      ccv_nnc_cmd_t cmd = CMD_CONVOLUTION_FORWARD(1, output_channels, kernel_depth, kernel_height, kernel_width, input_channels);
    2488
    0
      ccv_nnc_hint_t hint = ccv_nnc_no_hint;
    2489
    0
      hint.stride.dim[0] = 1;
    2490
    0
      hint.stride.dim[1] = 1;
    2491
    0
      hint.stride.dim[2] = 1;
    2492
    0
      hint.border.begin[0] = 0;
    2493
    0
      hint.border.end[0] = 0;
    2494
    0
      hint.border.begin[1] = padding_top;
    2495
    0
      hint.border.end[1] = padding_bottom;
    2496
    0
      hint.border.begin[2] = padding_left;
    2497
    0
      hint.border.end[2] = padding_right;
    2498
    0
      assert(ccv_nnc_hint_verify(hint, cmd.info, a->info, b->info) == 0);
    2499
    0
      ccv_nnc_tensor_t* w = ccv_nnc_tensor_new(0, CPU_TENSOR_NHWC(32F, output_channels, kernel_depth, kernel_height, kernel_width, input_channels), 0);
    2500
    0
      ccv_nnc_tensor_t* bias = ccv_nnc_tensor_new(0, CPU_TENSOR_NHWC(32F, output_channels), 0);
    2501
    0
      dsfmt_t dsfmt;
    2502
    0
      dsfmt_init_gen_rand(&dsfmt, 23);
    2503
    0
      int i;
    2504
    0
      for (i = 0; i < batch_size * input_depth * input_height * input_width * input_channels; i++)
    2505
    0
        a->data.f32[i] = dsfmt_genrand_open_close(&dsfmt) - 0.5;
    2506
    0
      for (i = 0; i < output_channels * kernel_depth * kernel_height * kernel_width * input_channels; i++)
    2507
    0
        w->data.f32[i] = (dsfmt_genrand_open_close(&dsfmt) - 0.5) * 0.1;
    2508
    0
      for (i = 0; i < output_channels; i++)
    2509
    0
        bias->data.f32[i] = (dsfmt_genrand_open_close(&dsfmt) - 0.5) * 0.1;
    2510
    0
      ccv_nnc_tensor_t* ga = ccv_nnc_tensor_new(0, GPU_TENSOR_NHWC(000, 32F, batch_size, input_depth, input_height, input_width, input_channels), 0);
    2511
    0
      ccv_nnc_tensor_t* gw = ccv_nnc_tensor_new(0, GPU_TENSOR_NHWC(000, 32F, output_channels, kernel_depth, kernel_height, kernel_width, input_channels), 0);
    2512
    0
      ccv_nnc_tensor_t* gwo = ccv_nnc_tensor_new(0, GPU_TENSOR_NCHW(000, 32F, output_channels, input_channels, kernel_depth, kernel_height, kernel_width), 0);
    2513
    0
      ccv_nnc_tensor_t* gbias = ccv_nnc_tensor_new(0, GPU_TENSOR_NHWC(000, 32F, output_channels), 0);
    2514
    0
      ccv_nnc_tensor_t* gc = ccv_nnc_tensor_new(0, GPU_TENSOR_NHWC(000, 32F, batch_size, output_depth, output_height, output_width, output_channels), 0);
    2515
    0
      ccv_nnc_cmd_t move = CMD_DATA_TRANSFER_FORWARD();
    2516
    0
      move.backend = CCV_NNC_BACKEND_MPS;
    2517
    0
      assert(move.backend >= 0);
    2518
    0
      ccv_nnc_cmd_exec(move, ccv_nnc_no_hint, 0, TENSOR_LIST(a, w, bias), TENSOR_LIST(ga, gw, gbias), 0);
    2519
    0
      ccv_nnc_cmd_t transform = CMD_FORMAT_TRANSFORM_FORWARD();
    2520
    0
      transform.backend = CCV_NNC_BACKEND_MPS;
    2521
    0
      assert(transform.backend >= 0);
    2522
    0
      ccv_nnc_stream_context_t* stream_context = ccv_nnc_stream_context_new(CCV_STREAM_CONTEXT_GPU);
    2523
    0
      ccv_nnc_cmd_exec(transform, ccv_nnc_no_hint, 0, TENSOR_LIST(gw), TENSOR_LIST(gwo), stream_context);
    2524
    0
      ccv_nnc_stream_context_wait(stream_context);
    2525
    0
      ccv_nnc_tensor_free(gw);
    2526
    0
      cmd.backend = CCV_NNC_BACKEND_MPS;
    2527
    0
      assert(cmd.backend >= 0);
    2528
    0
      cmd.algorithm = -1;
    2529
    0
      cmd = ccv_nnc_cmd_autotune(cmd, 1 * 1024 * 1024 * 1024, hint, 0, TENSOR_LIST(ga, gwo, gbias), TENSOR_LIST(gc), stream_context);
    2530
    0
      assert(CCV_NNC_EXEC_SUCCESS == ccv_nnc_cmd_exec(cmd, hint, 0, TENSOR_LIST(ga, gwo, gbias), TENSOR_LIST(gc), stream_context));
    2531
    0
      ccv_nnc_stream_context_wait(stream_context);
    2532
    0
      ccv_nnc_stream_context_free(stream_context);
    2533
    0
      ccv_nnc_tensor_t* c = ccv_nnc_tensor_new(0, CPU_TENSOR_NHWC(32F, batch_size, output_depth, output_height, output_width, output_channels), 0);
    2534
    0
      ccv_nnc_cmd_exec(move, ccv_nnc_no_hint, 0, TENSOR_LIST(gc), TENSOR_LIST(c), 0);
    2535
    0
      cmd.backend = CCV_NNC_BACKEND_CPU_REF;
    2536
    0
      assert(cmd.backend >= 0);
    2537
    0
      ccv_nnc_cmd_exec(cmd, hint, 0, TENSOR_LIST(a, w, bias), TENSOR_LIST(b), 0);
    2538
    0
      REQUIRE_ARRAY_EQ_WITH_TOLERANCE(float, b->data.f32, c->data.f32, batch_size * output_depth * output_height * output_width * output_channels, 1e-4, "output from mps should match from CPU");
    2539
    0
      ccv_nnc_tensor_free(c);
    2540
    0
      ccv_nnc_tensor_free(gc);
    2541
    0
      ccv_nnc_tensor_free(bias);
    2542
    0
      ccv_nnc_tensor_free(w);
    2543
    0
      ccv_nnc_tensor_free(b);
    2544
    0
      ccv_nnc_tensor_free(a);
    2545
    0
      ccv_nnc_tensor_free(gbias);
    2546
    0
      ccv_nnc_tensor_free(gwo);
    2547
    0
      ccv_nnc_tensor_free(ga);
    2548
    0
    }
    2549
    2550
    TEST_CASE("mps forward convolution 3d in nchw format")
    2551
    1
    {
    2552
    1
      GUARD_ELSE_RETURN(ccv_nnc_cmd_ok(CCV_NNC_CONVOLUTION_FORWARD, CCV_NNC_BACKEND_MPS));
    2553
    0
      ccv_nnc_tensor_t* a = ccv_nnc_tensor_new(0, CPU_TENSOR_NCHW(32F, BATCH_SIZE, INPUT_DIM, 5, INPUT_SIZE, INPUT_SIZE), 0);
    2554
    0
      ccv_nnc_tensor_t* b = ccv_nnc_tensor_new(0, CPU_TENSOR_NCHW(32F, BATCH_SIZE, OUTPUT_DIM, 3, OUTPUT_SIZE, OUTPUT_SIZE), 0);
    2555
    0
      ccv_nnc_cmd_t cmd = CMD_CONVOLUTION_FORWARD(1, OUTPUT_DIM, 3, KERNEL_SIZE, KERNEL_SIZE, INPUT_DIM);
    2556
    0
      ccv_nnc_hint_t hint = ccv_nnc_hint_auto(cmd.info, a->info, b->info);
    2557
    0
      hint.stride.dim[0] = 2;
    2558
    0
      hint.border.begin[0] = 1;
    2559
    0
      hint.border.end[0] = 1;
    2560
    0
      assert(ccv_nnc_hint_verify(hint, cmd.info, a->info, b->info) == 0);
    2561
    0
      ccv_nnc_tensor_t* w = ccv_nnc_tensor_new(0, CPU_TENSOR_NCHW(32F, OUTPUT_DIM, INPUT_DIM, 3, KERNEL_SIZE, KERNEL_SIZE), 0);
    2562
    0
      ccv_nnc_tensor_t* bias = ccv_nnc_tensor_new(0, CPU_TENSOR_NCHW(32F, OUTPUT_DIM), 0);
    2563
      // configure the inlets.
    2564
    0
      dsfmt_t dsfmt;
    2565
    0
      dsfmt_init_gen_rand(&dsfmt, 0);
    2566
    0
      int i;
    2567
    0
      for (i = 0; i < 3 * INPUT_DIM * KERNEL_SIZE * KERNEL_SIZE * OUTPUT_DIM; i++)
    2568
    0
        w->data.f32[i] = dsfmt_genrand_open_close(&dsfmt) / (INPUT_DIM * KERNEL_SIZE * KERNEL_SIZE);
    2569
    0
      for (i = 0; i < 5 * INPUT_SIZE * INPUT_SIZE * INPUT_DIM * ccv_max(1, BATCH_SIZE); i++)
    2570
    0
        a->data.f32[i] = dsfmt_genrand_open_close(&dsfmt);
    2571
    0
      for (i = 0; i < OUTPUT_DIM; i++)
    2572
    0
        bias->data.f32[i] = (float)i / OUTPUT_DIM;
    2573
      // Copy generated matrix values over to GPU.
    2574
    0
      ccv_nnc_tensor_t* ga = ccv_nnc_tensor_new(0, GPU_TENSOR_NCHW(000, 32F, BATCH_SIZE, INPUT_DIM, 5, INPUT_SIZE, INPUT_SIZE), 0);
    2575
    0
      ccv_nnc_tensor_t* gw = ccv_nnc_tensor_new(0, GPU_TENSOR_NCHW(000, 32F, OUTPUT_DIM, INPUT_DIM, 3, KERNEL_SIZE, KERNEL_SIZE), 0);
    2576
    0
      ccv_nnc_tensor_t* gbias = ccv_nnc_tensor_new(0, GPU_TENSOR_NCHW(000, 32F, OUTPUT_DIM), 0);
    2577
    0
      ccv_nnc_cmd_t move = CMD_DATA_TRANSFER_FORWARD();
    2578
    0
      move.backend = CCV_NNC_BACKEND_MPS;
    2579
    0
      assert(move.backend >= 0);
    2580
    0
      ccv_nnc_cmd_exec(move, ccv_nnc_no_hint, 0, TENSOR_LIST(a, w, bias), TENSOR_LIST(ga, gw, gbias), 0);
    2581
    0
      ccv_nnc_tensor_t* gc = ccv_nnc_tensor_new(0, GPU_TENSOR_NCHW(000, 32F, BATCH_SIZE, OUTPUT_DIM, 3, OUTPUT_SIZE, OUTPUT_SIZE), 0);
    2582
    2583
    0
      ccv_nnc_cmd_t transform = CMD_FORMAT_TRANSFORM_FORWARD();
    2584
    0
      transform.backend = CCV_NNC_BACKEND_MPS;
    2585
    0
      assert(transform.backend >= 0);
    2586
    0
      cmd.backend = CCV_NNC_BACKEND_MPS;
    2587
    0
      assert(cmd.backend >= 0);
    2588
    0
      cmd.algorithm = -1;
    2589
    0
      cmd = ccv_nnc_cmd_autotune(cmd, 1 * 1024 * 1024 * 1024, hint, 0, TENSOR_LIST(ga, gw, gbias), TENSOR_LIST(gc), 0);
    2590
    0
      assert(CCV_NNC_EXEC_SUCCESS == ccv_nnc_cmd_exec(cmd, hint, 0, TENSOR_LIST(ga, gw, gbias), TENSOR_LIST(gc), 0));
    2591
    0
      ccv_nnc_tensor_t* c = ccv_nnc_tensor_new(0, CPU_TENSOR_NCHW(32F, BATCH_SIZE, OUTPUT_DIM, 3, OUTPUT_SIZE, OUTPUT_SIZE), 0);
    2592
    0
      ccv_nnc_cmd_exec(move, ccv_nnc_no_hint, 0, TENSOR_LIST(gc), TENSOR_LIST(c), 0);
    2593
    0
      cmd.backend = CCV_NNC_BACKEND_CPU_REF;
    2594
    0
      assert(cmd.backend >= 0);
    2595
    0
      ccv_nnc_cmd_exec(cmd, hint, 0, TENSOR_LIST(a, w, bias), TENSOR_LIST(b), 0);
    2596
    0
      REQUIRE_ARRAY_EQ_WITH_TOLERANCE(float, b->data.f32, c->data.f32, BATCH_SIZE * 3 * OUTPUT_DIM * OUTPUT_SIZE * OUTPUT_SIZE, 1e-4, "output from mps should match from CPU");
    2597
    0
      ccv_nnc_tensor_free(c);
    2598
    0
      ccv_nnc_tensor_free(gc);
    2599
    0
      ccv_nnc_tensor_free(bias);
    2600
    0
      ccv_nnc_tensor_free(w);
    2601
    0
      ccv_nnc_tensor_free(b);
    2602
    0
      ccv_nnc_tensor_free(a);
    2603
    0
      ccv_nnc_tensor_free(gbias);
    2604
    0
      ccv_nnc_tensor_free(gw);
    2605
    0
      ccv_nnc_tensor_free(ga);
    2606
    0
    }
    2607
    2608
    TEST_CASE("compare softmax with mps")
    2609
    1
    {
    2610
    1
      GUARD_ELSE_RETURN(ccv_nnc_cmd_ok(CCV_NNC_SOFTMAX_FORWARD, CCV_NNC_BACKEND_MPS));
    2611
    0
      ccv_nnc_symbolic_graph_t* const symbolic_graph = ccv_nnc_symbolic_graph_new();
    2612
    0
      ccv_nnc_tensor_symbol_t a = ccv_nnc_tensor_symbol_new(symbolic_graph, GPU_TENSOR_NCHW(000, 32F, 20, 10), "a");
    2613
    0
      ccv_nnc_tensor_symbol_t b = ccv_nnc_tensor_symbol_new(symbolic_graph, GPU_TENSOR_NCHW(000, 32F, 20, 10), "b");
    2614
    0
      ccv_nnc_graph_exec_symbol_new(symbolic_graph, CMD_SOFTMAX_FORWARD(), TENSOR_SYMBOL_LIST(a), TENSOR_SYMBOL_LIST(b), "softmax");
    2615
    0
      ccv_nnc_graph_exec_symbol_autogen(symbolic_graph, 0, 0, CCV_NNC_AUTOGEN_ALL_EXECS | CCV_NNC_AUTOGEN_SOURCES_AND_DESTINATIONS);
    2616
    0
      SYMBOLIC_GRAPH_GEN(symbolic_graph, CCV_NNC_LONG_DOT_GRAPH);
    2617
    0
      ccv_nnc_graph_t* graph = 0;
    2618
    0
      ccv_nnc_tensor_arena_t* tensor_arena = 0;
    2619
    0
      ccv_nnc_graph_exec_arena_t* graph_exec_arena = 0;
    2620
    0
      ccv_nnc_symbolic_graph_compile(symbolic_graph, ccv_nnc_default_compile_params, 0, 0, 0, 0, SYMBOLIC_GRAPH_SOURCES(symbolic_graph), SYMBOLIC_GRAPH_DESTINATIONS(symbolic_graph), &graph, &tensor_arena, &graph_exec_arena);
    2621
    0
      GRAPH_GEN(graph, CCV_NNC_LONG_DOT_GRAPH);
    2622
    0
      ccv_nnc_tensor_t* const x_tensor = ccv_nnc_tensor_new(0, CPU_TENSOR_NCHW(32F, 20, 10), 0);
    2623
    0
      dsfmt_t dsfmt;
    2624
    0
      dsfmt_init_gen_rand(&dsfmt, 0);
    2625
    0
      int i;
    2626
    0
      for (i = 0; i < 20 * 10; i++)
    2627
    0
        x_tensor->data.f32[i] = dsfmt_genrand_open_close(&dsfmt);
    2628
    0
      ccv_nnc_tensor_t* const a_tensor = ccv_nnc_tensor_from_symbol(tensor_arena, a);
    2629
    0
      ccv_nnc_cmd_exec(CMD_DATA_TRANSFER_FORWARD(), ccv_nnc_no_hint, 0, TENSOR_LIST(x_tensor), TENSOR_LIST(a_tensor), 0);
    2630
    0
      ccv_nnc_graph_run(graph, 0, TRAVERSE_FULL, 0, 0);
    2631
    0
      ccv_nnc_tensor_t* const y_tensor = ccv_nnc_tensor_new(0, CPU_TENSOR_NCHW(32F, 20, 10), 0);
    2632
    0
      ccv_nnc_tensor_t* const b_tensor = ccv_nnc_tensor_from_symbol(tensor_arena, b);
    2633
    0
      ccv_nnc_cmd_exec(CMD_DATA_TRANSFER_FORWARD(), ccv_nnc_no_hint, 0, TENSOR_LIST(b_tensor), TENSOR_LIST(y_tensor), 0);
    2634
    0
      ccv_nnc_tensor_t* const ty = ccv_nnc_tensor_new(0, CPU_TENSOR_NCHW(32F, 20, 10), 0);
    2635
    0
      ccv_nnc_cmd_exec(CMD_SOFTMAX_FORWARD(), ccv_nnc_no_hint, 0, TENSOR_LIST(x_tensor), TENSOR_LIST(ty), 0);
    2636
    0
      REQUIRE_TENSOR_EQ(ty, y_tensor, "softmax from mps should match from CPU");
    2637
    0
      ccv_nnc_tensor_free(x_tensor);
    2638
    0
      ccv_nnc_tensor_free(y_tensor);
    2639
    0
      ccv_nnc_tensor_free(ty);
    2640
    0
      ccv_nnc_graph_free(graph);
    2641
    0
      ccv_nnc_tensor_arena_free(tensor_arena);
    2642
    0
      ccv_nnc_graph_exec_arena_free(graph_exec_arena);
    2643
    0
      ccv_nnc_symbolic_graph_free(symbolic_graph);
    2644
    0
    }
    2645
    2646
    TEST_CASE("compare softmax with mps in half precision")
    2647
    1
    {
    2648
    1
      GUARD_ELSE_RETURN(ccv_nnc_cmd_ok(CCV_NNC_SOFTMAX_FORWARD, CCV_NNC_BACKEND_MPS));
    2649
    0
      ccv_nnc_symbolic_graph_t* const symbolic_graph = ccv_nnc_symbolic_graph_new();
    2650
    0
      ccv_nnc_tensor_symbol_t a = ccv_nnc_tensor_symbol_new(symbolic_graph, GPU_TENSOR_NCHW(000, 16F, 20, 10), "a");
    2651
    0
      ccv_nnc_tensor_symbol_t b = ccv_nnc_tensor_symbol_new(symbolic_graph, GPU_TENSOR_NCHW(000, 16F, 20, 10), "b");
    2652
    0
      ccv_nnc_graph_exec_symbol_new(symbolic_graph, CMD_SOFTMAX_FORWARD(), TENSOR_SYMBOL_LIST(a), TENSOR_SYMBOL_LIST(b), "softmax");
    2653
    0
      ccv_nnc_graph_exec_symbol_autogen(symbolic_graph, 0, 0, CCV_NNC_AUTOGEN_ALL_EXECS | CCV_NNC_AUTOGEN_SOURCES_AND_DESTINATIONS);
    2654
    0
      SYMBOLIC_GRAPH_GEN(symbolic_graph, CCV_NNC_LONG_DOT_GRAPH);
    2655
    0
      ccv_nnc_graph_t* graph = 0;
    2656
    0
      ccv_nnc_tensor_arena_t* tensor_arena = 0;
    2657
    0
      ccv_nnc_graph_exec_arena_t* graph_exec_arena = 0;
    2658
    0
      ccv_nnc_symbolic_graph_compile(symbolic_graph, ccv_nnc_default_compile_params, 0, 0, 0, 0, SYMBOLIC_GRAPH_SOURCES(symbolic_graph), SYMBOLIC_GRAPH_DESTINATIONS(symbolic_graph), &graph, &tensor_arena, &graph_exec_arena);
    2659
    0
      GRAPH_GEN(graph, CCV_NNC_LONG_DOT_GRAPH);
    2660
    0
      ccv_nnc_tensor_t* const x_tensor = ccv_nnc_tensor_new(0, CPU_TENSOR_NCHW(32F, 20, 10), 0);
    2661
    0
      dsfmt_t dsfmt;
    2662
    0
      dsfmt_init_gen_rand(&dsfmt, 0);
    2663
    0
      int i;
    2664
    0
      for (i = 0; i < 20 * 10; i++)
    2665
    0
        x_tensor->data.f32[i] = dsfmt_genrand_open_close(&dsfmt);
    2666
    0
      ccv_nnc_tensor_t* const a_tensor = ccv_nnc_tensor_from_symbol(tensor_arena, a);
    2667
    0
      ccv_nnc_tensor_t* const x16_tensor = ccv_nnc_tensor_new(0, CPU_TENSOR_NCHW(16F, 20, 10), 0);
    2668
    0
      ccv_nnc_cmd_exec(CMD_DATATYPE_CONVERSION_FORWARD(), ccv_nnc_no_hint, 0, TENSOR_LIST(x_tensor), TENSOR_LIST(x16_tensor), 0);
    2669
    0
      ccv_nnc_cmd_exec(CMD_DATA_TRANSFER_FORWARD(), ccv_nnc_no_hint, 0, TENSOR_LIST(x16_tensor), TENSOR_LIST(a_tensor), 0);
    2670
    0
      ccv_nnc_graph_run(graph, 0, TRAVERSE_FULL, 0, 0);
    2671
    0
      ccv_nnc_tensor_t* const y16_tensor = ccv_nnc_tensor_new(0, CPU_TENSOR_NCHW(16F, 20, 10), 0);
    2672
    0
      ccv_nnc_tensor_t* const y_tensor = ccv_nnc_tensor_new(0, CPU_TENSOR_NCHW(32F, 20, 10), 0);
    2673
    0
      ccv_nnc_tensor_t* const b_tensor = ccv_nnc_tensor_from_symbol(tensor_arena, b);
    2674
    0
      ccv_nnc_cmd_exec(CMD_DATA_TRANSFER_FORWARD(), ccv_nnc_no_hint, 0, TENSOR_LIST(b_tensor), TENSOR_LIST(y16_tensor), 0);
    2675
    0
      ccv_nnc_cmd_exec(CMD_DATATYPE_CONVERSION_FORWARD(), ccv_nnc_no_hint, 0, TENSOR_LIST(y16_tensor), TENSOR_LIST(y_tensor), 0);
    2676
    0
      ccv_nnc_tensor_t* const ty = ccv_nnc_tensor_new(0, CPU_TENSOR_NCHW(32F, 20, 10), 0);
    2677
    0
      ccv_nnc_cmd_exec(CMD_SOFTMAX_FORWARD(), ccv_nnc_no_hint, 0, TENSOR_LIST(x_tensor), TENSOR_LIST(ty), 0);
    2678
    0
      REQUIRE_ARRAY_EQ_WITH_TOLERANCE(float, ty->data.f32, y_tensor->data.f32, 20 * 10, 1e-3, "softmax from mps should match from CPU");
    2679
    0
      ccv_nnc_tensor_free(x_tensor);
    2680
    0
      ccv_nnc_tensor_free(x16_tensor);
    2681
    0
      ccv_nnc_tensor_free(y16_tensor);
    2682
    0
      ccv_nnc_tensor_free(y_tensor);
    2683
    0
      ccv_nnc_tensor_free(ty);
    2684
    0
      ccv_nnc_graph_free(graph);
    2685
    0
      ccv_nnc_tensor_arena_free(tensor_arena);
    2686
    0
      ccv_nnc_graph_exec_arena_free(graph_exec_arena);
    2687
    0
      ccv_nnc_symbolic_graph_free(symbolic_graph);
    2688
    0
    }
    2689
    2690
    TEST_CASE("compare softmax gradient with mps")
    2691
    1
    {
    2692
    1
      GUARD_ELSE_RETURN(ccv_nnc_cmd_ok(CCV_NNC_SOFTMAX_FORWARD, CCV_NNC_BACKEND_MPS) &&
    2693
    1
        ccv_nnc_cmd_ok(CCV_NNC_SOFTMAX_BACKWARD, CCV_NNC_BACKEND_MPS));
    2694
    0
      ccv_nnc_symbolic_graph_t* const symbolic_graph = ccv_nnc_symbolic_graph_new();
    2695
    0
      ccv_nnc_tensor_symbol_t x = ccv_nnc_tensor_symbol_new(symbolic_graph, GPU_TENSOR_NHWC(000, 32F, 10, 100), "x");
    2696
    0
      ccv_nnc_tensor_symbol_t y = ccv_nnc_tensor_symbol_new(symbolic_graph, GPU_TENSOR_NHWC(000, 32F, 10, 100), "y");
    2697
    0
      ccv_nnc_graph_exec_symbol_new(symbolic_graph, CMD_SOFTMAX_FORWARD(), TENSOR_SYMBOL_LIST(x), TENSOR_SYMBOL_LIST(y), "softmax");
    2698
    0
      ccv_nnc_graph_exec_symbol_autogen(symbolic_graph, 0, 0, CCV_NNC_AUTOGEN_ALL_EXECS | CCV_NNC_AUTOGEN_SOURCES_AND_DESTINATIONS);
    2699
    0
      ccv_nnc_symbolic_graph_backward(symbolic_graph, TENSOR_SYMBOL_LIST(y), TENSOR_SYMBOL_LIST(x), SYMBOLIC_GRAPH_SOURCES(symbolic_graph), SYMBOLIC_GRAPH_DESTINATIONS(symbolic_graph));
    2700
    0
      ccv_nnc_graph_exec_symbol_autogen(symbolic_graph, 0, 0, CCV_NNC_AUTOGEN_ALL_EXECS | CCV_NNC_AUTOGEN_SOURCES_AND_DESTINATIONS);
    2701
    0
      SYMBOLIC_GRAPH_GEN(symbolic_graph, CCV_NNC_LONG_DOT_GRAPH);
    2702
    0
      ccv_nnc_tensor_symbol_t dy = ccv_nnc_tensor_symbol_for_backward(symbolic_graph, y);
    2703
    0
      ccv_nnc_tensor_symbol_t dx = ccv_nnc_tensor_symbol_for_backward(symbolic_graph, x);
    2704
    0
      ccv_nnc_tensor_t* const x_tensor = ccv_nnc_tensor_new(0, CPU_TENSOR_NHWC(32F, 10, 100), 0);
    2705
    0
      dsfmt_t dsfmt;
    2706
    0
      dsfmt_init_gen_rand(&dsfmt, 0);
    2707
    0
      int i;
    2708
    0
      for (i = 0; i < 10 * 100; i++)
    2709
    0
        x_tensor->data.f32[i] = dsfmt_genrand_open_close(&dsfmt);
    2710
    0
      ccv_nnc_tensor_t* const dy_tensor = ccv_nnc_tensor_new(0, CPU_TENSOR_NHWC(32F, 10, 100), 0);
    2711
    0
      for (i = 0; i < 10 * 100; i++)
    2712
    0
        dy_tensor->data.f32[i] = 0;
    2713
    0
      for (i = 0; i < 10; i++)
    2714
    0
        dy_tensor->data.f32[i * 100 + i] = 1;
    2715
    0
      ccv_nnc_tensor_t* const dyt = ccv_nnc_tensor_new(0, GPU_TENSOR_NHWC(000, 32F, 10, 100), 0);
    2716
    0
      ccv_nnc_cmd_exec(CMD_DATA_TRANSFER_FORWARD(), ccv_nnc_no_hint, 0, TENSOR_LIST(dy_tensor), TENSOR_LIST(dyt), 0);
    2717
    0
      ccv_nnc_graph_t* graph = 0;
    2718
    0
      ccv_nnc_tensor_arena_t* tensor_arena = 0;
    2719
    0
      ccv_nnc_graph_exec_arena_t* graph_exec_arena = 0;
    2720
    0
      ccv_nnc_symbolic_graph_compile(symbolic_graph, ccv_nnc_default_compile_params, TENSOR_BIND_MAP(KV(dy, dyt)), 0, 0, SYMBOLIC_GRAPH_SOURCES(symbolic_graph), SYMBOLIC_GRAPH_DESTINATIONS(symbolic_graph), &graph, &tensor_arena, &graph_exec_arena);
    2721
    0
      GRAPH_GEN(graph, CCV_NNC_LONG_DOT_GRAPH);
    2722
    0
      ccv_nnc_tensor_t* const xt = ccv_nnc_tensor_from_symbol(tensor_arena, x);
    2723
    0
      ccv_nnc_cmd_exec(CMD_DATA_TRANSFER_FORWARD(), ccv_nnc_no_hint, 0, TENSOR_LIST(x_tensor), TENSOR_LIST(xt), 0);
    2724
    0
      ccv_nnc_graph_run(graph, 0, TRAVERSE_FULL, 0, 0);
    2725
    0
      ccv_nnc_tensor_t* const dx_tensor = ccv_nnc_tensor_new(0, CPU_TENSOR_NHWC(32F, 10, 100), 0);
    2726
    0
      ccv_nnc_tensor_t* const dxt = ccv_nnc_tensor_from_symbol(tensor_arena, dx);
    2727
    0
      ccv_nnc_tensor_t* const y_tensor = ccv_nnc_tensor_new(0, CPU_TENSOR_NHWC(32F, 10, 100), 0);
    2728
    0
      ccv_nnc_tensor_t* const yt = ccv_nnc_tensor_from_symbol(tensor_arena, y);
    2729
    0
      ccv_nnc_cmd_exec(CMD_DATA_TRANSFER_FORWARD(), ccv_nnc_no_hint, 0, TENSOR_LIST(dxt), TENSOR_LIST(dx_tensor), 0);
    2730
    0
      ccv_nnc_cmd_exec(CMD_DATA_TRANSFER_FORWARD(), ccv_nnc_no_hint, 0, TENSOR_LIST(yt), TENSOR_LIST(y_tensor), 0);
    2731
    0
      ccv_nnc_tensor_t* const ty_tensor = ccv_nnc_tensor_new(0, CPU_TENSOR_NHWC(32F, 10, 100), 0);
    2732
    0
      ccv_nnc_cmd_exec(CMD_SOFTMAX_FORWARD(), ccv_nnc_no_hint, 0, TENSOR_LIST(x_tensor), TENSOR_LIST(ty_tensor), 0);
    2733
    0
      REQUIRE_TENSOR_EQ(ty_tensor, y_tensor, "forward pass should match");
    2734
    0
      ccv_nnc_tensor_t* const tdx_tensor = ccv_nnc_tensor_new(0, CPU_TENSOR_NHWC(32F, 10, 100), 0);
    2735
    0
      ccv_nnc_cmd_exec(CMD_SOFTMAX_BACKWARD(), ccv_nnc_no_hint, 0, TENSOR_LIST(dy_tensor, 0, ty_tensor), TENSOR_LIST(tdx_tensor), 0);
    2736
    0
      REQUIRE_TENSOR_EQ(tdx_tensor, dx_tensor, "backward pass should match");
    2737
    0
      ccv_nnc_tensor_free(x_tensor);
    2738
    0
      ccv_nnc_tensor_free(y_tensor);
    2739
    0
      ccv_nnc_tensor_free(dx_tensor);
    2740
    0
      ccv_nnc_tensor_free(dy_tensor);
    2741
    0
      ccv_nnc_tensor_free(ty_tensor);
    2742
    0
      ccv_nnc_tensor_free(tdx_tensor);
    2743
    0
      ccv_nnc_tensor_free(dyt);
    2744
    0
      ccv_nnc_graph_free(graph);
    2745
    0
      ccv_nnc_tensor_arena_free(tensor_arena);
    2746
    0
      ccv_nnc_graph_exec_arena_free(graph_exec_arena);
    2747
    0
      ccv_nnc_symbolic_graph_free(symbolic_graph);
    2748
    0
    }
    2749
    2750
    TEST_CASE("compare sigmoid with mps")
    2751
    1
    {
    2752
    1
      GUARD_ELSE_RETURN(ccv_nnc_cmd_ok(CCV_NNC_SIGMOID_FORWARD, CCV_NNC_BACKEND_MPS));
    2753
    0
      ccv_nnc_symbolic_graph_t* const symbolic_graph = ccv_nnc_symbolic_graph_new();
    2754
    0
      ccv_nnc_tensor_symbol_t a = ccv_nnc_tensor_symbol_new(symbolic_graph, GPU_TENSOR_NCHW(000, 32F, 20, 10), "a");
    2755
    0
      ccv_nnc_tensor_symbol_t b = ccv_nnc_tensor_symbol_new(symbolic_graph, GPU_TENSOR_NCHW(000, 32F, 20, 10), "b");
    2756
    0
      ccv_nnc_graph_exec_symbol_new(symbolic_graph, CMD_SIGMOID_FORWARD(), TENSOR_SYMBOL_LIST(a), TENSOR_SYMBOL_LIST(b), "sigmoid");
    2757
    0
      ccv_nnc_graph_exec_symbol_autogen(symbolic_graph, 0, 0, CCV_NNC_AUTOGEN_ALL_EXECS | CCV_NNC_AUTOGEN_SOURCES_AND_DESTINATIONS);
    2758
    0
      SYMBOLIC_GRAPH_GEN(symbolic_graph, CCV_NNC_LONG_DOT_GRAPH);
    2759
    0
      ccv_nnc_graph_t* graph = 0;
    2760
    0
      ccv_nnc_tensor_arena_t* tensor_arena = 0;
    2761
    0
      ccv_nnc_graph_exec_arena_t* graph_exec_arena = 0;
    2762
    0
      ccv_nnc_symbolic_graph_compile(symbolic_graph, ccv_nnc_default_compile_params, 0, 0, 0, 0, SYMBOLIC_GRAPH_SOURCES(symbolic_graph), SYMBOLIC_GRAPH_DESTINATIONS(symbolic_graph), &graph, &tensor_arena, &graph_exec_arena);
    2763
    0
      GRAPH_GEN(graph, CCV_NNC_LONG_DOT_GRAPH);
    2764
    0
      ccv_nnc_tensor_t* const x_tensor = ccv_nnc_tensor_new(0, CPU_TENSOR_NCHW(32F, 20, 10), 0);
    2765
    0
      dsfmt_t dsfmt;
    2766
    0
      dsfmt_init_gen_rand(&dsfmt, 0);
    2767
    0
      int i;
    2768
    0
      for (i = 0; i < 20 * 10; i++)
    2769
    0
        x_tensor->data.f32[i] = dsfmt_genrand_open_close(&dsfmt);
    2770
    0
      ccv_nnc_tensor_t* const a_tensor = ccv_nnc_tensor_from_symbol(tensor_arena, a);
    2771
    0
      ccv_nnc_cmd_exec(CMD_DATA_TRANSFER_FORWARD(), ccv_nnc_no_hint, 0, TENSOR_LIST(x_tensor), TENSOR_LIST(a_tensor), 0);
    2772
    0
      ccv_nnc_graph_run(graph, 0, TRAVERSE_FULL, 0, 0);
    2773
    0
      ccv_nnc_tensor_t* const y_tensor = ccv_nnc_tensor_new(0, CPU_TENSOR_NCHW(32F, 20, 10), 0);
    2774
    0
      ccv_nnc_tensor_t* const b_tensor = ccv_nnc_tensor_from_symbol(tensor_arena, b);
    2775
    0
      ccv_nnc_cmd_exec(CMD_DATA_TRANSFER_FORWARD(), ccv_nnc_no_hint, 0, TENSOR_LIST(b_tensor), TENSOR_LIST(y_tensor), 0);
    2776
    0
      ccv_nnc_tensor_t* const ty = ccv_nnc_tensor_new(0, CPU_TENSOR_NCHW(32F, 20, 10), 0);
    2777
    0
      ccv_nnc_cmd_exec(CMD_SIGMOID_FORWARD(), ccv_nnc_no_hint, 0, TENSOR_LIST(x_tensor), TENSOR_LIST(ty), 0);
    2778
    0
      REQUIRE_TENSOR_EQ(ty, y_tensor, "sigmoid from mps should match from CPU");
    2779
    0
      ccv_nnc_tensor_free(x_tensor);
    2780
    0
      ccv_nnc_tensor_free(y_tensor);
    2781
    0
      ccv_nnc_tensor_free(ty);
    2782
    0
      ccv_nnc_graph_free(graph);
    2783
    0
      ccv_nnc_tensor_arena_free(tensor_arena);
    2784
    0
      ccv_nnc_graph_exec_arena_free(graph_exec_arena);
    2785
    0
      ccv_nnc_symbolic_graph_free(symbolic_graph);
    2786
    0
    }
    2787
    2788
    TEST_CASE("compare sigmoid with mps in half precision")
    2789
    1
    {
    2790
    1
      GUARD_ELSE_RETURN(ccv_nnc_cmd_ok(CCV_NNC_SIGMOID_FORWARD, CCV_NNC_BACKEND_MPS));
    2791
    0
      ccv_nnc_symbolic_graph_t* const symbolic_graph = ccv_nnc_symbolic_graph_new();
    2792
    0
      ccv_nnc_tensor_symbol_t a = ccv_nnc_tensor_symbol_new(symbolic_graph, GPU_TENSOR_NCHW(000, 16F, 20, 10), "a");
    2793
    0
      ccv_nnc_tensor_symbol_t b = ccv_nnc_tensor_symbol_new(symbolic_graph, GPU_TENSOR_NCHW(000, 16F, 20, 10), "b");
    2794
    0
      ccv_nnc_graph_exec_symbol_new(symbolic_graph, CMD_SIGMOID_FORWARD(), TENSOR_SYMBOL_LIST(a), TENSOR_SYMBOL_LIST(b), "sigmoid");
    2795
    0
      ccv_nnc_graph_exec_symbol_autogen(symbolic_graph, 0, 0, CCV_NNC_AUTOGEN_ALL_EXECS | CCV_NNC_AUTOGEN_SOURCES_AND_DESTINATIONS);
    2796
    0
      SYMBOLIC_GRAPH_GEN(symbolic_graph, CCV_NNC_LONG_DOT_GRAPH);
    2797
    0
      ccv_nnc_graph_t* graph = 0;
    2798
    0
      ccv_nnc_tensor_arena_t* tensor_arena = 0;
    2799
    0
      ccv_nnc_graph_exec_arena_t* graph_exec_arena = 0;
    2800
    0
      ccv_nnc_symbolic_graph_compile(symbolic_graph, ccv_nnc_default_compile_params, 0, 0, 0, 0, SYMBOLIC_GRAPH_SOURCES(symbolic_graph), SYMBOLIC_GRAPH_DESTINATIONS(symbolic_graph), &graph, &tensor_arena, &graph_exec_arena);
    2801
    0
      GRAPH_GEN(graph, CCV_NNC_LONG_DOT_GRAPH);
    2802
    0
      ccv_nnc_tensor_t* const x_tensor = ccv_nnc_tensor_new(0, CPU_TENSOR_NCHW(32F, 20, 10), 0);
    2803
    0
      dsfmt_t dsfmt;
    2804
    0
      dsfmt_init_gen_rand(&dsfmt, 0);
    2805
    0
      int i;
    2806
    0
      for (i = 0; i < 20 * 10; i++)
    2807
    0
        x_tensor->data.f32[i] = dsfmt_genrand_open_close(&dsfmt);
    2808
    0
      ccv_nnc_tensor_t* const a_tensor = ccv_nnc_tensor_from_symbol(tensor_arena, a);
    2809
    0
      ccv_nnc_tensor_t* const x16_tensor = ccv_nnc_tensor_new(0, CPU_TENSOR_NCHW(16F, 20, 10), 0);
    2810
    0
      ccv_nnc_cmd_exec(CMD_DATATYPE_CONVERSION_FORWARD(), ccv_nnc_no_hint, 0, TENSOR_LIST(x_tensor), TENSOR_LIST(x16_tensor), 0);
    2811
    0
      ccv_nnc_cmd_exec(CMD_DATA_TRANSFER_FORWARD(), ccv_nnc_no_hint, 0, TENSOR_LIST(x16_tensor), TENSOR_LIST(a_tensor), 0);
    2812
    0
      ccv_nnc_graph_run(graph, 0, TRAVERSE_FULL, 0, 0);
    2813
    0
      ccv_nnc_tensor_t* const y16_tensor = ccv_nnc_tensor_new(0, CPU_TENSOR_NCHW(16F, 20, 10), 0);
    2814
    0
      ccv_nnc_tensor_t* const y_tensor = ccv_nnc_tensor_new(0, CPU_TENSOR_NCHW(32F, 20, 10), 0);
    2815
    0
      ccv_nnc_tensor_t* const b_tensor = ccv_nnc_tensor_from_symbol(tensor_arena, b);
    2816
    0
      ccv_nnc_cmd_exec(CMD_DATA_TRANSFER_FORWARD(), ccv_nnc_no_hint, 0, TENSOR_LIST(b_tensor), TENSOR_LIST(y16_tensor), 0);
    2817
    0
      ccv_nnc_cmd_exec(CMD_DATATYPE_CONVERSION_FORWARD(), ccv_nnc_no_hint, 0, TENSOR_LIST(y16_tensor), TENSOR_LIST(y_tensor), 0);
    2818
    0
      ccv_nnc_tensor_t* const ty = ccv_nnc_tensor_new(0, CPU_TENSOR_NCHW(32F, 20, 10), 0);
    2819
    0
      ccv_nnc_cmd_exec(CMD_SIGMOID_FORWARD(), ccv_nnc_no_hint, 0, TENSOR_LIST(x_tensor), TENSOR_LIST(ty), 0);
    2820
    0
      REQUIRE_ARRAY_EQ_WITH_TOLERANCE(float, ty->data.f32, y_tensor->data.f32, 20 * 10, 1e-3, "sigmoid from mps should match from CPU");
    2821
    0
      ccv_nnc_tensor_free(x_tensor);
    2822
    0
      ccv_nnc_tensor_free(x16_tensor);
    2823
    0
      ccv_nnc_tensor_free(y16_tensor);
    2824
    0
      ccv_nnc_tensor_free(y_tensor);
    2825
    0
      ccv_nnc_tensor_free(ty);
    2826
    0
      ccv_nnc_graph_free(graph);
    2827
    0
      ccv_nnc_tensor_arena_free(tensor_arena);
    2828
    0
      ccv_nnc_graph_exec_arena_free(graph_exec_arena);
    2829
    0
      ccv_nnc_symbolic_graph_free(symbolic_graph);
    2830
    0
    }
    2831
    2832
    TEST_CASE("compare sigmoid with mps and more vectorization cases")
    2833
    1
    {
    2834
    1
      GUARD_ELSE_RETURN(ccv_nnc_cmd_ok(CCV_NNC_SIGMOID_FORWARD, CCV_NNC_BACKEND_MPS));
    2835
    0
      dsfmt_t dsfmt;
    2836
    0
      dsfmt_init_gen_rand(&dsfmt, 0);
    2837
    0
      int i;
    2838
    0
      ccv_nnc_tensor_t* const a0 = ccv_nnc_tensor_new(0, GPU_TENSOR_NCHW(000, 32F, 32, 32), 0);
    2839
    0
      ccv_nnc_tensor_t* const b0 = ccv_nnc_tensor_new(0, GPU_TENSOR_NCHW(000, 32F, 32, 32), 0);
    2840
    0
      ccv_nnc_tensor_t* const ha0 = ccv_nnc_tensor_new(0, CPU_TENSOR_NCHW(32F, 32, 32), 0);
    2841
    0
      ccv_nnc_tensor_t* const hb0 = ccv_nnc_tensor_new(0, CPU_TENSOR_NCHW(32F, 32, 32), 0);
    2842
    0
      ccv_nnc_tensor_t* const tb0 = ccv_nnc_tensor_new(0, CPU_TENSOR_NCHW(32F, 32, 32), 0);
    2843
    0
      for (i = 0; i < 32 * 32; i++)
    2844
    0
        switch (i % 6)
    2845
    0
        {
    2846
    0
          case 0:
    2847
    0
            ha0->data.f32[i] = -80;
    2848
    0
            break;
    2849
    0
          case 1:
    2850
    0
            ha0->data.f32[i] = 80;
    2851
    0
            break;
    2852
    0
          case 2:
    2853
    0
            ha0->data.f32[i] = -20;
    2854
    0
            break;
    2855
    0
          case 3:
    2856
    0
            ha0->data.f32[i] = 20;
    2857
    0
            break;
    2858
    0
          default:
    2859
    0
            ha0->data.f32[i] = dsfmt_genrand_open_close(&dsfmt) * 12 - 6;
    2860
    0
            break;
    2861
    0
        }
    2862
    0
      ccv_nnc_cmd_exec(CMD_DATA_TRANSFER_FORWARD(), ccv_nnc_no_hint, 0, TENSOR_LIST(ha0), TENSOR_LIST(a0), 0);
    2863
    0
      ccv_nnc_cmd_exec(CMD_SIGMOID_FORWARD(), ccv_nnc_no_hint, 0, TENSOR_LIST(a0), TENSOR_LIST(b0), 0);
    2864
    0
      ccv_nnc_cmd_exec(CMD_SIGMOID_FORWARD(), ccv_nnc_no_hint, 0, TENSOR_LIST(ha0), TENSOR_LIST(tb0), 0);
    2865
    0
      ccv_nnc_cmd_exec(CMD_DATA_TRANSFER_FORWARD(), ccv_nnc_no_hint, 0, TENSOR_LIST(b0), TENSOR_LIST(hb0), 0);
    2866
    0
      REQUIRE_ARRAY_EQ_WITH_TOLERANCE(float, tb0->data.f32, hb0->data.f32, 32 * 32, 1e-6, "sigmoid from mps should match from CPU for the length %% 1024 == 0 case");
    2867
    0
      ccv_nnc_tensor_free(a0);
    2868
    0
      ccv_nnc_tensor_free(b0);
    2869
    0
      ccv_nnc_tensor_free(ha0);
    2870
    0
      ccv_nnc_tensor_free(hb0);
    2871
    0
      ccv_nnc_tensor_free(tb0);
    2872
    2873
    0
      ccv_nnc_tensor_t* const a1 = ccv_nnc_tensor_new(0, GPU_TENSOR_NHWC(000, 32F, 17, 61), 0);
    2874
    0
      ccv_nnc_tensor_t* const b1 = ccv_nnc_tensor_new(0, GPU_TENSOR_NHWC(000, 32F, 17, 61), 0);
    2875
    0
      ccv_nnc_tensor_t* const ha1 = ccv_nnc_tensor_new(0, CPU_TENSOR_NHWC(32F, 17, 61), 0);
    2876
    0
      ccv_nnc_tensor_t* const hb1 = ccv_nnc_tensor_new(0, CPU_TENSOR_NHWC(32F, 17, 61), 0);
    2877
    0
      ccv_nnc_tensor_t* const tb1 = ccv_nnc_tensor_new(0, CPU_TENSOR_NHWC(32F, 17, 61), 0);
    2878
    0
      for (i = 0; i < 17 * 61; i++)
    2879
    0
        switch (i % 6)
    2880
    0
        {
    2881
    0
          case 0:
    2882
    0
            ha1->data.f32[i] = -80;
    2883
    0
            break;
    2884
    0
          case 1:
    2885
    0
            ha1->data.f32[i] = 80;
    2886
    0
            break;
    2887
    0
          case 2:
    2888
    0
            ha1->data.f32[i] = -20;
    2889
    0
            break;
    2890
    0
          case 3:
    2891
    0
            ha1->data.f32[i] = 20;
    2892
    0
            break;
    2893
    0
          default:
    2894
    0
            ha1->data.f32[i] = dsfmt_genrand_open_close(&dsfmt) * 12 - 6;
    2895
    0
            break;
    2896
    0
        }
    2897
    0
      ccv_nnc_cmd_exec(CMD_DATA_TRANSFER_FORWARD(), ccv_nnc_no_hint, 0, TENSOR_LIST(ha1), TENSOR_LIST(a1), 0);
    2898
    0
      ccv_nnc_cmd_exec(CMD_SIGMOID_FORWARD(), ccv_nnc_no_hint, 0, TENSOR_LIST(a1), TENSOR_LIST(b1), 0);
    2899
    0
      ccv_nnc_cmd_exec(CMD_SIGMOID_FORWARD(), ccv_nnc_no_hint, 0, TENSOR_LIST(ha1), TENSOR_LIST(tb1), 0);
    2900
    0
      ccv_nnc_cmd_exec(CMD_DATA_TRANSFER_FORWARD(), ccv_nnc_no_hint, 0, TENSOR_LIST(b1), TENSOR_LIST(hb1), 0);
    2901
    0
      REQUIRE_ARRAY_EQ_WITH_TOLERANCE(float, tb1->data.f32, hb1->data.f32, 17 * 61, 1e-6, "sigmoid from mps should match from CPU for the tail case");
    2902
    0
      ccv_nnc_tensor_free(a1);
    2903
    0
      ccv_nnc_tensor_free(b1);
    2904
    0
      ccv_nnc_tensor_free(ha1);
    2905
    0
      ccv_nnc_tensor_free(hb1);
    2906
    0
      ccv_nnc_tensor_free(tb1);
    2907
    0
    }
    2908
    2909
    2910
    TEST_CASE("compare sigmoid gradient with mps")
    2911
    1
    {
    2912
    1
      GUARD_ELSE_RETURN(ccv_nnc_cmd_ok(CCV_NNC_SIGMOID_FORWARD, CCV_NNC_BACKEND_MPS) &&
    2913
    1
        ccv_nnc_cmd_ok(CCV_NNC_SIGMOID_BACKWARD, CCV_NNC_BACKEND_MPS));
    2914
    0
      ccv_nnc_symbolic_graph_t* const symbolic_graph = ccv_nnc_symbolic_graph_new();
    2915
    0
      ccv_nnc_tensor_symbol_t x = ccv_nnc_tensor_symbol_new(symbolic_graph, GPU_TENSOR_NHWC(000, 32F, 10, 100), "x");
    2916
    0
      ccv_nnc_tensor_symbol_t y = ccv_nnc_tensor_symbol_new(symbolic_graph, GPU_TENSOR_NHWC(000, 32F, 10, 100), "y");
    2917
    0
      ccv_nnc_graph_exec_symbol_new(symbolic_graph, CMD_SIGMOID_FORWARD(), TENSOR_SYMBOL_LIST(x), TENSOR_SYMBOL_LIST(y), "sigmoid");
    2918
    0
      ccv_nnc_graph_exec_symbol_autogen(symbolic_graph, 0, 0, CCV_NNC_AUTOGEN_ALL_EXECS | CCV_NNC_AUTOGEN_SOURCES_AND_DESTINATIONS);
    2919
    0
      ccv_nnc_symbolic_graph_backward(symbolic_graph, TENSOR_SYMBOL_LIST(y), TENSOR_SYMBOL_LIST(x), SYMBOLIC_GRAPH_SOURCES(symbolic_graph), SYMBOLIC_GRAPH_DESTINATIONS(symbolic_graph));
    2920
    0
      ccv_nnc_graph_exec_symbol_autogen(symbolic_graph, 0, 0, CCV_NNC_AUTOGEN_ALL_EXECS | CCV_NNC_AUTOGEN_SOURCES_AND_DESTINATIONS);
    2921
    0
      SYMBOLIC_GRAPH_GEN(symbolic_graph, CCV_NNC_LONG_DOT_GRAPH);
    2922
    0
      ccv_nnc_tensor_symbol_t dy = ccv_nnc_tensor_symbol_for_backward(symbolic_graph, y);
    2923
    0
      ccv_nnc_tensor_symbol_t dx = ccv_nnc_tensor_symbol_for_backward(symbolic_graph, x);
    2924
    0
      ccv_nnc_tensor_t* const x_tensor = ccv_nnc_tensor_new(0, CPU_TENSOR_NHWC(32F, 10, 100), 0);
    2925
    0
      dsfmt_t dsfmt;
    2926
    0
      dsfmt_init_gen_rand(&dsfmt, 0);
    2927
    0
      int i;
    2928
    0
      for (i = 0; i < 10 * 100; i++)
    2929
    0
        x_tensor->data.f32[i] = dsfmt_genrand_open_close(&dsfmt);
    2930
    0
      ccv_nnc_tensor_t* const dy_tensor = ccv_nnc_tensor_new(0, CPU_TENSOR_NHWC(32F, 10, 100), 0);
    2931
    0
      for (i = 0; i < 10 * 100; i++)
    2932
    0
        dy_tensor->data.f32[i] = 0;
    2933
    0
      for (i = 0; i < 10; i++)
    2934
    0
        dy_tensor->data.f32[i * 100 + i] = 1;
    2935
    0
      ccv_nnc_tensor_t* const dyt = ccv_nnc_tensor_new(0, GPU_TENSOR_NHWC(000, 32F, 10, 100), 0);
    2936
    0
      ccv_nnc_cmd_exec(CMD_DATA_TRANSFER_FORWARD(), ccv_nnc_no_hint, 0, TENSOR_LIST(dy_tensor), TENSOR_LIST(dyt), 0);
    2937
    0
      ccv_nnc_graph_t* graph = 0;
    2938
    0
      ccv_nnc_tensor_arena_t* tensor_arena = 0;
    2939
    0
      ccv_nnc_graph_exec_arena_t* graph_exec_arena = 0;
    2940
    0
      ccv_nnc_symbolic_graph_compile(symbolic_graph, ccv_nnc_default_compile_params, TENSOR_BIND_MAP(KV(dy, dyt)), TENSOR_SYMBOL_LIST(y), SYMBOLIC_GRAPH_SOURCES(symbolic_graph), SYMBOLIC_GRAPH_DESTINATIONS(symbolic_graph), &graph, &tensor_arena, &graph_exec_arena);
    2941
    0
      GRAPH_GEN(graph, CCV_NNC_LONG_DOT_GRAPH);
    2942
    0
      ccv_nnc_tensor_t* const xt = ccv_nnc_tensor_from_symbol(tensor_arena, x);
    2943
    0
      ccv_nnc_cmd_exec(CMD_DATA_TRANSFER_FORWARD(), ccv_nnc_no_hint, 0, TENSOR_LIST(x_tensor), TENSOR_LIST(xt), 0);
    2944
    0
      ccv_nnc_graph_run(graph, 0, TRAVERSE_FULL, 0, 0);
    2945
    0
      ccv_nnc_tensor_t* const dx_tensor = ccv_nnc_tensor_new(0, CPU_TENSOR_NHWC(32F, 10, 100), 0);
    2946
    0
      ccv_nnc_tensor_t* const dxt = ccv_nnc_tensor_from_symbol(tensor_arena, dx);
    2947
    0
      ccv_nnc_tensor_t* const y_tensor = ccv_nnc_tensor_new(0, CPU_TENSOR_NHWC(32F, 10, 100), 0);
    2948
    0
      ccv_nnc_tensor_t* const yt = ccv_nnc_tensor_from_symbol(tensor_arena, y);
    2949
    0
      ccv_nnc_cmd_exec(CMD_DATA_TRANSFER_FORWARD(), ccv_nnc_no_hint, 0, TENSOR_LIST(dxt), TENSOR_LIST(dx_tensor), 0);
    2950
    0
      ccv_nnc_cmd_exec(CMD_DATA_TRANSFER_FORWARD(), ccv_nnc_no_hint, 0, TENSOR_LIST(yt), TENSOR_LIST(y_tensor), 0);
    2951
    0
      ccv_nnc_tensor_t* const ty_tensor = ccv_nnc_tensor_new(0, CPU_TENSOR_NHWC(32F, 10, 100), 0);
    2952
    0
      ccv_nnc_cmd_exec(CMD_SIGMOID_FORWARD(), ccv_nnc_no_hint, 0, TENSOR_LIST(x_tensor), TENSOR_LIST(ty_tensor), 0);
    2953
    0
      REQUIRE_TENSOR_EQ(ty_tensor, y_tensor, "forward pass should match");
    2954
    0
      ccv_nnc_tensor_t* const tdx_tensor = ccv_nnc_tensor_new(0, CPU_TENSOR_NHWC(32F, 10, 100), 0);
    2955
    0
      ccv_nnc_cmd_exec(CMD_SIGMOID_BACKWARD(), ccv_nnc_no_hint, 0, TENSOR_LIST(dy_tensor, 0, ty_tensor), TENSOR_LIST(tdx_tensor), 0);
    2956
    0
      REQUIRE_TENSOR_EQ(tdx_tensor, dx_tensor, "backward pass should match");
    2957
    0
      ccv_nnc_tensor_free(x_tensor);
    2958
    0
      ccv_nnc_tensor_free(y_tensor);
    2959
    0
      ccv_nnc_tensor_free(dx_tensor);
    2960
    0
      ccv_nnc_tensor_free(dy_tensor);
    2961
    0
      ccv_nnc_tensor_free(ty_tensor);
    2962
    0
      ccv_nnc_tensor_free(tdx_tensor);
    2963
    0
      ccv_nnc_tensor_free(dyt);
    2964
    0
      ccv_nnc_graph_free(graph);
    2965
    0
      ccv_nnc_tensor_arena_free(tensor_arena);
    2966
    0
      ccv_nnc_graph_exec_arena_free(graph_exec_arena);
    2967
    0
      ccv_nnc_symbolic_graph_free(symbolic_graph);
    2968
    0
    }
    2969
    2970
    TEST_CASE("compare relu with mps")
    2971
    1
    {
    2972
    1
      GUARD_ELSE_RETURN(ccv_nnc_cmd_ok(CCV_NNC_RELU_FORWARD, CCV_NNC_BACKEND_MPS));
    2973
    0
      ccv_nnc_symbolic_graph_t* symbolic_graph = ccv_nnc_symbolic_graph_new();
    2974
    0
      ccv_nnc_tensor_symbol_t x = ccv_nnc_tensor_symbol_new(symbolic_graph, GPU_TENSOR_NHWC(000, 32F, 7, 7, 10), "x");
    2975
    0
      ccv_nnc_tensor_symbol_t y = ccv_nnc_tensor_symbol_new(symbolic_graph, GPU_TENSOR_NHWC(000, 32F, 7, 7, 10), "y");
    2976
    0
      ccv_nnc_graph_exec_symbol_new(symbolic_graph, CMD_RELU_FORWARD(), TENSOR_SYMBOL_LIST(x), TENSOR_SYMBOL_LIST(y), "relu");
    2977
    0
      ccv_nnc_graph_exec_symbol_autogen(symbolic_graph, 0, 0, CCV_NNC_AUTOGEN_ALL_EXECS | CCV_NNC_AUTOGEN_SOURCES_AND_DESTINATIONS);
    2978
    0
      ccv_nnc_graph_t* graph = 0;
    2979
    0
      ccv_nnc_tensor_arena_t* tensor_arena = 0;
    2980
    0
      ccv_nnc_graph_exec_arena_t* graph_exec_arena = 0;
    2981
    0
      ccv_nnc_symbolic_graph_compile(symbolic_graph, ccv_nnc_default_compile_params, 0, 0, 0, 0, SYMBOLIC_GRAPH_SOURCES(symbolic_graph), SYMBOLIC_GRAPH_DESTINATIONS(symbolic_graph), &graph, &tensor_arena, &graph_exec_arena);
    2982
    0
      SYMBOLIC_GRAPH_GEN(symbolic_graph, CCV_NNC_LONG_DOT_GRAPH);
    2983
    0
      GRAPH_GEN(graph, CCV_NNC_LONG_DOT_GRAPH);
    2984
    0
      dsfmt_t dsfmt;
    2985
    0
      dsfmt_init_gen_rand(&dsfmt, 0);
    2986
    0
      ccv_nnc_tensor_t* const x_tensor = ccv_nnc_tensor_new(0, CPU_TENSOR_NHWC(32F, 7, 7, 10), 0);
    2987
    0
      int i;
    2988
    0
      for (i = 0; i < 7 * 7 * 10; i++)
    2989
    0
        x_tensor->data.f32[i] = dsfmt_genrand_open_close(&dsfmt) * 2 - 1;
    2990
    0
      ccv_nnc_tensor_t* const xt = ccv_nnc_tensor_from_symbol(tensor_arena, x);
    2991
    0
      ccv_nnc_cmd_exec(CMD_DATA_TRANSFER_FORWARD(), ccv_nnc_no_hint, 0, TENSOR_LIST(x_tensor), TENSOR_LIST(xt), 0);
    2992
    0
      ccv_nnc_graph_run(graph, 0, TRAVERSE_FULL, 0, 0);
    2993
    0
      ccv_nnc_tensor_t* const y_tensor = ccv_nnc_tensor_new(0, CPU_TENSOR_NHWC(32F, 7, 7, 10), 0);
    2994
    0
      ccv_nnc_cmd_exec(CMD_RELU_FORWARD(), ccv_nnc_no_hint, 0, TENSOR_LIST(x_tensor), TENSOR_LIST(y_tensor), 0);
    2995
    0
      ccv_nnc_tensor_t* const yt = ccv_nnc_tensor_from_symbol(tensor_arena, y);
    2996
    0
      ccv_nnc_tensor_t* const cpu_y = ccv_nnc_tensor_new(0, CPU_TENSOR_NHWC(32F, 7, 7, 10), 0);
    2997
    0
      ccv_nnc_cmd_exec(CMD_DATA_TRANSFER_FORWARD(), ccv_nnc_no_hint, 0, TENSOR_LIST(yt), TENSOR_LIST(cpu_y), 0);
    2998
    0
      REQUIRE_TENSOR_EQ(y_tensor, cpu_y, "mps result should equal to cpu result");
    2999
    0
      ccv_nnc_symbolic_graph_free(symbolic_graph);
    3000
    0
      ccv_nnc_tensor_arena_free(tensor_arena);
    3001
    0
      ccv_nnc_graph_exec_arena_free(graph_exec_arena);
    3002
    0
      ccv_nnc_graph_free(graph);
    3003
    0
      ccv_nnc_tensor_free(x_tensor);
    3004
    0
      ccv_nnc_tensor_free(y_tensor);
    3005
    0
      ccv_nnc_tensor_free(cpu_y);
    3006
    0
    }
    3007
    3008
    TEST_CASE("compare relu with mps in half precision")
    3009
    1
    {
    3010
    1
      GUARD_ELSE_RETURN(ccv_nnc_cmd_ok(CCV_NNC_RELU_FORWARD, CCV_NNC_BACKEND_MPS));
    3011
    0
      ccv_nnc_symbolic_graph_t* symbolic_graph = ccv_nnc_symbolic_graph_new();
    3012
    0
      ccv_nnc_tensor_symbol_t x = ccv_nnc_tensor_symbol_new(symbolic_graph, GPU_TENSOR_NHWC(000, 16F, 7, 7, 10), "x");
    3013
    0
      ccv_nnc_tensor_symbol_t y = ccv_nnc_tensor_symbol_new(symbolic_graph, GPU_TENSOR_NHWC(000, 16F, 7, 7, 10), "y");
    3014
    0
      ccv_nnc_graph_exec_symbol_new(symbolic_graph, CMD_RELU_FORWARD(), TENSOR_SYMBOL_LIST(x), TENSOR_SYMBOL_LIST(y), "relu");
    3015
    0
      ccv_nnc_graph_exec_symbol_autogen(symbolic_graph, 0, 0, CCV_NNC_AUTOGEN_ALL_EXECS | CCV_NNC_AUTOGEN_SOURCES_AND_DESTINATIONS);
    3016
    0
      ccv_nnc_graph_t* graph = 0;
    3017
    0
      ccv_nnc_tensor_arena_t* tensor_arena = 0;
    3018
    0
      ccv_nnc_graph_exec_arena_t* graph_exec_arena = 0;
    3019
    0
      ccv_nnc_symbolic_graph_compile(symbolic_graph, ccv_nnc_default_compile_params, 0, 0, 0, 0, SYMBOLIC_GRAPH_SOURCES(symbolic_graph), SYMBOLIC_GRAPH_DESTINATIONS(symbolic_graph), &graph, &tensor_arena, &graph_exec_arena);
    3020
    0
      SYMBOLIC_GRAPH_GEN(symbolic_graph, CCV_NNC_LONG_DOT_GRAPH);
    3021
    0
      GRAPH_GEN(graph, CCV_NNC_LONG_DOT_GRAPH);
    3022
    0
      dsfmt_t dsfmt;
    3023
    0
      dsfmt_init_gen_rand(&dsfmt, 0);
    3024
    0
      ccv_nnc_tensor_t* const x_tensor = ccv_nnc_tensor_new(0, CPU_TENSOR_NHWC(32F, 7, 7, 10), 0);
    3025
    0
      int i;
    3026
    0
      for (i = 0; i < 7 * 7 * 10; i++)
    3027
    0
        x_tensor->data.f32[i] = dsfmt_genrand_open_close(&dsfmt) * 2 - 1;
    3028
    0
      ccv_nnc_tensor_t* const x16_tensor = ccv_nnc_tensor_new(0, CPU_TENSOR_NHWC(16F, 7, 7, 10), 0);
    3029
    0
      ccv_nnc_tensor_t* const xt = ccv_nnc_tensor_from_symbol(tensor_arena, x);
    3030
    0
      ccv_nnc_cmd_exec(CMD_DATATYPE_CONVERSION_FORWARD(), ccv_nnc_no_hint, 0, TENSOR_LIST(x_tensor), TENSOR_LIST(x16_tensor), 0);
    3031
    0
      ccv_nnc_cmd_exec(CMD_DATA_TRANSFER_FORWARD(), ccv_nnc_no_hint, 0, TENSOR_LIST(x16_tensor), TENSOR_LIST(xt), 0);
    3032
    0
      ccv_nnc_graph_run(graph, 0, TRAVERSE_FULL, 0, 0);
    3033
    0
      ccv_nnc_tensor_t* const y_tensor = ccv_nnc_tensor_new(0, CPU_TENSOR_NHWC(32F, 7, 7, 10), 0);
    3034
    0
      ccv_nnc_cmd_exec(CMD_RELU_FORWARD(), ccv_nnc_no_hint, 0, TENSOR_LIST(x_tensor), TENSOR_LIST(y_tensor), 0);
    3035
    0
      ccv_nnc_tensor_t* const yt = ccv_nnc_tensor_from_symbol(tensor_arena, y);
    3036
    0
      ccv_nnc_tensor_t* const cpu_y16 = ccv_nnc_tensor_new(0, CPU_TENSOR_NHWC(16F, 7, 7, 10), 0);
    3037
    0
      ccv_nnc_tensor_t* const cpu_y = ccv_nnc_tensor_new(0, CPU_TENSOR_NHWC(32F, 7, 7, 10), 0);
    3038
    0
      ccv_nnc_cmd_exec(CMD_DATA_TRANSFER_FORWARD(), ccv_nnc_no_hint, 0, TENSOR_LIST(yt), TENSOR_LIST(cpu_y16), 0);
    3039
    0
      ccv_nnc_cmd_exec(CMD_DATATYPE_CONVERSION_FORWARD(), ccv_nnc_no_hint, 0, TENSOR_LIST(cpu_y16), TENSOR_LIST(cpu_y), 0);
    3040
    0
      REQUIRE_ARRAY_EQ_WITH_TOLERANCE(float, y_tensor->data.f32, cpu_y->data.f32, 7 * 7 * 10, 1e-3, "mps result should equal to cpu result");
    3041
    0
      ccv_nnc_symbolic_graph_free(symbolic_graph);
    3042
    0
      ccv_nnc_tensor_arena_free(tensor_arena);
    3043
    0
      ccv_nnc_graph_exec_arena_free(graph_exec_arena);
    3044
    0
      ccv_nnc_graph_free(graph);
    3045
    0
      ccv_nnc_tensor_free(x_tensor);
    3046
    0
      ccv_nnc_tensor_free(x16_tensor);
    3047
    0
      ccv_nnc_tensor_free(y_tensor);
    3048
    0
      ccv_nnc_tensor_free(cpu_y);
    3049
    0
      ccv_nnc_tensor_free(cpu_y16);
    3050
    0
    }
    3051
    3052
    TEST_CASE("compare layer norm with mps")
    3053
    1
    {
    3054
    1
      GUARD_ELSE_RETURN(ccv_nnc_cmd_ok(CCV_NNC_LAYER_NORM_FORWARD, CCV_NNC_BACKEND_MPS) &&
    3055
    1
        ccv_nnc_cmd_ok(CCV_NNC_SET_FORWARD, CCV_NNC_BACKEND_MPS));
    3056
    0
      ccv_nnc_symbolic_graph_t* const symbolic_graph = ccv_nnc_symbolic_graph_new();
    3057
    0
      ccv_nnc_tensor_symbol_t x = ccv_nnc_tensor_symbol_new(symbolic_graph, CPU_TENSOR_NHWC(32F, 2, 2, 2, 10), "host x");
    3058
    0
      ccv_nnc_tensor_symbol_t bx = ccv_nnc_tensor_symbol_new(symbolic_graph, GPU_TENSOR_NHWC(000, 32F, 2, 2, 2, 10), "x");
    3059
    0
      ccv_nnc_tensor_symbol_t by = ccv_nnc_tensor_symbol_new(symbolic_graph, GPU_TENSOR_NHWC(000, 32F, 2, 2, 2, 10), "y");
    3060
    0
      ccv_nnc_tensor_symbol_t y = ccv_nnc_tensor_symbol_new(symbolic_graph, CPU_TENSOR_NHWC(32F, 2, 2, 2, 10), "host y");
    3061
    0
      ccv_nnc_tensor_symbol_t scale = ccv_nnc_tensor_symbol_new(symbolic_graph, GPU_TENSOR_NHWC(000, 32F, 1, 2, 2, 10), "scale");
    3062
    0
      ccv_nnc_tensor_symbol_t bias = ccv_nnc_tensor_symbol_new(symbolic_graph, GPU_TENSOR_NHWC(000, 32F, 1, 2, 2, 10), "bias");
    3063
    0
      ccv_nnc_tensor_symbol_t saved_mean = ccv_nnc_tensor_symbol_new(symbolic_graph, GPU_TENSOR_NHWC(000, 32F, 2, 1, 1, 1), "saved_mean");
    3064
    0
      ccv_nnc_tensor_symbol_t saved_inv_std = ccv_nnc_tensor_symbol_new(symbolic_graph, GPU_TENSOR_NHWC(000, 32F, 2, 1, 1, 1), "saved_inv_std");
    3065
    0
      ccv_nnc_graph_exec_symbol_new(symbolic_graph, CMD_DATA_TRANSFER_FORWARD(), TENSOR_SYMBOL_LIST(x), TENSOR_SYMBOL_LIST(bx), "transfer x");
    3066
    0
      ccv_nnc_graph_exec_symbol_new(symbolic_graph, CMD_LAYER_NORM_FORWARD(1e-6, 1, 1, 2, 3), TENSOR_SYMBOL_LIST(bx, scale, bias), TENSOR_SYMBOL_LIST(by, saved_mean, saved_inv_std), "layer_norm");
    3067
    0
      ccv_nnc_graph_exec_symbol_new(symbolic_graph, CMD_DATA_TRANSFER_FORWARD(), TENSOR_SYMBOL_LIST(by), TENSOR_SYMBOL_LIST(y), "transfer y");
    3068
    0
      ccv_nnc_graph_exec_symbol_autogen(symbolic_graph, 0, 0, CCV_NNC_AUTOGEN_ALL_EXECS | CCV_NNC_AUTOGEN_SOURCES_AND_DESTINATIONS);
    3069
    0
      ccv_nnc_graph_t* graph = 0;
    3070
    0
      ccv_nnc_tensor_arena_t* tensor_arena = 0;
    3071
    0
      ccv_nnc_graph_exec_arena_t* graph_exec_arena = 0;
    3072
    0
      ccv_nnc_symbolic_graph_compile(symbolic_graph, ccv_nnc_default_compile_params, 0, 0, 0, 0, SYMBOLIC_GRAPH_SOURCES(symbolic_graph), SYMBOLIC_GRAPH_DESTINATIONS(symbolic_graph), &graph, &tensor_arena, &graph_exec_arena);
    3073
    0
      SYMBOLIC_GRAPH_GEN(symbolic_graph, CCV_NNC_LONG_DOT_GRAPH);
    3074
    0
      GRAPH_GEN(graph, CCV_NNC_LONG_DOT_GRAPH);
    3075
    0
      dsfmt_t dsfmt;
    3076
    0
      float xdata[2 * 2 * 2 * 10];
    3077
    0
      ccv_nnc_tensor_t* const x_tensor = ccv_nnc_tensor_from_symbol(tensor_arena, x);
    3078
    0
      int i;
    3079
    0
      dsfmt_init_gen_rand(&dsfmt, 1);
    3080
    0
      for (i = 0; i < 2 * 2 * 2 * 10; i++)
    3081
    0
        x_tensor->data.f32[i] = xdata[i] = dsfmt_genrand_open_close(&dsfmt);
    3082
    0
      float scaledata[1 * 2 * 2 * 10];
    3083
    0
      float biasdata[1 * 2 * 2 * 10];
    3084
    0
      for (i = 0; i < 1 * 2 * 2 * 10; i++)
    3085
    0
      {
    3086
    0
        scaledata[i] = dsfmt_genrand_open_close(&dsfmt);
    3087
    0
        biasdata[i] = dsfmt_genrand_open_close(&dsfmt);
    3088
    0
      }
    3089
    0
      ccv_nnc_tensor_t scale_tensor = ccv_nnc_tensor(scaledata, CPU_TENSOR_NHWC(32F, 1, 2, 2, 10), 0);
    3090
    0
      ccv_nnc_tensor_t bias_tensor = ccv_nnc_tensor(biasdata, CPU_TENSOR_NHWC(32F, 1, 2, 2, 10), 0);
    3091
    0
      ccv_nnc_cmd_exec(CMD_DATA_TRANSFER_FORWARD(), ccv_nnc_no_hint, 0, TENSOR_LIST(&scale_tensor, &bias_tensor), TENSOR_LIST(ccv_nnc_tensor_from_symbol(tensor_arena, scale), ccv_nnc_tensor_from_symbol(tensor_arena, bias)), 0);
    3092
    0
      ccv_nnc_graph_run(graph, 0, TRAVERSE_FULL, 0, 0);
    3093
    0
      ccv_nnc_tensor_t* const y_tensor = ccv_nnc_tensor_from_symbol(tensor_arena, y);
    3094
    0
      ccv_nnc_symbolic_graph_free(symbolic_graph);
    3095
    0
      ccv_nnc_graph_exec_arena_free(graph_exec_arena);
    3096
    0
      ccv_nnc_graph_free(graph);
    3097
    0
      ccv_nnc_symbolic_graph_t* const cpu_symbolic_graph = ccv_nnc_symbolic_graph_new();
    3098
    0
      ccv_nnc_tensor_symbol_t cx = ccv_nnc_tensor_symbol_new(cpu_symbolic_graph, CPU_TENSOR_NHWC(32F, 2, 2, 2, 10), "x");
    3099
    0
      ccv_nnc_tensor_symbol_t cy = ccv_nnc_tensor_symbol_new(cpu_symbolic_graph, CPU_TENSOR_NHWC(32F, 2, 2, 2, 10), "y");
    3100
    0
      ccv_nnc_tensor_symbol_t cscale = ccv_nnc_tensor_symbol_new(cpu_symbolic_graph, CPU_TENSOR_NHWC(32F, 1, 2, 2, 10), "scale");
    3101
    0
      ccv_nnc_tensor_symbol_t cbias = ccv_nnc_tensor_symbol_new(cpu_symbolic_graph, CPU_TENSOR_NHWC(32F, 1, 2, 2, 10), "bias");
    3102
    0
      ccv_nnc_tensor_symbol_t csaved_mean = ccv_nnc_tensor_symbol_new(cpu_symbolic_graph, CPU_TENSOR_NHWC(32F, 2, 1, 1, 1), "saved_mean");
    3103
    0
      ccv_nnc_tensor_symbol_t csaved_inv_std = ccv_nnc_tensor_symbol_new(cpu_symbolic_graph, CPU_TENSOR_NHWC(32F, 2, 1, 1, 1), "saved_inv_std");
    3104
    0
      ccv_nnc_graph_exec_symbol_new(cpu_symbolic_graph, CMD_LAYER_NORM_FORWARD(1e-6, 1, 1, 2, 3), TENSOR_SYMBOL_LIST(cx, cscale, cbias), TENSOR_SYMBOL_LIST(cy, csaved_mean, csaved_inv_std), "layer_norm");
    3105
    0
      ccv_nnc_graph_exec_symbol_autogen(cpu_symbolic_graph, 0, 0, CCV_NNC_AUTOGEN_ALL_EXECS | CCV_NNC_AUTOGEN_SOURCES_AND_DESTINATIONS);
    3106
    0
      ccv_nnc_graph_t* cpu_graph = 0;
    3107
    0
      ccv_nnc_tensor_arena_t* cpu_tensor_arena = 0;
    3108
    0
      ccv_nnc_graph_exec_arena_t* cpu_graph_exec_arena = 0;
    3109
    0
      ccv_nnc_symbolic_graph_compile(cpu_symbolic_graph, ccv_nnc_default_compile_params, 0, 0, 0, 0, SYMBOLIC_GRAPH_SOURCES(cpu_symbolic_graph), SYMBOLIC_GRAPH_DESTINATIONS(cpu_symbolic_graph), &cpu_graph, &cpu_tensor_arena, &cpu_graph_exec_arena);
    3110
    0
      ccv_nnc_tensor_t* const cx_tensor = ccv_nnc_tensor_from_symbol(cpu_tensor_arena, cx);
    3111
    0
      memcpy(cx_tensor->data.f32, xdata, sizeof(float) * 2 * 2 * 2 * 10);
    3112
    0
      ccv_nnc_tensor_t* const cscale_tensor = ccv_nnc_tensor_from_symbol(cpu_tensor_arena, cscale);
    3113
    0
      memcpy(cscale_tensor->data.f32, scaledata, sizeof(float) * 1 * 2 * 2 * 10);
    3114
    0
      ccv_nnc_tensor_t* const cbias_tensor = ccv_nnc_tensor_from_symbol(cpu_tensor_arena, cbias);
    3115
    0
      memcpy(cbias_tensor->data.f32, biasdata, sizeof(float) * 1 * 2 * 2 * 10);
    3116
    0
      ccv_nnc_graph_run(cpu_graph, 0, TRAVERSE_FULL, 0, 0);
    3117
    0
      ccv_nnc_tensor_t* const cy_tensor = ccv_nnc_tensor_from_symbol(cpu_tensor_arena, cy);
    3118
      // Note that MPS and my other implementations treat epsilon differently.
    3119
    0
      REQUIRE_ARRAY_EQ_WITH_TOLERANCE(float, y_tensor->data.f32, cy_tensor->data.f32, 2 * 2 * 2 * 10, 1e-4, "layer norm result from mps should match the one from reference implementation");
    3120
    0
      ccv_nnc_symbolic_graph_free(cpu_symbolic_graph);
    3121
    0
      ccv_nnc_tensor_arena_free(tensor_arena);
    3122
    0
      ccv_nnc_tensor_arena_free(cpu_tensor_arena);
    3123
    0
      ccv_nnc_graph_exec_arena_free(cpu_graph_exec_arena);
    3124
    0
      ccv_nnc_graph_free(cpu_graph);
    3125
    0
    }
    3126
    3127
    TEST_CASE("compare layer norm with mps without scale / bias")
    3128
    1
    {
    3129
    1
      GUARD_ELSE_RETURN(ccv_nnc_cmd_ok(CCV_NNC_LAYER_NORM_FORWARD, CCV_NNC_BACKEND_MPS) &&
    3130
    1
        ccv_nnc_cmd_ok(CCV_NNC_SET_FORWARD, CCV_NNC_BACKEND_MPS));
    3131
    0
      ccv_nnc_symbolic_graph_t* const symbolic_graph = ccv_nnc_symbolic_graph_new();
    3132
    0
      ccv_nnc_tensor_symbol_t x = ccv_nnc_tensor_symbol_new(symbolic_graph, CPU_TENSOR_NHWC(32F, 2, 2, 2, 10), "host x");
    3133
    0
      ccv_nnc_tensor_symbol_t bx = ccv_nnc_tensor_symbol_new(symbolic_graph, GPU_TENSOR_NHWC(000, 32F, 2, 2, 2, 10), "x");
    3134
    0
      ccv_nnc_tensor_symbol_t by = ccv_nnc_tensor_symbol_new(symbolic_graph, GPU_TENSOR_NHWC(000, 32F, 2, 2, 2, 10), "y");
    3135
    0
      ccv_nnc_tensor_symbol_t y = ccv_nnc_tensor_symbol_new(symbolic_graph, CPU_TENSOR_NHWC(32F, 2, 2, 2, 10), "host y");
    3136
    0
      ccv_nnc_tensor_symbol_t saved_mean = ccv_nnc_tensor_symbol_new(symbolic_graph, GPU_TENSOR_NHWC(000, 32F, 2, 1, 1, 1), "saved_mean");
    3137
    0
      ccv_nnc_tensor_symbol_t saved_inv_std = ccv_nnc_tensor_symbol_new(symbolic_graph, GPU_TENSOR_NHWC(000, 32F, 2, 1, 1, 1), "saved_inv_std");
    3138
    0
      ccv_nnc_graph_exec_symbol_new(symbolic_graph, CMD_DATA_TRANSFER_FORWARD(), TENSOR_SYMBOL_LIST(x), TENSOR_SYMBOL_LIST(bx), "transfer x");
    3139
    0
      ccv_nnc_graph_exec_symbol_new(symbolic_graph, CMD_LAYER_NORM_FORWARD(1e-6, 0, 1, 2, 3), TENSOR_SYMBOL_LIST(bx), TENSOR_SYMBOL_LIST(by, saved_mean, saved_inv_std), "layer_norm");
    3140
    0
      ccv_nnc_graph_exec_symbol_new(symbolic_graph, CMD_DATA_TRANSFER_FORWARD(), TENSOR_SYMBOL_LIST(by), TENSOR_SYMBOL_LIST(y), "transfer y");
    3141
    0
      ccv_nnc_graph_exec_symbol_autogen(symbolic_graph, 0, 0, CCV_NNC_AUTOGEN_ALL_EXECS | CCV_NNC_AUTOGEN_SOURCES_AND_DESTINATIONS);
    3142
    0
      ccv_nnc_graph_t* graph = 0;
    3143
    0
      ccv_nnc_tensor_arena_t* tensor_arena = 0;
    3144
    0
      ccv_nnc_graph_exec_arena_t* graph_exec_arena = 0;
    3145
    0
      ccv_nnc_symbolic_graph_compile(symbolic_graph, ccv_nnc_default_compile_params, 0, 0, 0, 0, SYMBOLIC_GRAPH_SOURCES(symbolic_graph), SYMBOLIC_GRAPH_DESTINATIONS(symbolic_graph), &graph, &tensor_arena, &graph_exec_arena);
    3146
    0
      SYMBOLIC_GRAPH_GEN(symbolic_graph, CCV_NNC_LONG_DOT_GRAPH);
    3147
    0
      GRAPH_GEN(graph, CCV_NNC_LONG_DOT_GRAPH);
    3148
    0
      dsfmt_t dsfmt;
    3149
    0
      float xdata[2 * 2 * 2 * 10];
    3150
    0
      ccv_nnc_tensor_t* const x_tensor = ccv_nnc_tensor_from_symbol(tensor_arena, x);
    3151
    0
      int i;
    3152
    0
      dsfmt_init_gen_rand(&dsfmt, 1);
    3153
    0
      for (i = 0; i < 2 * 2 * 2 * 10; i++)
    3154
    0
        x_tensor->data.f32[i] = xdata[i] = dsfmt_genrand_open_close(&dsfmt);
    3155
    0
      ccv_nnc_graph_run(graph, 0, TRAVERSE_FULL, 0, 0);
    3156
    0
      ccv_nnc_tensor_t* const y_tensor = ccv_nnc_tensor_from_symbol(tensor_arena, y);
    3157
    0
      ccv_nnc_symbolic_graph_free(symbolic_graph);
    3158
    0
      ccv_nnc_graph_exec_arena_free(graph_exec_arena);
    3159
    0
      ccv_nnc_graph_free(graph);
    3160
    0
      ccv_nnc_symbolic_graph_t* const cpu_symbolic_graph = ccv_nnc_symbolic_graph_new();
    3161
    0
      ccv_nnc_tensor_symbol_t cx = ccv_nnc_tensor_symbol_new(cpu_symbolic_graph, CPU_TENSOR_NHWC(32F, 2, 2, 2, 10), "x");
    3162
    0
      ccv_nnc_tensor_symbol_t cy = ccv_nnc_tensor_symbol_new(cpu_symbolic_graph, CPU_TENSOR_NHWC(32F, 2, 2, 2, 10), "y");
    3163
    0
      ccv_nnc_tensor_symbol_t csaved_mean = ccv_nnc_tensor_symbol_new(cpu_symbolic_graph, CPU_TENSOR_NHWC(32F, 2, 1, 1, 1), "saved_mean");
    3164
    0
      ccv_nnc_tensor_symbol_t csaved_inv_std = ccv_nnc_tensor_symbol_new(cpu_symbolic_graph, CPU_TENSOR_NHWC(32F, 2, 1, 1, 1), "saved_inv_std");
    3165
    0
      ccv_nnc_graph_exec_symbol_new(cpu_symbolic_graph, CMD_LAYER_NORM_FORWARD(1e-6, 0, 1, 2, 3), TENSOR_SYMBOL_LIST(cx), TENSOR_SYMBOL_LIST(cy, csaved_mean, csaved_inv_std), "layer_norm");
    3166
    0
      ccv_nnc_graph_exec_symbol_autogen(cpu_symbolic_graph, 0, 0, CCV_NNC_AUTOGEN_ALL_EXECS | CCV_NNC_AUTOGEN_SOURCES_AND_DESTINATIONS);
    3167
    0
      ccv_nnc_graph_t* cpu_graph = 0;
    3168
    0
      ccv_nnc_tensor_arena_t* cpu_tensor_arena = 0;
    3169
    0
      ccv_nnc_graph_exec_arena_t* cpu_graph_exec_arena = 0;
    3170
    0
      ccv_nnc_symbolic_graph_compile(cpu_symbolic_graph, ccv_nnc_default_compile_params, 0, 0, 0, 0, SYMBOLIC_GRAPH_SOURCES(cpu_symbolic_graph), SYMBOLIC_GRAPH_DESTINATIONS(cpu_symbolic_graph), &cpu_graph, &cpu_tensor_arena, &cpu_graph_exec_arena);
    3171
    0
      ccv_nnc_tensor_t* const cx_tensor = ccv_nnc_tensor_from_symbol(cpu_tensor_arena, cx);
    3172
    0
      memcpy(cx_tensor->data.f32, xdata, sizeof(float) * 2 * 2 * 2 * 10);
    3173
    0
      ccv_nnc_graph_run(cpu_graph, 0, TRAVERSE_FULL, 0, 0);
    3174
    0
      ccv_nnc_tensor_t* const cy_tensor = ccv_nnc_tensor_from_symbol(cpu_tensor_arena, cy);
    3175
      // Note that MPS and my other implementations treat epsilon differently.
    3176
    0
      REQUIRE_ARRAY_EQ_WITH_TOLERANCE(float, y_tensor->data.f32, cy_tensor->data.f32, 2 * 2 * 2 * 10, 1e-4, "layer norm result from mps should match the one from reference implementation");
    3177
    0
      ccv_nnc_symbolic_graph_free(cpu_symbolic_graph);
    3178
    0
      ccv_nnc_tensor_arena_free(tensor_arena);
    3179
    0
      ccv_nnc_tensor_arena_free(cpu_tensor_arena);
    3180
    0
      ccv_nnc_graph_exec_arena_free(cpu_graph_exec_arena);
    3181
    0
      ccv_nnc_graph_free(cpu_graph);
    3182
    0
    }
    3183
    3184
    TEST_CASE("compare group norm with mps")
    3185
    1
    {
    3186
    1
      GUARD_ELSE_RETURN(ccv_nnc_cmd_ok(CCV_NNC_GROUP_NORM_FORWARD, CCV_NNC_BACKEND_MPS) &&
    3187
    1
        ccv_nnc_cmd_ok(CCV_NNC_SET_FORWARD, CCV_NNC_BACKEND_MPS));
    3188
    0
      ccv_nnc_symbolic_graph_t* const symbolic_graph = ccv_nnc_symbolic_graph_new();
    3189
    0
      ccv_nnc_tensor_symbol_t x = ccv_nnc_tensor_symbol_new(symbolic_graph, CPU_TENSOR_NHWC(32F, 2, 16, 2, 10), "host x");
    3190
    0
      ccv_nnc_tensor_symbol_t bx = ccv_nnc_tensor_symbol_new(symbolic_graph, GPU_TENSOR_NHWC(000, 32F, 2, 16, 2, 10), "x");
    3191
    0
      ccv_nnc_tensor_symbol_t by = ccv_nnc_tensor_symbol_new(symbolic_graph, GPU_TENSOR_NHWC(000, 32F, 2, 16, 2, 10), "y");
    3192
    0
      ccv_nnc_tensor_symbol_t y = ccv_nnc_tensor_symbol_new(symbolic_graph, CPU_TENSOR_NHWC(32F, 2, 16, 2, 10), "host y");
    3193
    0
      ccv_nnc_tensor_symbol_t scale = ccv_nnc_tensor_symbol_new(symbolic_graph, GPU_TENSOR_NHWC(000, 32F, 1, 16, 2, 10), "scale");
    3194
    0
      ccv_nnc_tensor_symbol_t bias = ccv_nnc_tensor_symbol_new(symbolic_graph, GPU_TENSOR_NHWC(000, 32F, 1, 16, 2, 10), "bias");
    3195
    0
      ccv_nnc_tensor_symbol_t saved_mean = ccv_nnc_tensor_symbol_new(symbolic_graph, GPU_TENSOR_NHWC(000, 32F, 2, 4, 2, 10), "saved_mean");
    3196
    0
      ccv_nnc_tensor_symbol_t saved_inv_std = ccv_nnc_tensor_symbol_new(symbolic_graph, GPU_TENSOR_NHWC(000, 32F, 2, 4, 2, 10), "saved_inv_std");
    3197
    0
      ccv_nnc_graph_exec_symbol_new(symbolic_graph, CMD_DATA_TRANSFER_FORWARD(), TENSOR_SYMBOL_LIST(x), TENSOR_SYMBOL_LIST(bx), "transfer x");
    3198
    0
      ccv_nnc_graph_exec_symbol_new(symbolic_graph, CMD_GROUP_NORM_FORWARD(1, 4, 1e-7, 1), TENSOR_SYMBOL_LIST(bx, scale, bias), TENSOR_SYMBOL_LIST(by, saved_mean, saved_inv_std), "group_norm");
    3199
    0
      ccv_nnc_graph_exec_symbol_new(symbolic_graph, CMD_DATA_TRANSFER_FORWARD(), TENSOR_SYMBOL_LIST(by), TENSOR_SYMBOL_LIST(y), "transfer y");
    3200
    0
      ccv_nnc_graph_exec_symbol_autogen(symbolic_graph, 0, 0, CCV_NNC_AUTOGEN_ALL_EXECS | CCV_NNC_AUTOGEN_SOURCES_AND_DESTINATIONS);
    3201
    0
      ccv_nnc_graph_t* graph = 0;
    3202
    0
      ccv_nnc_tensor_arena_t* tensor_arena = 0;
    3203
    0
      ccv_nnc_graph_exec_arena_t* graph_exec_arena = 0;
    3204
    0
      ccv_nnc_symbolic_graph_compile(symbolic_graph, ccv_nnc_default_compile_params, 0, 0, 0, 0, SYMBOLIC_GRAPH_SOURCES(symbolic_graph), SYMBOLIC_GRAPH_DESTINATIONS(symbolic_graph), &graph, &tensor_arena, &graph_exec_arena);
    3205
    0
      SYMBOLIC_GRAPH_GEN(symbolic_graph, CCV_NNC_LONG_DOT_GRAPH);
    3206
    0
      GRAPH_GEN(graph, CCV_NNC_LONG_DOT_GRAPH);
    3207
    0
      dsfmt_t dsfmt;
    3208
    0
      float xdata[2 * 16 * 2 * 10];
    3209
    0
      ccv_nnc_tensor_t* const x_tensor = ccv_nnc_tensor_from_symbol(tensor_arena, x);
    3210
    0
      int i;
    3211
    0
      dsfmt_init_gen_rand(&dsfmt, 1);
    3212
    0
      for (i = 0; i < 2 * 16 * 2 * 10; i++)
    3213
    0
        x_tensor->data.f32[i] = xdata[i] = dsfmt_genrand_open_close(&dsfmt);
    3214
    0
      float scaledata[1 * 16 * 2 * 10];
    3215
    0
      float biasdata[1 * 16 * 2 * 10];
    3216
    0
      for (i = 0; i < 1 * 16 * 2 * 10; i++)
    3217
    0
      {
    3218
    0
        scaledata[i] = dsfmt_genrand_open_close(&dsfmt);
    3219
    0
        biasdata[i] = dsfmt_genrand_open_close(&dsfmt);
    3220
    0
      }
    3221
    0
      ccv_nnc_tensor_t scale_tensor = ccv_nnc_tensor(scaledata, CPU_TENSOR_NHWC(32F, 1, 16, 2, 10), 0);
    3222
    0
      ccv_nnc_tensor_t bias_tensor = ccv_nnc_tensor(biasdata, CPU_TENSOR_NHWC(32F, 1, 16, 2, 10), 0);
    3223
    0
      ccv_nnc_cmd_exec(CMD_DATA_TRANSFER_FORWARD(), ccv_nnc_no_hint, 0, TENSOR_LIST(&scale_tensor, &bias_tensor), TENSOR_LIST(ccv_nnc_tensor_from_symbol(tensor_arena, scale), ccv_nnc_tensor_from_symbol(tensor_arena, bias)), 0);
    3224
    0
      ccv_nnc_graph_run(graph, 0, TRAVERSE_FULL, 0, 0);
    3225
    0
      ccv_nnc_tensor_t* const y_tensor = ccv_nnc_tensor_from_symbol(tensor_arena, y);
    3226
    0
      ccv_nnc_symbolic_graph_free(symbolic_graph);
    3227
    0
      ccv_nnc_graph_exec_arena_free(graph_exec_arena);
    3228
    0
      ccv_nnc_graph_free(graph);
    3229
    0
      ccv_nnc_symbolic_graph_t* const cpu_symbolic_graph = ccv_nnc_symbolic_graph_new();
    3230
    0
      ccv_nnc_tensor_symbol_t cx = ccv_nnc_tensor_symbol_new(cpu_symbolic_graph, CPU_TENSOR_NHWC(32F, 2, 16, 2, 10), "x");
    3231
    0
      ccv_nnc_tensor_symbol_t cy = ccv_nnc_tensor_symbol_new(cpu_symbolic_graph, CPU_TENSOR_NHWC(32F, 2, 16, 2, 10), "y");
    3232
    0
      ccv_nnc_tensor_symbol_t cscale = ccv_nnc_tensor_symbol_new(cpu_symbolic_graph, CPU_TENSOR_NHWC(32F, 1, 16, 2, 10), "scale");
    3233
    0
      ccv_nnc_tensor_symbol_t cbias = ccv_nnc_tensor_symbol_new(cpu_symbolic_graph, CPU_TENSOR_NHWC(32F, 1, 16, 2, 10), "bias");
    3234
    0
      ccv_nnc_tensor_symbol_t csaved_mean = ccv_nnc_tensor_symbol_new(cpu_symbolic_graph, CPU_TENSOR_NHWC(32F, 2, 4, 2, 10), "saved_mean");
    3235
    0
      ccv_nnc_tensor_symbol_t csaved_inv_std = ccv_nnc_tensor_symbol_new(cpu_symbolic_graph, CPU_TENSOR_NHWC(32F, 2, 4, 2, 10), "saved_inv_std");
    3236
    0
      ccv_nnc_graph_exec_symbol_new(cpu_symbolic_graph, CMD_GROUP_NORM_FORWARD(1, 4, 1e-7, 1), TENSOR_SYMBOL_LIST(cx, cscale, cbias), TENSOR_SYMBOL_LIST(cy, csaved_mean, csaved_inv_std), "group_norm");
    3237
    0
      ccv_nnc_graph_exec_symbol_autogen(cpu_symbolic_graph, 0, 0, CCV_NNC_AUTOGEN_ALL_EXECS | CCV_NNC_AUTOGEN_SOURCES_AND_DESTINATIONS);
    3238
    0
      ccv_nnc_graph_t* cpu_graph = 0;
    3239
    0
      ccv_nnc_tensor_arena_t* cpu_tensor_arena = 0;
    3240
    0
      ccv_nnc_graph_exec_arena_t* cpu_graph_exec_arena = 0;
    3241
    0
      ccv_nnc_symbolic_graph_compile(cpu_symbolic_graph, ccv_nnc_default_compile_params, 0, 0, 0, 0, SYMBOLIC_GRAPH_SOURCES(cpu_symbolic_graph), SYMBOLIC_GRAPH_DESTINATIONS(cpu_symbolic_graph), &cpu_graph, &cpu_tensor_arena, &cpu_graph_exec_arena);
    3242
    0
      ccv_nnc_tensor_t* const cx_tensor = ccv_nnc_tensor_from_symbol(cpu_tensor_arena, cx);
    3243
    0
      memcpy(cx_tensor->data.f32, xdata, sizeof(float) * 2 * 16 * 2 * 10);
    3244
    0
      ccv_nnc_tensor_t* const cscale_tensor = ccv_nnc_tensor_from_symbol(cpu_tensor_arena, cscale);
    3245
    0
      memcpy(cscale_tensor->data.f32, scaledata, sizeof(float) * 1 * 16 * 2 * 10);
    3246
    0
      ccv_nnc_tensor_t* const cbias_tensor = ccv_nnc_tensor_from_symbol(cpu_tensor_arena, cbias);
    3247
    0
      memcpy(cbias_tensor->data.f32, biasdata, sizeof(float) * 1 * 16 * 2 * 10);
    3248
    0
      ccv_nnc_graph_run(cpu_graph, 0, TRAVERSE_FULL, 0, 0);
    3249
    0
      ccv_nnc_tensor_t* const cy_tensor = ccv_nnc_tensor_from_symbol(cpu_tensor_arena, cy);
    3250
      // Note that MPS and my other implementations treat epsilon differently.
    3251
    0
      REQUIRE_ARRAY_EQ_WITH_TOLERANCE(float, y_tensor->data.f32, cy_tensor->data.f32, 2 * 16 * 2 * 10, 1e-3, "group norm result from mps should match the one from reference implementation");
    3252
    0
      ccv_nnc_symbolic_graph_free(cpu_symbolic_graph);
    3253
    0
      ccv_nnc_tensor_arena_free(tensor_arena);
    3254
    0
      ccv_nnc_tensor_arena_free(cpu_tensor_arena);
    3255
    0
      ccv_nnc_graph_exec_arena_free(cpu_graph_exec_arena);
    3256
    0
      ccv_nnc_graph_free(cpu_graph);
    3257
    0
    }
    3258
    3259
    TEST_CASE("compare group norm with mps without scale / bias")
    3260
    1
    {
    3261
    1
      GUARD_ELSE_RETURN(ccv_nnc_cmd_ok(CCV_NNC_GROUP_NORM_FORWARD, CCV_NNC_BACKEND_MPS) &&
    3262
    1
        ccv_nnc_cmd_ok(CCV_NNC_SET_FORWARD, CCV_NNC_BACKEND_MPS));
    3263
    0
      ccv_nnc_symbolic_graph_t* const symbolic_graph = ccv_nnc_symbolic_graph_new();
    3264
    0
      ccv_nnc_tensor_symbol_t x = ccv_nnc_tensor_symbol_new(symbolic_graph, CPU_TENSOR_NHWC(32F, 2, 16, 2, 10), "host x");
    3265
    0
      ccv_nnc_tensor_symbol_t bx = ccv_nnc_tensor_symbol_new(symbolic_graph, GPU_TENSOR_NHWC(000, 32F, 2, 16, 2, 10), "x");
    3266
    0
      ccv_nnc_tensor_symbol_t by = ccv_nnc_tensor_symbol_new(symbolic_graph, GPU_TENSOR_NHWC(000, 32F, 2, 16, 2, 10), "y");
    3267
    0
      ccv_nnc_tensor_symbol_t y = ccv_nnc_tensor_symbol_new(symbolic_graph, CPU_TENSOR_NHWC(32F, 2, 16, 2, 10), "host y");
    3268
    0
      ccv_nnc_tensor_symbol_t saved_mean = ccv_nnc_tensor_symbol_new(symbolic_graph, GPU_TENSOR_NHWC(000, 32F, 2, 4, 2, 10), "saved_mean");
    3269
    0
      ccv_nnc_tensor_symbol_t saved_inv_std = ccv_nnc_tensor_symbol_new(symbolic_graph, GPU_TENSOR_NHWC(000, 32F, 2, 4, 2, 10), "saved_inv_std");
    3270
    0
      ccv_nnc_graph_exec_symbol_new(symbolic_graph, CMD_DATA_TRANSFER_FORWARD(), TENSOR_SYMBOL_LIST(x), TENSOR_SYMBOL_LIST(bx), "transfer x");
    3271
    0
      ccv_nnc_graph_exec_symbol_new(symbolic_graph, CMD_GROUP_NORM_FORWARD(1, 4, 1e-7, 0), TENSOR_SYMBOL_LIST(bx), TENSOR_SYMBOL_LIST(by, saved_mean, saved_inv_std), "group_norm");
    3272
    0
      ccv_nnc_graph_exec_symbol_new(symbolic_graph, CMD_DATA_TRANSFER_FORWARD(), TENSOR_SYMBOL_LIST(by), TENSOR_SYMBOL_LIST(y), "transfer y");
    3273
    0
      ccv_nnc_graph_exec_symbol_autogen(symbolic_graph, 0, 0, CCV_NNC_AUTOGEN_ALL_EXECS | CCV_NNC_AUTOGEN_SOURCES_AND_DESTINATIONS);
    3274
    0
      ccv_nnc_graph_t* graph = 0;
    3275
    0
      ccv_nnc_tensor_arena_t* tensor_arena = 0;
    3276
    0
      ccv_nnc_graph_exec_arena_t* graph_exec_arena = 0;
    3277
    0
      ccv_nnc_symbolic_graph_compile(symbolic_graph, ccv_nnc_default_compile_params, 0, 0, 0, 0, SYMBOLIC_GRAPH_SOURCES(symbolic_graph), SYMBOLIC_GRAPH_DESTINATIONS(symbolic_graph), &graph, &tensor_arena, &graph_exec_arena);
    3278
    0
      SYMBOLIC_GRAPH_GEN(symbolic_graph, CCV_NNC_LONG_DOT_GRAPH);
    3279
    0
      GRAPH_GEN(graph, CCV_NNC_LONG_DOT_GRAPH);
    3280
    0
      dsfmt_t dsfmt;
    3281
    0
      float xdata[2 * 16 * 2 * 10];
    3282
    0
      ccv_nnc_tensor_t* const x_tensor = ccv_nnc_tensor_from_symbol(tensor_arena, x);
    3283
    0
      int i;
    3284
    0
      dsfmt_init_gen_rand(&dsfmt, 1);
    3285
    0
      for (i = 0; i < 2 * 16 * 2 * 10; i++)
    3286
    0
        x_tensor->data.f32[i] = xdata[i] = dsfmt_genrand_open_close(&dsfmt);
    3287
    0
      ccv_nnc_graph_run(graph, 0, TRAVERSE_FULL, 0, 0);
    3288
    0
      ccv_nnc_tensor_t* const y_tensor = ccv_nnc_tensor_from_symbol(tensor_arena, y);
    3289
    0
      ccv_nnc_symbolic_graph_free(symbolic_graph);
    3290
    0
      ccv_nnc_graph_exec_arena_free(graph_exec_arena);
    3291
    0
      ccv_nnc_graph_free(graph);
    3292
    0
      ccv_nnc_symbolic_graph_t* const cpu_symbolic_graph = ccv_nnc_symbolic_graph_new();
    3293
    0
      ccv_nnc_tensor_symbol_t cx = ccv_nnc_tensor_symbol_new(cpu_symbolic_graph, CPU_TENSOR_NHWC(32F, 2, 16, 2, 10), "x");
    3294
    0
      ccv_nnc_tensor_symbol_t cy = ccv_nnc_tensor_symbol_new(cpu_symbolic_graph, CPU_TENSOR_NHWC(32F, 2, 16, 2, 10), "y");
    3295
    0
      ccv_nnc_tensor_symbol_t csaved_mean = ccv_nnc_tensor_symbol_new(cpu_symbolic_graph, CPU_TENSOR_NHWC(32F, 2, 4, 2, 10), "saved_mean");
    3296
    0
      ccv_nnc_tensor_symbol_t csaved_inv_std = ccv_nnc_tensor_symbol_new(cpu_symbolic_graph, CPU_TENSOR_NHWC(32F, 2, 4, 2, 10), "saved_inv_std");
    3297
    0
      ccv_nnc_graph_exec_symbol_new(cpu_symbolic_graph, CMD_GROUP_NORM_FORWARD(1, 4, 1e-7, 0), TENSOR_SYMBOL_LIST(cx), TENSOR_SYMBOL_LIST(cy, csaved_mean, csaved_inv_std), "group_norm");
    3298
    0
      ccv_nnc_graph_exec_symbol_autogen(cpu_symbolic_graph, 0, 0, CCV_NNC_AUTOGEN_ALL_EXECS | CCV_NNC_AUTOGEN_SOURCES_AND_DESTINATIONS);
    3299
    0
      ccv_nnc_graph_t* cpu_graph = 0;
    3300
    0
      ccv_nnc_tensor_arena_t* cpu_tensor_arena = 0;
    3301
    0
      ccv_nnc_graph_exec_arena_t* cpu_graph_exec_arena = 0;
    3302
    0
      ccv_nnc_symbolic_graph_compile(cpu_symbolic_graph, ccv_nnc_default_compile_params, 0, 0, 0, 0, SYMBOLIC_GRAPH_SOURCES(cpu_symbolic_graph), SYMBOLIC_GRAPH_DESTINATIONS(cpu_symbolic_graph), &cpu_graph, &cpu_tensor_arena, &cpu_graph_exec_arena);
    3303
    0
      ccv_nnc_tensor_t* const cx_tensor = ccv_nnc_tensor_from_symbol(cpu_tensor_arena, cx);
    3304
    0
      memcpy(cx_tensor->data.f32, xdata, sizeof(float) * 2 * 16 * 2 * 10);
    3305
    0
      ccv_nnc_graph_run(cpu_graph, 0, TRAVERSE_FULL, 0, 0);
    3306
    0
      ccv_nnc_tensor_t* const cy_tensor = ccv_nnc_tensor_from_symbol(cpu_tensor_arena, cy);
    3307
      // Note that MPS and my other implementations treat epsilon differently.
    3308
    0
      REQUIRE_ARRAY_EQ_WITH_TOLERANCE(float, y_tensor->data.f32, cy_tensor->data.f32, 2 * 16 * 2 * 10, 1e-3, "group norm result from mps should match the one from reference implementation");
    3309
    0
      ccv_nnc_symbolic_graph_free(cpu_symbolic_graph);
    3310
    0
      ccv_nnc_tensor_arena_free(tensor_arena);
    3311
    0
      ccv_nnc_tensor_arena_free(cpu_tensor_arena);
    3312
    0
      ccv_nnc_graph_exec_arena_free(cpu_graph_exec_arena);
    3313
    0
      ccv_nnc_graph_free(cpu_graph);
    3314
    0
    }
    3315
    3316
    TEST_CASE("compare rmsnorm with mps")
    3317
    1
    {
    3318
    1
      GUARD_ELSE_RETURN(ccv_nnc_cmd_ok(CCV_NNC_RMSNORM_FORWARD, CCV_NNC_BACKEND_MPS) &&
    3319
    1
        ccv_nnc_cmd_ok(CCV_NNC_SET_FORWARD, CCV_NNC_BACKEND_MPS));
    3320
    0
      ccv_nnc_symbolic_graph_t* const symbolic_graph = ccv_nnc_symbolic_graph_new();
    3321
    0
      ccv_nnc_tensor_symbol_t x = ccv_nnc_tensor_symbol_new(symbolic_graph, CPU_TENSOR_NHWC(32F, 2, 2, 2, 10), "host x");
    3322
    0
      ccv_nnc_tensor_symbol_t bx = ccv_nnc_tensor_symbol_new(symbolic_graph, GPU_TENSOR_NHWC(000, 32F, 2, 2, 2, 10), "x");
    3323
    0
      ccv_nnc_tensor_symbol_t by = ccv_nnc_tensor_symbol_new(symbolic_graph, GPU_TENSOR_NHWC(000, 32F, 2, 2, 2, 10), "y");
    3324
    0
      ccv_nnc_tensor_symbol_t y = ccv_nnc_tensor_symbol_new(symbolic_graph, CPU_TENSOR_NHWC(32F, 2, 2, 2, 10), "host y");
    3325
    0
      ccv_nnc_tensor_symbol_t scale = ccv_nnc_tensor_symbol_new(symbolic_graph, GPU_TENSOR_NHWC(000, 32F, 1, 2, 2, 10), "scale");
    3326
    0
      ccv_nnc_tensor_symbol_t saved_inv_std = ccv_nnc_tensor_symbol_new(symbolic_graph, GPU_TENSOR_NHWC(000, 32F, 2, 1, 1, 1), "saved_inv_std");
    3327
    0
      ccv_nnc_graph_exec_symbol_new(symbolic_graph, CMD_DATA_TRANSFER_FORWARD(), TENSOR_SYMBOL_LIST(x), TENSOR_SYMBOL_LIST(bx), "transfer x");
    3328
    0
      ccv_nnc_graph_exec_symbol_new(symbolic_graph, CMD_RMSNORM_FORWARD(1e-6, 1, 1, 2, 3), TENSOR_SYMBOL_LIST(bx, scale), TENSOR_SYMBOL_LIST(by, saved_inv_std), "rmsnorm");
    3329
    0
      ccv_nnc_graph_exec_symbol_new(symbolic_graph, CMD_DATA_TRANSFER_FORWARD(), TENSOR_SYMBOL_LIST(by), TENSOR_SYMBOL_LIST(y), "transfer y");
    3330
    0
      ccv_nnc_graph_exec_symbol_autogen(symbolic_graph, 0, 0, CCV_NNC_AUTOGEN_ALL_EXECS | CCV_NNC_AUTOGEN_SOURCES_AND_DESTINATIONS);
    3331
    0
      ccv_nnc_graph_t* graph = 0;
    3332
    0
      ccv_nnc_tensor_arena_t* tensor_arena = 0;
    3333
    0
      ccv_nnc_graph_exec_arena_t* graph_exec_arena = 0;
    3334
    0
      ccv_nnc_symbolic_graph_compile(symbolic_graph, ccv_nnc_default_compile_params, 0, 0, 0, 0, SYMBOLIC_GRAPH_SOURCES(symbolic_graph), SYMBOLIC_GRAPH_DESTINATIONS(symbolic_graph), &graph, &tensor_arena, &graph_exec_arena);
    3335
    0
      SYMBOLIC_GRAPH_GEN(symbolic_graph, CCV_NNC_LONG_DOT_GRAPH);
    3336
    0
      GRAPH_GEN(graph, CCV_NNC_LONG_DOT_GRAPH);
    3337
    0
      dsfmt_t dsfmt;
    3338
    0
      float xdata[2 * 2 * 2 * 10];
    3339
    0
      ccv_nnc_tensor_t* const x_tensor = ccv_nnc_tensor_from_symbol(tensor_arena, x);
    3340
    0
      int i;
    3341
    0
      dsfmt_init_gen_rand(&dsfmt, 1);
    3342
    0
      for (i = 0; i < 2 * 2 * 2 * 10; i++)
    3343
    0
        x_tensor->data.f32[i] = xdata[i] = dsfmt_genrand_open_close(&dsfmt);
    3344
    0
      float scaledata[1 * 2 * 2 * 10];
    3345
    0
      for (i = 0; i < 1 * 2 * 2 * 10; i++)
    3346
    0
        scaledata[i] = dsfmt_genrand_open_close(&dsfmt);
    3347
    0
      ccv_nnc_tensor_t scale_tensor = ccv_nnc_tensor(scaledata, CPU_TENSOR_NHWC(32F, 1, 2, 2, 10), 0);
    3348
    0
      ccv_nnc_cmd_exec(CMD_DATA_TRANSFER_FORWARD(), ccv_nnc_no_hint, 0, TENSOR_LIST(&scale_tensor), TENSOR_LIST(ccv_nnc_tensor_from_symbol(tensor_arena, scale)), 0);
    3349
    0
      ccv_nnc_graph_run(graph, 0, TRAVERSE_FULL, 0, 0);
    3350
    0
      ccv_nnc_tensor_t* const y_tensor = ccv_nnc_tensor_from_symbol(tensor_arena, y);
    3351
    0
      ccv_nnc_symbolic_graph_free(symbolic_graph);
    3352
    0
      ccv_nnc_graph_exec_arena_free(graph_exec_arena);
    3353
    0
      ccv_nnc_graph_free(graph);
    3354
    0
      ccv_nnc_symbolic_graph_t* const cpu_symbolic_graph = ccv_nnc_symbolic_graph_new();
    3355
    0
      ccv_nnc_tensor_symbol_t cx = ccv_nnc_tensor_symbol_new(cpu_symbolic_graph, CPU_TENSOR_NHWC(32F, 2, 2, 2, 10), "x");
    3356
    0
      ccv_nnc_tensor_symbol_t cy = ccv_nnc_tensor_symbol_new(cpu_symbolic_graph, CPU_TENSOR_NHWC(32F, 2, 2, 2, 10), "y");
    3357
    0
      ccv_nnc_tensor_symbol_t cscale = ccv_nnc_tensor_symbol_new(cpu_symbolic_graph, CPU_TENSOR_NHWC(32F, 1, 2, 2, 10), "scale");
    3358
    0
      ccv_nnc_tensor_symbol_t csaved_inv_std = ccv_nnc_tensor_symbol_new(cpu_symbolic_graph, CPU_TENSOR_NHWC(32F, 2, 1, 1, 1), "saved_inv_std");
    3359
    0
      ccv_nnc_graph_exec_symbol_new(cpu_symbolic_graph, CMD_RMSNORM_FORWARD(1e-6, 1, 1, 2, 3), TENSOR_SYMBOL_LIST(cx, cscale), TENSOR_SYMBOL_LIST(cy, csaved_inv_std), "rmsnorm");
    3360
    0
      ccv_nnc_graph_exec_symbol_autogen(cpu_symbolic_graph, 0, 0, CCV_NNC_AUTOGEN_ALL_EXECS | CCV_NNC_AUTOGEN_SOURCES_AND_DESTINATIONS);
    3361
    0
      ccv_nnc_graph_t* cpu_graph = 0;
    3362
    0
      ccv_nnc_tensor_arena_t* cpu_tensor_arena = 0;
    3363
    0
      ccv_nnc_graph_exec_arena_t* cpu_graph_exec_arena = 0;
    3364
    0
      ccv_nnc_symbolic_graph_compile(cpu_symbolic_graph, ccv_nnc_default_compile_params, 0, 0, 0, 0, SYMBOLIC_GRAPH_SOURCES(cpu_symbolic_graph), SYMBOLIC_GRAPH_DESTINATIONS(cpu_symbolic_graph), &cpu_graph, &cpu_tensor_arena, &cpu_graph_exec_arena);
    3365
    0
      ccv_nnc_tensor_t* const cx_tensor = ccv_nnc_tensor_from_symbol(cpu_tensor_arena, cx);
    3366
    0
      memcpy(cx_tensor->data.f32, xdata, sizeof(float) * 2 * 2 * 2 * 10);
    3367
    0
      ccv_nnc_tensor_t* const cscale_tensor = ccv_nnc_tensor_from_symbol(cpu_tensor_arena, cscale);
    3368
    0
      memcpy(cscale_tensor->data.f32, scaledata, sizeof(float) * 1 * 2 * 2 * 10);
    3369
    0
      ccv_nnc_graph_run(cpu_graph, 0, TRAVERSE_FULL, 0, 0);
    3370
    0
      ccv_nnc_tensor_t* const cy_tensor = ccv_nnc_tensor_from_symbol(cpu_tensor_arena, cy);
    3371
      // Note that MPS and my other implementations treat epsilon differently.
    3372
    0
      REQUIRE_ARRAY_EQ_WITH_TOLERANCE(float, y_tensor->data.f32, cy_tensor->data.f32, 2 * 2 * 2 * 10, 1e-4, "rmsnorm result from mps should match the one from reference implementation");
    3373
    0
      ccv_nnc_symbolic_graph_free(cpu_symbolic_graph);
    3374
    0
      ccv_nnc_tensor_arena_free(tensor_arena);
    3375
    0
      ccv_nnc_tensor_arena_free(cpu_tensor_arena);
    3376
    0
      ccv_nnc_graph_exec_arena_free(cpu_graph_exec_arena);
    3377
    0
      ccv_nnc_graph_free(cpu_graph);
    3378
    0
    }
    3379
    3380
    TEST_CASE("compare rmsnorm with mps without scale")
    3381
    1
    {
    3382
    1
      GUARD_ELSE_RETURN(ccv_nnc_cmd_ok(CCV_NNC_RMSNORM_FORWARD, CCV_NNC_BACKEND_MPS) &&
    3383
    1
        ccv_nnc_cmd_ok(CCV_NNC_SET_FORWARD, CCV_NNC_BACKEND_MPS));
    3384
    0
      ccv_nnc_symbolic_graph_t* const symbolic_graph = ccv_nnc_symbolic_graph_new();
    3385
    0
      ccv_nnc_tensor_symbol_t x = ccv_nnc_tensor_symbol_new(symbolic_graph, CPU_TENSOR_NHWC(32F, 2, 2, 2, 10), "host x");
    3386
    0
      ccv_nnc_tensor_symbol_t bx = ccv_nnc_tensor_symbol_new(symbolic_graph, GPU_TENSOR_NHWC(000, 32F, 2, 2, 2, 10), "x");
    3387
    0
      ccv_nnc_tensor_symbol_t by = ccv_nnc_tensor_symbol_new(symbolic_graph, GPU_TENSOR_NHWC(000, 32F, 2, 2, 2, 10), "y");
    3388
    0
      ccv_nnc_tensor_symbol_t y = ccv_nnc_tensor_symbol_new(symbolic_graph, CPU_TENSOR_NHWC(32F, 2, 2, 2, 10), "host y");
    3389
    0
      ccv_nnc_tensor_symbol_t saved_inv_std = ccv_nnc_tensor_symbol_new(symbolic_graph, GPU_TENSOR_NHWC(000, 32F, 2, 1, 1, 1), "saved_inv_std");
    3390
    0
      ccv_nnc_graph_exec_symbol_new(symbolic_graph, CMD_DATA_TRANSFER_FORWARD(), TENSOR_SYMBOL_LIST(x), TENSOR_SYMBOL_LIST(bx), "transfer x");
    3391
    0
      ccv_nnc_graph_exec_symbol_new(symbolic_graph, CMD_RMSNORM_FORWARD(1e-6, 0, 1, 2, 3), TENSOR_SYMBOL_LIST(bx), TENSOR_SYMBOL_LIST(by, saved_inv_std), "rmsnorm");
    3392
    0
      ccv_nnc_graph_exec_symbol_new(symbolic_graph, CMD_DATA_TRANSFER_FORWARD(), TENSOR_SYMBOL_LIST(by), TENSOR_SYMBOL_LIST(y), "transfer y");
    3393
    0
      ccv_nnc_graph_exec_symbol_autogen(symbolic_graph, 0, 0, CCV_NNC_AUTOGEN_ALL_EXECS | CCV_NNC_AUTOGEN_SOURCES_AND_DESTINATIONS);
    3394
    0
      ccv_nnc_graph_t* graph = 0;
    3395
    0
      ccv_nnc_tensor_arena_t* tensor_arena = 0;
    3396
    0
      ccv_nnc_graph_exec_arena_t* graph_exec_arena = 0;
    3397
    0
      ccv_nnc_symbolic_graph_compile(symbolic_graph, ccv_nnc_default_compile_params, 0, 0, 0, 0, SYMBOLIC_GRAPH_SOURCES(symbolic_graph), SYMBOLIC_GRAPH_DESTINATIONS(symbolic_graph), &graph, &tensor_arena, &graph_exec_arena);
    3398
    0
      SYMBOLIC_GRAPH_GEN(symbolic_graph, CCV_NNC_LONG_DOT_GRAPH);
    3399
    0
      GRAPH_GEN(graph, CCV_NNC_LONG_DOT_GRAPH);
    3400
    0
      dsfmt_t dsfmt;
    3401
    0
      float xdata[2 * 2 * 2 * 10];
    3402
    0
      ccv_nnc_tensor_t* const x_tensor = ccv_nnc_tensor_from_symbol(tensor_arena, x);
    3403
    0
      int i;
    3404
    0
      dsfmt_init_gen_rand(&dsfmt, 1);
    3405
    0
      for (i = 0; i < 2 * 2 * 2 * 10; i++)
    3406
    0
        x_tensor->data.f32[i] = xdata[i] = dsfmt_genrand_open_close(&dsfmt);
    3407
    0
      ccv_nnc_graph_run(graph, 0, TRAVERSE_FULL, 0, 0);
    3408
    0
      ccv_nnc_tensor_t* const y_tensor = ccv_nnc_tensor_from_symbol(tensor_arena, y);
    3409
    0
      ccv_nnc_symbolic_graph_free(symbolic_graph);
    3410
    0
      ccv_nnc_graph_exec_arena_free(graph_exec_arena);
    3411
    0
      ccv_nnc_graph_free(graph);
    3412
    0
      ccv_nnc_symbolic_graph_t* const cpu_symbolic_graph = ccv_nnc_symbolic_graph_new();
    3413
    0
      ccv_nnc_tensor_symbol_t cx = ccv_nnc_tensor_symbol_new(cpu_symbolic_graph, CPU_TENSOR_NHWC(32F, 2, 2, 2, 10), "x");
    3414
    0
      ccv_nnc_tensor_symbol_t cy = ccv_nnc_tensor_symbol_new(cpu_symbolic_graph, CPU_TENSOR_NHWC(32F, 2, 2, 2, 10), "y");
    3415
    0
      ccv_nnc_tensor_symbol_t csaved_inv_std = ccv_nnc_tensor_symbol_new(cpu_symbolic_graph, CPU_TENSOR_NHWC(32F, 2, 1, 1, 1), "saved_inv_std");
    3416
    0
      ccv_nnc_graph_exec_symbol_new(cpu_symbolic_graph, CMD_RMSNORM_FORWARD(1e-6, 0, 1, 2, 3), TENSOR_SYMBOL_LIST(cx), TENSOR_SYMBOL_LIST(cy, csaved_inv_std), "rmsnorm");
    3417
    0
      ccv_nnc_graph_exec_symbol_autogen(cpu_symbolic_graph, 0, 0, CCV_NNC_AUTOGEN_ALL_EXECS | CCV_NNC_AUTOGEN_SOURCES_AND_DESTINATIONS);
    3418
    0
      ccv_nnc_graph_t* cpu_graph = 0;
    3419
    0
      ccv_nnc_tensor_arena_t* cpu_tensor_arena = 0;
    3420
    0
      ccv_nnc_graph_exec_arena_t* cpu_graph_exec_arena = 0;
    3421
    0
      ccv_nnc_symbolic_graph_compile(cpu_symbolic_graph, ccv_nnc_default_compile_params, 0, 0, 0, 0, SYMBOLIC_GRAPH_SOURCES(cpu_symbolic_graph), SYMBOLIC_GRAPH_DESTINATIONS(cpu_symbolic_graph), &cpu_graph, &cpu_tensor_arena, &cpu_graph_exec_arena);
    3422
    0
      ccv_nnc_tensor_t* const cx_tensor = ccv_nnc_tensor_from_symbol(cpu_tensor_arena, cx);
    3423
    0
      memcpy(cx_tensor->data.f32, xdata, sizeof(float) * 2 * 2 * 2 * 10);
    3424
    0
      ccv_nnc_graph_run(cpu_graph, 0, TRAVERSE_FULL, 0, 0);
    3425
    0
      ccv_nnc_tensor_t* const cy_tensor = ccv_nnc_tensor_from_symbol(cpu_tensor_arena, cy);
    3426
      // Note that MPS and my other implementations treat epsilon differently.
    3427
    0
      REQUIRE_ARRAY_EQ_WITH_TOLERANCE(float, y_tensor->data.f32, cy_tensor->data.f32, 2 * 2 * 2 * 10, 1e-4, "rmsnorm result from mps should match the one from reference implementation");
    3428
    0
      ccv_nnc_symbolic_graph_free(cpu_symbolic_graph);
    3429
    0
      ccv_nnc_tensor_arena_free(tensor_arena);
    3430
    0
      ccv_nnc_tensor_arena_free(cpu_tensor_arena);
    3431
    0
      ccv_nnc_graph_exec_arena_free(cpu_graph_exec_arena);
    3432
    0
      ccv_nnc_graph_free(cpu_graph);
    3433
    0
    }
    3434
    3435
    TEST_CASE("compare rmsnorm with mps in half precision for qwen shape")
    3436
    1
    {
    3437
    1
      GUARD_ELSE_RETURN(ccv_nnc_cmd_ok(CCV_NNC_RMSNORM_FORWARD, CCV_NNC_BACKEND_MPS) &&
    3438
    1
        ccv_nnc_cmd_ok(CCV_NNC_SET_FORWARD, CCV_NNC_BACKEND_MPS));
    3439
    0
      const int batch_size = 1;
    3440
    0
      const int sequence_length = 9;
    3441
    0
      const int channels = 3584;
    3442
    0
      const int element_count = batch_size * sequence_length * channels;
    3443
    0
      ccv_nnc_symbolic_graph_t* const symbolic_graph = ccv_nnc_symbolic_graph_new();
    3444
    0
      ccv_nnc_tensor_symbol_t a = ccv_nnc_tensor_symbol_new(symbolic_graph, GPU_TENSOR_NHWC(000, 16F, batch_size, sequence_length, channels), "a");
    3445
    0
      ccv_nnc_tensor_symbol_t b = ccv_nnc_tensor_symbol_new(symbolic_graph, GPU_TENSOR_NHWC(000, 16F, batch_size, sequence_length, channels), "b");
    3446
    0
      ccv_nnc_tensor_symbol_t scale = ccv_nnc_tensor_symbol_new(symbolic_graph, GPU_TENSOR_NHWC(000, 16F, 1, 1, channels), "scale");
    3447
    0
      ccv_nnc_tensor_symbol_t saved_inv_std = ccv_nnc_tensor_symbol_new(symbolic_graph, GPU_TENSOR_NHWC(000, 16F, batch_size, sequence_length, 1), "saved_inv_std");
    3448
    0
      ccv_nnc_graph_exec_symbol_new(symbolic_graph, CMD_RMSNORM_FORWARD(1e-6, 1, 2), TENSOR_SYMBOL_LIST(a, scale), TENSOR_SYMBOL_LIST(b, saved_inv_std), "rmsnorm");
    3449
    0
      ccv_nnc_graph_exec_symbol_autogen(symbolic_graph, 0, 0, CCV_NNC_AUTOGEN_ALL_EXECS | CCV_NNC_AUTOGEN_SOURCES_AND_DESTINATIONS);
    3450
    0
      ccv_nnc_graph_t* graph = 0;
    3451
    0
      ccv_nnc_tensor_arena_t* tensor_arena = 0;
    3452
    0
      ccv_nnc_graph_exec_arena_t* graph_exec_arena = 0;
    3453
    0
      ccv_nnc_symbolic_graph_compile(symbolic_graph, ccv_nnc_default_compile_params, 0, 0, 0, 0, SYMBOLIC_GRAPH_SOURCES(symbolic_graph), SYMBOLIC_GRAPH_DESTINATIONS(symbolic_graph), &graph, &tensor_arena, &graph_exec_arena);
    3454
    0
      SYMBOLIC_GRAPH_GEN(symbolic_graph, CCV_NNC_LONG_DOT_GRAPH);
    3455
    0
      GRAPH_GEN(graph, CCV_NNC_LONG_DOT_GRAPH);
    3456
    0
      dsfmt_t dsfmt;
    3457
    0
      dsfmt_init_gen_rand(&dsfmt, 1);
    3458
    0
      int i;
    3459
    0
      ccv_nnc_tensor_t* const x_tensor = ccv_nnc_tensor_new(0, CPU_TENSOR_NHWC(32F, batch_size, sequence_length, channels), 0);
    3460
    0
      for (i = 0; i < element_count; i++)
    3461
    0
        x_tensor->data.f32[i] = (dsfmt_genrand_open_close(&dsfmt) * 2 - 1) * 200;
    3462
    0
      ccv_nnc_tensor_t* const x16_tensor = ccv_nnc_tensor_new(0, CPU_TENSOR_NHWC(16F, batch_size, sequence_length, channels), 0);
    3463
    0
      ccv_nnc_cmd_exec(CMD_DATATYPE_CONVERSION_FORWARD(), ccv_nnc_no_hint, 0, TENSOR_LIST(x_tensor), TENSOR_LIST(x16_tensor), 0);
    3464
    0
      ccv_nnc_tensor_t* const a_tensor = ccv_nnc_tensor_from_symbol(tensor_arena, a);
    3465
    0
      ccv_nnc_cmd_exec(CMD_DATA_TRANSFER_FORWARD(), ccv_nnc_no_hint, 0, TENSOR_LIST(x16_tensor), TENSOR_LIST(a_tensor), 0);
    3466
    0
      ccv_nnc_tensor_t* const scale_tensor = ccv_nnc_tensor_new(0, CPU_TENSOR_NHWC(32F, 1, 1, channels), 0);
    3467
    0
      for (i = 0; i < channels; i++)
    3468
    0
        scale_tensor->data.f32[i] = (dsfmt_genrand_open_close(&dsfmt) * 2 - 1) * 2;
    3469
    0
      ccv_nnc_tensor_t* const scale16_tensor = ccv_nnc_tensor_new(0, CPU_TENSOR_NHWC(16F, 1, 1, channels), 0);
    3470
    0
      ccv_nnc_cmd_exec(CMD_DATATYPE_CONVERSION_FORWARD(), ccv_nnc_no_hint, 0, TENSOR_LIST(scale_tensor), TENSOR_LIST(scale16_tensor), 0);
    3471
    0
      ccv_nnc_cmd_exec(CMD_DATA_TRANSFER_FORWARD(), ccv_nnc_no_hint, 0, TENSOR_LIST(scale16_tensor), TENSOR_LIST(ccv_nnc_tensor_from_symbol(tensor_arena, scale)), 0);
    3472
    0
      ccv_nnc_graph_run(graph, 0, TRAVERSE_FULL, 0, 0);
    3473
    0
      ccv_nnc_tensor_t* const y16_tensor = ccv_nnc_tensor_new(0, CPU_TENSOR_NHWC(16F, batch_size, sequence_length, channels), 0);
    3474
    0
      ccv_nnc_tensor_t* const y_tensor = ccv_nnc_tensor_new(0, CPU_TENSOR_NHWC(32F, batch_size, sequence_length, channels), 0);
    3475
    0
      ccv_nnc_tensor_t* const b_tensor = ccv_nnc_tensor_from_symbol(tensor_arena, b);
    3476
    0
      ccv_nnc_cmd_exec(CMD_DATA_TRANSFER_FORWARD(), ccv_nnc_no_hint, 0, TENSOR_LIST(b_tensor), TENSOR_LIST(y16_tensor), 0);
    3477
    0
      ccv_nnc_cmd_exec(CMD_DATATYPE_CONVERSION_FORWARD(), ccv_nnc_no_hint, 0, TENSOR_LIST(y16_tensor), TENSOR_LIST(y_tensor), 0);
    3478
    0
      ccv_nnc_symbolic_graph_free(symbolic_graph);
    3479
    0
      ccv_nnc_graph_exec_arena_free(graph_exec_arena);
    3480
    0
      ccv_nnc_graph_free(graph);
    3481
    0
      ccv_nnc_symbolic_graph_t* const cpu_symbolic_graph = ccv_nnc_symbolic_graph_new();
    3482
    0
      ccv_nnc_tensor_symbol_t cx = ccv_nnc_tensor_symbol_new(cpu_symbolic_graph, CPU_TENSOR_NHWC(32F, batch_size, sequence_length, channels), "x");
    3483
    0
      ccv_nnc_tensor_symbol_t cy = ccv_nnc_tensor_symbol_new(cpu_symbolic_graph, CPU_TENSOR_NHWC(32F, batch_size, sequence_length, channels), "y");
    3484
    0
      ccv_nnc_tensor_symbol_t cscale = ccv_nnc_tensor_symbol_new(cpu_symbolic_graph, CPU_TENSOR_NHWC(32F, 1, 1, channels), "scale");
    3485
    0
      ccv_nnc_tensor_symbol_t csaved_inv_std = ccv_nnc_tensor_symbol_new(cpu_symbolic_graph, CPU_TENSOR_NHWC(32F, batch_size, sequence_length, 1), "saved_inv_std");
    3486
    0
      ccv_nnc_graph_exec_symbol_new(cpu_symbolic_graph, CMD_RMSNORM_FORWARD(1e-6, 1, 2), TENSOR_SYMBOL_LIST(cx, cscale), TENSOR_SYMBOL_LIST(cy, csaved_inv_std), "rmsnorm");
    3487
    0
      ccv_nnc_graph_exec_symbol_autogen(cpu_symbolic_graph, 0, 0, CCV_NNC_AUTOGEN_ALL_EXECS | CCV_NNC_AUTOGEN_SOURCES_AND_DESTINATIONS);
    3488
    0
      ccv_nnc_graph_t* cpu_graph = 0;
    3489
    0
      ccv_nnc_tensor_arena_t* cpu_tensor_arena = 0;
    3490
    0
      ccv_nnc_graph_exec_arena_t* cpu_graph_exec_arena = 0;
    3491
    0
      ccv_nnc_symbolic_graph_compile(cpu_symbolic_graph, ccv_nnc_default_compile_params, 0, 0, 0, 0, SYMBOLIC_GRAPH_SOURCES(cpu_symbolic_graph), SYMBOLIC_GRAPH_DESTINATIONS(cpu_symbolic_graph), &cpu_graph, &cpu_tensor_arena, &cpu_graph_exec_arena);
    3492
    0
      ccv_nnc_tensor_t* const cx_tensor = ccv_nnc_tensor_from_symbol(cpu_tensor_arena, cx);
    3493
    0
      memcpy(cx_tensor->data.f32, x_tensor->data.f32, sizeof(float) * element_count);
    3494
    0
      ccv_nnc_tensor_t* const cscale_tensor = ccv_nnc_tensor_from_symbol(cpu_tensor_arena, cscale);
    3495
    0
      memcpy(cscale_tensor->data.f32, scale_tensor->data.f32, sizeof(float) * channels);
    3496
    0
      ccv_nnc_graph_run(cpu_graph, 0, TRAVERSE_FULL, 0, 0);
    3497
    0
      ccv_nnc_tensor_t* const cy_tensor = ccv_nnc_tensor_from_symbol(cpu_tensor_arena, cy);
    3498
    0
      REQUIRE_ARRAY_EQ_WITH_TOLERANCE(float, y_tensor->data.f32, cy_tensor->data.f32, element_count, 1e-2, "qwen-like half precision rmsnorm result from mps should match the CPU reference implementation");
    3499
    0
      ccv_nnc_symbolic_graph_free(cpu_symbolic_graph);
    3500
    0
      ccv_nnc_tensor_arena_free(tensor_arena);
    3501
    0
      ccv_nnc_tensor_arena_free(cpu_tensor_arena);
    3502
    0
      ccv_nnc_graph_exec_arena_free(cpu_graph_exec_arena);
    3503
    0
      ccv_nnc_graph_free(cpu_graph);
    3504
    0
      ccv_nnc_tensor_free(x_tensor);
    3505
    0
      ccv_nnc_tensor_free(x16_tensor);
    3506
    0
      ccv_nnc_tensor_free(y16_tensor);
    3507
    0
      ccv_nnc_tensor_free(y_tensor);
    3508
    0
      ccv_nnc_tensor_free(scale_tensor);
    3509
    0
      ccv_nnc_tensor_free(scale16_tensor);
    3510
    0
    }
    3511
    3512
    TEST_CASE("compare layer norm with mps in bfloat precision through mfa and graph")
    3513
    1
    {
    3514
    1
      GUARD_ELSE_RETURN(ccv_nnc_cmd_ok(CCV_NNC_LAYER_NORM_FORWARD, CCV_NNC_BACKEND_MPS));
    3515
    0
      const int batch_size = 2;
    3516
    0
      const int sequence_length = 3;
    3517
    0
      const int channels = 257;
    3518
    0
      const int rows = batch_size * sequence_length;
    3519
    0
      const int element_count = rows * channels;
    3520
    0
      ccv_nnc_tensor_t* const a = ccv_nnc_tensor_new(0, GPU_TENSOR_NHWC(000, 16BF, batch_size, sequence_length, channels), 0);
    3521
    0
      ccv_nnc_tensor_t* const b = ccv_nnc_tensor_new(0, GPU_TENSOR_NHWC(000, 16BF, batch_size, sequence_length, channels), 0);
    3522
    0
      ccv_nnc_tensor_t* const scale = ccv_nnc_tensor_new(0, GPU_TENSOR_NHWC(000, 16BF, 1, 1, channels), 0);
    3523
    0
      ccv_nnc_tensor_t* const bias = ccv_nnc_tensor_new(0, GPU_TENSOR_NHWC(000, 16BF, 1, 1, channels), 0);
    3524
    0
      ccv_nnc_tensor_t* const saved_mean = ccv_nnc_tensor_new(0, GPU_TENSOR_NHWC(000, 16BF, batch_size, sequence_length, 1), 0);
    3525
    0
      ccv_nnc_tensor_t* const saved_inv_std = ccv_nnc_tensor_new(0, GPU_TENSOR_NHWC(000, 16BF, batch_size, sequence_length, 1), 0);
    3526
    0
      ccv_nnc_tensor_t* const ha = ccv_nnc_tensor_new(0, CPU_TENSOR_NHWC(32F, batch_size, sequence_length, channels), 0);
    3527
    0
      ccv_nnc_tensor_t* const hscale = ccv_nnc_tensor_new(0, CPU_TENSOR_NHWC(32F, 1, 1, channels), 0);
    3528
    0
      ccv_nnc_tensor_t* const hbias = ccv_nnc_tensor_new(0, CPU_TENSOR_NHWC(32F, 1, 1, channels), 0);
    3529
    0
      ccv_nnc_tensor_t* const ha16bf = ccv_nnc_tensor_new(0, CPU_TENSOR_NHWC(16BF, batch_size, sequence_length, channels), 0);
    3530
    0
      ccv_nnc_tensor_t* const hscale16bf = ccv_nnc_tensor_new(0, CPU_TENSOR_NHWC(16BF, 1, 1, channels), 0);
    3531
    0
      ccv_nnc_tensor_t* const hbias16bf = ccv_nnc_tensor_new(0, CPU_TENSOR_NHWC(16BF, 1, 1, channels), 0);
    3532
    0
      ccv_nnc_tensor_t* const hy = ccv_nnc_tensor_new(0, CPU_TENSOR_NHWC(32F, batch_size, sequence_length, channels), 0);
    3533
    0
      ccv_nnc_tensor_t* const hmean = ccv_nnc_tensor_new(0, CPU_TENSOR_NHWC(32F, batch_size, sequence_length, 1), 0);
    3534
    0
      ccv_nnc_tensor_t* const hinv = ccv_nnc_tensor_new(0, CPU_TENSOR_NHWC(32F, batch_size, sequence_length, 1), 0);
    3535
    0
      ccv_nnc_tensor_t* const hy16bf = ccv_nnc_tensor_new(0, CPU_TENSOR_NHWC(16BF, batch_size, sequence_length, channels), 0);
    3536
    0
      ccv_nnc_tensor_t* const hmean16bf = ccv_nnc_tensor_new(0, CPU_TENSOR_NHWC(16BF, batch_size, sequence_length, 1), 0);
    3537
    0
      ccv_nnc_tensor_t* const hinv16bf = ccv_nnc_tensor_new(0, CPU_TENSOR_NHWC(16BF, batch_size, sequence_length, 1), 0);
    3538
    0
      ccv_nnc_tensor_t* const expected_y = ccv_nnc_tensor_new(0, CPU_TENSOR_NHWC(32F, batch_size, sequence_length, channels), 0);
    3539
    0
      ccv_nnc_tensor_t* const expected_mean = ccv_nnc_tensor_new(0, CPU_TENSOR_NHWC(32F, batch_size, sequence_length, 1), 0);
    3540
    0
      ccv_nnc_tensor_t* const expected_inv = ccv_nnc_tensor_new(0, CPU_TENSOR_NHWC(32F, batch_size, sequence_length, 1), 0);
    3541
    0
      ccv_nnc_tensor_t* const expected_y16bf = ccv_nnc_tensor_new(0, CPU_TENSOR_NHWC(16BF, batch_size, sequence_length, channels), 0);
    3542
    0
      ccv_nnc_tensor_t* const expected_mean16bf = ccv_nnc_tensor_new(0, CPU_TENSOR_NHWC(16BF, batch_size, sequence_length, 1), 0);
    3543
    0
      ccv_nnc_tensor_t* const expected_inv16bf = ccv_nnc_tensor_new(0, CPU_TENSOR_NHWC(16BF, batch_size, sequence_length, 1), 0);
    3544
    0
      dsfmt_t dsfmt;
    3545
    0
      dsfmt_init_gen_rand(&dsfmt, 21);
    3546
    0
      int i, j;
    3547
    0
      for (i = 0; i < element_count; i++)
    3548
    0
        ha->data.f32[i] = (dsfmt_genrand_open_close(&dsfmt) * 2 - 1) * 3;
    3549
    0
      for (i = 0; i < channels; i++)
    3550
    0
      {
    3551
    0
        hscale->data.f32[i] = (dsfmt_genrand_open_close(&dsfmt) * 2 - 1) * 2;
    3552
    0
        hbias->data.f32[i] = (dsfmt_genrand_open_close(&dsfmt) * 2 - 1) * 0.25;
    3553
    0
      }
    3554
    0
      ccv_float_to_bfloat(ha->data.f32, (uint16_t*)ha16bf->data.f16, element_count);
    3555
    0
      ccv_float_to_bfloat(hscale->data.f32, (uint16_t*)hscale16bf->data.f16, channels);
    3556
    0
      ccv_float_to_bfloat(hbias->data.f32, (uint16_t*)hbias16bf->data.f16, channels);
    3557
    0
      ccv_bfloat_to_float((uint16_t*)ha16bf->data.f16, ha->data.f32, element_count);
    3558
    0
      ccv_bfloat_to_float((uint16_t*)hscale16bf->data.f16, hscale->data.f32, channels);
    3559
    0
      ccv_bfloat_to_float((uint16_t*)hbias16bf->data.f16, hbias->data.f32, channels);
    3560
    0
      ccv_nnc_cmd_exec(CMD_DATA_TRANSFER_FORWARD(), ccv_nnc_no_hint, 0, TENSOR_LIST(ha16bf, hscale16bf, hbias16bf), TENSOR_LIST(a, scale, bias), 0);
    3561
    0
      for (i = 0; i < rows; i++)
    3562
    0
      {
    3563
    0
        float mean = 0;
    3564
    0
        for (j = 0; j < channels; j++)
    3565
    0
          mean += ha->data.f32[i * channels + j];
    3566
    0
        mean = mean / channels;
    3567
    0
        float variance = 0;
    3568
    0
        for (j = 0; j < channels; j++)
    3569
    0
        {
    3570
    0
          const float centered = ha->data.f32[i * channels + j] - mean;
    3571
    0
          variance += centered * centered;
    3572
    0
        }
    3573
    0
        const float inv_std = 1.0f / sqrtf(variance / channels + 1e-6f);
    3574
    0
        expected_mean->data.f32[i] = mean;
    3575
    0
        expected_inv->data.f32[i] = inv_std;
    3576
    0
        for (j = 0; j < channels; j++)
    3577
    0
          expected_y->data.f32[i * channels + j] = (ha->data.f32[i * channels + j] - mean) * inv_std * hscale->data.f32[j] + hbias->data.f32[j];
    3578
    0
      }
    3579
    0
      ccv_float_to_bfloat(expected_y->data.f32, (uint16_t*)expected_y16bf->data.f16, element_count);
    3580
    0
      ccv_float_to_bfloat(expected_mean->data.f32, (uint16_t*)expected_mean16bf->data.f16, rows);
    3581
    0
      ccv_float_to_bfloat(expected_inv->data.f32, (uint16_t*)expected_inv16bf->data.f16, rows);
    3582
    0
      ccv_bfloat_to_float((uint16_t*)expected_y16bf->data.f16, expected_y->data.f32, element_count);
    3583
    0
      ccv_bfloat_to_float((uint16_t*)expected_mean16bf->data.f16, expected_mean->data.f32, rows);
    3584
    0
      ccv_bfloat_to_float((uint16_t*)expected_inv16bf->data.f16, expected_inv->data.f32, rows);
    3585
    0
      ccv_nnc_cmd_t cmd = CMD_LAYER_NORM_FORWARD(1e-6, 1, 2);
    3586
    0
      cmd.backend = CCV_NNC_BACKEND_MPS;
    3587
    0
      const uint64_t old_flags = ccv_nnc_flags();
    3588
    0
      if (old_flags & CCV_NNC_DISABLE_MFA)
    3589
    0
        ccv_nnc_disable_flag(CCV_NNC_DISABLE_MFA);
    3590
    0
      REQUIRE_EQ(CCV_NNC_EXEC_SUCCESS, ccv_nnc_cmd_exec(cmd, ccv_nnc_no_hint, 0, TENSOR_LIST(a, scale, bias), TENSOR_LIST(b, saved_mean, saved_inv_std), 0), "bfloat layer norm mfa should run");
    3591
    0
      ccv_nnc_cmd_exec(CMD_DATA_TRANSFER_FORWARD(), ccv_nnc_no_hint, 0, TENSOR_LIST(b, saved_mean, saved_inv_std), TENSOR_LIST(hy16bf, hmean16bf, hinv16bf), 0);
    3592
    0
      ccv_bfloat_to_float((uint16_t*)hy16bf->data.f16, hy->data.f32, element_count);
    3593
    0
      ccv_bfloat_to_float((uint16_t*)hmean16bf->data.f16, hmean->data.f32, rows);
    3594
    0
      ccv_bfloat_to_float((uint16_t*)hinv16bf->data.f16, hinv->data.f32, rows);
    3595
    0
      REQUIRE_ARRAY_EQ_WITH_TOLERANCE(float, hy->data.f32, expected_y->data.f32, element_count, 2e-2, "bfloat layer norm result from mfa should match fp32 reference rounded to bfloat");
    3596
    0
      REQUIRE_ARRAY_EQ_WITH_TOLERANCE(float, hmean->data.f32, expected_mean->data.f32, rows, 2e-2, "bfloat layer norm saved mean from mfa should match fp32 reference rounded to bfloat");
    3597
    0
      REQUIRE_ARRAY_EQ_WITH_TOLERANCE(float, hinv->data.f32, expected_inv->data.f32, rows, 2e-2, "bfloat layer norm saved inv std from mfa should match fp32 reference rounded to bfloat");
    3598
    0
      ccv_nnc_enable_flag(CCV_NNC_DISABLE_MFA);
    3599
    0
      REQUIRE_EQ(CCV_NNC_EXEC_SUCCESS, ccv_nnc_cmd_exec(cmd, ccv_nnc_no_hint, 0, TENSOR_LIST(a, scale, bias), TENSOR_LIST(b, saved_mean, saved_inv_std), 0), "bfloat layer norm graph fallback should run");
    3600
    0
      ccv_nnc_cmd_exec(CMD_DATA_TRANSFER_FORWARD(), ccv_nnc_no_hint, 0, TENSOR_LIST(b, saved_mean, saved_inv_std), TENSOR_LIST(hy16bf, hmean16bf, hinv16bf), 0);
    3601
    0
      ccv_bfloat_to_float((uint16_t*)hy16bf->data.f16, hy->data.f32, element_count);
    3602
    0
      ccv_bfloat_to_float((uint16_t*)hmean16bf->data.f16, hmean->data.f32, rows);
    3603
    0
      ccv_bfloat_to_float((uint16_t*)hinv16bf->data.f16, hinv->data.f32, rows);
    3604
    0
      REQUIRE_ARRAY_EQ_WITH_TOLERANCE(float, hy->data.f32, expected_y->data.f32, element_count, 2e-2, "bfloat layer norm result from graph fallback should match fp32 reference rounded to bfloat");
    3605
    0
      REQUIRE_ARRAY_EQ_WITH_TOLERANCE(float, hmean->data.f32, expected_mean->data.f32, rows, 2e-2, "bfloat layer norm saved mean from graph fallback should match fp32 reference rounded to bfloat");
    3606
    0
      REQUIRE_ARRAY_EQ_WITH_TOLERANCE(float, hinv->data.f32, expected_inv->data.f32, rows, 2e-2, "bfloat layer norm saved inv std from graph fallback should match fp32 reference rounded to bfloat");
    3607
    0
      if (old_flags & CCV_NNC_DISABLE_MFA)
    3608
    0
        ccv_nnc_enable_flag(CCV_NNC_DISABLE_MFA);
    3609
    0
      else
    3610
    0
        ccv_nnc_disable_flag(CCV_NNC_DISABLE_MFA);
    3611
    0
      ccv_nnc_tensor_free(a);
    3612
    0
      ccv_nnc_tensor_free(b);
    3613
    0
      ccv_nnc_tensor_free(scale);
    3614
    0
      ccv_nnc_tensor_free(bias);
    3615
    0
      ccv_nnc_tensor_free(saved_mean);
    3616
    0
      ccv_nnc_tensor_free(saved_inv_std);
    3617
    0
      ccv_nnc_tensor_free(ha);
    3618
    0
      ccv_nnc_tensor_free(hscale);
    3619
    0
      ccv_nnc_tensor_free(hbias);
    3620
    0
      ccv_nnc_tensor_free(ha16bf);
    3621
    0
      ccv_nnc_tensor_free(hscale16bf);
    3622
    0
      ccv_nnc_tensor_free(hbias16bf);
    3623
    0
      ccv_nnc_tensor_free(hy);
    3624
    0
      ccv_nnc_tensor_free(hmean);
    3625
    0
      ccv_nnc_tensor_free(hinv);
    3626
    0
      ccv_nnc_tensor_free(hy16bf);
    3627
    0
      ccv_nnc_tensor_free(hmean16bf);
    3628
    0
      ccv_nnc_tensor_free(hinv16bf);
    3629
    0
      ccv_nnc_tensor_free(expected_y);
    3630
    0
      ccv_nnc_tensor_free(expected_mean);
    3631
    0
      ccv_nnc_tensor_free(expected_inv);
    3632
    0
      ccv_nnc_tensor_free(expected_y16bf);
    3633
    0
      ccv_nnc_tensor_free(expected_mean16bf);
    3634
    0
      ccv_nnc_tensor_free(expected_inv16bf);
    3635
    0
    }
    3636
    3637
    TEST_CASE("compare rmsnorm with mps in bfloat precision through mfa and graph")
    3638
    1
    {
    3639
    1
      GUARD_ELSE_RETURN(ccv_nnc_cmd_ok(CCV_NNC_RMSNORM_FORWARD, CCV_NNC_BACKEND_MPS));
    3640
    0
      const int batch_size = 2;
    3641
    0
      const int sequence_length = 3;
    3642
    0
      const int channels = 257;
    3643
    0
      const int rows = batch_size * sequence_length;
    3644
    0
      const int element_count = rows * channels;
    3645
    0
      ccv_nnc_tensor_t* const a = ccv_nnc_tensor_new(0, GPU_TENSOR_NHWC(000, 16BF, batch_size, sequence_length, channels), 0);
    3646
    0
      ccv_nnc_tensor_t* const b = ccv_nnc_tensor_new(0, GPU_TENSOR_NHWC(000, 16BF, batch_size, sequence_length, channels), 0);
    3647
    0
      ccv_nnc_tensor_t* const scale = ccv_nnc_tensor_new(0, GPU_TENSOR_NHWC(000, 16BF, 1, 1, channels), 0);
    3648
    0
      ccv_nnc_tensor_t* const saved_inv_std = ccv_nnc_tensor_new(0, GPU_TENSOR_NHWC(000, 16BF, batch_size, sequence_length, 1), 0);
    3649
    0
      ccv_nnc_tensor_t* const ha = ccv_nnc_tensor_new(0, CPU_TENSOR_NHWC(32F, batch_size, sequence_length, channels), 0);
    3650
    0
      ccv_nnc_tensor_t* const hscale = ccv_nnc_tensor_new(0, CPU_TENSOR_NHWC(32F, 1, 1, channels), 0);
    3651
    0
      ccv_nnc_tensor_t* const ha16bf = ccv_nnc_tensor_new(0, CPU_TENSOR_NHWC(16BF, batch_size, sequence_length, channels), 0);
    3652
    0
      ccv_nnc_tensor_t* const hscale16bf = ccv_nnc_tensor_new(0, CPU_TENSOR_NHWC(16BF, 1, 1, channels), 0);
    3653
    0
      ccv_nnc_tensor_t* const hy = ccv_nnc_tensor_new(0, CPU_TENSOR_NHWC(32F, batch_size, sequence_length, channels), 0);
    3654
    0
      ccv_nnc_tensor_t* const hinv = ccv_nnc_tensor_new(0, CPU_TENSOR_NHWC(32F, batch_size, sequence_length, 1), 0);
    3655
    0
      ccv_nnc_tensor_t* const hy16bf = ccv_nnc_tensor_new(0, CPU_TENSOR_NHWC(16BF, batch_size, sequence_length, channels), 0);
    3656
    0
      ccv_nnc_tensor_t* const hinv16bf = ccv_nnc_tensor_new(0, CPU_TENSOR_NHWC(16BF, batch_size, sequence_length, 1), 0);
    3657
    0
      ccv_nnc_tensor_t* const expected_y = ccv_nnc_tensor_new(0, CPU_TENSOR_NHWC(32F, batch_size, sequence_length, channels), 0);
    3658
    0
      ccv_nnc_tensor_t* const expected_inv = ccv_nnc_tensor_new(0, CPU_TENSOR_NHWC(32F, batch_size, sequence_length, 1), 0);
    3659
    0
      ccv_nnc_tensor_t* const expected_y16bf = ccv_nnc_tensor_new(0, CPU_TENSOR_NHWC(16BF, batch_size, sequence_length, channels), 0);
    3660
    0
      ccv_nnc_tensor_t* const expected_inv16bf = ccv_nnc_tensor_new(0, CPU_TENSOR_NHWC(16BF, batch_size, sequence_length, 1), 0);
    3661
    0
      dsfmt_t dsfmt;
    3662
    0
      dsfmt_init_gen_rand(&dsfmt, 22);
    3663
    0
      int i, j;
    3664
    0
      for (i = 0; i < element_count; i++)
    3665
    0
        ha->data.f32[i] = (dsfmt_genrand_open_close(&dsfmt) * 2 - 1) * 3;
    3666
    0
      for (i = 0; i < channels; i++)
    3667
    0
        hscale->data.f32[i] = (dsfmt_genrand_open_close(&dsfmt) * 2 - 1) * 2;
    3668
    0
      ccv_float_to_bfloat(ha->data.f32, (uint16_t*)ha16bf->data.f16, element_count);
    3669
    0
      ccv_float_to_bfloat(hscale->data.f32, (uint16_t*)hscale16bf->data.f16, channels);
    3670
    0
      ccv_bfloat_to_float((uint16_t*)ha16bf->data.f16, ha->data.f32, element_count);
    3671
    0
      ccv_bfloat_to_float((uint16_t*)hscale16bf->data.f16, hscale->data.f32, channels);
    3672
    0
      ccv_nnc_cmd_exec(CMD_DATA_TRANSFER_FORWARD(), ccv_nnc_no_hint, 0, TENSOR_LIST(ha16bf, hscale16bf), TENSOR_LIST(a, scale), 0);
    3673
    0
      for (i = 0; i < rows; i++)
    3674
    0
      {
    3675
    0
        float variance = 0;
    3676
    0
        for (j = 0; j < channels; j++)
    3677
    0
        {
    3678
    0
          const float v = ha->data.f32[i * channels + j];
    3679
    0
          variance += v * v;
    3680
    0
        }
    3681
    0
        const float inv_std = 1.0f / sqrtf(variance / channels + 1e-6f);
    3682
    0
        expected_inv->data.f32[i] = inv_std;
    3683
    0
        for (j = 0; j < channels; j++)
    3684
    0
          expected_y->data.f32[i * channels + j] = ha->data.f32[i * channels + j] * inv_std * hscale->data.f32[j];
    3685
    0
      }
    3686
    0
      ccv_float_to_bfloat(expected_y->data.f32, (uint16_t*)expected_y16bf->data.f16, element_count);
    3687
    0
      ccv_float_to_bfloat(expected_inv->data.f32, (uint16_t*)expected_inv16bf->data.f16, rows);
    3688
    0
      ccv_bfloat_to_float((uint16_t*)expected_y16bf->data.f16, expected_y->data.f32, element_count);
    3689
    0
      ccv_bfloat_to_float((uint16_t*)expected_inv16bf->data.f16, expected_inv->data.f32, rows);
    3690
    0
      ccv_nnc_cmd_t cmd = CMD_RMSNORM_FORWARD(1e-6, 1, 2);
    3691
    0
      cmd.backend = CCV_NNC_BACKEND_MPS;
    3692
    0
      const uint64_t old_flags = ccv_nnc_flags();
    3693
    0
      if (old_flags & CCV_NNC_DISABLE_MFA)
    3694
    0
        ccv_nnc_disable_flag(CCV_NNC_DISABLE_MFA);
    3695
    0
      REQUIRE_EQ(CCV_NNC_EXEC_SUCCESS, ccv_nnc_cmd_exec(cmd, ccv_nnc_no_hint, 0, TENSOR_LIST(a, scale), TENSOR_LIST(b, saved_inv_std), 0), "bfloat rmsnorm mfa should run");
    3696
    0
      ccv_nnc_cmd_exec(CMD_DATA_TRANSFER_FORWARD(), ccv_nnc_no_hint, 0, TENSOR_LIST(b, saved_inv_std), TENSOR_LIST(hy16bf, hinv16bf), 0);
    3697
    0
      ccv_bfloat_to_float((uint16_t*)hy16bf->data.f16, hy->data.f32, element_count);
    3698
    0
      ccv_bfloat_to_float((uint16_t*)hinv16bf->data.f16, hinv->data.f32, rows);
    3699
    0
      REQUIRE_ARRAY_EQ_WITH_TOLERANCE(float, hy->data.f32, expected_y->data.f32, element_count, 2e-2, "bfloat rmsnorm result from mfa should match fp32 reference rounded to bfloat");
    3700
    0
      REQUIRE_ARRAY_EQ_WITH_TOLERANCE(float, hinv->data.f32, expected_inv->data.f32, rows, 2e-2, "bfloat rmsnorm saved inv std from mfa should match fp32 reference rounded to bfloat");
    3701
    0
      ccv_nnc_enable_flag(CCV_NNC_DISABLE_MFA);
    3702
    0
      REQUIRE_EQ(CCV_NNC_EXEC_SUCCESS, ccv_nnc_cmd_exec(cmd, ccv_nnc_no_hint, 0, TENSOR_LIST(a, scale), TENSOR_LIST(b, saved_inv_std), 0), "bfloat rmsnorm graph fallback should run");
    3703
    0
      ccv_nnc_cmd_exec(CMD_DATA_TRANSFER_FORWARD(), ccv_nnc_no_hint, 0, TENSOR_LIST(b, saved_inv_std), TENSOR_LIST(hy16bf, hinv16bf), 0);
    3704
    0
      ccv_bfloat_to_float((uint16_t*)hy16bf->data.f16, hy->data.f32, element_count);
    3705
    0
      ccv_bfloat_to_float((uint16_t*)hinv16bf->data.f16, hinv->data.f32, rows);
    3706
    0
      REQUIRE_ARRAY_EQ_WITH_TOLERANCE(float, hy->data.f32, expected_y->data.f32, element_count, 2e-2, "bfloat rmsnorm result from graph fallback should match fp32 reference rounded to bfloat");
    3707
    0
      REQUIRE_ARRAY_EQ_WITH_TOLERANCE(float, hinv->data.f32, expected_inv->data.f32, rows, 2e-2, "bfloat rmsnorm saved inv std from graph fallback should match fp32 reference rounded to bfloat");
    3708
    0
      if (old_flags & CCV_NNC_DISABLE_MFA)
    3709
    0
        ccv_nnc_enable_flag(CCV_NNC_DISABLE_MFA);
    3710
    0
      else
    3711
    0
        ccv_nnc_disable_flag(CCV_NNC_DISABLE_MFA);
    3712
    0
      ccv_nnc_tensor_free(a);
    3713
    0
      ccv_nnc_tensor_free(b);
    3714
    0
      ccv_nnc_tensor_free(scale);
    3715
    0
      ccv_nnc_tensor_free(saved_inv_std);
    3716
    0
      ccv_nnc_tensor_free(ha);
    3717
    0
      ccv_nnc_tensor_free(hscale);
    3718
    0
      ccv_nnc_tensor_free(ha16bf);
    3719
    0
      ccv_nnc_tensor_free(hscale16bf);
    3720
    0
      ccv_nnc_tensor_free(hy);
    3721
    0
      ccv_nnc_tensor_free(hinv);
    3722
    0
      ccv_nnc_tensor_free(hy16bf);
    3723
    0
      ccv_nnc_tensor_free(hinv16bf);
    3724
    0
      ccv_nnc_tensor_free(expected_y);
    3725
    0
      ccv_nnc_tensor_free(expected_inv);
    3726
    0
      ccv_nnc_tensor_free(expected_y16bf);
    3727
    0
      ccv_nnc_tensor_free(expected_inv16bf);
    3728
    0
    }
    3729
    3730
    TEST_CASE("compare add with mps")
    3731
    1
    {
    3732
    1
      GUARD_ELSE_RETURN(ccv_nnc_cmd_ok(CCV_NNC_ADD_FORWARD, CCV_NNC_BACKEND_MPS));
    3733
    0
      ccv_nnc_symbolic_graph_t* const symbolic_graph = ccv_nnc_symbolic_graph_new();
    3734
    0
      ccv_nnc_tensor_symbol_t x = ccv_nnc_tensor_symbol_new(symbolic_graph, CPU_TENSOR_NHWC(32F, 10, 5, 5, 3), "x");
    3735
    0
      ccv_nnc_tensor_symbol_t y = ccv_nnc_tensor_symbol_new(symbolic_graph, CPU_TENSOR_NHWC(32F, 10, 5, 1, 3), "y");
    3736
    0
      ccv_nnc_tensor_symbol_t a = ccv_nnc_tensor_symbol_new(symbolic_graph, GPU_TENSOR_NHWC(000, 32F, 10, 5, 5, 3), "a");
    3737
    0
      ccv_nnc_tensor_symbol_t b = ccv_nnc_tensor_symbol_new(symbolic_graph, GPU_TENSOR_NHWC(000, 32F, 10, 5, 1, 3), "b");
    3738
    0
      ccv_nnc_tensor_symbol_t c = ccv_nnc_tensor_symbol_new(symbolic_graph, GPU_TENSOR_NHWC(000, 32F, 10, 5, 5, 3), "c");
    3739
    0
      ccv_nnc_tensor_symbol_t z = ccv_nnc_tensor_symbol_new(symbolic_graph, CPU_TENSOR_NHWC(32F, 10, 5, 5, 3), "z");
    3740
    0
      ccv_nnc_graph_exec_symbol_new(symbolic_graph, CMD_DATA_TRANSFER_FORWARD(), TENSOR_SYMBOL_LIST(x, y), TENSOR_SYMBOL_LIST(a, b), "transfer");
    3741
    0
      ccv_nnc_graph_exec_symbol_new(symbolic_graph, CMD_ADD_FORWARD(0.5, 0.2), TENSOR_SYMBOL_LIST(a, b), TENSOR_SYMBOL_LIST(c), "add");
    3742
    0
      ccv_nnc_graph_exec_symbol_new(symbolic_graph, CMD_DATA_TRANSFER_FORWARD(), TENSOR_SYMBOL_LIST(c), TENSOR_SYMBOL_LIST(z), "transfer");
    3743
    0
      ccv_nnc_graph_exec_symbol_autogen(symbolic_graph, 0, 0, CCV_NNC_AUTOGEN_ALL_EXECS | CCV_NNC_AUTOGEN_SOURCES_AND_DESTINATIONS);
    3744
    0
      SYMBOLIC_GRAPH_GEN(symbolic_graph, CCV_NNC_LONG_DOT_GRAPH);
    3745
    0
      ccv_nnc_tensor_t* const x_tensor = ccv_nnc_tensor_new(0, CPU_TENSOR_NHWC(32F, 10, 5, 5, 3), 0);
    3746
    0
      ccv_nnc_tensor_t* const y_tensor = ccv_nnc_tensor_new(0, CPU_TENSOR_NHWC(32F, 10, 5, 1, 3), 0);
    3747
    0
      ccv_nnc_graph_t* graph = 0;
    3748
    0
      ccv_nnc_tensor_arena_t* tensor_arena = 0;
    3749
    0
      ccv_nnc_graph_exec_arena_t* graph_exec_arena = 0;
    3750
    0
      ccv_nnc_symbolic_graph_compile(symbolic_graph, ccv_nnc_default_compile_params, TENSOR_BIND_MAP(KV(x, x_tensor), KV(y, y_tensor)), TENSOR_SYMBOL_LIST(z), SYMBOLIC_GRAPH_SOURCES(symbolic_graph), SYMBOLIC_GRAPH_DESTINATIONS(symbolic_graph), &graph, &tensor_arena, &graph_exec_arena);
    3751
    0
      GRAPH_GEN(graph, CCV_NNC_LONG_DOT_GRAPH);
    3752
    0
      dsfmt_t dsfmt;
    3753
    0
      dsfmt_init_gen_rand(&dsfmt, 0);
    3754
    0
      int i;
    3755
    0
      for (i = 0; i < 10 * 5 * 5 * 3; i++)
    3756
    0
        x_tensor->data.f32[i] = dsfmt_genrand_open_close(&dsfmt);
    3757
    0
      for (i = 0; i < 10 * 5 * 1 * 3; i++)
    3758
    0
        y_tensor->data.f32[i] = dsfmt_genrand_open_close(&dsfmt);
    3759
    0
      ccv_nnc_tensor_t* zt = ccv_nnc_tensor_new(0, CPU_TENSOR_NHWC(32F, 10, 5, 5, 3), 0);
    3760
    0
      ccv_nnc_cmd_exec(CMD_ADD_FORWARD(0.5, 0.2), ccv_nnc_no_hint, 0, TENSOR_LIST(x_tensor, y_tensor), TENSOR_LIST(zt), 0);
    3761
    0
      ccv_nnc_graph_run(graph, 0, TRAVERSE_FULL, 0, 0);
    3762
    0
      ccv_nnc_tensor_t* const z_tensor = ccv_nnc_tensor_from_symbol(tensor_arena, z);
    3763
    0
      REQUIRE_TENSOR_EQ(zt, z_tensor, "add should match");
    3764
    0
      ccv_nnc_tensor_free(x_tensor);
    3765
    0
      ccv_nnc_tensor_free(y_tensor);
    3766
    0
      ccv_nnc_tensor_free(zt);
    3767
    0
      ccv_nnc_graph_free(graph);
    3768
    0
      ccv_nnc_tensor_arena_free(tensor_arena);
    3769
    0
      ccv_nnc_graph_exec_arena_free(graph_exec_arena);
    3770
    0
      ccv_nnc_symbolic_graph_free(symbolic_graph);
    3771
    0
    }
    3772
    3773
    TEST_CASE("compare add with mps in half precision")
    3774
    1
    {
    3775
    1
      GUARD_ELSE_RETURN(ccv_nnc_cmd_ok(CCV_NNC_ADD_FORWARD, CCV_NNC_BACKEND_MPS));
    3776
    0
      ccv_nnc_symbolic_graph_t* const symbolic_graph = ccv_nnc_symbolic_graph_new();
    3777
    0
      ccv_nnc_tensor_symbol_t x = ccv_nnc_tensor_symbol_new(symbolic_graph, CPU_TENSOR_NHWC(32F, 10, 5, 5, 3), "x");
    3778
    0
      ccv_nnc_tensor_symbol_t y = ccv_nnc_tensor_symbol_new(symbolic_graph, CPU_TENSOR_NHWC(32F, 10, 5, 1, 3), "y");
    3779
    0
      ccv_nnc_tensor_symbol_t x16 = ccv_nnc_tensor_symbol_new(symbolic_graph, CPU_TENSOR_NHWC(16F, 10, 5, 5, 3), "x 16");
    3780
    0
      ccv_nnc_tensor_symbol_t y16 = ccv_nnc_tensor_symbol_new(symbolic_graph, CPU_TENSOR_NHWC(16F, 10, 5, 1, 3), "y 16");
    3781
    0
      ccv_nnc_tensor_symbol_t a = ccv_nnc_tensor_symbol_new(symbolic_graph, GPU_TENSOR_NHWC(000, 16F, 10, 5, 5, 3), "a");
    3782
    0
      ccv_nnc_tensor_symbol_t b = ccv_nnc_tensor_symbol_new(symbolic_graph, GPU_TENSOR_NHWC(000, 16F, 10, 5, 1, 3), "b");
    3783
    0
      ccv_nnc_tensor_symbol_t c = ccv_nnc_tensor_symbol_new(symbolic_graph, GPU_TENSOR_NHWC(000, 16F, 10, 5, 5, 3), "c");
    3784
    0
      ccv_nnc_tensor_symbol_t z = ccv_nnc_tensor_symbol_new(symbolic_graph, CPU_TENSOR_NHWC(32F, 10, 5, 5, 3), "z");
    3785
    0
      ccv_nnc_tensor_symbol_t z16 = ccv_nnc_tensor_symbol_new(symbolic_graph, CPU_TENSOR_NHWC(16F, 10, 5, 5, 3), "z 16");
    3786
    0
      ccv_nnc_graph_exec_symbol_new(symbolic_graph, CMD_DATATYPE_CONVERSION_FORWARD(), TENSOR_SYMBOL_LIST(x, y), TENSOR_SYMBOL_LIST(x16, y16), "convert");
    3787
    0
      ccv_nnc_graph_exec_symbol_new(symbolic_graph, CMD_DATA_TRANSFER_FORWARD(), TENSOR_SYMBOL_LIST(x16, y16), TENSOR_SYMBOL_LIST(a, b), "transfer");
    3788
    0
      ccv_nnc_graph_exec_symbol_new(symbolic_graph, CMD_ADD_FORWARD(0.5, 0.2), TENSOR_SYMBOL_LIST(a, b), TENSOR_SYMBOL_LIST(c), "add");
    3789
    0
      ccv_nnc_graph_exec_symbol_new(symbolic_graph, CMD_DATA_TRANSFER_FORWARD(), TENSOR_SYMBOL_LIST(c), TENSOR_SYMBOL_LIST(z16), "transfer");
    3790
    0
      ccv_nnc_graph_exec_symbol_new(symbolic_graph, CMD_DATATYPE_CONVERSION_FORWARD(), TENSOR_SYMBOL_LIST(z16), TENSOR_SYMBOL_LIST(z), "convert");
    3791
    0
      ccv_nnc_graph_exec_symbol_autogen(symbolic_graph, 0, 0, CCV_NNC_AUTOGEN_ALL_EXECS | CCV_NNC_AUTOGEN_SOURCES_AND_DESTINATIONS);
    3792
    0
      SYMBOLIC_GRAPH_GEN(symbolic_graph, CCV_NNC_LONG_DOT_GRAPH);
    3793
    0
      ccv_nnc_tensor_t* const x_tensor = ccv_nnc_tensor_new(0, CPU_TENSOR_NHWC(32F, 10, 5, 5, 3), 0);
    3794
    0
      ccv_nnc_tensor_t* const y_tensor = ccv_nnc_tensor_new(0, CPU_TENSOR_NHWC(32F, 10, 5, 1, 3), 0);
    3795
    0
      ccv_nnc_graph_t* graph = 0;
    3796
    0
      ccv_nnc_tensor_arena_t* tensor_arena = 0;
    3797
    0
      ccv_nnc_graph_exec_arena_t* graph_exec_arena = 0;
    3798
    0
      ccv_nnc_symbolic_graph_compile(symbolic_graph, ccv_nnc_default_compile_params, TENSOR_BIND_MAP(KV(x, x_tensor), KV(y, y_tensor)), TENSOR_SYMBOL_LIST(z), SYMBOLIC_GRAPH_SOURCES(symbolic_graph), SYMBOLIC_GRAPH_DESTINATIONS(symbolic_graph), &graph, &tensor_arena, &graph_exec_arena);
    3799
    0
      GRAPH_GEN(graph, CCV_NNC_LONG_DOT_GRAPH);
    3800
    0
      dsfmt_t dsfmt;
    3801
    0
      dsfmt_init_gen_rand(&dsfmt, 0);
    3802
    0
      int i;
    3803
    0
      for (i = 0; i < 10 * 5 * 5 * 3; i++)
    3804
    0
        x_tensor->data.f32[i] = dsfmt_genrand_open_close(&dsfmt);
    3805
    0
      for (i = 0; i < 10 * 5 * 1 * 3; i++)
    3806
    0
        y_tensor->data.f32[i] = dsfmt_genrand_open_close(&dsfmt);
    3807
    0
      ccv_nnc_tensor_t* zt = ccv_nnc_tensor_new(0, CPU_TENSOR_NHWC(32F, 10, 5, 5, 3), 0);
    3808
    0
      ccv_nnc_cmd_exec(CMD_ADD_FORWARD(0.5, 0.2), ccv_nnc_no_hint, 0, TENSOR_LIST(x_tensor, y_tensor), TENSOR_LIST(zt), 0);
    3809
    0
      ccv_nnc_graph_run(graph, 0, TRAVERSE_FULL, 0, 0);
    3810
    0
      ccv_nnc_tensor_t* const z_tensor = ccv_nnc_tensor_from_symbol(tensor_arena, z);
    3811
    0
      REQUIRE_ARRAY_EQ_WITH_TOLERANCE(float, zt->data.f32, z_tensor->data.f32, 10 * 5 * 5 * 3, 1e-3, "add should match");
    3812
    0
      ccv_nnc_tensor_free(x_tensor);
    3813
    0
      ccv_nnc_tensor_free(y_tensor);
    3814
    0
      ccv_nnc_tensor_free(zt);
    3815
    0
      ccv_nnc_graph_free(graph);
    3816
    0
      ccv_nnc_tensor_arena_free(tensor_arena);
    3817
    0
      ccv_nnc_graph_exec_arena_free(graph_exec_arena);
    3818
    0
      ccv_nnc_symbolic_graph_free(symbolic_graph);
    3819
    0
    }
    3820
    3821
    TEST_CASE("compare add gradient with mps")
    3822
    1
    {
    3823
    1
      GUARD_ELSE_RETURN(ccv_nnc_cmd_ok(CCV_NNC_ADD_FORWARD, CCV_NNC_BACKEND_MPS) &&
    3824
    1
        ccv_nnc_cmd_ok(CCV_NNC_ADD_BACKWARD, CCV_NNC_BACKEND_MPS));
    3825
    0
      ccv_nnc_symbolic_graph_t* const symbolic_graph = ccv_nnc_symbolic_graph_new();
    3826
    0
      ccv_nnc_tensor_symbol_t x = ccv_nnc_tensor_symbol_new(symbolic_graph, CPU_TENSOR_NHWC(32F, 10, 5, 5, 3), "x");
    3827
    0
      ccv_nnc_tensor_symbol_t y = ccv_nnc_tensor_symbol_new(symbolic_graph, CPU_TENSOR_NHWC(32F, 10, 5, 1, 3), "y");
    3828
    0
      ccv_nnc_tensor_symbol_t a = ccv_nnc_tensor_symbol_new(symbolic_graph, GPU_TENSOR_NHWC(000, 32F, 10, 5, 5, 3), "a");
    3829
    0
      ccv_nnc_tensor_symbol_t b = ccv_nnc_tensor_symbol_new(symbolic_graph, GPU_TENSOR_NHWC(000, 32F, 10, 5, 1, 3), "b");
    3830
    0
      ccv_nnc_tensor_symbol_t c = ccv_nnc_tensor_symbol_new(symbolic_graph, GPU_TENSOR_NHWC(000, 32F, 10, 5, 5, 3), "c");
    3831
    0
      ccv_nnc_graph_exec_symbol_new(symbolic_graph, CMD_DATA_TRANSFER_FORWARD(), TENSOR_SYMBOL_LIST(x, y), TENSOR_SYMBOL_LIST(a, b), "transfer");
    3832
    0
      ccv_nnc_graph_exec_symbol_new(symbolic_graph, CMD_ADD_FORWARD(0.5, 0.2), TENSOR_SYMBOL_LIST(a, b), TENSOR_SYMBOL_LIST(c), "add");
    3833
    0
      ccv_nnc_graph_exec_symbol_autogen(symbolic_graph, 0, 0, CCV_NNC_AUTOGEN_ALL_EXECS | CCV_NNC_AUTOGEN_SOURCES_AND_DESTINATIONS);
    3834
    0
      ccv_nnc_symbolic_graph_backward(symbolic_graph, TENSOR_SYMBOL_LIST(c), TENSOR_SYMBOL_LIST(x, y), SYMBOLIC_GRAPH_SOURCES(symbolic_graph), SYMBOLIC_GRAPH_DESTINATIONS(symbolic_graph));
    3835
    0
      ccv_nnc_graph_exec_symbol_autogen(symbolic_graph, 0, 0, CCV_NNC_AUTOGEN_ALL_EXECS | CCV_NNC_AUTOGEN_SOURCES_AND_DESTINATIONS);
    3836
    0
      SYMBOLIC_GRAPH_GEN(symbolic_graph, CCV_NNC_LONG_DOT_GRAPH);
    3837
    0
      ccv_nnc_tensor_t* const x_tensor = ccv_nnc_tensor_new(0, CPU_TENSOR_NHWC(32F, 10, 5, 5, 3), 0);
    3838
    0
      ccv_nnc_tensor_t* const y_tensor = ccv_nnc_tensor_new(0, CPU_TENSOR_NHWC(32F, 10, 5, 1, 3), 0);
    3839
    0
      ccv_nnc_graph_t* graph = 0;
    3840
    0
      ccv_nnc_tensor_arena_t* tensor_arena = 0;
    3841
    0
      ccv_nnc_graph_exec_arena_t* graph_exec_arena = 0;
    3842
    0
      ccv_nnc_tensor_symbol_t dc = ccv_nnc_tensor_symbol_for_backward(symbolic_graph, c);
    3843
    0
      ccv_nnc_tensor_symbol_t dx = ccv_nnc_tensor_symbol_for_backward(symbolic_graph, x);
    3844
    0
      ccv_nnc_tensor_symbol_t dy = ccv_nnc_tensor_symbol_for_backward(symbolic_graph, y);
    3845
    0
      ccv_nnc_symbolic_graph_compile(symbolic_graph, ccv_nnc_default_compile_params, TENSOR_BIND_MAP(KV(x, x_tensor), KV(y, y_tensor)), TENSOR_SYMBOL_LIST(dx, dy), SYMBOLIC_GRAPH_SOURCES(symbolic_graph), SYMBOLIC_GRAPH_DESTINATIONS(symbolic_graph), &graph, &tensor_arena, &graph_exec_arena);
    3846
    0
      GRAPH_GEN(graph, CCV_NNC_LONG_DOT_GRAPH);
    3847
    0
      dsfmt_t dsfmt;
    3848
    0
      dsfmt_init_gen_rand(&dsfmt, 0);
    3849
    0
      int i;
    3850
    0
      for (i = 0; i < 10 * 5 * 5 * 3; i++)
    3851
    0
        x_tensor->data.f32[i] = dsfmt_genrand_open_close(&dsfmt);
    3852
    0
      for (i = 0; i < 10 * 5 * 1 * 3; i++)
    3853
    0
        y_tensor->data.f32[i] = dsfmt_genrand_open_close(&dsfmt);
    3854
    0
      ccv_nnc_tensor_t* dct = ccv_nnc_tensor_new(0, CPU_TENSOR_NHWC(32F, 10, 5, 5, 3), 0);
    3855
    0
      for (i = 0; i < 10 * 5 * 5 * 3; i++)
    3856
    0
        dct->data.f32[i] = dsfmt_genrand_open_close(&dsfmt);
    3857
    0
      ccv_nnc_tensor_t* const dc_tensor = ccv_nnc_tensor_from_symbol(tensor_arena, dc);
    3858
    0
      ccv_nnc_cmd_exec(CMD_DATA_TRANSFER_FORWARD(), ccv_nnc_no_hint, 0, TENSOR_LIST(dct), TENSOR_LIST(dc_tensor), 0);
    3859
    0
      ccv_nnc_graph_run(graph, 0, TRAVERSE_FULL, 0, 0);
    3860
    0
      ccv_nnc_tensor_t* zt = ccv_nnc_tensor_new(0, CPU_TENSOR_NHWC(32F, 10, 5, 5, 3), 0);
    3861
    0
      ccv_nnc_cmd_exec(CMD_ADD_FORWARD(0.5, 0.2), ccv_nnc_no_hint, 0, TENSOR_LIST(x_tensor, y_tensor), TENSOR_LIST(zt), 0);
    3862
    0
      ccv_nnc_tensor_t* dxt = ccv_nnc_tensor_new(0, CPU_TENSOR_NHWC(32F, 10, 5, 5, 3), 0);
    3863
    0
      ccv_nnc_tensor_t* dyt = ccv_nnc_tensor_new(0, CPU_TENSOR_NHWC(32F, 10, 5, 1, 3), 0);
    3864
    0
      ccv_nnc_cmd_exec(CMD_ADD_BACKWARD(0.5, 0.2), ccv_nnc_no_hint, 0, TENSOR_LIST(dct, x_tensor, y_tensor, zt), TENSOR_LIST(dxt, dyt), 0);
    3865
    0
      ccv_nnc_tensor_t* dx_tensor = ccv_nnc_tensor_from_symbol(tensor_arena, dx);
    3866
    0
      ccv_nnc_tensor_t* dy_tensor = ccv_nnc_tensor_from_symbol(tensor_arena, dy);
    3867
    0
      REQUIRE_TENSOR_EQ(dxt, dx_tensor, "backward pass should match");
    3868
    0
      REQUIRE_TENSOR_EQ(dyt, dy_tensor, "backward pass should match");
    3869
    0
      ccv_nnc_tensor_free(x_tensor);
    3870
    0
      ccv_nnc_tensor_free(y_tensor);
    3871
    0
      ccv_nnc_tensor_free(dct);
    3872
    0
      ccv_nnc_tensor_free(zt);
    3873
    0
      ccv_nnc_tensor_free(dxt);
    3874
    0
      ccv_nnc_tensor_free(dyt);
    3875
    0
      ccv_nnc_graph_free(graph);
    3876
    0
      ccv_nnc_tensor_arena_free(tensor_arena);
    3877
    0
      ccv_nnc_graph_exec_arena_free(graph_exec_arena);
    3878
    0
      ccv_nnc_symbolic_graph_free(symbolic_graph);
    3879
    0
    }
    3880
    3881
    TEST_CASE("compare add gradient with mps no dyt ")
    3882
    1
    {
    3883
    1
      GUARD_ELSE_RETURN(ccv_nnc_cmd_ok(CCV_NNC_ADD_FORWARD, CCV_NNC_BACKEND_MPS) &&
    3884
    1
        ccv_nnc_cmd_ok(CCV_NNC_ADD_BACKWARD, CCV_NNC_BACKEND_MPS));
    3885
    0
      ccv_nnc_symbolic_graph_t* const symbolic_graph = ccv_nnc_symbolic_graph_new();
    3886
    0
      ccv_nnc_tensor_symbol_t x = ccv_nnc_tensor_symbol_new(symbolic_graph, CPU_TENSOR_NHWC(32F, 10, 5, 5, 3), "x");
    3887
    0
      ccv_nnc_tensor_symbol_t y = ccv_nnc_tensor_symbol_new(symbolic_graph, CPU_TENSOR_NHWC(32F, 10, 5, 1, 3), "y");
    3888
    0
      ccv_nnc_tensor_symbol_t a = ccv_nnc_tensor_symbol_new(symbolic_graph, GPU_TENSOR_NHWC(000, 32F, 10, 5, 5, 3), "a");
    3889
    0
      ccv_nnc_tensor_symbol_t b = ccv_nnc_tensor_symbol_new(symbolic_graph, GPU_TENSOR_NHWC(000, 32F, 10, 5, 1, 3), "b");
    3890
    0
      ccv_nnc_tensor_symbol_t c = ccv_nnc_tensor_symbol_new(symbolic_graph, GPU_TENSOR_NHWC(000, 32F, 10, 5, 5, 3), "c");
    3891
    0
      ccv_nnc_graph_exec_symbol_new(symbolic_graph, CMD_DATA_TRANSFER_FORWARD(), TENSOR_SYMBOL_LIST(x, y), TENSOR_SYMBOL_LIST(a, b), "transfer");
    3892
    0
      ccv_nnc_graph_exec_symbol_new(symbolic_graph, CMD_ADD_FORWARD(0.5, 0.2), TENSOR_SYMBOL_LIST(a, b), TENSOR_SYMBOL_LIST(c), "add");
    3893
    0
      ccv_nnc_graph_exec_symbol_autogen(symbolic_graph, 0, 0, CCV_NNC_AUTOGEN_ALL_EXECS | CCV_NNC_AUTOGEN_SOURCES_AND_DESTINATIONS);
    3894
    0
      ccv_nnc_symbolic_graph_backward(symbolic_graph, TENSOR_SYMBOL_LIST(c), TENSOR_SYMBOL_LIST(x, y), SYMBOLIC_GRAPH_SOURCES(symbolic_graph), SYMBOLIC_GRAPH_DESTINATIONS(symbolic_graph));
    3895
    0
      ccv_nnc_graph_exec_symbol_autogen(symbolic_graph, 0, 0, CCV_NNC_AUTOGEN_ALL_EXECS | CCV_NNC_AUTOGEN_SOURCES_AND_DESTINATIONS);
    3896
    0
      SYMBOLIC_GRAPH_GEN(symbolic_graph, CCV_NNC_LONG_DOT_GRAPH);
    3897
    0
      ccv_nnc_tensor_t* const x_tensor = ccv_nnc_tensor_new(0, CPU_TENSOR_NHWC(32F, 10, 5, 5, 3), 0);
    3898
    0
      ccv_nnc_tensor_t* const y_tensor = ccv_nnc_tensor_new(0, CPU_TENSOR_NHWC(32F, 10, 5, 1, 3), 0);
    3899
    0
      ccv_nnc_graph_t* graph = 0;
    3900
    0
      ccv_nnc_tensor_arena_t* tensor_arena = 0;
    3901
    0
      ccv_nnc_graph_exec_arena_t* graph_exec_arena = 0;
    3902
    0
      ccv_nnc_tensor_symbol_t dc = ccv_nnc_tensor_symbol_for_backward(symbolic_graph, c);
    3903
    0
      ccv_nnc_tensor_symbol_t dx = ccv_nnc_tensor_symbol_for_backward(symbolic_graph, x);
    3904
    0
      ccv_nnc_tensor_symbol_t dy = ccv_nnc_tensor_symbol_for_backward(symbolic_graph, y);
    3905
    0
      ccv_nnc_symbolic_graph_compile(symbolic_graph, ccv_nnc_default_compile_params, TENSOR_BIND_MAP(KV(x, x_tensor), KV(y, y_tensor)), TENSOR_SYMBOL_LIST(dx, dy), SYMBOLIC_GRAPH_SOURCES(symbolic_graph), SYMBOLIC_GRAPH_DESTINATIONS(symbolic_graph), &graph, &tensor_arena, &graph_exec_arena);
    3906
    0
      GRAPH_GEN(graph, CCV_NNC_LONG_DOT_GRAPH);
    3907
    0
      dsfmt_t dsfmt;
    3908
    0
      dsfmt_init_gen_rand(&dsfmt, 0);
    3909
    0
      int i;
    3910
    0
      for (i = 0; i < 10 * 5 * 5 * 3; i++)
    3911
    0
        x_tensor->data.f32[i] = dsfmt_genrand_open_close(&dsfmt);
    3912
    0
      for (i = 0; i < 10 * 5 * 1 * 3; i++)
    3913
    0
        y_tensor->data.f32[i] = dsfmt_genrand_open_close(&dsfmt);
    3914
    0
      ccv_nnc_tensor_t* dct = ccv_nnc_tensor_new(0, CPU_TENSOR_NHWC(32F, 10, 5, 5, 3), 0);
    3915
    0
      for (i = 0; i < 10 * 5 * 5 * 3; i++)
    3916
    0
        dct->data.f32[i] = dsfmt_genrand_open_close(&dsfmt);
    3917
    0
      ccv_nnc_tensor_t* const dc_tensor = ccv_nnc_tensor_from_symbol(tensor_arena, dc);
    3918
    0
      ccv_nnc_cmd_exec(CMD_DATA_TRANSFER_FORWARD(), ccv_nnc_no_hint, 0, TENSOR_LIST(dct), TENSOR_LIST(dc_tensor), 0);
    3919
    0
      ccv_nnc_graph_run(graph, 0, TRAVERSE_FULL, 0, 0);
    3920
    0
      ccv_nnc_tensor_t* zt = ccv_nnc_tensor_new(0, CPU_TENSOR_NHWC(32F, 10, 5, 5, 3), 0);
    3921
    0
      ccv_nnc_cmd_exec(CMD_ADD_FORWARD(0.5, 0.2), ccv_nnc_no_hint, 0, TENSOR_LIST(x_tensor, y_tensor), TENSOR_LIST(zt), 0);
    3922
    0
      ccv_nnc_tensor_t* dxt = ccv_nnc_tensor_new(0, CPU_TENSOR_NHWC(32F, 10, 5, 5, 3), 0);
    3923
    0
      ccv_nnc_cmd_exec(CMD_ADD_BACKWARD(0.5, 0.2), ccv_nnc_no_hint, 0, TENSOR_LIST(dct, x_tensor, y_tensor, zt), TENSOR_LIST(dxt, 0), 0);
    3924
    0
      ccv_nnc_tensor_t* dx_tensor = ccv_nnc_tensor_from_symbol(tensor_arena, dx);
    3925
    0
      REQUIRE_TENSOR_EQ(dxt, dx_tensor, "backward pass should match");
    3926
    0
      ccv_nnc_tensor_free(x_tensor);
    3927
    0
      ccv_nnc_tensor_free(y_tensor);
    3928
    0
      ccv_nnc_tensor_free(dct);
    3929
    0
      ccv_nnc_tensor_free(zt);
    3930
    0
      ccv_nnc_tensor_free(dxt);
    3931
    0
      ccv_nnc_graph_free(graph);
    3932
    0
      ccv_nnc_tensor_arena_free(tensor_arena);
    3933
    0
      ccv_nnc_graph_exec_arena_free(graph_exec_arena);
    3934
    0
      ccv_nnc_symbolic_graph_free(symbolic_graph);
    3935
    0
    }
    3936
    3937
    TEST_CASE("broadcasting semantics for add backward mps (a,b)")
    3938
    1
    {
    3939
    1
      GUARD_ELSE_RETURN(ccv_nnc_cmd_ok(CCV_NNC_ADD_FORWARD, CCV_NNC_BACKEND_MPS) &&
    3940
    1
        ccv_nnc_cmd_ok(CCV_NNC_ADD_BACKWARD, CCV_NNC_BACKEND_MPS));
    3941
    0
      ccv_nnc_tensor_t* const a = ccv_nnc_tensor_new(0, CPU_TENSOR_NHWC(32F, 4, 1), 0);
    3942
    0
      ccv_nnc_tensor_t* const b = ccv_nnc_tensor_new(0, CPU_TENSOR_NHWC(32F, 2), 0);
    3943
    0
      ccv_nnc_tensor_t* const c = ccv_nnc_tensor_new(0, CPU_TENSOR_NHWC(32F, 4, 2), 0);
    3944
    0
      ccv_nnc_tensor_t* const da = ccv_nnc_tensor_new(0, CPU_TENSOR_NHWC(32F, 4, 1), 0);
    3945
    0
      ccv_nnc_tensor_t* const db = ccv_nnc_tensor_new(0, CPU_TENSOR_NHWC(32F, 2), 0);
    3946
    0
      ccv_nnc_tensor_t* const dat = ccv_nnc_tensor_new(0, CPU_TENSOR_NHWC(32F, 4, 1), 0);
    3947
    0
      ccv_nnc_tensor_t* const dbt = ccv_nnc_tensor_new(0, CPU_TENSOR_NHWC(32F, 2), 0);
    3948
    0
      a->data.f32[0] = 1;
    3949
    0
      a->data.f32[1] = 2;
    3950
    0
      a->data.f32[2] = 3;
    3951
    0
      a->data.f32[3] = 4;
    3952
    0
      b->data.f32[0] = 5;
    3953
    0
      b->data.f32[1] = 6;
    3954
    0
      float ctp[] = {
    3955
    0
        6, 7,
    3956
    0
        7, 8,
    3957
    0
        8, 9,
    3958
    0
        9, 10
    3959
    0
      };
    3960
    0
      memcpy(c->data.f32, ctp, sizeof(ctp));
    3961
    0
      ccv_nnc_tensor_t* const ga = ccv_nnc_tensor_new(0, GPU_TENSOR_NHWC(000, 32F, 4, 1), 0);
    3962
    0
      ccv_nnc_tensor_t* const gb = ccv_nnc_tensor_new(0, GPU_TENSOR_NHWC(000, 32F, 2), 0);
    3963
    0
      ccv_nnc_tensor_t* const gda = ccv_nnc_tensor_new(0, GPU_TENSOR_NHWC(000, 32F, 4, 1), 0);
    3964
    0
      ccv_nnc_tensor_t* const gdb = ccv_nnc_tensor_new(0, GPU_TENSOR_NHWC(000, 32F, 2), 0);
    3965
    0
      ccv_nnc_tensor_t* const gc = ccv_nnc_tensor_new(0, GPU_TENSOR_NHWC(000, 32F, 4, 2), 0);
    3966
    0
      ccv_nnc_cmd_exec(CMD_DATA_TRANSFER_FORWARD(), ccv_nnc_no_hint, 0, TENSOR_LIST(a, b, c), TENSOR_LIST(ga, gb, gc), 0);
    3967
    0
      ccv_nnc_cmd_exec(CMD_ADD_BACKWARD(0.5, 0.2), ccv_nnc_no_hint, 0, TENSOR_LIST(gc, ga, gb), TENSOR_LIST(gda, gdb), 0);
    3968
    0
      ccv_nnc_cmd_exec(CMD_DATA_TRANSFER_FORWARD(), ccv_nnc_no_hint, 0, TENSOR_LIST(gda, gdb), TENSOR_LIST(da, db), 0);
    3969
    0
      ccv_nnc_cmd_exec(CMD_ADD_BACKWARD(0.5, 0.2), ccv_nnc_no_hint, 0, TENSOR_LIST(c, a, b), TENSOR_LIST(dat, dbt), 0);
    3970
    0
      REQUIRE_TENSOR_EQ(dat, da, "gradient of a should be equal");
    3971
    0
      REQUIRE_TENSOR_EQ(dbt, db, "gradient of b should be equal");
    3972
    0
      ccv_nnc_tensor_free(a);
    3973
    0
      ccv_nnc_tensor_free(b);
    3974
    0
      ccv_nnc_tensor_free(c);
    3975
    0
      ccv_nnc_tensor_free(da);
    3976
    0
      ccv_nnc_tensor_free(db);
    3977
    0
      ccv_nnc_tensor_free(dat);
    3978
    0
      ccv_nnc_tensor_free(dbt);
    3979
    0
      ccv_nnc_tensor_free(ga);
    3980
    0
      ccv_nnc_tensor_free(gb);
    3981
    0
      ccv_nnc_tensor_free(gc);
    3982
    0
      ccv_nnc_tensor_free(gda);
    3983
    0
      ccv_nnc_tensor_free(gdb);
    3984
    0
    }
    3985
    3986
    TEST_CASE("broadcasting semantics for add backward mps (a, nil)")
    3987
    1
    {
    3988
    1
      GUARD_ELSE_RETURN(ccv_nnc_cmd_ok(CCV_NNC_ADD_FORWARD, CCV_NNC_BACKEND_MPS) &&
    3989
    1
        ccv_nnc_cmd_ok(CCV_NNC_ADD_BACKWARD, CCV_NNC_BACKEND_MPS));
    3990
    0
      ccv_nnc_tensor_t* const a = ccv_nnc_tensor_new(0, CPU_TENSOR_NHWC(32F, 4, 1), 0);
    3991
    0
      ccv_nnc_tensor_t* const b = ccv_nnc_tensor_new(0, CPU_TENSOR_NHWC(32F, 2), 0);
    3992
    0
      ccv_nnc_tensor_t* const c = ccv_nnc_tensor_new(0, CPU_TENSOR_NHWC(32F, 4, 2), 0);
    3993
    0
      ccv_nnc_tensor_t* const da = ccv_nnc_tensor_new(0, CPU_TENSOR_NHWC(32F, 4, 1), 0);
    3994
    0
      ccv_nnc_tensor_t* const dat = ccv_nnc_tensor_new(0, CPU_TENSOR_NHWC(32F, 4, 1), 0);
    3995
    0
      a->data.f32[0] = 1;
    3996
    0
      a->data.f32[1] = 2;
    3997
    0
      a->data.f32[2] = 3;
    3998
    0
      a->data.f32[3] = 4;
    3999
    0
      b->data.f32[0] = 5;
    4000
    0
      b->data.f32[1] = 6;
    4001
    0
      float ctp[] = {
    4002
    0
        6, 7,
    4003
    0
        7, 8,
    4004
    0
        8, 9,
    4005
    0
        9, 10
    4006
    0
      };
    4007
    0
      memcpy(c->data.f32, ctp, sizeof(ctp));
    4008
    0
      ccv_nnc_tensor_t* const ga = ccv_nnc_tensor_new(0, GPU_TENSOR_NHWC(000, 32F, 4, 1), 0);
    4009
    0
      ccv_nnc_tensor_t* const gb = ccv_nnc_tensor_new(0, GPU_TENSOR_NHWC(000, 32F, 2), 0);
    4010
    0
      ccv_nnc_tensor_t* const gda = ccv_nnc_tensor_new(0, GPU_TENSOR_NHWC(000, 32F, 4, 1), 0);
    4011
    0
      ccv_nnc_tensor_t* const gc = ccv_nnc_tensor_new(0, GPU_TENSOR_NHWC(000, 32F, 4, 2), 0);
    4012
    0
      ccv_nnc_cmd_exec(CMD_DATA_TRANSFER_FORWARD(), ccv_nnc_no_hint, 0, TENSOR_LIST(a, b, c), TENSOR_LIST(ga, gb, gc), 0);
    4013
    0
      ccv_nnc_cmd_exec(CMD_ADD_BACKWARD(0.5, 0.2), ccv_nnc_no_hint, 0, TENSOR_LIST(gc, ga, ), TENSOR_LIST(gda, ), 0);
    4014
    0
      ccv_nnc_cmd_exec(CMD_DATA_TRANSFER_FORWARD(), ccv_nnc_no_hint, 0, TENSOR_LIST(gda, ), TENSOR_LIST(da, ), 0);
    4015
    0
      ccv_nnc_cmd_exec(CMD_ADD_BACKWARD(0.5, 0.2), ccv_nnc_no_hint, 0, TENSOR_LIST(c, a, ), TENSOR_LIST(dat, ), 0);
    4016
    0
      REQUIRE_TENSOR_EQ(dat, da, "gradient of a should be equal");
    4017
    0
      ccv_nnc_tensor_free(a);
    4018
    0
      ccv_nnc_tensor_free(b);
    4019
    0
      ccv_nnc_tensor_free(c);
    4020
    0
      ccv_nnc_tensor_free(da);
    4021
    0
      ccv_nnc_tensor_free(dat);
    4022
    0
      ccv_nnc_tensor_free(ga);
    4023
    0
      ccv_nnc_tensor_free(gb);
    4024
    0
      ccv_nnc_tensor_free(gc);
    4025
    0
      ccv_nnc_tensor_free(gda);
    4026
    0
    }
    4027
    4028
    TEST_CASE("broadcasting semantics for add backward mps (nil,b)")
    4029
    1
    {
    4030
    1
      GUARD_ELSE_RETURN(ccv_nnc_cmd_ok(CCV_NNC_ADD_FORWARD, CCV_NNC_BACKEND_MPS) &&
    4031
    1
        ccv_nnc_cmd_ok(CCV_NNC_ADD_BACKWARD, CCV_NNC_BACKEND_MPS));
    4032
    0
      ccv_nnc_tensor_t* const a = ccv_nnc_tensor_new(0, CPU_TENSOR_NHWC(32F, 4, 1), 0);
    4033
    0
      ccv_nnc_tensor_t* const b = ccv_nnc_tensor_new(0, CPU_TENSOR_NHWC(32F, 2), 0);
    4034
    0
      ccv_nnc_tensor_t* const c = ccv_nnc_tensor_new(0, CPU_TENSOR_NHWC(32F, 4, 2), 0);
    4035
    0
      ccv_nnc_tensor_t* const db = ccv_nnc_tensor_new(0, CPU_TENSOR_NHWC(32F, 2), 0);
    4036
    0
      ccv_nnc_tensor_t* const dbt = ccv_nnc_tensor_new(0, CPU_TENSOR_NHWC(32F, 2), 0);
    4037
    0
      a->data.f32[0] = 1;
    4038
    0
      a->data.f32[1] = 2;
    4039
    0
      a->data.f32[2] = 3;
    4040
    0
      a->data.f32[3] = 4;
    4041
    0
      b->data.f32[0] = 5;
    4042
    0
      b->data.f32[1] = 6;
    4043
    0
      float ctp[] = {
    4044
    0
        6, 7,
    4045
    0
        7, 8,
    4046
    0
        8, 9,
    4047
    0
        9, 10
    4048
    0
      };
    4049
    0
      memcpy(c->data.f32, ctp, sizeof(ctp));
    4050
    0
      ccv_nnc_tensor_t* const ga = ccv_nnc_tensor_new(0, GPU_TENSOR_NHWC(000, 32F, 4, 1), 0);
    4051
    0
      ccv_nnc_tensor_t* const gb = ccv_nnc_tensor_new(0, GPU_TENSOR_NHWC(000, 32F, 2), 0);
    4052
    0
      ccv_nnc_tensor_t* const gdb = ccv_nnc_tensor_new(0, GPU_TENSOR_NHWC(000, 32F, 2), 0);
    4053
    0
      ccv_nnc_tensor_t* const gc = ccv_nnc_tensor_new(0, GPU_TENSOR_NHWC(000, 32F, 4, 2), 0);
    4054
    0
      ccv_nnc_cmd_exec(CMD_DATA_TRANSFER_FORWARD(), ccv_nnc_no_hint, 0, TENSOR_LIST(a, b, c), TENSOR_LIST(ga, gb, gc), 0);
    4055
    0
      ccv_nnc_cmd_exec(CMD_ADD_BACKWARD(0.5, 0.2), ccv_nnc_no_hint, 0, TENSOR_LIST(gc, ga, gb), TENSOR_LIST(0, gdb), 0);
    4056
    0
      ccv_nnc_cmd_exec(CMD_DATA_TRANSFER_FORWARD(), ccv_nnc_no_hint, 0, TENSOR_LIST(gdb), TENSOR_LIST(db), 0);
    4057
    0
      ccv_nnc_cmd_exec(CMD_ADD_BACKWARD(0.5, 0.2), ccv_nnc_no_hint, 0, TENSOR_LIST(c, 0, b), TENSOR_LIST(0, dbt), 0);
    4058
    0
      REQUIRE_TENSOR_EQ(dbt, db, "gradient of b should be equal");
    4059
    0
      ccv_nnc_tensor_free(a);
    4060
    0
      ccv_nnc_tensor_free(b);
    4061
    0
      ccv_nnc_tensor_free(c);
    4062
    0
      ccv_nnc_tensor_free(db);
    4063
    0
      ccv_nnc_tensor_free(dbt);
    4064
    0
      ccv_nnc_tensor_free(ga);
    4065
    0
      ccv_nnc_tensor_free(gb);
    4066
    0
      ccv_nnc_tensor_free(gc);
    4067
    0
      ccv_nnc_tensor_free(gdb);
    4068
    0
    }
    4069
    4070
    TEST_CASE("compare ewsum with mps")
    4071
    1
    {
    4072
    1
      GUARD_ELSE_RETURN(ccv_nnc_cmd_ok(CCV_NNC_EWSUM_FORWARD, CCV_NNC_BACKEND_MPS));
    4073
    0
      ccv_nnc_tensor_t* const a = ccv_nnc_tensor_new(0, GPU_TENSOR_NHWC(000, 32F, 100), 0);
    4074
    0
      ccv_nnc_tensor_t* const b = ccv_nnc_tensor_new(0, GPU_TENSOR_NHWC(000, 32F, 100), 0);
    4075
    0
      ccv_nnc_tensor_t* const c = ccv_nnc_tensor_new(0, GPU_TENSOR_NHWC(000, 32F, 100), 0);
    4076
    0
      ccv_nnc_tensor_t* const d = ccv_nnc_tensor_new(0, GPU_TENSOR_NHWC(000, 32F, 100), 0);
    4077
    0
      ccv_nnc_tensor_t* const ha = ccv_nnc_tensor_new(0, CPU_TENSOR_NHWC(32F, 100), 0);
    4078
    0
      ccv_nnc_tensor_t* const hb = ccv_nnc_tensor_new(0, CPU_TENSOR_NHWC(32F, 100), 0);
    4079
    0
      ccv_nnc_tensor_t* const hc = ccv_nnc_tensor_new(0, CPU_TENSOR_NHWC(32F, 100), 0);
    4080
    0
      ccv_nnc_tensor_t* const hd = ccv_nnc_tensor_new(0, CPU_TENSOR_NHWC(32F, 100), 0);
    4081
    0
      ccv_nnc_tensor_t* const gd = ccv_nnc_tensor_new(0, CPU_TENSOR_NHWC(32F, 100), 0);
    4082
    0
      int i;
    4083
    0
      for (i = 0; i < 100; i++)
    4084
    0
      {
    4085
    0
        ha->data.f32[i] = 1;
    4086
    0
        hb->data.f32[i] = 0.5;
    4087
    0
        hc->data.f32[i] = 0.25;
    4088
    0
        gd->data.f32[i] = 1.75;
    4089
    0
      }
    4090
    0
      ccv_nnc_cmd_exec(CMD_DATA_TRANSFER_FORWARD(), ccv_nnc_no_hint, 0, TENSOR_LIST(ha, hb, hc), TENSOR_LIST(a, b, c), 0);
    4091
    0
      ccv_nnc_cmd_exec(CMD_EWSUM_FORWARD(), ccv_nnc_no_hint, 0, TENSOR_LIST(a, b, c), TENSOR_LIST(d), 0);
    4092
    0
      ccv_nnc_cmd_exec(CMD_DATA_TRANSFER_FORWARD(), ccv_nnc_no_hint, 0, TENSOR_LIST(d), TENSOR_LIST(hd), 0);
    4093
    0
      REQUIRE_TENSOR_EQ(hd, gd, "ewsum result should be the same");
    4094
    0
      ccv_nnc_tensor_free(a);
    4095
    0
      ccv_nnc_tensor_free(b);
    4096
    0
      ccv_nnc_tensor_free(c);
    4097
    0
      ccv_nnc_tensor_free(d);
    4098
    0
      ccv_nnc_tensor_free(ha);
    4099
    0
      ccv_nnc_tensor_free(hb);
    4100
    0
      ccv_nnc_tensor_free(hc);
    4101
    0
      ccv_nnc_tensor_free(hd);
    4102
    0
      ccv_nnc_tensor_free(gd);
    4103
    0
    }
    4104
    4105
    TEST_CASE("compare ewsum with mps in half precision")
    4106
    1
    {
    4107
    1
      GUARD_ELSE_RETURN(ccv_nnc_cmd_ok(CCV_NNC_EWSUM_FORWARD, CCV_NNC_BACKEND_MPS));
    4108
    0
      ccv_nnc_tensor_t* const a = ccv_nnc_tensor_new(0, GPU_TENSOR_NHWC(000, 16F, 100), 0);
    4109
    0
      ccv_nnc_tensor_t* const b = ccv_nnc_tensor_new(0, GPU_TENSOR_NHWC(000, 16F, 100), 0);
    4110
    0
      ccv_nnc_tensor_t* const c = ccv_nnc_tensor_new(0, GPU_TENSOR_NHWC(000, 16F, 100), 0);
    4111
    0
      ccv_nnc_tensor_t* const d = ccv_nnc_tensor_new(0, GPU_TENSOR_NHWC(000, 16F, 100), 0);
    4112
    0
      ccv_nnc_tensor_t* const ha = ccv_nnc_tensor_new(0, CPU_TENSOR_NHWC(32F, 100), 0);
    4113
    0
      ccv_nnc_tensor_t* const hb = ccv_nnc_tensor_new(0, CPU_TENSOR_NHWC(32F, 100), 0);
    4114
    0
      ccv_nnc_tensor_t* const hc = ccv_nnc_tensor_new(0, CPU_TENSOR_NHWC(32F, 100), 0);
    4115
    0
      ccv_nnc_tensor_t* const hd = ccv_nnc_tensor_new(0, CPU_TENSOR_NHWC(32F, 100), 0);
    4116
    0
      ccv_nnc_tensor_t* const ha16 = ccv_nnc_tensor_new(0, CPU_TENSOR_NHWC(16F, 100), 0);
    4117
    0
      ccv_nnc_tensor_t* const hb16 = ccv_nnc_tensor_new(0, CPU_TENSOR_NHWC(16F, 100), 0);
    4118
    0
      ccv_nnc_tensor_t* const hc16 = ccv_nnc_tensor_new(0, CPU_TENSOR_NHWC(16F, 100), 0);
    4119
    0
      ccv_nnc_tensor_t* const hd16 = ccv_nnc_tensor_new(0, CPU_TENSOR_NHWC(16F, 100), 0);
    4120
    0
      ccv_nnc_tensor_t* const gd = ccv_nnc_tensor_new(0, CPU_TENSOR_NHWC(32F, 100), 0);
    4121
    0
      int i;
    4122
    0
      for (i = 0; i < 100; i++)
    4123
    0
      {
    4124
    0
        ha->data.f32[i] = 1;
    4125
    0
        hb->data.f32[i] = 0.5;
    4126
    0
        hc->data.f32[i] = 0.25;
    4127
    0
        gd->data.f32[i] = 1.75;
    4128
    0
      }
    4129
    0
      ccv_nnc_cmd_exec(CMD_DATATYPE_CONVERSION_FORWARD(), ccv_nnc_no_hint, 0, TENSOR_LIST(ha, hb, hc), TENSOR_LIST(ha16, hb16, hc16), 0);
    4130
    0
      ccv_nnc_cmd_exec(CMD_DATA_TRANSFER_FORWARD(), ccv_nnc_no_hint, 0, TENSOR_LIST(ha16, hb16, hc16), TENSOR_LIST(a, b, c), 0);
    4131
    0
      ccv_nnc_cmd_exec(CMD_EWSUM_FORWARD(), ccv_nnc_no_hint, 0, TENSOR_LIST(a, b, c), TENSOR_LIST(d), 0);
    4132
    0
      ccv_nnc_cmd_exec(CMD_DATA_TRANSFER_FORWARD(), ccv_nnc_no_hint, 0, TENSOR_LIST(d), TENSOR_LIST(hd16), 0);
    4133
    0
      ccv_nnc_cmd_exec(CMD_DATATYPE_CONVERSION_FORWARD(), ccv_nnc_no_hint, 0, TENSOR_LIST(hd16), TENSOR_LIST(hd), 0);
    4134
    0
      REQUIRE_ARRAY_EQ_WITH_TOLERANCE(float, hd->data.f32, gd->data.f32, 100, 1e-3, "ewsum result should be the same");
    4135
    0
      ccv_nnc_tensor_free(a);
    4136
    0
      ccv_nnc_tensor_free(b);
    4137
    0
      ccv_nnc_tensor_free(c);
    4138
    0
      ccv_nnc_tensor_free(d);
    4139
    0
      ccv_nnc_tensor_free(ha);
    4140
    0
      ccv_nnc_tensor_free(hb);
    4141
    0
      ccv_nnc_tensor_free(hc);
    4142
    0
      ccv_nnc_tensor_free(hd);
    4143
    0
      ccv_nnc_tensor_free(ha16);
    4144
    0
      ccv_nnc_tensor_free(hb16);
    4145
    0
      ccv_nnc_tensor_free(hc16);
    4146
    0
      ccv_nnc_tensor_free(hd16);
    4147
    0
      ccv_nnc_tensor_free(gd);
    4148
    0
    }
    4149
    4150
    TEST_CASE("compare ewsum with mps in bfloat precision")
    4151
    1
    {
    4152
    1
      GUARD_ELSE_RETURN(ccv_nnc_cmd_ok(CCV_NNC_EWSUM_FORWARD, CCV_NNC_BACKEND_MPS));
    4153
    0
      ccv_nnc_tensor_t* const a = ccv_nnc_tensor_new(0, GPU_TENSOR_NHWC(000, 16BF, 100), 0);
    4154
    0
      ccv_nnc_tensor_t* const b = ccv_nnc_tensor_new(0, GPU_TENSOR_NHWC(000, 16BF, 100), 0);
    4155
    0
      ccv_nnc_tensor_t* const c = ccv_nnc_tensor_new(0, GPU_TENSOR_NHWC(000, 16BF, 100), 0);
    4156
    0
      ccv_nnc_tensor_t* const d = ccv_nnc_tensor_new(0, GPU_TENSOR_NHWC(000, 16BF, 100), 0);
    4157
    0
      ccv_nnc_tensor_t* const ha = ccv_nnc_tensor_new(0, CPU_TENSOR_NHWC(32F, 100), 0);
    4158
    0
      ccv_nnc_tensor_t* const hb = ccv_nnc_tensor_new(0, CPU_TENSOR_NHWC(32F, 100), 0);
    4159
    0
      ccv_nnc_tensor_t* const hc = ccv_nnc_tensor_new(0, CPU_TENSOR_NHWC(32F, 100), 0);
    4160
    0
      ccv_nnc_tensor_t* const hd = ccv_nnc_tensor_new(0, CPU_TENSOR_NHWC(32F, 100), 0);
    4161
    0
      ccv_nnc_tensor_t* const ha16bf = ccv_nnc_tensor_new(0, CPU_TENSOR_NHWC(16BF, 100), 0);
    4162
    0
      ccv_nnc_tensor_t* const hb16bf = ccv_nnc_tensor_new(0, CPU_TENSOR_NHWC(16BF, 100), 0);
    4163
    0
      ccv_nnc_tensor_t* const hc16bf = ccv_nnc_tensor_new(0, CPU_TENSOR_NHWC(16BF, 100), 0);
    4164
    0
      ccv_nnc_tensor_t* const hd16bf = ccv_nnc_tensor_new(0, CPU_TENSOR_NHWC(16BF, 100), 0);
    4165
    0
      ccv_nnc_tensor_t* const gd = ccv_nnc_tensor_new(0, CPU_TENSOR_NHWC(32F, 100), 0);
    4166
    0
      int i;
    4167
    0
      for (i = 0; i < 100; i++)
    4168
    0
      {
    4169
    0
        ha->data.f32[i] = 1;
    4170
    0
        hb->data.f32[i] = 0.5;
    4171
    0
        hc->data.f32[i] = 0.25;
    4172
    0
        gd->data.f32[i] = 1.75;
    4173
    0
      }
    4174
    0
      ccv_float_to_bfloat(ha->data.f32, (uint16_t*)ha16bf->data.f16, 100);
    4175
    0
      ccv_float_to_bfloat(hb->data.f32, (uint16_t*)hb16bf->data.f16, 100);
    4176
    0
      ccv_float_to_bfloat(hc->data.f32, (uint16_t*)hc16bf->data.f16, 100);
    4177
    0
      ccv_nnc_cmd_exec(CMD_DATA_TRANSFER_FORWARD(), ccv_nnc_no_hint, 0, TENSOR_LIST(ha16bf, hb16bf, hc16bf), TENSOR_LIST(a, b, c), 0);
    4178
    0
      ccv_nnc_cmd_exec(CMD_EWSUM_FORWARD(), ccv_nnc_no_hint, 0, TENSOR_LIST(a, b, c), TENSOR_LIST(d), 0);
    4179
    0
      ccv_nnc_cmd_exec(CMD_DATA_TRANSFER_FORWARD(), ccv_nnc_no_hint, 0, TENSOR_LIST(d), TENSOR_LIST(hd16bf), 0);
    4180
    0
      ccv_bfloat_to_float((uint16_t*)hd16bf->data.f16, hd->data.f32, 100);
    4181
    0
      REQUIRE_ARRAY_EQ_WITH_TOLERANCE(float, hd->data.f32, gd->data.f32, 100, 1e-3, "ewsum result should be the same");
    4182
    0
      ccv_nnc_tensor_free(a);
    4183
    0
      ccv_nnc_tensor_free(b);
    4184
    0
      ccv_nnc_tensor_free(c);
    4185
    0
      ccv_nnc_tensor_free(d);
    4186
    0
      ccv_nnc_tensor_free(ha);
    4187
    0
      ccv_nnc_tensor_free(hb);
    4188
    0
      ccv_nnc_tensor_free(hc);
    4189
    0
      ccv_nnc_tensor_free(hd);
    4190
    0
      ccv_nnc_tensor_free(ha16bf);
    4191
    0
      ccv_nnc_tensor_free(hb16bf);
    4192
    0
      ccv_nnc_tensor_free(hc16bf);
    4193
    0
      ccv_nnc_tensor_free(hd16bf);
    4194
    0
      ccv_nnc_tensor_free(gd);
    4195
    0
    }
    4196
    4197
    TEST_CASE("compare ewsum with mps in bfloat precision with MFA disabled")
    4198
    1
    {
    4199
    1
      GUARD_ELSE_RETURN(ccv_nnc_cmd_ok(CCV_NNC_EWSUM_FORWARD, CCV_NNC_BACKEND_MPS));
    4200
    0
      ccv_nnc_tensor_t* const a = ccv_nnc_tensor_new(0, GPU_TENSOR_NHWC(000, 16BF, 100), 0);
    4201
    0
      ccv_nnc_tensor_t* const b = ccv_nnc_tensor_new(0, GPU_TENSOR_NHWC(000, 16BF, 100), 0);
    4202
    0
      ccv_nnc_tensor_t* const c = ccv_nnc_tensor_new(0, GPU_TENSOR_NHWC(000, 16BF, 100), 0);
    4203
    0
      ccv_nnc_tensor_t* const d = ccv_nnc_tensor_new(0, GPU_TENSOR_NHWC(000, 16BF, 100), 0);
    4204
    0
      ccv_nnc_tensor_t* const ha = ccv_nnc_tensor_new(0, CPU_TENSOR_NHWC(32F, 100), 0);
    4205
    0
      ccv_nnc_tensor_t* const hb = ccv_nnc_tensor_new(0, CPU_TENSOR_NHWC(32F, 100), 0);
    4206
    0
      ccv_nnc_tensor_t* const hc = ccv_nnc_tensor_new(0, CPU_TENSOR_NHWC(32F, 100), 0);
    4207
    0
      ccv_nnc_tensor_t* const hd = ccv_nnc_tensor_new(0, CPU_TENSOR_NHWC(32F, 100), 0);
    4208
    0
      ccv_nnc_tensor_t* const ha16bf = ccv_nnc_tensor_new(0, CPU_TENSOR_NHWC(16BF, 100), 0);
    4209
    0
      ccv_nnc_tensor_t* const hb16bf = ccv_nnc_tensor_new(0, CPU_TENSOR_NHWC(16BF, 100), 0);
    4210
    0
      ccv_nnc_tensor_t* const hc16bf = ccv_nnc_tensor_new(0, CPU_TENSOR_NHWC(16BF, 100), 0);
    4211
    0
      ccv_nnc_tensor_t* const hd16bf = ccv_nnc_tensor_new(0, CPU_TENSOR_NHWC(16BF, 100), 0);
    4212
    0
      ccv_nnc_tensor_t* const gd = ccv_nnc_tensor_new(0, CPU_TENSOR_NHWC(32F, 100), 0);
    4213
    0
      int i;
    4214
    0
      for (i = 0; i < 100; i++)
    4215
    0
      {
    4216
    0
        ha->data.f32[i] = 1;
    4217
    0
        hb->data.f32[i] = 0.5;
    4218
    0
        hc->data.f32[i] = 0.25;
    4219
    0
        gd->data.f32[i] = 1.75;
    4220
    0
      }
    4221
    0
      ccv_float_to_bfloat(ha->data.f32, (uint16_t*)ha16bf->data.f16, 100);
    4222
    0
      ccv_float_to_bfloat(hb->data.f32, (uint16_t*)hb16bf->data.f16, 100);
    4223
    0
      ccv_float_to_bfloat(hc->data.f32, (uint16_t*)hc16bf->data.f16, 100);
    4224
    0
      ccv_nnc_cmd_exec(CMD_DATA_TRANSFER_FORWARD(), ccv_nnc_no_hint, 0, TENSOR_LIST(ha16bf, hb16bf, hc16bf), TENSOR_LIST(a, b, c), 0);
    4225
    0
      const uint64_t old_flags = ccv_nnc_flags();
    4226
    0
      ccv_nnc_enable_flag(CCV_NNC_DISABLE_MFA);
    4227
    0
      ccv_nnc_cmd_exec(CMD_EWSUM_FORWARD(), ccv_nnc_no_hint, 0, TENSOR_LIST(a, b, c), TENSOR_LIST(d), 0);
    4228
    0
      if (!(old_flags & CCV_NNC_DISABLE_MFA))
    4229
    0
        ccv_nnc_disable_flag(CCV_NNC_DISABLE_MFA);
    4230
    0
      ccv_nnc_cmd_exec(CMD_DATA_TRANSFER_FORWARD(), ccv_nnc_no_hint, 0, TENSOR_LIST(d), TENSOR_LIST(hd16bf), 0);
    4231
    0
      ccv_bfloat_to_float((uint16_t*)hd16bf->data.f16, hd->data.f32, 100);
    4232
    0
      REQUIRE_ARRAY_EQ_WITH_TOLERANCE(float, hd->data.f32, gd->data.f32, 100, 1e-3, "ewsum result should be the same");
    4233
    0
      ccv_nnc_tensor_free(a);
    4234
    0
      ccv_nnc_tensor_free(b);
    4235
    0
      ccv_nnc_tensor_free(c);
    4236
    0
      ccv_nnc_tensor_free(d);
    4237
    0
      ccv_nnc_tensor_free(ha);
    4238
    0
      ccv_nnc_tensor_free(hb);
    4239
    0
      ccv_nnc_tensor_free(hc);
    4240
    0
      ccv_nnc_tensor_free(hd);
    4241
    0
      ccv_nnc_tensor_free(ha16bf);
    4242
    0
      ccv_nnc_tensor_free(hb16bf);
    4243
    0
      ccv_nnc_tensor_free(hc16bf);
    4244
    0
      ccv_nnc_tensor_free(hd16bf);
    4245
    0
      ccv_nnc_tensor_free(gd);
    4246
    0
    }
    4247
    4248
    TEST_CASE("compare transpose two tensor views")
    4249
    1
    {
    4250
    1
      GUARD_ELSE_RETURN(ccv_nnc_cmd_ok(CCV_NNC_TRANSPOSE_FORWARD, CCV_NNC_BACKEND_MPS));
    4251
    0
      ccv_nnc_tensor_t* const ha = ccv_nnc_tensor_new(0, CPU_TENSOR_NHWC(32F, 7, 6, 5, 4), 0);
    4252
    0
      memset(ha->data.f32, 0, sizeof(float) * 7 * 6 * 5 * 4);
    4253
    0
      ccv_nnc_tensor_view_t ha_view = ccv_nnc_tensor_view(ha, CPU_TENSOR_NHWC(32F, 4, 3, 2, 2), DIM_ALLOC(3, 2, 1, 0), DIM_ALLOC(6 * 5 * 4, 5 * 4, 4, 1));
    4254
    0
      ccv_nnc_tensor_t* const hb = ccv_nnc_tensor_new(0, CPU_TENSOR_NHWC(32F, 8, 7, 6, 5), 0);
    4255
    0
      memset(hb->data.f32, 0, sizeof(float) * 8 * 7 * 6 * 5);
    4256
    0
      ccv_nnc_tensor_view_t hb_view = ccv_nnc_tensor_view(hb, CPU_TENSOR_NHWC(32F, 4, 2, 2, 3), DIM_ALLOC(3, 2, 1, 0), DIM_ALLOC(7 * 6 * 5, 6 * 5, 5, 1));
    4257
    0
      int i, j, k, l;
    4258
    0
      for (i = 0; i < 4; i++)
    4259
    0
        for (j = 0; j < 3; j++)
    4260
    0
          for (k = 0; k < 2; k++)
    4261
    0
            for (l = 0; l < 2; l++)
    4262
    0
              ha->data.f32[(i + 3) * 6 * 5 * 4 + (j + 2) * 5 * 4 + (k + 1) * 4 + l] = i * 3 * 2 * 2 + j * 2 * 2 + k * 2 + l;
    4263
    0
      ccv_nnc_cmd_exec(CMD_TRANSPOSE_FORWARD(1, 3), ccv_nnc_no_hint, 0, TENSOR_LIST((ccv_nnc_tensor_t*)&ha_view), TENSOR_LIST((ccv_nnc_tensor_t*)&hb_view), 0);
    4264
    0
      ccv_nnc_tensor_t* hd = ccv_nnc_tensor_new(0, CPU_TENSOR_NHWC(32F, 7, 6, 5, 4), 0);
    4265
    0
      memset(hd->data.f32, 0, sizeof(float) * 7 * 6 * 5 * 4);
    4266
    0
      ccv_nnc_tensor_view_t hd_view = ccv_nnc_tensor_view(hd, CPU_TENSOR_NHWC(32F, 4, 3, 2, 2), DIM_ALLOC(3, 2, 1, 0), DIM_ALLOC(6 * 5 * 4, 5 * 4, 4, 1));
    4267
    0
      ccv_nnc_cmd_exec(CMD_TRANSPOSE_FORWARD(1, 3), ccv_nnc_no_hint, 0, TENSOR_LIST((ccv_nnc_tensor_t*)&hb_view), TENSOR_LIST((ccv_nnc_tensor_t*)&hd_view), 0);
    4268
    0
      REQUIRE_TENSOR_EQ(hd, ha, "4x3x2x2 tensor should be exactly the same.");
    4269
    0
      ccv_nnc_tensor_t* const a = ccv_nnc_tensor_new(0, GPU_TENSOR_NHWC(000, 32F, 7, 6, 5, 4), 0);
    4270
    0
      ccv_nnc_cmd_exec(CMD_DATA_TRANSFER_FORWARD(), ccv_nnc_no_hint, 0, TENSOR_LIST(ha), TENSOR_LIST(a), 0);
    4271
    0
      ccv_nnc_tensor_view_t a_view = ccv_nnc_tensor_view(a, GPU_TENSOR_NHWC(000, 32F, 4, 3, 2, 2), DIM_ALLOC(3, 2, 1, 0), DIM_ALLOC(6 * 5 * 4, 5 * 4, 4, 1));
    4272
    0
      ccv_nnc_tensor_t* const b = ccv_nnc_tensor_new(0, GPU_TENSOR_NHWC(000, 32F, 8, 7, 6, 5), 0);
    4273
    0
      ccv_nnc_cmd_exec(CMD_SET_FORWARD(0), ccv_nnc_no_hint, 0, TENSOR_LIST(), TENSOR_LIST(b), 0);
    4274
    0
      ccv_nnc_tensor_view_t b_view = ccv_nnc_tensor_view(b, GPU_TENSOR_NHWC(000, 32F, 4, 2, 2, 3), DIM_ALLOC(3, 2, 1, 0), DIM_ALLOC(7 * 6 * 5, 6 * 5, 5, 1));
    4275
    0
      ccv_nnc_cmd_exec(CMD_TRANSPOSE_FORWARD(1, 3), ccv_nnc_no_hint, 0, TENSOR_LIST((ccv_nnc_tensor_t*)&a_view), TENSOR_LIST((ccv_nnc_tensor_t*)&b_view), 0);
    4276
    0
      ccv_nnc_tensor_t* d = ccv_nnc_tensor_new(0, GPU_TENSOR_NHWC(000, 32F, 7, 6, 5, 4), 0);
    4277
    0
      ccv_nnc_cmd_exec(CMD_SET_FORWARD(0), ccv_nnc_no_hint, 0, TENSOR_LIST(), TENSOR_LIST(d), 0);
    4278
    0
      ccv_nnc_tensor_view_t d_view = ccv_nnc_tensor_view(d, GPU_TENSOR_NHWC(000, 32F, 4, 3, 2, 2), DIM_ALLOC(3, 2, 1, 0), DIM_ALLOC(6 * 5 * 4, 5 * 4, 4, 1));
    4279
    0
      ccv_nnc_cmd_exec(CMD_TRANSPOSE_FORWARD(1, 3), ccv_nnc_no_hint, 0, TENSOR_LIST((ccv_nnc_tensor_t*)&b_view), TENSOR_LIST((ccv_nnc_tensor_t*)&d_view), 0);
    4280
    0
      ccv_nnc_tensor_t* const hbt = ccv_nnc_tensor_new(0, CPU_TENSOR_NHWC(32F, 8, 7, 6, 5), 0);
    4281
    0
      ccv_nnc_tensor_t* const hdt = ccv_nnc_tensor_new(0, CPU_TENSOR_NHWC(32F, 7, 6, 5, 4), 0);
    4282
    0
      ccv_nnc_cmd_exec(CMD_DATA_TRANSFER_FORWARD(), ccv_nnc_no_hint, 0, TENSOR_LIST(b, d), TENSOR_LIST(hbt, hdt), 0);
    4283
    0
      REQUIRE_TENSOR_EQ(hbt, hb, "4x2x2x3 tensor should be exactly the same.");
    4284
    0
      REQUIRE_TENSOR_EQ(hdt, hd, "4x3x2x2 tensor should be exactly the same.");
    4285
    0
      ccv_nnc_tensor_free(ha);
    4286
    0
      ccv_nnc_tensor_free(hb);
    4287
    0
      ccv_nnc_tensor_free(hd);
    4288
    0
      ccv_nnc_tensor_free(hbt);
    4289
    0
      ccv_nnc_tensor_free(hdt);
    4290
    0
      ccv_nnc_tensor_free(a);
    4291
    0
      ccv_nnc_tensor_free(b);
    4292
    0
      ccv_nnc_tensor_free(d);
    4293
    0
    }
    4294
    4295
    TEST_CASE("broadcasting semantics for add [[1, 2, 3], [4, 5, 6]] + [7, 8, 9]")
    4296
    1
    {
    4297
    1
      GUARD_ELSE_RETURN(ccv_nnc_cmd_ok(CCV_NNC_ADD_FORWARD, CCV_NNC_BACKEND_MPS));
    4298
    0
      ccv_nnc_tensor_t* const a = ccv_nnc_tensor_new(0, CPU_TENSOR_NHWC(32F, 2, 3), 0);
    4299
    0
      ccv_nnc_tensor_t* const b = ccv_nnc_tensor_new(0, CPU_TENSOR_NHWC(32F, 3), 0);
    4300
    0
      ccv_nnc_tensor_t* const c = ccv_nnc_tensor_new(0, CPU_TENSOR_NHWC(32F, 2, 3), 0);
    4301
    0
      a->data.f32[0] = 1;
    4302
    0
      a->data.f32[1] = 2;
    4303
    0
      a->data.f32[2] = 3;
    4304
    0
      a->data.f32[3] = 4;
    4305
    0
      a->data.f32[4] = 5;
    4306
    0
      a->data.f32[5] = 6;
    4307
    0
      b->data.f32[0] = 7;
    4308
    0
      b->data.f32[1] = 8;
    4309
    0
      b->data.f32[2] = 9;
    4310
    0
      ccv_nnc_tensor_t* const ga = ccv_nnc_tensor_new(0, GPU_TENSOR_NHWC(000, 32F, 2, 3), 0);
    4311
    0
      ccv_nnc_tensor_t* const gb = ccv_nnc_tensor_new(0, GPU_TENSOR_NHWC(000, 32F, 3), 0);
    4312
    0
      ccv_nnc_tensor_t* const gc = ccv_nnc_tensor_new(0, GPU_TENSOR_NHWC(000, 32F, 2, 3), 0);
    4313
    0
      ccv_nnc_cmd_exec(CMD_DATA_TRANSFER_FORWARD(), ccv_nnc_no_hint, 0, TENSOR_LIST(a, b), TENSOR_LIST(ga, gb), 0);
    4314
    0
      ccv_nnc_cmd_exec(CMD_ADD_FORWARD(1, 1), ccv_nnc_no_hint, 0, TENSOR_LIST(ga, gb), TENSOR_LIST(gc), 0);
    4315
    0
      ccv_nnc_cmd_exec(CMD_DATA_TRANSFER_FORWARD(), ccv_nnc_no_hint, 0, TENSOR_LIST(gc), TENSOR_LIST(c), 0);
    4316
    0
      float ctp[] = {
    4317
    0
        8, 10, 12,
    4318
    0
        11, 13, 15
    4319
    0
      };
    4320
    0
      ccv_nnc_tensor_t ct = ccv_nnc_tensor(ctp, CPU_TENSOR_NHWC(32F, 2, 3), 0);
    4321
    0
      REQUIRE_TENSOR_EQ(c, &ct, "result should be equal");
    4322
    0
      ccv_nnc_tensor_free(a);
    4323
    0
      ccv_nnc_tensor_free(b);
    4324
    0
      ccv_nnc_tensor_free(c);
    4325
    0
      ccv_nnc_tensor_free(ga);
    4326
    0
      ccv_nnc_tensor_free(gb);
    4327
    0
      ccv_nnc_tensor_free(gc);
    4328
    0
    }
    4329
    4330
    TEST_CASE("broadcasting semantics for add [[1], [2], [3], [4]] + [5, 6]")
    4331
    1
    {
    4332
    1
      GUARD_ELSE_RETURN(ccv_nnc_cmd_ok(CCV_NNC_ADD_FORWARD, CCV_NNC_BACKEND_MPS));
    4333
    0
      ccv_nnc_tensor_t* const a = ccv_nnc_tensor_new(0, CPU_TENSOR_NHWC(32F, 4, 1), 0);
    4334
    0
      ccv_nnc_tensor_t* const b = ccv_nnc_tensor_new(0, CPU_TENSOR_NHWC(32F, 2), 0);
    4335
    0
      ccv_nnc_tensor_t* const c = ccv_nnc_tensor_new(0, CPU_TENSOR_NHWC(32F, 4, 2), 0);
    4336
    0
      a->data.f32[0] = 1;
    4337
    0
      a->data.f32[1] = 2;
    4338
    0
      a->data.f32[2] = 3;
    4339
    0
      a->data.f32[3] = 4;
    4340
    0
      b->data.f32[0] = 5;
    4341
    0
      b->data.f32[1] = 6;
    4342
    0
      ccv_nnc_tensor_t* const ga = ccv_nnc_tensor_new(0, GPU_TENSOR_NHWC(000, 32F, 4, 1), 0);
    4343
    0
      ccv_nnc_tensor_t* const gb = ccv_nnc_tensor_new(0, GPU_TENSOR_NHWC(000, 32F, 2), 0);
    4344
    0
      ccv_nnc_tensor_t* const gc = ccv_nnc_tensor_new(0, GPU_TENSOR_NHWC(000, 32F, 4, 2), 0);
    4345
    0
      ccv_nnc_cmd_exec(CMD_DATA_TRANSFER_FORWARD(), ccv_nnc_no_hint, 0, TENSOR_LIST(a, b), TENSOR_LIST(ga, gb), 0);
    4346
    0
      ccv_nnc_cmd_exec(CMD_ADD_FORWARD(1, 1), ccv_nnc_no_hint, 0, TENSOR_LIST(ga, gb), TENSOR_LIST(gc), 0);
    4347
    0
      ccv_nnc_cmd_exec(CMD_DATA_TRANSFER_FORWARD(), ccv_nnc_no_hint, 0, TENSOR_LIST(gc), TENSOR_LIST(c), 0);
    4348
    0
      float ctp[] = {
    4349
    0
        6, 7,
    4350
    0
        7, 8,
    4351
    0
        8, 9,
    4352
    0
        9, 10
    4353
    0
      };
    4354
    0
      ccv_nnc_tensor_t ct = ccv_nnc_tensor(ctp, CPU_TENSOR_NHWC(32F, 4, 2), 0);
    4355
    0
      REQUIRE_TENSOR_EQ(c, &ct, "result should be equal");
    4356
    0
      ccv_nnc_tensor_free(a);
    4357
    0
      ccv_nnc_tensor_free(b);
    4358
    0
      ccv_nnc_tensor_free(c);
    4359
    0
      ccv_nnc_tensor_free(ga);
    4360
    0
      ccv_nnc_tensor_free(gb);
    4361
    0
      ccv_nnc_tensor_free(gc);
    4362
    0
    }
    4363
    4364
    TEST_CASE("broadcasting semantics for mul [[1, 2, 3], [4, 5, 6]] * [7, 8, 9]")
    4365
    1
    {
    4366
    1
      GUARD_ELSE_RETURN(ccv_nnc_cmd_ok(CCV_NNC_MUL_FORWARD, CCV_NNC_BACKEND_MPS));
    4367
    0
      ccv_nnc_tensor_t* const a = ccv_nnc_tensor_new(0, CPU_TENSOR_NHWC(32F, 2, 3), 0);
    4368
    0
      ccv_nnc_tensor_t* const b = ccv_nnc_tensor_new(0, CPU_TENSOR_NHWC(32F, 3), 0);
    4369
    0
      ccv_nnc_tensor_t* const c = ccv_nnc_tensor_new(0, CPU_TENSOR_NHWC(32F, 2, 3), 0);
    4370
    0
      a->data.f32[0] = 1;
    4371
    0
      a->data.f32[1] = 2;
    4372
    0
      a->data.f32[2] = 3;
    4373
    0
      a->data.f32[3] = 4;
    4374
    0
      a->data.f32[4] = 5;
    4375
    0
      a->data.f32[5] = 6;
    4376
    0
      b->data.f32[0] = 7;
    4377
    0
      b->data.f32[1] = 8;
    4378
    0
      b->data.f32[2] = 9;
    4379
    0
      ccv_nnc_tensor_t* const ga = ccv_nnc_tensor_new(0, GPU_TENSOR_NHWC(000, 32F, 2, 3), 0);
    4380
    0
      ccv_nnc_tensor_t* const gb = ccv_nnc_tensor_new(0, GPU_TENSOR_NHWC(000, 32F, 3), 0);
    4381
    0
      ccv_nnc_tensor_t* const gc = ccv_nnc_tensor_new(0, GPU_TENSOR_NHWC(000, 32F, 2, 3), 0);
    4382
    0
      ccv_nnc_cmd_exec(CMD_DATA_TRANSFER_FORWARD(), ccv_nnc_no_hint, 0, TENSOR_LIST(a, b), TENSOR_LIST(ga, gb), 0);
    4383
    0
      ccv_nnc_cmd_exec(CMD_MUL_FORWARD(1), ccv_nnc_no_hint, 0, TENSOR_LIST(ga, gb), TENSOR_LIST(gc), 0);
    4384
    0
      ccv_nnc_cmd_exec(CMD_DATA_TRANSFER_FORWARD(), ccv_nnc_no_hint, 0, TENSOR_LIST(gc), TENSOR_LIST(c), 0);
    4385
    0
      float ctp[] = {
    4386
    0
        7, 16, 27,
    4387
    0
        28, 40, 54
    4388
    0
      };
    4389
    0
      ccv_nnc_tensor_t ct = ccv_nnc_tensor(ctp, CPU_TENSOR_NHWC(32F, 2, 3), 0);
    4390
    0
      REQUIRE_TENSOR_EQ(c, &ct, "result should be equal");
    4391
    0
      ccv_nnc_tensor_free(a);
    4392
    0
      ccv_nnc_tensor_free(b);
    4393
    0
      ccv_nnc_tensor_free(c);
    4394
    0
      ccv_nnc_tensor_free(ga);
    4395
    0
      ccv_nnc_tensor_free(gb);
    4396
    0
      ccv_nnc_tensor_free(gc);
    4397
    0
    }
    4398
    4399
    TEST_CASE("broadcasting semantics for mul [[1], [2], [3], [4]] * [5, 6]")
    4400
    1
    {
    4401
    1
      GUARD_ELSE_RETURN(ccv_nnc_cmd_ok(CCV_NNC_MUL_FORWARD, CCV_NNC_BACKEND_MPS));
    4402
    0
      ccv_nnc_tensor_t* const a = ccv_nnc_tensor_new(0, CPU_TENSOR_NHWC(32F, 4, 1), 0);
    4403
    0
      ccv_nnc_tensor_t* const b = ccv_nnc_tensor_new(0, CPU_TENSOR_NHWC(32F, 2), 0);
    4404
    0
      ccv_nnc_tensor_t* const c = ccv_nnc_tensor_new(0, CPU_TENSOR_NHWC(32F, 4, 2), 0);
    4405
    0
      a->data.f32[0] = 1;
    4406
    0
      a->data.f32[1] = 2;
    4407
    0
      a->data.f32[2] = 3;
    4408
    0
      a->data.f32[3] = 4;
    4409
    0
      b->data.f32[0] = 5;
    4410
    0
      b->data.f32[1] = 6;
    4411
    0
      ccv_nnc_tensor_t* const ga = ccv_nnc_tensor_new(0, GPU_TENSOR_NHWC(000, 32F, 4, 1), 0);
    4412
    0
      ccv_nnc_tensor_t* const gb = ccv_nnc_tensor_new(0, GPU_TENSOR_NHWC(000, 32F, 2), 0);
    4413
    0
      ccv_nnc_tensor_t* const gc = ccv_nnc_tensor_new(0, GPU_TENSOR_NHWC(000, 32F, 4, 2), 0);
    4414
    0
      ccv_nnc_cmd_exec(CMD_DATA_TRANSFER_FORWARD(), ccv_nnc_no_hint, 0, TENSOR_LIST(a, b), TENSOR_LIST(ga, gb), 0);
    4415
    0
      ccv_nnc_cmd_exec(CMD_MUL_FORWARD(1), ccv_nnc_no_hint, 0, TENSOR_LIST(ga, gb), TENSOR_LIST(gc), 0);
    4416
    0
      ccv_nnc_cmd_exec(CMD_DATA_TRANSFER_FORWARD(), ccv_nnc_no_hint, 0, TENSOR_LIST(gc), TENSOR_LIST(c), 0);
    4417
    0
      float ctp[] = {
    4418
    0
        5, 6,
    4419
    0
        10, 12,
    4420
    0
        15, 18,
    4421
    0
        20, 24
    4422
    0
      };
    4423
    0
      ccv_nnc_tensor_t ct = ccv_nnc_tensor(ctp, CPU_TENSOR_NHWC(32F, 4, 2), 0);
    4424
    0
      REQUIRE_TENSOR_EQ(c, &ct, "result should be equal");
    4425
    0
      ccv_nnc_tensor_free(a);
    4426
    0
      ccv_nnc_tensor_free(b);
    4427
    0
      ccv_nnc_tensor_free(c);
    4428
    0
      ccv_nnc_tensor_free(ga);
    4429
    0
      ccv_nnc_tensor_free(gb);
    4430
    0
      ccv_nnc_tensor_free(gc);
    4431
    0
    }
    4432
    4433
    TEST_CASE("scalar mul [[1, 2, 3], [4, 5, 6]] * 0.3")
    4434
    1
    {
    4435
    1
      GUARD_ELSE_RETURN(ccv_nnc_cmd_ok(CCV_NNC_MUL_FORWARD, CCV_NNC_BACKEND_MPS));
    4436
    0
      ccv_nnc_tensor_t* const a = ccv_nnc_tensor_new(0, CPU_TENSOR_NHWC(32F, 2, 3), 0);
    4437
    0
      ccv_nnc_tensor_t* const c = ccv_nnc_tensor_new(0, CPU_TENSOR_NHWC(32F, 2, 3), 0);
    4438
    0
      a->data.f32[0] = 1;
    4439
    0
      a->data.f32[1] = 2;
    4440
    0
      a->data.f32[2] = 3;
    4441
    0
      a->data.f32[3] = 4;
    4442
    0
      a->data.f32[4] = 5;
    4443
    0
      a->data.f32[5] = 6;
    4444
    0
      ccv_nnc_tensor_t* const ga = ccv_nnc_tensor_new(0, GPU_TENSOR_NHWC(000, 32F, 2, 3), 0);
    4445
    0
      ccv_nnc_tensor_t* const gc = ccv_nnc_tensor_new(0, GPU_TENSOR_NHWC(000, 32F, 2, 3), 0);
    4446
    0
      ccv_nnc_cmd_exec(CMD_DATA_TRANSFER_FORWARD(), ccv_nnc_no_hint, 0, TENSOR_LIST(a), TENSOR_LIST(ga), 0);
    4447
    0
      ccv_nnc_cmd_exec(CMD_SCALAR_MUL_FORWARD(0.3), ccv_nnc_no_hint, 0, TENSOR_LIST(ga), TENSOR_LIST(gc), 0);
    4448
    0
      ccv_nnc_cmd_exec(CMD_DATA_TRANSFER_FORWARD(), ccv_nnc_no_hint, 0, TENSOR_LIST(gc), TENSOR_LIST(c), 0);
    4449
    0
      float ctp[] = {
    4450
    0
        0.3, 0.6, 0.9,
    4451
    0
        1.2, 1.5, 1.8,
    4452
    0
      };
    4453
    0
      ccv_nnc_tensor_t ct = ccv_nnc_tensor(ctp, CPU_TENSOR_NHWC(32F, 2, 3), 0);
    4454
    0
      REQUIRE_TENSOR_EQ(c, &ct, "result should be equal");
    4455
    0
      ccv_nnc_tensor_free(a);
    4456
    0
      ccv_nnc_tensor_free(c);
    4457
    0
      ccv_nnc_tensor_free(ga);
    4458
    0
      ccv_nnc_tensor_free(gc);
    4459
    0
    }
    4460
    4461
    TEST_CASE("scalar mul [[1, 2, 3], [4, 5, 6]] * 0.5, int")
    4462
    1
    {
    4463
    1
      GUARD_ELSE_RETURN(ccv_nnc_cmd_ok(CCV_NNC_MUL_FORWARD, CCV_NNC_BACKEND_MPS));
    4464
    0
      ccv_nnc_tensor_t* const a = ccv_nnc_tensor_new(0, CPU_TENSOR_NHWC(32S, 2, 3), 0);
    4465
    0
      ccv_nnc_tensor_t* const c = ccv_nnc_tensor_new(0, CPU_TENSOR_NHWC(32S, 2, 3), 0);
    4466
    0
      ccv_nnc_tensor_t* const ct = ccv_nnc_tensor_new(0, CPU_TENSOR_NHWC(32S, 2, 3), 0);
    4467
    0
      a->data.i32[0] = 1;
    4468
    0
      a->data.i32[1] = 2;
    4469
    0
      a->data.i32[2] = 3;
    4470
    0
      a->data.i32[3] = 4;
    4471
    0
      a->data.i32[4] = 5;
    4472
    0
      a->data.i32[5] = 6;
    4473
    0
      ccv_nnc_tensor_t* const ga = ccv_nnc_tensor_new(0, GPU_TENSOR_NHWC(000, 32S, 2, 3), 0);
    4474
    0
      ccv_nnc_tensor_t* const gc = ccv_nnc_tensor_new(0, GPU_TENSOR_NHWC(000, 32S, 2, 3), 0);
    4475
    0
      ccv_nnc_cmd_exec(CMD_DATA_TRANSFER_FORWARD(), ccv_nnc_no_hint, 0, TENSOR_LIST(a), TENSOR_LIST(ga), 0);
    4476
    0
      ccv_nnc_cmd_exec(CMD_SCALAR_MUL_FORWARD(0.5), ccv_nnc_no_hint, 0, TENSOR_LIST(ga), TENSOR_LIST(gc), 0);
    4477
    0
      ccv_nnc_cmd_exec(CMD_DATA_TRANSFER_FORWARD(), ccv_nnc_no_hint, 0, TENSOR_LIST(gc), TENSOR_LIST(c), 0);
    4478
    0
      ccv_nnc_cmd_exec(CMD_SCALAR_MUL_FORWARD(0.5), ccv_nnc_no_hint, 0, TENSOR_LIST(a), TENSOR_LIST(ct), 0);
    4479
    0
      REQUIRE_TENSOR_EQ(c, ct, "result should be equal");
    4480
    0
      ccv_nnc_tensor_free(a);
    4481
    0
      ccv_nnc_tensor_free(c);
    4482
    0
      ccv_nnc_tensor_free(ct);
    4483
    0
      ccv_nnc_tensor_free(ga);
    4484
    0
      ccv_nnc_tensor_free(gc);
    4485
    0
    }
    4486
    4487
    TEST_CASE("compare average pooling with mps")
    4488
    1
    {
    4489
    1
      GUARD_ELSE_RETURN(ccv_nnc_cmd_ok(CCV_NNC_AVERAGE_POOL_FORWARD, CCV_NNC_BACKEND_MPS));
    4490
    0
      ccv_nnc_symbolic_graph_t* symbolic_graph = ccv_nnc_symbolic_graph_new();
    4491
    0
      ccv_nnc_tensor_symbol_t x = ccv_nnc_tensor_symbol_new(symbolic_graph, GPU_TENSOR_NHWC(000, 32F, 7, 7, 10), "x");
    4492
    0
      ccv_nnc_tensor_symbol_t y = ccv_nnc_tensor_symbol_new(symbolic_graph, GPU_TENSOR_NHWC(000, 32F, 3, 3, 10), "y");
    4493
    0
      ccv_nnc_graph_exec_symbol_t avg_pool = ccv_nnc_graph_exec_symbol_new(symbolic_graph, CMD_AVERAGE_POOL_FORWARD(5, 5), TENSOR_SYMBOL_LIST(x), TENSOR_SYMBOL_LIST(y), "avg_pool");
    4494
    0
      ccv_nnc_graph_exec_symbol_set_hint(symbolic_graph, avg_pool, HINT((2, 2), (1, 1)));
    4495
    0
      ccv_nnc_graph_exec_symbol_autogen(symbolic_graph, 0, 0, CCV_NNC_AUTOGEN_ALL_EXECS | CCV_NNC_AUTOGEN_SOURCES_AND_DESTINATIONS);
    4496
    0
      ccv_nnc_graph_t* graph = 0;
    4497
    0
      ccv_nnc_tensor_arena_t* tensor_arena = 0;
    4498
    0
      ccv_nnc_graph_exec_arena_t* graph_exec_arena = 0;
    4499
    0
      ccv_nnc_symbolic_graph_compile(symbolic_graph, ccv_nnc_default_compile_params, 0, 0, 0, 0, SYMBOLIC_GRAPH_SOURCES(symbolic_graph), SYMBOLIC_GRAPH_DESTINATIONS(symbolic_graph), &graph, &tensor_arena, &graph_exec_arena);
    4500
    0
      SYMBOLIC_GRAPH_GEN(symbolic_graph, CCV_NNC_LONG_DOT_GRAPH);
    4501
    0
      GRAPH_GEN(graph, CCV_NNC_LONG_DOT_GRAPH);
    4502
    0
      dsfmt_t dsfmt;
    4503
    0
      dsfmt_init_gen_rand(&dsfmt, 0);
    4504
    0
      ccv_nnc_tensor_t* const x_tensor = ccv_nnc_tensor_new(0, CPU_TENSOR_NHWC(32F, 7, 7, 10), 0);
    4505
    0
      int i;
    4506
    0
      for (i = 0; i < 7 * 7 * 10; i++)
    4507
    0
        x_tensor->data.f32[i] = dsfmt_genrand_open_close(&dsfmt);
    4508
    0
      ccv_nnc_tensor_t* const xt = ccv_nnc_tensor_from_symbol(tensor_arena, x);
    4509
    0
      ccv_nnc_cmd_exec(CMD_DATA_TRANSFER_FORWARD(), ccv_nnc_no_hint, 0, TENSOR_LIST(x_tensor), TENSOR_LIST(xt), 0);
    4510
    0
      ccv_nnc_graph_run(graph, 0, TRAVERSE_FULL, 0, 0);
    4511
    0
      ccv_nnc_tensor_t* const y_tensor = ccv_nnc_tensor_new(0, CPU_TENSOR_NHWC(32F, 3, 3, 10), 0);
    4512
    0
      ccv_nnc_cmd_exec(CMD_AVERAGE_POOL_FORWARD(5, 5), HINT((2, 2), (1, 1)), 0, TENSOR_LIST(x_tensor), TENSOR_LIST(y_tensor), 0);
    4513
    0
      ccv_nnc_tensor_t* const yt = ccv_nnc_tensor_from_symbol(tensor_arena, y);
    4514
    0
      ccv_nnc_tensor_t* const cpu_y = ccv_nnc_tensor_new(0, CPU_TENSOR_NHWC(32F, 3, 3, 10), 0);
    4515
    0
      ccv_nnc_cmd_exec(CMD_DATA_TRANSFER_FORWARD(), ccv_nnc_no_hint, 0, TENSOR_LIST(yt), TENSOR_LIST(cpu_y), 0);
    4516
    0
      REQUIRE_TENSOR_EQ(y_tensor, cpu_y, "mps result should equal to cpu result");
    4517
    0
      ccv_nnc_symbolic_graph_free(symbolic_graph);
    4518
    0
      ccv_nnc_tensor_arena_free(tensor_arena);
    4519
    0
      ccv_nnc_graph_exec_arena_free(graph_exec_arena);
    4520
    0
      ccv_nnc_graph_free(graph);
    4521
    0
      ccv_nnc_tensor_free(x_tensor);
    4522
    0
      ccv_nnc_tensor_free(y_tensor);
    4523
    0
      ccv_nnc_tensor_free(cpu_y);
    4524
    0
    }
    4525
    4526
    TEST_CASE("compare average pooling with mps in half precision")
    4527
    1
    {
    4528
    1
      GUARD_ELSE_RETURN(ccv_nnc_cmd_ok(CCV_NNC_AVERAGE_POOL_FORWARD, CCV_NNC_BACKEND_MPS));
    4529
    0
      ccv_nnc_symbolic_graph_t* symbolic_graph = ccv_nnc_symbolic_graph_new();
    4530
    0
      ccv_nnc_tensor_symbol_t x = ccv_nnc_tensor_symbol_new(symbolic_graph, GPU_TENSOR_NHWC(000, 16F, 7, 7, 10), "x");
    4531
    0
      ccv_nnc_tensor_symbol_t y = ccv_nnc_tensor_symbol_new(symbolic_graph, GPU_TENSOR_NHWC(000, 16F, 3, 3, 10), "y");
    4532
    0
      ccv_nnc_graph_exec_symbol_t avg_pool = ccv_nnc_graph_exec_symbol_new(symbolic_graph, CMD_AVERAGE_POOL_FORWARD(5, 5), TENSOR_SYMBOL_LIST(x), TENSOR_SYMBOL_LIST(y), "avg_pool");
    4533
    0
      ccv_nnc_graph_exec_symbol_set_hint(symbolic_graph, avg_pool, HINT((2, 2), (1, 1)));
    4534
    0
      ccv_nnc_graph_exec_symbol_autogen(symbolic_graph, 0, 0, CCV_NNC_AUTOGEN_ALL_EXECS | CCV_NNC_AUTOGEN_SOURCES_AND_DESTINATIONS);
    4535
    0
      ccv_nnc_graph_t* graph = 0;
    4536
    0
      ccv_nnc_tensor_arena_t* tensor_arena = 0;
    4537
    0
      ccv_nnc_graph_exec_arena_t* graph_exec_arena = 0;
    4538
    0
      ccv_nnc_symbolic_graph_compile(symbolic_graph, ccv_nnc_default_compile_params, 0, 0, 0, 0, SYMBOLIC_GRAPH_SOURCES(symbolic_graph), SYMBOLIC_GRAPH_DESTINATIONS(symbolic_graph), &graph, &tensor_arena, &graph_exec_arena);
    4539
    0
      SYMBOLIC_GRAPH_GEN(symbolic_graph, CCV_NNC_LONG_DOT_GRAPH);
    4540
    0
      GRAPH_GEN(graph, CCV_NNC_LONG_DOT_GRAPH);
    4541
    0
      dsfmt_t dsfmt;
    4542
    0
      dsfmt_init_gen_rand(&dsfmt, 0);
    4543
    0
      ccv_nnc_tensor_t* const x_tensor = ccv_nnc_tensor_new(0, CPU_TENSOR_NHWC(32F, 7, 7, 10), 0);
    4544
    0
      int i;
    4545
    0
      for (i = 0; i < 7 * 7 * 10; i++)
    4546
    0
        x_tensor->data.f32[i] = dsfmt_genrand_open_close(&dsfmt);
    4547
    0
      ccv_nnc_tensor_t* const x16_tensor = ccv_nnc_tensor_new(0, CPU_TENSOR_NHWC(16F, 7, 7, 10), 0);
    4548
    0
      ccv_nnc_tensor_t* const xt = ccv_nnc_tensor_from_symbol(tensor_arena, x);
    4549
    0
      ccv_nnc_cmd_exec(CMD_DATATYPE_CONVERSION_FORWARD(), ccv_nnc_no_hint, 0, TENSOR_LIST(x_tensor), TENSOR_LIST(x16_tensor), 0);
    4550
    0
      ccv_nnc_cmd_exec(CMD_DATA_TRANSFER_FORWARD(), ccv_nnc_no_hint, 0, TENSOR_LIST(x16_tensor), TENSOR_LIST(xt), 0);
    4551
    0
      ccv_nnc_graph_run(graph, 0, TRAVERSE_FULL, 0, 0);
    4552
    0
      ccv_nnc_tensor_t* const y_tensor = ccv_nnc_tensor_new(0, CPU_TENSOR_NHWC(32F, 3, 3, 10), 0);
    4553
    0
      ccv_nnc_cmd_exec(CMD_AVERAGE_POOL_FORWARD(5, 5), HINT((2, 2), (1, 1)), 0, TENSOR_LIST(x_tensor), TENSOR_LIST(y_tensor), 0);
    4554
    0
      ccv_nnc_tensor_t* const yt = ccv_nnc_tensor_from_symbol(tensor_arena, y);
    4555
    0
      ccv_nnc_tensor_t* const cpu_y = ccv_nnc_tensor_new(0, CPU_TENSOR_NHWC(32F, 3, 3, 10), 0);
    4556
    0
      ccv_nnc_tensor_t* const cpu_y16 = ccv_nnc_tensor_new(0, CPU_TENSOR_NHWC(16F, 3, 3, 10), 0);
    4557
    0
      ccv_nnc_cmd_exec(CMD_DATA_TRANSFER_FORWARD(), ccv_nnc_no_hint, 0, TENSOR_LIST(yt), TENSOR_LIST(cpu_y16), 0);
    4558
    0
      ccv_nnc_cmd_exec(CMD_DATATYPE_CONVERSION_FORWARD(), ccv_nnc_no_hint, 0, TENSOR_LIST(cpu_y16), TENSOR_LIST(cpu_y), 0);
    4559
    0
      REQUIRE_ARRAY_EQ_WITH_TOLERANCE(float, y_tensor->data.f32, cpu_y->data.f32, 3 * 3 * 10, 1e-3, "mps result should equal to cpu result");
    4560
    0
      ccv_nnc_symbolic_graph_free(symbolic_graph);
    4561
    0
      ccv_nnc_tensor_arena_free(tensor_arena);
    4562
    0
      ccv_nnc_graph_exec_arena_free(graph_exec_arena);
    4563
    0
      ccv_nnc_graph_free(graph);
    4564
    0
      ccv_nnc_tensor_free(x_tensor);
    4565
    0
      ccv_nnc_tensor_free(x16_tensor);
    4566
    0
      ccv_nnc_tensor_free(y_tensor);
    4567
    0
      ccv_nnc_tensor_free(cpu_y);
    4568
    0
      ccv_nnc_tensor_free(cpu_y16);
    4569
    0
    }
    4570
    4571
    TEST_CASE("compare max pooling with mps")
    4572
    1
    {
    4573
    1
      GUARD_ELSE_RETURN(ccv_nnc_cmd_ok(CCV_NNC_MAX_POOL_FORWARD, CCV_NNC_BACKEND_MPS));
    4574
    0
      ccv_nnc_symbolic_graph_t* symbolic_graph = ccv_nnc_symbolic_graph_new();
    4575
    0
      ccv_nnc_tensor_symbol_t x = ccv_nnc_tensor_symbol_new(symbolic_graph, GPU_TENSOR_NHWC(000, 32F, 7, 7, 10), "x");
    4576
    0
      ccv_nnc_tensor_symbol_t y = ccv_nnc_tensor_symbol_new(symbolic_graph, GPU_TENSOR_NHWC(000, 32F, 3, 3, 10), "y");
    4577
    0
      ccv_nnc_graph_exec_symbol_t max_pool = ccv_nnc_graph_exec_symbol_new(symbolic_graph, CMD_MAX_POOL_FORWARD(5, 5), TENSOR_SYMBOL_LIST(x), TENSOR_SYMBOL_LIST(y), "max_pool");
    4578
    0
      ccv_nnc_graph_exec_symbol_set_hint(symbolic_graph, max_pool, HINT((2, 2), (1, 1)));
    4579
    0
      ccv_nnc_graph_exec_symbol_autogen(symbolic_graph, 0, 0, CCV_NNC_AUTOGEN_ALL_EXECS | CCV_NNC_AUTOGEN_SOURCES_AND_DESTINATIONS);
    4580
    0
      ccv_nnc_graph_t* graph = 0;
    4581
    0
      ccv_nnc_tensor_arena_t* tensor_arena = 0;
    4582
    0
      ccv_nnc_graph_exec_arena_t* graph_exec_arena = 0;
    4583
    0
      ccv_nnc_symbolic_graph_compile(symbolic_graph, ccv_nnc_default_compile_params, 0, 0, 0, 0, SYMBOLIC_GRAPH_SOURCES(symbolic_graph), SYMBOLIC_GRAPH_DESTINATIONS(symbolic_graph), &graph, &tensor_arena, &graph_exec_arena);
    4584
    0
      SYMBOLIC_GRAPH_GEN(symbolic_graph, CCV_NNC_LONG_DOT_GRAPH);
    4585
    0
      GRAPH_GEN(graph, CCV_NNC_LONG_DOT_GRAPH);
    4586
    0
      dsfmt_t dsfmt;
    4587
    0
      dsfmt_init_gen_rand(&dsfmt, 0);
    4588
    0
      ccv_nnc_tensor_t* const x_tensor = ccv_nnc_tensor_new(0, CPU_TENSOR_NHWC(32F, 7, 7, 10), 0);
    4589
    0
      int i;
    4590
    0
      for (i = 0; i < 7 * 7 * 10; i++)
    4591
    0
        x_tensor->data.f32[i] = dsfmt_genrand_open_close(&dsfmt);
    4592
    0
      ccv_nnc_tensor_t* const xt = ccv_nnc_tensor_from_symbol(tensor_arena, x);
    4593
    0
      ccv_nnc_cmd_exec(CMD_DATA_TRANSFER_FORWARD(), ccv_nnc_no_hint, 0, TENSOR_LIST(x_tensor), TENSOR_LIST(xt), 0);
    4594
    0
      ccv_nnc_graph_run(graph, 0, TRAVERSE_FULL, 0, 0);
    4595
    0
      ccv_nnc_tensor_t* const y_tensor = ccv_nnc_tensor_new(0, CPU_TENSOR_NHWC(32F, 3, 3, 10), 0);
    4596
    0
      ccv_nnc_cmd_exec(CMD_MAX_POOL_FORWARD(5, 5), HINT((2, 2), (1, 1)), 0, TENSOR_LIST(x_tensor), TENSOR_LIST(y_tensor), 0);
    4597
    0
      ccv_nnc_tensor_t* const yt = ccv_nnc_tensor_from_symbol(tensor_arena, y);
    4598
    0
      ccv_nnc_tensor_t* const cpu_y = ccv_nnc_tensor_new(0, CPU_TENSOR_NHWC(32F, 3, 3, 10), 0);
    4599
    0
      ccv_nnc_cmd_exec(CMD_DATA_TRANSFER_FORWARD(), ccv_nnc_no_hint, 0, TENSOR_LIST(yt), TENSOR_LIST(cpu_y), 0);
    4600
    0
      REQUIRE_TENSOR_EQ(y_tensor, cpu_y, "mps result should equal to cpu result");
    4601
    0
      ccv_nnc_symbolic_graph_free(symbolic_graph);
    4602
    0
      ccv_nnc_tensor_arena_free(tensor_arena);
    4603
    0
      ccv_nnc_graph_exec_arena_free(graph_exec_arena);
    4604
    0
      ccv_nnc_graph_free(graph);
    4605
    0
      ccv_nnc_tensor_free(x_tensor);
    4606
    0
      ccv_nnc_tensor_free(y_tensor);
    4607
    0
      ccv_nnc_tensor_free(cpu_y);
    4608
    0
    }
    4609
    4610
    TEST_CASE("compare max pooling with mps in half precision")
    4611
    1
    {
    4612
    1
      GUARD_ELSE_RETURN(ccv_nnc_cmd_ok(CCV_NNC_MAX_POOL_FORWARD, CCV_NNC_BACKEND_MPS));
    4613
    0
      ccv_nnc_symbolic_graph_t* symbolic_graph = ccv_nnc_symbolic_graph_new();
    4614
    0
      ccv_nnc_tensor_symbol_t x = ccv_nnc_tensor_symbol_new(symbolic_graph, GPU_TENSOR_NHWC(000, 16F, 7, 7, 10), "x");
    4615
    0
      ccv_nnc_tensor_symbol_t y = ccv_nnc_tensor_symbol_new(symbolic_graph, GPU_TENSOR_NHWC(000, 16F, 3, 3, 10), "y");
    4616
    0
      ccv_nnc_graph_exec_symbol_t max_pool = ccv_nnc_graph_exec_symbol_new(symbolic_graph, CMD_MAX_POOL_FORWARD(5, 5), TENSOR_SYMBOL_LIST(x), TENSOR_SYMBOL_LIST(y), "max_pool");
    4617
    0
      ccv_nnc_graph_exec_symbol_set_hint(symbolic_graph, max_pool, HINT((2, 2), (1, 1)));
    4618
    0
      ccv_nnc_graph_exec_symbol_autogen(symbolic_graph, 0, 0, CCV_NNC_AUTOGEN_ALL_EXECS | CCV_NNC_AUTOGEN_SOURCES_AND_DESTINATIONS);
    4619
    0
      ccv_nnc_graph_t* graph = 0;
    4620
    0
      ccv_nnc_tensor_arena_t* tensor_arena = 0;
    4621
    0
      ccv_nnc_graph_exec_arena_t* graph_exec_arena = 0;
    4622
    0
      ccv_nnc_symbolic_graph_compile(symbolic_graph, ccv_nnc_default_compile_params, 0, 0, 0, 0, SYMBOLIC_GRAPH_SOURCES(symbolic_graph), SYMBOLIC_GRAPH_DESTINATIONS(symbolic_graph), &graph, &tensor_arena, &graph_exec_arena);
    4623
    0
      SYMBOLIC_GRAPH_GEN(symbolic_graph, CCV_NNC_LONG_DOT_GRAPH);
    4624
    0
      GRAPH_GEN(graph, CCV_NNC_LONG_DOT_GRAPH);
    4625
    0
      dsfmt_t dsfmt;
    4626
    0
      dsfmt_init_gen_rand(&dsfmt, 0);
    4627
    0
      ccv_nnc_tensor_t* const x_tensor = ccv_nnc_tensor_new(0, CPU_TENSOR_NHWC(32F, 7, 7, 10), 0);
    4628
    0
      int i;
    4629
    0
      for (i = 0; i < 7 * 7 * 10; i++)
    4630
    0
        x_tensor->data.f32[i] = dsfmt_genrand_open_close(&dsfmt);
    4631
    0
      ccv_nnc_tensor_t* const xt = ccv_nnc_tensor_from_symbol(tensor_arena, x);
    4632
    0
      ccv_nnc_tensor_t* const x16_tensor = ccv_nnc_tensor_new(0, CPU_TENSOR_NHWC(16F, 7, 7, 10), 0);
    4633
    0
      ccv_nnc_cmd_exec(CMD_DATATYPE_CONVERSION_FORWARD(), ccv_nnc_no_hint, 0, TENSOR_LIST(x_tensor), TENSOR_LIST(x16_tensor), 0);
    4634
    0
      ccv_nnc_cmd_exec(CMD_DATA_TRANSFER_FORWARD(), ccv_nnc_no_hint, 0, TENSOR_LIST(x16_tensor), TENSOR_LIST(xt), 0);
    4635
    0
      ccv_nnc_graph_run(graph, 0, TRAVERSE_FULL, 0, 0);
    4636
    0
      ccv_nnc_tensor_t* const y_tensor = ccv_nnc_tensor_new(0, CPU_TENSOR_NHWC(32F, 3, 3, 10), 0);
    4637
    0
      ccv_nnc_cmd_exec(CMD_MAX_POOL_FORWARD(5, 5), HINT((2, 2), (1, 1)), 0, TENSOR_LIST(x_tensor), TENSOR_LIST(y_tensor), 0);
    4638
    0
      ccv_nnc_tensor_t* const yt = ccv_nnc_tensor_from_symbol(tensor_arena, y);
    4639
    0
      ccv_nnc_tensor_t* const cpu_y16 = ccv_nnc_tensor_new(0, CPU_TENSOR_NHWC(16F, 3, 3, 10), 0);
    4640
    0
      ccv_nnc_tensor_t* const cpu_y = ccv_nnc_tensor_new(0, CPU_TENSOR_NHWC(32F, 3, 3, 10), 0);
    4641
    0
      ccv_nnc_cmd_exec(CMD_DATA_TRANSFER_FORWARD(), ccv_nnc_no_hint, 0, TENSOR_LIST(yt), TENSOR_LIST(cpu_y16), 0);
    4642
    0
      ccv_nnc_cmd_exec(CMD_DATATYPE_CONVERSION_FORWARD(), ccv_nnc_no_hint, 0, TENSOR_LIST(cpu_y16), TENSOR_LIST(cpu_y), 0);
    4643
    0
      REQUIRE_ARRAY_EQ_WITH_TOLERANCE(float, y_tensor->data.f32, cpu_y->data.f32, 3 * 3 * 10, 1e-3, "mps result should equal to cpu result");
    4644
    0
      ccv_nnc_symbolic_graph_free(symbolic_graph);
    4645
    0
      ccv_nnc_tensor_arena_free(tensor_arena);
    4646
    0
      ccv_nnc_graph_exec_arena_free(graph_exec_arena);
    4647
    0
      ccv_nnc_graph_free(graph);
    4648
    0
      ccv_nnc_tensor_free(x_tensor);
    4649
    0
      ccv_nnc_tensor_free(x16_tensor);
    4650
    0
      ccv_nnc_tensor_free(y_tensor);
    4651
    0
      ccv_nnc_tensor_free(cpu_y);
    4652
    0
      ccv_nnc_tensor_free(cpu_y16);
    4653
    0
    }
    4654
    4655
    TEST_CASE("compare max pooling 2x2 with mps")
    4656
    1
    {
    4657
    1
      GUARD_ELSE_RETURN(ccv_nnc_cmd_ok(CCV_NNC_MAX_POOL_FORWARD, CCV_NNC_BACKEND_MPS));
    4658
    0
      ccv_nnc_symbolic_graph_t* symbolic_graph = ccv_nnc_symbolic_graph_new();
    4659
    0
      ccv_nnc_tensor_symbol_t x = ccv_nnc_tensor_symbol_new(symbolic_graph, GPU_TENSOR_NCHW(000, 32F, 10, 6, 6), "x");
    4660
    0
      ccv_nnc_tensor_symbol_t y = ccv_nnc_tensor_symbol_new(symbolic_graph, GPU_TENSOR_NCHW(000, 32F, 10, 3, 3), "y");
    4661
    0
      ccv_nnc_graph_exec_symbol_t max_pool = ccv_nnc_graph_exec_symbol_new(symbolic_graph, CMD_MAX_POOL_FORWARD(2, 2), TENSOR_SYMBOL_LIST(x), TENSOR_SYMBOL_LIST(y), "max_pool");
    4662
    0
      ccv_nnc_graph_exec_symbol_set_hint(symbolic_graph, max_pool, HINT((2, 2), (0, 0)));
    4663
    0
      ccv_nnc_graph_exec_symbol_autogen(symbolic_graph, 0, 0, CCV_NNC_AUTOGEN_ALL_EXECS | CCV_NNC_AUTOGEN_SOURCES_AND_DESTINATIONS);
    4664
    0
      ccv_nnc_graph_t* graph = 0;
    4665
    0
      ccv_nnc_tensor_arena_t* tensor_arena = 0;
    4666
    0
      ccv_nnc_graph_exec_arena_t* graph_exec_arena = 0;
    4667
    0
      ccv_nnc_symbolic_graph_compile(symbolic_graph, ccv_nnc_default_compile_params, 0, 0, 0, 0, SYMBOLIC_GRAPH_SOURCES(symbolic_graph), SYMBOLIC_GRAPH_DESTINATIONS(symbolic_graph), &graph, &tensor_arena, &graph_exec_arena);
    4668
    0
      SYMBOLIC_GRAPH_GEN(symbolic_graph, CCV_NNC_LONG_DOT_GRAPH);
    4669
    0
      GRAPH_GEN(graph, CCV_NNC_LONG_DOT_GRAPH);
    4670
    0
      dsfmt_t dsfmt;
    4671
    0
      dsfmt_init_gen_rand(&dsfmt, 0);
    4672
    0
      ccv_nnc_tensor_t* const x_tensor = ccv_nnc_tensor_new(0, CPU_TENSOR_NCHW(32F, 10, 6, 6), 0);
    4673
    0
      int i, j;
    4674
    0
      for (i = 0; i < 6 * 6 * 10; i++)
    4675
    0
        x_tensor->data.f32[i] = dsfmt_genrand_open_close(&dsfmt);
    4676
    0
      ccv_nnc_tensor_t* const gt_x = ccv_nnc_tensor_new(0, CPU_TENSOR_NHWC(32F, 6, 6, 10), 0);
    4677
    0
      for (i = 0; i < 10; i++)
    4678
    0
        for (j = 0; j < 6 * 6; j++)
    4679
    0
          gt_x->data.f32[j * 10 + i] = x_tensor->data.f32[i * 6 * 6 + j];
    4680
    0
      ccv_nnc_tensor_t* const xt = ccv_nnc_tensor_from_symbol(tensor_arena, x);
    4681
    0
      ccv_nnc_cmd_exec(CMD_DATA_TRANSFER_FORWARD(), ccv_nnc_no_hint, 0, TENSOR_LIST(x_tensor), TENSOR_LIST(xt), 0);
    4682
    0
      ccv_nnc_graph_run(graph, 0, TRAVERSE_FULL, 0, 0);
    4683
    0
      ccv_nnc_tensor_t* const gt_y= ccv_nnc_tensor_new(0, CPU_TENSOR_NHWC(32F, 3, 3, 10), 0);
    4684
    0
      ccv_nnc_cmd_exec(CMD_MAX_POOL_FORWARD(2, 2), HINT((2, 2), (0, 0)), 0, TENSOR_LIST(gt_x), TENSOR_LIST(gt_y), 0);
    4685
    0
      ccv_nnc_tensor_t* const yt = ccv_nnc_tensor_from_symbol(tensor_arena, y);
    4686
    0
      ccv_nnc_tensor_t* const cpu_y = ccv_nnc_tensor_new(0, CPU_TENSOR_NCHW(32F, 10, 3, 3), 0);
    4687
    0
      ccv_nnc_cmd_exec(CMD_DATA_TRANSFER_FORWARD(), ccv_nnc_no_hint, 0, TENSOR_LIST(yt), TENSOR_LIST(cpu_y), 0);
    4688
    0
      ccv_nnc_tensor_t* const y_tensor = ccv_nnc_tensor_new(0, CPU_TENSOR_NCHW(32F, 10, 3, 3), 0);
    4689
    0
      for (i = 0; i < 10; i++)
    4690
    0
        for (j = 0; j < 3 * 3; j++)
    4691
    0
          y_tensor->data.f32[i * 3 * 3 + j] = gt_y->data.f32[j * 10 + i];
    4692
    0
      REQUIRE_TENSOR_EQ(y_tensor, cpu_y, "mps result should equal to cpu result");
    4693
    0
      ccv_nnc_symbolic_graph_free(symbolic_graph);
    4694
    0
      ccv_nnc_tensor_arena_free(tensor_arena);
    4695
    0
      ccv_nnc_graph_exec_arena_free(graph_exec_arena);
    4696
    0
      ccv_nnc_graph_free(graph);
    4697
    0
      ccv_nnc_tensor_free(x_tensor);
    4698
    0
      ccv_nnc_tensor_free(y_tensor);
    4699
    0
      ccv_nnc_tensor_free(cpu_y);
    4700
    0
    }
    4701
    4702
    TEST_CASE("compare max pooling 2x2 with mps in half precision")
    4703
    1
    {
    4704
    1
      GUARD_ELSE_RETURN(ccv_nnc_cmd_ok(CCV_NNC_MAX_POOL_FORWARD, CCV_NNC_BACKEND_MPS));
    4705
    0
      ccv_nnc_symbolic_graph_t* symbolic_graph = ccv_nnc_symbolic_graph_new();
    4706
    0
      ccv_nnc_tensor_symbol_t x = ccv_nnc_tensor_symbol_new(symbolic_graph, GPU_TENSOR_NCHW(000, 16F, 10, 6, 6), "x");
    4707
    0
      ccv_nnc_tensor_symbol_t y = ccv_nnc_tensor_symbol_new(symbolic_graph, GPU_TENSOR_NCHW(000, 16F, 10, 3, 3), "y");
    4708
    0
      ccv_nnc_graph_exec_symbol_t max_pool = ccv_nnc_graph_exec_symbol_new(symbolic_graph, CMD_MAX_POOL_FORWARD(2, 2), TENSOR_SYMBOL_LIST(x), TENSOR_SYMBOL_LIST(y), "max_pool");
    4709
    0
      ccv_nnc_graph_exec_symbol_set_hint(symbolic_graph, max_pool, HINT((2, 2), (0, 0)));
    4710
    0
      ccv_nnc_graph_exec_symbol_autogen(symbolic_graph, 0, 0, CCV_NNC_AUTOGEN_ALL_EXECS | CCV_NNC_AUTOGEN_SOURCES_AND_DESTINATIONS);
    4711
    0
      ccv_nnc_graph_t* graph = 0;
    4712
    0
      ccv_nnc_tensor_arena_t* tensor_arena = 0;
    4713
    0
      ccv_nnc_graph_exec_arena_t* graph_exec_arena = 0;
    4714
    0
      ccv_nnc_symbolic_graph_compile(symbolic_graph, ccv_nnc_default_compile_params, 0, 0, 0, 0, SYMBOLIC_GRAPH_SOURCES(symbolic_graph), SYMBOLIC_GRAPH_DESTINATIONS(symbolic_graph), &graph, &tensor_arena, &graph_exec_arena);
    4715
    0
      SYMBOLIC_GRAPH_GEN(symbolic_graph, CCV_NNC_LONG_DOT_GRAPH);
    4716
    0
      GRAPH_GEN(graph, CCV_NNC_LONG_DOT_GRAPH);
    4717
    0
      dsfmt_t dsfmt;
    4718
    0
      dsfmt_init_gen_rand(&dsfmt, 0);
    4719
    0
      ccv_nnc_tensor_t* const x_tensor = ccv_nnc_tensor_new(0, CPU_TENSOR_NCHW(32F, 10, 6, 6), 0);
    4720
    0
      int i, j;
    4721
    0
      for (i = 0; i < 6 * 6 * 10; i++)
    4722
    0
        x_tensor->data.f32[i] = dsfmt_genrand_open_close(&dsfmt);
    4723
    0
      ccv_nnc_tensor_t* const gt_x = ccv_nnc_tensor_new(0, CPU_TENSOR_NHWC(32F, 6, 6, 10), 0);
    4724
    0
      for (i = 0; i < 10; i++)
    4725
    0
        for (j = 0; j < 6 * 6; j++)
    4726
    0
          gt_x->data.f32[j * 10 + i] = x_tensor->data.f32[i * 6 * 6 + j];
    4727
    0
      ccv_nnc_tensor_t* const xt = ccv_nnc_tensor_from_symbol(tensor_arena, x);
    4728
    0
      ccv_nnc_tensor_t* const x16_tensor = ccv_nnc_tensor_new(0, CPU_TENSOR_NCHW(16F, 10, 6, 6), 0);
    4729
    0
      ccv_nnc_cmd_exec(CMD_DATATYPE_CONVERSION_FORWARD(), ccv_nnc_no_hint, 0, TENSOR_LIST(x_tensor), TENSOR_LIST(x16_tensor), 0);
    4730
    0
      ccv_nnc_cmd_exec(CMD_DATA_TRANSFER_FORWARD(), ccv_nnc_no_hint, 0, TENSOR_LIST(x16_tensor), TENSOR_LIST(xt), 0);
    4731
    0
      ccv_nnc_graph_run(graph, 0, TRAVERSE_FULL, 0, 0);
    4732
    0
      ccv_nnc_tensor_t* const gt_y= ccv_nnc_tensor_new(0, CPU_TENSOR_NHWC(32F, 3, 3, 10), 0);
    4733
    0
      ccv_nnc_cmd_exec(CMD_MAX_POOL_FORWARD(2, 2), HINT((2, 2), (0, 0)), 0, TENSOR_LIST(gt_x), TENSOR_LIST(gt_y), 0);
    4734
    0
      ccv_nnc_tensor_t* const yt = ccv_nnc_tensor_from_symbol(tensor_arena, y);
    4735
    0
      ccv_nnc_tensor_t* const cpu_y16 = ccv_nnc_tensor_new(0, CPU_TENSOR_NCHW(16F, 10, 3, 3), 0);
    4736
    0
      ccv_nnc_tensor_t* const cpu_y = ccv_nnc_tensor_new(0, CPU_TENSOR_NCHW(32F, 10, 3, 3), 0);
    4737
    0
      ccv_nnc_cmd_exec(CMD_DATA_TRANSFER_FORWARD(), ccv_nnc_no_hint, 0, TENSOR_LIST(yt), TENSOR_LIST(cpu_y16), 0);
    4738
    0
      ccv_nnc_cmd_exec(CMD_DATATYPE_CONVERSION_FORWARD(), ccv_nnc_no_hint, 0, TENSOR_LIST(cpu_y16), TENSOR_LIST(cpu_y), 0);
    4739
    0
      ccv_nnc_tensor_t* const y_tensor = ccv_nnc_tensor_new(0, CPU_TENSOR_NCHW(32F, 10, 3, 3), 0);
    4740
    0
      for (i = 0; i < 10; i++)
    4741
    0
        for (j = 0; j < 3 * 3; j++)
    4742
    0
          y_tensor->data.f32[i * 3 * 3 + j] = gt_y->data.f32[j * 10 + i];
    4743
    0
      REQUIRE_ARRAY_EQ_WITH_TOLERANCE(float, y_tensor->data.f32, cpu_y->data.f32, 10 * 3 * 3, 1e-3, "mps result should equal to cpu result");
    4744
    0
      ccv_nnc_symbolic_graph_free(symbolic_graph);
    4745
    0
      ccv_nnc_tensor_arena_free(tensor_arena);
    4746
    0
      ccv_nnc_graph_exec_arena_free(graph_exec_arena);
    4747
    0
      ccv_nnc_graph_free(graph);
    4748
    0
      ccv_nnc_tensor_free(x_tensor);
    4749
    0
      ccv_nnc_tensor_free(x16_tensor);
    4750
    0
      ccv_nnc_tensor_free(y_tensor);
    4751
    0
      ccv_nnc_tensor_free(cpu_y);
    4752
    0
      ccv_nnc_tensor_free(cpu_y16);
    4753
    0
    }
    4754
    4755
    4756
    TEST_CASE("mps mse mean loss forward")
    4757
    1
    {
    4758
    1
      GUARD_ELSE_RETURN(ccv_nnc_cmd_ok(CCV_NNC_MSE_FORWARD, CCV_NNC_BACKEND_MPS));
    4759
    0
      ccv_nnc_tensor_t* a = ccv_nnc_tensor_new(0, GPU_TENSOR_NHWC(000, 32F, 10, 100), 0);
    4760
    0
      ccv_nnc_tensor_t* b = ccv_nnc_tensor_new(0, GPU_TENSOR_NHWC(000, 32F, 10, 100), 0);
    4761
    0
      ccv_nnc_tensor_t* c = ccv_nnc_tensor_new(0, GPU_TENSOR_NHWC(000, 32F, 10, 1), 0);
    4762
    0
      ccv_nnc_tensor_t* ha = ccv_nnc_tensor_new(0, CPU_TENSOR_NHWC(32F, 10, 100), 0);
    4763
    0
      ccv_nnc_tensor_t* hb = ccv_nnc_tensor_new(0, CPU_TENSOR_NHWC(32F, 10, 100), 0);
    4764
    0
      ccv_nnc_tensor_t* hc = ccv_nnc_tensor_new(0, CPU_TENSOR_NHWC(32F, 10, 1), 0);
    4765
    0
      dsfmt_t dsfmt;
    4766
    0
      dsfmt_init_gen_rand(&dsfmt, 0);
    4767
    0
      int i;
    4768
    0
      for (i = 0; i < 1000; i++)
    4769
    0
        ha->data.f32[i] = dsfmt_genrand_open_close(&dsfmt);
    4770
    0
      for (i = 0; i < 1000; i++)
    4771
    0
        hb->data.f32[i] = dsfmt_genrand_open_close(&dsfmt);
    4772
    0
      ccv_nnc_cmd_exec(CMD_DATA_TRANSFER_FORWARD(), ccv_nnc_no_hint, 0, TENSOR_LIST(ha, hb), TENSOR_LIST(a, b), 0);
    4773
    0
      ccv_nnc_cmd_exec(CMD_MSE_FORWARD(CCV_NNC_MSE_REDUCE_MEAN), ccv_nnc_no_hint, 0, TENSOR_LIST(ha, hb), TENSOR_LIST(hc), 0);
    4774
    0
      ccv_nnc_cmd_exec(CMD_MSE_FORWARD(CCV_NNC_MSE_REDUCE_MEAN), ccv_nnc_no_hint, 0, TENSOR_LIST(a, b), TENSOR_LIST(c), 0);
    4775
    0
      ccv_nnc_tensor_t* tc = ccv_nnc_tensor_new(0, CPU_TENSOR_NHWC(32F, 10, 1), 0);
    4776
    0
      ccv_nnc_cmd_exec(CMD_DATA_TRANSFER_FORWARD(), ccv_nnc_no_hint, 0, TENSOR_LIST(c), TENSOR_LIST(tc), 0);
    4777
    0
      REQUIRE_TENSOR_EQ(tc, hc, "MPS computed output should be the same as CPU computed ones");
    4778
    0
      ccv_nnc_tensor_free(a);
    4779
    0
      ccv_nnc_tensor_free(b);
    4780
    0
      ccv_nnc_tensor_free(c);
    4781
    0
      ccv_nnc_tensor_free(ha);
    4782
    0
      ccv_nnc_tensor_free(hb);
    4783
    0
      ccv_nnc_tensor_free(hc);
    4784
    0
      ccv_nnc_tensor_free(tc);
    4785
    0
    }
    4786
    4787
    TEST_CASE("mps mse sum loss forward")
    4788
    1
    {
    4789
    1
      GUARD_ELSE_RETURN(ccv_nnc_cmd_ok(CCV_NNC_MSE_FORWARD, CCV_NNC_BACKEND_MPS));
    4790
    0
      ccv_nnc_tensor_t* a = ccv_nnc_tensor_new(0, GPU_TENSOR_NHWC(000, 32F, 10, 100), 0);
    4791
    0
      ccv_nnc_tensor_t* b = ccv_nnc_tensor_new(0, GPU_TENSOR_NHWC(000, 32F, 10, 100), 0);
    4792
    0
      ccv_nnc_tensor_t* c = ccv_nnc_tensor_new(0, GPU_TENSOR_NHWC(000, 32F, 10, 1), 0);
    4793
    0
      ccv_nnc_tensor_t* ha = ccv_nnc_tensor_new(0, CPU_TENSOR_NHWC(32F, 10, 100), 0);
    4794
    0
      ccv_nnc_tensor_t* hb = ccv_nnc_tensor_new(0, CPU_TENSOR_NHWC(32F, 10, 100), 0);
    4795
    0
      ccv_nnc_tensor_t* hc = ccv_nnc_tensor_new(0, CPU_TENSOR_NHWC(32F, 10, 1), 0);
    4796
    0
      dsfmt_t dsfmt;
    4797
    0
      dsfmt_init_gen_rand(&dsfmt, 0);
    4798
    0
      int i;
    4799
    0
      for (i = 0; i < 1000; i++)
    4800
    0
        ha->data.f32[i] = dsfmt_genrand_open_close(&dsfmt);
    4801
    0
      for (i = 0; i < 1000; i++)
    4802
    0
        hb->data.f32[i] = dsfmt_genrand_open_close(&dsfmt);
    4803
    4804
    0
      ccv_nnc_cmd_exec(CMD_DATA_TRANSFER_FORWARD(), ccv_nnc_no_hint, 0, TENSOR_LIST(ha, hb), TENSOR_LIST(a, b), 0);
    4805
    0
      ccv_nnc_cmd_exec(CMD_MSE_FORWARD(CCV_NNC_MSE_REDUCE_SUM), ccv_nnc_no_hint, 0, TENSOR_LIST(ha, hb), TENSOR_LIST(hc), 0);
    4806
    0
      ccv_nnc_cmd_exec(CMD_MSE_FORWARD(CCV_NNC_MSE_REDUCE_SUM), ccv_nnc_no_hint, 0, TENSOR_LIST(a, b), TENSOR_LIST(c), 0);
    4807
    0
      ccv_nnc_tensor_t* tc = ccv_nnc_tensor_new(0, CPU_TENSOR_NHWC(32F, 10, 1), 0);
    4808
    0
      ccv_nnc_cmd_exec(CMD_DATA_TRANSFER_FORWARD(), ccv_nnc_no_hint, 0, TENSOR_LIST(c), TENSOR_LIST(tc), 0);
    4809
    4810
    0
      REQUIRE_TENSOR_EQ(tc, hc, "MPS computed output should be the same as CPU computed ones");
    4811
    0
      ccv_nnc_tensor_free(a);
    4812
    0
      ccv_nnc_tensor_free(b);
    4813
    0
      ccv_nnc_tensor_free(c);
    4814
    0
      ccv_nnc_tensor_free(ha);
    4815
    0
      ccv_nnc_tensor_free(hb);
    4816
    0
      ccv_nnc_tensor_free(hc);
    4817
    0
      ccv_nnc_tensor_free(tc);
    4818
    0
    }
    4819
    4820
    TEST_CASE("mps mse mean loss backward")
    4821
    1
    {
    4822
    1
      GUARD_ELSE_RETURN(ccv_nnc_cmd_ok(CCV_NNC_MSE_FORWARD, CCV_NNC_BACKEND_MPS) &&
    4823
    1
      ccv_nnc_cmd_ok(CCV_NNC_MSE_BACKWARD, CCV_NNC_BACKEND_MPS));
    4824
    4825
    0
      ccv_nnc_tensor_t* a = ccv_nnc_tensor_new(0, GPU_TENSOR_NHWC(000, 32F, 10, 100), 0);
    4826
    0
      ccv_nnc_tensor_t* b = ccv_nnc_tensor_new(0, GPU_TENSOR_NHWC(000, 32F, 10, 100), 0);
    4827
    0
      ccv_nnc_tensor_t* c = ccv_nnc_tensor_new(0, GPU_TENSOR_NHWC(000, 32F, 10), 0);
    4828
    0
      ccv_nnc_tensor_t* da = ccv_nnc_tensor_new(0, GPU_TENSOR_NHWC(000, 32F, 10, 100), 0);
    4829
    0
      ccv_nnc_tensor_t* db = ccv_nnc_tensor_new(0, GPU_TENSOR_NHWC(000, 32F, 10, 100), 0);
    4830
    0
      ccv_nnc_tensor_t* g = ccv_nnc_tensor_new(0, GPU_TENSOR_NHWC(000, 32F, 10), 0);
    4831
    0
      ccv_nnc_tensor_t* ha = ccv_nnc_tensor_new(0, CPU_TENSOR_NHWC(32F, 10, 100), 0);
    4832
    0
      ccv_nnc_tensor_t* hb = ccv_nnc_tensor_new(0, CPU_TENSOR_NHWC(32F, 10, 100), 0);
    4833
    0
      ccv_nnc_tensor_t* hc = ccv_nnc_tensor_new(0, CPU_TENSOR_NHWC(32F, 10), 0);
    4834
    0
      ccv_nnc_tensor_t* hda = ccv_nnc_tensor_new(0, CPU_TENSOR_NHWC(32F, 10, 100), 0);
    4835
    0
      ccv_nnc_tensor_t* hdb = ccv_nnc_tensor_new(0, CPU_TENSOR_NHWC(32F, 10, 100), 0);
    4836
    0
      ccv_nnc_tensor_t* hg = ccv_nnc_tensor_new(0, CPU_TENSOR_NHWC(32F, 10), 0);
    4837
    0
      dsfmt_t dsfmt;
    4838
    0
      dsfmt_init_gen_rand(&dsfmt, 0);
    4839
    0
      int i;
    4840
    0
      for (i = 0; i < 1000; i++)
    4841
    0
        ha->data.f32[i] = dsfmt_genrand_open_close(&dsfmt);
    4842
    0
      for (i = 0; i < 1000; i++)
    4843
    0
        hb->data.f32[i] = dsfmt_genrand_open_close(&dsfmt);
    4844
    0
      for (i = 0; i < 10; i++)
    4845
    0
        hg->data.f32[i] = 1;
    4846
    0
      ccv_nnc_cmd_exec(CMD_DATA_TRANSFER_FORWARD(), ccv_nnc_no_hint, 0, TENSOR_LIST(ha, hb, hg), TENSOR_LIST(a, b, g), 0);
    4847
    0
      ccv_nnc_cmd_exec(CMD_MSE_FORWARD(CCV_NNC_MSE_REDUCE_MEAN), ccv_nnc_no_hint, 0, TENSOR_LIST(ha, hb), TENSOR_LIST(hc), 0);
    4848
    0
      ccv_nnc_cmd_exec(CMD_MSE_BACKWARD(CCV_NNC_MSE_REDUCE_MEAN), ccv_nnc_no_hint, 0, TENSOR_LIST(hg, ha, hb), TENSOR_LIST(hda, hdb), 0);
    4849
    0
      ccv_nnc_cmd_exec(CMD_MSE_FORWARD(CCV_NNC_MSE_REDUCE_MEAN), ccv_nnc_no_hint, 0, TENSOR_LIST(a, b), TENSOR_LIST(c), 0);
    4850
    0
      ccv_nnc_cmd_exec(CMD_MSE_BACKWARD(CCV_NNC_MSE_REDUCE_MEAN), ccv_nnc_no_hint, 0, TENSOR_LIST(g, a, b), TENSOR_LIST(da, db), 0);
    4851
    0
      ccv_nnc_tensor_t* tda = ccv_nnc_tensor_new(0, CPU_TENSOR_NHWC(32F, 10, 100), 0);
    4852
    0
      ccv_nnc_tensor_t* tdb = ccv_nnc_tensor_new(0, CPU_TENSOR_NHWC(32F, 10, 100), 0);
    4853
    0
      ccv_nnc_cmd_exec(CMD_DATA_TRANSFER_FORWARD(), ccv_nnc_no_hint, 0, TENSOR_LIST(da, db), TENSOR_LIST(tda, tdb), 0);
    4854
    4855
    0
      REQUIRE_TENSOR_EQ(tda, hda, "MPS computed output should be the same as CPU computed ones");
    4856
    0
      REQUIRE_TENSOR_EQ(tdb, hdb, "MPS computed output should be the same as CPU computed ones");
    4857
    4858
    0
      ccv_nnc_tensor_free(a);
    4859
    0
      ccv_nnc_tensor_free(b);
    4860
    0
      ccv_nnc_tensor_free(c);
    4861
    0
      ccv_nnc_tensor_free(da);
    4862
    0
      ccv_nnc_tensor_free(db);
    4863
    0
      ccv_nnc_tensor_free(g);
    4864
    0
      ccv_nnc_tensor_free(ha);
    4865
    0
      ccv_nnc_tensor_free(hb);
    4866
    0
      ccv_nnc_tensor_free(hc);
    4867
    0
      ccv_nnc_tensor_free(hda);
    4868
    0
      ccv_nnc_tensor_free(hdb);
    4869
    0
      ccv_nnc_tensor_free(hg);
    4870
    0
      ccv_nnc_tensor_free(tda);
    4871
    0
      ccv_nnc_tensor_free(tdb);
    4872
    0
    }
    4873
    4874
    TEST_CASE("mps mse sum loss backward")
    4875
    1
    {
    4876
    1
      GUARD_ELSE_RETURN(ccv_nnc_cmd_ok(CCV_NNC_MSE_FORWARD, CCV_NNC_BACKEND_MPS) &&
    4877
    1
        ccv_nnc_cmd_ok(CCV_NNC_MSE_BACKWARD, CCV_NNC_BACKEND_MPS));
    4878
    0
      ccv_nnc_tensor_t* a = ccv_nnc_tensor_new(0, GPU_TENSOR_NHWC(000, 32F, 10, 100), 0);
    4879
    0
      ccv_nnc_tensor_t* b = ccv_nnc_tensor_new(0, GPU_TENSOR_NHWC(000, 32F, 10, 100), 0);
    4880
    0
      ccv_nnc_tensor_t* c = ccv_nnc_tensor_new(0, GPU_TENSOR_NHWC(000, 32F, 10), 0);
    4881
    0
      ccv_nnc_tensor_t* da = ccv_nnc_tensor_new(0, GPU_TENSOR_NHWC(000, 32F, 10, 100), 0);
    4882
    0
      ccv_nnc_tensor_t* db = ccv_nnc_tensor_new(0, GPU_TENSOR_NHWC(000, 32F, 10, 100), 0);
    4883
    0
      ccv_nnc_tensor_t* g = ccv_nnc_tensor_new(0, GPU_TENSOR_NHWC(000, 32F, 10), 0);
    4884
    0
      ccv_nnc_tensor_t* ha = ccv_nnc_tensor_new(0, CPU_TENSOR_NHWC(32F, 10, 100), 0);
    4885
    0
      ccv_nnc_tensor_t* hb = ccv_nnc_tensor_new(0, CPU_TENSOR_NHWC(32F, 10, 100), 0);
    4886
    0
      ccv_nnc_tensor_t* hc = ccv_nnc_tensor_new(0, CPU_TENSOR_NHWC(32F, 10), 0);
    4887
    0
      ccv_nnc_tensor_t* hda = ccv_nnc_tensor_new(0, CPU_TENSOR_NHWC(32F, 10, 100), 0);
    4888
    0
      ccv_nnc_tensor_t* hdb = ccv_nnc_tensor_new(0, CPU_TENSOR_NHWC(32F, 10, 100), 0);
    4889
    0
      ccv_nnc_tensor_t* hg = ccv_nnc_tensor_new(0, CPU_TENSOR_NHWC(32F, 10), 0);
    4890
    0
      dsfmt_t dsfmt;
    4891
    0
      dsfmt_init_gen_rand(&dsfmt, 0);
    4892
    0
      int i;
    4893
    0
      for (i = 0; i < 1000; i++)
    4894
    0
        ha->data.f32[i] = dsfmt_genrand_open_close(&dsfmt);
    4895
    0
      for (i = 0; i < 1000; i++)
    4896
    0
        hb->data.f32[i] = dsfmt_genrand_open_close(&dsfmt);
    4897
    0
      for (i = 0; i < 10; i++)
    4898
    0
        hg->data.f32[i] = 1;
    4899
    0
      ccv_nnc_cmd_exec(CMD_DATA_TRANSFER_FORWARD(), ccv_nnc_no_hint, 0, TENSOR_LIST(ha, hb, hg), TENSOR_LIST(a, b, g), 0);
    4900
    0
      ccv_nnc_cmd_exec(CMD_MSE_FORWARD(CCV_NNC_MSE_REDUCE_SUM), ccv_nnc_no_hint, 0, TENSOR_LIST(ha, hb), TENSOR_LIST(hc), 0);
    4901
    0
      ccv_nnc_cmd_exec(CMD_MSE_BACKWARD(CCV_NNC_MSE_REDUCE_SUM), ccv_nnc_no_hint, 0, TENSOR_LIST(hg, ha, hb), TENSOR_LIST(hda, hdb), 0);
    4902
    0
      ccv_nnc_cmd_exec(CMD_MSE_FORWARD(CCV_NNC_MSE_REDUCE_SUM), ccv_nnc_no_hint, 0, TENSOR_LIST(a, b), TENSOR_LIST(c), 0);
    4903
    0
      ccv_nnc_cmd_exec(CMD_MSE_BACKWARD(CCV_NNC_MSE_REDUCE_SUM), ccv_nnc_no_hint, 0, TENSOR_LIST(g, a, b), TENSOR_LIST(da, db), 0);
    4904
    0
      ccv_nnc_tensor_t* tda = ccv_nnc_tensor_new(0, CPU_TENSOR_NHWC(32F, 10, 100), 0);
    4905
    0
      ccv_nnc_tensor_t* tdb = ccv_nnc_tensor_new(0, CPU_TENSOR_NHWC(32F, 10, 100), 0);
    4906
    0
      ccv_nnc_cmd_exec(CMD_DATA_TRANSFER_FORWARD(), ccv_nnc_no_hint, 0, TENSOR_LIST(da, db), TENSOR_LIST(tda, tdb), 0);
    4907
    0
      REQUIRE_TENSOR_EQ(tda, hda, "MPS computed output should be the same as CPU computed ones");
    4908
    0
      REQUIRE_TENSOR_EQ(tdb, hdb, "MPS computed output should be the same as CPU computed ones");
    4909
    0
      ccv_nnc_tensor_free(a);
    4910
    0
      ccv_nnc_tensor_free(b);
    4911
    0
      ccv_nnc_tensor_free(c);
    4912
    0
      ccv_nnc_tensor_free(da);
    4913
    0
      ccv_nnc_tensor_free(db);
    4914
    0
      ccv_nnc_tensor_free(g);
    4915
    0
      ccv_nnc_tensor_free(ha);
    4916
    0
      ccv_nnc_tensor_free(hb);
    4917
    0
      ccv_nnc_tensor_free(hc);
    4918
    0
      ccv_nnc_tensor_free(hda);
    4919
    0
      ccv_nnc_tensor_free(hdb);
    4920
    0
      ccv_nnc_tensor_free(hg);
    4921
    0
      ccv_nnc_tensor_free(tda);
    4922
    0
      ccv_nnc_tensor_free(tdb);
    4923
    0
    }
    4924
    4925
    4926
    TEST_CASE("mps mse sum loss backward (no output db)")
    4927
    1
    {
    4928
    1
      GUARD_ELSE_RETURN(ccv_nnc_cmd_ok(CCV_NNC_MSE_FORWARD, CCV_NNC_BACKEND_MPS) &&
    4929
    1
        ccv_nnc_cmd_ok(CCV_NNC_MSE_BACKWARD, CCV_NNC_BACKEND_MPS));
    4930
    0
      ccv_nnc_tensor_t* a = ccv_nnc_tensor_new(0, GPU_TENSOR_NHWC(000, 32F, 10, 100), 0);
    4931
    0
      ccv_nnc_tensor_t* b = ccv_nnc_tensor_new(0, GPU_TENSOR_NHWC(000, 32F, 10, 100), 0);
    4932
    0
      ccv_nnc_tensor_t* c = ccv_nnc_tensor_new(0, GPU_TENSOR_NHWC(000, 32F, 10), 0);
    4933
    0
      ccv_nnc_tensor_t* da = ccv_nnc_tensor_new(0, GPU_TENSOR_NHWC(000, 32F, 10, 100), 0);
    4934
    0
      ccv_nnc_tensor_t* g = ccv_nnc_tensor_new(0, GPU_TENSOR_NHWC(000, 32F, 10), 0);
    4935
    0
      ccv_nnc_tensor_t* ha = ccv_nnc_tensor_new(0, CPU_TENSOR_NHWC(32F, 10, 100), 0);
    4936
    0
      ccv_nnc_tensor_t* hb = ccv_nnc_tensor_new(0, CPU_TENSOR_NHWC(32F, 10, 100), 0);
    4937
    0
      ccv_nnc_tensor_t* hc = ccv_nnc_tensor_new(0, CPU_TENSOR_NHWC(32F, 10), 0);
    4938
    0
      ccv_nnc_tensor_t* hda = ccv_nnc_tensor_new(0, CPU_TENSOR_NHWC(32F, 10, 100), 0);
    4939
    0
      ccv_nnc_tensor_t* hg = ccv_nnc_tensor_new(0, CPU_TENSOR_NHWC(32F, 10), 0);
    4940
    0
      dsfmt_t dsfmt;
    4941
    0
      dsfmt_init_gen_rand(&dsfmt, 0);
    4942
    0
      int i;
    4943
    0
      for (i = 0; i < 1000; i++)
    4944
    0
        ha->data.f32[i] = dsfmt_genrand_open_close(&dsfmt);
    4945
    0
      for (i = 0; i < 1000; i++)
    4946
    0
        hb->data.f32[i] = dsfmt_genrand_open_close(&dsfmt);
    4947
    0
      for (i = 0; i < 10; i++)
    4948
    0
        hg->data.f32[i] = 1;
    4949
    0
      ccv_nnc_cmd_exec(CMD_DATA_TRANSFER_FORWARD(), ccv_nnc_no_hint, 0, TENSOR_LIST(ha, hb, hg), TENSOR_LIST(a, b, g), 0);
    4950
    0
      ccv_nnc_cmd_exec(CMD_MSE_FORWARD(CCV_NNC_MSE_REDUCE_SUM), ccv_nnc_no_hint, 0, TENSOR_LIST(ha, hb), TENSOR_LIST(hc), 0);
    4951
    0
      ccv_nnc_cmd_exec(CMD_MSE_BACKWARD(CCV_NNC_MSE_REDUCE_SUM), ccv_nnc_no_hint, 0, TENSOR_LIST(hg, ha, hb), TENSOR_LIST(hda, 0), 0);
    4952
    0
      ccv_nnc_cmd_exec(CMD_MSE_FORWARD(CCV_NNC_MSE_REDUCE_SUM), ccv_nnc_no_hint, 0, TENSOR_LIST(a, b), TENSOR_LIST(c), 0);
    4953
    0
      ccv_nnc_cmd_exec(CMD_MSE_BACKWARD(CCV_NNC_MSE_REDUCE_SUM), ccv_nnc_no_hint, 0, TENSOR_LIST(g, a, b), TENSOR_LIST(da, 0), 0);
    4954
    0
      ccv_nnc_tensor_t* tda = ccv_nnc_tensor_new(0, CPU_TENSOR_NHWC(32F, 10, 100), 0);
    4955
    0
      ccv_nnc_tensor_t* tdb = ccv_nnc_tensor_new(0, CPU_TENSOR_NHWC(32F, 10, 100), 0);
    4956
    0
      ccv_nnc_cmd_exec(CMD_DATA_TRANSFER_FORWARD(), ccv_nnc_no_hint, 0, TENSOR_LIST(da, 0), TENSOR_LIST(tda, 0), 0);
    4957
    0
      REQUIRE_TENSOR_EQ(tda, hda, "MPS computed output should be the same as CPU computed ones");
    4958
    0
      ccv_nnc_tensor_free(a);
    4959
    0
      ccv_nnc_tensor_free(b);
    4960
    0
      ccv_nnc_tensor_free(c);
    4961
    0
      ccv_nnc_tensor_free(da);
    4962
    0
      ccv_nnc_tensor_free(g);
    4963
    0
      ccv_nnc_tensor_free(ha);
    4964
    0
      ccv_nnc_tensor_free(hb);
    4965
    0
      ccv_nnc_tensor_free(hc);
    4966
    0
      ccv_nnc_tensor_free(hda);
    4967
    0
      ccv_nnc_tensor_free(hg);
    4968
    0
      ccv_nnc_tensor_free(tda);
    4969
    0
      ccv_nnc_tensor_free(tdb);
    4970
    0
    }
    4971
    4972
    TEST_CASE("mps leaky relu gradient in float")
    4973
    1
    {
    4974
    1
      GUARD_ELSE_RETURN(ccv_nnc_cmd_ok(CCV_NNC_LEAKY_RELU_FORWARD, CCV_NNC_BACKEND_MPS) &&
    4975
    1
        ccv_nnc_cmd_ok(CCV_NNC_LEAKY_RELU_BACKWARD, CCV_NNC_BACKEND_MPS));
    4976
    0
      ccv_nnc_symbolic_graph_t* const symbolic_graph = ccv_nnc_symbolic_graph_new();
    4977
    0
      ccv_nnc_tensor_symbol_t x = ccv_nnc_tensor_symbol_new(symbolic_graph, GPU_TENSOR_NHWC(000, 32F, 10, 100), "x");
    4978
    0
      ccv_nnc_tensor_symbol_t y = ccv_nnc_tensor_symbol_new(symbolic_graph, GPU_TENSOR_NHWC(000, 32F, 10, 100), "y");
    4979
    0
      ccv_nnc_graph_exec_symbol_new(symbolic_graph, CMD_LEAKY_RELU_FORWARD(0.2), TENSOR_SYMBOL_LIST(x), TENSOR_SYMBOL_LIST(y), "leaky relu");
    4980
    0
      ccv_nnc_graph_exec_symbol_autogen(symbolic_graph, 0, 0, CCV_NNC_AUTOGEN_ALL_EXECS | CCV_NNC_AUTOGEN_SOURCES_AND_DESTINATIONS);
    4981
    0
      ccv_nnc_symbolic_graph_backward(symbolic_graph, TENSOR_SYMBOL_LIST(y), TENSOR_SYMBOL_LIST(x), SYMBOLIC_GRAPH_SOURCES(symbolic_graph), SYMBOLIC_GRAPH_DESTINATIONS(symbolic_graph));
    4982
    0
      ccv_nnc_graph_exec_symbol_autogen(symbolic_graph, 0, 0, CCV_NNC_AUTOGEN_ALL_EXECS | CCV_NNC_AUTOGEN_SOURCES_AND_DESTINATIONS);
    4983
    0
      SYMBOLIC_GRAPH_GEN(symbolic_graph, CCV_NNC_LONG_DOT_GRAPH);
    4984
    0
      ccv_nnc_tensor_symbol_t dy = ccv_nnc_tensor_symbol_for_backward(symbolic_graph, y);
    4985
    0
      ccv_nnc_tensor_symbol_t dx = ccv_nnc_tensor_symbol_for_backward(symbolic_graph, x);
    4986
    0
      ccv_nnc_tensor_t* const x_tensor = ccv_nnc_tensor_new(0, CPU_TENSOR_NHWC(32F, 10, 100), 0);
    4987
    0
      dsfmt_t dsfmt;
    4988
    0
      dsfmt_init_gen_rand(&dsfmt, 0);
    4989
    0
      int i;
    4990
    0
      for (i = 0; i < 10 * 100; i++)
    4991
    0
        x_tensor->data.f32[i] = dsfmt_genrand_open_close(&dsfmt);
    4992
    0
      ccv_nnc_tensor_t* const dy_tensor = ccv_nnc_tensor_new(0, CPU_TENSOR_NHWC(32F, 10, 100), 0);
    4993
    0
      for (i = 0; i < 10 * 100; i++)
    4994
    0
        dy_tensor->data.f32[i] = 0;
    4995
    0
      for (i = 0; i < 10; i++)
    4996
    0
        dy_tensor->data.f32[i * 100 + i] = 1;
    4997
    0
      ccv_nnc_tensor_t* const dyt = ccv_nnc_tensor_new(0, GPU_TENSOR_NHWC(000, 32F, 10, 100), 0);
    4998
    0
      ccv_nnc_cmd_exec(CMD_DATA_TRANSFER_FORWARD(), ccv_nnc_no_hint, 0, TENSOR_LIST(dy_tensor), TENSOR_LIST(dyt), 0);
    4999
    0
      ccv_nnc_graph_t* graph = 0;
    5000
    0
      ccv_nnc_tensor_arena_t* tensor_arena = 0;
    5001
    0
      ccv_nnc_graph_exec_arena_t* graph_exec_arena = 0;
    5002
    0
      ccv_nnc_symbolic_graph_compile(symbolic_graph, ccv_nnc_default_compile_params, TENSOR_BIND_MAP(KV(dy, dyt)), TENSOR_SYMBOL_LIST(y), SYMBOLIC_GRAPH_SOURCES(symbolic_graph), SYMBOLIC_GRAPH_DESTINATIONS(symbolic_graph), &graph, &tensor_arena, &graph_exec_arena);
    5003
    0
      GRAPH_GEN(graph, CCV_NNC_LONG_DOT_GRAPH);
    5004
    0
      ccv_nnc_tensor_t* const xt = ccv_nnc_tensor_from_symbol(tensor_arena, x);
    5005
    0
      ccv_nnc_cmd_exec(CMD_DATA_TRANSFER_FORWARD(), ccv_nnc_no_hint, 0, TENSOR_LIST(x_tensor), TENSOR_LIST(xt), 0);
    5006
    0
      ccv_nnc_graph_run(graph, 0, TRAVERSE_FULL, 0, 0);
    5007
    0
      ccv_nnc_tensor_t* const dx_tensor = ccv_nnc_tensor_new(0, CPU_TENSOR_NHWC(32F, 10, 100), 0);
    5008
    0
      ccv_nnc_tensor_t* const dxt = ccv_nnc_tensor_from_symbol(tensor_arena, dx);
    5009
    0
      ccv_nnc_tensor_t* const y_tensor = ccv_nnc_tensor_new(0, CPU_TENSOR_NHWC(32F, 10, 100), 0);
    5010
    0
      ccv_nnc_tensor_t* const yt = ccv_nnc_tensor_from_symbol(tensor_arena, y);
    5011
    0
      ccv_nnc_cmd_exec(CMD_DATA_TRANSFER_FORWARD(), ccv_nnc_no_hint, 0, TENSOR_LIST(dxt), TENSOR_LIST(dx_tensor), 0);
    5012
    0
      ccv_nnc_cmd_exec(CMD_DATA_TRANSFER_FORWARD(), ccv_nnc_no_hint, 0, TENSOR_LIST(yt), TENSOR_LIST(y_tensor), 0);
    5013
    0
      ccv_nnc_tensor_t* const ty_tensor = ccv_nnc_tensor_new(0, CPU_TENSOR_NHWC(32F, 10, 100), 0);
    5014
    0
      ccv_nnc_cmd_exec(CMD_LEAKY_RELU_FORWARD(0.2), ccv_nnc_no_hint, 0, TENSOR_LIST(x_tensor), TENSOR_LIST(ty_tensor), 0);
    5015
    0
      REQUIRE_TENSOR_EQ(ty_tensor, y_tensor, "forward pass should match");
    5016
    0
      ccv_nnc_tensor_t* const tdx_tensor = ccv_nnc_tensor_new(0, CPU_TENSOR_NHWC(32F, 10, 100), 0);
    5017
    0
      ccv_nnc_cmd_exec(CMD_LEAKY_RELU_BACKWARD(0.2), ccv_nnc_no_hint, 0, TENSOR_LIST(dy_tensor, 0, y_tensor), TENSOR_LIST(tdx_tensor), 0);
    5018
    0
      REQUIRE_TENSOR_EQ(tdx_tensor, dx_tensor, "backward pass should match");
    5019
    0
      ccv_nnc_tensor_free(x_tensor);
    5020
    0
      ccv_nnc_tensor_free(y_tensor);
    5021
    0
      ccv_nnc_tensor_free(dx_tensor);
    5022
    0
      ccv_nnc_tensor_free(dy_tensor);
    5023
    0
      ccv_nnc_tensor_free(ty_tensor);
    5024
    0
      ccv_nnc_tensor_free(tdx_tensor);
    5025
    0
      ccv_nnc_tensor_free(dyt);
    5026
    0
      ccv_nnc_graph_free(graph);
    5027
    0
      ccv_nnc_tensor_arena_free(tensor_arena);
    5028
    0
      ccv_nnc_graph_exec_arena_free(graph_exec_arena);
    5029
    0
      ccv_nnc_symbolic_graph_free(symbolic_graph);
    5030
    0
    }
    5031
    5032
    TEST_CASE("compare layer norm gradient with mps")
    5033
    1
    {
    5034
    1
      GUARD_ELSE_RETURN(ccv_nnc_cmd_ok(CCV_NNC_LAYER_NORM_FORWARD, CCV_NNC_BACKEND_MPS) &&
    5035
    1
        ccv_nnc_cmd_ok(CCV_NNC_LAYER_NORM_BACKWARD, CCV_NNC_BACKEND_MPS) &&
    5036
    1
        (ccv_nnc_cmd_ok(CCV_NNC_SET_FORWARD, CCV_NNC_BACKEND_MPS) || ccv_nnc_cmd_ok(CCV_NNC_SET_FORWARD, CCV_NNC_BACKEND_MPS)));
    5037
    0
      ccv_nnc_symbolic_graph_t* const symbolic_graph = ccv_nnc_symbolic_graph_new();
    5038
    0
      ccv_nnc_tensor_symbol_t bx = ccv_nnc_tensor_symbol_new(symbolic_graph, GPU_TENSOR_NHWC(000, 32F, 2, 2, 2, LN_DIM), "x");
    5039
    0
      ccv_nnc_tensor_symbol_t by = ccv_nnc_tensor_symbol_new(symbolic_graph, GPU_TENSOR_NHWC(000, 32F, 2, 2, 2, LN_DIM), "y");
    5040
    0
      ccv_nnc_tensor_symbol_t scale = ccv_nnc_tensor_symbol_new(symbolic_graph, GPU_TENSOR_NHWC(000, 32F, 1, 2, 2, LN_DIM), "scale");
    5041
    0
      ccv_nnc_tensor_symbol_t bias = ccv_nnc_tensor_symbol_new(symbolic_graph, GPU_TENSOR_NHWC(000, 32F, 1, 2, 2, LN_DIM), "bias");
    5042
    0
      ccv_nnc_tensor_symbol_t saved_mean = ccv_nnc_tensor_symbol_new(symbolic_graph, GPU_TENSOR_NHWC(000, 32F, 2, 1, 1, 1), "saved_mean");
    5043
    0
      ccv_nnc_tensor_symbol_t saved_inv_std = ccv_nnc_tensor_symbol_new(symbolic_graph, GPU_TENSOR_NHWC(000, 32F, 2, 1, 1, 1), "saved_inv_std");
    5044
    0
      ccv_nnc_graph_exec_symbol_new(symbolic_graph, CMD_LAYER_NORM_FORWARD(1e-4, 1, 1, 2, 3), TENSOR_SYMBOL_LIST(bx, scale, bias), TENSOR_SYMBOL_LIST(by, saved_mean, saved_inv_std), "layer_norm");
    5045
    0
      ccv_nnc_graph_exec_symbol_autogen(symbolic_graph, 0, 0, CCV_NNC_AUTOGEN_ALL_EXECS | CCV_NNC_AUTOGEN_SOURCES_AND_DESTINATIONS);
    5046
    0
      ccv_nnc_symbolic_graph_backward(symbolic_graph, TENSOR_SYMBOL_LIST(by), TENSOR_SYMBOL_LIST(bx, scale, bias), SYMBOLIC_GRAPH_SOURCES(symbolic_graph), SYMBOLIC_GRAPH_DESTINATIONS(symbolic_graph));
    5047
    0
      ccv_nnc_graph_exec_symbol_autogen(symbolic_graph, 0, 0, CCV_NNC_AUTOGEN_ALL_EXECS | CCV_NNC_AUTOGEN_SOURCES_AND_DESTINATIONS);
    5048
    0
      ccv_nnc_tensor_symbol_t dby = ccv_nnc_tensor_symbol_for_backward(symbolic_graph, by);
    5049
    0
      ccv_nnc_tensor_symbol_t dbx = ccv_nnc_tensor_symbol_for_backward(symbolic_graph, bx);
    5050
    0
      ccv_nnc_graph_t* graph = 0;
    5051
    0
      ccv_nnc_tensor_arena_t* tensor_arena = 0;
    5052
    0
      ccv_nnc_graph_exec_arena_t* graph_exec_arena = 0;
    5053
    0
      ccv_nnc_symbolic_graph_compile(symbolic_graph, ccv_nnc_default_compile_params, 0, 0, 0, 0, SYMBOLIC_GRAPH_SOURCES(symbolic_graph), SYMBOLIC_GRAPH_DESTINATIONS(symbolic_graph), &graph, &tensor_arena, &graph_exec_arena);
    5054
    0
      SYMBOLIC_GRAPH_GEN(symbolic_graph, CCV_NNC_LONG_DOT_GRAPH);
    5055
    0
      GRAPH_GEN(graph, CCV_NNC_LONG_DOT_GRAPH);
    5056
    0
      ccv_nnc_tensor_t* const bx_tensor = ccv_nnc_tensor_from_symbol(tensor_arena, bx);
    5057
    0
      dsfmt_t dsfmt;
    5058
    0
      ccv_nnc_tensor_t* const x_tensor = ccv_nnc_tensor_new(0, CPU_TENSOR_NHWC(32F, 2, 2, 2, LN_DIM), 0);
    5059
    0
      int i;
    5060
    0
      dsfmt_init_gen_rand(&dsfmt, 1);
    5061
    0
      for (i = 0; i < 2 * 2 * 2 * LN_DIM; i++)
    5062
    0
        x_tensor->data.f32[i] = dsfmt_genrand_open_close(&dsfmt) * 100;
    5063
    5064
    0
      ccv_nnc_cmd_exec(CMD_DATA_TRANSFER_FORWARD(), ccv_nnc_no_hint, 0, TENSOR_LIST(x_tensor), TENSOR_LIST(bx_tensor), 0);
    5065
    0
      float scaledata[1 * 2 * 2 * LN_DIM];
    5066
    0
      float biasdata[1 * 2 * 2 * LN_DIM];
    5067
    0
      for (i = 0; i < 1 * 2 * 2 * LN_DIM; i++)
    5068
    0
        scaledata[i] = dsfmt_genrand_open_close(&dsfmt);
    5069
    0
      for (i = 0; i < 1 * 2 * 2 * LN_DIM; i++)
    5070
    0
        biasdata[i] = dsfmt_genrand_open_close(&dsfmt);
    5071
    5072
    0
      ccv_nnc_tensor_t scale_tensor = ccv_nnc_tensor(scaledata, CPU_TENSOR_NHWC(32F, 1, 2, 2, LN_DIM), 0);
    5073
    0
      ccv_nnc_tensor_t bias_tensor = ccv_nnc_tensor(biasdata, CPU_TENSOR_NHWC(32F, 1, 2, 2, LN_DIM), 0);
    5074
    0
      ccv_nnc_cmd_exec(CMD_DATA_TRANSFER_FORWARD(), ccv_nnc_no_hint, 0, TENSOR_LIST(&scale_tensor, &bias_tensor), TENSOR_LIST(ccv_nnc_tensor_from_symbol(tensor_arena, scale), ccv_nnc_tensor_from_symbol(tensor_arena, bias)), 0);
    5075
    0
      ccv_nnc_tensor_t* const dy_tensor = ccv_nnc_tensor_new(0, CPU_TENSOR_NHWC(32F, 2, 2, 2, LN_DIM), 0);
    5076
    0
      ccv_nnc_tensor_t* const dby_tensor = ccv_nnc_tensor_from_symbol(tensor_arena, dby);
    5077
    0
      for (i = 0; i < 2 * 2 * 2 * LN_DIM; i++)
    5078
    0
        dy_tensor->data.f32[i] = dsfmt_genrand_open_close(&dsfmt) * 2 - 1;
    5079
    0
      ccv_nnc_cmd_exec(CMD_DATA_TRANSFER_FORWARD(), ccv_nnc_no_hint, 0, TENSOR_LIST(dy_tensor), TENSOR_LIST(dby_tensor), 0);
    5080
    0
      ccv_nnc_graph_run(graph, 0, TRAVERSE_FULL, 0, 0);
    5081
    0
      ccv_nnc_tensor_t* const dbx_tensor = ccv_nnc_tensor_from_symbol(tensor_arena, dbx);
    5082
    0
      ccv_nnc_tensor_t* const dx_tensor = ccv_nnc_tensor_new(0, CPU_TENSOR_NHWC(32F, 2, 2, 2, LN_DIM), 0);
    5083
    0
      ccv_nnc_cmd_exec(CMD_DATA_TRANSFER_FORWARD(), ccv_nnc_no_hint, 0, TENSOR_LIST(dbx_tensor), TENSOR_LIST(dx_tensor), 0);
    5084
    0
      ccv_nnc_tensor_t* const dbscale_tensor = ccv_nnc_tensor_from_symbol(tensor_arena, ccv_nnc_tensor_symbol_for_backward(symbolic_graph, scale));
    5085
    0
      ccv_nnc_tensor_t* const dbbias_tensor = ccv_nnc_tensor_from_symbol(tensor_arena, ccv_nnc_tensor_symbol_for_backward(symbolic_graph, bias));
    5086
    0
      ccv_nnc_tensor_t* const dscale_tensor = ccv_nnc_tensor_new(0, CPU_TENSOR_NHWC(32F, 1, 2, 2, LN_DIM), 0);
    5087
    0
      ccv_nnc_tensor_t* const dbias_tensor = ccv_nnc_tensor_new(0, CPU_TENSOR_NHWC(32F, 1, 2, 2, LN_DIM), 0);
    5088
    0
      ccv_nnc_cmd_exec(CMD_DATA_TRANSFER_FORWARD(), ccv_nnc_no_hint, 0, TENSOR_LIST(dbscale_tensor, dbbias_tensor), TENSOR_LIST(dscale_tensor, dbias_tensor), 0);
    5089
    0
      ccv_nnc_symbolic_graph_free(symbolic_graph);
    5090
    0
      ccv_nnc_tensor_arena_free(tensor_arena);
    5091
    0
      ccv_nnc_graph_exec_arena_free(graph_exec_arena);
    5092
    0
      ccv_nnc_graph_free(graph);
    5093
    0
      ccv_nnc_symbolic_graph_t* const cpu_symbolic_graph = ccv_nnc_symbolic_graph_new();
    5094
    0
      ccv_nnc_tensor_symbol_t cx = ccv_nnc_tensor_symbol_new(cpu_symbolic_graph, CPU_TENSOR_NHWC(32F, 2, 2, 2, LN_DIM), "x");
    5095
    0
      ccv_nnc_tensor_symbol_t cy = ccv_nnc_tensor_symbol_new(cpu_symbolic_graph, CPU_TENSOR_NHWC(32F, 2, 2, 2, LN_DIM), "y");
    5096
    0
      ccv_nnc_tensor_symbol_t cscale = ccv_nnc_tensor_symbol_new(cpu_symbolic_graph, CPU_TENSOR_NHWC(32F, 1, 2, 2, LN_DIM), "scale");
    5097
    0
      ccv_nnc_tensor_symbol_t cbias = ccv_nnc_tensor_symbol_new(cpu_symbolic_graph, CPU_TENSOR_NHWC(32F, 1, 2, 2, LN_DIM), "bias");
    5098
    0
      ccv_nnc_tensor_symbol_t csaved_mean = ccv_nnc_tensor_symbol_new(cpu_symbolic_graph, CPU_TENSOR_NHWC(32F, 2, 1, 1, 1), "saved_mean");
    5099
    0
      ccv_nnc_tensor_symbol_t csaved_inv_std = ccv_nnc_tensor_symbol_new(cpu_symbolic_graph, CPU_TENSOR_NHWC(32F, 2, 1, 1, 1), "saved_inv_std");
    5100
    0
      ccv_nnc_graph_exec_symbol_new(cpu_symbolic_graph, CMD_LAYER_NORM_FORWARD(1e-4, 1, 1, 2, 3), TENSOR_SYMBOL_LIST(cx, cscale, cbias), TENSOR_SYMBOL_LIST(cy, csaved_mean, csaved_inv_std), "layer_norm");
    5101
    0
      ccv_nnc_graph_exec_symbol_autogen(cpu_symbolic_graph, 0, 0, CCV_NNC_AUTOGEN_ALL_EXECS | CCV_NNC_AUTOGEN_SOURCES_AND_DESTINATIONS);
    5102
    0
      ccv_nnc_symbolic_graph_backward(cpu_symbolic_graph, TENSOR_SYMBOL_LIST(cy), TENSOR_SYMBOL_LIST(cx, cscale, cbias), SYMBOLIC_GRAPH_SOURCES(cpu_symbolic_graph), SYMBOLIC_GRAPH_DESTINATIONS(cpu_symbolic_graph));
    5103
    0
      ccv_nnc_graph_exec_symbol_autogen(cpu_symbolic_graph, 0, 0, CCV_NNC_AUTOGEN_ALL_EXECS | CCV_NNC_AUTOGEN_SOURCES_AND_DESTINATIONS);
    5104
    0
      ccv_nnc_tensor_symbol_t dcy = ccv_nnc_tensor_symbol_for_backward(cpu_symbolic_graph, cy);
    5105
    0
      ccv_nnc_tensor_symbol_t dcx = ccv_nnc_tensor_symbol_for_backward(cpu_symbolic_graph, cx);
    5106
    0
      ccv_nnc_tensor_symbol_t dcscale = ccv_nnc_tensor_symbol_for_backward(cpu_symbolic_graph, cscale);
    5107
    0
      ccv_nnc_tensor_symbol_t dcbias = ccv_nnc_tensor_symbol_for_backward(cpu_symbolic_graph, cbias);
    5108
    0
      ccv_nnc_graph_t* cpu_graph = 0;
    5109
    0
      ccv_nnc_tensor_arena_t* cpu_tensor_arena = 0;
    5110
    0
      ccv_nnc_graph_exec_arena_t* cpu_graph_exec_arena = 0;
    5111
    0
      ccv_nnc_symbolic_graph_compile(cpu_symbolic_graph, ccv_nnc_default_compile_params, 0, 0, 0, 0, SYMBOLIC_GRAPH_SOURCES(cpu_symbolic_graph), SYMBOLIC_GRAPH_DESTINATIONS(cpu_symbolic_graph), &cpu_graph, &cpu_tensor_arena, &cpu_graph_exec_arena);
    5112
    0
      ccv_nnc_tensor_t* const cx_tensor = ccv_nnc_tensor_from_symbol(cpu_tensor_arena, cx);
    5113
    0
      memcpy(cx_tensor->data.f32, x_tensor->data.f32, sizeof(float) * 2 * 2 * 2 * LN_DIM);
    5114
    0
      ccv_nnc_tensor_t* const dcy_tensor = ccv_nnc_tensor_from_symbol(cpu_tensor_arena, dcy);
    5115
    0
      memcpy(dcy_tensor->data.f32, dy_tensor->data.f32, sizeof(float) * 2 * 2 * 2 * LN_DIM);
    5116
    0
      ccv_nnc_tensor_t* const cscale_tensor = ccv_nnc_tensor_from_symbol(cpu_tensor_arena, cscale);
    5117
    0
      memcpy(cscale_tensor->data.f32, scaledata, sizeof(float) * 1 * 2 * 2 * LN_DIM);
    5118
    0
      ccv_nnc_tensor_t* const cbias_tensor = ccv_nnc_tensor_from_symbol(cpu_tensor_arena, cbias);
    5119
    0
      memcpy(cbias_tensor->data.f32, biasdata, sizeof(float) * 1 * 2 * 2 * LN_DIM);
    5120
    0
      ccv_nnc_graph_run(cpu_graph, 0, TRAVERSE_FULL, 0, 0);
    5121
    0
      ccv_nnc_tensor_t* const dcscale_tensor = ccv_nnc_tensor_from_symbol(cpu_tensor_arena, dcscale);
    5122
    0
      ccv_nnc_tensor_t* const dcbias_tensor = ccv_nnc_tensor_from_symbol(cpu_tensor_arena, dcbias);
    5123
    0
      ccv_nnc_tensor_t* const dcx_tensor = ccv_nnc_tensor_from_symbol(cpu_tensor_arena, dcx);
    5124
    5125
    0
      REQUIRE_TENSOR_EQ(dx_tensor, dcx_tensor, "layer norm gradient result from mps should match the one from reference implementation");
    5126
    0
      REQUIRE_TENSOR_EQ(dscale_tensor, dcscale_tensor, "layer norm scale gradient result from mps should match the one from reference implementation");
    5127
    0
      REQUIRE_TENSOR_EQ(dbias_tensor, dcbias_tensor, "layer norm bias gradient result from mps should match the one from reference implementation");
    5128
    0
      ccv_nnc_symbolic_graph_free(cpu_symbolic_graph);
    5129
    0
      ccv_nnc_tensor_arena_free(cpu_tensor_arena);
    5130
    0
      ccv_nnc_graph_exec_arena_free(cpu_graph_exec_arena);
    5131
    0
      ccv_nnc_graph_free(cpu_graph);
    5132
    0
      ccv_nnc_tensor_free(x_tensor);
    5133
    0
      ccv_nnc_tensor_free(dy_tensor);
    5134
    0
      ccv_nnc_tensor_free(dx_tensor);
    5135
    0
      ccv_nnc_tensor_free(dscale_tensor);
    5136
    0
      ccv_nnc_tensor_free(dbias_tensor);
    5137
    0
    }
    5138
    5139
    TEST_CASE("compare layer norm gradient with mps (no bias)")
    5140
    1
    {
    5141
    1
      GUARD_ELSE_RETURN(ccv_nnc_cmd_ok(CCV_NNC_LAYER_NORM_FORWARD, CCV_NNC_BACKEND_MPS) &&
    5142
    1
        ccv_nnc_cmd_ok(CCV_NNC_LAYER_NORM_BACKWARD, CCV_NNC_BACKEND_MPS) &&
    5143
    1
        (ccv_nnc_cmd_ok(CCV_NNC_SET_FORWARD, CCV_NNC_BACKEND_MPS) || ccv_nnc_cmd_ok(CCV_NNC_SET_FORWARD, CCV_NNC_BACKEND_MPS)));
    5144
    0
      ccv_nnc_symbolic_graph_t* const symbolic_graph = ccv_nnc_symbolic_graph_new();
    5145
    0
      ccv_nnc_tensor_symbol_t bx = ccv_nnc_tensor_symbol_new(symbolic_graph, GPU_TENSOR_NHWC(000, 32F, 2, 2, 2, LN_DIM), "x");
    5146
    0
      ccv_nnc_tensor_symbol_t by = ccv_nnc_tensor_symbol_new(symbolic_graph, GPU_TENSOR_NHWC(000, 32F, 2, 2, 2, LN_DIM), "y");
    5147
    0
      ccv_nnc_tensor_symbol_t scale = ccv_nnc_tensor_symbol_new(symbolic_graph, GPU_TENSOR_NHWC(000, 32F, 1, 2, 2, LN_DIM), "scale");
    5148
    0
      ccv_nnc_tensor_symbol_t bias = ccv_nnc_tensor_symbol_new(symbolic_graph, GPU_TENSOR_NHWC(000, 32F, 1, 2, 2, LN_DIM), "bias");
    5149
    0
      ccv_nnc_tensor_symbol_t saved_mean = ccv_nnc_tensor_symbol_new(symbolic_graph, GPU_TENSOR_NHWC(000, 32F, 2, 1, 1, 1), "saved_mean");
    5150
    0
      ccv_nnc_tensor_symbol_t saved_inv_std = ccv_nnc_tensor_symbol_new(symbolic_graph, GPU_TENSOR_NHWC(000, 32F, 2, 1, 1, 1), "saved_inv_std");
    5151
    0
      ccv_nnc_graph_exec_symbol_new(symbolic_graph, CMD_LAYER_NORM_FORWARD(1e-4, 1, 1, 2, 3), TENSOR_SYMBOL_LIST(bx, scale, bias), TENSOR_SYMBOL_LIST(by, saved_mean, saved_inv_std), "layer_norm");
    5152
    0
      ccv_nnc_graph_exec_symbol_autogen(symbolic_graph, 0, 0, CCV_NNC_AUTOGEN_ALL_EXECS | CCV_NNC_AUTOGEN_SOURCES_AND_DESTINATIONS);
    5153
    0
      ccv_nnc_symbolic_graph_backward(symbolic_graph, TENSOR_SYMBOL_LIST(by), TENSOR_SYMBOL_LIST(bx, scale), SYMBOLIC_GRAPH_SOURCES(symbolic_graph), SYMBOLIC_GRAPH_DESTINATIONS(symbolic_graph));
    5154
    0
      ccv_nnc_graph_exec_symbol_autogen(symbolic_graph, 0, 0, CCV_NNC_AUTOGEN_ALL_EXECS | CCV_NNC_AUTOGEN_SOURCES_AND_DESTINATIONS);
    5155
    0
      ccv_nnc_tensor_symbol_t dby = ccv_nnc_tensor_symbol_for_backward(symbolic_graph, by);
    5156
    0
      ccv_nnc_tensor_symbol_t dbx = ccv_nnc_tensor_symbol_for_backward(symbolic_graph, bx);
    5157
    0
      ccv_nnc_graph_t* graph = 0;
    5158
    0
      ccv_nnc_tensor_arena_t* tensor_arena = 0;
    5159
    0
      ccv_nnc_graph_exec_arena_t* graph_exec_arena = 0;
    5160
    0
      ccv_nnc_symbolic_graph_compile(symbolic_graph, ccv_nnc_default_compile_params, 0, 0, 0, 0, SYMBOLIC_GRAPH_SOURCES(symbolic_graph), SYMBOLIC_GRAPH_DESTINATIONS(symbolic_graph), &graph, &tensor_arena, &graph_exec_arena);
    5161
    0
      SYMBOLIC_GRAPH_GEN(symbolic_graph, CCV_NNC_LONG_DOT_GRAPH);
    5162
    0
      GRAPH_GEN(graph, CCV_NNC_LONG_DOT_GRAPH);
    5163
    0
      ccv_nnc_tensor_t* const bx_tensor = ccv_nnc_tensor_from_symbol(tensor_arena, bx);
    5164
    0
      dsfmt_t dsfmt;
    5165
    0
      ccv_nnc_tensor_t* const x_tensor = ccv_nnc_tensor_new(0, CPU_TENSOR_NHWC(32F, 2, 2, 2, LN_DIM), 0);
    5166
    0
      int i;
    5167
    0
      dsfmt_init_gen_rand(&dsfmt, 1);
    5168
    0
      for (i = 0; i < 2 * 2 * 2 * LN_DIM; i++)
    5169
    0
        x_tensor->data.f32[i] = dsfmt_genrand_open_close(&dsfmt) * 100;
    5170
    5171
    0
      ccv_nnc_cmd_exec(CMD_DATA_TRANSFER_FORWARD(), ccv_nnc_no_hint, 0, TENSOR_LIST(x_tensor), TENSOR_LIST(bx_tensor), 0);
    5172
    0
      float scaledata[1 * 2 * 2 * LN_DIM];
    5173
    0
      float biasdata[1 * 2 * 2 * LN_DIM];
    5174
    0
      for (i = 0; i < 1 * 2 * 2 * LN_DIM; i++)
    5175
    0
        scaledata[i] = dsfmt_genrand_open_close(&dsfmt);
    5176
    0
      for (i = 0; i < 1 * 2 * 2 * LN_DIM; i++)
    5177
    0
        biasdata[i] = dsfmt_genrand_open_close(&dsfmt);
    5178
    5179
    0
      ccv_nnc_tensor_t scale_tensor = ccv_nnc_tensor(scaledata, CPU_TENSOR_NHWC(32F, 1, 2, 2, LN_DIM), 0);
    5180
    0
      ccv_nnc_tensor_t bias_tensor = ccv_nnc_tensor(biasdata, CPU_TENSOR_NHWC(32F, 1, 2, 2, LN_DIM), 0);
    5181
    0
      ccv_nnc_cmd_exec(CMD_DATA_TRANSFER_FORWARD(), ccv_nnc_no_hint, 0, TENSOR_LIST(&scale_tensor, &bias_tensor), TENSOR_LIST(ccv_nnc_tensor_from_symbol(tensor_arena, scale), ccv_nnc_tensor_from_symbol(tensor_arena, bias)), 0);
    5182
    0
      ccv_nnc_tensor_t* const dy_tensor = ccv_nnc_tensor_new(0, CPU_TENSOR_NHWC(32F, 2, 2, 2, LN_DIM), 0);
    5183
    0
      ccv_nnc_tensor_t* const dby_tensor = ccv_nnc_tensor_from_symbol(tensor_arena, dby);
    5184
    0
      for (i = 0; i < 2 * 2 * 2 * LN_DIM; i++)
    5185
    0
        dy_tensor->data.f32[i] = dsfmt_genrand_open_close(&dsfmt) * 2 - 1;
    5186
    0
      ccv_nnc_cmd_exec(CMD_DATA_TRANSFER_FORWARD(), ccv_nnc_no_hint, 0, TENSOR_LIST(dy_tensor), TENSOR_LIST(dby_tensor), 0);
    5187
    0
      ccv_nnc_graph_run(graph, 0, TRAVERSE_FULL, 0, 0);
    5188
    0
      ccv_nnc_tensor_t* const dbx_tensor = ccv_nnc_tensor_from_symbol(tensor_arena, dbx);
    5189
    0
      ccv_nnc_tensor_t* const dx_tensor = ccv_nnc_tensor_new(0, CPU_TENSOR_NHWC(32F, 2, 2, 2, LN_DIM), 0);
    5190
    0
      ccv_nnc_cmd_exec(CMD_DATA_TRANSFER_FORWARD(), ccv_nnc_no_hint, 0, TENSOR_LIST(dbx_tensor), TENSOR_LIST(dx_tensor), 0);
    5191
    0
      ccv_nnc_tensor_t* const dbscale_tensor = ccv_nnc_tensor_from_symbol(tensor_arena, ccv_nnc_tensor_symbol_for_backward(symbolic_graph, scale));
    5192
    0
      ccv_nnc_tensor_t* const dscale_tensor = ccv_nnc_tensor_new(0, CPU_TENSOR_NHWC(32F, 1, 2, 2, LN_DIM), 0);
    5193
    0
      ccv_nnc_cmd_exec(CMD_DATA_TRANSFER_FORWARD(), ccv_nnc_no_hint, 0, TENSOR_LIST(dbscale_tensor), TENSOR_LIST(dscale_tensor), 0);
    5194
    0
      ccv_nnc_symbolic_graph_free(symbolic_graph);
    5195
    0
      ccv_nnc_tensor_arena_free(tensor_arena);
    5196
    0
      ccv_nnc_graph_exec_arena_free(graph_exec_arena);
    5197
    0
      ccv_nnc_graph_free(graph);
    5198
    0
      ccv_nnc_symbolic_graph_t* const cpu_symbolic_graph = ccv_nnc_symbolic_graph_new();
    5199
    0
      ccv_nnc_tensor_symbol_t cx = ccv_nnc_tensor_symbol_new(cpu_symbolic_graph, CPU_TENSOR_NHWC(32F, 2, 2, 2, LN_DIM), "x");
    5200
    0
      ccv_nnc_tensor_symbol_t cy = ccv_nnc_tensor_symbol_new(cpu_symbolic_graph, CPU_TENSOR_NHWC(32F, 2, 2, 2, LN_DIM), "y");
    5201
    0
      ccv_nnc_tensor_symbol_t cscale = ccv_nnc_tensor_symbol_new(cpu_symbolic_graph, CPU_TENSOR_NHWC(32F, 1, 2, 2, LN_DIM), "scale");
    5202
    0
      ccv_nnc_tensor_symbol_t cbias = ccv_nnc_tensor_symbol_new(cpu_symbolic_graph, CPU_TENSOR_NHWC(32F, 1, 2, 2, LN_DIM), "bias");
    5203
    0
      ccv_nnc_tensor_symbol_t csaved_mean = ccv_nnc_tensor_symbol_new(cpu_symbolic_graph, CPU_TENSOR_NHWC(32F, 2, 1, 1, 1), "saved_mean");
    5204
    0
      ccv_nnc_tensor_symbol_t csaved_inv_std = ccv_nnc_tensor_symbol_new(cpu_symbolic_graph, CPU_TENSOR_NHWC(32F, 2, 1, 1, 1), "saved_inv_std");
    5205
    0
      ccv_nnc_graph_exec_symbol_new(cpu_symbolic_graph, CMD_LAYER_NORM_FORWARD(1e-4, 1, 1, 2, 3), TENSOR_SYMBOL_LIST(cx, cscale, cbias), TENSOR_SYMBOL_LIST(cy, csaved_mean, csaved_inv_std), "layer_norm");
    5206
    0
      ccv_nnc_graph_exec_symbol_autogen(cpu_symbolic_graph, 0, 0, CCV_NNC_AUTOGEN_ALL_EXECS | CCV_NNC_AUTOGEN_SOURCES_AND_DESTINATIONS);
    5207
    0
      ccv_nnc_symbolic_graph_backward(cpu_symbolic_graph, TENSOR_SYMBOL_LIST(cy), TENSOR_SYMBOL_LIST(cx, cscale), SYMBOLIC_GRAPH_SOURCES(cpu_symbolic_graph), SYMBOLIC_GRAPH_DESTINATIONS(cpu_symbolic_graph));
    5208
    0
      ccv_nnc_graph_exec_symbol_autogen(cpu_symbolic_graph, 0, 0, CCV_NNC_AUTOGEN_ALL_EXECS | CCV_NNC_AUTOGEN_SOURCES_AND_DESTINATIONS);
    5209
    0
      ccv_nnc_tensor_symbol_t dcy = ccv_nnc_tensor_symbol_for_backward(cpu_symbolic_graph, cy);
    5210
    0
      ccv_nnc_tensor_symbol_t dcx = ccv_nnc_tensor_symbol_for_backward(cpu_symbolic_graph, cx);
    5211
    0
      ccv_nnc_tensor_symbol_t dcscale = ccv_nnc_tensor_symbol_for_backward(cpu_symbolic_graph, cscale);
    5212
    0
      ccv_nnc_graph_t* cpu_graph = 0;
    5213
    0
      ccv_nnc_tensor_arena_t* cpu_tensor_arena = 0;
    5214
    0
      ccv_nnc_graph_exec_arena_t* cpu_graph_exec_arena = 0;
    5215
    0
      ccv_nnc_symbolic_graph_compile(cpu_symbolic_graph, ccv_nnc_default_compile_params, 0, 0, 0, 0, SYMBOLIC_GRAPH_SOURCES(cpu_symbolic_graph), SYMBOLIC_GRAPH_DESTINATIONS(cpu_symbolic_graph), &cpu_graph, &cpu_tensor_arena, &cpu_graph_exec_arena);
    5216
    0
      ccv_nnc_tensor_t* const cx_tensor = ccv_nnc_tensor_from_symbol(cpu_tensor_arena, cx);
    5217
    0
      memcpy(cx_tensor->data.f32, x_tensor->data.f32, sizeof(float) * 2 * 2 * 2 * LN_DIM);
    5218
    0
      ccv_nnc_tensor_t* const dcy_tensor = ccv_nnc_tensor_from_symbol(cpu_tensor_arena, dcy);
    5219
    0
      memcpy(dcy_tensor->data.f32, dy_tensor->data.f32, sizeof(float) * 2 * 2 * 2 * LN_DIM);
    5220
    0
      ccv_nnc_tensor_t* const cscale_tensor = ccv_nnc_tensor_from_symbol(cpu_tensor_arena, cscale);
    5221
    0
      memcpy(cscale_tensor->data.f32, scaledata, sizeof(float) * 1 * 2 * 2 * LN_DIM);
    5222
    0
      ccv_nnc_graph_run(cpu_graph, 0, TRAVERSE_FULL, 0, 0);
    5223
    0
      ccv_nnc_tensor_t* const dcscale_tensor = ccv_nnc_tensor_from_symbol(cpu_tensor_arena, dcscale);
    5224
    0
      ccv_nnc_tensor_t* const dcx_tensor = ccv_nnc_tensor_from_symbol(cpu_tensor_arena, dcx);
    5225
    5226
    0
      REQUIRE_TENSOR_EQ(dx_tensor, dcx_tensor, "layer norm gradient result from mps should match the one from reference implementation");
    5227
    0
      REQUIRE_TENSOR_EQ(dscale_tensor, dcscale_tensor, "layer norm scale gradient result from mps should match the one from reference implementation");
    5228
    0
      ccv_nnc_symbolic_graph_free(cpu_symbolic_graph);
    5229
    0
      ccv_nnc_tensor_arena_free(cpu_tensor_arena);
    5230
    0
      ccv_nnc_graph_exec_arena_free(cpu_graph_exec_arena);
    5231
    0
      ccv_nnc_graph_free(cpu_graph);
    5232
    0
      ccv_nnc_tensor_free(x_tensor);
    5233
    0
      ccv_nnc_tensor_free(dy_tensor);
    5234
    0
      ccv_nnc_tensor_free(dx_tensor);
    5235
    0
      ccv_nnc_tensor_free(dscale_tensor);
    5236
    0
    }
    5237
    5238
    TEST_CASE("compare layer norm gradient with mps without scale / bias")
    5239
    1
    {
    5240
    1
      GUARD_ELSE_RETURN(ccv_nnc_cmd_ok(CCV_NNC_LAYER_NORM_FORWARD, CCV_NNC_BACKEND_MPS) &&
    5241
    1
        ccv_nnc_cmd_ok(CCV_NNC_LAYER_NORM_BACKWARD, CCV_NNC_BACKEND_MPS) &&
    5242
    1
        (ccv_nnc_cmd_ok(CCV_NNC_SET_FORWARD, CCV_NNC_BACKEND_MPS) || ccv_nnc_cmd_ok(CCV_NNC_SET_FORWARD, CCV_NNC_BACKEND_MPS)));
    5243
    0
      ccv_nnc_symbolic_graph_t* const symbolic_graph = ccv_nnc_symbolic_graph_new();
    5244
    0
      ccv_nnc_tensor_symbol_t bx = ccv_nnc_tensor_symbol_new(symbolic_graph, GPU_TENSOR_NHWC(000, 32F, 2, 2, 2, LN_DIM), "x");
    5245
    0
      ccv_nnc_tensor_symbol_t by = ccv_nnc_tensor_symbol_new(symbolic_graph, GPU_TENSOR_NHWC(000, 32F, 2, 2, 2, LN_DIM), "y");
    5246
    0
      ccv_nnc_tensor_symbol_t saved_mean = ccv_nnc_tensor_symbol_new(symbolic_graph, GPU_TENSOR_NHWC(000, 32F, 2, 1, 1, 1), "saved_mean");
    5247
    0
      ccv_nnc_tensor_symbol_t saved_inv_std = ccv_nnc_tensor_symbol_new(symbolic_graph, GPU_TENSOR_NHWC(000, 32F, 2, 1, 1, 1), "saved_inv_std");
    5248
    0
      ccv_nnc_graph_exec_symbol_new(symbolic_graph, CMD_LAYER_NORM_FORWARD(1e-4, 0, 1, 2, 3), TENSOR_SYMBOL_LIST(bx), TENSOR_SYMBOL_LIST(by, saved_mean, saved_inv_std), "layer_norm");
    5249
    0
      ccv_nnc_graph_exec_symbol_autogen(symbolic_graph, 0, 0, CCV_NNC_AUTOGEN_ALL_EXECS | CCV_NNC_AUTOGEN_SOURCES_AND_DESTINATIONS);
    5250
    0
      ccv_nnc_symbolic_graph_backward(symbolic_graph, TENSOR_SYMBOL_LIST(by), TENSOR_SYMBOL_LIST(bx), SYMBOLIC_GRAPH_SOURCES(symbolic_graph), SYMBOLIC_GRAPH_DESTINATIONS(symbolic_graph));
    5251
    0
      ccv_nnc_graph_exec_symbol_autogen(symbolic_graph, 0, 0, CCV_NNC_AUTOGEN_ALL_EXECS | CCV_NNC_AUTOGEN_SOURCES_AND_DESTINATIONS);
    5252
    0
      ccv_nnc_tensor_symbol_t dby = ccv_nnc_tensor_symbol_for_backward(symbolic_graph, by);
    5253
    0
      ccv_nnc_tensor_symbol_t dbx = ccv_nnc_tensor_symbol_for_backward(symbolic_graph, bx);
    5254
    0
      ccv_nnc_graph_t* graph = 0;
    5255
    0
      ccv_nnc_tensor_arena_t* tensor_arena = 0;
    5256
    0
      ccv_nnc_graph_exec_arena_t* graph_exec_arena = 0;
    5257
    0
      ccv_nnc_symbolic_graph_compile(symbolic_graph, ccv_nnc_default_compile_params, 0, 0, 0, 0, SYMBOLIC_GRAPH_SOURCES(symbolic_graph), SYMBOLIC_GRAPH_DESTINATIONS(symbolic_graph), &graph, &tensor_arena, &graph_exec_arena);
    5258
    0
      SYMBOLIC_GRAPH_GEN(symbolic_graph, CCV_NNC_LONG_DOT_GRAPH);
    5259
    0
      GRAPH_GEN(graph, CCV_NNC_LONG_DOT_GRAPH);
    5260
    0
      ccv_nnc_tensor_t* const bx_tensor = ccv_nnc_tensor_from_symbol(tensor_arena, bx);
    5261
    0
      dsfmt_t dsfmt;
    5262
    0
      ccv_nnc_tensor_t* const x_tensor = ccv_nnc_tensor_new(0, CPU_TENSOR_NHWC(32F, 2, 2, 2, LN_DIM), 0);
    5263
    0
      int i;
    5264
    0
      dsfmt_init_gen_rand(&dsfmt, 1);
    5265
    0
      for (i = 0; i < 2 * 2 * 2 * LN_DIM; i++)
    5266
    0
        x_tensor->data.f32[i] = dsfmt_genrand_open_close(&dsfmt) * 100;
    5267
    5268
    0
      ccv_nnc_cmd_exec(CMD_DATA_TRANSFER_FORWARD(), ccv_nnc_no_hint, 0, TENSOR_LIST(x_tensor), TENSOR_LIST(bx_tensor), 0);
    5269
    0
      ccv_nnc_tensor_t* const dy_tensor = ccv_nnc_tensor_new(0, CPU_TENSOR_NHWC(32F, 2, 2, 2, LN_DIM), 0);
    5270
    0
      ccv_nnc_tensor_t* const dby_tensor = ccv_nnc_tensor_from_symbol(tensor_arena, dby);
    5271
    0
      for (i = 0; i < 2 * 2 * 2 * LN_DIM; i++)
    5272
    0
        dy_tensor->data.f32[i] = dsfmt_genrand_open_close(&dsfmt) * 2 - 1;
    5273
    0
      ccv_nnc_cmd_exec(CMD_DATA_TRANSFER_FORWARD(), ccv_nnc_no_hint, 0, TENSOR_LIST(dy_tensor), TENSOR_LIST(dby_tensor), 0);
    5274
    0
      ccv_nnc_graph_run(graph, 0, TRAVERSE_FULL, 0, 0);
    5275
    0
      ccv_nnc_tensor_t* const dbx_tensor = ccv_nnc_tensor_from_symbol(tensor_arena, dbx);
    5276
    0
      ccv_nnc_tensor_t* const dx_tensor = ccv_nnc_tensor_new(0, CPU_TENSOR_NHWC(32F, 2, 2, 2, LN_DIM), 0);
    5277
    0
      ccv_nnc_cmd_exec(CMD_DATA_TRANSFER_FORWARD(), ccv_nnc_no_hint, 0, TENSOR_LIST(dbx_tensor), TENSOR_LIST(dx_tensor), 0);
    5278
    0
      ccv_nnc_symbolic_graph_free(symbolic_graph);
    5279
    0
      ccv_nnc_tensor_arena_free(tensor_arena);
    5280
    0
      ccv_nnc_graph_exec_arena_free(graph_exec_arena);
    5281
    0
      ccv_nnc_graph_free(graph);
    5282
    0
      ccv_nnc_symbolic_graph_t* const cpu_symbolic_graph = ccv_nnc_symbolic_graph_new();
    5283
    0
      ccv_nnc_tensor_symbol_t cx = ccv_nnc_tensor_symbol_new(cpu_symbolic_graph, CPU_TENSOR_NHWC(32F, 2, 2, 2, LN_DIM), "x");
    5284
    0
      ccv_nnc_tensor_symbol_t cy = ccv_nnc_tensor_symbol_new(cpu_symbolic_graph, CPU_TENSOR_NHWC(32F, 2, 2, 2, LN_DIM), "y");
    5285
    0
      ccv_nnc_tensor_symbol_t csaved_mean = ccv_nnc_tensor_symbol_new(cpu_symbolic_graph, CPU_TENSOR_NHWC(32F, 2, 1, 1, 1), "saved_mean");
    5286
    0
      ccv_nnc_tensor_symbol_t csaved_inv_std = ccv_nnc_tensor_symbol_new(cpu_symbolic_graph, CPU_TENSOR_NHWC(32F, 2, 1, 1, 1), "saved_inv_std");
    5287
    0
      ccv_nnc_graph_exec_symbol_new(cpu_symbolic_graph, CMD_LAYER_NORM_FORWARD(1e-4, 0, 1, 2, 3), TENSOR_SYMBOL_LIST(cx), TENSOR_SYMBOL_LIST(cy, csaved_mean, csaved_inv_std), "layer_norm");
    5288
    0
      ccv_nnc_graph_exec_symbol_autogen(cpu_symbolic_graph, 0, 0, CCV_NNC_AUTOGEN_ALL_EXECS | CCV_NNC_AUTOGEN_SOURCES_AND_DESTINATIONS);
    5289
    0
      ccv_nnc_symbolic_graph_backward(cpu_symbolic_graph, TENSOR_SYMBOL_LIST(cy), TENSOR_SYMBOL_LIST(cx), SYMBOLIC_GRAPH_SOURCES(cpu_symbolic_graph), SYMBOLIC_GRAPH_DESTINATIONS(cpu_symbolic_graph));
    5290
    0
      ccv_nnc_graph_exec_symbol_autogen(cpu_symbolic_graph, 0, 0, CCV_NNC_AUTOGEN_ALL_EXECS | CCV_NNC_AUTOGEN_SOURCES_AND_DESTINATIONS);
    5291
    0
      ccv_nnc_tensor_symbol_t dcy = ccv_nnc_tensor_symbol_for_backward(cpu_symbolic_graph, cy);
    5292
    0
      ccv_nnc_tensor_symbol_t dcx = ccv_nnc_tensor_symbol_for_backward(cpu_symbolic_graph, cx);
    5293
    0
      ccv_nnc_graph_t* cpu_graph = 0;
    5294
    0
      ccv_nnc_tensor_arena_t* cpu_tensor_arena = 0;
    5295
    0
      ccv_nnc_graph_exec_arena_t* cpu_graph_exec_arena = 0;
    5296
    0
      ccv_nnc_symbolic_graph_compile(cpu_symbolic_graph, ccv_nnc_default_compile_params, 0, 0, 0, 0, SYMBOLIC_GRAPH_SOURCES(cpu_symbolic_graph), SYMBOLIC_GRAPH_DESTINATIONS(cpu_symbolic_graph), &cpu_graph, &cpu_tensor_arena, &cpu_graph_exec_arena);
    5297
    0
      ccv_nnc_tensor_t* const cx_tensor = ccv_nnc_tensor_from_symbol(cpu_tensor_arena, cx);
    5298
    0
      memcpy(cx_tensor->data.f32, x_tensor->data.f32, sizeof(float) * 2 * 2 * 2 * LN_DIM);
    5299
    0
      ccv_nnc_tensor_t* const dcy_tensor = ccv_nnc_tensor_from_symbol(cpu_tensor_arena, dcy);
    5300
    0
      memcpy(dcy_tensor->data.f32, dy_tensor->data.f32, sizeof(float) * 2 * 2 * 2 * LN_DIM);
    5301
    0
      ccv_nnc_graph_run(cpu_graph, 0, TRAVERSE_FULL, 0, 0);
    5302
    0
      ccv_nnc_tensor_t* const dcx_tensor = ccv_nnc_tensor_from_symbol(cpu_tensor_arena, dcx);
    5303
    5304
    0
      REQUIRE_TENSOR_EQ(dx_tensor, dcx_tensor, "layer norm gradient result from mps should match the one from reference implementation");
    5305
    0
      ccv_nnc_symbolic_graph_free(cpu_symbolic_graph);
    5306
    0
      ccv_nnc_tensor_arena_free(cpu_tensor_arena);
    5307
    0
      ccv_nnc_graph_exec_arena_free(cpu_graph_exec_arena);
    5308
    0
      ccv_nnc_graph_free(cpu_graph);
    5309
    0
      ccv_nnc_tensor_free(x_tensor);
    5310
    0
      ccv_nnc_tensor_free(dy_tensor);
    5311
    0
      ccv_nnc_tensor_free(dx_tensor);
    5312
    0
    }
    5313
    5314
    TEST_CASE("compare layer norm gradient with mps (no bias) without scale / bias")
    5315
    1
    {
    5316
    1
      GUARD_ELSE_RETURN(ccv_nnc_cmd_ok(CCV_NNC_LAYER_NORM_FORWARD, CCV_NNC_BACKEND_MPS) &&
    5317
    1
        ccv_nnc_cmd_ok(CCV_NNC_LAYER_NORM_BACKWARD, CCV_NNC_BACKEND_MPS) &&
    5318
    1
        (ccv_nnc_cmd_ok(CCV_NNC_SET_FORWARD, CCV_NNC_BACKEND_MPS) || ccv_nnc_cmd_ok(CCV_NNC_SET_FORWARD, CCV_NNC_BACKEND_MPS)));
    5319
    0
      ccv_nnc_symbolic_graph_t* const symbolic_graph = ccv_nnc_symbolic_graph_new();
    5320
    0
      ccv_nnc_tensor_symbol_t bx = ccv_nnc_tensor_symbol_new(symbolic_graph, GPU_TENSOR_NHWC(000, 32F, 2, 2, 2, LN_DIM), "x");
    5321
    0
      ccv_nnc_tensor_symbol_t by = ccv_nnc_tensor_symbol_new(symbolic_graph, GPU_TENSOR_NHWC(000, 32F, 2, 2, 2, LN_DIM), "y");
    5322
    0
      ccv_nnc_tensor_symbol_t saved_mean = ccv_nnc_tensor_symbol_new(symbolic_graph, GPU_TENSOR_NHWC(000, 32F, 2, 1, 1, 1), "saved_mean");
    5323
    0
      ccv_nnc_tensor_symbol_t saved_inv_std = ccv_nnc_tensor_symbol_new(symbolic_graph, GPU_TENSOR_NHWC(000, 32F, 2, 1, 1, 1), "saved_inv_std");
    5324
    0
      ccv_nnc_graph_exec_symbol_new(symbolic_graph, CMD_LAYER_NORM_FORWARD(1e-4, 0, 1, 2, 3), TENSOR_SYMBOL_LIST(bx), TENSOR_SYMBOL_LIST(by, saved_mean, saved_inv_std), "layer_norm");
    5325
    0
      ccv_nnc_graph_exec_symbol_autogen(symbolic_graph, 0, 0, CCV_NNC_AUTOGEN_ALL_EXECS | CCV_NNC_AUTOGEN_SOURCES_AND_DESTINATIONS);
    5326
    0
      ccv_nnc_symbolic_graph_backward(symbolic_graph, TENSOR_SYMBOL_LIST(by), TENSOR_SYMBOL_LIST(bx), SYMBOLIC_GRAPH_SOURCES(symbolic_graph), SYMBOLIC_GRAPH_DESTINATIONS(symbolic_graph));
    5327
    0
      ccv_nnc_graph_exec_symbol_autogen(symbolic_graph, 0, 0, CCV_NNC_AUTOGEN_ALL_EXECS | CCV_NNC_AUTOGEN_SOURCES_AND_DESTINATIONS);
    5328
    0
      ccv_nnc_tensor_symbol_t dby = ccv_nnc_tensor_symbol_for_backward(symbolic_graph, by);
    5329
    0
      ccv_nnc_tensor_symbol_t dbx = ccv_nnc_tensor_symbol_for_backward(symbolic_graph, bx);
    5330
    0
      ccv_nnc_graph_t* graph = 0;
    5331
    0
      ccv_nnc_tensor_arena_t* tensor_arena = 0;
    5332
    0
      ccv_nnc_graph_exec_arena_t* graph_exec_arena = 0;
    5333
    0
      ccv_nnc_symbolic_graph_compile(symbolic_graph, ccv_nnc_default_compile_params, 0, 0, 0, 0, SYMBOLIC_GRAPH_SOURCES(symbolic_graph), SYMBOLIC_GRAPH_DESTINATIONS(symbolic_graph), &graph, &tensor_arena, &graph_exec_arena);
    5334
    0
      SYMBOLIC_GRAPH_GEN(symbolic_graph, CCV_NNC_LONG_DOT_GRAPH);
    5335
    0
      GRAPH_GEN(graph, CCV_NNC_LONG_DOT_GRAPH);
    5336
    0
      ccv_nnc_tensor_t* const bx_tensor = ccv_nnc_tensor_from_symbol(tensor_arena, bx);
    5337
    0
      dsfmt_t dsfmt;
    5338
    0
      ccv_nnc_tensor_t* const x_tensor = ccv_nnc_tensor_new(0, CPU_TENSOR_NHWC(32F, 2, 2, 2, LN_DIM), 0);
    5339
    0
      int i;
    5340
    0
      dsfmt_init_gen_rand(&dsfmt, 1);
    5341
    0
      for (i = 0; i < 2 * 2 * 2 * LN_DIM; i++)
    5342
    0
        x_tensor->data.f32[i] = dsfmt_genrand_open_close(&dsfmt) * 100;
    5343
    5344
    0
      ccv_nnc_cmd_exec(CMD_DATA_TRANSFER_FORWARD(), ccv_nnc_no_hint, 0, TENSOR_LIST(x_tensor), TENSOR_LIST(bx_tensor), 0);
    5345
    0
      ccv_nnc_tensor_t* const dy_tensor = ccv_nnc_tensor_new(0, CPU_TENSOR_NHWC(32F, 2, 2, 2, LN_DIM), 0);
    5346
    0
      ccv_nnc_tensor_t* const dby_tensor = ccv_nnc_tensor_from_symbol(tensor_arena, dby);
    5347
    0
      for (i = 0; i < 2 * 2 * 2 * LN_DIM; i++)
    5348
    0
        dy_tensor->data.f32[i] = dsfmt_genrand_open_close(&dsfmt) * 2 - 1;
    5349
    0
      ccv_nnc_cmd_exec(CMD_DATA_TRANSFER_FORWARD(), ccv_nnc_no_hint, 0, TENSOR_LIST(dy_tensor), TENSOR_LIST(dby_tensor), 0);
    5350
    0
      ccv_nnc_graph_run(graph, 0, TRAVERSE_FULL, 0, 0);
    5351
    0
      ccv_nnc_tensor_t* const dbx_tensor = ccv_nnc_tensor_from_symbol(tensor_arena, dbx);
    5352
    0
      ccv_nnc_tensor_t* const dx_tensor = ccv_nnc_tensor_new(0, CPU_TENSOR_NHWC(32F, 2, 2, 2, LN_DIM), 0);
    5353
    0
      ccv_nnc_cmd_exec(CMD_DATA_TRANSFER_FORWARD(), ccv_nnc_no_hint, 0, TENSOR_LIST(dbx_tensor), TENSOR_LIST(dx_tensor), 0);
    5354
    0
      ccv_nnc_symbolic_graph_free(symbolic_graph);
    5355
    0
      ccv_nnc_tensor_arena_free(tensor_arena);
    5356
    0
      ccv_nnc_graph_exec_arena_free(graph_exec_arena);
    5357
    0
      ccv_nnc_graph_free(graph);
    5358
    0
      ccv_nnc_symbolic_graph_t* const cpu_symbolic_graph = ccv_nnc_symbolic_graph_new();
    5359
    0
      ccv_nnc_tensor_symbol_t cx = ccv_nnc_tensor_symbol_new(cpu_symbolic_graph, CPU_TENSOR_NHWC(32F, 2, 2, 2, LN_DIM), "x");
    5360
    0
      ccv_nnc_tensor_symbol_t cy = ccv_nnc_tensor_symbol_new(cpu_symbolic_graph, CPU_TENSOR_NHWC(32F, 2, 2, 2, LN_DIM), "y");
    5361
    0
      ccv_nnc_tensor_symbol_t csaved_mean = ccv_nnc_tensor_symbol_new(cpu_symbolic_graph, CPU_TENSOR_NHWC(32F, 2, 1, 1, 1), "saved_mean");
    5362
    0
      ccv_nnc_tensor_symbol_t csaved_inv_std = ccv_nnc_tensor_symbol_new(cpu_symbolic_graph, CPU_TENSOR_NHWC(32F, 2, 1, 1, 1), "saved_inv_std");
    5363
    0
      ccv_nnc_graph_exec_symbol_new(cpu_symbolic_graph, CMD_LAYER_NORM_FORWARD(1e-4, 0, 1, 2, 3), TENSOR_SYMBOL_LIST(cx), TENSOR_SYMBOL_LIST(cy, csaved_mean, csaved_inv_std), "layer_norm");
    5364
    0
      ccv_nnc_graph_exec_symbol_autogen(cpu_symbolic_graph, 0, 0, CCV_NNC_AUTOGEN_ALL_EXECS | CCV_NNC_AUTOGEN_SOURCES_AND_DESTINATIONS);
    5365
    0
      ccv_nnc_symbolic_graph_backward(cpu_symbolic_graph, TENSOR_SYMBOL_LIST(cy), TENSOR_SYMBOL_LIST(cx), SYMBOLIC_GRAPH_SOURCES(cpu_symbolic_graph), SYMBOLIC_GRAPH_DESTINATIONS(cpu_symbolic_graph));
    5366
    0
      ccv_nnc_graph_exec_symbol_autogen(cpu_symbolic_graph, 0, 0, CCV_NNC_AUTOGEN_ALL_EXECS | CCV_NNC_AUTOGEN_SOURCES_AND_DESTINATIONS);
    5367
    0
      ccv_nnc_tensor_symbol_t dcy = ccv_nnc_tensor_symbol_for_backward(cpu_symbolic_graph, cy);
    5368
    0
      ccv_nnc_tensor_symbol_t dcx = ccv_nnc_tensor_symbol_for_backward(cpu_symbolic_graph, cx);
    5369
    0
      ccv_nnc_graph_t* cpu_graph = 0;
    5370
    0
      ccv_nnc_tensor_arena_t* cpu_tensor_arena = 0;
    5371
    0
      ccv_nnc_graph_exec_arena_t* cpu_graph_exec_arena = 0;
    5372
    0
      ccv_nnc_symbolic_graph_compile(cpu_symbolic_graph, ccv_nnc_default_compile_params, 0, 0, 0, 0, SYMBOLIC_GRAPH_SOURCES(cpu_symbolic_graph), SYMBOLIC_GRAPH_DESTINATIONS(cpu_symbolic_graph), &cpu_graph, &cpu_tensor_arena, &cpu_graph_exec_arena);
    5373
    0
      ccv_nnc_tensor_t* const cx_tensor = ccv_nnc_tensor_from_symbol(cpu_tensor_arena, cx);
    5374
    0
      memcpy(cx_tensor->data.f32, x_tensor->data.f32, sizeof(float) * 2 * 2 * 2 * LN_DIM);
    5375
    0
      ccv_nnc_tensor_t* const dcy_tensor = ccv_nnc_tensor_from_symbol(cpu_tensor_arena, dcy);
    5376
    0
      memcpy(dcy_tensor->data.f32, dy_tensor->data.f32, sizeof(float) * 2 * 2 * 2 * LN_DIM);
    5377
    0
      ccv_nnc_graph_run(cpu_graph, 0, TRAVERSE_FULL, 0, 0);
    5378
    0
      ccv_nnc_tensor_t* const dcx_tensor = ccv_nnc_tensor_from_symbol(cpu_tensor_arena, dcx);
    5379
    5380
    0
      REQUIRE_TENSOR_EQ(dx_tensor, dcx_tensor, "layer norm gradient result from mps should match the one from reference implementation");
    5381
    0
      ccv_nnc_symbolic_graph_free(cpu_symbolic_graph);
    5382
    0
      ccv_nnc_tensor_arena_free(cpu_tensor_arena);
    5383
    0
      ccv_nnc_graph_exec_arena_free(cpu_graph_exec_arena);
    5384
    0
      ccv_nnc_graph_free(cpu_graph);
    5385
    0
      ccv_nnc_tensor_free(x_tensor);
    5386
    0
      ccv_nnc_tensor_free(dy_tensor);
    5387
    0
      ccv_nnc_tensor_free(dx_tensor);
    5388
    0
    }
    5389
    5390
    TEST_CASE("compare rmsnorm gradient with mps")
    5391
    1
    {
    5392
    1
      GUARD_ELSE_RETURN(ccv_nnc_cmd_ok(CCV_NNC_RMSNORM_FORWARD, CCV_NNC_BACKEND_MPS) &&
    5393
    1
        ccv_nnc_cmd_ok(CCV_NNC_RMSNORM_BACKWARD, CCV_NNC_BACKEND_MPS) &&
    5394
    1
        (ccv_nnc_cmd_ok(CCV_NNC_SET_FORWARD, CCV_NNC_BACKEND_MPS) || ccv_nnc_cmd_ok(CCV_NNC_SET_FORWARD, CCV_NNC_BACKEND_MPS)));
    5395
    0
      ccv_nnc_symbolic_graph_t* const symbolic_graph = ccv_nnc_symbolic_graph_new();
    5396
    0
      ccv_nnc_tensor_symbol_t bx = ccv_nnc_tensor_symbol_new(symbolic_graph, GPU_TENSOR_NHWC(000, 32F, 2, 2, 2, LN_DIM), "x");
    5397
    0
      ccv_nnc_tensor_symbol_t by = ccv_nnc_tensor_symbol_new(symbolic_graph, GPU_TENSOR_NHWC(000, 32F, 2, 2, 2, LN_DIM), "y");
    5398
    0
      ccv_nnc_tensor_symbol_t scale = ccv_nnc_tensor_symbol_new(symbolic_graph, GPU_TENSOR_NHWC(000, 32F, 1, 2, 2, LN_DIM), "scale");
    5399
    0
      ccv_nnc_tensor_symbol_t saved_inv_std = ccv_nnc_tensor_symbol_new(symbolic_graph, GPU_TENSOR_NHWC(000, 32F, 2, 1, 1, 1), "saved_inv_std");
    5400
    0
      ccv_nnc_graph_exec_symbol_new(symbolic_graph, CMD_RMSNORM_FORWARD(1e-4, 1, 1, 2, 3), TENSOR_SYMBOL_LIST(bx, scale), TENSOR_SYMBOL_LIST(by, saved_inv_std), "rmsnorm");
    5401
    0
      ccv_nnc_graph_exec_symbol_autogen(symbolic_graph, 0, 0, CCV_NNC_AUTOGEN_ALL_EXECS | CCV_NNC_AUTOGEN_SOURCES_AND_DESTINATIONS);
    5402
    0
      ccv_nnc_symbolic_graph_backward(symbolic_graph, TENSOR_SYMBOL_LIST(by), TENSOR_SYMBOL_LIST(bx, scale), SYMBOLIC_GRAPH_SOURCES(symbolic_graph), SYMBOLIC_GRAPH_DESTINATIONS(symbolic_graph));
    5403
    0
      ccv_nnc_graph_exec_symbol_autogen(symbolic_graph, 0, 0, CCV_NNC_AUTOGEN_ALL_EXECS | CCV_NNC_AUTOGEN_SOURCES_AND_DESTINATIONS);
    5404
    0
      ccv_nnc_tensor_symbol_t dby = ccv_nnc_tensor_symbol_for_backward(symbolic_graph, by);
    5405
    0
      ccv_nnc_tensor_symbol_t dbx = ccv_nnc_tensor_symbol_for_backward(symbolic_graph, bx);
    5406
    0
      ccv_nnc_graph_t* graph = 0;
    5407
    0
      ccv_nnc_tensor_arena_t* tensor_arena = 0;
    5408
    0
      ccv_nnc_graph_exec_arena_t* graph_exec_arena = 0;
    5409
    0
      ccv_nnc_symbolic_graph_compile(symbolic_graph, ccv_nnc_default_compile_params, 0, 0, 0, 0, SYMBOLIC_GRAPH_SOURCES(symbolic_graph), SYMBOLIC_GRAPH_DESTINATIONS(symbolic_graph), &graph, &tensor_arena, &graph_exec_arena);
    5410
    0
      SYMBOLIC_GRAPH_GEN(symbolic_graph, CCV_NNC_LONG_DOT_GRAPH);
    5411
    0
      GRAPH_GEN(graph, CCV_NNC_LONG_DOT_GRAPH);
    5412
    0
      ccv_nnc_tensor_t* const bx_tensor = ccv_nnc_tensor_from_symbol(tensor_arena, bx);
    5413
    0
      dsfmt_t dsfmt;
    5414
    0
      ccv_nnc_tensor_t* const x_tensor = ccv_nnc_tensor_new(0, CPU_TENSOR_NHWC(32F, 2, 2, 2, LN_DIM), 0);
    5415
    0
      int i;
    5416
    0
      dsfmt_init_gen_rand(&dsfmt, 1);
    5417
    0
      for (i = 0; i < 2 * 2 * 2 * LN_DIM; i++)
    5418
    0
        x_tensor->data.f32[i] = dsfmt_genrand_open_close(&dsfmt) * 100;
    5419
    5420
    0
      ccv_nnc_cmd_exec(CMD_DATA_TRANSFER_FORWARD(), ccv_nnc_no_hint, 0, TENSOR_LIST(x_tensor), TENSOR_LIST(bx_tensor), 0);
    5421
    0
      float scaledata[1 * 2 * 2 * LN_DIM];
    5422
    0
      for (i = 0; i < 1 * 2 * 2 * LN_DIM; i++)
    5423
    0
        scaledata[i] = dsfmt_genrand_open_close(&dsfmt);
    5424
    5425
    0
      ccv_nnc_tensor_t scale_tensor = ccv_nnc_tensor(scaledata, CPU_TENSOR_NHWC(32F, 1, 2, 2, LN_DIM), 0);
    5426
    0
      ccv_nnc_cmd_exec(CMD_DATA_TRANSFER_FORWARD(), ccv_nnc_no_hint, 0, TENSOR_LIST(&scale_tensor), TENSOR_LIST(ccv_nnc_tensor_from_symbol(tensor_arena, scale)), 0);
    5427
    0
      ccv_nnc_tensor_t* const dy_tensor = ccv_nnc_tensor_new(0, CPU_TENSOR_NHWC(32F, 2, 2, 2, LN_DIM), 0);
    5428
    0
      ccv_nnc_tensor_t* const dby_tensor = ccv_nnc_tensor_from_symbol(tensor_arena, dby);
    5429
    0
      for (i = 0; i < 2 * 2 * 2 * LN_DIM; i++)
    5430
    0
        dy_tensor->data.f32[i] = dsfmt_genrand_open_close(&dsfmt) * 2 - 1;
    5431
    0
      ccv_nnc_cmd_exec(CMD_DATA_TRANSFER_FORWARD(), ccv_nnc_no_hint, 0, TENSOR_LIST(dy_tensor), TENSOR_LIST(dby_tensor), 0);
    5432
    0
      ccv_nnc_graph_run(graph, 0, TRAVERSE_FULL, 0, 0);
    5433
    0
      ccv_nnc_tensor_t* const dbx_tensor = ccv_nnc_tensor_from_symbol(tensor_arena, dbx);
    5434
    0
      ccv_nnc_tensor_t* const dx_tensor = ccv_nnc_tensor_new(0, CPU_TENSOR_NHWC(32F, 2, 2, 2, LN_DIM), 0);
    5435
    0
      ccv_nnc_cmd_exec(CMD_DATA_TRANSFER_FORWARD(), ccv_nnc_no_hint, 0, TENSOR_LIST(dbx_tensor), TENSOR_LIST(dx_tensor), 0);
    5436
    0
      ccv_nnc_tensor_t* const dbscale_tensor = ccv_nnc_tensor_from_symbol(tensor_arena, ccv_nnc_tensor_symbol_for_backward(symbolic_graph, scale));
    5437
    0
      ccv_nnc_tensor_t* const dscale_tensor = ccv_nnc_tensor_new(0, CPU_TENSOR_NHWC(32F, 1, 2, 2, LN_DIM), 0);
    5438
    0
      ccv_nnc_cmd_exec(CMD_DATA_TRANSFER_FORWARD(), ccv_nnc_no_hint, 0, TENSOR_LIST(dbscale_tensor), TENSOR_LIST(dscale_tensor), 0);
    5439
    0
      ccv_nnc_symbolic_graph_free(symbolic_graph);
    5440
    0
      ccv_nnc_tensor_arena_free(tensor_arena);
    5441
    0
      ccv_nnc_graph_exec_arena_free(graph_exec_arena);
    5442
    0
      ccv_nnc_graph_free(graph);
    5443
    0
      ccv_nnc_symbolic_graph_t* const cpu_symbolic_graph = ccv_nnc_symbolic_graph_new();
    5444
    0
      ccv_nnc_tensor_symbol_t cx = ccv_nnc_tensor_symbol_new(cpu_symbolic_graph, CPU_TENSOR_NHWC(32F, 2, 2, 2, LN_DIM), "x");
    5445
    0
      ccv_nnc_tensor_symbol_t cy = ccv_nnc_tensor_symbol_new(cpu_symbolic_graph, CPU_TENSOR_NHWC(32F, 2, 2, 2, LN_DIM), "y");
    5446
    0
      ccv_nnc_tensor_symbol_t cscale = ccv_nnc_tensor_symbol_new(cpu_symbolic_graph, CPU_TENSOR_NHWC(32F, 1, 2, 2, LN_DIM), "scale");
    5447
    0
      ccv_nnc_tensor_symbol_t csaved_inv_std = ccv_nnc_tensor_symbol_new(cpu_symbolic_graph, CPU_TENSOR_NHWC(32F, 2, 1, 1, 1), "saved_inv_std");
    5448
    0
      ccv_nnc_graph_exec_symbol_new(cpu_symbolic_graph, CMD_RMSNORM_FORWARD(1e-4, 1, 1, 2, 3), TENSOR_SYMBOL_LIST(cx, cscale), TENSOR_SYMBOL_LIST(cy, csaved_inv_std), "layer_norm");
    5449
    0
      ccv_nnc_graph_exec_symbol_autogen(cpu_symbolic_graph, 0, 0, CCV_NNC_AUTOGEN_ALL_EXECS | CCV_NNC_AUTOGEN_SOURCES_AND_DESTINATIONS);
    5450
    0
      ccv_nnc_symbolic_graph_backward(cpu_symbolic_graph, TENSOR_SYMBOL_LIST(cy), TENSOR_SYMBOL_LIST(cx, cscale), SYMBOLIC_GRAPH_SOURCES(cpu_symbolic_graph), SYMBOLIC_GRAPH_DESTINATIONS(cpu_symbolic_graph));
    5451
    0
      ccv_nnc_graph_exec_symbol_autogen(cpu_symbolic_graph, 0, 0, CCV_NNC_AUTOGEN_ALL_EXECS | CCV_NNC_AUTOGEN_SOURCES_AND_DESTINATIONS);
    5452
    0
      ccv_nnc_tensor_symbol_t dcy = ccv_nnc_tensor_symbol_for_backward(cpu_symbolic_graph, cy);
    5453
    0
      ccv_nnc_tensor_symbol_t dcx = ccv_nnc_tensor_symbol_for_backward(cpu_symbolic_graph, cx);
    5454
    0
      ccv_nnc_tensor_symbol_t dcscale = ccv_nnc_tensor_symbol_for_backward(cpu_symbolic_graph, cscale);
    5455
    0
      ccv_nnc_graph_t* cpu_graph = 0;
    5456
    0
      ccv_nnc_tensor_arena_t* cpu_tensor_arena = 0;
    5457
    0
      ccv_nnc_graph_exec_arena_t* cpu_graph_exec_arena = 0;
    5458
    0
      ccv_nnc_symbolic_graph_compile(cpu_symbolic_graph, ccv_nnc_default_compile_params, 0, 0, 0, 0, SYMBOLIC_GRAPH_SOURCES(cpu_symbolic_graph), SYMBOLIC_GRAPH_DESTINATIONS(cpu_symbolic_graph), &cpu_graph, &cpu_tensor_arena, &cpu_graph_exec_arena);
    5459
    0
      ccv_nnc_tensor_t* const cx_tensor = ccv_nnc_tensor_from_symbol(cpu_tensor_arena, cx);
    5460
    0
      memcpy(cx_tensor->data.f32, x_tensor->data.f32, sizeof(float) * 2 * 2 * 2 * LN_DIM);
    5461
    0
      ccv_nnc_tensor_t* const dcy_tensor = ccv_nnc_tensor_from_symbol(cpu_tensor_arena, dcy);
    5462
    0
      memcpy(dcy_tensor->data.f32, dy_tensor->data.f32, sizeof(float) * 2 * 2 * 2 * LN_DIM);
    5463
    0
      ccv_nnc_tensor_t* const cscale_tensor = ccv_nnc_tensor_from_symbol(cpu_tensor_arena, cscale);
    5464
    0
      memcpy(cscale_tensor->data.f32, scaledata, sizeof(float) * 1 * 2 * 2 * LN_DIM);
    5465
    0
      ccv_nnc_graph_run(cpu_graph, 0, TRAVERSE_FULL, 0, 0);
    5466
    0
      ccv_nnc_tensor_t* const dcscale_tensor = ccv_nnc_tensor_from_symbol(cpu_tensor_arena, dcscale);
    5467
    0
      ccv_nnc_tensor_t* const dcx_tensor = ccv_nnc_tensor_from_symbol(cpu_tensor_arena, dcx);
    5468
    5469
    0
      REQUIRE_TENSOR_EQ(dx_tensor, dcx_tensor, "layer norm gradient result from mps should match the one from reference implementation");
    5470
    0
      REQUIRE_TENSOR_EQ(dscale_tensor, dcscale_tensor, "layer norm scale gradient result from mps should match the one from reference implementation");
    5471
    0
      ccv_nnc_symbolic_graph_free(cpu_symbolic_graph);
    5472
    0
      ccv_nnc_tensor_arena_free(cpu_tensor_arena);
    5473
    0
      ccv_nnc_graph_exec_arena_free(cpu_graph_exec_arena);
    5474
    0
      ccv_nnc_graph_free(cpu_graph);
    5475
    0
      ccv_nnc_tensor_free(x_tensor);
    5476
    0
      ccv_nnc_tensor_free(dy_tensor);
    5477
    0
      ccv_nnc_tensor_free(dx_tensor);
    5478
    0
      ccv_nnc_tensor_free(dscale_tensor);
    5479
    0
    }
    5480
    5481
    TEST_CASE("compare rmsnorm gradient with mps without scale")
    5482
    1
    {
    5483
    1
      GUARD_ELSE_RETURN(ccv_nnc_cmd_ok(CCV_NNC_RMSNORM_FORWARD, CCV_NNC_BACKEND_MPS) &&
    5484
    1
        ccv_nnc_cmd_ok(CCV_NNC_RMSNORM_BACKWARD, CCV_NNC_BACKEND_MPS) &&
    5485
    1
        (ccv_nnc_cmd_ok(CCV_NNC_SET_FORWARD, CCV_NNC_BACKEND_MPS) || ccv_nnc_cmd_ok(CCV_NNC_SET_FORWARD, CCV_NNC_BACKEND_MPS)));
    5486
    0
      ccv_nnc_symbolic_graph_t* const symbolic_graph = ccv_nnc_symbolic_graph_new();
    5487
    0
      ccv_nnc_tensor_symbol_t bx = ccv_nnc_tensor_symbol_new(symbolic_graph, GPU_TENSOR_NHWC(000, 32F, 2, 2, 2, LN_DIM), "x");
    5488
    0
      ccv_nnc_tensor_symbol_t by = ccv_nnc_tensor_symbol_new(symbolic_graph, GPU_TENSOR_NHWC(000, 32F, 2, 2, 2, LN_DIM), "y");
    5489
    0
      ccv_nnc_tensor_symbol_t saved_inv_std = ccv_nnc_tensor_symbol_new(symbolic_graph, GPU_TENSOR_NHWC(000, 32F, 2, 1, 1, 1), "saved_inv_std");
    5490
    0
      ccv_nnc_graph_exec_symbol_new(symbolic_graph, CMD_RMSNORM_FORWARD(1e-4, 0, 1, 2, 3), TENSOR_SYMBOL_LIST(bx), TENSOR_SYMBOL_LIST(by, saved_inv_std), "rmsnorm");
    5491
    0
      ccv_nnc_graph_exec_symbol_autogen(symbolic_graph, 0, 0, CCV_NNC_AUTOGEN_ALL_EXECS | CCV_NNC_AUTOGEN_SOURCES_AND_DESTINATIONS);
    5492
    0
      ccv_nnc_symbolic_graph_backward(symbolic_graph, TENSOR_SYMBOL_LIST(by), TENSOR_SYMBOL_LIST(bx), SYMBOLIC_GRAPH_SOURCES(symbolic_graph), SYMBOLIC_GRAPH_DESTINATIONS(symbolic_graph));
    5493
    0
      ccv_nnc_graph_exec_symbol_autogen(symbolic_graph, 0, 0, CCV_NNC_AUTOGEN_ALL_EXECS | CCV_NNC_AUTOGEN_SOURCES_AND_DESTINATIONS);
    5494
    0
      ccv_nnc_tensor_symbol_t dby = ccv_nnc_tensor_symbol_for_backward(symbolic_graph, by);
    5495
    0
      ccv_nnc_tensor_symbol_t dbx = ccv_nnc_tensor_symbol_for_backward(symbolic_graph, bx);
    5496
    0
      ccv_nnc_graph_t* graph = 0;
    5497
    0
      ccv_nnc_tensor_arena_t* tensor_arena = 0;
    5498
    0
      ccv_nnc_graph_exec_arena_t* graph_exec_arena = 0;
    5499
    0
      ccv_nnc_symbolic_graph_compile(symbolic_graph, ccv_nnc_default_compile_params, 0, 0, 0, 0, SYMBOLIC_GRAPH_SOURCES(symbolic_graph), SYMBOLIC_GRAPH_DESTINATIONS(symbolic_graph), &graph, &tensor_arena, &graph_exec_arena);
    5500
    0
      SYMBOLIC_GRAPH_GEN(symbolic_graph, CCV_NNC_LONG_DOT_GRAPH);
    5501
    0
      GRAPH_GEN(graph, CCV_NNC_LONG_DOT_GRAPH);
    5502
    0
      ccv_nnc_tensor_t* const bx_tensor = ccv_nnc_tensor_from_symbol(tensor_arena, bx);
    5503
    0
      dsfmt_t dsfmt;
    5504
    0
      ccv_nnc_tensor_t* const x_tensor = ccv_nnc_tensor_new(0, CPU_TENSOR_NHWC(32F, 2, 2, 2, LN_DIM), 0);
    5505
    0
      int i;
    5506
    0
      dsfmt_init_gen_rand(&dsfmt, 1);
    5507
    0
      for (i = 0; i < 2 * 2 * 2 * LN_DIM; i++)
    5508
    0
        x_tensor->data.f32[i] = dsfmt_genrand_open_close(&dsfmt) * 100;
    5509
    5510
    0
      ccv_nnc_cmd_exec(CMD_DATA_TRANSFER_FORWARD(), ccv_nnc_no_hint, 0, TENSOR_LIST(x_tensor), TENSOR_LIST(bx_tensor), 0);
    5511
    0
      ccv_nnc_tensor_t* const dy_tensor = ccv_nnc_tensor_new(0, CPU_TENSOR_NHWC(32F, 2, 2, 2, LN_DIM), 0);
    5512
    0
      ccv_nnc_tensor_t* const dby_tensor = ccv_nnc_tensor_from_symbol(tensor_arena, dby);
    5513
    0
      for (i = 0; i < 2 * 2 * 2 * LN_DIM; i++)
    5514
    0
        dy_tensor->data.f32[i] = dsfmt_genrand_open_close(&dsfmt) * 2 - 1;
    5515
    0
      ccv_nnc_cmd_exec(CMD_DATA_TRANSFER_FORWARD(), ccv_nnc_no_hint, 0, TENSOR_LIST(dy_tensor), TENSOR_LIST(dby_tensor), 0);
    5516
    0
      ccv_nnc_graph_run(graph, 0, TRAVERSE_FULL, 0, 0);
    5517
    0
      ccv_nnc_tensor_t* const dbx_tensor = ccv_nnc_tensor_from_symbol(tensor_arena, dbx);
    5518
    0
      ccv_nnc_tensor_t* const dx_tensor = ccv_nnc_tensor_new(0, CPU_TENSOR_NHWC(32F, 2, 2, 2, LN_DIM), 0);
    5519
    0
      ccv_nnc_cmd_exec(CMD_DATA_TRANSFER_FORWARD(), ccv_nnc_no_hint, 0, TENSOR_LIST(dbx_tensor), TENSOR_LIST(dx_tensor), 0);
    5520
    0
      ccv_nnc_symbolic_graph_free(symbolic_graph);
    5521
    0
      ccv_nnc_tensor_arena_free(tensor_arena);
    5522
    0
      ccv_nnc_graph_exec_arena_free(graph_exec_arena);
    5523
    0
      ccv_nnc_graph_free(graph);
    5524
    0
      ccv_nnc_symbolic_graph_t* const cpu_symbolic_graph = ccv_nnc_symbolic_graph_new();
    5525
    0
      ccv_nnc_tensor_symbol_t cx = ccv_nnc_tensor_symbol_new(cpu_symbolic_graph, CPU_TENSOR_NHWC(32F, 2, 2, 2, LN_DIM), "x");
    5526
    0
      ccv_nnc_tensor_symbol_t cy = ccv_nnc_tensor_symbol_new(cpu_symbolic_graph, CPU_TENSOR_NHWC(32F, 2, 2, 2, LN_DIM), "y");
    5527
    0
      ccv_nnc_tensor_symbol_t csaved_inv_std = ccv_nnc_tensor_symbol_new(cpu_symbolic_graph, CPU_TENSOR_NHWC(32F, 2, 1, 1, 1), "saved_inv_std");
    5528
    0
      ccv_nnc_graph_exec_symbol_new(cpu_symbolic_graph, CMD_RMSNORM_FORWARD(1e-4, 0, 1, 2, 3), TENSOR_SYMBOL_LIST(cx), TENSOR_SYMBOL_LIST(cy, csaved_inv_std), "layer_norm");
    5529
    0
      ccv_nnc_graph_exec_symbol_autogen(cpu_symbolic_graph, 0, 0, CCV_NNC_AUTOGEN_ALL_EXECS | CCV_NNC_AUTOGEN_SOURCES_AND_DESTINATIONS);
    5530
    0
      ccv_nnc_symbolic_graph_backward(cpu_symbolic_graph, TENSOR_SYMBOL_LIST(cy), TENSOR_SYMBOL_LIST(cx), SYMBOLIC_GRAPH_SOURCES(cpu_symbolic_graph), SYMBOLIC_GRAPH_DESTINATIONS(cpu_symbolic_graph));
    5531
    0
      ccv_nnc_graph_exec_symbol_autogen(cpu_symbolic_graph, 0, 0, CCV_NNC_AUTOGEN_ALL_EXECS | CCV_NNC_AUTOGEN_SOURCES_AND_DESTINATIONS);
    5532
    0
      ccv_nnc_tensor_symbol_t dcy = ccv_nnc_tensor_symbol_for_backward(cpu_symbolic_graph, cy);
    5533
    0
      ccv_nnc_tensor_symbol_t dcx = ccv_nnc_tensor_symbol_for_backward(cpu_symbolic_graph, cx);
    5534
    0
      ccv_nnc_graph_t* cpu_graph = 0;
    5535
    0
      ccv_nnc_tensor_arena_t* cpu_tensor_arena = 0;
    5536
    0
      ccv_nnc_graph_exec_arena_t* cpu_graph_exec_arena = 0;
    5537
    0
      ccv_nnc_symbolic_graph_compile(cpu_symbolic_graph, ccv_nnc_default_compile_params, 0, 0, 0, 0, SYMBOLIC_GRAPH_SOURCES(cpu_symbolic_graph), SYMBOLIC_GRAPH_DESTINATIONS(cpu_symbolic_graph), &cpu_graph, &cpu_tensor_arena, &cpu_graph_exec_arena);
    5538
    0
      ccv_nnc_tensor_t* const cx_tensor = ccv_nnc_tensor_from_symbol(cpu_tensor_arena, cx);
    5539
    0
      memcpy(cx_tensor->data.f32, x_tensor->data.f32, sizeof(float) * 2 * 2 * 2 * LN_DIM);
    5540
    0
      ccv_nnc_tensor_t* const dcy_tensor = ccv_nnc_tensor_from_symbol(cpu_tensor_arena, dcy);
    5541
    0
      memcpy(dcy_tensor->data.f32, dy_tensor->data.f32, sizeof(float) * 2 * 2 * 2 * LN_DIM);
    5542
    0
      ccv_nnc_graph_run(cpu_graph, 0, TRAVERSE_FULL, 0, 0);
    5543
    0
      ccv_nnc_tensor_t* const dcx_tensor = ccv_nnc_tensor_from_symbol(cpu_tensor_arena, dcx);
    5544
    5545
    0
      REQUIRE_TENSOR_EQ(dx_tensor, dcx_tensor, "layer norm gradient result from mps should match the one from reference implementation");
    5546
    0
      ccv_nnc_symbolic_graph_free(cpu_symbolic_graph);
    5547
    0
      ccv_nnc_tensor_arena_free(cpu_tensor_arena);
    5548
    0
      ccv_nnc_graph_exec_arena_free(cpu_graph_exec_arena);
    5549
    0
      ccv_nnc_graph_free(cpu_graph);
    5550
    0
      ccv_nnc_tensor_free(x_tensor);
    5551
    0
      ccv_nnc_tensor_free(dy_tensor);
    5552
    0
      ccv_nnc_tensor_free(dx_tensor);
    5553
    0
    }
    5554
    5555
    TEST_CASE("mps backward convolution in nchw format")
    5556
    1
    {
    5557
    1
      GUARD_ELSE_RETURN(ccv_nnc_cmd_ok(CCV_NNC_CONVOLUTION_BACKWARD, CCV_NNC_BACKEND_MPS));
    5558
    0
      ccv_nnc_tensor_t* a = ccv_nnc_tensor_new(0, CPU_TENSOR_NHWC(32F, BATCH_SIZE, INPUT_SIZE, INPUT_SIZE, INPUT_DIM), 0);
    5559
    0
      ccv_nnc_tensor_t* h = ccv_nnc_tensor_new(0, CPU_TENSOR_NHWC(32F, BATCH_SIZE, INPUT_SIZE, INPUT_SIZE, INPUT_DIM), 0);
    5560
    0
      ccv_nnc_tensor_t* g = ccv_nnc_tensor_new(0, CPU_TENSOR_NHWC(32F, BATCH_SIZE, OUTPUT_SIZE, OUTPUT_SIZE, OUTPUT_DIM), 0);
    5561
    0
      ccv_nnc_cmd_t cmd = CMD_CONVOLUTION_BACKWARD(1, OUTPUT_DIM, KERNEL_SIZE, KERNEL_SIZE, INPUT_DIM);
    5562
    0
      cmd.backend = CCV_NNC_BACKEND_CPU_REF;
    5563
    0
      assert(cmd.backend >= 0);
    5564
    0
      ccv_nnc_hint_t hint = ccv_nnc_hint_auto(cmd.info, a->info, g->info);
    5565
    0
      assert(ccv_nnc_hint_verify(hint, cmd.info, a->info, g->info) == 0);
    5566
    0
      ccv_nnc_tensor_t* w = ccv_nnc_tensor_new(0, CPU_TENSOR_NHWC(32F, OUTPUT_DIM, KERNEL_SIZE, KERNEL_SIZE, INPUT_DIM), 0);
    5567
    0
      ccv_nnc_tensor_t* dw = ccv_nnc_tensor_new(0, CPU_TENSOR_NHWC(32F, OUTPUT_DIM, KERNEL_SIZE, KERNEL_SIZE, INPUT_DIM), 0);
    5568
    0
      ccv_nnc_tensor_t* dbias = ccv_nnc_tensor_new(0, CPU_TENSOR_NHWC(32F, 1, 1, 1, OUTPUT_DIM), 0);
    5569
      // configure the inlets.
    5570
    0
      dsfmt_t dsfmt;
    5571
    0
      dsfmt_init_gen_rand(&dsfmt, 0);
    5572
    0
      int i;
    5573
    0
      for (i = 0; i < INPUT_DIM * KERNEL_SIZE * KERNEL_SIZE * OUTPUT_DIM; i++)
    5574
    0
        w->data.f32[i] = dsfmt_genrand_open_close(&dsfmt) / (INPUT_DIM * KERNEL_SIZE * KERNEL_SIZE);
    5575
    0
      for (i = 0; i < INPUT_SIZE * INPUT_SIZE * INPUT_DIM * ccv_max(1, BATCH_SIZE); i++)
    5576
    0
        a->data.f32[i] = dsfmt_genrand_open_close(&dsfmt);
    5577
    0
      for (i = 0; i < OUTPUT_SIZE * OUTPUT_SIZE * OUTPUT_DIM * ccv_max(1, BATCH_SIZE); i++)
    5578
    0
        g->data.f32[i] = dsfmt_genrand_open_close(&dsfmt) / OUTPUT_DIM; // (OUTPUT_SIZE * OUTPUT_SIZE * OUTPUT_DIM);
    5579
      // Copy generated matrix values over to GPU.
    5580
    0
      ccv_nnc_tensor_t* ga = ccv_nnc_tensor_new(0, GPU_TENSOR_NHWC(000, 32F, BATCH_SIZE, INPUT_SIZE, INPUT_SIZE, INPUT_DIM), 0);
    5581
    0
      ccv_nnc_tensor_t* gg = ccv_nnc_tensor_new(0, GPU_TENSOR_NHWC(000, 32F, BATCH_SIZE, OUTPUT_SIZE, OUTPUT_SIZE, OUTPUT_DIM), 0);
    5582
    0
      ccv_nnc_tensor_t* gh = ccv_nnc_tensor_new(0, GPU_TENSOR_NHWC(000, 32F, BATCH_SIZE, INPUT_SIZE, INPUT_SIZE, INPUT_DIM), 0);
    5583
    0
      ccv_nnc_tensor_t* gw = ccv_nnc_tensor_new(0, GPU_TENSOR_NHWC(000, 32F, OUTPUT_DIM, KERNEL_SIZE, KERNEL_SIZE, INPUT_DIM), 0);
    5584
    0
      ccv_nnc_tensor_t* gbias = ccv_nnc_tensor_new(0, GPU_TENSOR_NHWC(000, 32F, 1, 1, 1, OUTPUT_DIM), 0);
    5585
    0
      ccv_nnc_tensor_t* gdw = ccv_nnc_tensor_new(0, GPU_TENSOR_NHWC(000, 32F, OUTPUT_DIM, KERNEL_SIZE, KERNEL_SIZE, INPUT_DIM), 0);
    5586
    0
      ccv_nnc_tensor_t* gdbias = ccv_nnc_tensor_new(0, GPU_TENSOR_NHWC(000, 32F, 1, 1, 1, OUTPUT_DIM), 0);
    5587
    0
      ccv_nnc_tensor_t* gao = ccv_nnc_tensor_new(0, GPU_TENSOR_NCHW(000, 32F, BATCH_SIZE, INPUT_DIM, INPUT_SIZE, INPUT_SIZE), 0);
    5588
    0
      ccv_nnc_tensor_t* ggo = ccv_nnc_tensor_new(0, GPU_TENSOR_NCHW(000, 32F, BATCH_SIZE, OUTPUT_DIM, OUTPUT_SIZE, OUTPUT_SIZE), 0);
    5589
    0
      ccv_nnc_tensor_t* gho = ccv_nnc_tensor_new(0, GPU_TENSOR_NCHW(000, 32F, BATCH_SIZE, INPUT_DIM, INPUT_SIZE, INPUT_SIZE), 0);
    5590
    0
      ccv_nnc_tensor_t* gwo = ccv_nnc_tensor_new(0, GPU_TENSOR_NCHW(000, 32F, OUTPUT_DIM, INPUT_DIM, KERNEL_SIZE, KERNEL_SIZE), 0);
    5591
    0
      ccv_nnc_tensor_t* gbiaso = ccv_nnc_tensor_new(0, GPU_TENSOR_NCHW(000, 32F, 1, OUTPUT_DIM, 1, 1), 0);
    5592
    0
      ccv_nnc_tensor_t* gdwo = ccv_nnc_tensor_new(0, GPU_TENSOR_NCHW(000, 32F, OUTPUT_DIM, INPUT_DIM, KERNEL_SIZE, KERNEL_SIZE), 0);
    5593
    0
      ccv_nnc_tensor_t* gdbiaso = ccv_nnc_tensor_new(0, GPU_TENSOR_NCHW(000, 32F, 1, OUTPUT_DIM, 1, 1), 0);
    5594
    0
      ccv_nnc_cmd_exec(CMD_DATA_TRANSFER_FORWARD(), ccv_nnc_no_hint, 0, TENSOR_LIST(a, w, g), TENSOR_LIST(ga, gw, gg), 0);
    5595
    0
      ccv_nnc_cmd_exec(cmd, hint, 0, TENSOR_LIST(g, a, w), TENSOR_LIST(h, dw, dbias), 0);
    5596
    0
      ccv_nnc_cmd_exec(CMD_FORMAT_TRANSFORM_FORWARD(), ccv_nnc_no_hint, 0, TENSOR_LIST(ga, gw, gg), TENSOR_LIST(gao, gwo, ggo), 0);
    5597
    0
      cmd.backend = CCV_NNC_BACKEND_MPS;
    5598
    5599
    0
      assert(cmd.backend >= 0);
    5600
    0
      cmd.algorithm = -1;
    5601
    0
      ccv_nnc_stream_context_t* stream_context = ccv_nnc_stream_context_new(CCV_STREAM_CONTEXT_GPU);
    5602
    5603
    0
      cmd = ccv_nnc_cmd_autotune(cmd, 1 * 1024 * 1024 * 1024, hint, 0, TENSOR_LIST(ggo, gao, gwo), TENSOR_LIST(gho, gdwo, gdbiaso), stream_context);
    5604
    0
      assert(CCV_NNC_EXEC_SUCCESS == ccv_nnc_cmd_exec(cmd, hint, 0, TENSOR_LIST(ggo, gao, gwo), TENSOR_LIST(gho, gdwo, gdbiaso), stream_context));
    5605
    0
      ccv_nnc_stream_context_wait(stream_context);
    5606
    0
      ccv_nnc_stream_context_free(stream_context);
    5607
    0
      ccv_nnc_tensor_t* ch = ccv_nnc_tensor_new(0, CPU_TENSOR_NHWC(32F, BATCH_SIZE, INPUT_SIZE, INPUT_SIZE, INPUT_DIM), 0);
    5608
    0
      ccv_nnc_tensor_t* cdw = ccv_nnc_tensor_new(0, CPU_TENSOR_NHWC(32F, OUTPUT_DIM, KERNEL_SIZE, KERNEL_SIZE, INPUT_DIM), 0);
    5609
    0
      ccv_nnc_tensor_t* cdbias = ccv_nnc_tensor_new(0, CPU_TENSOR_NHWC(32F, 1, OUTPUT_DIM, 1, 1), 0);
    5610
    5611
    0
      ccv_nnc_cmd_exec(CMD_FORMAT_TRANSFORM_FORWARD(), ccv_nnc_no_hint, 0, TENSOR_LIST(gho, gdwo, gdbiaso), TENSOR_LIST(gh, gdw, gdbias), 0);
    5612
    0
      ccv_nnc_cmd_exec(CMD_DATA_TRANSFER_FORWARD(), ccv_nnc_no_hint, 0, TENSOR_LIST(gh, gdw, gdbias), TENSOR_LIST(ch, cdw, cdbias), 0);
    5613
    5614
    0
      REQUIRE_ARRAY_EQ_WITH_TOLERANCE(float, dw->data.f32, cdw->data.f32, INPUT_DIM * OUTPUT_DIM * KERNEL_SIZE * KERNEL_SIZE, 5e-1, "output from mps should match from CPU");
    5615
    0
      REQUIRE_ARRAY_EQ_WITH_TOLERANCE(float, dbias->data.f32, cdbias->data.f32, OUTPUT_DIM, 5e-1, "output from mps should match from CPU");
    5616
    0
      REQUIRE_ARRAY_EQ_WITH_TOLERANCE(float, h->data.f32, ch->data.f32, BATCH_SIZE * INPUT_DIM * INPUT_SIZE * INPUT_SIZE, 1e-4, "output from mps should match from CPU");
    5617
    0
      ccv_nnc_tensor_free(gao);
    5618
    0
      ccv_nnc_tensor_free(ggo);
    5619
    0
      ccv_nnc_tensor_free(gho);
    5620
    0
      ccv_nnc_tensor_free(gwo);
    5621
    0
      ccv_nnc_tensor_free(gbiaso);
    5622
    0
      ccv_nnc_tensor_free(gdwo);
    5623
    0
      ccv_nnc_tensor_free(gdbiaso);
    5624
    0
      ccv_nnc_tensor_free(h);
    5625
    0
      ccv_nnc_tensor_free(gh);
    5626
    0
      ccv_nnc_tensor_free(w);
    5627
    0
      ccv_nnc_tensor_free(g);
    5628
    0
      ccv_nnc_tensor_free(a);
    5629
    0
      ccv_nnc_tensor_free(gbias);
    5630
    0
      ccv_nnc_tensor_free(gdbias);
    5631
    0
      ccv_nnc_tensor_free(gdw);
    5632
    0
      ccv_nnc_tensor_free(gw);
    5633
    0
      ccv_nnc_tensor_free(gg);
    5634
    0
      ccv_nnc_tensor_free(ga);
    5635
    0
      ccv_nnc_tensor_free(ch);
    5636
    0
      ccv_nnc_tensor_free(cdw);
    5637
    0
      ccv_nnc_tensor_free(cdbias);
    5638
    0
    }
    5639
    5640
    TEST_CASE("mps backward convolution in nhwc format")
    5641
    1
    {
    5642
    1
      GUARD_ELSE_RETURN(ccv_nnc_cmd_ok(CCV_NNC_CONVOLUTION_BACKWARD, CCV_NNC_BACKEND_MPS));
    5643
    0
      ccv_nnc_tensor_t* a = ccv_nnc_tensor_new(0, CPU_TENSOR_NHWC(32F, BATCH_SIZE, INPUT_SIZE, INPUT_SIZE, INPUT_DIM), 0);
    5644
    0
      ccv_nnc_tensor_t* h = ccv_nnc_tensor_new(0, CPU_TENSOR_NHWC(32F, BATCH_SIZE, INPUT_SIZE, INPUT_SIZE, INPUT_DIM), 0);
    5645
    0
      ccv_nnc_tensor_t* g = ccv_nnc_tensor_new(0, CPU_TENSOR_NHWC(32F, BATCH_SIZE, OUTPUT_SIZE, OUTPUT_SIZE, OUTPUT_DIM), 0);
    5646
    0
      ccv_nnc_cmd_t cmd = CMD_CONVOLUTION_BACKWARD(1, OUTPUT_DIM, KERNEL_SIZE, KERNEL_SIZE, INPUT_DIM);
    5647
    0
      cmd.backend = CCV_NNC_BACKEND_CPU_REF;
    5648
    0
      assert(cmd.backend >= 0);
    5649
    0
      ccv_nnc_hint_t hint = ccv_nnc_hint_auto(cmd.info, a->info, g->info);
    5650
    0
      assert(ccv_nnc_hint_verify(hint, cmd.info, a->info, g->info) == 0);
    5651
    0
      ccv_nnc_tensor_t* w = ccv_nnc_tensor_new(0, CPU_TENSOR_NHWC(32F, OUTPUT_DIM, KERNEL_SIZE, KERNEL_SIZE, INPUT_DIM), 0);
    5652
    0
      ccv_nnc_tensor_t* dw = ccv_nnc_tensor_new(0, CPU_TENSOR_NHWC(32F, OUTPUT_DIM, KERNEL_SIZE, KERNEL_SIZE, INPUT_DIM), 0);
    5653
    0
      ccv_nnc_tensor_t* dbias = ccv_nnc_tensor_new(0, CPU_TENSOR_NHWC(32F, OUTPUT_DIM), 0);
    5654
      // configure the inlets.
    5655
    0
      dsfmt_t dsfmt;
    5656
    0
      dsfmt_init_gen_rand(&dsfmt, 0);
    5657
    0
      int i;
    5658
    0
      for (i = 0; i < INPUT_DIM * KERNEL_SIZE * KERNEL_SIZE * OUTPUT_DIM; i++)
    5659
    0
        w->data.f32[i] = dsfmt_genrand_open_close(&dsfmt) / (INPUT_DIM * KERNEL_SIZE * KERNEL_SIZE);
    5660
    0
      for (i = 0; i < INPUT_SIZE * INPUT_SIZE * INPUT_DIM * ccv_max(1, BATCH_SIZE); i++)
    5661
    0
        a->data.f32[i] = dsfmt_genrand_open_close(&dsfmt);
    5662
    0
      for (i = 0; i < OUTPUT_SIZE * OUTPUT_SIZE * OUTPUT_DIM * ccv_max(1, BATCH_SIZE); i++)
    5663
    0
        g->data.f32[i] = dsfmt_genrand_open_close(&dsfmt) / OUTPUT_DIM; // (OUTPUT_SIZE * OUTPUT_SIZE * OUTPUT_DIM);
    5664
      // Copy generated matrix values over to GPU.
    5665
    0
      ccv_nnc_tensor_t* ga = ccv_nnc_tensor_new(0, GPU_TENSOR_NHWC(000, 32F, BATCH_SIZE, INPUT_SIZE, INPUT_SIZE, INPUT_DIM), 0);
    5666
    0
      ccv_nnc_tensor_t* gg = ccv_nnc_tensor_new(0, GPU_TENSOR_NHWC(000, 32F, BATCH_SIZE, OUTPUT_SIZE, OUTPUT_SIZE, OUTPUT_DIM), 0);
    5667
    0
      ccv_nnc_tensor_t* gh = ccv_nnc_tensor_new(0, GPU_TENSOR_NHWC(000, 32F, BATCH_SIZE, INPUT_SIZE, INPUT_SIZE, INPUT_DIM), 0);
    5668
    0
      ccv_nnc_tensor_t* gw = ccv_nnc_tensor_new(0, GPU_TENSOR_NHWC(000, 32F, OUTPUT_DIM, KERNEL_SIZE, KERNEL_SIZE, INPUT_DIM), 0);
    5669
    0
      ccv_nnc_tensor_t* gbias = ccv_nnc_tensor_new(0, GPU_TENSOR_NHWC(000, 32F, 1, 1, 1, OUTPUT_DIM), 0);
    5670
    0
      ccv_nnc_tensor_t* gdw = ccv_nnc_tensor_new(0, GPU_TENSOR_NHWC(000, 32F, OUTPUT_DIM, KERNEL_SIZE, KERNEL_SIZE, INPUT_DIM), 0);
    5671
    0
      ccv_nnc_tensor_t* gdbias = ccv_nnc_tensor_new(0, GPU_TENSOR_NHWC(000, 32F, 1, 1, 1, OUTPUT_DIM), 0);
    5672
    0
      ccv_nnc_tensor_t* gwo = ccv_nnc_tensor_new(0, GPU_TENSOR_NCHW(000, 32F, OUTPUT_DIM, INPUT_DIM, KERNEL_SIZE, KERNEL_SIZE), 0);
    5673
    0
      ccv_nnc_tensor_t* gdwo = ccv_nnc_tensor_new(0, GPU_TENSOR_NCHW(000, 32F, OUTPUT_DIM, INPUT_DIM, KERNEL_SIZE, KERNEL_SIZE), 0);
    5674
    5675
    0
      ccv_nnc_cmd_exec(CMD_DATA_TRANSFER_FORWARD(), ccv_nnc_no_hint, 0, TENSOR_LIST(a, w, g), TENSOR_LIST(ga, gw, gg), 0);
    5676
    0
      ccv_nnc_cmd_exec(cmd, hint, 0, TENSOR_LIST(g, a, w), TENSOR_LIST(h, dw, dbias), 0);
    5677
    0
      ccv_nnc_cmd_exec(CMD_FORMAT_TRANSFORM_FORWARD(), ccv_nnc_no_hint, 0, TENSOR_LIST(gw), TENSOR_LIST(gwo), 0);
    5678
    0
      cmd.backend = CCV_NNC_BACKEND_MPS;
    5679
    5680
    0
      assert(cmd.backend >= 0);
    5681
    0
      cmd.algorithm = -1;
    5682
    0
      ccv_nnc_stream_context_t* stream_context = ccv_nnc_stream_context_new(CCV_STREAM_CONTEXT_GPU);
    5683
    5684
    0
      cmd = ccv_nnc_cmd_autotune(cmd, 1 * 1024 * 1024 * 1024, hint, 0, TENSOR_LIST(gg, ga, gwo), TENSOR_LIST(gh, gdwo, gdbias), stream_context);
    5685
    0
      assert(CCV_NNC_EXEC_SUCCESS == ccv_nnc_cmd_exec(cmd, hint, 0, TENSOR_LIST(gg, ga, gwo), TENSOR_LIST(gh, gdwo, gdbias), stream_context));
    5686
    0
      ccv_nnc_stream_context_wait(stream_context);
    5687
    0
      ccv_nnc_stream_context_free(stream_context);
    5688
    0
      ccv_nnc_tensor_t* ch = ccv_nnc_tensor_new(0, CPU_TENSOR_NHWC(32F, BATCH_SIZE, INPUT_SIZE, INPUT_SIZE, INPUT_DIM), 0);
    5689
    0
      ccv_nnc_tensor_t* cdw = ccv_nnc_tensor_new(0, CPU_TENSOR_NHWC(32F, OUTPUT_DIM, KERNEL_SIZE, KERNEL_SIZE, INPUT_DIM), 0);
    5690
    0
      ccv_nnc_tensor_t* cdbias = ccv_nnc_tensor_new(0, CPU_TENSOR_NHWC(32F, 1, 1, 1,  OUTPUT_DIM), 0);
    5691
    5692
    0
      ccv_nnc_cmd_exec(CMD_FORMAT_TRANSFORM_FORWARD(), ccv_nnc_no_hint, 0, TENSOR_LIST(gdwo), TENSOR_LIST(gdw), 0);
    5693
    0
      ccv_nnc_cmd_exec(CMD_DATA_TRANSFER_FORWARD(), ccv_nnc_no_hint, 0, TENSOR_LIST(gh, gdw, gdbias), TENSOR_LIST(ch, cdw, cdbias), 0);
    5694
    5695
    0
      REQUIRE_ARRAY_EQ_WITH_TOLERANCE(float, dw->data.f32, cdw->data.f32, INPUT_DIM * OUTPUT_DIM * KERNEL_SIZE * KERNEL_SIZE, 5e-1, "output from mps should match from CPU");
    5696
    0
      REQUIRE_ARRAY_EQ_WITH_TOLERANCE(float, dbias->data.f32, cdbias->data.f32, OUTPUT_DIM, 5e-1, "output from mps should match from CPU");
    5697
    0
      REQUIRE_ARRAY_EQ_WITH_TOLERANCE(float, h->data.f32, ch->data.f32, BATCH_SIZE * INPUT_DIM * INPUT_SIZE * INPUT_SIZE, 1e-4, "output from mps should match from CPU");
    5698
    5699
    0
      ccv_nnc_tensor_free(gwo);
    5700
    0
      ccv_nnc_tensor_free(gdwo);
    5701
    0
      ccv_nnc_tensor_free(h);
    5702
    0
      ccv_nnc_tensor_free(gh);
    5703
    0
      ccv_nnc_tensor_free(w);
    5704
    0
      ccv_nnc_tensor_free(g);
    5705
    0
      ccv_nnc_tensor_free(a);
    5706
    0
      ccv_nnc_tensor_free(gbias);
    5707
    0
      ccv_nnc_tensor_free(gdbias);
    5708
    0
      ccv_nnc_tensor_free(gdw);
    5709
    0
      ccv_nnc_tensor_free(gw);
    5710
    0
      ccv_nnc_tensor_free(gg);
    5711
    0
      ccv_nnc_tensor_free(ga);
    5712
    0
      ccv_nnc_tensor_free(ch);
    5713
    0
      ccv_nnc_tensor_free(cdw);
    5714
    0
      ccv_nnc_tensor_free(cdbias);
    5715
    0
    }
    5716
    5717
    TEST_CASE("mps backward convolution in bfloat precision")
    5718
    1
    {
    5719
    1
      GUARD_ELSE_RETURN(ccv_nnc_cmd_ok(CCV_NNC_CONVOLUTION_BACKWARD, CCV_NNC_BACKEND_MPS));
    5720
    0
      const int batch_size = 2;
    5721
    0
      const int input_size = 8;
    5722
    0
      const int kernel_size = 3;
    5723
    0
      const int output_size = 6;
    5724
    0
      const int input_dim = 4;
    5725
    0
      const int output_dim = 8;
    5726
    0
      ccv_nnc_tensor_t* a = ccv_nnc_tensor_new(0, CPU_TENSOR_NHWC(32F, batch_size, input_size, input_size, input_dim), 0);
    5727
    0
      ccv_nnc_tensor_t* h = ccv_nnc_tensor_new(0, CPU_TENSOR_NHWC(32F, batch_size, input_size, input_size, input_dim), 0);
    5728
    0
      ccv_nnc_tensor_t* g = ccv_nnc_tensor_new(0, CPU_TENSOR_NHWC(32F, batch_size, output_size, output_size, output_dim), 0);
    5729
    0
      ccv_nnc_cmd_t cmd = CMD_CONVOLUTION_BACKWARD(1, output_dim, kernel_size, kernel_size, input_dim);
    5730
    0
      cmd.backend = CCV_NNC_BACKEND_CPU_REF;
    5731
    0
      assert(cmd.backend >= 0);
    5732
    0
      ccv_nnc_hint_t hint = ccv_nnc_hint_auto(cmd.info, a->info, g->info);
    5733
    0
      assert(ccv_nnc_hint_verify(hint, cmd.info, a->info, g->info) == 0);
    5734
    0
      ccv_nnc_tensor_t* w = ccv_nnc_tensor_new(0, CPU_TENSOR_NHWC(32F, output_dim, kernel_size, kernel_size, input_dim), 0);
    5735
    0
      ccv_nnc_tensor_t* dw = ccv_nnc_tensor_new(0, CPU_TENSOR_NHWC(32F, output_dim, kernel_size, kernel_size, input_dim), 0);
    5736
    0
      ccv_nnc_tensor_t* dbias = ccv_nnc_tensor_new(0, CPU_TENSOR_NHWC(32F, output_dim), 0);
    5737
    0
      dsfmt_t dsfmt;
    5738
    0
      dsfmt_init_gen_rand(&dsfmt, 12);
    5739
    0
      int i;
    5740
    0
      for (i = 0; i < input_dim * kernel_size * kernel_size * output_dim; i++)
    5741
    0
        w->data.f32[i] = (dsfmt_genrand_open_close(&dsfmt) - 0.5) / (input_dim * kernel_size * kernel_size);
    5742
    0
      for (i = 0; i < input_size * input_size * input_dim * batch_size; i++)
    5743
    0
        a->data.f32[i] = dsfmt_genrand_open_close(&dsfmt) - 0.5;
    5744
    0
      for (i = 0; i < output_size * output_size * output_dim * batch_size; i++)
    5745
    0
        g->data.f32[i] = (dsfmt_genrand_open_close(&dsfmt) - 0.5) / output_dim;
    5746
    0
      const int input_count = batch_size * input_size * input_size * input_dim;
    5747
    0
      const int gradient_count = batch_size * output_size * output_size * output_dim;
    5748
    0
      const int weight_count = output_dim * kernel_size * kernel_size * input_dim;
    5749
    0
      ccv_nnc_tensor_t* a16bf = ccv_nnc_tensor_new(0, CPU_TENSOR_NHWC(16BF, batch_size, input_size, input_size, input_dim), 0);
    5750
    0
      ccv_nnc_tensor_t* w16bf = ccv_nnc_tensor_new(0, CPU_TENSOR_NHWC(16BF, output_dim, kernel_size, kernel_size, input_dim), 0);
    5751
    0
      ccv_nnc_tensor_t* g16bf = ccv_nnc_tensor_new(0, CPU_TENSOR_NHWC(16BF, batch_size, output_size, output_size, output_dim), 0);
    5752
    0
      ccv_float_to_bfloat(a->data.f32, (uint16_t*)a16bf->data.f16, input_count);
    5753
    0
      ccv_float_to_bfloat(w->data.f32, (uint16_t*)w16bf->data.f16, weight_count);
    5754
    0
      ccv_float_to_bfloat(g->data.f32, (uint16_t*)g16bf->data.f16, gradient_count);
    5755
    0
      ccv_bfloat_to_float((uint16_t*)a16bf->data.f16, a->data.f32, input_count);
    5756
    0
      ccv_bfloat_to_float((uint16_t*)w16bf->data.f16, w->data.f32, weight_count);
    5757
    0
      ccv_bfloat_to_float((uint16_t*)g16bf->data.f16, g->data.f32, gradient_count);
    5758
    0
      ccv_nnc_cmd_exec(cmd, hint, 0, TENSOR_LIST(g, a, w), TENSOR_LIST(h, dw, dbias), 0);
    5759
    0
      ccv_nnc_tensor_t* h16bf = ccv_nnc_tensor_new(0, CPU_TENSOR_NHWC(16BF, batch_size, input_size, input_size, input_dim), 0);
    5760
    0
      ccv_nnc_tensor_t* dw16bf = ccv_nnc_tensor_new(0, CPU_TENSOR_NHWC(16BF, output_dim, kernel_size, kernel_size, input_dim), 0);
    5761
    0
      ccv_nnc_tensor_t* dbias16bf = ccv_nnc_tensor_new(0, CPU_TENSOR_NHWC(16BF, output_dim), 0);
    5762
    0
      ccv_float_to_bfloat(h->data.f32, (uint16_t*)h16bf->data.f16, input_count);
    5763
    0
      ccv_float_to_bfloat(dw->data.f32, (uint16_t*)dw16bf->data.f16, weight_count);
    5764
    0
      ccv_float_to_bfloat(dbias->data.f32, (uint16_t*)dbias16bf->data.f16, output_dim);
    5765
    0
      ccv_bfloat_to_float((uint16_t*)h16bf->data.f16, h->data.f32, input_count);
    5766
    0
      ccv_bfloat_to_float((uint16_t*)dw16bf->data.f16, dw->data.f32, weight_count);
    5767
    0
      ccv_bfloat_to_float((uint16_t*)dbias16bf->data.f16, dbias->data.f32, output_dim);
    5768
    0
      ccv_nnc_tensor_t* ga = ccv_nnc_tensor_new(0, GPU_TENSOR_NHWC(000, 16BF, batch_size, input_size, input_size, input_dim), 0);
    5769
    0
      ccv_nnc_tensor_t* gg = ccv_nnc_tensor_new(0, GPU_TENSOR_NHWC(000, 16BF, batch_size, output_size, output_size, output_dim), 0);
    5770
    0
      ccv_nnc_tensor_t* gh = ccv_nnc_tensor_new(0, GPU_TENSOR_NHWC(000, 16BF, batch_size, input_size, input_size, input_dim), 0);
    5771
    0
      ccv_nnc_tensor_t* gw = ccv_nnc_tensor_new(0, GPU_TENSOR_NHWC(000, 16BF, output_dim, kernel_size, kernel_size, input_dim), 0);
    5772
    0
      ccv_nnc_tensor_t* gwo = ccv_nnc_tensor_new(0, GPU_TENSOR_NCHW(000, 16BF, output_dim, input_dim, kernel_size, kernel_size), 0);
    5773
    0
      ccv_nnc_tensor_t* gdw = ccv_nnc_tensor_new(0, GPU_TENSOR_NHWC(000, 16BF, output_dim, kernel_size, kernel_size, input_dim), 0);
    5774
    0
      ccv_nnc_tensor_t* gdwo = ccv_nnc_tensor_new(0, GPU_TENSOR_NCHW(000, 16BF, output_dim, input_dim, kernel_size, kernel_size), 0);
    5775
    0
      ccv_nnc_tensor_t* gdbias = ccv_nnc_tensor_new(0, GPU_TENSOR_NHWC(000, 16BF, 1, 1, 1, output_dim), 0);
    5776
    0
      ccv_nnc_cmd_t move = CMD_DATA_TRANSFER_FORWARD();
    5777
    0
      move.backend = CCV_NNC_BACKEND_MPS;
    5778
    0
      assert(move.backend >= 0);
    5779
    0
      ccv_nnc_cmd_exec(move, ccv_nnc_no_hint, 0, TENSOR_LIST(a16bf, w16bf, g16bf), TENSOR_LIST(ga, gw, gg), 0);
    5780
    0
      ccv_nnc_cmd_t transform = CMD_FORMAT_TRANSFORM_FORWARD();
    5781
    0
      transform.backend = CCV_NNC_BACKEND_MPS;
    5782
    0
      assert(transform.backend >= 0);
    5783
    0
      ccv_nnc_cmd_exec(transform, ccv_nnc_no_hint, 0, TENSOR_LIST(gw), TENSOR_LIST(gwo), 0);
    5784
    0
      cmd.backend = CCV_NNC_BACKEND_MPS;
    5785
    0
      assert(cmd.backend >= 0);
    5786
    0
      cmd.algorithm = -1;
    5787
    0
      ccv_nnc_stream_context_t* stream_context = ccv_nnc_stream_context_new(CCV_STREAM_CONTEXT_GPU);
    5788
    0
      cmd = ccv_nnc_cmd_autotune(cmd, 1 * 1024 * 1024 * 1024, hint, 0, TENSOR_LIST(gg, ga, gwo), TENSOR_LIST(gh, gdwo, gdbias), stream_context);
    5789
    0
      assert(CCV_NNC_EXEC_SUCCESS == ccv_nnc_cmd_exec(cmd, hint, 0, TENSOR_LIST(gg, ga, gwo), TENSOR_LIST(gh, gdwo, gdbias), stream_context));
    5790
    0
      ccv_nnc_stream_context_wait(stream_context);
    5791
    0
      ccv_nnc_stream_context_free(stream_context);
    5792
    0
      ccv_nnc_cmd_exec(transform, ccv_nnc_no_hint, 0, TENSOR_LIST(gdwo), TENSOR_LIST(gdw), 0);
    5793
    0
      ccv_nnc_tensor_t* ch16bf = ccv_nnc_tensor_new(0, CPU_TENSOR_NHWC(16BF, batch_size, input_size, input_size, input_dim), 0);
    5794
    0
      ccv_nnc_tensor_t* cdw16bf = ccv_nnc_tensor_new(0, CPU_TENSOR_NHWC(16BF, output_dim, kernel_size, kernel_size, input_dim), 0);
    5795
    0
      ccv_nnc_tensor_t* cdbias16bf = ccv_nnc_tensor_new(0, CPU_TENSOR_NHWC(16BF, 1, 1, 1, output_dim), 0);
    5796
    0
      ccv_nnc_tensor_t* ch = ccv_nnc_tensor_new(0, CPU_TENSOR_NHWC(32F, batch_size, input_size, input_size, input_dim), 0);
    5797
    0
      ccv_nnc_tensor_t* cdw = ccv_nnc_tensor_new(0, CPU_TENSOR_NHWC(32F, output_dim, kernel_size, kernel_size, input_dim), 0);
    5798
    0
      ccv_nnc_tensor_t* cdbias = ccv_nnc_tensor_new(0, CPU_TENSOR_NHWC(32F, 1, 1, 1, output_dim), 0);
    5799
    0
      ccv_nnc_cmd_exec(move, ccv_nnc_no_hint, 0, TENSOR_LIST(gh, gdw, gdbias), TENSOR_LIST(ch16bf, cdw16bf, cdbias16bf), 0);
    5800
    0
      ccv_bfloat_to_float((uint16_t*)ch16bf->data.f16, ch->data.f32, input_count);
    5801
    0
      ccv_bfloat_to_float((uint16_t*)cdw16bf->data.f16, cdw->data.f32, weight_count);
    5802
    0
      ccv_bfloat_to_float((uint16_t*)cdbias16bf->data.f16, cdbias->data.f32, output_dim);
    5803
    0
      REQUIRE_ARRAY_EQ_WITH_TOLERANCE(float, h->data.f32, ch->data.f32, input_count, 5e-2, "bfloat input gradient from mps should match CPU reference rounded to bfloat");
    5804
    0
      REQUIRE_ARRAY_EQ_WITH_TOLERANCE(float, dw->data.f32, cdw->data.f32, weight_count, 1e-1, "bfloat weight gradient from mps should match CPU reference rounded to bfloat");
    5805
    0
      REQUIRE_ARRAY_EQ_WITH_TOLERANCE(float, dbias->data.f32, cdbias->data.f32, output_dim, 1e-1, "bfloat bias gradient from mps should match CPU reference rounded to bfloat");
    5806
    0
      ccv_nnc_tensor_free(cdbias);
    5807
    0
      ccv_nnc_tensor_free(cdw);
    5808
    0
      ccv_nnc_tensor_free(ch);
    5809
    0
      ccv_nnc_tensor_free(cdbias16bf);
    5810
    0
      ccv_nnc_tensor_free(cdw16bf);
    5811
    0
      ccv_nnc_tensor_free(ch16bf);
    5812
    0
      ccv_nnc_tensor_free(gdbias);
    5813
    0
      ccv_nnc_tensor_free(gdwo);
    5814
    0
      ccv_nnc_tensor_free(gdw);
    5815
    0
      ccv_nnc_tensor_free(gwo);
    5816
    0
      ccv_nnc_tensor_free(gw);
    5817
    0
      ccv_nnc_tensor_free(gh);
    5818
    0
      ccv_nnc_tensor_free(gg);
    5819
    0
      ccv_nnc_tensor_free(ga);
    5820
    0
      ccv_nnc_tensor_free(dbias16bf);
    5821
    0
      ccv_nnc_tensor_free(dw16bf);
    5822
    0
      ccv_nnc_tensor_free(h16bf);
    5823
    0
      ccv_nnc_tensor_free(g16bf);
    5824
    0
      ccv_nnc_tensor_free(w16bf);
    5825
    0
      ccv_nnc_tensor_free(a16bf);
    5826
    0
      ccv_nnc_tensor_free(dbias);
    5827
    0
      ccv_nnc_tensor_free(dw);
    5828
    0
      ccv_nnc_tensor_free(w);
    5829
    0
      ccv_nnc_tensor_free(g);
    5830
    0
      ccv_nnc_tensor_free(h);
    5831
    0
      ccv_nnc_tensor_free(a);
    5832
    0
    }
    5833
    5834
    TEST_CASE("mps backward convolution in nchw format with dilation 2, 3")
    5835
    1
    {
    5836
    1
      GUARD_ELSE_RETURN(ccv_nnc_cmd_ok(CCV_NNC_CONVOLUTION_BACKWARD, CCV_NNC_BACKEND_MPS));
    5837
    0
      ccv_nnc_tensor_t* a = ccv_nnc_tensor_new(0, CPU_TENSOR_NHWC(32F, BATCH_SIZE, INPUT_SIZE, INPUT_SIZE, INPUT_DIM), 0);
    5838
    0
      ccv_nnc_tensor_t* h = ccv_nnc_tensor_new(0, CPU_TENSOR_NHWC(32F, BATCH_SIZE, INPUT_SIZE, INPUT_SIZE, INPUT_DIM), 0);
    5839
    0
      ccv_nnc_tensor_t* g = ccv_nnc_tensor_new(0, CPU_TENSOR_NHWC(32F, BATCH_SIZE, OUTPUT_SIZE, OUTPUT_SIZE, OUTPUT_DIM), 0);
    5840
    0
      ccv_nnc_cmd_t cmd = CMD_CONVOLUTION_BACKWARD(1, OUTPUT_DIM, KERNEL_SIZE, KERNEL_SIZE, INPUT_DIM);
    5841
    0
      cmd.info.convolution.dilation[0] = 2;
    5842
    0
      cmd.info.convolution.dilation[1] = 3;
    5843
    0
      cmd.backend = CCV_NNC_BACKEND_CPU_REF;
    5844
    0
      assert(cmd.backend >= 0);
    5845
    0
      ccv_nnc_cmd_param_t modified_cmd = cmd.info;
    5846
    0
      modified_cmd.size.dim[0] = (cmd.info.size.dim[0] - 1) * ccv_max(cmd.info.convolution.dilation[0], 1) + 1;
    5847
    0
      modified_cmd.size.dim[1] = (cmd.info.size.dim[1] - 1) * ccv_max(cmd.info.convolution.dilation[1], 1) + 1;
    5848
    0
      ccv_nnc_hint_t hint = ccv_nnc_hint_auto(modified_cmd, a->info, g->info);
    5849
    0
      assert(ccv_nnc_hint_verify(hint, modified_cmd, a->info, g->info) == 0);
    5850
    0
      ccv_nnc_tensor_t* w = ccv_nnc_tensor_new(0, CPU_TENSOR_NHWC(32F, OUTPUT_DIM, KERNEL_SIZE, KERNEL_SIZE, INPUT_DIM), 0);
    5851
    0
      ccv_nnc_tensor_t* dw = ccv_nnc_tensor_new(0, CPU_TENSOR_NHWC(32F, OUTPUT_DIM, KERNEL_SIZE, KERNEL_SIZE, INPUT_DIM), 0);
    5852
    0
      ccv_nnc_tensor_t* dbias = ccv_nnc_tensor_new(0, CPU_TENSOR_NHWC(32F, 1, 1, 1, OUTPUT_DIM), 0);
    5853
      // configure the inlets.
    5854
    0
      dsfmt_t dsfmt;
    5855
    0
      dsfmt_init_gen_rand(&dsfmt, 0);
    5856
    0
      int i;
    5857
    0
      for (i = 0; i < INPUT_DIM * KERNEL_SIZE * KERNEL_SIZE * OUTPUT_DIM; i++)
    5858
    0
        w->data.f32[i] = dsfmt_genrand_open_close(&dsfmt) / (INPUT_DIM * KERNEL_SIZE * KERNEL_SIZE);
    5859
    0
      for (i = 0; i < INPUT_SIZE * INPUT_SIZE * INPUT_DIM * ccv_max(1, BATCH_SIZE); i++)
    5860
    0
        a->data.f32[i] = dsfmt_genrand_open_close(&dsfmt);
    5861
    0
      for (i = 0; i < OUTPUT_SIZE * OUTPUT_SIZE * OUTPUT_DIM * ccv_max(1, BATCH_SIZE); i++)
    5862
    0
        g->data.f32[i] = dsfmt_genrand_open_close(&dsfmt) / OUTPUT_DIM; // (OUTPUT_SIZE * OUTPUT_SIZE * OUTPUT_DIM);
    5863
      // Copy generated matrix values over to GPU.
    5864
    0
      ccv_nnc_tensor_t* ga = ccv_nnc_tensor_new(0, GPU_TENSOR_NHWC(000, 32F, BATCH_SIZE, INPUT_SIZE, INPUT_SIZE, INPUT_DIM), 0);
    5865
    0
      ccv_nnc_tensor_t* gg = ccv_nnc_tensor_new(0, GPU_TENSOR_NHWC(000, 32F, BATCH_SIZE, OUTPUT_SIZE, OUTPUT_SIZE, OUTPUT_DIM), 0);
    5866
    0
      ccv_nnc_tensor_t* gh = ccv_nnc_tensor_new(0, GPU_TENSOR_NHWC(000, 32F, BATCH_SIZE, INPUT_SIZE, INPUT_SIZE, INPUT_DIM), 0);
    5867
    0
      ccv_nnc_tensor_t* gw = ccv_nnc_tensor_new(0, GPU_TENSOR_NHWC(000, 32F, OUTPUT_DIM, KERNEL_SIZE, KERNEL_SIZE, INPUT_DIM), 0);
    5868
    0
      ccv_nnc_tensor_t* gbias = ccv_nnc_tensor_new(0, GPU_TENSOR_NHWC(000, 32F, 1, 1, 1, OUTPUT_DIM), 0);
    5869
    0
      ccv_nnc_tensor_t* gdw = ccv_nnc_tensor_new(0, GPU_TENSOR_NHWC(000, 32F, OUTPUT_DIM, KERNEL_SIZE, KERNEL_SIZE, INPUT_DIM), 0);
    5870
    0
      ccv_nnc_tensor_t* gdbias = ccv_nnc_tensor_new(0, GPU_TENSOR_NHWC(000, 32F, 1, 1, 1, OUTPUT_DIM), 0);
    5871
    0
      ccv_nnc_tensor_t* gao = ccv_nnc_tensor_new(0, GPU_TENSOR_NCHW(000, 32F, BATCH_SIZE, INPUT_DIM, INPUT_SIZE, INPUT_SIZE), 0);
    5872
    0
      ccv_nnc_tensor_t* ggo = ccv_nnc_tensor_new(0, GPU_TENSOR_NCHW(000, 32F, BATCH_SIZE, OUTPUT_DIM, OUTPUT_SIZE, OUTPUT_SIZE), 0);
    5873
    0
      ccv_nnc_tensor_t* gho = ccv_nnc_tensor_new(0, GPU_TENSOR_NCHW(000, 32F, BATCH_SIZE, INPUT_DIM, INPUT_SIZE, INPUT_SIZE), 0);
    5874
    0
      ccv_nnc_tensor_t* gwo = ccv_nnc_tensor_new(0, GPU_TENSOR_NCHW(000, 32F, OUTPUT_DIM, INPUT_DIM, KERNEL_SIZE, KERNEL_SIZE), 0);
    5875
    0
      ccv_nnc_tensor_t* gbiaso = ccv_nnc_tensor_new(0, GPU_TENSOR_NCHW(000, 32F, 1, OUTPUT_DIM, 1, 1), 0);
    5876
    0
      ccv_nnc_tensor_t* gdwo = ccv_nnc_tensor_new(0, GPU_TENSOR_NCHW(000, 32F, OUTPUT_DIM, INPUT_DIM, KERNEL_SIZE, KERNEL_SIZE), 0);
    5877
    0
      ccv_nnc_tensor_t* gdbiaso = ccv_nnc_tensor_new(0, GPU_TENSOR_NCHW(000, 32F, 1, OUTPUT_DIM, 1, 1), 0);
    5878
    0
      ccv_nnc_cmd_exec(CMD_DATA_TRANSFER_FORWARD(), ccv_nnc_no_hint, 0, TENSOR_LIST(a, w, g), TENSOR_LIST(ga, gw, gg), 0);
    5879
    0
      ccv_nnc_cmd_exec(cmd, hint, 0, TENSOR_LIST(g, a, w), TENSOR_LIST(h, dw, dbias), 0);
    5880
    0
      ccv_nnc_cmd_exec(CMD_FORMAT_TRANSFORM_FORWARD(), ccv_nnc_no_hint, 0, TENSOR_LIST(ga, gw, gg), TENSOR_LIST(gao, gwo, ggo), 0);
    5881
    0
      cmd.backend = CCV_NNC_BACKEND_MPS;
    5882
    5883
    0
      assert(cmd.backend >= 0);
    5884
    0
      cmd.algorithm = -1;
    5885
    0
      ccv_nnc_stream_context_t* stream_context = ccv_nnc_stream_context_new(CCV_STREAM_CONTEXT_GPU);
    5886
    5887
    0
      cmd = ccv_nnc_cmd_autotune(cmd, 1 * 1024 * 1024 * 1024, hint, 0, TENSOR_LIST(ggo, gao, gwo), TENSOR_LIST(gho, gdwo, gdbiaso), stream_context);
    5888
    0
      assert(CCV_NNC_EXEC_SUCCESS == ccv_nnc_cmd_exec(cmd, hint, 0, TENSOR_LIST(ggo, gao, gwo), TENSOR_LIST(gho, gdwo, gdbiaso), stream_context));
    5889
    0
      ccv_nnc_stream_context_wait(stream_context);
    5890
    0
      ccv_nnc_stream_context_free(stream_context);
    5891
    0
      ccv_nnc_tensor_t* ch = ccv_nnc_tensor_new(0, CPU_TENSOR_NHWC(32F, BATCH_SIZE, INPUT_SIZE, INPUT_SIZE, INPUT_DIM), 0);
    5892
    0
      ccv_nnc_tensor_t* cdw = ccv_nnc_tensor_new(0, CPU_TENSOR_NHWC(32F, OUTPUT_DIM, KERNEL_SIZE, KERNEL_SIZE, INPUT_DIM), 0);
    5893
    0
      ccv_nnc_tensor_t* cdbias = ccv_nnc_tensor_new(0, CPU_TENSOR_NHWC(32F, 1, OUTPUT_DIM, 1, 1), 0);
    5894
    5895
    0
      ccv_nnc_cmd_exec(CMD_FORMAT_TRANSFORM_FORWARD(), ccv_nnc_no_hint, 0, TENSOR_LIST(gho, gdwo, gdbiaso), TENSOR_LIST(gh, gdw, gdbias), 0);
    5896
    0
      ccv_nnc_cmd_exec(CMD_DATA_TRANSFER_FORWARD(), ccv_nnc_no_hint, 0, TENSOR_LIST(gh, gdw, gdbias), TENSOR_LIST(ch, cdw, cdbias), 0);
    5897
    5898
    0
      REQUIRE_ARRAY_EQ_WITH_TOLERANCE(float, dw->data.f32, cdw->data.f32, INPUT_DIM * OUTPUT_DIM * KERNEL_SIZE * KERNEL_SIZE, 5e-1, "output from mps should match from CPU");
    5899
    0
      REQUIRE_ARRAY_EQ_WITH_TOLERANCE(float, dbias->data.f32, cdbias->data.f32, OUTPUT_DIM, 5e-1, "output from mps should match from CPU");
    5900
    0
      REQUIRE_ARRAY_EQ_WITH_TOLERANCE(float, h->data.f32, ch->data.f32, BATCH_SIZE * INPUT_DIM * INPUT_SIZE * INPUT_SIZE, 1e-4, "output from mps should match from CPU");
    5901
    0
      ccv_nnc_tensor_free(gao);
    5902
    0
      ccv_nnc_tensor_free(ggo);
    5903
    0
      ccv_nnc_tensor_free(gho);
    5904
    0
      ccv_nnc_tensor_free(gwo);
    5905
    0
      ccv_nnc_tensor_free(gbiaso);
    5906
    0
      ccv_nnc_tensor_free(gdwo);
    5907
    0
      ccv_nnc_tensor_free(gdbiaso);
    5908
    0
      ccv_nnc_tensor_free(h);
    5909
    0
      ccv_nnc_tensor_free(gh);
    5910
    0
      ccv_nnc_tensor_free(w);
    5911
    0
      ccv_nnc_tensor_free(g);
    5912
    0
      ccv_nnc_tensor_free(a);
    5913
    0
      ccv_nnc_tensor_free(gbias);
    5914
    0
      ccv_nnc_tensor_free(gdbias);
    5915
    0
      ccv_nnc_tensor_free(gdw);
    5916
    0
      ccv_nnc_tensor_free(gw);
    5917
    0
      ccv_nnc_tensor_free(gg);
    5918
    0
      ccv_nnc_tensor_free(ga);
    5919
    0
      ccv_nnc_tensor_free(ch);
    5920
    0
      ccv_nnc_tensor_free(cdw);
    5921
    0
      ccv_nnc_tensor_free(cdbias);
    5922
    0
    }
    5923
    5924
    TEST_CASE("compare group norm gradient with mps")
    5925
    1
    {
    5926
    1
      GUARD_ELSE_RETURN(ccv_nnc_cmd_ok(CCV_NNC_GROUP_NORM_FORWARD, CCV_NNC_BACKEND_MPS) &&
    5927
    1
        ccv_nnc_cmd_ok(CCV_NNC_GROUP_NORM_BACKWARD, CCV_NNC_BACKEND_MPS) &&
    5928
    1
        (ccv_nnc_cmd_ok(CCV_NNC_SET_FORWARD, CCV_NNC_BACKEND_MPS) || ccv_nnc_cmd_ok(CCV_NNC_SET_FORWARD, CCV_NNC_BACKEND_MPS)));
    5929
    0
      ccv_nnc_symbolic_graph_t* const symbolic_graph = ccv_nnc_symbolic_graph_new();
    5930
    0
      ccv_nnc_tensor_symbol_t bx = ccv_nnc_tensor_symbol_new(symbolic_graph, GPU_TENSOR_NHWC(000, 32F, 2, GN_C_DIM, 2, LN_DIM), "x");
    5931
    0
      ccv_nnc_tensor_symbol_t by = ccv_nnc_tensor_symbol_new(symbolic_graph, GPU_TENSOR_NHWC(000, 32F, 2, GN_C_DIM, 2, LN_DIM), "y");
    5932
    0
      ccv_nnc_tensor_symbol_t scale = ccv_nnc_tensor_symbol_new(symbolic_graph, GPU_TENSOR_NHWC(000, 32F, 1, GN_C_DIM, 2, LN_DIM), "scale");
    5933
    0
      ccv_nnc_tensor_symbol_t bias = ccv_nnc_tensor_symbol_new(symbolic_graph, GPU_TENSOR_NHWC(000, 32F, 1, GN_C_DIM, 2, LN_DIM), "bias");
    5934
    0
      ccv_nnc_tensor_symbol_t saved_mean = ccv_nnc_tensor_symbol_new(symbolic_graph, GPU_TENSOR_NHWC(000, 32F, 2, GN_RC_DIM, 2, LN_DIM), "saved_mean");
    5935
    0
      ccv_nnc_tensor_symbol_t saved_inv_std = ccv_nnc_tensor_symbol_new(symbolic_graph, GPU_TENSOR_NHWC(000, 32F, 2, GN_RC_DIM, 2, LN_DIM), "saved_inv_std");
    5936
    0
      ccv_nnc_graph_exec_symbol_new(symbolic_graph, CMD_GROUP_NORM_FORWARD(1, 4, 1e-5, 1), TENSOR_SYMBOL_LIST(bx, scale, bias), TENSOR_SYMBOL_LIST(by, saved_mean, saved_inv_std), "group_norm");
    5937
    0
      ccv_nnc_graph_exec_symbol_autogen(symbolic_graph, 0, 0, CCV_NNC_AUTOGEN_ALL_EXECS | CCV_NNC_AUTOGEN_SOURCES_AND_DESTINATIONS);
    5938
    0
      ccv_nnc_symbolic_graph_backward(symbolic_graph, TENSOR_SYMBOL_LIST(by), TENSOR_SYMBOL_LIST(bx, scale, bias), SYMBOLIC_GRAPH_SOURCES(symbolic_graph), SYMBOLIC_GRAPH_DESTINATIONS(symbolic_graph));
    5939
    0
      ccv_nnc_graph_exec_symbol_autogen(symbolic_graph, 0, 0, CCV_NNC_AUTOGEN_ALL_EXECS | CCV_NNC_AUTOGEN_SOURCES_AND_DESTINATIONS);
    5940
    0
      ccv_nnc_tensor_symbol_t dby = ccv_nnc_tensor_symbol_for_backward(symbolic_graph, by);
    5941
    0
      ccv_nnc_tensor_symbol_t dbx = ccv_nnc_tensor_symbol_for_backward(symbolic_graph, bx);
    5942
    0
      ccv_nnc_graph_t* graph = 0;
    5943
    0
      ccv_nnc_tensor_arena_t* tensor_arena = 0;
    5944
    0
      ccv_nnc_graph_exec_arena_t* graph_exec_arena = 0;
    5945
    0
      ccv_nnc_symbolic_graph_compile(symbolic_graph, ccv_nnc_default_compile_params, 0, 0, 0, 0, SYMBOLIC_GRAPH_SOURCES(symbolic_graph), SYMBOLIC_GRAPH_DESTINATIONS(symbolic_graph), &graph, &tensor_arena, &graph_exec_arena);
    5946
    0
      SYMBOLIC_GRAPH_GEN(symbolic_graph, CCV_NNC_LONG_DOT_GRAPH);
    5947
    0
      GRAPH_GEN(graph, CCV_NNC_LONG_DOT_GRAPH);
    5948
    0
      ccv_nnc_tensor_t* const bx_tensor = ccv_nnc_tensor_from_symbol(tensor_arena, bx);
    5949
    0
      dsfmt_t dsfmt;
    5950
    0
      ccv_nnc_tensor_t* const x_tensor = ccv_nnc_tensor_new(0, CPU_TENSOR_NHWC(32F, 2, GN_C_DIM, 2, LN_DIM), 0);
    5951
    0
      int i;
    5952
    0
      dsfmt_init_gen_rand(&dsfmt, 1);
    5953
    0
      for (i = 0; i < 2 * GN_C_DIM * 2 * LN_DIM; i++)
    5954
    0
        x_tensor->data.f32[i] = dsfmt_genrand_open_close(&dsfmt) * 100;
    5955
    0
      ccv_nnc_cmd_exec(CMD_DATA_TRANSFER_FORWARD(), ccv_nnc_no_hint, 0, TENSOR_LIST(x_tensor), TENSOR_LIST(bx_tensor), 0);
    5956
    0
      float scaledata[1 * GN_C_DIM * 2 * LN_DIM];
    5957
    0
      float biasdata[1 * GN_C_DIM * 2 * LN_DIM];
    5958
    0
      for (i = 0; i < 1 * GN_C_DIM * 2 * LN_DIM; i++)
    5959
    0
      {
    5960
    0
        scaledata[i] = dsfmt_genrand_open_close(&dsfmt);
    5961
    0
        biasdata[i] = dsfmt_genrand_open_close(&dsfmt);
    5962
    0
      }
    5963
    0
      ccv_nnc_tensor_t scale_tensor = ccv_nnc_tensor(scaledata, CPU_TENSOR_NHWC(32F, 1, GN_C_DIM, 2, LN_DIM), 0);
    5964
    0
      ccv_nnc_tensor_t bias_tensor = ccv_nnc_tensor(biasdata, CPU_TENSOR_NHWC(32F, 1, GN_C_DIM, 2, LN_DIM), 0);
    5965
    0
      ccv_nnc_cmd_exec(CMD_DATA_TRANSFER_FORWARD(), ccv_nnc_no_hint, 0, TENSOR_LIST(&scale_tensor, &bias_tensor), TENSOR_LIST(ccv_nnc_tensor_from_symbol(tensor_arena, scale), ccv_nnc_tensor_from_symbol(tensor_arena, bias)), 0);
    5966
      // ccv_nnc_graph_run(graph, 0, TRAVERSE_FULL, 0, 0);
    5967
    0
      ccv_nnc_tensor_t* const dy_tensor = ccv_nnc_tensor_new(0, CPU_TENSOR_NHWC(32F, 2, GN_C_DIM, 2, LN_DIM), 0);
    5968
    0
      ccv_nnc_tensor_t* const dby_tensor = ccv_nnc_tensor_from_symbol(tensor_arena, dby);
    5969
    0
      for (i = 0; i < 2 * GN_C_DIM * 2 * LN_DIM; i++)
    5970
    0
        dy_tensor->data.f32[i] = dsfmt_genrand_open_close(&dsfmt) * 2 - 1;
    5971
    0
      ccv_nnc_cmd_exec(CMD_DATA_TRANSFER_FORWARD(), ccv_nnc_no_hint, 0, TENSOR_LIST(dy_tensor), TENSOR_LIST(dby_tensor), 0);
    5972
    0
      ccv_nnc_graph_run(graph, 0, TRAVERSE_FULL, 0, 0);
    5973
    0
      ccv_nnc_tensor_t* const dbx_tensor = ccv_nnc_tensor_from_symbol(tensor_arena, dbx);
    5974
    0
      ccv_nnc_tensor_t* const dx_tensor = ccv_nnc_tensor_new(0, CPU_TENSOR_NHWC(32F, 2, GN_C_DIM, 2, LN_DIM), 0);
    5975
    0
      ccv_nnc_cmd_exec(CMD_DATA_TRANSFER_FORWARD(), ccv_nnc_no_hint, 0, TENSOR_LIST(dbx_tensor), TENSOR_LIST(dx_tensor), 0);
    5976
    0
      ccv_nnc_tensor_t* const dbscale_tensor = ccv_nnc_tensor_from_symbol(tensor_arena, ccv_nnc_tensor_symbol_for_backward(symbolic_graph, scale));
    5977
    0
      ccv_nnc_tensor_t* const dbbias_tensor = ccv_nnc_tensor_from_symbol(tensor_arena, ccv_nnc_tensor_symbol_for_backward(symbolic_graph, bias));
    5978
    0
      ccv_nnc_tensor_t* const dscale_tensor = ccv_nnc_tensor_new(0, CPU_TENSOR_NHWC(32F, 1, GN_C_DIM, 2, LN_DIM), 0);
    5979
    0
      ccv_nnc_tensor_t* const dbias_tensor = ccv_nnc_tensor_new(0, CPU_TENSOR_NHWC(32F, 1, GN_C_DIM, 2, LN_DIM), 0);
    5980
    0
      ccv_nnc_cmd_exec(CMD_DATA_TRANSFER_FORWARD(), ccv_nnc_no_hint, 0, TENSOR_LIST(dbscale_tensor, dbbias_tensor), TENSOR_LIST(dscale_tensor, dbias_tensor), 0);
    5981
    0
      ccv_nnc_symbolic_graph_free(symbolic_graph);
    5982
    0
      ccv_nnc_tensor_arena_free(tensor_arena);
    5983
    0
      ccv_nnc_graph_exec_arena_free(graph_exec_arena);
    5984
    0
      ccv_nnc_graph_free(graph);
    5985
    0
      ccv_nnc_symbolic_graph_t* const cpu_symbolic_graph = ccv_nnc_symbolic_graph_new();
    5986
    0
      ccv_nnc_tensor_symbol_t cx = ccv_nnc_tensor_symbol_new(cpu_symbolic_graph, CPU_TENSOR_NHWC(32F, 2, GN_C_DIM, 2, LN_DIM), "x");
    5987
    0
      ccv_nnc_tensor_symbol_t cy = ccv_nnc_tensor_symbol_new(cpu_symbolic_graph, CPU_TENSOR_NHWC(32F, 2, GN_C_DIM, 2, LN_DIM), "y");
    5988
    0
      ccv_nnc_tensor_symbol_t cscale = ccv_nnc_tensor_symbol_new(cpu_symbolic_graph, CPU_TENSOR_NHWC(32F, 1, GN_C_DIM, 2, LN_DIM), "scale");
    5989
    0
      ccv_nnc_tensor_symbol_t cbias = ccv_nnc_tensor_symbol_new(cpu_symbolic_graph, CPU_TENSOR_NHWC(32F, 1, GN_C_DIM, 2, LN_DIM), "bias");
    5990
    0
      ccv_nnc_tensor_symbol_t csaved_mean = ccv_nnc_tensor_symbol_new(cpu_symbolic_graph, CPU_TENSOR_NHWC(32F, 2, GN_RC_DIM, 2, LN_DIM), "saved_mean");
    5991
    0
      ccv_nnc_tensor_symbol_t csaved_inv_std = ccv_nnc_tensor_symbol_new(cpu_symbolic_graph, CPU_TENSOR_NHWC(32F, 2, GN_RC_DIM, 2, LN_DIM), "saved_inv_std");
    5992
    0
      ccv_nnc_graph_exec_symbol_new(cpu_symbolic_graph, CMD_GROUP_NORM_FORWARD(1, GN_RC_DIM, 1e-5, 1), TENSOR_SYMBOL_LIST(cx, cscale, cbias), TENSOR_SYMBOL_LIST(cy, csaved_mean, csaved_inv_std), "group_norm");
    5993
    0
      ccv_nnc_graph_exec_symbol_autogen(cpu_symbolic_graph, 0, 0, CCV_NNC_AUTOGEN_ALL_EXECS | CCV_NNC_AUTOGEN_SOURCES_AND_DESTINATIONS);
    5994
    0
      ccv_nnc_symbolic_graph_backward(cpu_symbolic_graph, TENSOR_SYMBOL_LIST(cy), TENSOR_SYMBOL_LIST(cx, cscale, cbias), SYMBOLIC_GRAPH_SOURCES(cpu_symbolic_graph), SYMBOLIC_GRAPH_DESTINATIONS(cpu_symbolic_graph));
    5995
    0
      ccv_nnc_graph_exec_symbol_autogen(cpu_symbolic_graph, 0, 0, CCV_NNC_AUTOGEN_ALL_EXECS | CCV_NNC_AUTOGEN_SOURCES_AND_DESTINATIONS);
    5996
    0
      ccv_nnc_tensor_symbol_t dcy = ccv_nnc_tensor_symbol_for_backward(cpu_symbolic_graph, cy);
    5997
    0
      ccv_nnc_tensor_symbol_t dcx = ccv_nnc_tensor_symbol_for_backward(cpu_symbolic_graph, cx);
    5998
    0
      ccv_nnc_tensor_symbol_t dcscale = ccv_nnc_tensor_symbol_for_backward(cpu_symbolic_graph, cscale);
    5999
    0
      ccv_nnc_tensor_symbol_t dcbias = ccv_nnc_tensor_symbol_for_backward(cpu_symbolic_graph, cbias);
    6000
    0
      ccv_nnc_graph_t* cpu_graph = 0;
    6001
    0
      ccv_nnc_tensor_arena_t* cpu_tensor_arena = 0;
    6002
    0
      ccv_nnc_graph_exec_arena_t* cpu_graph_exec_arena = 0;
    6003
    0
      ccv_nnc_symbolic_graph_compile(cpu_symbolic_graph, ccv_nnc_default_compile_params, 0, 0, 0, 0, SYMBOLIC_GRAPH_SOURCES(cpu_symbolic_graph), SYMBOLIC_GRAPH_DESTINATIONS(cpu_symbolic_graph), &cpu_graph, &cpu_tensor_arena, &cpu_graph_exec_arena);
    6004
    0
      ccv_nnc_tensor_t* const cx_tensor = ccv_nnc_tensor_from_symbol(cpu_tensor_arena, cx);
    6005
    0
      memcpy(cx_tensor->data.f32, x_tensor->data.f32, sizeof(float) * 2 * GN_C_DIM * 2 * LN_DIM);
    6006
    0
      ccv_nnc_tensor_t* const dcy_tensor = ccv_nnc_tensor_from_symbol(cpu_tensor_arena, dcy);
    6007
    0
      memcpy(dcy_tensor->data.f32, dy_tensor->data.f32, sizeof(float) * 2 * GN_C_DIM * 2 * LN_DIM);
    6008
    0
      ccv_nnc_tensor_t* const cscale_tensor = ccv_nnc_tensor_from_symbol(cpu_tensor_arena, cscale);
    6009
    0
      memcpy(cscale_tensor->data.f32, scaledata, sizeof(float) * 1 * GN_C_DIM * 2 * LN_DIM);
    6010
    0
      ccv_nnc_tensor_t* const cbias_tensor = ccv_nnc_tensor_from_symbol(cpu_tensor_arena, cbias);
    6011
    0
      memcpy(cbias_tensor->data.f32, biasdata, sizeof(float) * 1 * GN_C_DIM * 2 * LN_DIM);
    6012
    6013
    0
      ccv_nnc_graph_run(cpu_graph, 0, TRAVERSE_FULL, 0, 0);
    6014
    6015
    0
      ccv_nnc_tensor_t* const dcx_tensor = ccv_nnc_tensor_from_symbol(cpu_tensor_arena, dcx);
    6016
    0
      ccv_nnc_tensor_t* const dcscale_tensor = ccv_nnc_tensor_from_symbol(cpu_tensor_arena, dcscale);
    6017
    0
      ccv_nnc_tensor_t* const dcbias_tensor = ccv_nnc_tensor_from_symbol(cpu_tensor_arena, dcbias);
    6018
    6019
    0
      REQUIRE_ARRAY_EQ_WITH_TOLERANCE(float, dx_tensor->data.f32, dcx_tensor->data.f32, 2 * GN_C_DIM * 2 * LN_DIM, 1e-5, "group norm output from mps should match from CPU");
    6020
    0
      REQUIRE_ARRAY_EQ_WITH_TOLERANCE(float, dbias_tensor->data.f32, dcbias_tensor->data.f32, 1 * GN_C_DIM * 2 * LN_DIM, 1e-5, "group norm output from mps should match from CPU");
    6021
    0
      REQUIRE_ARRAY_EQ_WITH_TOLERANCE(float, dscale_tensor->data.f32, dcscale_tensor->data.f32, 1 * GN_C_DIM * 2 * LN_DIM, 1e-5, "group norm output from mps should match from CPU");
    6022
    6023
    0
      ccv_nnc_symbolic_graph_free(cpu_symbolic_graph);
    6024
    0
      ccv_nnc_tensor_arena_free(cpu_tensor_arena);
    6025
    0
      ccv_nnc_graph_exec_arena_free(cpu_graph_exec_arena);
    6026
    0
      ccv_nnc_graph_free(cpu_graph);
    6027
    0
      ccv_nnc_tensor_free(x_tensor);
    6028
    0
      ccv_nnc_tensor_free(dy_tensor);
    6029
    0
      ccv_nnc_tensor_free(dx_tensor);
    6030
    0
      ccv_nnc_tensor_free(dscale_tensor);
    6031
    0
      ccv_nnc_tensor_free(dbias_tensor);
    6032
    0
    }
    6033
    6034
    TEST_CASE("compare group norm gradient with mps, variant 1")
    6035
    1
    {
    6036
    1
      GUARD_ELSE_RETURN(ccv_nnc_cmd_ok(CCV_NNC_GROUP_NORM_FORWARD, CCV_NNC_BACKEND_MPS) &&
    6037
    1
        ccv_nnc_cmd_ok(CCV_NNC_GROUP_NORM_BACKWARD, CCV_NNC_BACKEND_MPS) &&
    6038
    1
        (ccv_nnc_cmd_ok(CCV_NNC_SET_FORWARD, CCV_NNC_BACKEND_MPS) || ccv_nnc_cmd_ok(CCV_NNC_SET_FORWARD, CCV_NNC_BACKEND_MPS)));
    6039
    0
      ccv_nnc_symbolic_graph_t* const symbolic_graph = ccv_nnc_symbolic_graph_new();
    6040
    0
      ccv_nnc_tensor_symbol_t bx = ccv_nnc_tensor_symbol_new(symbolic_graph, GPU_TENSOR_NHWC(000, 32F, 2, GN_C_DIM, 2, LN_DIM), "x");
    6041
    0
      ccv_nnc_tensor_symbol_t by = ccv_nnc_tensor_symbol_new(symbolic_graph, GPU_TENSOR_NHWC(000, 32F, 2, GN_C_DIM, 2, LN_DIM), "y");
    6042
    0
      ccv_nnc_tensor_symbol_t scale = ccv_nnc_tensor_symbol_new(symbolic_graph, GPU_TENSOR_NHWC(000, 32F, 1, GN_C_DIM, 1, 1), "scale");
    6043
    0
      ccv_nnc_tensor_symbol_t bias = ccv_nnc_tensor_symbol_new(symbolic_graph, GPU_TENSOR_NHWC(000, 32F, 1, GN_C_DIM, 1, 1), "bias");
    6044
    0
      ccv_nnc_tensor_symbol_t saved_mean = ccv_nnc_tensor_symbol_new(symbolic_graph, GPU_TENSOR_NHWC(000, 32F, 2, GN_RC_DIM, 1, 1), "saved_mean");
    6045
    0
      ccv_nnc_tensor_symbol_t saved_inv_std = ccv_nnc_tensor_symbol_new(symbolic_graph, GPU_TENSOR_NHWC(000, 32F, 2, GN_RC_DIM, 1, 1), "saved_inv_std");
    6046
    0
      ccv_nnc_graph_exec_symbol_new(symbolic_graph, CMD_GROUP_NORM_FORWARD(1, 4, 1e-5, 1), TENSOR_SYMBOL_LIST(bx, scale, bias), TENSOR_SYMBOL_LIST(by, saved_mean, saved_inv_std), "group_norm");
    6047
    0
      ccv_nnc_graph_exec_symbol_autogen(symbolic_graph, 0, 0, CCV_NNC_AUTOGEN_ALL_EXECS | CCV_NNC_AUTOGEN_SOURCES_AND_DESTINATIONS);
    6048
    0
      ccv_nnc_symbolic_graph_backward(symbolic_graph, TENSOR_SYMBOL_LIST(by), TENSOR_SYMBOL_LIST(bx, scale, bias), SYMBOLIC_GRAPH_SOURCES(symbolic_graph), SYMBOLIC_GRAPH_DESTINATIONS(symbolic_graph));
    6049
    0
      ccv_nnc_graph_exec_symbol_autogen(symbolic_graph, 0, 0, CCV_NNC_AUTOGEN_ALL_EXECS | CCV_NNC_AUTOGEN_SOURCES_AND_DESTINATIONS);
    6050
    0
      ccv_nnc_tensor_symbol_t dby = ccv_nnc_tensor_symbol_for_backward(symbolic_graph, by);
    6051
    0
      ccv_nnc_tensor_symbol_t dbx = ccv_nnc_tensor_symbol_for_backward(symbolic_graph, bx);
    6052
    0
      ccv_nnc_graph_t* graph = 0;
    6053
    0
      ccv_nnc_tensor_arena_t* tensor_arena = 0;
    6054
    0
      ccv_nnc_graph_exec_arena_t* graph_exec_arena = 0;
    6055
    0
      ccv_nnc_symbolic_graph_compile(symbolic_graph, ccv_nnc_default_compile_params, 0, 0, 0, 0, SYMBOLIC_GRAPH_SOURCES(symbolic_graph), SYMBOLIC_GRAPH_DESTINATIONS(symbolic_graph), &graph, &tensor_arena, &graph_exec_arena);
    6056
    0
      SYMBOLIC_GRAPH_GEN(symbolic_graph, CCV_NNC_LONG_DOT_GRAPH);
    6057
    0
      GRAPH_GEN(graph, CCV_NNC_LONG_DOT_GRAPH);
    6058
    0
      ccv_nnc_tensor_t* const bx_tensor = ccv_nnc_tensor_from_symbol(tensor_arena, bx);
    6059
    0
      dsfmt_t dsfmt;
    6060
    0
      ccv_nnc_tensor_t* const x_tensor = ccv_nnc_tensor_new(0, CPU_TENSOR_NHWC(32F, 2, GN_C_DIM, 2, LN_DIM), 0);
    6061
    0
      int i;
    6062
    0
      dsfmt_init_gen_rand(&dsfmt, 1);
    6063
    0
      for (i = 0; i < 2 * GN_C_DIM * 2 * LN_DIM; i++)
    6064
    0
        x_tensor->data.f32[i] = dsfmt_genrand_open_close(&dsfmt) * 100;
    6065
    0
      ccv_nnc_cmd_exec(CMD_DATA_TRANSFER_FORWARD(), ccv_nnc_no_hint, 0, TENSOR_LIST(x_tensor), TENSOR_LIST(bx_tensor), 0);
    6066
    0
      float scaledata[1 * GN_C_DIM * 1 * 1];
    6067
    0
      float biasdata[1 * GN_C_DIM * 1 * 1];
    6068
    0
      for (i = 0; i < 1 * GN_C_DIM * 1 * 1; i++)
    6069
    0
      {
    6070
    0
        scaledata[i] = dsfmt_genrand_open_close(&dsfmt);
    6071
    0
        biasdata[i] = dsfmt_genrand_open_close(&dsfmt);
    6072
    0
      }
    6073
    0
      ccv_nnc_tensor_t scale_tensor = ccv_nnc_tensor(scaledata, CPU_TENSOR_NHWC(32F, 1, GN_C_DIM, 1, 1), 0);
    6074
    0
      ccv_nnc_tensor_t bias_tensor = ccv_nnc_tensor(biasdata, CPU_TENSOR_NHWC(32F, 1, GN_C_DIM, 1, 1), 0);
    6075
    0
      ccv_nnc_cmd_exec(CMD_DATA_TRANSFER_FORWARD(), ccv_nnc_no_hint, 0, TENSOR_LIST(&scale_tensor, &bias_tensor), TENSOR_LIST(ccv_nnc_tensor_from_symbol(tensor_arena, scale), ccv_nnc_tensor_from_symbol(tensor_arena, bias)), 0);
    6076
      // ccv_nnc_graph_run(graph, 0, TRAVERSE_FULL, 0, 0);
    6077
    0
      ccv_nnc_tensor_t* const dy_tensor = ccv_nnc_tensor_new(0, CPU_TENSOR_NHWC(32F, 2, GN_C_DIM, 2, LN_DIM), 0);
    6078
    0
      ccv_nnc_tensor_t* const dby_tensor = ccv_nnc_tensor_from_symbol(tensor_arena, dby);
    6079
    0
      for (i = 0; i < 2 * GN_C_DIM * 2 * LN_DIM; i++)
    6080
    0
        dy_tensor->data.f32[i] = dsfmt_genrand_open_close(&dsfmt) * 2 - 1;
    6081
    0
      ccv_nnc_cmd_exec(CMD_DATA_TRANSFER_FORWARD(), ccv_nnc_no_hint, 0, TENSOR_LIST(dy_tensor), TENSOR_LIST(dby_tensor), 0);
    6082
    0
      ccv_nnc_graph_run(graph, 0, TRAVERSE_FULL, 0, 0);
    6083
    0
      ccv_nnc_tensor_t* const dbx_tensor = ccv_nnc_tensor_from_symbol(tensor_arena, dbx);
    6084
    0
      ccv_nnc_tensor_t* const dx_tensor = ccv_nnc_tensor_new(0, CPU_TENSOR_NHWC(32F, 2, GN_C_DIM, 2, LN_DIM), 0);
    6085
    0
      ccv_nnc_cmd_exec(CMD_DATA_TRANSFER_FORWARD(), ccv_nnc_no_hint, 0, TENSOR_LIST(dbx_tensor), TENSOR_LIST(dx_tensor), 0);
    6086
    0
      ccv_nnc_tensor_t* const dbscale_tensor = ccv_nnc_tensor_from_symbol(tensor_arena, ccv_nnc_tensor_symbol_for_backward(symbolic_graph, scale));
    6087
    0
      ccv_nnc_tensor_t* const dbbias_tensor = ccv_nnc_tensor_from_symbol(tensor_arena, ccv_nnc_tensor_symbol_for_backward(symbolic_graph, bias));
    6088
    0
      ccv_nnc_tensor_t* const dscale_tensor = ccv_nnc_tensor_new(0, CPU_TENSOR_NHWC(32F, 1, GN_C_DIM, 1, 1), 0);
    6089
    0
      ccv_nnc_tensor_t* const dbias_tensor = ccv_nnc_tensor_new(0, CPU_TENSOR_NHWC(32F, 1, GN_C_DIM, 1, 1), 0);
    6090
    0
      ccv_nnc_cmd_exec(CMD_DATA_TRANSFER_FORWARD(), ccv_nnc_no_hint, 0, TENSOR_LIST(dbscale_tensor, dbbias_tensor), TENSOR_LIST(dscale_tensor, dbias_tensor), 0);
    6091
    0
      ccv_nnc_symbolic_graph_free(symbolic_graph);
    6092
    0
      ccv_nnc_tensor_arena_free(tensor_arena);
    6093
    0
      ccv_nnc_graph_exec_arena_free(graph_exec_arena);
    6094
    0
      ccv_nnc_graph_free(graph);
    6095
    0
      ccv_nnc_symbolic_graph_t* const cpu_symbolic_graph = ccv_nnc_symbolic_graph_new();
    6096
    0
      ccv_nnc_tensor_symbol_t cx = ccv_nnc_tensor_symbol_new(cpu_symbolic_graph, CPU_TENSOR_NHWC(32F, 2, GN_C_DIM, 2, LN_DIM), "x");
    6097
    0
      ccv_nnc_tensor_symbol_t cy = ccv_nnc_tensor_symbol_new(cpu_symbolic_graph, CPU_TENSOR_NHWC(32F, 2, GN_C_DIM, 2, LN_DIM), "y");
    6098
    0
      ccv_nnc_tensor_symbol_t cscale = ccv_nnc_tensor_symbol_new(cpu_symbolic_graph, CPU_TENSOR_NHWC(32F, 1, GN_C_DIM, 1, 1), "scale");
    6099
    0
      ccv_nnc_tensor_symbol_t cbias = ccv_nnc_tensor_symbol_new(cpu_symbolic_graph, CPU_TENSOR_NHWC(32F, 1, GN_C_DIM, 1, 1), "bias");
    6100
    0
      ccv_nnc_tensor_symbol_t csaved_mean = ccv_nnc_tensor_symbol_new(cpu_symbolic_graph, CPU_TENSOR_NHWC(32F, 2, GN_RC_DIM, 1, 1), "saved_mean");
    6101
    0
      ccv_nnc_tensor_symbol_t csaved_inv_std = ccv_nnc_tensor_symbol_new(cpu_symbolic_graph, CPU_TENSOR_NHWC(32F, 2, GN_RC_DIM, 1, 1), "saved_inv_std");
    6102
    0
      ccv_nnc_graph_exec_symbol_new(cpu_symbolic_graph, CMD_GROUP_NORM_FORWARD(1, GN_RC_DIM, 1e-5, 1), TENSOR_SYMBOL_LIST(cx, cscale, cbias), TENSOR_SYMBOL_LIST(cy, csaved_mean, csaved_inv_std), "group_norm");
    6103
    0
      ccv_nnc_graph_exec_symbol_autogen(cpu_symbolic_graph, 0, 0, CCV_NNC_AUTOGEN_ALL_EXECS | CCV_NNC_AUTOGEN_SOURCES_AND_DESTINATIONS);
    6104
    0
      ccv_nnc_symbolic_graph_backward(cpu_symbolic_graph, TENSOR_SYMBOL_LIST(cy), TENSOR_SYMBOL_LIST(cx, cscale, cbias), SYMBOLIC_GRAPH_SOURCES(cpu_symbolic_graph), SYMBOLIC_GRAPH_DESTINATIONS(cpu_symbolic_graph));
    6105
    0
      ccv_nnc_graph_exec_symbol_autogen(cpu_symbolic_graph, 0, 0, CCV_NNC_AUTOGEN_ALL_EXECS | CCV_NNC_AUTOGEN_SOURCES_AND_DESTINATIONS);
    6106
    0
      ccv_nnc_tensor_symbol_t dcy = ccv_nnc_tensor_symbol_for_backward(cpu_symbolic_graph, cy);
    6107
    0
      ccv_nnc_tensor_symbol_t dcx = ccv_nnc_tensor_symbol_for_backward(cpu_symbolic_graph, cx);
    6108
    0
      ccv_nnc_tensor_symbol_t dcscale = ccv_nnc_tensor_symbol_for_backward(cpu_symbolic_graph, cscale);
    6109
    0
      ccv_nnc_tensor_symbol_t dcbias = ccv_nnc_tensor_symbol_for_backward(cpu_symbolic_graph, cbias);
    6110
    0
      ccv_nnc_graph_t* cpu_graph = 0;
    6111
    0
      ccv_nnc_tensor_arena_t* cpu_tensor_arena = 0;
    6112
    0
      ccv_nnc_graph_exec_arena_t* cpu_graph_exec_arena = 0;
    6113
    0
      ccv_nnc_symbolic_graph_compile(cpu_symbolic_graph, ccv_nnc_default_compile_params, 0, 0, 0, 0, SYMBOLIC_GRAPH_SOURCES(cpu_symbolic_graph), SYMBOLIC_GRAPH_DESTINATIONS(cpu_symbolic_graph), &cpu_graph, &cpu_tensor_arena, &cpu_graph_exec_arena);
    6114
    0
      ccv_nnc_tensor_t* const cx_tensor = ccv_nnc_tensor_from_symbol(cpu_tensor_arena, cx);
    6115
    0
      memcpy(cx_tensor->data.f32, x_tensor->data.f32, sizeof(float) * 2 * GN_C_DIM * 2 * LN_DIM);
    6116
    0
      ccv_nnc_tensor_t* const dcy_tensor = ccv_nnc_tensor_from_symbol(cpu_tensor_arena, dcy);
    6117
    0
      memcpy(dcy_tensor->data.f32, dy_tensor->data.f32, sizeof(float) * 2 * GN_C_DIM * 2 * LN_DIM);
    6118
    0
      ccv_nnc_tensor_t* const cscale_tensor = ccv_nnc_tensor_from_symbol(cpu_tensor_arena, cscale);
    6119
    0
      memcpy(cscale_tensor->data.f32, scaledata, sizeof(float) * 1 * GN_C_DIM * 1 * 1);
    6120
    0
      ccv_nnc_tensor_t* const cbias_tensor = ccv_nnc_tensor_from_symbol(cpu_tensor_arena, cbias);
    6121
    0
      memcpy(cbias_tensor->data.f32, biasdata, sizeof(float) * 1 * GN_C_DIM * 1 * 1);
    6122
    6123
    0
      ccv_nnc_graph_run(cpu_graph, 0, TRAVERSE_FULL, 0, 0);
    6124
    6125
    0
      ccv_nnc_tensor_t* const dcx_tensor = ccv_nnc_tensor_from_symbol(cpu_tensor_arena, dcx);
    6126
    0
      ccv_nnc_tensor_t* const dcscale_tensor = ccv_nnc_tensor_from_symbol(cpu_tensor_arena, dcscale);
    6127
    0
      ccv_nnc_tensor_t* const dcbias_tensor = ccv_nnc_tensor_from_symbol(cpu_tensor_arena, dcbias);
    6128
    6129
    0
      REQUIRE_ARRAY_EQ_WITH_TOLERANCE(float, dx_tensor->data.f32, dcx_tensor->data.f32, 2 * GN_C_DIM * 2 * LN_DIM, 1e-5, "group norm output from mps should match from CPU");
    6130
    0
      REQUIRE_ARRAY_EQ_WITH_TOLERANCE(float, dbias_tensor->data.f32, dcbias_tensor->data.f32, 1 * GN_C_DIM * 1 * 1, 1e-5, "group norm output from mps should match from CPU");
    6131
    0
      REQUIRE_ARRAY_EQ_WITH_TOLERANCE(float, dscale_tensor->data.f32, dcscale_tensor->data.f32, 1 * GN_C_DIM * 1 * 1, 1e-5, "group norm output from mps should match from CPU");
    6132
    6133
    0
      ccv_nnc_symbolic_graph_free(cpu_symbolic_graph);
    6134
    0
      ccv_nnc_tensor_arena_free(cpu_tensor_arena);
    6135
    0
      ccv_nnc_graph_exec_arena_free(cpu_graph_exec_arena);
    6136
    0
      ccv_nnc_graph_free(cpu_graph);
    6137
    0
      ccv_nnc_tensor_free(x_tensor);
    6138
    0
      ccv_nnc_tensor_free(dy_tensor);
    6139
    0
      ccv_nnc_tensor_free(dx_tensor);
    6140
    0
      ccv_nnc_tensor_free(dscale_tensor);
    6141
    0
      ccv_nnc_tensor_free(dbias_tensor);
    6142
    0
    }
    6143
    6144
    TEST_CASE("compare group norm gradient with mps (no dbias)")
    6145
    1
    {
    6146
    1
      GUARD_ELSE_RETURN(ccv_nnc_cmd_ok(CCV_NNC_GROUP_NORM_FORWARD, CCV_NNC_BACKEND_MPS) &&
    6147
    1
        ccv_nnc_cmd_ok(CCV_NNC_GROUP_NORM_BACKWARD, CCV_NNC_BACKEND_MPS) &&
    6148
    1
        (ccv_nnc_cmd_ok(CCV_NNC_SET_FORWARD, CCV_NNC_BACKEND_MPS) || ccv_nnc_cmd_ok(CCV_NNC_SET_FORWARD, CCV_NNC_BACKEND_MPS)));
    6149
    0
      ccv_nnc_symbolic_graph_t* const symbolic_graph = ccv_nnc_symbolic_graph_new();
    6150
    0
      ccv_nnc_tensor_symbol_t bx = ccv_nnc_tensor_symbol_new(symbolic_graph, GPU_TENSOR_NHWC(000, 32F, 2, GN_C_DIM, 2, LN_DIM), "x");
    6151
    0
      ccv_nnc_tensor_symbol_t by = ccv_nnc_tensor_symbol_new(symbolic_graph, GPU_TENSOR_NHWC(000, 32F, 2, GN_C_DIM, 2, LN_DIM), "y");
    6152
    0
      ccv_nnc_tensor_symbol_t scale = ccv_nnc_tensor_symbol_new(symbolic_graph, GPU_TENSOR_NHWC(000, 32F, 1, GN_C_DIM, 2, LN_DIM), "scale");
    6153
    0
      ccv_nnc_tensor_symbol_t bias = ccv_nnc_tensor_symbol_new(symbolic_graph, GPU_TENSOR_NHWC(000, 32F, 1, GN_C_DIM, 2, LN_DIM), "bias");
    6154
    0
      ccv_nnc_tensor_symbol_t saved_mean = ccv_nnc_tensor_symbol_new(symbolic_graph, GPU_TENSOR_NHWC(000, 32F, 2, GN_RC_DIM, 2, LN_DIM), "saved_mean");
    6155
    0
      ccv_nnc_tensor_symbol_t saved_inv_std = ccv_nnc_tensor_symbol_new(symbolic_graph, GPU_TENSOR_NHWC(000, 32F, 2, GN_RC_DIM, 2, LN_DIM), "saved_inv_std");
    6156
    0
      ccv_nnc_graph_exec_symbol_new(symbolic_graph, CMD_GROUP_NORM_FORWARD(1, 4, 1e-5, 1), TENSOR_SYMBOL_LIST(bx, scale, bias), TENSOR_SYMBOL_LIST(by, saved_mean, saved_inv_std), "group_norm");
    6157
    0
      ccv_nnc_graph_exec_symbol_autogen(symbolic_graph, 0, 0, CCV_NNC_AUTOGEN_ALL_EXECS | CCV_NNC_AUTOGEN_SOURCES_AND_DESTINATIONS);
    6158
    0
      ccv_nnc_symbolic_graph_backward(symbolic_graph, TENSOR_SYMBOL_LIST(by), TENSOR_SYMBOL_LIST(bx, scale), SYMBOLIC_GRAPH_SOURCES(symbolic_graph), SYMBOLIC_GRAPH_DESTINATIONS(symbolic_graph));
    6159
    0
      ccv_nnc_graph_exec_symbol_autogen(symbolic_graph, 0, 0, CCV_NNC_AUTOGEN_ALL_EXECS | CCV_NNC_AUTOGEN_SOURCES_AND_DESTINATIONS);
    6160
    0
      ccv_nnc_tensor_symbol_t dby = ccv_nnc_tensor_symbol_for_backward(symbolic_graph, by);
    6161
    0
      ccv_nnc_tensor_symbol_t dbx = ccv_nnc_tensor_symbol_for_backward(symbolic_graph, bx);
    6162
    0
      ccv_nnc_graph_t* graph = 0;
    6163
    0
      ccv_nnc_tensor_arena_t* tensor_arena = 0;
    6164
    0
      ccv_nnc_graph_exec_arena_t* graph_exec_arena = 0;
    6165
    0
      ccv_nnc_symbolic_graph_compile(symbolic_graph, ccv_nnc_default_compile_params, 0, 0, 0, 0, SYMBOLIC_GRAPH_SOURCES(symbolic_graph), SYMBOLIC_GRAPH_DESTINATIONS(symbolic_graph), &graph, &tensor_arena, &graph_exec_arena);
    6166
    0
      SYMBOLIC_GRAPH_GEN(symbolic_graph, CCV_NNC_LONG_DOT_GRAPH);
    6167
    0
      GRAPH_GEN(graph, CCV_NNC_LONG_DOT_GRAPH);
    6168
    0
      ccv_nnc_tensor_t* const bx_tensor = ccv_nnc_tensor_from_symbol(tensor_arena, bx);
    6169
    0
      dsfmt_t dsfmt;
    6170
    0
      ccv_nnc_tensor_t* const x_tensor = ccv_nnc_tensor_new(0, CPU_TENSOR_NHWC(32F, 2, GN_C_DIM, 2, LN_DIM), 0);
    6171
    0
      int i;
    6172
    0
      dsfmt_init_gen_rand(&dsfmt, 1);
    6173
    0
      for (i = 0; i < 2 * GN_C_DIM * 2 * LN_DIM; i++)
    6174
    0
        x_tensor->data.f32[i] = dsfmt_genrand_open_close(&dsfmt) * 100;
    6175
    0
      ccv_nnc_cmd_exec(CMD_DATA_TRANSFER_FORWARD(), ccv_nnc_no_hint, 0, TENSOR_LIST(x_tensor), TENSOR_LIST(bx_tensor), 0);
    6176
    0
      float scaledata[1 * GN_C_DIM * 2 * LN_DIM];
    6177
    0
      float biasdata[1 * GN_C_DIM * 2 * LN_DIM];
    6178
    0
      for (i = 0; i < 1 * GN_C_DIM * 2 * LN_DIM; i++)
    6179
    0
      {
    6180
    0
        scaledata[i] = dsfmt_genrand_open_close(&dsfmt);
    6181
    0
        biasdata[i] = dsfmt_genrand_open_close(&dsfmt);
    6182
    0
      }
    6183
    0
      ccv_nnc_tensor_t scale_tensor = ccv_nnc_tensor(scaledata, CPU_TENSOR_NHWC(32F, 1, GN_C_DIM, 2, LN_DIM), 0);
    6184
    0
      ccv_nnc_tensor_t bias_tensor = ccv_nnc_tensor(biasdata, CPU_TENSOR_NHWC(32F, 1, GN_C_DIM, 2, LN_DIM), 0);
    6185
    0
      ccv_nnc_cmd_exec(CMD_DATA_TRANSFER_FORWARD(), ccv_nnc_no_hint, 0, TENSOR_LIST(&scale_tensor, &bias_tensor), TENSOR_LIST(ccv_nnc_tensor_from_symbol(tensor_arena, scale), ccv_nnc_tensor_from_symbol(tensor_arena, bias)), 0);
    6186
      // ccv_nnc_graph_run(graph, 0, TRAVERSE_FULL, 0, 0);
    6187
    0
      ccv_nnc_tensor_t* const dy_tensor = ccv_nnc_tensor_new(0, CPU_TENSOR_NHWC(32F, 2, GN_C_DIM, 2, LN_DIM), 0);
    6188
    0
      ccv_nnc_tensor_t* const dby_tensor = ccv_nnc_tensor_from_symbol(tensor_arena, dby);
    6189
    0
      for (i = 0; i < 2 * GN_C_DIM * 2 * LN_DIM; i++)
    6190
    0
        dy_tensor->data.f32[i] = dsfmt_genrand_open_close(&dsfmt) * 2 - 1;
    6191
    0
      ccv_nnc_cmd_exec(CMD_DATA_TRANSFER_FORWARD(), ccv_nnc_no_hint, 0, TENSOR_LIST(dy_tensor), TENSOR_LIST(dby_tensor), 0);
    6192
    0
      ccv_nnc_graph_run(graph, 0, TRAVERSE_FULL, 0, 0);
    6193
    0
      ccv_nnc_tensor_t* const dbx_tensor = ccv_nnc_tensor_from_symbol(tensor_arena, dbx);
    6194
    0
      ccv_nnc_tensor_t* const dx_tensor = ccv_nnc_tensor_new(0, CPU_TENSOR_NHWC(32F, 2, GN_C_DIM, 2, LN_DIM), 0);
    6195
    0
      ccv_nnc_cmd_exec(CMD_DATA_TRANSFER_FORWARD(), ccv_nnc_no_hint, 0, TENSOR_LIST(dbx_tensor), TENSOR_LIST(dx_tensor), 0);
    6196
    0
      ccv_nnc_tensor_t* const dbscale_tensor = ccv_nnc_tensor_from_symbol(tensor_arena, ccv_nnc_tensor_symbol_for_backward(symbolic_graph, scale));
    6197
    0
      ccv_nnc_tensor_t* const dscale_tensor = ccv_nnc_tensor_new(0, CPU_TENSOR_NHWC(32F, 1, GN_C_DIM, 2, LN_DIM), 0);
    6198
    0
      ccv_nnc_cmd_exec(CMD_DATA_TRANSFER_FORWARD(), ccv_nnc_no_hint, 0, TENSOR_LIST(dbscale_tensor), TENSOR_LIST(dscale_tensor), 0);
    6199
    0
      ccv_nnc_symbolic_graph_free(symbolic_graph);
    6200
    0
      ccv_nnc_tensor_arena_free(tensor_arena);
    6201
    0
      ccv_nnc_graph_exec_arena_free(graph_exec_arena);
    6202
    0
      ccv_nnc_graph_free(graph);
    6203
    0
      ccv_nnc_symbolic_graph_t* const cpu_symbolic_graph = ccv_nnc_symbolic_graph_new();
    6204
    0
      ccv_nnc_tensor_symbol_t cx = ccv_nnc_tensor_symbol_new(cpu_symbolic_graph, CPU_TENSOR_NHWC(32F, 2, GN_C_DIM, 2, LN_DIM), "x");
    6205
    0
      ccv_nnc_tensor_symbol_t cy = ccv_nnc_tensor_symbol_new(cpu_symbolic_graph, CPU_TENSOR_NHWC(32F, 2, GN_C_DIM, 2, LN_DIM), "y");
    6206
    0
      ccv_nnc_tensor_symbol_t cscale = ccv_nnc_tensor_symbol_new(cpu_symbolic_graph, CPU_TENSOR_NHWC(32F, 1, GN_C_DIM, 2, LN_DIM), "scale");
    6207
    0
      ccv_nnc_tensor_symbol_t cbias = ccv_nnc_tensor_symbol_new(cpu_symbolic_graph, CPU_TENSOR_NHWC(32F, 1, GN_C_DIM, 2, LN_DIM), "bias");
    6208
    0
      ccv_nnc_tensor_symbol_t csaved_mean = ccv_nnc_tensor_symbol_new(cpu_symbolic_graph, CPU_TENSOR_NHWC(32F, 2, GN_RC_DIM, 2, LN_DIM), "saved_mean");
    6209
    0
      ccv_nnc_tensor_symbol_t csaved_inv_std = ccv_nnc_tensor_symbol_new(cpu_symbolic_graph, CPU_TENSOR_NHWC(32F, 2, GN_RC_DIM, 2, LN_DIM), "saved_inv_std");
    6210
    0
      ccv_nnc_graph_exec_symbol_new(cpu_symbolic_graph, CMD_GROUP_NORM_FORWARD(1, GN_RC_DIM, 1e-5, 1), TENSOR_SYMBOL_LIST(cx, cscale, cbias), TENSOR_SYMBOL_LIST(cy, csaved_mean, csaved_inv_std), "group_norm");
    6211
    0
      ccv_nnc_graph_exec_symbol_autogen(cpu_symbolic_graph, 0, 0, CCV_NNC_AUTOGEN_ALL_EXECS | CCV_NNC_AUTOGEN_SOURCES_AND_DESTINATIONS);
    6212
    0
      ccv_nnc_symbolic_graph_backward(cpu_symbolic_graph, TENSOR_SYMBOL_LIST(cy), TENSOR_SYMBOL_LIST(cx, cscale), SYMBOLIC_GRAPH_SOURCES(cpu_symbolic_graph), SYMBOLIC_GRAPH_DESTINATIONS(cpu_symbolic_graph));
    6213
    0
      ccv_nnc_graph_exec_symbol_autogen(cpu_symbolic_graph, 0, 0, CCV_NNC_AUTOGEN_ALL_EXECS | CCV_NNC_AUTOGEN_SOURCES_AND_DESTINATIONS);
    6214
    0
      ccv_nnc_tensor_symbol_t dcy = ccv_nnc_tensor_symbol_for_backward(cpu_symbolic_graph, cy);
    6215
    0
      ccv_nnc_tensor_symbol_t dcx = ccv_nnc_tensor_symbol_for_backward(cpu_symbolic_graph, cx);
    6216
    0
      ccv_nnc_tensor_symbol_t dcscale = ccv_nnc_tensor_symbol_for_backward(cpu_symbolic_graph, cscale);
    6217
    0
      ccv_nnc_graph_t* cpu_graph = 0;
    6218
    0
      ccv_nnc_tensor_arena_t* cpu_tensor_arena = 0;
    6219
    0
      ccv_nnc_graph_exec_arena_t* cpu_graph_exec_arena = 0;
    6220
    0
      ccv_nnc_symbolic_graph_compile(cpu_symbolic_graph, ccv_nnc_default_compile_params, 0, 0, 0, 0, SYMBOLIC_GRAPH_SOURCES(cpu_symbolic_graph), SYMBOLIC_GRAPH_DESTINATIONS(cpu_symbolic_graph), &cpu_graph, &cpu_tensor_arena, &cpu_graph_exec_arena);
    6221
    0
      ccv_nnc_tensor_t* const cx_tensor = ccv_nnc_tensor_from_symbol(cpu_tensor_arena, cx);
    6222
    0
      memcpy(cx_tensor->data.f32, x_tensor->data.f32, sizeof(float) * 2 * GN_C_DIM * 2 * LN_DIM);
    6223
    0
      ccv_nnc_tensor_t* const dcy_tensor = ccv_nnc_tensor_from_symbol(cpu_tensor_arena, dcy);
    6224
    0
      memcpy(dcy_tensor->data.f32, dy_tensor->data.f32, sizeof(float) * 2 * GN_C_DIM * 2 * LN_DIM);
    6225
    0
      ccv_nnc_tensor_t* const cscale_tensor = ccv_nnc_tensor_from_symbol(cpu_tensor_arena, cscale);
    6226
    0
      memcpy(cscale_tensor->data.f32, scaledata, sizeof(float) * 1 * GN_C_DIM * 2 * LN_DIM);
    6227
    6228
    0
      ccv_nnc_graph_run(cpu_graph, 0, TRAVERSE_FULL, 0, 0);
    6229
    6230
    0
      ccv_nnc_tensor_t* const dcx_tensor = ccv_nnc_tensor_from_symbol(cpu_tensor_arena, dcx);
    6231
    0
      ccv_nnc_tensor_t* const dcscale_tensor = ccv_nnc_tensor_from_symbol(cpu_tensor_arena, dcscale);
    6232
    6233
    0
      REQUIRE_ARRAY_EQ_WITH_TOLERANCE(float, dx_tensor->data.f32, dcx_tensor->data.f32, 2 * GN_C_DIM * 2 * LN_DIM, 1e-5, "group norm output from mps should match from CPU");
    6234
    0
      REQUIRE_ARRAY_EQ_WITH_TOLERANCE(float, dscale_tensor->data.f32, dcscale_tensor->data.f32, 1 * GN_C_DIM * 2 * LN_DIM, 1e-5, "group norm output from mps should match from CPU");
    6235
    6236
    0
      ccv_nnc_symbolic_graph_free(cpu_symbolic_graph);
    6237
    0
      ccv_nnc_tensor_arena_free(cpu_tensor_arena);
    6238
    0
      ccv_nnc_graph_exec_arena_free(cpu_graph_exec_arena);
    6239
    0
      ccv_nnc_graph_free(cpu_graph);
    6240
    0
      ccv_nnc_tensor_free(x_tensor);
    6241
    0
      ccv_nnc_tensor_free(dy_tensor);
    6242
    0
      ccv_nnc_tensor_free(dx_tensor);
    6243
    0
      ccv_nnc_tensor_free(dscale_tensor);
    6244
    0
    }
    6245
    6246
    TEST_CASE("compare group norm gradient with mps without scale / bias")
    6247
    1
    {
    6248
    1
      GUARD_ELSE_RETURN(ccv_nnc_cmd_ok(CCV_NNC_GROUP_NORM_FORWARD, CCV_NNC_BACKEND_MPS) &&
    6249
    1
        ccv_nnc_cmd_ok(CCV_NNC_GROUP_NORM_BACKWARD, CCV_NNC_BACKEND_MPS) &&
    6250
    1
        (ccv_nnc_cmd_ok(CCV_NNC_SET_FORWARD, CCV_NNC_BACKEND_MPS) || ccv_nnc_cmd_ok(CCV_NNC_SET_FORWARD, CCV_NNC_BACKEND_MPS)));
    6251
    0
      ccv_nnc_symbolic_graph_t* const symbolic_graph = ccv_nnc_symbolic_graph_new();
    6252
    0
      ccv_nnc_tensor_symbol_t bx = ccv_nnc_tensor_symbol_new(symbolic_graph, GPU_TENSOR_NHWC(000, 32F, 2, GN_C_DIM, 2, LN_DIM), "x");
    6253
    0
      ccv_nnc_tensor_symbol_t by = ccv_nnc_tensor_symbol_new(symbolic_graph, GPU_TENSOR_NHWC(000, 32F, 2, GN_C_DIM, 2, LN_DIM), "y");
    6254
    0
      ccv_nnc_tensor_symbol_t saved_mean = ccv_nnc_tensor_symbol_new(symbolic_graph, GPU_TENSOR_NHWC(000, 32F, 2, GN_RC_DIM, 2, LN_DIM), "saved_mean");
    6255
    0
      ccv_nnc_tensor_symbol_t saved_inv_std = ccv_nnc_tensor_symbol_new(symbolic_graph, GPU_TENSOR_NHWC(000, 32F, 2, GN_RC_DIM, 2, LN_DIM), "saved_inv_std");
    6256
    0
      ccv_nnc_graph_exec_symbol_new(symbolic_graph, CMD_GROUP_NORM_FORWARD(1, 4, 1e-5, 0), TENSOR_SYMBOL_LIST(bx), TENSOR_SYMBOL_LIST(by, saved_mean, saved_inv_std), "group_norm");
    6257
    0
      ccv_nnc_graph_exec_symbol_autogen(symbolic_graph, 0, 0, CCV_NNC_AUTOGEN_ALL_EXECS | CCV_NNC_AUTOGEN_SOURCES_AND_DESTINATIONS);
    6258
    0
      ccv_nnc_symbolic_graph_backward(symbolic_graph, TENSOR_SYMBOL_LIST(by), TENSOR_SYMBOL_LIST(bx), SYMBOLIC_GRAPH_SOURCES(symbolic_graph), SYMBOLIC_GRAPH_DESTINATIONS(symbolic_graph));
    6259
    0
      ccv_nnc_graph_exec_symbol_autogen(symbolic_graph, 0, 0, CCV_NNC_AUTOGEN_ALL_EXECS | CCV_NNC_AUTOGEN_SOURCES_AND_DESTINATIONS);
    6260
    0
      ccv_nnc_tensor_symbol_t dby = ccv_nnc_tensor_symbol_for_backward(symbolic_graph, by);
    6261
    0
      ccv_nnc_tensor_symbol_t dbx = ccv_nnc_tensor_symbol_for_backward(symbolic_graph, bx);
    6262
    0
      ccv_nnc_graph_t* graph = 0;
    6263
    0
      ccv_nnc_tensor_arena_t* tensor_arena = 0;
    6264
    0
      ccv_nnc_graph_exec_arena_t* graph_exec_arena = 0;
    6265
    0
      ccv_nnc_symbolic_graph_compile(symbolic_graph, ccv_nnc_default_compile_params, 0, 0, 0, 0, SYMBOLIC_GRAPH_SOURCES(symbolic_graph), SYMBOLIC_GRAPH_DESTINATIONS(symbolic_graph), &graph, &tensor_arena, &graph_exec_arena);
    6266
    0
      SYMBOLIC_GRAPH_GEN(symbolic_graph, CCV_NNC_LONG_DOT_GRAPH);
    6267
    0
      GRAPH_GEN(graph, CCV_NNC_LONG_DOT_GRAPH);
    6268
    0
      ccv_nnc_tensor_t* const bx_tensor = ccv_nnc_tensor_from_symbol(tensor_arena, bx);
    6269
    0
      dsfmt_t dsfmt;
    6270
    0
      ccv_nnc_tensor_t* const x_tensor = ccv_nnc_tensor_new(0, CPU_TENSOR_NHWC(32F, 2, GN_C_DIM, 2, LN_DIM), 0);
    6271
    0
      int i;
    6272
    0
      dsfmt_init_gen_rand(&dsfmt, 1);
    6273
    0
      for (i = 0; i < 2 * GN_C_DIM * 2 * LN_DIM; i++)
    6274
    0
        x_tensor->data.f32[i] = dsfmt_genrand_open_close(&dsfmt) * 100;
    6275
    0
      ccv_nnc_cmd_exec(CMD_DATA_TRANSFER_FORWARD(), ccv_nnc_no_hint, 0, TENSOR_LIST(x_tensor), TENSOR_LIST(bx_tensor), 0);
    6276
      // ccv_nnc_graph_run(graph, 0, TRAVERSE_FULL, 0, 0);
    6277
    0
      ccv_nnc_tensor_t* const dy_tensor = ccv_nnc_tensor_new(0, CPU_TENSOR_NHWC(32F, 2, GN_C_DIM, 2, LN_DIM), 0);
    6278
    0
      ccv_nnc_tensor_t* const dby_tensor = ccv_nnc_tensor_from_symbol(tensor_arena, dby);
    6279
    0
      for (i = 0; i < 2 * GN_C_DIM * 2 * LN_DIM; i++)
    6280
    0
        dy_tensor->data.f32[i] = dsfmt_genrand_open_close(&dsfmt) * 2 - 1;
    6281
    0
      ccv_nnc_cmd_exec(CMD_DATA_TRANSFER_FORWARD(), ccv_nnc_no_hint, 0, TENSOR_LIST(dy_tensor), TENSOR_LIST(dby_tensor), 0);
    6282
    0
      ccv_nnc_graph_run(graph, 0, TRAVERSE_FULL, 0, 0);
    6283
    0
      ccv_nnc_tensor_t* const dbx_tensor = ccv_nnc_tensor_from_symbol(tensor_arena, dbx);
    6284
    0
      ccv_nnc_tensor_t* const dx_tensor = ccv_nnc_tensor_new(0, CPU_TENSOR_NHWC(32F, 2, GN_C_DIM, 2, LN_DIM), 0);
    6285
    0
      ccv_nnc_cmd_exec(CMD_DATA_TRANSFER_FORWARD(), ccv_nnc_no_hint, 0, TENSOR_LIST(dbx_tensor), TENSOR_LIST(dx_tensor), 0);
    6286
    0
      ccv_nnc_symbolic_graph_free(symbolic_graph);
    6287
    0
      ccv_nnc_tensor_arena_free(tensor_arena);
    6288
    0
      ccv_nnc_graph_exec_arena_free(graph_exec_arena);
    6289
    0
      ccv_nnc_graph_free(graph);
    6290
    0
      ccv_nnc_symbolic_graph_t* const cpu_symbolic_graph = ccv_nnc_symbolic_graph_new();
    6291
    0
      ccv_nnc_tensor_symbol_t cx = ccv_nnc_tensor_symbol_new(cpu_symbolic_graph, CPU_TENSOR_NHWC(32F, 2, GN_C_DIM, 2, LN_DIM), "x");
    6292
    0
      ccv_nnc_tensor_symbol_t cy = ccv_nnc_tensor_symbol_new(cpu_symbolic_graph, CPU_TENSOR_NHWC(32F, 2, GN_C_DIM, 2, LN_DIM), "y");
    6293
    0
      ccv_nnc_tensor_symbol_t csaved_mean = ccv_nnc_tensor_symbol_new(cpu_symbolic_graph, CPU_TENSOR_NHWC(32F, 2, GN_RC_DIM, 2, LN_DIM), "saved_mean");
    6294
    0
      ccv_nnc_tensor_symbol_t csaved_inv_std = ccv_nnc_tensor_symbol_new(cpu_symbolic_graph, CPU_TENSOR_NHWC(32F, 2, GN_RC_DIM, 2, LN_DIM), "saved_inv_std");
    6295
    0
      ccv_nnc_graph_exec_symbol_new(cpu_symbolic_graph, CMD_GROUP_NORM_FORWARD(1, GN_RC_DIM, 1e-5, 0), TENSOR_SYMBOL_LIST(cx), TENSOR_SYMBOL_LIST(cy, csaved_mean, csaved_inv_std), "group_norm");
    6296
    0
      ccv_nnc_graph_exec_symbol_autogen(cpu_symbolic_graph, 0, 0, CCV_NNC_AUTOGEN_ALL_EXECS | CCV_NNC_AUTOGEN_SOURCES_AND_DESTINATIONS);
    6297
    0
      ccv_nnc_symbolic_graph_backward(cpu_symbolic_graph, TENSOR_SYMBOL_LIST(cy), TENSOR_SYMBOL_LIST(cx), SYMBOLIC_GRAPH_SOURCES(cpu_symbolic_graph), SYMBOLIC_GRAPH_DESTINATIONS(cpu_symbolic_graph));
    6298
    0
      ccv_nnc_graph_exec_symbol_autogen(cpu_symbolic_graph, 0, 0, CCV_NNC_AUTOGEN_ALL_EXECS | CCV_NNC_AUTOGEN_SOURCES_AND_DESTINATIONS);
    6299
    0
      ccv_nnc_tensor_symbol_t dcy = ccv_nnc_tensor_symbol_for_backward(cpu_symbolic_graph, cy);
    6300
    0
      ccv_nnc_tensor_symbol_t dcx = ccv_nnc_tensor_symbol_for_backward(cpu_symbolic_graph, cx);
    6301
    0
      ccv_nnc_graph_t* cpu_graph = 0;
    6302
    0
      ccv_nnc_tensor_arena_t* cpu_tensor_arena = 0;
    6303
    0
      ccv_nnc_graph_exec_arena_t* cpu_graph_exec_arena = 0;
    6304
    0
      ccv_nnc_symbolic_graph_compile(cpu_symbolic_graph, ccv_nnc_default_compile_params, 0, 0, 0, 0, SYMBOLIC_GRAPH_SOURCES(cpu_symbolic_graph), SYMBOLIC_GRAPH_DESTINATIONS(cpu_symbolic_graph), &cpu_graph, &cpu_tensor_arena, &cpu_graph_exec_arena);
    6305
    0
      ccv_nnc_tensor_t* const cx_tensor = ccv_nnc_tensor_from_symbol(cpu_tensor_arena, cx);
    6306
    0
      memcpy(cx_tensor->data.f32, x_tensor->data.f32, sizeof(float) * 2 * GN_C_DIM * 2 * LN_DIM);
    6307
    0
      ccv_nnc_tensor_t* const dcy_tensor = ccv_nnc_tensor_from_symbol(cpu_tensor_arena, dcy);
    6308
    0
      memcpy(dcy_tensor->data.f32, dy_tensor->data.f32, sizeof(float) * 2 * GN_C_DIM * 2 * LN_DIM);
    6309
    6310
    0
      ccv_nnc_graph_run(cpu_graph, 0, TRAVERSE_FULL, 0, 0);
    6311
    6312
    0
      ccv_nnc_tensor_t* const dcx_tensor = ccv_nnc_tensor_from_symbol(cpu_tensor_arena, dcx);
    6313
    6314
    0
      REQUIRE_ARRAY_EQ_WITH_TOLERANCE(float, dx_tensor->data.f32, dcx_tensor->data.f32, 2 * GN_C_DIM * 2 * LN_DIM, 1e-5, "group norm output from mps should match from CPU");
    6315
    6316
    0
      ccv_nnc_symbolic_graph_free(cpu_symbolic_graph);
    6317
    0
      ccv_nnc_tensor_arena_free(cpu_tensor_arena);
    6318
    0
      ccv_nnc_graph_exec_arena_free(cpu_graph_exec_arena);
    6319
    0
      ccv_nnc_graph_free(cpu_graph);
    6320
    0
      ccv_nnc_tensor_free(x_tensor);
    6321
    0
      ccv_nnc_tensor_free(dy_tensor);
    6322
    0
      ccv_nnc_tensor_free(dx_tensor);
    6323
    0
    }
    6324
    6325
    TEST_CASE("compare group norm gradient with mps, variant 1 without scale / bias")
    6326
    1
    {
    6327
    1
      GUARD_ELSE_RETURN(ccv_nnc_cmd_ok(CCV_NNC_GROUP_NORM_FORWARD, CCV_NNC_BACKEND_MPS) &&
    6328
    1
        ccv_nnc_cmd_ok(CCV_NNC_GROUP_NORM_BACKWARD, CCV_NNC_BACKEND_MPS) &&
    6329
    1
        (ccv_nnc_cmd_ok(CCV_NNC_SET_FORWARD, CCV_NNC_BACKEND_MPS) || ccv_nnc_cmd_ok(CCV_NNC_SET_FORWARD, CCV_NNC_BACKEND_MPS)));
    6330
    0
      ccv_nnc_symbolic_graph_t* const symbolic_graph = ccv_nnc_symbolic_graph_new();
    6331
    0
      ccv_nnc_tensor_symbol_t bx = ccv_nnc_tensor_symbol_new(symbolic_graph, GPU_TENSOR_NHWC(000, 32F, 2, GN_C_DIM, 2, LN_DIM), "x");
    6332
    0
      ccv_nnc_tensor_symbol_t by = ccv_nnc_tensor_symbol_new(symbolic_graph, GPU_TENSOR_NHWC(000, 32F, 2, GN_C_DIM, 2, LN_DIM), "y");
    6333
    0
      ccv_nnc_tensor_symbol_t saved_mean = ccv_nnc_tensor_symbol_new(symbolic_graph, GPU_TENSOR_NHWC(000, 32F, 2, GN_RC_DIM, 1, 1), "saved_mean");
    6334
    0
      ccv_nnc_tensor_symbol_t saved_inv_std = ccv_nnc_tensor_symbol_new(symbolic_graph, GPU_TENSOR_NHWC(000, 32F, 2, GN_RC_DIM, 1, 1), "saved_inv_std");
    6335
    0
      ccv_nnc_graph_exec_symbol_new(symbolic_graph, CMD_GROUP_NORM_FORWARD(1, 4, 1e-5, 0), TENSOR_SYMBOL_LIST(bx), TENSOR_SYMBOL_LIST(by, saved_mean, saved_inv_std), "group_norm");
    6336
    0
      ccv_nnc_graph_exec_symbol_autogen(symbolic_graph, 0, 0, CCV_NNC_AUTOGEN_ALL_EXECS | CCV_NNC_AUTOGEN_SOURCES_AND_DESTINATIONS);
    6337
    0
      ccv_nnc_symbolic_graph_backward(symbolic_graph, TENSOR_SYMBOL_LIST(by), TENSOR_SYMBOL_LIST(bx), SYMBOLIC_GRAPH_SOURCES(symbolic_graph), SYMBOLIC_GRAPH_DESTINATIONS(symbolic_graph));
    6338
    0
      ccv_nnc_graph_exec_symbol_autogen(symbolic_graph, 0, 0, CCV_NNC_AUTOGEN_ALL_EXECS | CCV_NNC_AUTOGEN_SOURCES_AND_DESTINATIONS);
    6339
    0
      ccv_nnc_tensor_symbol_t dby = ccv_nnc_tensor_symbol_for_backward(symbolic_graph, by);
    6340
    0
      ccv_nnc_tensor_symbol_t dbx = ccv_nnc_tensor_symbol_for_backward(symbolic_graph, bx);
    6341
    0
      ccv_nnc_graph_t* graph = 0;
    6342
    0
      ccv_nnc_tensor_arena_t* tensor_arena = 0;
    6343
    0
      ccv_nnc_graph_exec_arena_t* graph_exec_arena = 0;
    6344
    0
      ccv_nnc_symbolic_graph_compile(symbolic_graph, ccv_nnc_default_compile_params, 0, 0, 0, 0, SYMBOLIC_GRAPH_SOURCES(symbolic_graph), SYMBOLIC_GRAPH_DESTINATIONS(symbolic_graph), &graph, &tensor_arena, &graph_exec_arena);
    6345
    0
      SYMBOLIC_GRAPH_GEN(symbolic_graph, CCV_NNC_LONG_DOT_GRAPH);
    6346
    0
      GRAPH_GEN(graph, CCV_NNC_LONG_DOT_GRAPH);
    6347
    0
      ccv_nnc_tensor_t* const bx_tensor = ccv_nnc_tensor_from_symbol(tensor_arena, bx);
    6348
    0
      dsfmt_t dsfmt;
    6349
    0
      ccv_nnc_tensor_t* const x_tensor = ccv_nnc_tensor_new(0, CPU_TENSOR_NHWC(32F, 2, GN_C_DIM, 2, LN_DIM), 0);
    6350
    0
      int i;
    6351
    0
      dsfmt_init_gen_rand(&dsfmt, 1);
    6352
    0
      for (i = 0; i < 2 * GN_C_DIM * 2 * LN_DIM; i++)
    6353
    0
        x_tensor->data.f32[i] = dsfmt_genrand_open_close(&dsfmt) * 100;
    6354
    0
      ccv_nnc_cmd_exec(CMD_DATA_TRANSFER_FORWARD(), ccv_nnc_no_hint, 0, TENSOR_LIST(x_tensor), TENSOR_LIST(bx_tensor), 0);
    6355
      // ccv_nnc_graph_run(graph, 0, TRAVERSE_FULL, 0, 0);
    6356
    0
      ccv_nnc_tensor_t* const dy_tensor = ccv_nnc_tensor_new(0, CPU_TENSOR_NHWC(32F, 2, GN_C_DIM, 2, LN_DIM), 0);
    6357
    0
      ccv_nnc_tensor_t* const dby_tensor = ccv_nnc_tensor_from_symbol(tensor_arena, dby);
    6358
    0
      for (i = 0; i < 2 * GN_C_DIM * 2 * LN_DIM; i++)
    6359
    0
        dy_tensor->data.f32[i] = dsfmt_genrand_open_close(&dsfmt) * 2 - 1;
    6360
    0
      ccv_nnc_cmd_exec(CMD_DATA_TRANSFER_FORWARD(), ccv_nnc_no_hint, 0, TENSOR_LIST(dy_tensor), TENSOR_LIST(dby_tensor), 0);
    6361
    0
      ccv_nnc_graph_run(graph, 0, TRAVERSE_FULL, 0, 0);
    6362
    0
      ccv_nnc_tensor_t* const dbx_tensor = ccv_nnc_tensor_from_symbol(tensor_arena, dbx);
    6363
    0
      ccv_nnc_tensor_t* const dx_tensor = ccv_nnc_tensor_new(0, CPU_TENSOR_NHWC(32F, 2, GN_C_DIM, 2, LN_DIM), 0);
    6364
    0
      ccv_nnc_cmd_exec(CMD_DATA_TRANSFER_FORWARD(), ccv_nnc_no_hint, 0, TENSOR_LIST(dbx_tensor), TENSOR_LIST(dx_tensor), 0);
    6365
    0
      ccv_nnc_symbolic_graph_free(symbolic_graph);
    6366
    0
      ccv_nnc_tensor_arena_free(tensor_arena);
    6367
    0
      ccv_nnc_graph_exec_arena_free(graph_exec_arena);
    6368
    0
      ccv_nnc_graph_free(graph);
    6369
    0
      ccv_nnc_symbolic_graph_t* const cpu_symbolic_graph = ccv_nnc_symbolic_graph_new();
    6370
    0
      ccv_nnc_tensor_symbol_t cx = ccv_nnc_tensor_symbol_new(cpu_symbolic_graph, CPU_TENSOR_NHWC(32F, 2, GN_C_DIM, 2, LN_DIM), "x");
    6371
    0
      ccv_nnc_tensor_symbol_t cy = ccv_nnc_tensor_symbol_new(cpu_symbolic_graph, CPU_TENSOR_NHWC(32F, 2, GN_C_DIM, 2, LN_DIM), "y");
    6372
    0
      ccv_nnc_tensor_symbol_t csaved_mean = ccv_nnc_tensor_symbol_new(cpu_symbolic_graph, CPU_TENSOR_NHWC(32F, 2, GN_RC_DIM, 1, 1), "saved_mean");
    6373
    0
      ccv_nnc_tensor_symbol_t csaved_inv_std = ccv_nnc_tensor_symbol_new(cpu_symbolic_graph, CPU_TENSOR_NHWC(32F, 2, GN_RC_DIM, 1, 1), "saved_inv_std");
    6374
    0
      ccv_nnc_graph_exec_symbol_new(cpu_symbolic_graph, CMD_GROUP_NORM_FORWARD(1, GN_RC_DIM, 1e-5, 0), TENSOR_SYMBOL_LIST(cx), TENSOR_SYMBOL_LIST(cy, csaved_mean, csaved_inv_std), "group_norm");
    6375
    0
      ccv_nnc_graph_exec_symbol_autogen(cpu_symbolic_graph, 0, 0, CCV_NNC_AUTOGEN_ALL_EXECS | CCV_NNC_AUTOGEN_SOURCES_AND_DESTINATIONS);
    6376
    0
      ccv_nnc_symbolic_graph_backward(cpu_symbolic_graph, TENSOR_SYMBOL_LIST(cy), TENSOR_SYMBOL_LIST(cx), SYMBOLIC_GRAPH_SOURCES(cpu_symbolic_graph), SYMBOLIC_GRAPH_DESTINATIONS(cpu_symbolic_graph));
    6377
    0
      ccv_nnc_graph_exec_symbol_autogen(cpu_symbolic_graph, 0, 0, CCV_NNC_AUTOGEN_ALL_EXECS | CCV_NNC_AUTOGEN_SOURCES_AND_DESTINATIONS);
    6378
    0
      ccv_nnc_tensor_symbol_t dcy = ccv_nnc_tensor_symbol_for_backward(cpu_symbolic_graph, cy);
    6379
    0
      ccv_nnc_tensor_symbol_t dcx = ccv_nnc_tensor_symbol_for_backward(cpu_symbolic_graph, cx);
    6380
    0
      ccv_nnc_graph_t* cpu_graph = 0;
    6381
    0
      ccv_nnc_tensor_arena_t* cpu_tensor_arena = 0;
    6382
    0
      ccv_nnc_graph_exec_arena_t* cpu_graph_exec_arena = 0;
    6383
    0
      ccv_nnc_symbolic_graph_compile(cpu_symbolic_graph, ccv_nnc_default_compile_params, 0, 0, 0, 0, SYMBOLIC_GRAPH_SOURCES(cpu_symbolic_graph), SYMBOLIC_GRAPH_DESTINATIONS(cpu_symbolic_graph), &cpu_graph, &cpu_tensor_arena, &cpu_graph_exec_arena);
    6384
    0
      ccv_nnc_tensor_t* const cx_tensor = ccv_nnc_tensor_from_symbol(cpu_tensor_arena, cx);
    6385
    0
      memcpy(cx_tensor->data.f32, x_tensor->data.f32, sizeof(float) * 2 * GN_C_DIM * 2 * LN_DIM);
    6386
    0
      ccv_nnc_tensor_t* const dcy_tensor = ccv_nnc_tensor_from_symbol(cpu_tensor_arena, dcy);
    6387
    0
      memcpy(dcy_tensor->data.f32, dy_tensor->data.f32, sizeof(float) * 2 * GN_C_DIM * 2 * LN_DIM);
    6388
    6389
    0
      ccv_nnc_graph_run(cpu_graph, 0, TRAVERSE_FULL, 0, 0);
    6390
    6391
    0
      ccv_nnc_tensor_t* const dcx_tensor = ccv_nnc_tensor_from_symbol(cpu_tensor_arena, dcx);
    6392
    6393
    0
      REQUIRE_ARRAY_EQ_WITH_TOLERANCE(float, dx_tensor->data.f32, dcx_tensor->data.f32, 2 * GN_C_DIM * 2 * LN_DIM, 1e-5, "group norm output from mps should match from CPU");
    6394
    6395
    0
      ccv_nnc_symbolic_graph_free(cpu_symbolic_graph);
    6396
    0
      ccv_nnc_tensor_arena_free(cpu_tensor_arena);
    6397
    0
      ccv_nnc_graph_exec_arena_free(cpu_graph_exec_arena);
    6398
    0
      ccv_nnc_graph_free(cpu_graph);
    6399
    0
      ccv_nnc_tensor_free(x_tensor);
    6400
    0
      ccv_nnc_tensor_free(dy_tensor);
    6401
    0
      ccv_nnc_tensor_free(dx_tensor);
    6402
    0
    }
    6403
    6404
    TEST_CASE("compare group norm gradient with mps (no dbias) without scale / bias")
    6405
    1
    {
    6406
    1
      GUARD_ELSE_RETURN(ccv_nnc_cmd_ok(CCV_NNC_GROUP_NORM_FORWARD, CCV_NNC_BACKEND_MPS) &&
    6407
    1
        ccv_nnc_cmd_ok(CCV_NNC_GROUP_NORM_BACKWARD, CCV_NNC_BACKEND_MPS) &&
    6408
    1
        (ccv_nnc_cmd_ok(CCV_NNC_SET_FORWARD, CCV_NNC_BACKEND_MPS) || ccv_nnc_cmd_ok(CCV_NNC_SET_FORWARD, CCV_NNC_BACKEND_MPS)));
    6409
    0
      ccv_nnc_symbolic_graph_t* const symbolic_graph = ccv_nnc_symbolic_graph_new();
    6410
    0
      ccv_nnc_tensor_symbol_t bx = ccv_nnc_tensor_symbol_new(symbolic_graph, GPU_TENSOR_NHWC(000, 32F, 2, GN_C_DIM, 2, LN_DIM), "x");
    6411
    0
      ccv_nnc_tensor_symbol_t by = ccv_nnc_tensor_symbol_new(symbolic_graph, GPU_TENSOR_NHWC(000, 32F, 2, GN_C_DIM, 2, LN_DIM), "y");
    6412
    0
      ccv_nnc_tensor_symbol_t saved_mean = ccv_nnc_tensor_symbol_new(symbolic_graph, GPU_TENSOR_NHWC(000, 32F, 2, GN_RC_DIM, 2, LN_DIM), "saved_mean");
    6413
    0
      ccv_nnc_tensor_symbol_t saved_inv_std = ccv_nnc_tensor_symbol_new(symbolic_graph, GPU_TENSOR_NHWC(000, 32F, 2, GN_RC_DIM, 2, LN_DIM), "saved_inv_std");
    6414
    0
      ccv_nnc_graph_exec_symbol_new(symbolic_graph, CMD_GROUP_NORM_FORWARD(1, 4, 1e-5, 0), TENSOR_SYMBOL_LIST(bx), TENSOR_SYMBOL_LIST(by, saved_mean, saved_inv_std), "group_norm");
    6415
    0
      ccv_nnc_graph_exec_symbol_autogen(symbolic_graph, 0, 0, CCV_NNC_AUTOGEN_ALL_EXECS | CCV_NNC_AUTOGEN_SOURCES_AND_DESTINATIONS);
    6416
    0
      ccv_nnc_symbolic_graph_backward(symbolic_graph, TENSOR_SYMBOL_LIST(by), TENSOR_SYMBOL_LIST(bx), SYMBOLIC_GRAPH_SOURCES(symbolic_graph), SYMBOLIC_GRAPH_DESTINATIONS(symbolic_graph));
    6417
    0
      ccv_nnc_graph_exec_symbol_autogen(symbolic_graph, 0, 0, CCV_NNC_AUTOGEN_ALL_EXECS | CCV_NNC_AUTOGEN_SOURCES_AND_DESTINATIONS);
    6418
    0
      ccv_nnc_tensor_symbol_t dby = ccv_nnc_tensor_symbol_for_backward(symbolic_graph, by);
    6419
    0
      ccv_nnc_tensor_symbol_t dbx = ccv_nnc_tensor_symbol_for_backward(symbolic_graph, bx);
    6420
    0
      ccv_nnc_graph_t* graph = 0;
    6421
    0
      ccv_nnc_tensor_arena_t* tensor_arena = 0;
    6422
    0
      ccv_nnc_graph_exec_arena_t* graph_exec_arena = 0;
    6423
    0
      ccv_nnc_symbolic_graph_compile(symbolic_graph, ccv_nnc_default_compile_params, 0, 0, 0, 0, SYMBOLIC_GRAPH_SOURCES(symbolic_graph), SYMBOLIC_GRAPH_DESTINATIONS(symbolic_graph), &graph, &tensor_arena, &graph_exec_arena);
    6424
    0
      SYMBOLIC_GRAPH_GEN(symbolic_graph, CCV_NNC_LONG_DOT_GRAPH);
    6425
    0
      GRAPH_GEN(graph, CCV_NNC_LONG_DOT_GRAPH);
    6426
    0
      ccv_nnc_tensor_t* const bx_tensor = ccv_nnc_tensor_from_symbol(tensor_arena, bx);
    6427
    0
      dsfmt_t dsfmt;
    6428
    0
      ccv_nnc_tensor_t* const x_tensor = ccv_nnc_tensor_new(0, CPU_TENSOR_NHWC(32F, 2, GN_C_DIM, 2, LN_DIM), 0);
    6429
    0
      int i;
    6430
    0
      dsfmt_init_gen_rand(&dsfmt, 1);
    6431
    0
      for (i = 0; i < 2 * GN_C_DIM * 2 * LN_DIM; i++)
    6432
    0
        x_tensor->data.f32[i] = dsfmt_genrand_open_close(&dsfmt) * 100;
    6433
    0
      ccv_nnc_cmd_exec(CMD_DATA_TRANSFER_FORWARD(), ccv_nnc_no_hint, 0, TENSOR_LIST(x_tensor), TENSOR_LIST(bx_tensor), 0);
    6434
      // ccv_nnc_graph_run(graph, 0, TRAVERSE_FULL, 0, 0);
    6435
    0
      ccv_nnc_tensor_t* const dy_tensor = ccv_nnc_tensor_new(0, CPU_TENSOR_NHWC(32F, 2, GN_C_DIM, 2, LN_DIM), 0);
    6436
    0
      ccv_nnc_tensor_t* const dby_tensor = ccv_nnc_tensor_from_symbol(tensor_arena, dby);
    6437
    0
      for (i = 0; i < 2 * GN_C_DIM * 2 * LN_DIM; i++)
    6438
    0
        dy_tensor->data.f32[i] = dsfmt_genrand_open_close(&dsfmt) * 2 - 1;
    6439
    0
      ccv_nnc_cmd_exec(CMD_DATA_TRANSFER_FORWARD(), ccv_nnc_no_hint, 0, TENSOR_LIST(dy_tensor), TENSOR_LIST(dby_tensor), 0);
    6440
    0
      ccv_nnc_graph_run(graph, 0, TRAVERSE_FULL, 0, 0);
    6441
    0
      ccv_nnc_tensor_t* const dbx_tensor = ccv_nnc_tensor_from_symbol(tensor_arena, dbx);
    6442
    0
      ccv_nnc_tensor_t* const dx_tensor = ccv_nnc_tensor_new(0, CPU_TENSOR_NHWC(32F, 2, GN_C_DIM, 2, LN_DIM), 0);
    6443
    0
      ccv_nnc_cmd_exec(CMD_DATA_TRANSFER_FORWARD(), ccv_nnc_no_hint, 0, TENSOR_LIST(dbx_tensor), TENSOR_LIST(dx_tensor), 0);
    6444
    0
      ccv_nnc_symbolic_graph_free(symbolic_graph);
    6445
    0
      ccv_nnc_tensor_arena_free(tensor_arena);
    6446
    0
      ccv_nnc_graph_exec_arena_free(graph_exec_arena);
    6447
    0
      ccv_nnc_graph_free(graph);
    6448
    0
      ccv_nnc_symbolic_graph_t* const cpu_symbolic_graph = ccv_nnc_symbolic_graph_new();
    6449
    0
      ccv_nnc_tensor_symbol_t cx = ccv_nnc_tensor_symbol_new(cpu_symbolic_graph, CPU_TENSOR_NHWC(32F, 2, GN_C_DIM, 2, LN_DIM), "x");
    6450
    0
      ccv_nnc_tensor_symbol_t cy = ccv_nnc_tensor_symbol_new(cpu_symbolic_graph, CPU_TENSOR_NHWC(32F, 2, GN_C_DIM, 2, LN_DIM), "y");
    6451
    0
      ccv_nnc_tensor_symbol_t csaved_mean = ccv_nnc_tensor_symbol_new(cpu_symbolic_graph, CPU_TENSOR_NHWC(32F, 2, GN_RC_DIM, 2, LN_DIM), "saved_mean");
    6452
    0
      ccv_nnc_tensor_symbol_t csaved_inv_std = ccv_nnc_tensor_symbol_new(cpu_symbolic_graph, CPU_TENSOR_NHWC(32F, 2, GN_RC_DIM, 2, LN_DIM), "saved_inv_std");
    6453
    0
      ccv_nnc_graph_exec_symbol_new(cpu_symbolic_graph, CMD_GROUP_NORM_FORWARD(1, GN_RC_DIM, 1e-5, 0), TENSOR_SYMBOL_LIST(cx), TENSOR_SYMBOL_LIST(cy, csaved_mean, csaved_inv_std), "group_norm");
    6454
    0
      ccv_nnc_graph_exec_symbol_autogen(cpu_symbolic_graph, 0, 0, CCV_NNC_AUTOGEN_ALL_EXECS | CCV_NNC_AUTOGEN_SOURCES_AND_DESTINATIONS);
    6455
    0
      ccv_nnc_symbolic_graph_backward(cpu_symbolic_graph, TENSOR_SYMBOL_LIST(cy), TENSOR_SYMBOL_LIST(cx), SYMBOLIC_GRAPH_SOURCES(cpu_symbolic_graph), SYMBOLIC_GRAPH_DESTINATIONS(cpu_symbolic_graph));
    6456
    0
      ccv_nnc_graph_exec_symbol_autogen(cpu_symbolic_graph, 0, 0, CCV_NNC_AUTOGEN_ALL_EXECS | CCV_NNC_AUTOGEN_SOURCES_AND_DESTINATIONS);
    6457
    0
      ccv_nnc_tensor_symbol_t dcy = ccv_nnc_tensor_symbol_for_backward(cpu_symbolic_graph, cy);
    6458
    0
      ccv_nnc_tensor_symbol_t dcx = ccv_nnc_tensor_symbol_for_backward(cpu_symbolic_graph, cx);
    6459
    0
      ccv_nnc_graph_t* cpu_graph = 0;
    6460
    0
      ccv_nnc_tensor_arena_t* cpu_tensor_arena = 0;
    6461
    0
      ccv_nnc_graph_exec_arena_t* cpu_graph_exec_arena = 0;
    6462
    0
      ccv_nnc_symbolic_graph_compile(cpu_symbolic_graph, ccv_nnc_default_compile_params, 0, 0, 0, 0, SYMBOLIC_GRAPH_SOURCES(cpu_symbolic_graph), SYMBOLIC_GRAPH_DESTINATIONS(cpu_symbolic_graph), &cpu_graph, &cpu_tensor_arena, &cpu_graph_exec_arena);
    6463
    0
      ccv_nnc_tensor_t* const cx_tensor = ccv_nnc_tensor_from_symbol(cpu_tensor_arena, cx);
    6464
    0
      memcpy(cx_tensor->data.f32, x_tensor->data.f32, sizeof(float) * 2 * GN_C_DIM * 2 * LN_DIM);
    6465
    0
      ccv_nnc_tensor_t* const dcy_tensor = ccv_nnc_tensor_from_symbol(cpu_tensor_arena, dcy);
    6466
    0
      memcpy(dcy_tensor->data.f32, dy_tensor->data.f32, sizeof(float) * 2 * GN_C_DIM * 2 * LN_DIM);
    6467
    6468
    0
      ccv_nnc_graph_run(cpu_graph, 0, TRAVERSE_FULL, 0, 0);
    6469
    6470
    0
      ccv_nnc_tensor_t* const dcx_tensor = ccv_nnc_tensor_from_symbol(cpu_tensor_arena, dcx);
    6471
    6472
    0
      REQUIRE_ARRAY_EQ_WITH_TOLERANCE(float, dx_tensor->data.f32, dcx_tensor->data.f32, 2 * GN_C_DIM * 2 * LN_DIM, 1e-5, "group norm output from mps should match from CPU");
    6473
    6474
    0
      ccv_nnc_symbolic_graph_free(cpu_symbolic_graph);
    6475
    0
      ccv_nnc_tensor_arena_free(cpu_tensor_arena);
    6476
    0
      ccv_nnc_graph_exec_arena_free(cpu_graph_exec_arena);
    6477
    0
      ccv_nnc_graph_free(cpu_graph);
    6478
    0
      ccv_nnc_tensor_free(x_tensor);
    6479
    0
      ccv_nnc_tensor_free(dy_tensor);
    6480
    0
      ccv_nnc_tensor_free(dx_tensor);
    6481
    0
    }
    6482
    6483
    TEST_CASE("broadcasting semantics for mul backward (a,b)")
    6484
    1
    {
    6485
    1
      GUARD_ELSE_RETURN(ccv_nnc_cmd_ok(CCV_NNC_MUL_FORWARD, CCV_NNC_BACKEND_MPS) &&
    6486
    1
        ccv_nnc_cmd_ok(CCV_NNC_MUL_BACKWARD, CCV_NNC_BACKEND_MPS));
    6487
    0
      ccv_nnc_tensor_t* const a = ccv_nnc_tensor_new(0, CPU_TENSOR_NHWC(32F, 4, 1), 0);
    6488
    0
      ccv_nnc_tensor_t* const b = ccv_nnc_tensor_new(0, CPU_TENSOR_NHWC(32F, 2), 0);
    6489
    0
      ccv_nnc_tensor_t* const c = ccv_nnc_tensor_new(0, CPU_TENSOR_NHWC(32F, 4, 2), 0);
    6490
    0
      ccv_nnc_tensor_t* const da = ccv_nnc_tensor_new(0, CPU_TENSOR_NHWC(32F, 4, 1), 0);
    6491
    0
      ccv_nnc_tensor_t* const db = ccv_nnc_tensor_new(0, CPU_TENSOR_NHWC(32F, 2), 0);
    6492
    0
      ccv_nnc_tensor_t* const dat = ccv_nnc_tensor_new(0, CPU_TENSOR_NHWC(32F, 4, 1), 0);
    6493
    0
      ccv_nnc_tensor_t* const dbt = ccv_nnc_tensor_new(0, CPU_TENSOR_NHWC(32F, 2), 0);
    6494
    0
      a->data.f32[0] = 1;
    6495
    0
      a->data.f32[1] = 2;
    6496
    0
      a->data.f32[2] = 3;
    6497
    0
      a->data.f32[3] = 4;
    6498
    0
      b->data.f32[0] = 5;
    6499
    0
      b->data.f32[1] = 6;
    6500
    0
      float ctp[] = {
    6501
    0
        6, 7,
    6502
    0
        7, 8,
    6503
    0
        8, 9,
    6504
    0
        9, 10
    6505
    0
      };
    6506
    0
      memcpy(c->data.f32, ctp, sizeof(ctp));
    6507
    0
      ccv_nnc_tensor_t* const ga = ccv_nnc_tensor_new(0, GPU_TENSOR_NHWC(000, 32F, 4, 1), 0);
    6508
    0
      ccv_nnc_tensor_t* const gb = ccv_nnc_tensor_new(0, GPU_TENSOR_NHWC(000, 32F, 2), 0);
    6509
    0
      ccv_nnc_tensor_t* const gda = ccv_nnc_tensor_new(0, GPU_TENSOR_NHWC(000, 32F, 4, 1), 0);
    6510
    0
      ccv_nnc_tensor_t* const gdb = ccv_nnc_tensor_new(0, GPU_TENSOR_NHWC(000, 32F, 2), 0);
    6511
    0
      ccv_nnc_tensor_t* const gc = ccv_nnc_tensor_new(0, GPU_TENSOR_NHWC(000, 32F, 4, 2), 0);
    6512
    0
      ccv_nnc_cmd_exec(CMD_DATA_TRANSFER_FORWARD(), ccv_nnc_no_hint, 0, TENSOR_LIST(a, b, c), TENSOR_LIST(ga, gb, gc), 0);
    6513
    0
      ccv_nnc_cmd_exec(CMD_MUL_BACKWARD(0.5), ccv_nnc_no_hint, 0, TENSOR_LIST(gc, ga, gb), TENSOR_LIST(gda, gdb), 0);
    6514
    0
      ccv_nnc_cmd_exec(CMD_DATA_TRANSFER_FORWARD(), ccv_nnc_no_hint, 0, TENSOR_LIST(gda, gdb), TENSOR_LIST(da, db), 0);
    6515
    0
      ccv_nnc_cmd_exec(CMD_MUL_BACKWARD(0.5), ccv_nnc_no_hint, 0, TENSOR_LIST(c, a, b), TENSOR_LIST(dat, dbt), 0);
    6516
    6517
    0
      REQUIRE_TENSOR_EQ(dat, da, "gradient of a should be equal");
    6518
    0
      REQUIRE_TENSOR_EQ(dbt, db, "gradient of b should be equal");
    6519
    0
      ccv_nnc_tensor_free(a);
    6520
    0
      ccv_nnc_tensor_free(b);
    6521
    0
      ccv_nnc_tensor_free(c);
    6522
    0
      ccv_nnc_tensor_free(da);
    6523
    0
      ccv_nnc_tensor_free(db);
    6524
    0
      ccv_nnc_tensor_free(dat);
    6525
    0
      ccv_nnc_tensor_free(dbt);
    6526
    0
      ccv_nnc_tensor_free(ga);
    6527
    0
      ccv_nnc_tensor_free(gb);
    6528
    0
      ccv_nnc_tensor_free(gc);
    6529
    0
      ccv_nnc_tensor_free(gda);
    6530
    0
      ccv_nnc_tensor_free(gdb);
    6531
    0
    }
    6532
    6533
    TEST_CASE("broadcasting semantics for mul backward (a, nil)")
    6534
    1
    {
    6535
    1
      GUARD_ELSE_RETURN(ccv_nnc_cmd_ok(CCV_NNC_MUL_FORWARD, CCV_NNC_BACKEND_MPS) &&
    6536
    1
        ccv_nnc_cmd_ok(CCV_NNC_MUL_BACKWARD, CCV_NNC_BACKEND_MPS));
    6537
    0
      ccv_nnc_tensor_t* const a = ccv_nnc_tensor_new(0, CPU_TENSOR_NHWC(32F, 4, 1), 0);
    6538
    0
      ccv_nnc_tensor_t* const b = ccv_nnc_tensor_new(0, CPU_TENSOR_NHWC(32F, 2), 0);
    6539
    0
      ccv_nnc_tensor_t* const c = ccv_nnc_tensor_new(0, CPU_TENSOR_NHWC(32F, 4, 2), 0);
    6540
    0
      ccv_nnc_tensor_t* const da = ccv_nnc_tensor_new(0, CPU_TENSOR_NHWC(32F, 4, 1), 0);
    6541
    0
      ccv_nnc_tensor_t* const dat = ccv_nnc_tensor_new(0, CPU_TENSOR_NHWC(32F, 4, 1), 0);
    6542
    0
      a->data.f32[0] = 1;
    6543
    0
      a->data.f32[1] = 2;
    6544
    0
      a->data.f32[2] = 3;
    6545
    0
      a->data.f32[3] = 4;
    6546
    0
      b->data.f32[0] = 5;
    6547
    0
      b->data.f32[1] = 6;
    6548
    0
      float ctp[] = {
    6549
    0
        6, 7,
    6550
    0
        7, 8,
    6551
    0
        8, 9,
    6552
    0
        9, 10
    6553
    0
      };
    6554
    0
      memcpy(c->data.f32, ctp, sizeof(ctp));
    6555
    0
      ccv_nnc_tensor_t* const ga = ccv_nnc_tensor_new(0, GPU_TENSOR_NHWC(000, 32F, 4, 1), 0);
    6556
    0
      ccv_nnc_tensor_t* const gb = ccv_nnc_tensor_new(0, GPU_TENSOR_NHWC(000, 32F, 2), 0);
    6557
    0
      ccv_nnc_tensor_t* const gda = ccv_nnc_tensor_new(0, GPU_TENSOR_NHWC(000, 32F, 4, 1), 0);
    6558
    0
      ccv_nnc_tensor_t* const gc = ccv_nnc_tensor_new(0, GPU_TENSOR_NHWC(000, 32F, 4, 2), 0);
    6559
    0
      ccv_nnc_cmd_exec(CMD_DATA_TRANSFER_FORWARD(), ccv_nnc_no_hint, 0, TENSOR_LIST(a, b, c), TENSOR_LIST(ga, gb, gc), 0);
    6560
    0
      ccv_nnc_cmd_exec(CMD_MUL_BACKWARD(0.5), ccv_nnc_no_hint, 0, TENSOR_LIST(gc, ga, gb), TENSOR_LIST(gda, 0), 0);
    6561
    0
      ccv_nnc_cmd_exec(CMD_DATA_TRANSFER_FORWARD(), ccv_nnc_no_hint, 0, TENSOR_LIST(gda, 0), TENSOR_LIST(da, 0), 0);
    6562
    0
      ccv_nnc_cmd_exec(CMD_MUL_BACKWARD(0.5), ccv_nnc_no_hint, 0, TENSOR_LIST(c, a, b), TENSOR_LIST(dat, 0), 0);
    6563
    6564
    0
      REQUIRE_TENSOR_EQ(dat, da, "gradient of a should be equal");
    6565
    0
      ccv_nnc_tensor_free(a);
    6566
    0
      ccv_nnc_tensor_free(b);
    6567
    0
      ccv_nnc_tensor_free(c);
    6568
    0
      ccv_nnc_tensor_free(da);
    6569
    0
      ccv_nnc_tensor_free(dat);
    6570
    0
      ccv_nnc_tensor_free(ga);
    6571
    0
      ccv_nnc_tensor_free(gb);
    6572
    0
      ccv_nnc_tensor_free(gc);
    6573
    0
      ccv_nnc_tensor_free(gda);
    6574
    0
    }
    6575
    6576
    TEST_CASE("broadcasting semantics for mul backward (nil,b)")
    6577
    1
    {
    6578
    1
      GUARD_ELSE_RETURN(ccv_nnc_cmd_ok(CCV_NNC_MUL_FORWARD, CCV_NNC_BACKEND_MPS) &&
    6579
    1
        ccv_nnc_cmd_ok(CCV_NNC_MUL_BACKWARD, CCV_NNC_BACKEND_MPS));
    6580
    0
      ccv_nnc_tensor_t* const a = ccv_nnc_tensor_new(0, CPU_TENSOR_NHWC(32F, 4, 1), 0);
    6581
    0
      ccv_nnc_tensor_t* const b = ccv_nnc_tensor_new(0, CPU_TENSOR_NHWC(32F, 2), 0);
    6582
    0
      ccv_nnc_tensor_t* const c = ccv_nnc_tensor_new(0, CPU_TENSOR_NHWC(32F, 4, 2), 0);
    6583
    0
      ccv_nnc_tensor_t* const db = ccv_nnc_tensor_new(0, CPU_TENSOR_NHWC(32F, 2), 0);
    6584
    0
      ccv_nnc_tensor_t* const dbt = ccv_nnc_tensor_new(0, CPU_TENSOR_NHWC(32F, 2), 0);
    6585
    0
      a->data.f32[0] = 1;
    6586
    0
      a->data.f32[1] = 2;
    6587
    0
      a->data.f32[2] = 3;
    6588
    0
      a->data.f32[3] = 4;
    6589
    0
      b->data.f32[0] = 5;
    6590
    0
      b->data.f32[1] = 6;
    6591
    0
      float ctp[] = {
    6592
    0
        6, 7,
    6593
    0
        7, 8,
    6594
    0
        8, 9,
    6595
    0
        9, 10
    6596
    0
      };
    6597
    0
      memcpy(c->data.f32, ctp, sizeof(ctp));
    6598
    0
      ccv_nnc_tensor_t* const ga = ccv_nnc_tensor_new(0, GPU_TENSOR_NHWC(000, 32F, 4, 1), 0);
    6599
    0
      ccv_nnc_tensor_t* const gb = ccv_nnc_tensor_new(0, GPU_TENSOR_NHWC(000, 32F, 2), 0);
    6600
    0
      ccv_nnc_tensor_t* const gdb = ccv_nnc_tensor_new(0, GPU_TENSOR_NHWC(000, 32F, 2), 0);
    6601
    0
      ccv_nnc_tensor_t* const gc = ccv_nnc_tensor_new(0, GPU_TENSOR_NHWC(000, 32F, 4, 2), 0);
    6602
    0
      ccv_nnc_cmd_exec(CMD_DATA_TRANSFER_FORWARD(), ccv_nnc_no_hint, 0, TENSOR_LIST(a, b, c), TENSOR_LIST(ga, gb, gc), 0);
    6603
    0
      ccv_nnc_cmd_exec(CMD_MUL_BACKWARD(0.5), ccv_nnc_no_hint, 0, TENSOR_LIST(gc, ga, gb), TENSOR_LIST(0, gdb), 0);
    6604
    0
      ccv_nnc_cmd_exec(CMD_DATA_TRANSFER_FORWARD(), ccv_nnc_no_hint, 0, TENSOR_LIST(0, gdb), TENSOR_LIST(0, db), 0);
    6605
    0
      ccv_nnc_cmd_exec(CMD_MUL_BACKWARD(0.5), ccv_nnc_no_hint, 0, TENSOR_LIST(c, a, b), TENSOR_LIST(0, dbt), 0);
    6606
    6607
    0
      REQUIRE_TENSOR_EQ(dbt, db, "gradient of b should be equal");
    6608
    0
      ccv_nnc_tensor_free(a);
    6609
    0
      ccv_nnc_tensor_free(b);
    6610
    0
      ccv_nnc_tensor_free(c);
    6611
    0
      ccv_nnc_tensor_free(db);
    6612
    0
      ccv_nnc_tensor_free(dbt);
    6613
    0
      ccv_nnc_tensor_free(ga);
    6614
    0
      ccv_nnc_tensor_free(gb);
    6615
    0
      ccv_nnc_tensor_free(gc);
    6616
    0
      ccv_nnc_tensor_free(gdb);
    6617
    0
    }
    6618
    6619
    TEST_CASE("broadcasting semantics for mul backward (no output db)")
    6620
    1
    {
    6621
    1
      GUARD_ELSE_RETURN(ccv_nnc_cmd_ok(CCV_NNC_MUL_FORWARD, CCV_NNC_BACKEND_MPS) &&
    6622
    1
        ccv_nnc_cmd_ok(CCV_NNC_MUL_BACKWARD, CCV_NNC_BACKEND_MPS));
    6623
    0
      ccv_nnc_tensor_t* const a = ccv_nnc_tensor_new(0, CPU_TENSOR_NHWC(32F, 4, 1), 0);
    6624
    0
      ccv_nnc_tensor_t* const b = ccv_nnc_tensor_new(0, CPU_TENSOR_NHWC(32F, 2), 0);
    6625
    0
      ccv_nnc_tensor_t* const c = ccv_nnc_tensor_new(0, CPU_TENSOR_NHWC(32F, 4, 2), 0);
    6626
    0
      ccv_nnc_tensor_t* const da = ccv_nnc_tensor_new(0, CPU_TENSOR_NHWC(32F, 4, 1), 0);
    6627
    0
      ccv_nnc_tensor_t* const dat = ccv_nnc_tensor_new(0, CPU_TENSOR_NHWC(32F, 4, 1), 0);
    6628
    0
      a->data.f32[0] = 1;
    6629
    0
      a->data.f32[1] = 2;
    6630
    0
      a->data.f32[2] = 3;
    6631
    0
      a->data.f32[3] = 4;
    6632
    0
      b->data.f32[0] = 5;
    6633
    0
      b->data.f32[1] = 6;
    6634
    0
      float ctp[] = {
    6635
    0
        6, 7,
    6636
    0
        7, 8,
    6637
    0
        8, 9,
    6638
    0
        9, 10
    6639
    0
      };
    6640
    0
      memcpy(c->data.f32, ctp, sizeof(ctp));
    6641
    0
      ccv_nnc_tensor_t* const ga = ccv_nnc_tensor_new(0, GPU_TENSOR_NHWC(000, 32F, 4, 1), 0);
    6642
    0
      ccv_nnc_tensor_t* const gb = ccv_nnc_tensor_new(0, GPU_TENSOR_NHWC(000, 32F, 2), 0);
    6643
    0
      ccv_nnc_tensor_t* const gda = ccv_nnc_tensor_new(0, GPU_TENSOR_NHWC(000, 32F, 4, 1), 0);
    6644
    0
      ccv_nnc_tensor_t* const gc = ccv_nnc_tensor_new(0, GPU_TENSOR_NHWC(000, 32F, 4, 2), 0);
    6645
    0
      ccv_nnc_cmd_exec(CMD_DATA_TRANSFER_FORWARD(), ccv_nnc_no_hint, 0, TENSOR_LIST(a, b, c), TENSOR_LIST(ga, gb, gc), 0);
    6646
    0
      ccv_nnc_cmd_exec(CMD_MUL_BACKWARD(0.5), ccv_nnc_no_hint, 0, TENSOR_LIST(gc, ga, gb), TENSOR_LIST(gda, 0), 0);
    6647
    0
      ccv_nnc_cmd_exec(CMD_DATA_TRANSFER_FORWARD(), ccv_nnc_no_hint, 0, TENSOR_LIST(gda, 0), TENSOR_LIST(da, 0), 0);
    6648
    0
      ccv_nnc_cmd_exec(CMD_MUL_BACKWARD(0.5), ccv_nnc_no_hint, 0, TENSOR_LIST(c, a, b), TENSOR_LIST(dat, 0), 0);
    6649
    6650
    0
      REQUIRE_TENSOR_EQ(dat, da, "gradient of a should be equal");
    6651
    0
      ccv_nnc_tensor_free(a);
    6652
    0
      ccv_nnc_tensor_free(b);
    6653
    0
      ccv_nnc_tensor_free(c);
    6654
    0
      ccv_nnc_tensor_free(da);
    6655
    0
      ccv_nnc_tensor_free(dat);
    6656
    0
      ccv_nnc_tensor_free(ga);
    6657
    0
      ccv_nnc_tensor_free(gb);
    6658
    0
      ccv_nnc_tensor_free(gc);
    6659
    0
      ccv_nnc_tensor_free(gda);
    6660
    0
    }
    6661
    6662
    TEST_CASE("broadcasting semantics for mul backward (no input grad)")
    6663
    1
    {
    6664
    1
      GUARD_ELSE_RETURN(ccv_nnc_cmd_ok(CCV_NNC_MUL_FORWARD, CCV_NNC_BACKEND_MPS) &&
    6665
    1
        ccv_nnc_cmd_ok(CCV_NNC_MUL_BACKWARD, CCV_NNC_BACKEND_MPS));
    6666
    0
      ccv_nnc_tensor_t* const a = ccv_nnc_tensor_new(0, CPU_TENSOR_NHWC(32F, 4, 1), 0);
    6667
    0
      ccv_nnc_tensor_t* const b = ccv_nnc_tensor_new(0, CPU_TENSOR_NHWC(32F, 2), 0);
    6668
    0
      ccv_nnc_tensor_t* const da = ccv_nnc_tensor_new(0, CPU_TENSOR_NHWC(32F, 4, 1), 0);
    6669
    0
      ccv_nnc_tensor_t* const db = ccv_nnc_tensor_new(0, CPU_TENSOR_NHWC(32F, 2), 0);
    6670
    0
      ccv_nnc_tensor_t* const dat = ccv_nnc_tensor_new(0, CPU_TENSOR_NHWC(32F, 4, 1), 0);
    6671
    0
      ccv_nnc_tensor_t* const dbt = ccv_nnc_tensor_new(0, CPU_TENSOR_NHWC(32F, 2), 0);
    6672
    0
      a->data.f32[0] = 1;
    6673
    0
      a->data.f32[1] = 2;
    6674
    0
      a->data.f32[2] = 3;
    6675
    0
      a->data.f32[3] = 4;
    6676
    0
      b->data.f32[0] = 5;
    6677
    0
      b->data.f32[1] = 6;
    6678
    0
      ccv_nnc_tensor_t* const ga = ccv_nnc_tensor_new(0, GPU_TENSOR_NHWC(000, 32F, 4, 1), 0);
    6679
    0
      ccv_nnc_tensor_t* const gb = ccv_nnc_tensor_new(0, GPU_TENSOR_NHWC(000, 32F, 2), 0);
    6680
    0
      ccv_nnc_tensor_t* const gda = ccv_nnc_tensor_new(0, GPU_TENSOR_NHWC(000, 32F, 4, 1), 0);
    6681
    0
      ccv_nnc_tensor_t* const gdb = ccv_nnc_tensor_new(0, GPU_TENSOR_NHWC(000, 32F, 2), 0);
    6682
    0
      ccv_nnc_cmd_exec(CMD_DATA_TRANSFER_FORWARD(), ccv_nnc_no_hint, 0, TENSOR_LIST(a, b), TENSOR_LIST(ga, gb), 0);
    6683
    0
      ccv_nnc_cmd_exec(CMD_MUL_BACKWARD(0.5), ccv_nnc_no_hint, 0, TENSOR_LIST(0, ga, gb), TENSOR_LIST(gda, gdb), 0);
    6684
    0
      ccv_nnc_cmd_exec(CMD_DATA_TRANSFER_FORWARD(), ccv_nnc_no_hint, 0, TENSOR_LIST(gda, gdb), TENSOR_LIST(da, db), 0);
    6685
    0
      ccv_nnc_cmd_exec(CMD_MUL_BACKWARD(0.5), ccv_nnc_no_hint, 0, TENSOR_LIST(0, a, b), TENSOR_LIST(dat, dbt), 0);
    6686
    6687
    6688
    0
      REQUIRE_TENSOR_EQ(dat, da, "gradient of a should be equal");
    6689
    0
      REQUIRE_TENSOR_EQ(dbt, db, "gradient of b should be equal");
    6690
    0
      ccv_nnc_tensor_free(a);
    6691
    0
      ccv_nnc_tensor_free(b);
    6692
    0
      ccv_nnc_tensor_free(da);
    6693
    0
      ccv_nnc_tensor_free(db);
    6694
    0
      ccv_nnc_tensor_free(dat);
    6695
    0
      ccv_nnc_tensor_free(dbt);
    6696
    0
      ccv_nnc_tensor_free(ga);
    6697
    0
      ccv_nnc_tensor_free(gb);
    6698
    0
      ccv_nnc_tensor_free(gda);
    6699
    0
      ccv_nnc_tensor_free(gdb);
    6700
    0
    }
    6701
    6702
    6703
    TEST_CASE("broadcasting semantics for mul backward (no input grad) for b")
    6704
    1
    {
    6705
    1
      GUARD_ELSE_RETURN(ccv_nnc_cmd_ok(CCV_NNC_MUL_FORWARD, CCV_NNC_BACKEND_MPS) &&
    6706
    1
        ccv_nnc_cmd_ok(CCV_NNC_MUL_BACKWARD, CCV_NNC_BACKEND_MPS));
    6707
    0
      ccv_nnc_tensor_t* const a = ccv_nnc_tensor_new(0, CPU_TENSOR_NHWC(32F, 2, 3), 0);
    6708
    0
      ccv_nnc_tensor_t* const b = ccv_nnc_tensor_new(0, CPU_TENSOR_NHWC(32F, 3), 0);
    6709
    0
      ccv_nnc_tensor_t* const da = ccv_nnc_tensor_new(0, CPU_TENSOR_NHWC(32F, 2, 3), 0);
    6710
    0
      ccv_nnc_tensor_t* const db = ccv_nnc_tensor_new(0, CPU_TENSOR_NHWC(32F, 3), 0);
    6711
    0
      ccv_nnc_tensor_t* const dat = ccv_nnc_tensor_new(0, CPU_TENSOR_NHWC(32F, 2, 3), 0);
    6712
    0
      ccv_nnc_tensor_t* const dbt = ccv_nnc_tensor_new(0, CPU_TENSOR_NHWC(32F, 3), 0);
    6713
    0
      a->data.f32[0] = 1;
    6714
    0
      a->data.f32[1] = 2;
    6715
    0
      a->data.f32[2] = 3;
    6716
    0
      a->data.f32[3] = 4;
    6717
    0
      a->data.f32[4] = 5;
    6718
    0
      a->data.f32[5] = 6;
    6719
    0
      b->data.f32[0] = 7;
    6720
    0
      b->data.f32[1] = 8;
    6721
    0
      b->data.f32[2] = 9;
    6722
    0
      ccv_nnc_tensor_t* const ga = ccv_nnc_tensor_new(0, GPU_TENSOR_NHWC(000, 32F, 2, 3), 0);
    6723
    0
      ccv_nnc_tensor_t* const gb = ccv_nnc_tensor_new(0, GPU_TENSOR_NHWC(000, 32F, 3), 0);
    6724
    0
      ccv_nnc_tensor_t* const gda = ccv_nnc_tensor_new(0, GPU_TENSOR_NHWC(000, 32F, 2, 3), 0);
    6725
    0
      ccv_nnc_tensor_t* const gdb = ccv_nnc_tensor_new(0, GPU_TENSOR_NHWC(000, 32F, 3), 0);
    6726
    0
      ccv_nnc_cmd_exec(CMD_DATA_TRANSFER_FORWARD(), ccv_nnc_no_hint, 0, TENSOR_LIST(a, b), TENSOR_LIST(ga, gb), 0);
    6727
    0
      ccv_nnc_cmd_exec(CMD_MUL_BACKWARD(0.5), ccv_nnc_no_hint, 0, TENSOR_LIST(0, ga, gb), TENSOR_LIST(gda, gdb), 0);
    6728
    0
      ccv_nnc_cmd_exec(CMD_DATA_TRANSFER_FORWARD(), ccv_nnc_no_hint, 0, TENSOR_LIST(gda, gdb), TENSOR_LIST(da, db), 0);
    6729
    0
      ccv_nnc_cmd_exec(CMD_MUL_BACKWARD(0.5), ccv_nnc_no_hint, 0, TENSOR_LIST(0, a, b), TENSOR_LIST(dat, dbt), 0);
    6730
    6731
    0
      REQUIRE_TENSOR_EQ(dat, da, "gradient of a should be equal");
    6732
    0
      REQUIRE_TENSOR_EQ(dbt, db, "gradient of b should be equal");
    6733
    0
      ccv_nnc_tensor_free(a);
    6734
    0
      ccv_nnc_tensor_free(b);
    6735
    0
      ccv_nnc_tensor_free(da);
    6736
    0
      ccv_nnc_tensor_free(db);
    6737
    0
      ccv_nnc_tensor_free(dat);
    6738
    0
      ccv_nnc_tensor_free(dbt);
    6739
    0
      ccv_nnc_tensor_free(ga);
    6740
    0
      ccv_nnc_tensor_free(gb);
    6741
    0
      ccv_nnc_tensor_free(gda);
    6742
    0
      ccv_nnc_tensor_free(gdb);
    6743
    0
    }
    6744
    6745
    TEST_CASE("broadcasting semantics for mul backward (no input grad) for a")
    6746
    1
    {
    6747
    1
      GUARD_ELSE_RETURN(ccv_nnc_cmd_ok(CCV_NNC_MUL_FORWARD, CCV_NNC_BACKEND_MPS) &&
    6748
    1
        ccv_nnc_cmd_ok(CCV_NNC_MUL_BACKWARD, CCV_NNC_BACKEND_MPS));
    6749
    0
      ccv_nnc_tensor_t* const b = ccv_nnc_tensor_new(0, CPU_TENSOR_NHWC(32F, 2, 3), 0);
    6750
    0
      ccv_nnc_tensor_t* const a = ccv_nnc_tensor_new(0, CPU_TENSOR_NHWC(32F, 3), 0);
    6751
    0
      ccv_nnc_tensor_t* const db = ccv_nnc_tensor_new(0, CPU_TENSOR_NHWC(32F, 2, 3), 0);
    6752
    0
      ccv_nnc_tensor_t* const da = ccv_nnc_tensor_new(0, CPU_TENSOR_NHWC(32F, 3), 0);
    6753
    0
      ccv_nnc_tensor_t* const dbt = ccv_nnc_tensor_new(0, CPU_TENSOR_NHWC(32F, 2, 3), 0);
    6754
    0
      ccv_nnc_tensor_t* const dat = ccv_nnc_tensor_new(0, CPU_TENSOR_NHWC(32F, 3), 0);
    6755
    0
      b->data.f32[0] = 1;
    6756
    0
      b->data.f32[1] = 2;
    6757
    0
      b->data.f32[2] = 3;
    6758
    0
      b->data.f32[3] = 4;
    6759
    0
      b->data.f32[4] = 5;
    6760
    0
      b->data.f32[5] = 6;
    6761
    0
      a->data.f32[0] = 7;
    6762
    0
      a->data.f32[1] = 8;
    6763
    0
      a->data.f32[2] = 9;
    6764
    0
      ccv_nnc_tensor_t* const gb = ccv_nnc_tensor_new(0, GPU_TENSOR_NHWC(000, 32F, 2, 3), 0);
    6765
    0
      ccv_nnc_tensor_t* const ga = ccv_nnc_tensor_new(0, GPU_TENSOR_NHWC(000, 32F, 3), 0);
    6766
    0
      ccv_nnc_tensor_t* const gdb = ccv_nnc_tensor_new(0, GPU_TENSOR_NHWC(000, 32F, 2, 3), 0);
    6767
    0
      ccv_nnc_tensor_t* const gda = ccv_nnc_tensor_new(0, GPU_TENSOR_NHWC(000, 32F, 3), 0);
    6768
    0
      ccv_nnc_cmd_exec(CMD_DATA_TRANSFER_FORWARD(), ccv_nnc_no_hint, 0, TENSOR_LIST(a, b), TENSOR_LIST(ga, gb), 0);
    6769
    0
      ccv_nnc_cmd_exec(CMD_MUL_BACKWARD(0.5), ccv_nnc_no_hint, 0, TENSOR_LIST(0, ga, gb), TENSOR_LIST(gda, gdb), 0);
    6770
    0
      ccv_nnc_cmd_exec(CMD_DATA_TRANSFER_FORWARD(), ccv_nnc_no_hint, 0, TENSOR_LIST(gda, gdb), TENSOR_LIST(da, db), 0);
    6771
    0
      ccv_nnc_cmd_exec(CMD_MUL_BACKWARD(0.5), ccv_nnc_no_hint, 0, TENSOR_LIST(0, a, b), TENSOR_LIST(dat, dbt), 0);
    6772
    6773
    0
      REQUIRE_TENSOR_EQ(dat, da, "gradient of a should be equal");
    6774
    0
      REQUIRE_TENSOR_EQ(dbt, db, "gradient of b should be equal");
    6775
    0
      ccv_nnc_tensor_free(a);
    6776
    0
      ccv_nnc_tensor_free(b);
    6777
    0
      ccv_nnc_tensor_free(da);
    6778
    0
      ccv_nnc_tensor_free(db);
    6779
    0
      ccv_nnc_tensor_free(dat);
    6780
    0
      ccv_nnc_tensor_free(dbt);
    6781
    0
      ccv_nnc_tensor_free(ga);
    6782
    0
      ccv_nnc_tensor_free(gb);
    6783
    0
      ccv_nnc_tensor_free(gda);
    6784
    0
      ccv_nnc_tensor_free(gdb);
    6785
    0
    }
    6786
    6787
    TEST_CASE("mps scalar mul forward")
    6788
    1
    {
    6789
    1
      GUARD_ELSE_RETURN(ccv_nnc_cmd_ok(CCV_NNC_SCALAR_MUL_BACKWARD, CCV_NNC_BACKEND_MPS) &&
    6790
    1
        ccv_nnc_cmd_ok(CCV_NNC_SCALAR_MUL_FORWARD, CCV_NNC_BACKEND_MPS));
    6791
    6792
    0
      ccv_nnc_tensor_t* const x = ccv_nnc_tensor_new(0, CPU_TENSOR_NHWC(32F, 4), 0);
    6793
    0
      ccv_nnc_tensor_t* const gx = ccv_nnc_tensor_new(0, GPU_TENSOR_NHWC(000, 32F, 4), 0);
    6794
    6795
    0
      dsfmt_t dsfmt;
    6796
    0
      dsfmt_init_gen_rand(&dsfmt, 0);
    6797
    0
      int i;
    6798
    0
      for (i = 0; i < 4; i++)
    6799
    0
          x->data.f32[i] = dsfmt_genrand_open_close(&dsfmt);
    6800
    0
      ccv_nnc_cmd_exec(CMD_DATA_TRANSFER_FORWARD(), ccv_nnc_no_hint, 0, TENSOR_LIST(x), TENSOR_LIST(gx), 0);
    6801
    6802
    0
      ccv_nnc_tensor_t* const gy = ccv_nnc_tensor_new(0, GPU_TENSOR_NHWC(000, 32F, 4), 0);
    6803
    6804
    0
      ccv_nnc_cmd_exec(CMD_SCALAR_MUL_FORWARD(1.1), ccv_nnc_no_hint, 0, TENSOR_LIST(gx), TENSOR_LIST(gy), 0);
    6805
    6806
    0
      ccv_nnc_tensor_t* const y = ccv_nnc_tensor_new(0, CPU_TENSOR_NHWC(32F, 4), 0);
    6807
    0
      ccv_nnc_cmd_exec(CMD_DATA_TRANSFER_FORWARD(), ccv_nnc_no_hint, 0, TENSOR_LIST(gy), TENSOR_LIST(y), 0);
    6808
    0
      for (i = 0; i < 4; i++) {
    6809
    0
          REQUIRE_EQ_WITH_TOLERANCE(x->data.f32[i] * 1.1, y->data.f32[i], 1e-5, "scalarmul forward cy has to be 1.1 * x");
    6810
    0
      }
    6811
    6812
    0
      ccv_nnc_tensor_free(x);
    6813
    0
      ccv_nnc_tensor_free(gx);
    6814
    0
      ccv_nnc_tensor_free(gy);
    6815
    0
      ccv_nnc_tensor_free(y);
    6816
    0
    }
    6817
    6818
    TEST_CASE("mps scalar mul backward")
    6819
    1
    {
    6820
    1
      GUARD_ELSE_RETURN(
    6821
    1
        ccv_nnc_cmd_ok(CCV_NNC_SCALAR_MUL_FORWARD, CCV_NNC_BACKEND_MPS));
    6822
    6823
    0
      ccv_nnc_tensor_t* const y = ccv_nnc_tensor_new(0, CPU_TENSOR_NHWC(32F, 4), 0);
    6824
    6825
    0
      dsfmt_t dsfmt;
    6826
    0
      dsfmt_init_gen_rand(&dsfmt, 0);
    6827
    0
      int i;
    6828
    0
      for (i = 0; i < 4; i++)
    6829
    0
          y->data.f32[i] = dsfmt_genrand_open_close(&dsfmt);
    6830
    0
      ccv_nnc_tensor_t* const gy = ccv_nnc_tensor_new(0, GPU_TENSOR_NHWC(000, 32F, 4), 0);
    6831
    0
      ccv_nnc_tensor_t* const gdx = ccv_nnc_tensor_new(0, GPU_TENSOR_NHWC(000, 32F, 4), 0);
    6832
    0
      ccv_nnc_cmd_exec(CMD_DATA_TRANSFER_FORWARD(), ccv_nnc_no_hint, 0, TENSOR_LIST(y), TENSOR_LIST(gy), 0);
    6833
    0
      ccv_nnc_cmd_exec(CMD_SCALAR_MUL_BACKWARD(1.1), ccv_nnc_no_hint, 0, TENSOR_LIST(gy), TENSOR_LIST(gdx), 0);
    6834
    6835
    0
      ccv_nnc_tensor_t* const dx = ccv_nnc_tensor_new(0, CPU_TENSOR_NHWC(32F, 4), 0);
    6836
    0
      ccv_nnc_cmd_exec(CMD_DATA_TRANSFER_FORWARD(), ccv_nnc_no_hint, 0, TENSOR_LIST(gdx), TENSOR_LIST(dx), 0);
    6837
    6838
    0
      for (i = 0; i < 4; i++) {
    6839
    0
          REQUIRE_EQ_WITH_TOLERANCE(dx->data.f32[i], y->data.f32[i] * 1.1, 1e-5, "scalarmul backward dx has to be 1.1 * dy");
    6840
    0
      }
    6841
    6842
    0
      ccv_nnc_tensor_free(y);
    6843
    0
      ccv_nnc_tensor_free(gy);
    6844
    0
      ccv_nnc_tensor_free(gdx);
    6845
    0
      ccv_nnc_tensor_free(dx);
    6846
    0
    }
    6847
    6848
    TEST_CASE("mps scalar mul backward, no input")
    6849
    1
    {
    6850
    1
      GUARD_ELSE_RETURN(ccv_nnc_cmd_ok(CCV_NNC_SCALAR_MUL_BACKWARD, CCV_NNC_BACKEND_MPS));
    6851
    6852
    0
      ccv_nnc_tensor_t* const gdx = ccv_nnc_tensor_new(0, GPU_TENSOR_NHWC(000, 32F, 4), 0);
    6853
    0
      ccv_nnc_cmd_exec(CMD_SCALAR_MUL_BACKWARD(1.1), ccv_nnc_no_hint, 0, TENSOR_LIST(0), TENSOR_LIST(gdx), 0);
    6854
    0
      ccv_nnc_tensor_t* const dx = ccv_nnc_tensor_new(0, CPU_TENSOR_NHWC(32F, 4), 0);
    6855
    0
      ccv_nnc_cmd_exec(CMD_DATA_TRANSFER_FORWARD(), ccv_nnc_no_hint, 0, TENSOR_LIST(gdx), TENSOR_LIST(dx), 0);
    6856
    6857
    0
      for (int i = 0; i < 4; i++)
    6858
    0
          REQUIRE_EQ_WITH_TOLERANCE(dx->data.f32[i], 1.1, 1e-5, "scalar mul backward without input should be 1.1 ");
    6859
    0
      ccv_nnc_tensor_free(gdx);
    6860
    0
      ccv_nnc_tensor_free(dx);
    6861
    0
    }
    6862
    6863
    TEST_CASE("mps forward convolution transpose")
    6864
    1
    {
    6865
    1
      GUARD_ELSE_RETURN(ccv_nnc_cmd_ok(CCV_NNC_CONVOLUTION_TRANSPOSE_FORWARD, CCV_NNC_BACKEND_MPS));
    6866
    0
      ccv_nnc_tensor_t* a = ccv_nnc_tensor_new(0, CPU_TENSOR_NHWC(32F, BATCH_SIZE, OUTPUT_SIZE, OUTPUT_SIZE, OUTPUT_DIM), 0);
    6867
    0
      ccv_nnc_tensor_t* b = ccv_nnc_tensor_new(0, CPU_TENSOR_NHWC(32F, BATCH_SIZE, INPUT_SIZE, INPUT_SIZE, INPUT_DIM), 0);
    6868
    0
      ccv_nnc_cmd_t cmd = CMD_CONVOLUTION_TRANSPOSE_FORWARD(1, INPUT_DIM, 0, KERNEL_SIZE, KERNEL_SIZE, OUTPUT_DIM);
    6869
    0
      cmd.backend = CCV_NNC_BACKEND_CPU_REF;
    6870
    0
      assert(cmd.backend >= 0);
    6871
    0
      ccv_nnc_hint_t hint = ccv_nnc_hint_auto(cmd.info, b->info, a->info);
    6872
    0
      ccv_nnc_tensor_t* w = ccv_nnc_tensor_new(0, CPU_TENSOR_NHWC(32F, OUTPUT_DIM, KERNEL_SIZE, KERNEL_SIZE, INPUT_DIM), 0);
    6873
    0
      ccv_nnc_tensor_t* bias = ccv_nnc_tensor_new(0, CPU_TENSOR_NHWC(32F, INPUT_DIM), 0);
    6874
      // configure the inlets.
    6875
    0
      dsfmt_t dsfmt;
    6876
    0
      dsfmt_init_gen_rand(&dsfmt, 0);
    6877
    0
      int i;
    6878
    0
      for (i = 0; i < INPUT_DIM * KERNEL_SIZE * KERNEL_SIZE * OUTPUT_DIM; i++)
    6879
    0
        w->data.f32[i] = dsfmt_genrand_open_close(&dsfmt) / (INPUT_DIM * KERNEL_SIZE * KERNEL_SIZE);
    6880
    0
      for (i = 0; i < OUTPUT_SIZE * OUTPUT_SIZE * OUTPUT_DIM * ccv_max(1, BATCH_SIZE); i++)
    6881
    0
        a->data.f32[i] = dsfmt_genrand_open_close(&dsfmt);
    6882
    0
      for (i = 0; i < INPUT_DIM; i++)
    6883
    0
        bias->data.f32[i] = (float)i / INPUT_DIM;
    6884
      // Copy generated matrix values over to GPU.
    6885
    0
      ccv_nnc_tensor_t* ga = ccv_nnc_tensor_new(0, GPU_TENSOR_NHWC(000, 32F, BATCH_SIZE, OUTPUT_SIZE, OUTPUT_SIZE, OUTPUT_DIM), 0);
    6886
    0
      ccv_nnc_tensor_t* gw = ccv_nnc_tensor_new(0, GPU_TENSOR_NHWC(000, 32F, OUTPUT_DIM, KERNEL_SIZE, KERNEL_SIZE, INPUT_DIM), 0);
    6887
    0
      ccv_nnc_tensor_t* gwo = ccv_nnc_tensor_new(0, GPU_TENSOR_NCHW(000, 32F, OUTPUT_DIM, INPUT_DIM, KERNEL_SIZE, KERNEL_SIZE), 0);
    6888
    0
      ccv_nnc_tensor_t* gbias = ccv_nnc_tensor_new(0, GPU_TENSOR_NHWC(000, 32F, INPUT_DIM), 0);
    6889
    0
      ccv_nnc_cmd_t move = CMD_DATA_TRANSFER_FORWARD();
    6890
    0
      move.backend = CCV_NNC_BACKEND_MPS;
    6891
    0
      assert(move.backend >= 0);
    6892
    0
      ccv_nnc_cmd_exec(move, ccv_nnc_no_hint, 0, TENSOR_LIST(a, w, bias), TENSOR_LIST(ga, gw, gbias), 0);
    6893
    0
      ccv_nnc_cmd_exec(cmd, hint, 0, TENSOR_LIST(a, w, bias), TENSOR_LIST(b), 0);
    6894
    0
      ccv_nnc_tensor_t* gc = ccv_nnc_tensor_new(0, GPU_TENSOR_NHWC(000, 32F, BATCH_SIZE, INPUT_SIZE, INPUT_SIZE, INPUT_DIM), 0);
    6895
    6896
    0
      ccv_nnc_cmd_t transform = CMD_FORMAT_TRANSFORM_FORWARD();
    6897
    0
      transform.backend = CCV_NNC_BACKEND_MPS;
    6898
    0
      assert(transform.backend >= 0);
    6899
    0
      ccv_nnc_stream_context_t* stream_context = ccv_nnc_stream_context_new(CCV_STREAM_CONTEXT_GPU);
    6900
    0
      ccv_nnc_cmd_exec(transform, ccv_nnc_no_hint, 0, TENSOR_LIST(gw), TENSOR_LIST(gwo), stream_context);
    6901
    0
      ccv_nnc_stream_context_wait(stream_context);
    6902
    0
      ccv_nnc_tensor_free(gw);
    6903
    6904
    0
      cmd.backend = CCV_NNC_BACKEND_MPS;
    6905
    0
      assert(cmd.backend >= 0);
    6906
    0
      cmd.algorithm = -1;
    6907
    0
      cmd = ccv_nnc_cmd_autotune(cmd, 1 * 1024 * 1024 * 1024, hint, 0, TENSOR_LIST(ga, gwo, gbias), TENSOR_LIST(gc), stream_context);
    6908
    0
      assert(CCV_NNC_EXEC_SUCCESS == ccv_nnc_cmd_exec(cmd, hint, 0, TENSOR_LIST(ga, gwo, gbias), TENSOR_LIST(gc), stream_context));
    6909
    0
      ccv_nnc_stream_context_wait(stream_context);
    6910
    0
      ccv_nnc_stream_context_free(stream_context);
    6911
    0
      ccv_nnc_tensor_t* c = ccv_nnc_tensor_new(0, CPU_TENSOR_NHWC(32F, BATCH_SIZE, INPUT_SIZE, INPUT_SIZE, INPUT_DIM), 0);
    6912
    0
      ccv_nnc_cmd_exec(move, ccv_nnc_no_hint, 0, TENSOR_LIST(gc), TENSOR_LIST(c), 0);
    6913
    0
      REQUIRE_ARRAY_EQ_WITH_TOLERANCE(float, b->data.f32, c->data.f32, BATCH_SIZE * INPUT_DIM * INPUT_SIZE * INPUT_SIZE, 1e-4, "output from mps should match from CPU");
    6914
    0
      ccv_nnc_tensor_free(c);
    6915
    0
      ccv_nnc_tensor_free(gc);
    6916
    0
      ccv_nnc_tensor_free(bias);
    6917
    0
      ccv_nnc_tensor_free(w);
    6918
    0
      ccv_nnc_tensor_free(b);
    6919
    0
      ccv_nnc_tensor_free(a);
    6920
    0
      ccv_nnc_tensor_free(gbias);
    6921
    0
      ccv_nnc_tensor_free(gwo);
    6922
    0
      ccv_nnc_tensor_free(ga);
    6923
    0
    }
    6924
    6925
    TEST_CASE("mps forward convolution transpose in nchw format")
    6926
    1
    {
    6927
    1
      GUARD_ELSE_RETURN(ccv_nnc_cmd_ok(CCV_NNC_CONVOLUTION_TRANSPOSE_FORWARD, CCV_NNC_BACKEND_MPS));
    6928
    0
      ccv_nnc_tensor_t* a = ccv_nnc_tensor_new(0, CPU_TENSOR_NCHW(32F, BATCH_SIZE, OUTPUT_DIM, OUTPUT_SIZE, OUTPUT_SIZE), 0);
    6929
    0
      ccv_nnc_tensor_t* b = ccv_nnc_tensor_new(0, CPU_TENSOR_NCHW(32F, BATCH_SIZE, INPUT_DIM, INPUT_SIZE, INPUT_SIZE), 0);
    6930
    0
      ccv_nnc_cmd_t cmd = CMD_CONVOLUTION_TRANSPOSE_FORWARD(1, INPUT_DIM, 0, KERNEL_SIZE, KERNEL_SIZE, OUTPUT_DIM);
    6931
    0
      cmd.backend = CCV_NNC_BACKEND_CPU_REF;
    6932
    0
      assert(cmd.backend >= 0);
    6933
    0
      ccv_nnc_hint_t hint = ccv_nnc_hint_auto(cmd.info, b->info, a->info);
    6934
    0
      ccv_nnc_tensor_t* w = ccv_nnc_tensor_new(0, CPU_TENSOR_NCHW(32F, OUTPUT_DIM, INPUT_DIM, KERNEL_SIZE, KERNEL_SIZE), 0);
    6935
    0
      ccv_nnc_tensor_t* bias = ccv_nnc_tensor_new(0, CPU_TENSOR_NCHW(32F, INPUT_DIM), 0);
    6936
      // configure the inlets.
    6937
    0
      dsfmt_t dsfmt;
    6938
    0
      dsfmt_init_gen_rand(&dsfmt, 0);
    6939
    0
      int i;
    6940
    0
      for (i = 0; i < INPUT_DIM * KERNEL_SIZE * KERNEL_SIZE * OUTPUT_DIM; i++)
    6941
    0
        w->data.f32[i] = dsfmt_genrand_open_close(&dsfmt) / (INPUT_DIM * KERNEL_SIZE * KERNEL_SIZE);
    6942
    0
      for (i = 0; i < OUTPUT_SIZE * OUTPUT_SIZE * OUTPUT_DIM * ccv_max(1, BATCH_SIZE); i++)
    6943
    0
        a->data.f32[i] = dsfmt_genrand_open_close(&dsfmt);
    6944
    0
      for (i = 0; i < INPUT_DIM; i++)
    6945
    0
        bias->data.f32[i] = (float)i / INPUT_DIM;
    6946
      // Copy generated matrix values over to GPU.
    6947
    0
      ccv_nnc_tensor_t* ga = ccv_nnc_tensor_new(0, GPU_TENSOR_NCHW(000, 32F, BATCH_SIZE, OUTPUT_DIM, OUTPUT_SIZE, OUTPUT_SIZE), 0);
    6948
    0
      ccv_nnc_tensor_t* gw = ccv_nnc_tensor_new(0, GPU_TENSOR_NCHW(000, 32F, OUTPUT_DIM, INPUT_DIM, KERNEL_SIZE, KERNEL_SIZE), 0);
    6949
    0
      ccv_nnc_tensor_t* gbias = ccv_nnc_tensor_new(0, GPU_TENSOR_NCHW(000, 32F, INPUT_DIM), 0);
    6950
    0
      ccv_nnc_cmd_t move = CMD_DATA_TRANSFER_FORWARD();
    6951
    0
      move.backend = CCV_NNC_BACKEND_MPS;
    6952
    0
      assert(move.backend >= 0);
    6953
    0
      ccv_nnc_cmd_exec(move, ccv_nnc_no_hint, 0, TENSOR_LIST(a, w, bias), TENSOR_LIST(ga, gw, gbias), 0);
    6954
    0
      ccv_nnc_cmd_exec(cmd, hint, 0, TENSOR_LIST(a, w, bias), TENSOR_LIST(b), 0);
    6955
    0
      ccv_nnc_tensor_t* gc = ccv_nnc_tensor_new(0, GPU_TENSOR_NCHW(000, 32F, BATCH_SIZE, INPUT_DIM, INPUT_SIZE, INPUT_SIZE), 0);
    6956
    6957
    0
      ccv_nnc_cmd_t transform = CMD_FORMAT_TRANSFORM_FORWARD();
    6958
    0
      transform.backend = CCV_NNC_BACKEND_MPS;
    6959
    0
      assert(transform.backend >= 0);
    6960
    0
      cmd.backend = CCV_NNC_BACKEND_MPS;
    6961
    0
      assert(cmd.backend >= 0);
    6962
    0
      cmd.algorithm = -1;
    6963
    0
      cmd = ccv_nnc_cmd_autotune(cmd, 1 * 1024 * 1024 * 1024, hint, 0, TENSOR_LIST(ga, gw, gbias), TENSOR_LIST(gc), 0);
    6964
    0
      assert(CCV_NNC_EXEC_SUCCESS == ccv_nnc_cmd_exec(cmd, hint, 0, TENSOR_LIST(ga, gw, gbias), TENSOR_LIST(gc), 0));
    6965
    0
      ccv_nnc_tensor_t* c = ccv_nnc_tensor_new(0, CPU_TENSOR_NCHW(32F, BATCH_SIZE, INPUT_DIM, INPUT_SIZE, INPUT_SIZE), 0);
    6966
    0
      ccv_nnc_cmd_exec(move, ccv_nnc_no_hint, 0, TENSOR_LIST(gc), TENSOR_LIST(c), 0);
    6967
    0
      REQUIRE_ARRAY_EQ_WITH_TOLERANCE(float, b->data.f32, c->data.f32, BATCH_SIZE * INPUT_DIM * INPUT_SIZE * INPUT_SIZE, 1e-5, "output from mps should match from CPU");
    6968
    0
      ccv_nnc_tensor_free(c);
    6969
    0
      ccv_nnc_tensor_free(gc);
    6970
    0
      ccv_nnc_tensor_free(bias);
    6971
    0
      ccv_nnc_tensor_free(w);
    6972
    0
      ccv_nnc_tensor_free(b);
    6973
    0
      ccv_nnc_tensor_free(a);
    6974
    0
      ccv_nnc_tensor_free(gbias);
    6975
    0
      ccv_nnc_tensor_free(gw);
    6976
    0
      ccv_nnc_tensor_free(ga);
    6977
    0
    }
    6978
    6979
    TEST_CASE("mps forward convolution transpose in half precision")
    6980
    1
    {
    6981
    1
      GUARD_ELSE_RETURN(ccv_nnc_cmd_ok(CCV_NNC_CONVOLUTION_TRANSPOSE_FORWARD, CCV_NNC_BACKEND_MPS));
    6982
    0
      ccv_nnc_tensor_t* a = ccv_nnc_tensor_new(0, CPU_TENSOR_NHWC(32F, BATCH_SIZE, OUTPUT_SIZE, OUTPUT_SIZE, OUTPUT_DIM), 0);
    6983
    0
      ccv_nnc_tensor_t* b = ccv_nnc_tensor_new(0, CPU_TENSOR_NHWC(32F, BATCH_SIZE, INPUT_SIZE, INPUT_SIZE, INPUT_DIM), 0);
    6984
    0
      ccv_nnc_cmd_t cmd = CMD_CONVOLUTION_TRANSPOSE_FORWARD(1, INPUT_DIM, 0, KERNEL_SIZE, KERNEL_SIZE, OUTPUT_DIM);
    6985
    0
      cmd.backend = CCV_NNC_BACKEND_CPU_REF;
    6986
    0
      assert(cmd.backend >= 0);
    6987
    0
      ccv_nnc_hint_t hint = ccv_nnc_hint_auto(cmd.info, b->info, a->info);
    6988
    0
      ccv_nnc_tensor_t* w = ccv_nnc_tensor_new(0, CPU_TENSOR_NHWC(32F, OUTPUT_DIM, KERNEL_SIZE, KERNEL_SIZE, INPUT_DIM), 0);
    6989
    0
      ccv_nnc_tensor_t* bias = ccv_nnc_tensor_new(0, CPU_TENSOR_NHWC(32F, INPUT_DIM), 0);
    6990
      // configure the inlets.
    6991
    0
      dsfmt_t dsfmt;
    6992
    0
      dsfmt_init_gen_rand(&dsfmt, 0);
    6993
    0
      int i;
    6994
    0
      for (i = 0; i < INPUT_DIM * KERNEL_SIZE * KERNEL_SIZE * OUTPUT_DIM; i++)
    6995
    0
        w->data.f32[i] = dsfmt_genrand_open_close(&dsfmt) / (INPUT_DIM * KERNEL_SIZE * KERNEL_SIZE);
    6996
    0
      for (i = 0; i < OUTPUT_SIZE * OUTPUT_SIZE * OUTPUT_DIM * ccv_max(1, BATCH_SIZE); i++)
    6997
    0
        a->data.f32[i] = dsfmt_genrand_open_close(&dsfmt);
    6998
    0
      for (i = 0; i < INPUT_DIM; i++)
    6999
    0
        bias->data.f32[i] = (float)i / INPUT_DIM;
    7000
    0
      ccv_nnc_tensor_t* a1 = ccv_nnc_tensor_new(0, CPU_TENSOR_NHWC(16F, BATCH_SIZE, OUTPUT_SIZE, OUTPUT_SIZE, OUTPUT_DIM), 0);
    7001
    0
      ccv_nnc_tensor_t* w1 = ccv_nnc_tensor_new(0, CPU_TENSOR_NHWC(16F, OUTPUT_DIM, KERNEL_SIZE, KERNEL_SIZE, INPUT_DIM), 0);
    7002
    0
      ccv_nnc_tensor_t* bias1 = ccv_nnc_tensor_new(0, CPU_TENSOR_NHWC(16F, INPUT_DIM), 0);
    7003
    0
      ccv_nnc_cmd_exec(CMD_DATATYPE_CONVERSION_FORWARD(), ccv_nnc_no_hint, 0, TENSOR_LIST(a, w, bias), TENSOR_LIST(a1, w1, bias1), 0);
    7004
      // Copy generated matrix values over to GPU.
    7005
    0
      ccv_nnc_tensor_t* ga = ccv_nnc_tensor_new(0, GPU_TENSOR_NHWC(000, 16F, BATCH_SIZE, OUTPUT_SIZE, OUTPUT_SIZE, OUTPUT_DIM), 0);
    7006
    0
      ccv_nnc_tensor_t* gw = ccv_nnc_tensor_new(0, GPU_TENSOR_NHWC(000, 16F, OUTPUT_DIM, KERNEL_SIZE, KERNEL_SIZE, INPUT_DIM), 0);
    7007
    0
      ccv_nnc_tensor_t* gwo = ccv_nnc_tensor_new(0, GPU_TENSOR_NCHW(000, 16F, OUTPUT_DIM, INPUT_DIM, KERNEL_SIZE, KERNEL_SIZE), 0);
    7008
    0
      ccv_nnc_tensor_t* gbias = ccv_nnc_tensor_new(0, GPU_TENSOR_NHWC(000, 16F, INPUT_DIM), 0);
    7009
    0
      ccv_nnc_cmd_exec(CMD_DATA_TRANSFER_FORWARD(), ccv_nnc_no_hint, 0, TENSOR_LIST(a1, w1, bias1), TENSOR_LIST(ga, gw, gbias), 0);
    7010
    0
      ccv_nnc_cmd_exec(cmd, hint, 0, TENSOR_LIST(a, w, bias), TENSOR_LIST(b), 0);
    7011
    0
      ccv_nnc_tensor_t* gc = ccv_nnc_tensor_new(0, GPU_TENSOR_NHWC(000, 16F, BATCH_SIZE, INPUT_SIZE, INPUT_SIZE, INPUT_DIM), 0);
    7012
    7013
    0
      ccv_nnc_cmd_t transform = CMD_FORMAT_TRANSFORM_FORWARD();
    7014
    0
      transform.backend = CCV_NNC_BACKEND_MPS;
    7015
    0
      assert(transform.backend >= 0);
    7016
    0
      ccv_nnc_stream_context_t* stream_context = ccv_nnc_stream_context_new(CCV_STREAM_CONTEXT_GPU);
    7017
    0
      ccv_nnc_cmd_exec(transform, ccv_nnc_no_hint, 0, TENSOR_LIST(gw), TENSOR_LIST(gwo), stream_context);
    7018
    0
      ccv_nnc_stream_context_wait(stream_context);
    7019
    0
      ccv_nnc_tensor_free(gw);
    7020
    7021
    0
      cmd.backend = CCV_NNC_BACKEND_MPS;
    7022
    0
      assert(cmd.backend >= 0);
    7023
    0
      cmd.algorithm = -1;
    7024
    0
      cmd = ccv_nnc_cmd_autotune(cmd, 512 * 1024 * 1024, hint, 0, TENSOR_LIST(ga, gwo, gbias), TENSOR_LIST(gc), stream_context);
    7025
    0
      assert(CCV_NNC_EXEC_SUCCESS == ccv_nnc_cmd_exec(cmd, hint, 0, TENSOR_LIST(ga, gwo, gbias), TENSOR_LIST(gc), stream_context));
    7026
    0
      ccv_nnc_stream_context_wait(stream_context);
    7027
    0
      ccv_nnc_stream_context_free(stream_context);
    7028
    0
      ccv_nnc_tensor_t* c1 = ccv_nnc_tensor_new(0, CPU_TENSOR_NHWC(16F, BATCH_SIZE, INPUT_SIZE, INPUT_SIZE, INPUT_DIM), 0);
    7029
    0
      ccv_nnc_cmd_exec(CMD_DATA_TRANSFER_FORWARD(), ccv_nnc_no_hint, 0, TENSOR_LIST(gc), TENSOR_LIST(c1), 0);
    7030
    0
      ccv_nnc_tensor_t* c = ccv_nnc_tensor_new(0, CPU_TENSOR_NHWC(32F, BATCH_SIZE, INPUT_SIZE, INPUT_SIZE, INPUT_DIM), 0);
    7031
    0
      ccv_nnc_cmd_exec(CMD_DATATYPE_CONVERSION_FORWARD(), ccv_nnc_no_hint, 0, TENSOR_LIST(c1), TENSOR_LIST(c), 0);
    7032
    0
      REQUIRE_ARRAY_EQ_WITH_TOLERANCE(float, b->data.f32, c->data.f32, BATCH_SIZE * INPUT_DIM * INPUT_SIZE * INPUT_SIZE, 5e-3, "output from mps should match from CPU");
    7033
    0
      ccv_nnc_tensor_free(c);
    7034
    0
      ccv_nnc_tensor_free(gc);
    7035
    0
      ccv_nnc_tensor_free(bias);
    7036
    0
      ccv_nnc_tensor_free(w);
    7037
    0
      ccv_nnc_tensor_free(b);
    7038
    0
      ccv_nnc_tensor_free(a);
    7039
    0
      ccv_nnc_tensor_free(c1);
    7040
    0
      ccv_nnc_tensor_free(bias1);
    7041
    0
      ccv_nnc_tensor_free(w1);
    7042
    0
      ccv_nnc_tensor_free(a1);
    7043
    0
      ccv_nnc_tensor_free(gbias);
    7044
    0
      ccv_nnc_tensor_free(gwo);
    7045
    0
      ccv_nnc_tensor_free(ga);
    7046
    0
    }
    7047
    7048
    TEST_CASE("compare tanh with mps")
    7049
    1
    {
    7050
    1
      GUARD_ELSE_RETURN(ccv_nnc_cmd_ok(CCV_NNC_TANH_FORWARD, CCV_NNC_BACKEND_MPS));
    7051
    0
      ccv_nnc_symbolic_graph_t* const symbolic_graph = ccv_nnc_symbolic_graph_new();
    7052
    0
      ccv_nnc_tensor_symbol_t a = ccv_nnc_tensor_symbol_new(symbolic_graph, GPU_TENSOR_NCHW(000, 32F, 20, 10), "a");
    7053
    0
      ccv_nnc_tensor_symbol_t b = ccv_nnc_tensor_symbol_new(symbolic_graph, GPU_TENSOR_NCHW(000, 32F, 20, 10), "b");
    7054
    0
      ccv_nnc_graph_exec_symbol_new(symbolic_graph, CMD_TANH_FORWARD(), TENSOR_SYMBOL_LIST(a), TENSOR_SYMBOL_LIST(b), "tanh");
    7055
    0
      ccv_nnc_graph_exec_symbol_autogen(symbolic_graph, 0, 0, CCV_NNC_AUTOGEN_ALL_EXECS | CCV_NNC_AUTOGEN_SOURCES_AND_DESTINATIONS);
    7056
    0
      SYMBOLIC_GRAPH_GEN(symbolic_graph, CCV_NNC_LONG_DOT_GRAPH);
    7057
    0
      ccv_nnc_graph_t* graph = 0;
    7058
    0
      ccv_nnc_tensor_arena_t* tensor_arena = 0;
    7059
    0
      ccv_nnc_graph_exec_arena_t* graph_exec_arena = 0;
    7060
    0
      ccv_nnc_symbolic_graph_compile(symbolic_graph, ccv_nnc_default_compile_params, 0, 0, 0, 0, SYMBOLIC_GRAPH_SOURCES(symbolic_graph), SYMBOLIC_GRAPH_DESTINATIONS(symbolic_graph), &graph, &tensor_arena, &graph_exec_arena);
    7061
    0
      GRAPH_GEN(graph, CCV_NNC_LONG_DOT_GRAPH);
    7062
    0
      ccv_nnc_tensor_t* const x_tensor = ccv_nnc_tensor_new(0, CPU_TENSOR_NCHW(32F, 20, 10), 0);
    7063
    0
      dsfmt_t dsfmt;
    7064
    0
      dsfmt_init_gen_rand(&dsfmt, 0);
    7065
    0
      int i;
    7066
    0
      for (i = 0; i < 20 * 10; i++)
    7067
    0
        x_tensor->data.f32[i] = dsfmt_genrand_open_close(&dsfmt);
    7068
    0
      ccv_nnc_tensor_t* const a_tensor = ccv_nnc_tensor_from_symbol(tensor_arena, a);
    7069
    0
      ccv_nnc_cmd_exec(CMD_DATA_TRANSFER_FORWARD(), ccv_nnc_no_hint, 0, TENSOR_LIST(x_tensor), TENSOR_LIST(a_tensor), 0);
    7070
    0
      ccv_nnc_graph_run(graph, 0, TRAVERSE_FULL, 0, 0);
    7071
    0
      ccv_nnc_tensor_t* const y_tensor = ccv_nnc_tensor_new(0, CPU_TENSOR_NCHW(32F, 20, 10), 0);
    7072
    0
      ccv_nnc_tensor_t* const b_tensor = ccv_nnc_tensor_from_symbol(tensor_arena, b);
    7073
    0
      ccv_nnc_cmd_exec(CMD_DATA_TRANSFER_FORWARD(), ccv_nnc_no_hint, 0, TENSOR_LIST(b_tensor), TENSOR_LIST(y_tensor), 0);
    7074
    0
      ccv_nnc_tensor_t* const ty = ccv_nnc_tensor_new(0, CPU_TENSOR_NCHW(32F, 20, 10), 0);
    7075
    0
      ccv_nnc_cmd_exec(CMD_TANH_FORWARD(), ccv_nnc_no_hint, 0, TENSOR_LIST(x_tensor), TENSOR_LIST(ty), 0);
    7076
    0
      REQUIRE_TENSOR_EQ(ty, y_tensor, "tanh from mps should match from CPU");
    7077
    0
      ccv_nnc_tensor_free(x_tensor);
    7078
    0
      ccv_nnc_tensor_free(y_tensor);
    7079
    0
      ccv_nnc_tensor_free(ty);
    7080
    0
      ccv_nnc_graph_free(graph);
    7081
    0
      ccv_nnc_tensor_arena_free(tensor_arena);
    7082
    0
      ccv_nnc_graph_exec_arena_free(graph_exec_arena);
    7083
    0
      ccv_nnc_symbolic_graph_free(symbolic_graph);
    7084
    0
    }
    7085
    7086
    TEST_CASE("compare tanh with mps in half precision")
    7087
    1
    {
    7088
    1
      GUARD_ELSE_RETURN(ccv_nnc_cmd_ok(CCV_NNC_TANH_FORWARD, CCV_NNC_BACKEND_MPS));
    7089
    0
      ccv_nnc_symbolic_graph_t* const symbolic_graph = ccv_nnc_symbolic_graph_new();
    7090
    0
      ccv_nnc_tensor_symbol_t a = ccv_nnc_tensor_symbol_new(symbolic_graph, GPU_TENSOR_NCHW(000, 16F, 20, 10), "a");
    7091
    0
      ccv_nnc_tensor_symbol_t b = ccv_nnc_tensor_symbol_new(symbolic_graph, GPU_TENSOR_NCHW(000, 16F, 20, 10), "b");
    7092
    0
      ccv_nnc_graph_exec_symbol_new(symbolic_graph, CMD_TANH_FORWARD(), TENSOR_SYMBOL_LIST(a), TENSOR_SYMBOL_LIST(b), "tanh");
    7093
    0
      ccv_nnc_graph_exec_symbol_autogen(symbolic_graph, 0, 0, CCV_NNC_AUTOGEN_ALL_EXECS | CCV_NNC_AUTOGEN_SOURCES_AND_DESTINATIONS);
    7094
    0
      SYMBOLIC_GRAPH_GEN(symbolic_graph, CCV_NNC_LONG_DOT_GRAPH);
    7095
    0
      ccv_nnc_graph_t* graph = 0;
    7096
    0
      ccv_nnc_tensor_arena_t* tensor_arena = 0;
    7097
    0
      ccv_nnc_graph_exec_arena_t* graph_exec_arena = 0;
    7098
    0
      ccv_nnc_symbolic_graph_compile(symbolic_graph, ccv_nnc_default_compile_params, 0, 0, 0, 0, SYMBOLIC_GRAPH_SOURCES(symbolic_graph), SYMBOLIC_GRAPH_DESTINATIONS(symbolic_graph), &graph, &tensor_arena, &graph_exec_arena);
    7099
    0
      GRAPH_GEN(graph, CCV_NNC_LONG_DOT_GRAPH);
    7100
    0
      ccv_nnc_tensor_t* const x_tensor = ccv_nnc_tensor_new(0, CPU_TENSOR_NCHW(32F, 20, 10), 0);
    7101
    0
      dsfmt_t dsfmt;
    7102
    0
      dsfmt_init_gen_rand(&dsfmt, 0);
    7103
    0
      int i;
    7104
    0
      for (i = 0; i < 20 * 10; i++)
    7105
    0
        x_tensor->data.f32[i] = dsfmt_genrand_open_close(&dsfmt);
    7106
    0
      ccv_nnc_tensor_t* const a_tensor = ccv_nnc_tensor_from_symbol(tensor_arena, a);
    7107
    0
      ccv_nnc_tensor_t* const x16_tensor = ccv_nnc_tensor_new(0, CPU_TENSOR_NCHW(16F, 20, 10), 0);
    7108
    0
      ccv_nnc_cmd_exec(CMD_DATATYPE_CONVERSION_FORWARD(), ccv_nnc_no_hint, 0, TENSOR_LIST(x_tensor), TENSOR_LIST(x16_tensor), 0);
    7109
    0
      ccv_nnc_cmd_exec(CMD_DATA_TRANSFER_FORWARD(), ccv_nnc_no_hint, 0, TENSOR_LIST(x16_tensor), TENSOR_LIST(a_tensor), 0);
    7110
    0
      ccv_nnc_graph_run(graph, 0, TRAVERSE_FULL, 0, 0);
    7111
    0
      ccv_nnc_tensor_t* const y16_tensor = ccv_nnc_tensor_new(0, CPU_TENSOR_NCHW(16F, 20, 10), 0);
    7112
    0
      ccv_nnc_tensor_t* const y_tensor = ccv_nnc_tensor_new(0, CPU_TENSOR_NCHW(32F, 20, 10), 0);
    7113
    0
      ccv_nnc_tensor_t* const b_tensor = ccv_nnc_tensor_from_symbol(tensor_arena, b);
    7114
    0
      ccv_nnc_cmd_exec(CMD_DATA_TRANSFER_FORWARD(), ccv_nnc_no_hint, 0, TENSOR_LIST(b_tensor), TENSOR_LIST(y16_tensor), 0);
    7115
    0
      ccv_nnc_cmd_exec(CMD_DATATYPE_CONVERSION_FORWARD(), ccv_nnc_no_hint, 0, TENSOR_LIST(y16_tensor), TENSOR_LIST(y_tensor), 0);
    7116
    0
      ccv_nnc_tensor_t* const ty = ccv_nnc_tensor_new(0, CPU_TENSOR_NCHW(32F, 20, 10), 0);
    7117
    0
      ccv_nnc_cmd_exec(CMD_TANH_FORWARD(), ccv_nnc_no_hint, 0, TENSOR_LIST(x_tensor), TENSOR_LIST(ty), 0);
    7118
    0
      REQUIRE_ARRAY_EQ_WITH_TOLERANCE(float, ty->data.f32, y_tensor->data.f32, 20 * 10, 1e-3, "tanh from mps should match from CPU");
    7119
    0
      ccv_nnc_tensor_free(x_tensor);
    7120
    0
      ccv_nnc_tensor_free(x16_tensor);
    7121
    0
      ccv_nnc_tensor_free(y16_tensor);
    7122
    0
      ccv_nnc_tensor_free(y_tensor);
    7123
    0
      ccv_nnc_tensor_free(ty);
    7124
    0
      ccv_nnc_graph_free(graph);
    7125
    0
      ccv_nnc_tensor_arena_free(tensor_arena);
    7126
    0
      ccv_nnc_graph_exec_arena_free(graph_exec_arena);
    7127
    0
      ccv_nnc_symbolic_graph_free(symbolic_graph);
    7128
    0
    }
    7129
    7130
    TEST_CASE("compare tanh gradient with mps")
    7131
    1
    {
    7132
    1
      GUARD_ELSE_RETURN(ccv_nnc_cmd_ok(CCV_NNC_TANH_FORWARD, CCV_NNC_BACKEND_MPS) &&
    7133
    1
        ccv_nnc_cmd_ok(CCV_NNC_TANH_BACKWARD, CCV_NNC_BACKEND_MPS));
    7134
    0
      ccv_nnc_symbolic_graph_t* const symbolic_graph = ccv_nnc_symbolic_graph_new();
    7135
    0
      ccv_nnc_tensor_symbol_t x = ccv_nnc_tensor_symbol_new(symbolic_graph, GPU_TENSOR_NHWC(000, 32F, 10, 100), "x");
    7136
    0
      ccv_nnc_tensor_symbol_t y = ccv_nnc_tensor_symbol_new(symbolic_graph, GPU_TENSOR_NHWC(000, 32F, 10, 100), "y");
    7137
    0
      ccv_nnc_graph_exec_symbol_new(symbolic_graph, CMD_TANH_FORWARD(), TENSOR_SYMBOL_LIST(x), TENSOR_SYMBOL_LIST(y), "tanh");
    7138
    0
      ccv_nnc_graph_exec_symbol_autogen(symbolic_graph, 0, 0, CCV_NNC_AUTOGEN_ALL_EXECS | CCV_NNC_AUTOGEN_SOURCES_AND_DESTINATIONS);
    7139
    0
      ccv_nnc_symbolic_graph_backward(symbolic_graph, TENSOR_SYMBOL_LIST(y), TENSOR_SYMBOL_LIST(x), SYMBOLIC_GRAPH_SOURCES(symbolic_graph), SYMBOLIC_GRAPH_DESTINATIONS(symbolic_graph));
    7140
    0
      ccv_nnc_graph_exec_symbol_autogen(symbolic_graph, 0, 0, CCV_NNC_AUTOGEN_ALL_EXECS | CCV_NNC_AUTOGEN_SOURCES_AND_DESTINATIONS);
    7141
    0
      SYMBOLIC_GRAPH_GEN(symbolic_graph, CCV_NNC_LONG_DOT_GRAPH);
    7142
    0
      ccv_nnc_tensor_symbol_t dy = ccv_nnc_tensor_symbol_for_backward(symbolic_graph, y);
    7143
    0
      ccv_nnc_tensor_symbol_t dx = ccv_nnc_tensor_symbol_for_backward(symbolic_graph, x);
    7144
    0
      ccv_nnc_tensor_t* const x_tensor = ccv_nnc_tensor_new(0, CPU_TENSOR_NHWC(32F, 10, 100), 0);
    7145
    0
      dsfmt_t dsfmt;
    7146
    0
      dsfmt_init_gen_rand(&dsfmt, 0);
    7147
    0
      int i;
    7148
    0
      for (i = 0; i < 10 * 100; i++)
    7149
    0
        x_tensor->data.f32[i] = dsfmt_genrand_open_close(&dsfmt);
    7150
    0
      ccv_nnc_tensor_t* const dy_tensor = ccv_nnc_tensor_new(0, CPU_TENSOR_NHWC(32F, 10, 100), 0);
    7151
    0
      for (i = 0; i < 10 * 100; i++)
    7152
    0
        dy_tensor->data.f32[i] = 0;
    7153
    0
      for (i = 0; i < 10; i++)
    7154
    0
        dy_tensor->data.f32[i * 100 + i] = 1;
    7155
    0
      ccv_nnc_tensor_t* const dyt = ccv_nnc_tensor_new(0, GPU_TENSOR_NHWC(000, 32F, 10, 100), 0);
    7156
    0
      ccv_nnc_cmd_exec(CMD_DATA_TRANSFER_FORWARD(), ccv_nnc_no_hint, 0, TENSOR_LIST(dy_tensor), TENSOR_LIST(dyt), 0);
    7157
    0
      ccv_nnc_graph_t* graph = 0;
    7158
    0
      ccv_nnc_tensor_arena_t* tensor_arena = 0;
    7159
    0
      ccv_nnc_graph_exec_arena_t* graph_exec_arena = 0;
    7160
    0
      ccv_nnc_symbolic_graph_compile(symbolic_graph, ccv_nnc_default_compile_params, TENSOR_BIND_MAP(KV(dy, dyt)), TENSOR_SYMBOL_LIST(y), SYMBOLIC_GRAPH_SOURCES(symbolic_graph), SYMBOLIC_GRAPH_DESTINATIONS(symbolic_graph), &graph, &tensor_arena, &graph_exec_arena);
    7161
    0
      GRAPH_GEN(graph, CCV_NNC_LONG_DOT_GRAPH);
    7162
    0
      ccv_nnc_tensor_t* const xt = ccv_nnc_tensor_from_symbol(tensor_arena, x);
    7163
    0
      ccv_nnc_cmd_exec(CMD_DATA_TRANSFER_FORWARD(), ccv_nnc_no_hint, 0, TENSOR_LIST(x_tensor), TENSOR_LIST(xt), 0);
    7164
    0
      ccv_nnc_graph_run(graph, 0, TRAVERSE_FULL, 0, 0);
    7165
    0
      ccv_nnc_tensor_t* const dx_tensor = ccv_nnc_tensor_new(0, CPU_TENSOR_NHWC(32F, 10, 100), 0);
    7166
    0
      ccv_nnc_tensor_t* const dxt = ccv_nnc_tensor_from_symbol(tensor_arena, dx);
    7167
    0
      ccv_nnc_tensor_t* const y_tensor = ccv_nnc_tensor_new(0, CPU_TENSOR_NHWC(32F, 10, 100), 0);
    7168
    0
      ccv_nnc_tensor_t* const yt = ccv_nnc_tensor_from_symbol(tensor_arena, y);
    7169
    0
      ccv_nnc_cmd_exec(CMD_DATA_TRANSFER_FORWARD(), ccv_nnc_no_hint, 0, TENSOR_LIST(dxt), TENSOR_LIST(dx_tensor), 0);
    7170
    0
      ccv_nnc_cmd_exec(CMD_DATA_TRANSFER_FORWARD(), ccv_nnc_no_hint, 0, TENSOR_LIST(yt), TENSOR_LIST(y_tensor), 0);
    7171
    0
      ccv_nnc_tensor_t* const ty_tensor = ccv_nnc_tensor_new(0, CPU_TENSOR_NHWC(32F, 10, 100), 0);
    7172
    0
      ccv_nnc_cmd_exec(CMD_TANH_FORWARD(), ccv_nnc_no_hint, 0, TENSOR_LIST(x_tensor), TENSOR_LIST(ty_tensor), 0);
    7173
    0
      REQUIRE_TENSOR_EQ(ty_tensor, y_tensor, "forward pass should match");
    7174
    0
      ccv_nnc_tensor_t* const tdx_tensor = ccv_nnc_tensor_new(0, CPU_TENSOR_NHWC(32F, 10, 100), 0);
    7175
    0
      ccv_nnc_cmd_exec(CMD_TANH_BACKWARD(), ccv_nnc_no_hint, 0, TENSOR_LIST(dy_tensor, 0, ty_tensor), TENSOR_LIST(tdx_tensor), 0);
    7176
    0
      REQUIRE_TENSOR_EQ(tdx_tensor, dx_tensor, "backward pass should match");
    7177
    0
      ccv_nnc_tensor_free(x_tensor);
    7178
    0
      ccv_nnc_tensor_free(y_tensor);
    7179
    0
      ccv_nnc_tensor_free(dx_tensor);
    7180
    0
      ccv_nnc_tensor_free(dy_tensor);
    7181
    0
      ccv_nnc_tensor_free(ty_tensor);
    7182
    0
      ccv_nnc_tensor_free(tdx_tensor);
    7183
    0
      ccv_nnc_tensor_free(dyt);
    7184
    0
      ccv_nnc_graph_free(graph);
    7185
    0
      ccv_nnc_tensor_arena_free(tensor_arena);
    7186
    0
      ccv_nnc_graph_exec_arena_free(graph_exec_arena);
    7187
    0
      ccv_nnc_symbolic_graph_free(symbolic_graph);
    7188
    0
    }
    7189
    7190
    TEST_CASE("compare tanh gradient with mps in half precision")
    7191
    1
    {
    7192
    1
      GUARD_ELSE_RETURN(ccv_nnc_cmd_ok(CCV_NNC_TANH_FORWARD, CCV_NNC_BACKEND_MPS) &&
    7193
    1
        ccv_nnc_cmd_ok(CCV_NNC_TANH_BACKWARD, CCV_NNC_BACKEND_MPS));
    7194
    0
      ccv_nnc_symbolic_graph_t* const symbolic_graph = ccv_nnc_symbolic_graph_new();
    7195
    0
      ccv_nnc_tensor_symbol_t x = ccv_nnc_tensor_symbol_new(symbolic_graph, GPU_TENSOR_NHWC(000, 16F, 10, 100), "x");
    7196
    0
      ccv_nnc_tensor_symbol_t y = ccv_nnc_tensor_symbol_new(symbolic_graph, GPU_TENSOR_NHWC(000, 16F, 10, 100), "y");
    7197
    0
      ccv_nnc_graph_exec_symbol_new(symbolic_graph, CMD_TANH_FORWARD(), TENSOR_SYMBOL_LIST(x), TENSOR_SYMBOL_LIST(y), "tanh");
    7198
    0
      ccv_nnc_graph_exec_symbol_autogen(symbolic_graph, 0, 0, CCV_NNC_AUTOGEN_ALL_EXECS | CCV_NNC_AUTOGEN_SOURCES_AND_DESTINATIONS);
    7199
    0
      ccv_nnc_symbolic_graph_backward(symbolic_graph, TENSOR_SYMBOL_LIST(y), TENSOR_SYMBOL_LIST(x), SYMBOLIC_GRAPH_SOURCES(symbolic_graph), SYMBOLIC_GRAPH_DESTINATIONS(symbolic_graph));
    7200
    0
      ccv_nnc_graph_exec_symbol_autogen(symbolic_graph, 0, 0, CCV_NNC_AUTOGEN_ALL_EXECS | CCV_NNC_AUTOGEN_SOURCES_AND_DESTINATIONS);
    7201
    0
      SYMBOLIC_GRAPH_GEN(symbolic_graph, CCV_NNC_LONG_DOT_GRAPH);
    7202
    0
      ccv_nnc_tensor_symbol_t dy = ccv_nnc_tensor_symbol_for_backward(symbolic_graph, y);
    7203
    0
      ccv_nnc_tensor_symbol_t dx = ccv_nnc_tensor_symbol_for_backward(symbolic_graph, x);
    7204
    0
      ccv_nnc_tensor_t* const x_tensor = ccv_nnc_tensor_new(0, CPU_TENSOR_NHWC(32F, 10, 100), 0);
    7205
    0
      dsfmt_t dsfmt;
    7206
    0
      dsfmt_init_gen_rand(&dsfmt, 0);
    7207
    0
      int i;
    7208
    0
      for (i = 0; i < 10 * 100; i++)
    7209
    0
        x_tensor->data.f32[i] = dsfmt_genrand_open_close(&dsfmt);
    7210
    0
      ccv_nnc_tensor_t* const dy_tensor = ccv_nnc_tensor_new(0, CPU_TENSOR_NHWC(32F, 10, 100), 0);
    7211
    0
      for (i = 0; i < 10 * 100; i++)
    7212
    0
        dy_tensor->data.f32[i] = 0;
    7213
    0
      for (i = 0; i < 10; i++)
    7214
    0
        dy_tensor->data.f32[i * 100 + i] = 1;
    7215
    0
      ccv_nnc_tensor_t* const dy16_tensor = ccv_nnc_tensor_new(0, CPU_TENSOR_NHWC(16F, 10, 100), 0);
    7216
    0
      ccv_nnc_tensor_t* const dyt = ccv_nnc_tensor_new(0, GPU_TENSOR_NHWC(000, 16F, 10, 100), 0);
    7217
    0
      ccv_nnc_cmd_exec(CMD_DATATYPE_CONVERSION_FORWARD(), ccv_nnc_no_hint, 0, TENSOR_LIST(dy_tensor), TENSOR_LIST(dy16_tensor), 0);
    7218
    0
      ccv_nnc_cmd_exec(CMD_DATA_TRANSFER_FORWARD(), ccv_nnc_no_hint, 0, TENSOR_LIST(dy16_tensor), TENSOR_LIST(dyt), 0);
    7219
    0
      ccv_nnc_graph_t* graph = 0;
    7220
    0
      ccv_nnc_tensor_arena_t* tensor_arena = 0;
    7221
    0
      ccv_nnc_graph_exec_arena_t* graph_exec_arena = 0;
    7222
    0
      ccv_nnc_symbolic_graph_compile(symbolic_graph, ccv_nnc_default_compile_params, TENSOR_BIND_MAP(KV(dy, dyt)), TENSOR_SYMBOL_LIST(y), SYMBOLIC_GRAPH_SOURCES(symbolic_graph), SYMBOLIC_GRAPH_DESTINATIONS(symbolic_graph), &graph, &tensor_arena, &graph_exec_arena);
    7223
    0
      GRAPH_GEN(graph, CCV_NNC_LONG_DOT_GRAPH);
    7224
    0
      ccv_nnc_tensor_t* const xt = ccv_nnc_tensor_from_symbol(tensor_arena, x);
    7225
    0
      ccv_nnc_tensor_t* const x16_tensor = ccv_nnc_tensor_new(0, CPU_TENSOR_NHWC(16F, 10, 100), 0);
    7226
    0
      ccv_nnc_cmd_exec(CMD_DATATYPE_CONVERSION_FORWARD(), ccv_nnc_no_hint, 0, TENSOR_LIST(x_tensor), TENSOR_LIST(x16_tensor), 0);
    7227
    0
      ccv_nnc_cmd_exec(CMD_DATA_TRANSFER_FORWARD(), ccv_nnc_no_hint, 0, TENSOR_LIST(x16_tensor), TENSOR_LIST(xt), 0);
    7228
    0
      ccv_nnc_graph_run(graph, 0, TRAVERSE_FULL, 0, 0);
    7229
    0
      ccv_nnc_tensor_t* const dx16_tensor = ccv_nnc_tensor_new(0, CPU_TENSOR_NHWC(16F, 10, 100), 0);
    7230
    0
      ccv_nnc_tensor_t* const dx_tensor = ccv_nnc_tensor_new(0, CPU_TENSOR_NHWC(32F, 10, 100), 0);
    7231
    0
      ccv_nnc_tensor_t* const dxt = ccv_nnc_tensor_from_symbol(tensor_arena, dx);
    7232
    0
      ccv_nnc_tensor_t* const y16_tensor = ccv_nnc_tensor_new(0, CPU_TENSOR_NHWC(16F, 10, 100), 0);
    7233
    0
      ccv_nnc_tensor_t* const y_tensor = ccv_nnc_tensor_new(0, CPU_TENSOR_NHWC(32F, 10, 100), 0);
    7234
    0
      ccv_nnc_tensor_t* const yt = ccv_nnc_tensor_from_symbol(tensor_arena, y);
    7235
    0
      ccv_nnc_cmd_exec(CMD_DATA_TRANSFER_FORWARD(), ccv_nnc_no_hint, 0, TENSOR_LIST(dxt), TENSOR_LIST(dx16_tensor), 0);
    7236
    0
      ccv_nnc_cmd_exec(CMD_DATATYPE_CONVERSION_FORWARD(), ccv_nnc_no_hint, 0, TENSOR_LIST(dx16_tensor), TENSOR_LIST(dx_tensor), 0);
    7237
    0
      ccv_nnc_cmd_exec(CMD_DATA_TRANSFER_FORWARD(), ccv_nnc_no_hint, 0, TENSOR_LIST(yt), TENSOR_LIST(y16_tensor), 0);
    7238
    0
      ccv_nnc_cmd_exec(CMD_DATATYPE_CONVERSION_FORWARD(), ccv_nnc_no_hint, 0, TENSOR_LIST(y16_tensor), TENSOR_LIST(y_tensor), 0);
    7239
    0
      ccv_nnc_tensor_t* const ty_tensor = ccv_nnc_tensor_new(0, CPU_TENSOR_NHWC(32F, 10, 100), 0);
    7240
    0
      ccv_nnc_cmd_exec(CMD_TANH_FORWARD(), ccv_nnc_no_hint, 0, TENSOR_LIST(x_tensor), TENSOR_LIST(ty_tensor), 0);
    7241
    0
      REQUIRE_ARRAY_EQ_WITH_TOLERANCE(float, ty_tensor->data.f32, y_tensor->data.f32, 10 * 100, 1e-3, "forward pass should match");
    7242
    0
      ccv_nnc_tensor_t* const tdx_tensor = ccv_nnc_tensor_new(0, CPU_TENSOR_NHWC(32F, 10, 100), 0);
    7243
    0
      ccv_nnc_cmd_exec(CMD_TANH_BACKWARD(), ccv_nnc_no_hint, 0, TENSOR_LIST(dy_tensor, 0, ty_tensor), TENSOR_LIST(tdx_tensor), 0);
    7244
    0
      REQUIRE_ARRAY_EQ_WITH_TOLERANCE(float, tdx_tensor->data.f32, dx_tensor->data.f32, 10 * 100, 1e-3, "backward pass should match");
    7245
    0
      ccv_nnc_tensor_free(x_tensor);
    7246
    0
      ccv_nnc_tensor_free(x16_tensor);
    7247
    0
      ccv_nnc_tensor_free(y_tensor);
    7248
    0
      ccv_nnc_tensor_free(y16_tensor);
    7249
    0
      ccv_nnc_tensor_free(dx_tensor);
    7250
    0
      ccv_nnc_tensor_free(dx16_tensor);
    7251
    0
      ccv_nnc_tensor_free(dy_tensor);
    7252
    0
      ccv_nnc_tensor_free(dy16_tensor);
    7253
    0
      ccv_nnc_tensor_free(ty_tensor);
    7254
    0
      ccv_nnc_tensor_free(tdx_tensor);
    7255
    0
      ccv_nnc_tensor_free(dyt);
    7256
    0
      ccv_nnc_graph_free(graph);
    7257
    0
      ccv_nnc_tensor_arena_free(tensor_arena);
    7258
    0
      ccv_nnc_graph_exec_arena_free(graph_exec_arena);
    7259
    0
      ccv_nnc_symbolic_graph_free(symbolic_graph);
    7260
    0
    }
    7261
    7262
    #include "case_main.h"