## Coverage Report

#### Created: 2026-05-04 15:30

next uncovered line (L), next uncovered region (R), next uncovered branch (B)

    /home/liu/actions-runner/_work/ccv/ccv/test/int/nnc/cublas.tests.c

    Line
    Count
    Source
    1
    #include "case.h"
    2
    #include "ccv_case.h"
    3
    #include "ccv_nnc_case.h"
    4
    #include <ccv.h>
    5
    #include <nnc/ccv_nnc.h>
    6
    #include <nnc/ccv_nnc_easy.h>
    7
    #include <3rdparty/dsfmt/dSFMT.h>
    8
    9
    TEST_SETUP()
    10
    {
    11
      ccv_nnc_init();
    12
    }
    13
    14
    TEST_CASE("gemm no transpose")
    15
    1
    {
    16
    1
      GUARD_ELSE_RETURN(ccv_nnc_cmd_ok(CCV_NNC_GEMM_FORWARD, CCV_NNC_BACKEND_GPU_CUBLAS));
    17
    1
      float ap[] = {
    18
    1
        1, 2,
    19
    1
        3, 4,
    20
    1
        5, 6,
    21
    1
        7, 8,
    22
    1
      };
    23
    1
      ccv_nnc_tensor_t* const a = ccv_nnc_tensor_new(ap, CPU_TENSOR_NHWC(32F, 4, 2), 0);
    24
    1
      float bp[] = {
    25
    1
        7, 8, 9,
    26
    1
        10, 11, 12,
    27
    1
      };
    28
    1
      ccv_nnc_tensor_t* const b = ccv_nnc_tensor_new(bp, CPU_TENSOR_NHWC(32F, 2, 3), 0);
    29
    1
      ccv_nnc_tensor_t* const c = ccv_nnc_tensor_new(0, CPU_TENSOR_NHWC(32F, 4, 3), 0);
    30
    1
      ccv_nnc_tensor_t* ga = ccv_nnc_tensor_new(0, GPU_TENSOR_NHWC(000, 32F, 4, 2), 0);
    31
    1
      ccv_nnc_tensor_t* gb = ccv_nnc_tensor_new(0, GPU_TENSOR_NHWC(000, 32F, 2, 3), 0);
    32
    1
      ccv_nnc_tensor_t* gc = ccv_nnc_tensor_new(0, GPU_TENSOR_NHWC(000, 32F, 4, 3), 0);
    33
    1
      ccv_nnc_cmd_exec(CMD_DATA_TRANSFER_FORWARD(), ccv_nnc_no_hint, 0, TENSOR_LIST(a, b), TENSOR_LIST(ga, gb), 0);
    34
    1
      ccv_nnc_cmd_exec(CMD_GEMM_FORWARD(), ccv_nnc_no_hint, 0, TENSOR_LIST(ga, gb), TENSOR_LIST(gc), 0);
    35
    1
      ccv_nnc_cmd_exec(CMD_DATA_TRANSFER_FORWARD(), ccv_nnc_no_hint, 0, TENSOR_LIST(gc), TENSOR_LIST(c), 0);
    36
    1
      float ctp[] = {
    37
    1
        1 * 7 + 2 * 10, 1 * 8 + 2 * 11, 1 * 9 + 2 * 12,
    38
    1
        3 * 7 + 4 * 10, 3 * 8 + 4 * 11, 3 * 9 + 4 * 12,
    39
    1
        5 * 7 + 6 * 10, 5 * 8 + 6 * 11, 5 * 9 + 6 * 12,
    40
    1
        7 * 7 + 8 * 10, 7 * 8 + 8 * 11, 7 * 9 + 8 * 12,
    41
    1
      };
    42
    1
      ccv_nnc_tensor_t ct = ccv_nnc_tensor(ctp, CPU_TENSOR_NHWC(32F, 4, 3), 0);
    43
    1
      REQUIRE_TENSOR_EQ(c, &ct, "result should be equal");
    44
    1
      ccv_nnc_tensor_free(a);
    45
    1
      ccv_nnc_tensor_free(b);
    46
    1
      ccv_nnc_tensor_free(c);
    47
    1
      ccv_nnc_tensor_free(ga);
    48
    1
      ccv_nnc_tensor_free(gb);
    49
    1
      ccv_nnc_tensor_free(gc);
    50
    1
    }
    51
    52
    TEST_CASE("gemm transpose a")
    53
    1
    {
    54
    1
      GUARD_ELSE_RETURN(ccv_nnc_cmd_ok(CCV_NNC_GEMM_FORWARD, CCV_NNC_BACKEND_GPU_CUBLAS));
    55
    1
      float ap[] = {
    56
    1
        1, 3, 5, 7,
    57
    1
        2, 4, 6, 8,
    58
    1
      };
    59
    1
      ccv_nnc_tensor_t* const a = ccv_nnc_tensor_new(ap, CPU_TENSOR_NHWC(32F, 2, 4), 0);
    60
    1
      float bp[] = {
    61
    1
        7, 8, 9,
    62
    1
        10, 11, 12,
    63
    1
      };
    64
    1
      ccv_nnc_tensor_t* const b = ccv_nnc_tensor_new(bp, CPU_TENSOR_NHWC(32F, 2, 3), 0);
    65
    1
      ccv_nnc_tensor_t* const c = ccv_nnc_tensor_new(0, CPU_TENSOR_NHWC(32F, 4, 3), 0);
    66
    1
      ccv_nnc_tensor_t* ga = ccv_nnc_tensor_new(0, GPU_TENSOR_NHWC(000, 32F, 2, 4), 0);
    67
    1
      ccv_nnc_tensor_t* gb = ccv_nnc_tensor_new(0, GPU_TENSOR_NHWC(000, 32F, 2, 3), 0);
    68
    1
      ccv_nnc_tensor_t* gc = ccv_nnc_tensor_new(0, GPU_TENSOR_NHWC(000, 32F, 4, 3), 0);
    69
    1
      ccv_nnc_cmd_exec(CMD_DATA_TRANSFER_FORWARD(), ccv_nnc_no_hint, 0, TENSOR_LIST(a, b), TENSOR_LIST(ga, gb), 0);
    70
    1
      ccv_nnc_cmd_exec(CMD_GEMM_FORWARD(TRANSPOSE(0, 1)), ccv_nnc_no_hint, 0, TENSOR_LIST(ga, gb), TENSOR_LIST(gc), 0);
    71
    1
      ccv_nnc_cmd_exec(CMD_DATA_TRANSFER_FORWARD(), ccv_nnc_no_hint, 0, TENSOR_LIST(gc), TENSOR_LIST(c), 0);
    72
    1
      float ctp[] = {
    73
    1
        1 * 7 + 2 * 10, 1 * 8 + 2 * 11, 1 * 9 + 2 * 12,
    74
    1
        3 * 7 + 4 * 10, 3 * 8 + 4 * 11, 3 * 9 + 4 * 12,
    75
    1
        5 * 7 + 6 * 10, 5 * 8 + 6 * 11, 5 * 9 + 6 * 12,
    76
    1
        7 * 7 + 8 * 10, 7 * 8 + 8 * 11, 7 * 9 + 8 * 12,
    77
    1
      };
    78
    1
      ccv_nnc_tensor_t ct = ccv_nnc_tensor(ctp, CPU_TENSOR_NHWC(32F, 4, 3), 0);
    79
    1
      REQUIRE_TENSOR_EQ(c, &ct, "result should be equal");
    80
    1
      ccv_nnc_tensor_free(a);
    81
    1
      ccv_nnc_tensor_free(b);
    82
    1
      ccv_nnc_tensor_free(c);
    83
    1
      ccv_nnc_tensor_free(ga);
    84
    1
      ccv_nnc_tensor_free(gb);
    85
    1
      ccv_nnc_tensor_free(gc);
    86
    1
    }
    87
    88
    TEST_CASE("gemm transpose b")
    89
    1
    {
    90
    1
      GUARD_ELSE_RETURN(ccv_nnc_cmd_ok(CCV_NNC_GEMM_FORWARD, CCV_NNC_BACKEND_GPU_CUBLAS));
    91
    1
      float ap[] = {
    92
    1
        1, 2,
    93
    1
        3, 4,
    94
    1
        5, 6,
    95
    1
        7, 8,
    96
    1
      };
    97
    1
      ccv_nnc_tensor_t* const a = ccv_nnc_tensor_new(ap, CPU_TENSOR_NHWC(32F, 4, 2), 0);
    98
    1
      float bp[] = {
    99
    1
        7, 10,
    100
    1
        8, 11,
    101
    1
        9, 12,
    102
    1
      };
    103
    1
      ccv_nnc_tensor_t* const b = ccv_nnc_tensor_new(bp, CPU_TENSOR_NHWC(32F, 3, 2), 0);
    104
    1
      ccv_nnc_tensor_t* const c = ccv_nnc_tensor_new(0, CPU_TENSOR_NHWC(32F, 4, 3), 0);
    105
    1
      ccv_nnc_tensor_t* ga = ccv_nnc_tensor_new(0, GPU_TENSOR_NHWC(000, 32F, 4, 2), 0);
    106
    1
      ccv_nnc_tensor_t* gb = ccv_nnc_tensor_new(0, GPU_TENSOR_NHWC(000, 32F, 3, 2), 0);
    107
    1
      ccv_nnc_tensor_t* gc = ccv_nnc_tensor_new(0, GPU_TENSOR_NHWC(000, 32F, 4, 3), 0);
    108
    1
      ccv_nnc_cmd_exec(CMD_DATA_TRANSFER_FORWARD(), ccv_nnc_no_hint, 0, TENSOR_LIST(a, b), TENSOR_LIST(ga, gb), 0);
    109
    1
      ccv_nnc_cmd_exec(CMD_GEMM_FORWARD(NO_TRANSPOSE, TRANSPOSE(0, 1)), ccv_nnc_no_hint, 0, TENSOR_LIST(ga, gb), TENSOR_LIST(gc), 0);
    110
    1
      ccv_nnc_cmd_exec(CMD_DATA_TRANSFER_FORWARD(), ccv_nnc_no_hint, 0, TENSOR_LIST(gc), TENSOR_LIST(c), 0);
    111
    1
      float ctp[] = {
    112
    1
        1 * 7 + 2 * 10, 1 * 8 + 2 * 11, 1 * 9 + 2 * 12,
    113
    1
        3 * 7 + 4 * 10, 3 * 8 + 4 * 11, 3 * 9 + 4 * 12,
    114
    1
        5 * 7 + 6 * 10, 5 * 8 + 6 * 11, 5 * 9 + 6 * 12,
    115
    1
        7 * 7 + 8 * 10, 7 * 8 + 8 * 11, 7 * 9 + 8 * 12,
    116
    1
      };
    117
    1
      ccv_nnc_tensor_t ct = ccv_nnc_tensor(ctp, CPU_TENSOR_NHWC(32F, 4, 3), 0);
    118
    1
      REQUIRE_TENSOR_EQ(c, &ct, "result should be equal");
    119
    1
      ccv_nnc_tensor_free(a);
    120
    1
      ccv_nnc_tensor_free(b);
    121
    1
      ccv_nnc_tensor_free(c);
    122
    1
      ccv_nnc_tensor_free(ga);
    123
    1
      ccv_nnc_tensor_free(gb);
    124
    1
      ccv_nnc_tensor_free(gc);
    125
    1
    }
    126
    127
    TEST_CASE("gemm transpose a and b")
    128
    1
    {
    129
    1
      GUARD_ELSE_RETURN(ccv_nnc_cmd_ok(CCV_NNC_GEMM_FORWARD, CCV_NNC_BACKEND_GPU_CUBLAS));
    130
    1
      float ap[] = {
    131
    1
        1, 3, 5, 7,
    132
    1
        2, 4, 6, 8,
    133
    1
      };
    134
    1
      ccv_nnc_tensor_t* const a = ccv_nnc_tensor_new(ap, CPU_TENSOR_NHWC(32F, 2, 4), 0);
    135
    1
      float bp[] = {
    136
    1
        7, 10,
    137
    1
        8, 11,
    138
    1
        9, 12,
    139
    1
      };
    140
    1
      ccv_nnc_tensor_t* const b = ccv_nnc_tensor_new(bp, CPU_TENSOR_NHWC(32F, 3, 2), 0);
    141
    1
      ccv_nnc_tensor_t* const c = ccv_nnc_tensor_new(0, CPU_TENSOR_NHWC(32F, 4, 3), 0);
    142
    1
      ccv_nnc_tensor_t* ga = ccv_nnc_tensor_new(0, GPU_TENSOR_NHWC(000, 32F, 2, 4), 0);
    143
    1
      ccv_nnc_tensor_t* gb = ccv_nnc_tensor_new(0, GPU_TENSOR_NHWC(000, 32F, 3, 2), 0);
    144
    1
      ccv_nnc_tensor_t* gc = ccv_nnc_tensor_new(0, GPU_TENSOR_NHWC(000, 32F, 4, 3), 0);
    145
    1
      ccv_nnc_cmd_exec(CMD_DATA_TRANSFER_FORWARD(), ccv_nnc_no_hint, 0, TENSOR_LIST(a, b), TENSOR_LIST(ga, gb), 0);
    146
    1
      ccv_nnc_cmd_exec(CMD_GEMM_FORWARD(TRANSPOSE(0, 1), TRANSPOSE(0, 1)), ccv_nnc_no_hint, 0, TENSOR_LIST(ga, gb), TENSOR_LIST(gc), 0);
    147
    1
      ccv_nnc_cmd_exec(CMD_DATA_TRANSFER_FORWARD(), ccv_nnc_no_hint, 0, TENSOR_LIST(gc), TENSOR_LIST(c), 0);
    148
    1
      float ctp[] = {
    149
    1
        1 * 7 + 2 * 10, 1 * 8 + 2 * 11, 1 * 9 + 2 * 12,
    150
    1
        3 * 7 + 4 * 10, 3 * 8 + 4 * 11, 3 * 9 + 4 * 12,
    151
    1
        5 * 7 + 6 * 10, 5 * 8 + 6 * 11, 5 * 9 + 6 * 12,
    152
    1
        7 * 7 + 8 * 10, 7 * 8 + 8 * 11, 7 * 9 + 8 * 12,
    153
    1
      };
    154
    1
      ccv_nnc_tensor_t ct = ccv_nnc_tensor(ctp, CPU_TENSOR_NHWC(32F, 4, 3), 0);
    155
    1
      REQUIRE_TENSOR_EQ(c, &ct, "result should be equal");
    156
    1
      ccv_nnc_tensor_free(a);
    157
    1
      ccv_nnc_tensor_free(b);
    158
    1
      ccv_nnc_tensor_free(c);
    159
    1
      ccv_nnc_tensor_free(ga);
    160
    1
      ccv_nnc_tensor_free(gb);
    161
    1
      ccv_nnc_tensor_free(gc);
    162
    1
    }
    163
    164
    TEST_CASE("gemm no transpose with bias")
    165
    1
    {
    166
    1
      GUARD_ELSE_RETURN(ccv_nnc_cmd_ok(CCV_NNC_GEMM_FORWARD, CCV_NNC_BACKEND_GPU_CUBLAS));
    167
    1
      float ap[] = {
    168
    1
        1, 2,
    169
    1
        3, 4,
    170
    1
        5, 6,
    171
    1
        7, 8,
    172
    1
      };
    173
    1
      ccv_nnc_tensor_t* const a = ccv_nnc_tensor_new(ap, CPU_TENSOR_NHWC(32F, 4, 2), 0);
    174
    1
      float bp[] = {
    175
    1
        7, 8, 9,
    176
    1
        10, 11, 12,
    177
    1
      };
    178
    1
      ccv_nnc_tensor_t* const b = ccv_nnc_tensor_new(bp, CPU_TENSOR_NHWC(32F, 2, 3), 0);
    179
    1
      float dp[] = {
    180
    1
        1, -1, 1,
    181
    1
        1, -1, 1,
    182
    1
        1, -1, 1,
    183
    1
        1, -1, 1,
    184
    1
      };
    185
    1
      ccv_nnc_tensor_t* const d = ccv_nnc_tensor_new(dp, CPU_TENSOR_NHWC(32F, 4, 3), 0);
    186
    1
      ccv_nnc_tensor_t* const c = ccv_nnc_tensor_new(0, CPU_TENSOR_NHWC(32F, 4, 3), 0);
    187
    1
      ccv_nnc_tensor_t* ga = ccv_nnc_tensor_new(0, GPU_TENSOR_NHWC(000, 32F, 4, 2), 0);
    188
    1
      ccv_nnc_tensor_t* gb = ccv_nnc_tensor_new(0, GPU_TENSOR_NHWC(000, 32F, 2, 3), 0);
    189
    1
      ccv_nnc_tensor_t* gd = ccv_nnc_tensor_new(0, GPU_TENSOR_NHWC(000, 32F, 4, 3), 0);
    190
    1
      ccv_nnc_tensor_t* gc = ccv_nnc_tensor_new(0, GPU_TENSOR_NHWC(000, 32F, 4, 3), 0);
    191
    1
      ccv_nnc_cmd_exec(CMD_DATA_TRANSFER_FORWARD(), ccv_nnc_no_hint, 0, TENSOR_LIST(a, b, d), TENSOR_LIST(ga, gb, gd), 0);
    192
    1
      ccv_nnc_cmd_exec(CMD_GEMM_FORWARD(), ccv_nnc_no_hint, 0, TENSOR_LIST(ga, gb, gd), TENSOR_LIST(gc), 0);
    193
    1
      ccv_nnc_cmd_exec(CMD_DATA_TRANSFER_FORWARD(), ccv_nnc_no_hint, 0, TENSOR_LIST(gc), TENSOR_LIST(c), 0);
    194
    1
      float ctp[] = {
    195
    1
        1 * 7 + 2 * 10 + 1, 1 * 8 + 2 * 11 - 1, 1 * 9 + 2 * 12 + 1,
    196
    1
        3 * 7 + 4 * 10 + 1, 3 * 8 + 4 * 11 - 1, 3 * 9 + 4 * 12 + 1,
    197
    1
        5 * 7 + 6 * 10 + 1, 5 * 8 + 6 * 11 - 1, 5 * 9 + 6 * 12 + 1,
    198
    1
        7 * 7 + 8 * 10 + 1, 7 * 8 + 8 * 11 - 1, 7 * 9 + 8 * 12 + 1,
    199
    1
      };
    200
    1
      ccv_nnc_tensor_t ct = ccv_nnc_tensor(ctp, CPU_TENSOR_NHWC(32F, 4, 3), 0);
    201
    1
      REQUIRE_TENSOR_EQ(c, &ct, "result should be equal");
    202
    1
      ccv_nnc_tensor_free(a);
    203
    1
      ccv_nnc_tensor_free(b);
    204
    1
      ccv_nnc_tensor_free(c);
    205
    1
      ccv_nnc_tensor_free(d);
    206
    1
      ccv_nnc_tensor_free(ga);
    207
    1
      ccv_nnc_tensor_free(gb);
    208
    1
      ccv_nnc_tensor_free(gc);
    209
    1
      ccv_nnc_tensor_free(gd);
    210
    1
    }
    211
    212
    TEST_CASE("gemm no transpose with bias and palettize weights")
    213
    1
    {
    214
    1
      GUARD_ELSE_RETURN(ccv_nnc_cmd_ok(CCV_NNC_GEMM_FORWARD, CCV_NNC_BACKEND_GPU_CUBLAS));
    215
    1
      float ap[] = {
    216
    1
        1, 2,
    217
    1
        3, 4,
    218
    1
        5, 6,
    219
    1
        7, 8,
    220
    1
      };
    221
    1
      ccv_nnc_tensor_t* const a = ccv_nnc_tensor_new(ap, CPU_TENSOR_NHWC(32F, 4, 2), 0);
    222
    1
      float bp[] = {
    223
    1
        7, 8, 9,
    224
    1
        10, 11, 12,
    225
    1
      };
    226
    1
      ccv_nnc_tensor_t* const b = ccv_nnc_tensor_new(bp, CPU_TENSOR_NHWC(32F, 2, 3), 0);
    227
    1
      float dp[] = {
    228
    1
        1, -1, 1,
    229
    1
        1, -1, 1,
    230
    1
        1, -1, 1,
    231
    1
        1, -1, 1,
    232
    1
      };
    233
    1
      ccv_nnc_tensor_t* const pb = ccv_nnc_tensor_new(0, ccv_nnc_tensor_palettize(CPU_TENSOR_NHWC(32F, 2, 3), 4, 128), 0);
    234
    1
      (void)ccv_nnc_palettize(b->data.u8, CCV_32F, CCV_TENSOR_CPU_MEMORY, 6, 4, 128, pb->data.u8, ccv_nnc_tensor_data_size_without_padding(pb->info));
    235
    1
      ccv_nnc_tensor_t* const d = ccv_nnc_tensor_new(dp, CPU_TENSOR_NHWC(32F, 4, 3), 0);
    236
    1
      ccv_nnc_tensor_t* const c = ccv_nnc_tensor_new(0, CPU_TENSOR_NHWC(32F, 4, 3), 0);
    237
    1
      ccv_nnc_tensor_t* ga = ccv_nnc_tensor_new(0, GPU_TENSOR_NHWC(000, 32F, 4, 2), 0);
    238
    1
      ccv_nnc_tensor_t* gb = ccv_nnc_tensor_new(0, ccv_nnc_tensor_palettize(GPU_TENSOR_NHWC(000, 32F, 2, 3), 4, 128), 0);
    239
    1
      ccv_nnc_tensor_t* gd = ccv_nnc_tensor_new(0, GPU_TENSOR_NHWC(000, 32F, 4, 3), 0);
    240
    1
      ccv_nnc_tensor_t* gc = ccv_nnc_tensor_new(0, GPU_TENSOR_NHWC(000, 32F, 4, 3), 0);
    241
    1
      ccv_nnc_cmd_exec(CMD_DATA_TRANSFER_FORWARD(), ccv_nnc_no_hint, 0, TENSOR_LIST(a, pb, d), TENSOR_LIST(ga, gb, gd), 0);
    242
    1
      ccv_nnc_cmd_exec(CMD_GEMM_FORWARD(), ccv_nnc_no_hint, 0, TENSOR_LIST(ga, gb, gd), TENSOR_LIST(gc), 0);
    243
    1
      ccv_nnc_cmd_exec(CMD_DATA_TRANSFER_FORWARD(), ccv_nnc_no_hint, 0, TENSOR_LIST(gc), TENSOR_LIST(c), 0);
    244
    1
      float ctp[] = {
    245
    1
        1 * 7 + 2 * 10 + 1, 1 * 8 + 2 * 11 - 1, 1 * 9 + 2 * 12 + 1,
    246
    1
        3 * 7 + 4 * 10 + 1, 3 * 8 + 4 * 11 - 1, 3 * 9 + 4 * 12 + 1,
    247
    1
        5 * 7 + 6 * 10 + 1, 5 * 8 + 6 * 11 - 1, 5 * 9 + 6 * 12 + 1,
    248
    1
        7 * 7 + 8 * 10 + 1, 7 * 8 + 8 * 11 - 1, 7 * 9 + 8 * 12 + 1,
    249
    1
      };
    250
    1
      ccv_nnc_tensor_t ct = ccv_nnc_tensor(ctp, CPU_TENSOR_NHWC(32F, 4, 3), 0);
    251
    1
      REQUIRE_TENSOR_EQ(c, &ct, "result should be equal");
    252
    1
      ccv_nnc_tensor_free(a);
    253
    1
      ccv_nnc_tensor_free(b);
    254
    1
      ccv_nnc_tensor_free(pb);
    255
    1
      ccv_nnc_tensor_free(c);
    256
    1
      ccv_nnc_tensor_free(d);
    257
    1
      ccv_nnc_tensor_free(ga);
    258
    1
      ccv_nnc_tensor_free(gb);
    259
    1
      ccv_nnc_tensor_free(gc);
    260
    1
      ccv_nnc_tensor_free(gd);
    261
    1
    }
    262
    263
    TEST_CASE("gemm no transpose with bias and row-wise int8 weights")
    264
    1
    {
    265
    1
      GUARD_ELSE_RETURN(ccv_nnc_cmd_ok(CCV_NNC_GEMM_FORWARD, CCV_NNC_BACKEND_GPU_CUBLAS));
    266
    1
      float ap[] = {
    267
    1
        1, 2,
    268
    1
        3, 4,
    269
    1
        5, 6,
    270
    1
        7, 8,
    271
    1
      };
    272
    1
      ccv_nnc_tensor_t* const a = ccv_nnc_tensor_new(ap, CPU_TENSOR_NHWC(32F, 4, 2), 0);
    273
    1
      float bp[] = {
    274
    1
        7, 8, 9,
    275
    1
        10, 11, 12,
    276
    1
      };
    277
    1
      ccv_nnc_tensor_t* const b = ccv_nnc_tensor_new(bp, CPU_TENSOR_NHWC(32F, 2, 3), 0);
    278
    1
      ccv_nnc_tensor_t* const qb = ccv_nnc_tensor_new(0, ccv_nnc_tensor_8i_rowwise(CPU_TENSOR_NHWC(32F, 2, 3)), 0);
    279
    1
      const size_t qsize = ccv_nnc_quantize_8i_rowwise(b->data.u8, CCV_32F, CCV_TENSOR_CPU_MEMORY, 6, 3, qb->data.u8, ccv_nnc_tensor_data_size_without_padding(qb->info));
    280
    1
      ccv_nnc_tensor_t* const dq_b = ccv_nnc_tensor_new(0, CPU_TENSOR_NHWC(32F, 2, 3), 0);
    281
    1
      ccv_nnc_dequantize_8i_rowwise(qb->data.u8, CCV_32F, CCV_TENSOR_CPU_MEMORY, qsize, 3, dq_b->data.u8, 6);
    282
    1
      float dp[] = {
    283
    1
        1, -1, 1,
    284
    1
        1, -1, 1,
    285
    1
        1, -1, 1,
    286
    1
        1, -1, 1,
    287
    1
      };
    288
    1
      ccv_nnc_tensor_t* const d = ccv_nnc_tensor_new(dp, CPU_TENSOR_NHWC(32F, 4, 3), 0);
    289
    1
      ccv_nnc_tensor_t* const ct = ccv_nnc_tensor_new(0, CPU_TENSOR_NHWC(32F, 4, 3), 0);
    290
    1
      ccv_nnc_cmd_t cmd = CMD_GEMM_FORWARD();
    291
    1
      cmd.backend = CCV_NNC_BACKEND_CPU_REF;
    292
    1
      assert(cmd.backend >= 0);
    293
    1
      ccv_nnc_cmd_exec(cmd, ccv_nnc_no_hint, 0, TENSOR_LIST(a, dq_b, d), TENSOR_LIST(ct), 0);
    294
    1
      ccv_nnc_tensor_t* const c = ccv_nnc_tensor_new(0, CPU_TENSOR_NHWC(32F, 4, 3), 0);
    295
    1
      ccv_nnc_tensor_t* ga = ccv_nnc_tensor_new(0, GPU_TENSOR_NHWC(000, 32F, 4, 2), 0);
    296
    1
      ccv_nnc_tensor_t* gqb = ccv_nnc_tensor_new(0, ccv_nnc_tensor_8i_rowwise(GPU_TENSOR_NHWC(000, 32F, 2, 3)), 0);
    297
    1
      ccv_nnc_tensor_t* gd = ccv_nnc_tensor_new(0, GPU_TENSOR_NHWC(000, 32F, 4, 3), 0);
    298
    1
      ccv_nnc_tensor_t* gc = ccv_nnc_tensor_new(0, GPU_TENSOR_NHWC(000, 32F, 4, 3), 0);
    299
    1
      ccv_nnc_cmd_exec(CMD_DATA_TRANSFER_FORWARD(), ccv_nnc_no_hint, 0, TENSOR_LIST(a, qb, d), TENSOR_LIST(ga, gqb, gd), 0);
    300
    1
      ccv_nnc_cmd_exec(CMD_GEMM_FORWARD(), ccv_nnc_no_hint, 0, TENSOR_LIST(ga, gqb, gd), TENSOR_LIST(gc), 0);
    301
    1
      ccv_nnc_cmd_exec(CMD_DATA_TRANSFER_FORWARD(), ccv_nnc_no_hint, 0, TENSOR_LIST(gc), TENSOR_LIST(c), 0);
    302
    1
      REQUIRE_ARRAY_EQ_WITH_TOLERANCE(float, ct->data.f32, c->data.f32, 12, 1e-5, "result should match CPU with dequantized row-wise weights");
    303
    1
      ccv_nnc_tensor_free(a);
    304
    1
      ccv_nnc_tensor_free(b);
    305
    1
      ccv_nnc_tensor_free(qb);
    306
    1
      ccv_nnc_tensor_free(dq_b);
    307
    1
      ccv_nnc_tensor_free(d);
    308
    1
      ccv_nnc_tensor_free(ct);
    309
    1
      ccv_nnc_tensor_free(c);
    310
    1
      ccv_nnc_tensor_free(ga);
    311
    1
      ccv_nnc_tensor_free(gqb);
    312
    1
      ccv_nnc_tensor_free(gd);
    313
    1
      ccv_nnc_tensor_free(gc);
    314
    1
    }
    315
    316
    TEST_CASE("backward gemm with no transpose")
    317
    1
    {
    318
    1
      GUARD_ELSE_RETURN(ccv_nnc_cmd_ok(CCV_NNC_GEMM_FORWARD, CCV_NNC_BACKEND_GPU_CUBLAS) &&
    319
    1
        ccv_nnc_cmd_ok(CCV_NNC_GEMM_BACKWARD, CCV_NNC_BACKEND_GPU_CUBLAS));
    320
    1
      float gp[] = {
    321
    1
        1, 2, 3,
    322
    1
        4, 5, 6,
    323
    1
        7, 8, 9,
    324
    1
        10, 11, 12,
    325
    1
      };
    326
    1
      ccv_nnc_tensor_t* const g = ccv_nnc_tensor_new(gp, CPU_TENSOR_NHWC(32F, 4, 3), 0);
    327
    1
      float ap[] = {
    328
    1
        13, 14,
    329
    1
        15, 16,
    330
    1
        17, 18,
    331
    1
        19, 20,
    332
    1
      };
    333
    1
      ccv_nnc_tensor_t* const a = ccv_nnc_tensor_new(ap, CPU_TENSOR_NHWC(32F, 4, 2), 0);
    334
    1
      float bp[] = {
    335
    1
        21, 22, 23,
    336
    1
        24, 25, 26,
    337
    1
      };
    338
    1
      ccv_nnc_tensor_t* const b = ccv_nnc_tensor_new(bp, CPU_TENSOR_NHWC(32F, 2, 3), 0);
    339
    1
      ccv_nnc_tensor_t* const h = ccv_nnc_tensor_new(0, CPU_TENSOR_NHWC(32F, 4, 2), 0);
    340
    1
      ccv_nnc_tensor_t* const db = ccv_nnc_tensor_new(0, CPU_TENSOR_NHWC(32F, 2, 3), 0);
    341
    1
      ccv_nnc_tensor_t* const dbias = ccv_nnc_tensor_new(0, CPU_TENSOR_NHWC(32F, 3), 0);
    342
    1
      ccv_nnc_tensor_t* gg = ccv_nnc_tensor_new(0, GPU_TENSOR_NHWC(000, 32F, 4, 3), 0);
    343
    1
      ccv_nnc_tensor_t* ga = ccv_nnc_tensor_new(0, GPU_TENSOR_NHWC(000, 32F, 4, 2), 0);
    344
    1
      ccv_nnc_tensor_t* gb = ccv_nnc_tensor_new(0, GPU_TENSOR_NHWC(000, 32F, 2, 3), 0);
    345
    1
      ccv_nnc_tensor_t* gh = ccv_nnc_tensor_new(0, GPU_TENSOR_NHWC(000, 32F, 4, 2), 0);
    346
    1
      ccv_nnc_tensor_t* gdb = ccv_nnc_tensor_new(0, GPU_TENSOR_NHWC(000, 32F, 2, 3), 0);
    347
    1
      ccv_nnc_tensor_t* gdbias = ccv_nnc_tensor_new(0, GPU_TENSOR_NHWC(000, 32F, 3), 0);
    348
    1
      ccv_nnc_cmd_exec(CMD_DATA_TRANSFER_FORWARD(), ccv_nnc_no_hint, 0, TENSOR_LIST(g, a, b), TENSOR_LIST(gg, ga, gb), 0);
    349
    1
      ccv_nnc_cmd_exec(CMD_GEMM_BACKWARD(), ccv_nnc_no_hint, 0, TENSOR_LIST(gg, ga, gb), TENSOR_LIST(gh, gdb, gdbias), 0);
    350
    1
      ccv_nnc_cmd_exec(CMD_DATA_TRANSFER_FORWARD(), ccv_nnc_no_hint, 0, TENSOR_LIST(gh, gdb, gdbias), TENSOR_LIST(h, db, dbias), 0);
    351
    1
      float dbiastp[] = {
    352
    1
        22, 26, 30,
    353
    1
      };
    354
    1
      ccv_nnc_tensor_t dbiast = ccv_nnc_tensor(dbiastp, CPU_TENSOR_NHWC(32F, 3), 0);
    355
    1
      REQUIRE_TENSOR_EQ(dbias, &dbiast, "bias should be equal");
    356
    1
      float htp[] = {
    357
    1
        1 * 21 + 2 * 22 + 3 * 23, 1 * 24 + 2 * 25 + 3 * 26,
    358
    1
        4 * 21 + 5 * 22 + 6 * 23, 4 * 24 + 5 * 25 + 6 * 26,
    359
    1
        7 * 21 + 8 * 22 + 9 * 23, 7 * 24 + 8 * 25 + 9 * 26,
    360
    1
        10 * 21 + 11 * 22 + 12 * 23, 10 * 24 + 11 * 25 + 12 * 26,
    361
    1
      };
    362
    1
      ccv_nnc_tensor_t ht = ccv_nnc_tensor(htp, CPU_TENSOR_NHWC(32F, 4, 2), 0);
    363
    1
      REQUIRE_TENSOR_EQ(h, &ht, "h should be equal");
    364
    1
      float dbtp[] = {
    365
    1
        1 * 13 + 4 * 15 + 7 * 17 + 10 * 19, 2 * 13 + 5 * 15 + 8 * 17 + 11 * 19, 3 * 13 + 6 * 15 + 9 * 17 + 12 * 19,
    366
    1
        1 * 14 + 4 * 16 + 7 * 18 + 10 * 20, 2 * 14 + 5 * 16 + 8 * 18 + 11 * 20, 3 * 14 + 6 * 16 + 9 * 18 + 12 * 20,
    367
    1
      };
    368
    1
      ccv_nnc_tensor_t dbt = ccv_nnc_tensor(dbtp, CPU_TENSOR_NHWC(32F, 2, 3), 0);
    369
    1
      REQUIRE_TENSOR_EQ(db, &dbt, "db should be equal");
    370
    1
      ccv_nnc_tensor_free(g);
    371
    1
      ccv_nnc_tensor_free(a);
    372
    1
      ccv_nnc_tensor_free(b);
    373
    1
      ccv_nnc_tensor_free(h);
    374
    1
      ccv_nnc_tensor_free(db);
    375
    1
      ccv_nnc_tensor_free(dbias);
    376
    1
      ccv_nnc_tensor_free(gg);
    377
    1
      ccv_nnc_tensor_free(ga);
    378
    1
      ccv_nnc_tensor_free(gb);
    379
    1
      ccv_nnc_tensor_free(gh);
    380
    1
      ccv_nnc_tensor_free(gdb);
    381
    1
      ccv_nnc_tensor_free(gdbias);
    382
    1
    }
    383
    384
    TEST_CASE("backward gemm with no transpose and palettize weights")
    385
    1
    {
    386
    1
      GUARD_ELSE_RETURN(ccv_nnc_cmd_ok(CCV_NNC_GEMM_FORWARD, CCV_NNC_BACKEND_GPU_CUBLAS) &&
    387
    1
        ccv_nnc_cmd_ok(CCV_NNC_GEMM_BACKWARD, CCV_NNC_BACKEND_GPU_CUBLAS));
    388
    1
      float gp[] = {
    389
    1
        1, 2, 3,
    390
    1
        4, 5, 6,
    391
    1
        7, 8, 9,
    392
    1
        10, 11, 12,
    393
    1
      };
    394
    1
      ccv_nnc_tensor_t* const g = ccv_nnc_tensor_new(gp, CPU_TENSOR_NHWC(32F, 4, 3), 0);
    395
    1
      float ap[] = {
    396
    1
        13, 14,
    397
    1
        15, 16,
    398
    1
        17, 18,
    399
    1
        19, 20,
    400
    1
      };
    401
    1
      ccv_nnc_tensor_t* const a = ccv_nnc_tensor_new(ap, CPU_TENSOR_NHWC(32F, 4, 2), 0);
    402
    1
      float bp[] = {
    403
    1
        21, 22, 23,
    404
    1
        24, 25, 26,
    405
    1
      };
    406
    1
      ccv_nnc_tensor_t* const b = ccv_nnc_tensor_new(bp, CPU_TENSOR_NHWC(32F, 2, 3), 0);
    407
    1
      ccv_nnc_tensor_t* const pb = ccv_nnc_tensor_new(0, ccv_nnc_tensor_palettize(CPU_TENSOR_NHWC(32F, 2, 3), 4, 128), 0);
    408
    1
      (void)ccv_nnc_palettize(b->data.u8, CCV_32F, CCV_TENSOR_CPU_MEMORY, 6, 4, 128, pb->data.u8, ccv_nnc_tensor_data_size_without_padding(pb->info));
    409
    1
      ccv_nnc_tensor_t* const h = ccv_nnc_tensor_new(0, CPU_TENSOR_NHWC(32F, 4, 2), 0);
    410
    1
      ccv_nnc_tensor_t* const db = ccv_nnc_tensor_new(0, CPU_TENSOR_NHWC(32F, 2, 3), 0);
    411
    1
      ccv_nnc_tensor_t* const dbias = ccv_nnc_tensor_new(0, CPU_TENSOR_NHWC(32F, 3), 0);
    412
    1
      ccv_nnc_tensor_t* gg = ccv_nnc_tensor_new(0, GPU_TENSOR_NHWC(000, 32F, 4, 3), 0);
    413
    1
      ccv_nnc_tensor_t* ga = ccv_nnc_tensor_new(0, GPU_TENSOR_NHWC(000, 32F, 4, 2), 0);
    414
    1
      ccv_nnc_tensor_t* gb = ccv_nnc_tensor_new(0, ccv_nnc_tensor_palettize(GPU_TENSOR_NHWC(000, 32F, 2, 3), 4, 128), 0);
    415
    1
      ccv_nnc_tensor_t* gh = ccv_nnc_tensor_new(0, GPU_TENSOR_NHWC(000, 32F, 4, 2), 0);
    416
    1
      ccv_nnc_tensor_t* gdb = ccv_nnc_tensor_new(0, GPU_TENSOR_NHWC(000, 32F, 2, 3), 0);
    417
    1
      ccv_nnc_tensor_t* gdbias = ccv_nnc_tensor_new(0, GPU_TENSOR_NHWC(000, 32F, 3), 0);
    418
    1
      ccv_nnc_cmd_exec(CMD_DATA_TRANSFER_FORWARD(), ccv_nnc_no_hint, 0, TENSOR_LIST(g, a, pb), TENSOR_LIST(gg, ga, gb), 0);
    419
    1
      ccv_nnc_cmd_exec(CMD_GEMM_BACKWARD(), ccv_nnc_no_hint, 0, TENSOR_LIST(gg, ga, gb), TENSOR_LIST(gh, gdb, gdbias), 0);
    420
    1
      ccv_nnc_cmd_exec(CMD_DATA_TRANSFER_FORWARD(), ccv_nnc_no_hint, 0, TENSOR_LIST(gh, gdb, gdbias), TENSOR_LIST(h, db, dbias), 0);
    421
    1
      float dbiastp[] = {
    422
    1
        22, 26, 30,
    423
    1
      };
    424
    1
      ccv_nnc_tensor_t dbiast = ccv_nnc_tensor(dbiastp, CPU_TENSOR_NHWC(32F, 3), 0);
    425
    1
      REQUIRE_TENSOR_EQ(dbias, &dbiast, "bias should be equal");
    426
    1
      float htp[] = {
    427
    1
        1 * 21 + 2 * 22 + 3 * 23, 1 * 24 + 2 * 25 + 3 * 26,
    428
    1
        4 * 21 + 5 * 22 + 6 * 23, 4 * 24 + 5 * 25 + 6 * 26,
    429
    1
        7 * 21 + 8 * 22 + 9 * 23, 7 * 24 + 8 * 25 + 9 * 26,
    430
    1
        10 * 21 + 11 * 22 + 12 * 23, 10 * 24 + 11 * 25 + 12 * 26,
    431
    1
      };
    432
    1
      ccv_nnc_tensor_t ht = ccv_nnc_tensor(htp, CPU_TENSOR_NHWC(32F, 4, 2), 0);
    433
    1
      REQUIRE_TENSOR_EQ(h, &ht, "h should be equal");
    434
    1
      float dbtp[] = {
    435
    1
        1 * 13 + 4 * 15 + 7 * 17 + 10 * 19, 2 * 13 + 5 * 15 + 8 * 17 + 11 * 19, 3 * 13 + 6 * 15 + 9 * 17 + 12 * 19,
    436
    1
        1 * 14 + 4 * 16 + 7 * 18 + 10 * 20, 2 * 14 + 5 * 16 + 8 * 18 + 11 * 20, 3 * 14 + 6 * 16 + 9 * 18 + 12 * 20,
    437
    1
      };
    438
    1
      ccv_nnc_tensor_t dbt = ccv_nnc_tensor(dbtp, CPU_TENSOR_NHWC(32F, 2, 3), 0);
    439
    1
      REQUIRE_TENSOR_EQ(db, &dbt, "db should be equal");
    440
    1
      ccv_nnc_tensor_free(g);
    441
    1
      ccv_nnc_tensor_free(a);
    442
    1
      ccv_nnc_tensor_free(b);
    443
    1
      ccv_nnc_tensor_free(pb);
    444
    1
      ccv_nnc_tensor_free(h);
    445
    1
      ccv_nnc_tensor_free(db);
    446
    1
      ccv_nnc_tensor_free(dbias);
    447
    1
      ccv_nnc_tensor_free(gg);
    448
    1
      ccv_nnc_tensor_free(ga);
    449
    1
      ccv_nnc_tensor_free(gb);
    450
    1
      ccv_nnc_tensor_free(gh);
    451
    1
      ccv_nnc_tensor_free(gdb);
    452
    1
      ccv_nnc_tensor_free(gdbias);
    453
    1
    }
    454
    455
    TEST_CASE("backward gemm with no transpose and row-wise int8 weights")
    456
    1
    {
    457
    1
      GUARD_ELSE_RETURN(ccv_nnc_cmd_ok(CCV_NNC_GEMM_FORWARD, CCV_NNC_BACKEND_GPU_CUBLAS) &&
    458
    1
        ccv_nnc_cmd_ok(CCV_NNC_GEMM_BACKWARD, CCV_NNC_BACKEND_GPU_CUBLAS));
    459
    1
      float gp[] = {
    460
    1
        1, 2, 3,
    461
    1
        4, 5, 6,
    462
    1
        7, 8, 9,
    463
    1
        10, 11, 12,
    464
    1
      };
    465
    1
      ccv_nnc_tensor_t* const g = ccv_nnc_tensor_new(gp, CPU_TENSOR_NHWC(32F, 4, 3), 0);
    466
    1
      float ap[] = {
    467
    1
        13, 14,
    468
    1
        15, 16,
    469
    1
        17, 18,
    470
    1
        19, 20,
    471
    1
      };
    472
    1
      ccv_nnc_tensor_t* const a = ccv_nnc_tensor_new(ap, CPU_TENSOR_NHWC(32F, 4, 2), 0);
    473
    1
      float bp[] = {
    474
    1
        21, 22, 23,
    475
    1
        24, 25, 26,
    476
    1
      };
    477
    1
      ccv_nnc_tensor_t* const b = ccv_nnc_tensor_new(bp, CPU_TENSOR_NHWC(32F, 2, 3), 0);
    478
    1
      ccv_nnc_tensor_t* const qb = ccv_nnc_tensor_new(0, ccv_nnc_tensor_8i_rowwise(CPU_TENSOR_NHWC(32F, 2, 3)), 0);
    479
    1
      const size_t qsize = ccv_nnc_quantize_8i_rowwise(b->data.u8, CCV_32F, CCV_TENSOR_CPU_MEMORY, 6, 3, qb->data.u8, ccv_nnc_tensor_data_size_without_padding(qb->info));
    480
    1
      ccv_nnc_tensor_t* const dq_b = ccv_nnc_tensor_new(0, CPU_TENSOR_NHWC(32F, 2, 3), 0);
    481
    1
      ccv_nnc_dequantize_8i_rowwise(qb->data.u8, CCV_32F, CCV_TENSOR_CPU_MEMORY, qsize, 3, dq_b->data.u8, 6);
    482
    1
      ccv_nnc_tensor_t* const ht = ccv_nnc_tensor_new(0, CPU_TENSOR_NHWC(32F, 4, 2), 0);
    483
    1
      ccv_nnc_tensor_t* const dbt = ccv_nnc_tensor_new(0, CPU_TENSOR_NHWC(32F, 2, 3), 0);
    484
    1
      ccv_nnc_tensor_t* const dbiast = ccv_nnc_tensor_new(0, CPU_TENSOR_NHWC(32F, 3), 0);
    485
    1
      ccv_nnc_cmd_t cmd = CMD_GEMM_BACKWARD();
    486
    1
      cmd.backend = CCV_NNC_BACKEND_CPU_REF;
    487
    1
      assert(cmd.backend >= 0);
    488
    1
      ccv_nnc_cmd_exec(cmd, ccv_nnc_no_hint, 0, TENSOR_LIST(g, a, dq_b), TENSOR_LIST(ht, dbt, dbiast), 0);
    489
    1
      ccv_nnc_tensor_t* const h = ccv_nnc_tensor_new(0, CPU_TENSOR_NHWC(32F, 4, 2), 0);
    490
    1
      ccv_nnc_tensor_t* const db = ccv_nnc_tensor_new(0, CPU_TENSOR_NHWC(32F, 2, 3), 0);
    491
    1
      ccv_nnc_tensor_t* const dbias = ccv_nnc_tensor_new(0, CPU_TENSOR_NHWC(32F, 3), 0);
    492
    1
      ccv_nnc_tensor_t* gg = ccv_nnc_tensor_new(0, GPU_TENSOR_NHWC(000, 32F, 4, 3), 0);
    493
    1
      ccv_nnc_tensor_t* ga = ccv_nnc_tensor_new(0, GPU_TENSOR_NHWC(000, 32F, 4, 2), 0);
    494
    1
      ccv_nnc_tensor_t* gqb = ccv_nnc_tensor_new(0, ccv_nnc_tensor_8i_rowwise(GPU_TENSOR_NHWC(000, 32F, 2, 3)), 0);
    495
    1
      ccv_nnc_tensor_t* gh = ccv_nnc_tensor_new(0, GPU_TENSOR_NHWC(000, 32F, 4, 2), 0);
    496
    1
      ccv_nnc_tensor_t* gdb = ccv_nnc_tensor_new(0, GPU_TENSOR_NHWC(000, 32F, 2, 3), 0);
    497
    1
      ccv_nnc_tensor_t* gdbias = ccv_nnc_tensor_new(0, GPU_TENSOR_NHWC(000, 32F, 3), 0);
    498
    1
      ccv_nnc_cmd_exec(CMD_DATA_TRANSFER_FORWARD(), ccv_nnc_no_hint, 0, TENSOR_LIST(g, a, qb), TENSOR_LIST(gg, ga, gqb), 0);
    499
    1
      ccv_nnc_cmd_exec(CMD_GEMM_BACKWARD(), ccv_nnc_no_hint, 0, TENSOR_LIST(gg, ga, gqb), TENSOR_LIST(gh, gdb, gdbias), 0);
    500
    1
      ccv_nnc_cmd_exec(CMD_DATA_TRANSFER_FORWARD(), ccv_nnc_no_hint, 0, TENSOR_LIST(gh, gdb, gdbias), TENSOR_LIST(h, db, dbias), 0);
    501
    1
      REQUIRE_ARRAY_EQ_WITH_TOLERANCE(float, ht->data.f32, h->data.f32, 8, 1e-5, "h should match CPU with dequantized row-wise weights");
    502
    1
      REQUIRE_ARRAY_EQ_WITH_TOLERANCE(float, dbt->data.f32, db->data.f32, 6, 1e-5, "db should match CPU with dequantized row-wise weights");
    503
    1
      REQUIRE_ARRAY_EQ_WITH_TOLERANCE(float, dbiast->data.f32, dbias->data.f32, 3, 1e-5, "dbias should match CPU with dequantized row-wise weights");
    504
    1
      ccv_nnc_tensor_free(g);
    505
    1
      ccv_nnc_tensor_free(a);
    506
    1
      ccv_nnc_tensor_free(b);
    507
    1
      ccv_nnc_tensor_free(qb);
    508
    1
      ccv_nnc_tensor_free(dq_b);
    509
    1
      ccv_nnc_tensor_free(ht);
    510
    1
      ccv_nnc_tensor_free(dbt);
    511
    1
      ccv_nnc_tensor_free(dbiast);
    512
    1
      ccv_nnc_tensor_free(h);
    513
    1
      ccv_nnc_tensor_free(db);
    514
    1
      ccv_nnc_tensor_free(dbias);
    515
    1
      ccv_nnc_tensor_free(gg);
    516
    1
      ccv_nnc_tensor_free(ga);
    517
    1
      ccv_nnc_tensor_free(gqb);
    518
    1
      ccv_nnc_tensor_free(gh);
    519
    1
      ccv_nnc_tensor_free(gdb);
    520
    1
      ccv_nnc_tensor_free(gdbias);
    521
    1
    }
    522
    523
    TEST_CASE("backward gemm with transpose a")
    524
    1
    {
    525
    1
      GUARD_ELSE_RETURN(ccv_nnc_cmd_ok(CCV_NNC_GEMM_FORWARD, CCV_NNC_BACKEND_GPU_CUBLAS) &&
    526
    1
        ccv_nnc_cmd_ok(CCV_NNC_GEMM_BACKWARD, CCV_NNC_BACKEND_GPU_CUBLAS));
    527
    1
      float gp[] = {
    528
    1
        1, 2, 3,
    529
    1
        4, 5, 6,
    530
    1
        7, 8, 9,
    531
    1
        10, 11, 12,
    532
    1
      };
    533
    1
      ccv_nnc_tensor_t* const g = ccv_nnc_tensor_new(gp, CPU_TENSOR_NHWC(32F, 4, 3), 0);
    534
    1
      float ap[] = {
    535
    1
        13, 15, 17, 19,
    536
    1
        14, 16, 18, 20,
    537
    1
      };
    538
    1
      ccv_nnc_tensor_t* const a = ccv_nnc_tensor_new(ap, CPU_TENSOR_NHWC(32F, 2, 4), 0);
    539
    1
      float bp[] = {
    540
    1
        21, 22, 23,
    541
    1
        24, 25, 26,
    542
    1
      };
    543
    1
      ccv_nnc_tensor_t* const b = ccv_nnc_tensor_new(bp, CPU_TENSOR_NHWC(32F, 2, 3), 0);
    544
    1
      ccv_nnc_tensor_t* const h = ccv_nnc_tensor_new(0, CPU_TENSOR_NHWC(32F, 2, 4), 0);
    545
    1
      ccv_nnc_tensor_t* const db = ccv_nnc_tensor_new(0, CPU_TENSOR_NHWC(32F, 2, 3), 0);
    546
    1
      ccv_nnc_tensor_t* const dbias = ccv_nnc_tensor_new(0, CPU_TENSOR_NHWC(32F, 3), 0);
    547
    1
      ccv_nnc_tensor_t* gg = ccv_nnc_tensor_new(0, GPU_TENSOR_NHWC(000, 32F, 4, 3), 0);
    548
    1
      ccv_nnc_tensor_t* ga = ccv_nnc_tensor_new(0, GPU_TENSOR_NHWC(000, 32F, 2, 4), 0);
    549
    1
      ccv_nnc_tensor_t* gb = ccv_nnc_tensor_new(0, GPU_TENSOR_NHWC(000, 32F, 2, 3), 0);
    550
    1
      ccv_nnc_tensor_t* gh = ccv_nnc_tensor_new(0, GPU_TENSOR_NHWC(000, 32F, 2, 4), 0);
    551
    1
      ccv_nnc_tensor_t* gdb = ccv_nnc_tensor_new(0, GPU_TENSOR_NHWC(000, 32F, 2, 3), 0);
    552
    1
      ccv_nnc_tensor_t* gdbias = ccv_nnc_tensor_new(0, GPU_TENSOR_NHWC(000, 32F, 3), 0);
    553
    1
      ccv_nnc_cmd_exec(CMD_DATA_TRANSFER_FORWARD(), ccv_nnc_no_hint, 0, TENSOR_LIST(g, a, b), TENSOR_LIST(gg, ga, gb), 0);
    554
    1
      ccv_nnc_cmd_exec(CMD_GEMM_BACKWARD(TRANSPOSE(0, 1)), ccv_nnc_no_hint, 0, TENSOR_LIST(gg, ga, gb), TENSOR_LIST(gh, gdb, gdbias), 0);
    555
    1
      ccv_nnc_cmd_exec(CMD_DATA_TRANSFER_FORWARD(), ccv_nnc_no_hint, 0, TENSOR_LIST(gh, gdb, gdbias), TENSOR_LIST(h, db, dbias), 0);
    556
    1
      float dbiastp[] = {
    557
    1
        22, 26, 30,
    558
    1
      };
    559
    1
      ccv_nnc_tensor_t dbiast = ccv_nnc_tensor(dbiastp, CPU_TENSOR_NHWC(32F, 3), 0);
    560
    1
      REQUIRE_TENSOR_EQ(dbias, &dbiast, "bias should be equal");
    561
    1
      float htp[] = {
    562
    1
        1 * 21 + 2 * 22 + 3 * 23, 4 * 21 + 5 * 22 + 6 * 23, 7 * 21 + 8 * 22 + 9 * 23, 10 * 21 + 11 * 22 + 12 * 23,
    563
    1
        1 * 24 + 2 * 25 + 3 * 26, 4 * 24 + 5 * 25 + 6 * 26, 7 * 24 + 8 * 25 + 9 * 26, 10 * 24 + 11 * 25 + 12 * 26,
    564
    1
      };
    565
    1
      ccv_nnc_tensor_t ht = ccv_nnc_tensor(htp, CPU_TENSOR_NHWC(32F, 2, 4), 0);
    566
    1
      REQUIRE_TENSOR_EQ(h, &ht, "h should be equal");
    567
    1
      float dbtp[] = {
    568
    1
        1 * 13 + 4 * 15 + 7 * 17 + 10 * 19, 2 * 13 + 5 * 15 + 8 * 17 + 11 * 19, 3 * 13 + 6 * 15 + 9 * 17 + 12 * 19,
    569
    1
        1 * 14 + 4 * 16 + 7 * 18 + 10 * 20, 2 * 14 + 5 * 16 + 8 * 18 + 11 * 20, 3 * 14 + 6 * 16 + 9 * 18 + 12 * 20,
    570
    1
      };
    571
    1
      ccv_nnc_tensor_t dbt = ccv_nnc_tensor(dbtp, CPU_TENSOR_NHWC(32F, 2, 3), 0);
    572
    1
      REQUIRE_TENSOR_EQ(db, &dbt, "db should be equal");
    573
    1
      ccv_nnc_tensor_free(g);
    574
    1
      ccv_nnc_tensor_free(a);
    575
    1
      ccv_nnc_tensor_free(b);
    576
    1
      ccv_nnc_tensor_free(h);
    577
    1
      ccv_nnc_tensor_free(db);
    578
    1
      ccv_nnc_tensor_free(dbias);
    579
    1
      ccv_nnc_tensor_free(gg);
    580
    1
      ccv_nnc_tensor_free(ga);
    581
    1
      ccv_nnc_tensor_free(gb);
    582
    1
      ccv_nnc_tensor_free(gh);
    583
    1
      ccv_nnc_tensor_free(gdb);
    584
    1
      ccv_nnc_tensor_free(gdbias);
    585
    1
    }
    586
    587
    TEST_CASE("backward gemm with transpose b")
    588
    1
    {
    589
    1
      GUARD_ELSE_RETURN(ccv_nnc_cmd_ok(CCV_NNC_GEMM_FORWARD, CCV_NNC_BACKEND_GPU_CUBLAS) &&
    590
    1
        ccv_nnc_cmd_ok(CCV_NNC_GEMM_BACKWARD, CCV_NNC_BACKEND_GPU_CUBLAS));
    591
    1
      float gp[] = {
    592
    1
        1, 2, 3,
    593
    1
        4, 5, 6,
    594
    1
        7, 8, 9,
    595
    1
        10, 11, 12,
    596
    1
      };
    597
    1
      ccv_nnc_tensor_t* const g = ccv_nnc_tensor_new(gp, CPU_TENSOR_NHWC(32F, 4, 3), 0);
    598
    1
      float ap[] = {
    599
    1
        13, 14,
    600
    1
        15, 16,
    601
    1
        17, 18,
    602
    1
        19, 20,
    603
    1
      };
    604
    1
      ccv_nnc_tensor_t* const a = ccv_nnc_tensor_new(ap, CPU_TENSOR_NHWC(32F, 4, 2), 0);
    605
    1
      float bp[] = {
    606
    1
        21, 24,
    607
    1
        22, 25,
    608
    1
        23, 26,
    609
    1
      };
    610
    1
      ccv_nnc_tensor_t* const b = ccv_nnc_tensor_new(bp, CPU_TENSOR_NHWC(32F, 3, 2), 0);
    611
    1
      ccv_nnc_tensor_t* const h = ccv_nnc_tensor_new(0, CPU_TENSOR_NHWC(32F, 4, 2), 0);
    612
    1
      ccv_nnc_tensor_t* const db = ccv_nnc_tensor_new(0, CPU_TENSOR_NHWC(32F, 3, 2), 0);
    613
    1
      ccv_nnc_tensor_t* const dbias = ccv_nnc_tensor_new(0, CPU_TENSOR_NHWC(32F, 3), 0);
    614
    1
      ccv_nnc_tensor_t* gg = ccv_nnc_tensor_new(0, GPU_TENSOR_NHWC(000, 32F, 4, 3), 0);
    615
    1
      ccv_nnc_tensor_t* ga = ccv_nnc_tensor_new(0, GPU_TENSOR_NHWC(000, 32F, 4, 2), 0);
    616
    1
      ccv_nnc_tensor_t* gb = ccv_nnc_tensor_new(0, GPU_TENSOR_NHWC(000, 32F, 3, 2), 0);
    617
    1
      ccv_nnc_tensor_t* gh = ccv_nnc_tensor_new(0, GPU_TENSOR_NHWC(000, 32F, 4, 2), 0);
    618
    1
      ccv_nnc_tensor_t* gdb = ccv_nnc_tensor_new(0, GPU_TENSOR_NHWC(000, 32F, 3, 2), 0);
    619
    1
      ccv_nnc_tensor_t* gdbias = ccv_nnc_tensor_new(0, GPU_TENSOR_NHWC(000, 32F, 3), 0);
    620
    1
      ccv_nnc_cmd_exec(CMD_DATA_TRANSFER_FORWARD(), ccv_nnc_no_hint, 0, TENSOR_LIST(g, a, b), TENSOR_LIST(gg, ga, gb), 0);
    621
    1
      ccv_nnc_cmd_exec(CMD_GEMM_BACKWARD(NO_TRANSPOSE, TRANSPOSE(0, 1)), ccv_nnc_no_hint, 0, TENSOR_LIST(gg, ga, gb), TENSOR_LIST(gh, gdb, gdbias), 0);
    622
    1
      ccv_nnc_cmd_exec(CMD_DATA_TRANSFER_FORWARD(), ccv_nnc_no_hint, 0, TENSOR_LIST(gh, gdb, gdbias), TENSOR_LIST(h, db, dbias), 0);
    623
    1
      float dbiastp[] = {
    624
    1
        22, 26, 30,
    625
    1
      };
    626
    1
      ccv_nnc_tensor_t dbiast = ccv_nnc_tensor(dbiastp, CPU_TENSOR_NHWC(32F, 3), 0);
    627
    1
      REQUIRE_TENSOR_EQ(dbias, &dbiast, "bias should be equal");
    628
    1
      float htp[] = {
    629
    1
        1 * 21 + 2 * 22 + 3 * 23, 1 * 24 + 2 * 25 + 3 * 26,
    630
    1
        4 * 21 + 5 * 22 + 6 * 23, 4 * 24 + 5 * 25 + 6 * 26,
    631
    1
        7 * 21 + 8 * 22 + 9 * 23, 7 * 24 + 8 * 25 + 9 * 26,
    632
    1
        10 * 21 + 11 * 22 + 12 * 23, 10 * 24 + 11 * 25 + 12 * 26,
    633
    1
      };
    634
    1
      ccv_nnc_tensor_t ht = ccv_nnc_tensor(htp, CPU_TENSOR_NHWC(32F, 4, 2), 0);
    635
    1
      REQUIRE_TENSOR_EQ(h, &ht, "h should be equal");
    636
    1
      float dbtp[] = {
    637
    1
        1 * 13 + 4 * 15 + 7 * 17 + 10 * 19, 1 * 14 + 4 * 16 + 7 * 18 + 10 * 20,
    638
    1
        2 * 13 + 5 * 15 + 8 * 17 + 11 * 19, 2 * 14 + 5 * 16 + 8 * 18 + 11 * 20,
    639
    1
        3 * 13 + 6 * 15 + 9 * 17 + 12 * 19, 3 * 14 + 6 * 16 + 9 * 18 + 12 * 20,
    640
    1
      };
    641
    1
      ccv_nnc_tensor_t dbt = ccv_nnc_tensor(dbtp, CPU_TENSOR_NHWC(32F, 3, 2), 0);
    642
    1
      REQUIRE_TENSOR_EQ(db, &dbt, "db should be equal");
    643
    1
      ccv_nnc_tensor_free(g);
    644
    1
      ccv_nnc_tensor_free(a);
    645
    1
      ccv_nnc_tensor_free(b);
    646
    1
      ccv_nnc_tensor_free(h);
    647
    1
      ccv_nnc_tensor_free(db);
    648
    1
      ccv_nnc_tensor_free(dbias);
    649
    1
      ccv_nnc_tensor_free(gg);
    650
    1
      ccv_nnc_tensor_free(ga);
    651
    1
      ccv_nnc_tensor_free(gb);
    652
    1
      ccv_nnc_tensor_free(gh);
    653
    1
      ccv_nnc_tensor_free(gdb);
    654
    1
      ccv_nnc_tensor_free(gdbias);
    655
    1
    }
    656
    657
    TEST_CASE("backward gemm with transpose a and b")
    658
    1
    {
    659
    1
      GUARD_ELSE_RETURN(ccv_nnc_cmd_ok(CCV_NNC_GEMM_FORWARD, CCV_NNC_BACKEND_GPU_CUBLAS) &&
    660
    1
        ccv_nnc_cmd_ok(CCV_NNC_GEMM_BACKWARD, CCV_NNC_BACKEND_GPU_CUBLAS));
    661
    1
      float gp[] = {
    662
    1
        1, 2, 3,
    663
    1
        4, 5, 6,
    664
    1
        7, 8, 9,
    665
    1
        10, 11, 12,
    666
    1
      };
    667
    1
      ccv_nnc_tensor_t* const g = ccv_nnc_tensor_new(gp, CPU_TENSOR_NHWC(32F, 4, 3), 0);
    668
    1
      float ap[] = {
    669
    1
        13, 15, 17, 19,
    670
    1
        14, 16, 18, 20,
    671
    1
      };
    672
    1
      ccv_nnc_tensor_t* const a = ccv_nnc_tensor_new(ap, CPU_TENSOR_NHWC(32F, 2, 4), 0);
    673
    1
      float bp[] = {
    674
    1
        21, 24,
    675
    1
        22, 25,
    676
    1
        23, 26,
    677
    1
      };
    678
    1
      ccv_nnc_tensor_t* const b = ccv_nnc_tensor_new(bp, CPU_TENSOR_NHWC(32F, 3, 2), 0);
    679
    1
      ccv_nnc_tensor_t* const h = ccv_nnc_tensor_new(0, CPU_TENSOR_NHWC(32F, 2, 4), 0);
    680
    1
      ccv_nnc_tensor_t* const db = ccv_nnc_tensor_new(0, CPU_TENSOR_NHWC(32F, 3, 2), 0);
    681
    1
      ccv_nnc_tensor_t* const dbias = ccv_nnc_tensor_new(0, CPU_TENSOR_NHWC(32F, 3), 0);
    682
    1
      ccv_nnc_tensor_t* gg = ccv_nnc_tensor_new(0, GPU_TENSOR_NHWC(000, 32F, 4, 3), 0);
    683
    1
      ccv_nnc_tensor_t* ga = ccv_nnc_tensor_new(0, GPU_TENSOR_NHWC(000, 32F, 2, 4), 0);
    684
    1
      ccv_nnc_tensor_t* gb = ccv_nnc_tensor_new(0, GPU_TENSOR_NHWC(000, 32F, 3, 2), 0);
    685
    1
      ccv_nnc_tensor_t* gh = ccv_nnc_tensor_new(0, GPU_TENSOR_NHWC(000, 32F, 2, 4), 0);
    686
    1
      ccv_nnc_tensor_t* gdb = ccv_nnc_tensor_new(0, GPU_TENSOR_NHWC(000, 32F, 3, 2), 0);
    687
    1
      ccv_nnc_tensor_t* gdbias = ccv_nnc_tensor_new(0, GPU_TENSOR_NHWC(000, 32F, 3), 0);
    688
    1
      ccv_nnc_cmd_exec(CMD_DATA_TRANSFER_FORWARD(), ccv_nnc_no_hint, 0, TENSOR_LIST(g, a, b), TENSOR_LIST(gg, ga, gb), 0);
    689
    1
      ccv_nnc_cmd_exec(CMD_GEMM_BACKWARD(TRANSPOSE(0, 1), TRANSPOSE(0, 1)), ccv_nnc_no_hint, 0, TENSOR_LIST(gg, ga, gb), TENSOR_LIST(gh, gdb, gdbias), 0);
    690
    1
      ccv_nnc_cmd_exec(CMD_DATA_TRANSFER_FORWARD(), ccv_nnc_no_hint, 0, TENSOR_LIST(gh, gdb, gdbias), TENSOR_LIST(h, db, dbias), 0);
    691
    1
      float dbiastp[] = {
    692
    1
        22, 26, 30,
    693
    1
      };
    694
    1
      ccv_nnc_tensor_t dbiast = ccv_nnc_tensor(dbiastp, CPU_TENSOR_NHWC(32F, 3), 0);
    695
    1
      REQUIRE_TENSOR_EQ(dbias, &dbiast, "bias should be equal");
    696
    1
      float htp[] = {
    697
    1
        1 * 21 + 2 * 22 + 3 * 23, 4 * 21 + 5 * 22 + 6 * 23, 7 * 21 + 8 * 22 + 9 * 23, 10 * 21 + 11 * 22 + 12 * 23,
    698
    1
        1 * 24 + 2 * 25 + 3 * 26, 4 * 24 + 5 * 25 + 6 * 26, 7 * 24 + 8 * 25 + 9 * 26, 10 * 24 + 11 * 25 + 12 * 26,
    699
    1
      };
    700
    1
      ccv_nnc_tensor_t ht = ccv_nnc_tensor(htp, CPU_TENSOR_NHWC(32F, 2, 4), 0);
    701
    1
      REQUIRE_TENSOR_EQ(h, &ht, "h should be equal");
    702
    1
      float dbtp[] = {
    703
    1
        1 * 13 + 4 * 15 + 7 * 17 + 10 * 19, 1 * 14 + 4 * 16 + 7 * 18 + 10 * 20,
    704
    1
        2 * 13 + 5 * 15 + 8 * 17 + 11 * 19, 2 * 14 + 5 * 16 + 8 * 18 + 11 * 20,
    705
    1
        3 * 13 + 6 * 15 + 9 * 17 + 12 * 19, 3 * 14 + 6 * 16 + 9 * 18 + 12 * 20,
    706
    1
      };
    707
    1
      ccv_nnc_tensor_t dbt = ccv_nnc_tensor(dbtp, CPU_TENSOR_NHWC(32F, 3, 2), 0);
    708
    1
      REQUIRE_TENSOR_EQ(db, &dbt, "db should be equal");
    709
    1
      ccv_nnc_tensor_free(g);
    710
    1
      ccv_nnc_tensor_free(a);
    711
    1
      ccv_nnc_tensor_free(b);
    712
    1
      ccv_nnc_tensor_free(h);
    713
    1
      ccv_nnc_tensor_free(db);
    714
    1
      ccv_nnc_tensor_free(dbias);
    715
    1
      ccv_nnc_tensor_free(gg);
    716
    1
      ccv_nnc_tensor_free(ga);
    717
    1
      ccv_nnc_tensor_free(gb);
    718
    1
      ccv_nnc_tensor_free(gh);
    719
    1
      ccv_nnc_tensor_free(gdb);
    720
    1
      ccv_nnc_tensor_free(gdbias);
    721
    1
    }
    722
    723
    TEST_CASE("gemm no transpose batch 2")
    724
    1
    {
    725
    1
      GUARD_ELSE_RETURN(ccv_nnc_cmd_ok(CCV_NNC_GEMM_FORWARD, CCV_NNC_BACKEND_GPU_CUBLAS));
    726
    1
      float ap[] = {
    727
    1
        1, 2,
    728
    1
        3, 4,
    729
    1
        5, 6,
    730
    1
        7, 8,
    731
    1
        2, 3,
    732
    1
        4, 5,
    733
    1
        6, 7,
    734
    1
        8, 9
    735
    1
      };
    736
    1
      ccv_nnc_tensor_t* const a = ccv_nnc_tensor_new(ap, CPU_TENSOR_NHWC(32F, 2, 4, 2), 0);
    737
    1
      float bp[] = {
    738
    1
        7, 8, 9,
    739
    1
        10, 11, 12,
    740
    1
      };
    741
    1
      ccv_nnc_tensor_t* const b = ccv_nnc_tensor_new(bp, CPU_TENSOR_NHWC(32F, 2, 3), 0);
    742
    1
      ccv_nnc_tensor_t* const c = ccv_nnc_tensor_new(0, CPU_TENSOR_NHWC(32F, 2, 4, 3), 0);
    743
    1
      ccv_nnc_tensor_t* ga = ccv_nnc_tensor_new(0, GPU_TENSOR_NHWC(000, 32F, 2, 4, 2), 0);
    744
    1
      ccv_nnc_tensor_t* gb = ccv_nnc_tensor_new(0, GPU_TENSOR_NHWC(000, 32F, 2, 3), 0);
    745
    1
      ccv_nnc_tensor_t* gc = ccv_nnc_tensor_new(0, GPU_TENSOR_NHWC(000, 32F, 2, 4, 3), 0);
    746
    1
      ccv_nnc_cmd_exec(CMD_DATA_TRANSFER_FORWARD(), ccv_nnc_no_hint, 0, TENSOR_LIST(a, b), TENSOR_LIST(ga, gb), 0);
    747
    1
      ccv_nnc_cmd_exec(CMD_GEMM_FORWARD(), ccv_nnc_no_hint, 0, TENSOR_LIST(ga, gb), TENSOR_LIST(gc), 0);
    748
    1
      ccv_nnc_cmd_exec(CMD_DATA_TRANSFER_FORWARD(), ccv_nnc_no_hint, 0, TENSOR_LIST(gc), TENSOR_LIST(c), 0);
    749
    1
      float ctp[] = {
    750
    1
        1 * 7 + 2 * 10, 1 * 8 + 2 * 11, 1 * 9 + 2 * 12,
    751
    1
        3 * 7 + 4 * 10, 3 * 8 + 4 * 11, 3 * 9 + 4 * 12,
    752
    1
        5 * 7 + 6 * 10, 5 * 8 + 6 * 11, 5 * 9 + 6 * 12,
    753
    1
        7 * 7 + 8 * 10, 7 * 8 + 8 * 11, 7 * 9 + 8 * 12,
    754
    1
        2 * 7 + 3 * 10, 2 * 8 + 3 * 11, 2 * 9 + 3 * 12,
    755
    1
        4 * 7 + 5 * 10, 4 * 8 + 5 * 11, 4 * 9 + 5 * 12,
    756
    1
        6 * 7 + 7 * 10, 6 * 8 + 7 * 11, 6 * 9 + 7 * 12,
    757
    1
        8 * 7 + 9 * 10, 8 * 8 + 9 * 11, 8 * 9 + 9 * 12,
    758
    1
      };
    759
    1
      ccv_nnc_tensor_t ct = ccv_nnc_tensor(ctp, CPU_TENSOR_NHWC(32F, 2, 4, 3), 0);
    760
    1
      REQUIRE_TENSOR_EQ(c, &ct, "result should be equal");
    761
    1
      ccv_nnc_tensor_free(a);
    762
    1
      ccv_nnc_tensor_free(b);
    763
    1
      ccv_nnc_tensor_free(c);
    764
    1
      ccv_nnc_tensor_free(ga);
    765
    1
      ccv_nnc_tensor_free(gb);
    766
    1
      ccv_nnc_tensor_free(gc);
    767
    1
    }
    768
    769
    TEST_CASE("gemm transpose a batch 2")
    770
    1
    {
    771
    1
      GUARD_ELSE_RETURN(ccv_nnc_cmd_ok(CCV_NNC_GEMM_FORWARD, CCV_NNC_BACKEND_GPU_CUBLAS));
    772
    1
      float ap[] = {
    773
    1
        1, 3, 5, 7,
    774
    1
        2, 4, 6, 8,
    775
    1
        2, 4, 6, 8,
    776
    1
        3, 5, 7, 9,
    777
    1
      };
    778
    1
      ccv_nnc_tensor_t* const a = ccv_nnc_tensor_new(ap, CPU_TENSOR_NHWC(32F, 2, 2, 4), 0);
    779
    1
      float bp[] = {
    780
    1
        7, 8, 9,
    781
    1
        10, 11, 12,
    782
    1
      };
    783
    1
      ccv_nnc_tensor_t* const b = ccv_nnc_tensor_new(bp, CPU_TENSOR_NHWC(32F, 2, 3), 0);
    784
    1
      float dp[] = {
    785
    1
        -1, 0, 1,
    786
    1
      };
    787
    1
      ccv_nnc_tensor_t* const d = ccv_nnc_tensor_new(dp, CPU_TENSOR_NHWC(32F, 3), 0);
    788
    1
      ccv_nnc_tensor_t* const c = ccv_nnc_tensor_new(0, CPU_TENSOR_NHWC(32F, 2, 4, 3), 0);
    789
    1
      ccv_nnc_tensor_t* ga = ccv_nnc_tensor_new(0, GPU_TENSOR_NHWC(000, 32F, 2, 2, 4), 0);
    790
    1
      ccv_nnc_tensor_t* gb = ccv_nnc_tensor_new(0, GPU_TENSOR_NHWC(000, 32F, 2, 3), 0);
    791
    1
      ccv_nnc_tensor_t* gc = ccv_nnc_tensor_new(0, GPU_TENSOR_NHWC(000, 32F, 2, 4, 3), 0);
    792
    1
      ccv_nnc_tensor_t* gd = ccv_nnc_tensor_new(0, GPU_TENSOR_NHWC(000, 32F, 3), 0);
    793
    1
      ccv_nnc_cmd_exec(CMD_DATA_TRANSFER_FORWARD(), ccv_nnc_no_hint, 0, TENSOR_LIST(a, b, d), TENSOR_LIST(ga, gb, gd), 0);
    794
    1
      ccv_nnc_cmd_exec(CMD_GEMM_FORWARD(TRANSPOSE(1, 2)), ccv_nnc_no_hint, 0, TENSOR_LIST(ga, gb, gd), TENSOR_LIST(gc), 0);
    795
    1
      ccv_nnc_cmd_exec(CMD_DATA_TRANSFER_FORWARD(), ccv_nnc_no_hint, 0, TENSOR_LIST(gc), TENSOR_LIST(c), 0);
    796
    1
      float ctp[] = {
    797
    1
        1 * 7 + 2 * 10 - 1, 1 * 8 + 2 * 11, 1 * 9 + 2 * 12 + 1,
    798
    1
        3 * 7 + 4 * 10 - 1, 3 * 8 + 4 * 11, 3 * 9 + 4 * 12 + 1,
    799
    1
        5 * 7 + 6 * 10 - 1, 5 * 8 + 6 * 11, 5 * 9 + 6 * 12 + 1,
    800
    1
        7 * 7 + 8 * 10 - 1, 7 * 8 + 8 * 11, 7 * 9 + 8 * 12 + 1,
    801
    1
        2 * 7 + 3 * 10 - 1, 2 * 8 + 3 * 11, 2 * 9 + 3 * 12 + 1,
    802
    1
        4 * 7 + 5 * 10 - 1, 4 * 8 + 5 * 11, 4 * 9 + 5 * 12 + 1,
    803
    1
        6 * 7 + 7 * 10 - 1, 6 * 8 + 7 * 11, 6 * 9 + 7 * 12 + 1,
    804
    1
        8 * 7 + 9 * 10 - 1, 8 * 8 + 9 * 11, 8 * 9 + 9 * 12 + 1,
    805
    1
      };
    806
    1
      ccv_nnc_tensor_t ct = ccv_nnc_tensor(ctp, CPU_TENSOR_NHWC(32F, 2, 4, 3), 0);
    807
    1
      REQUIRE_TENSOR_EQ(c, &ct, "result should be equal");
    808
    1
      ccv_nnc_tensor_free(a);
    809
    1
      ccv_nnc_tensor_free(b);
    810
    1
      ccv_nnc_tensor_free(c);
    811
    1
      ccv_nnc_tensor_free(d);
    812
    1
      ccv_nnc_tensor_free(ga);
    813
    1
      ccv_nnc_tensor_free(gb);
    814
    1
      ccv_nnc_tensor_free(gc);
    815
    1
      ccv_nnc_tensor_free(gd);
    816
    1
    }
    817
    818
    TEST_CASE("gemm transpose b batch 2")
    819
    1
    {
    820
    1
      GUARD_ELSE_RETURN(ccv_nnc_cmd_ok(CCV_NNC_GEMM_FORWARD, CCV_NNC_BACKEND_GPU_CUBLAS));
    821
    1
      float ap[] = {
    822
    1
        1, 2,
    823
    1
        3, 4,
    824
    1
        5, 6,
    825
    1
        7, 8,
    826
    1
        2, 3,
    827
    1
        4, 5,
    828
    1
        6, 7,
    829
    1
        8, 9
    830
    1
      };
    831
    1
      ccv_nnc_tensor_t* const a = ccv_nnc_tensor_new(ap, CPU_TENSOR_NHWC(32F, 2, 4, 2), 0);
    832
    1
      float bp[] = {
    833
    1
        7, 10,
    834
    1
        8, 11,
    835
    1
        9, 12,
    836
    1
        80, 110,
    837
    1
        90, 120,
    838
    1
        10, 13,
    839
    1
      };
    840
    1
      ccv_nnc_tensor_t* const b = ccv_nnc_tensor_new(bp, CPU_TENSOR_NHWC(32F, 2, 3, 2), 0);
    841
    1
      float dp[] = {
    842
    1
        -1, 0, 1,
    843
    1
        2, 3, -4,
    844
    1
      };
    845
    1
      ccv_nnc_tensor_t* const d = ccv_nnc_tensor_new(dp, CPU_TENSOR_NHWC(32F, 2, 1, 3), 0);
    846
    1
      ccv_nnc_tensor_t* const c = ccv_nnc_tensor_new(0, CPU_TENSOR_NHWC(32F, 2, 4, 3), 0);
    847
    1
      ccv_nnc_tensor_t* ga = ccv_nnc_tensor_new(0, GPU_TENSOR_NHWC(000, 32F, 2, 4, 2), 0);
    848
    1
      ccv_nnc_tensor_t* gb = ccv_nnc_tensor_new(0, GPU_TENSOR_NHWC(000, 32F, 2, 3, 2), 0);
    849
    1
      ccv_nnc_tensor_t* gc = ccv_nnc_tensor_new(0, GPU_TENSOR_NHWC(000, 32F, 2, 4, 3), 0);
    850
    1
      ccv_nnc_tensor_t* gd = ccv_nnc_tensor_new(0, GPU_TENSOR_NHWC(000, 32F, 2, 1, 3), 0);
    851
    1
      ccv_nnc_cmd_exec(CMD_DATA_TRANSFER_FORWARD(), ccv_nnc_no_hint, 0, TENSOR_LIST(a, b, d), TENSOR_LIST(ga, gb, gd), 0);
    852
    1
      ccv_nnc_cmd_exec(CMD_GEMM_FORWARD(NO_TRANSPOSE, TRANSPOSE(1, 2)), ccv_nnc_no_hint, 0, TENSOR_LIST(ga, gb, gd), TENSOR_LIST(gc), 0);
    853
    1
      ccv_nnc_cmd_exec(CMD_DATA_TRANSFER_FORWARD(), ccv_nnc_no_hint, 0, TENSOR_LIST(gc), TENSOR_LIST(c), 0);
    854
    1
      float ctp[] = {
    855
    1
        1 * 7 + 2 * 10 - 1, 1 * 8 + 2 * 11, 1 * 9 + 2 * 12 + 1,
    856
    1
        3 * 7 + 4 * 10 - 1, 3 * 8 + 4 * 11, 3 * 9 + 4 * 12 + 1,
    857
    1
        5 * 7 + 6 * 10 - 1, 5 * 8 + 6 * 11, 5 * 9 + 6 * 12 + 1,
    858
    1
        7 * 7 + 8 * 10 - 1, 7 * 8 + 8 * 11, 7 * 9 + 8 * 12 + 1,
    859
    1
        2 * 80 + 3 * 110 + 2, 2 * 90 + 3 * 120 + 3, 2 * 10 + 3 * 13 - 4,
    860
    1
        4 * 80 + 5 * 110 + 2, 4 * 90 + 5 * 120 + 3, 4 * 10 + 5 * 13 - 4,
    861
    1
        6 * 80 + 7 * 110 + 2, 6 * 90 + 7 * 120 + 3, 6 * 10 + 7 * 13 - 4,
    862
    1
        8 * 80 + 9 * 110 + 2, 8 * 90 + 9 * 120 + 3, 8 * 10 + 9 * 13 - 4,
    863
    1
      };
    864
    1
      ccv_nnc_tensor_t ct = ccv_nnc_tensor(ctp, CPU_TENSOR_NHWC(32F, 2, 4, 3), 0);
    865
    1
      REQUIRE_TENSOR_EQ(c, &ct, "result should be equal");
    866
    1
      ccv_nnc_tensor_free(a);
    867
    1
      ccv_nnc_tensor_free(b);
    868
    1
      ccv_nnc_tensor_free(c);
    869
    1
      ccv_nnc_tensor_free(d);
    870
    1
      ccv_nnc_tensor_free(ga);
    871
    1
      ccv_nnc_tensor_free(gb);
    872
    1
      ccv_nnc_tensor_free(gc);
    873
    1
      ccv_nnc_tensor_free(gd);
    874
    1
    }
    875
    876
    TEST_CASE("backward gemm with no transpose batch 2, same b")
    877
    1
    {
    878
    1
      GUARD_ELSE_RETURN(ccv_nnc_cmd_ok(CCV_NNC_GEMM_FORWARD, CCV_NNC_BACKEND_GPU_CUBLAS) &&
    879
    1
        ccv_nnc_cmd_ok(CCV_NNC_GEMM_BACKWARD, CCV_NNC_BACKEND_GPU_CUBLAS));
    880
    1
      float gp[] = {
    881
    1
        1, 2, 3,
    882
    1
        4, 5, 6,
    883
    1
        7, 8, 9,
    884
    1
        10, 11, 12,
    885
    1
        10, 20, 30,
    886
    1
        40, 50, 60,
    887
    1
        70, 80, 90,
    888
    1
        100, 110, 120,
    889
    1
      };
    890
    1
      ccv_nnc_tensor_t* const g = ccv_nnc_tensor_new(gp, CPU_TENSOR_NHWC(32F, 2, 4, 3), 0);
    891
    1
      float ap[] = {
    892
    1
        13, 14,
    893
    1
        15, 16,
    894
    1
        17, 18,
    895
    1
        19, 20,
    896
    1
        131, 141,
    897
    1
        151, 161,
    898
    1
        171, 181,
    899
    1
        191, 201,
    900
    1
      };
    901
    1
      ccv_nnc_tensor_t* const a = ccv_nnc_tensor_new(ap, CPU_TENSOR_NHWC(32F, 2, 4, 2), 0);
    902
    1
      float bp[] = {
    903
    1
        21, 22, 23,
    904
    1
        24, 25, 26,
    905
    1
      };
    906
    1
      ccv_nnc_tensor_t* const b = ccv_nnc_tensor_new(bp, CPU_TENSOR_NHWC(32F, 2, 3), 0);
    907
    1
      ccv_nnc_tensor_t* const h = ccv_nnc_tensor_new(0, CPU_TENSOR_NHWC(32F, 2, 4, 2), 0);
    908
    1
      ccv_nnc_tensor_t* const db = ccv_nnc_tensor_new(0, CPU_TENSOR_NHWC(32F, 2, 3), 0);
    909
    1
      ccv_nnc_tensor_t* const dbias = ccv_nnc_tensor_new(0, CPU_TENSOR_NHWC(32F, 3), 0);
    910
    1
      ccv_nnc_tensor_t* gg = ccv_nnc_tensor_new(0, GPU_TENSOR_NHWC(000, 32F, 2, 4, 3), 0);
    911
    1
      ccv_nnc_tensor_t* ga = ccv_nnc_tensor_new(0, GPU_TENSOR_NHWC(000, 32F, 2, 4, 2), 0);
    912
    1
      ccv_nnc_tensor_t* gb = ccv_nnc_tensor_new(0, GPU_TENSOR_NHWC(000, 32F, 2, 3), 0);
    913
    1
      ccv_nnc_tensor_t* gh = ccv_nnc_tensor_new(0, GPU_TENSOR_NHWC(000, 32F, 2, 4, 2), 0);
    914
    1
      ccv_nnc_tensor_t* gdb = ccv_nnc_tensor_new(0, GPU_TENSOR_NHWC(000, 32F, 2, 3), 0);
    915
    1
      ccv_nnc_tensor_t* gdbias = ccv_nnc_tensor_new(0, GPU_TENSOR_NHWC(000, 32F, 3), 0);
    916
    1
      ccv_nnc_cmd_exec(CMD_DATA_TRANSFER_FORWARD(), ccv_nnc_no_hint, 0, TENSOR_LIST(g, a, b), TENSOR_LIST(gg, ga, gb), 0);
    917
    1
      ccv_nnc_cmd_exec(CMD_GEMM_BACKWARD(), ccv_nnc_no_hint, 0, TENSOR_LIST(gg, ga, gb), TENSOR_LIST(gh, gdb, gdbias), 0);
    918
    1
      ccv_nnc_cmd_exec(CMD_DATA_TRANSFER_FORWARD(), ccv_nnc_no_hint, 0, TENSOR_LIST(gh, gdb, gdbias), TENSOR_LIST(h, db, dbias), 0);
    919
    1
      float dbiastp[] = {
    920
    1
        22 + 220, 26 + 260, 30 + 300,
    921
    1
      };
    922
    1
      ccv_nnc_tensor_t dbiast = ccv_nnc_tensor(dbiastp, CPU_TENSOR_NHWC(32F, 3), 0);
    923
    1
      REQUIRE_TENSOR_EQ(dbias, &dbiast, "bias should be equal");
    924
    1
      float htp[] = {
    925
    1
        1 * 21 + 2 * 22 + 3 * 23, 1 * 24 + 2 * 25 + 3 * 26,
    926
    1
        4 * 21 + 5 * 22 + 6 * 23, 4 * 24 + 5 * 25 + 6 * 26,
    927
    1
        7 * 21 + 8 * 22 + 9 * 23, 7 * 24 + 8 * 25 + 9 * 26,
    928
    1
        10 * 21 + 11 * 22 + 12 * 23, 10 * 24 + 11 * 25 + 12 * 26,
    929
    1
        10 * 21 + 20 * 22 + 30 * 23, 10 * 24 + 20 * 25 + 30 * 26,
    930
    1
        40 * 21 + 50 * 22 + 60 * 23, 40 * 24 + 50 * 25 + 60 * 26,
    931
    1
        70 * 21 + 80 * 22 + 90 * 23, 70 * 24 + 80 * 25 + 90 * 26,
    932
    1
        100 * 21 + 110 * 22 + 120 * 23, 100 * 24 + 110 * 25 + 120 * 26,
    933
    1
      };
    934
    1
      ccv_nnc_tensor_t ht = ccv_nnc_tensor(htp, CPU_TENSOR_NHWC(32F, 2, 4, 2), 0);
    935
    1
      REQUIRE_TENSOR_EQ(h, &ht, "h should be equal");
    936
    1
      float dbtp[] = {
    937
    1
        1 * 13 + 4 * 15 + 7 * 17 + 10 * 19 + 10 * 131 + 40 * 151 + 70 * 171 + 100 * 191, 2 * 13 + 5 * 15 + 8 * 17 + 11 * 19 + 20 * 131 + 50 * 151 + 80 * 171 + 110 * 191, 3 * 13 + 6 * 15 + 9 * 17 + 12 * 19 + 30 * 131 + 60 * 151 + 90 * 171 + 120 * 191,
    938
    1
        1 * 14 + 4 * 16 + 7 * 18 + 10 * 20 + 10 * 141 + 40 * 161 + 70 * 181 + 100 * 201, 2 * 14 + 5 * 16 + 8 * 18 + 11 * 20 + 20 * 141 + 50 * 161 + 80 * 181 + 110 * 201, 3 * 14 + 6 * 16 + 9 * 18 + 12 * 20 + 30 * 141 + 60 * 161 + 90 * 181 + 120 * 201,
    939
    1
      };
    940
    1
      ccv_nnc_tensor_t dbt = ccv_nnc_tensor(dbtp, CPU_TENSOR_NHWC(32F, 2, 3), 0);
    941
    1
      REQUIRE_TENSOR_EQ(db, &dbt, "db should be equal");
    942
    1
      ccv_nnc_tensor_free(g);
    943
    1
      ccv_nnc_tensor_free(a);
    944
    1
      ccv_nnc_tensor_free(b);
    945
    1
      ccv_nnc_tensor_free(h);
    946
    1
      ccv_nnc_tensor_free(db);
    947
    1
      ccv_nnc_tensor_free(dbias);
    948
    1
      ccv_nnc_tensor_free(gg);
    949
    1
      ccv_nnc_tensor_free(ga);
    950
    1
      ccv_nnc_tensor_free(gb);
    951
    1
      ccv_nnc_tensor_free(gh);
    952
    1
      ccv_nnc_tensor_free(gdb);
    953
    1
      ccv_nnc_tensor_free(gdbias);
    954
    1
    }
    955
    956
    TEST_CASE("backward gemm with no transpose batch 2, batched b")
    957
    1
    {
    958
    1
      GUARD_ELSE_RETURN(ccv_nnc_cmd_ok(CCV_NNC_GEMM_FORWARD, CCV_NNC_BACKEND_GPU_CUBLAS) &&
    959
    1
        ccv_nnc_cmd_ok(CCV_NNC_GEMM_BACKWARD, CCV_NNC_BACKEND_GPU_CUBLAS));
    960
    1
      float gp[] = {
    961
    1
        1, 2, 3,
    962
    1
        4, 5, 6,
    963
    1
        7, 8, 9,
    964
    1
        10, 11, 12,
    965
    1
        10, 20, 30,
    966
    1
        40, 50, 60,
    967
    1
        70, 80, 90,
    968
    1
        100, 110, 120,
    969
    1
      };
    970
    1
      ccv_nnc_tensor_t* const g = ccv_nnc_tensor_new(gp, CPU_TENSOR_NHWC(32F, 2, 4, 3), 0);
    971
    1
      float ap[] = {
    972
    1
        13, 14,
    973
    1
        15, 16,
    974
    1
        17, 18,
    975
    1
        19, 20,
    976
    1
        131, 141,
    977
    1
        151, 161,
    978
    1
        171, 181,
    979
    1
        191, 201,
    980
    1
      };
    981
    1
      ccv_nnc_tensor_t* const a = ccv_nnc_tensor_new(ap, CPU_TENSOR_NHWC(32F, 2, 4, 2), 0);
    982
    1
      float bp[] = {
    983
    1
        21, 22, 23,
    984
    1
        24, 25, 26,
    985
    1
        212, 222, 232,
    986
    1
        242, 252, 262,
    987
    1
      };
    988
    1
      ccv_nnc_tensor_t* const b = ccv_nnc_tensor_new(bp, CPU_TENSOR_NHWC(32F, 2, 2, 3), 0);
    989
    1
      ccv_nnc_tensor_t* const h = ccv_nnc_tensor_new(0, CPU_TENSOR_NHWC(32F, 2, 4, 2), 0);
    990
    1
      ccv_nnc_tensor_t* const db = ccv_nnc_tensor_new(0, CPU_TENSOR_NHWC(32F, 2, 2, 3), 0);
    991
    1
      ccv_nnc_tensor_t* const dbias = ccv_nnc_tensor_new(0, CPU_TENSOR_NHWC(32F, 2, 1, 3), 0);
    992
    1
      ccv_nnc_tensor_t* const gg = ccv_nnc_tensor_new(0, GPU_TENSOR_NHWC(000, 32F, 2, 4, 3), 0);
    993
    1
      ccv_nnc_tensor_t* const ga = ccv_nnc_tensor_new(0, GPU_TENSOR_NHWC(000, 32F, 2, 4, 2), 0);
    994
    1
      ccv_nnc_tensor_t* const gb = ccv_nnc_tensor_new(0, GPU_TENSOR_NHWC(000, 32F, 2, 2, 3), 0);
    995
    1
      ccv_nnc_tensor_t* const gh = ccv_nnc_tensor_new(0, GPU_TENSOR_NHWC(000, 32F, 2, 4, 2), 0);
    996
    1
      ccv_nnc_tensor_t* const gdb = ccv_nnc_tensor_new(0, GPU_TENSOR_NHWC(000, 32F, 2, 2, 3), 0);
    997
    1
      ccv_nnc_tensor_t* const gdbias = ccv_nnc_tensor_new(0, GPU_TENSOR_NHWC(000, 32F, 2, 1, 3), 0);
    998
    1
      ccv_nnc_cmd_exec(CMD_DATA_TRANSFER_FORWARD(), ccv_nnc_no_hint, 0, TENSOR_LIST(g, a, b), TENSOR_LIST(gg, ga, gb), 0);
    999
    1
      ccv_nnc_cmd_exec(CMD_GEMM_BACKWARD(), ccv_nnc_no_hint, 0, TENSOR_LIST(gg, ga, gb), TENSOR_LIST(gh, gdb, gdbias), 0);
    1000
    1
      ccv_nnc_cmd_exec(CMD_DATA_TRANSFER_FORWARD(), ccv_nnc_no_hint, 0, TENSOR_LIST(gh, gdb, gdbias), TENSOR_LIST(h, db, dbias), 0);
    1001
    1
      float dbiastp[] = {
    1002
    1
        22, 26, 30,
    1003
    1
        220, 260, 300,
    1004
    1
      };
    1005
    1
      ccv_nnc_tensor_t dbiast = ccv_nnc_tensor(dbiastp, CPU_TENSOR_NHWC(32F, 2, 1, 3), 0);
    1006
    1
      REQUIRE_TENSOR_EQ(dbias, &dbiast, "bias should be equal");
    1007
    1
      float htp[] = {
    1008
    1
        1 * 21 + 2 * 22 + 3 * 23, 1 * 24 + 2 * 25 + 3 * 26,
    1009
    1
        4 * 21 + 5 * 22 + 6 * 23, 4 * 24 + 5 * 25 + 6 * 26,
    1010
    1
        7 * 21 + 8 * 22 + 9 * 23, 7 * 24 + 8 * 25 + 9 * 26,
    1011
    1
        10 * 21 + 11 * 22 + 12 * 23, 10 * 24 + 11 * 25 + 12 * 26,
    1012
    1
        10 * 212 + 20 * 222 + 30 * 232, 10 * 242 + 20 * 252 + 30 * 262,
    1013
    1
        40 * 212 + 50 * 222 + 60 * 232, 40 * 242 + 50 * 252 + 60 * 262,
    1014
    1
        70 * 212 + 80 * 222 + 90 * 232, 70 * 242 + 80 * 252 + 90 * 262,
    1015
    1
        100 * 212 + 110 * 222 + 120 * 232, 100 * 242 + 110 * 252 + 120 * 262,
    1016
    1
      };
    1017
    1
      ccv_nnc_tensor_t ht = ccv_nnc_tensor(htp, CPU_TENSOR_NHWC(32F, 2, 4, 2), 0);
    1018
    1
      REQUIRE_TENSOR_EQ(h, &ht, "h should be equal");
    1019
    1
      float dbtp[] = {
    1020
    1
        1 * 13 + 4 * 15 + 7 * 17 + 10 * 19, 2 * 13 + 5 * 15 + 8 * 17 + 11 * 19, 3 * 13 + 6 * 15 + 9 * 17 + 12 * 19,
    1021
    1
        1 * 14 + 4 * 16 + 7 * 18 + 10 * 20, 2 * 14 + 5 * 16 + 8 * 18 + 11 * 20, 3 * 14 + 6 * 16 + 9 * 18 + 12 * 20,
    1022
    1
        10 * 131 + 40 * 151 + 70 * 171 + 100 * 191, 20 * 131 + 50 * 151 + 80 * 171 + 110 * 191, 30 * 131 + 60 * 151 + 90 * 171 + 120 * 191,
    1023
    1
        10 * 141 + 40 * 161 + 70 * 181 + 100 * 201, 20 * 141 + 50 * 161 + 80 * 181 + 110 * 201, 30 * 141 + 60 * 161 + 90 * 181 + 120 * 201,
    1024
    1
      };
    1025
    1
      ccv_nnc_tensor_t dbt = ccv_nnc_tensor(dbtp, CPU_TENSOR_NHWC(32F, 2, 2, 3), 0);
    1026
    1
      REQUIRE_TENSOR_EQ(db, &dbt, "db should be equal");
    1027
    1
      ccv_nnc_tensor_free(g);
    1028
    1
      ccv_nnc_tensor_free(a);
    1029
    1
      ccv_nnc_tensor_free(b);
    1030
    1
      ccv_nnc_tensor_free(h);
    1031
    1
      ccv_nnc_tensor_free(db);
    1032
    1
      ccv_nnc_tensor_free(dbias);
    1033
    1
      ccv_nnc_tensor_free(gg);
    1034
    1
      ccv_nnc_tensor_free(ga);
    1035
    1
      ccv_nnc_tensor_free(gb);
    1036
    1
      ccv_nnc_tensor_free(gh);
    1037
    1
      ccv_nnc_tensor_free(gdb);
    1038
    1
      ccv_nnc_tensor_free(gdbias);
    1039
    1
    }
    1040
    1041
    TEST_CASE("backward gemm with transpose a batch 2, same b")
    1042
    1
    {
    1043
    1
      GUARD_ELSE_RETURN(ccv_nnc_cmd_ok(CCV_NNC_GEMM_FORWARD, CCV_NNC_BACKEND_GPU_CUBLAS) &&
    1044
    1
        ccv_nnc_cmd_ok(CCV_NNC_GEMM_BACKWARD, CCV_NNC_BACKEND_GPU_CUBLAS));
    1045
    1
      float gp[] = {
    1046
    1
        1, 2, 3,
    1047
    1
        4, 5, 6,
    1048
    1
        7, 8, 9,
    1049
    1
        10, 11, 12,
    1050
    1
        10, 20, 30,
    1051
    1
        40, 50, 60,
    1052
    1
        70, 80, 90,
    1053
    1
        100, 110, 120,
    1054
    1
      };
    1055
    1
      ccv_nnc_tensor_t* const g = ccv_nnc_tensor_new(gp, CPU_TENSOR_NHWC(32F, 2, 4, 3), 0);
    1056
    1
      float ap[] = {
    1057
    1
        13, 15, 17, 19,
    1058
    1
        14, 16, 18, 20,
    1059
    1
        131, 151, 171, 191,
    1060
    1
        141, 161, 181, 201,
    1061
    1
      };
    1062
    1
      ccv_nnc_tensor_t* const a = ccv_nnc_tensor_new(ap, CPU_TENSOR_NHWC(32F, 2, 2, 4), 0);
    1063
    1
      float bp[] = {
    1064
    1
        21, 22, 23,
    1065
    1
        24, 25, 26,
    1066
    1
      };
    1067
    1
      ccv_nnc_tensor_t* const b = ccv_nnc_tensor_new(bp, CPU_TENSOR_NHWC(32F, 2, 3), 0);
    1068
    1
      ccv_nnc_tensor_t* const h = ccv_nnc_tensor_new(0, CPU_TENSOR_NHWC(32F, 2, 2, 4), 0);
    1069
    1
      ccv_nnc_tensor_t* const db = ccv_nnc_tensor_new(0, CPU_TENSOR_NHWC(32F, 2, 3), 0);
    1070
    1
      ccv_nnc_tensor_t* const dbias = ccv_nnc_tensor_new(0, CPU_TENSOR_NHWC(32F, 3), 0);
    1071
    1
      ccv_nnc_tensor_t* const gg = ccv_nnc_tensor_new(0, GPU_TENSOR_NHWC(000, 32F, 2, 4, 3), 0);
    1072
    1
      ccv_nnc_tensor_t* const ga = ccv_nnc_tensor_new(0, GPU_TENSOR_NHWC(000, 32F, 2, 2, 4), 0);
    1073
    1
      ccv_nnc_tensor_t* const gb = ccv_nnc_tensor_new(0, GPU_TENSOR_NHWC(000, 32F, 2, 3), 0);
    1074
    1
      ccv_nnc_tensor_t* const gh = ccv_nnc_tensor_new(0, GPU_TENSOR_NHWC(000, 32F, 2, 2, 4), 0);
    1075
    1
      ccv_nnc_tensor_t* const gdb = ccv_nnc_tensor_new(0, GPU_TENSOR_NHWC(000, 32F, 2, 3), 0);
    1076
    1
      ccv_nnc_tensor_t* const gdbias = ccv_nnc_tensor_new(0, GPU_TENSOR_NHWC(000, 32F, 3), 0);
    1077
    1
      ccv_nnc_cmd_exec(CMD_DATA_TRANSFER_FORWARD(), ccv_nnc_no_hint, 0, TENSOR_LIST(g, a, b), TENSOR_LIST(gg, ga, gb), 0);
    1078
    1
      ccv_nnc_cmd_exec(CMD_GEMM_BACKWARD(TRANSPOSE(1, 2)), ccv_nnc_no_hint, 0, TENSOR_LIST(gg, ga, gb), TENSOR_LIST(gh, gdb, gdbias), 0);
    1079
    1
      ccv_nnc_cmd_exec(CMD_DATA_TRANSFER_FORWARD(), ccv_nnc_no_hint, 0, TENSOR_LIST(gh, gdb, gdbias), TENSOR_LIST(h, db, dbias), 0);
    1080
    1
      float dbiastp[] = {
    1081
    1
        22 + 220, 26 + 260, 30 + 300,
    1082
    1
      };
    1083
    1
      ccv_nnc_tensor_t dbiast = ccv_nnc_tensor(dbiastp, CPU_TENSOR_NHWC(32F, 3), 0);
    1084
    1
      REQUIRE_TENSOR_EQ(dbias, &dbiast, "bias should be equal");
    1085
    1
      float htp[] = {
    1086
    1
        1 * 21 + 2 * 22 + 3 * 23, 4 * 21 + 5 * 22 + 6 * 23, 7 * 21 + 8 * 22 + 9 * 23, 10 * 21 + 11 * 22 + 12 * 23,
    1087
    1
        1 * 24 + 2 * 25 + 3 * 26, 4 * 24 + 5 * 25 + 6 * 26, 7 * 24 + 8 * 25 + 9 * 26, 10 * 24 + 11 * 25 + 12 * 26,
    1088
    1
        10 * 21 + 20 * 22 + 30 * 23, 40 * 21 + 50 * 22 + 60 * 23, 70 * 21 + 80 * 22 + 90 * 23, 100 * 21 + 110 * 22 + 120 * 23,
    1089
    1
        10 * 24 + 20 * 25 + 30 * 26, 40 * 24 + 50 * 25 + 60 * 26, 70 * 24 + 80 * 25 + 90 * 26, 100 * 24 + 110 * 25 + 120 * 26,
    1090
    1
      };
    1091
    1
      ccv_nnc_tensor_t ht = ccv_nnc_tensor(htp, CPU_TENSOR_NHWC(32F, 2, 2, 4), 0);
    1092
    1
      REQUIRE_TENSOR_EQ(h, &ht, "h should be equal");
    1093
    1
      float dbtp[] = {
    1094
    1
        1 * 13 + 4 * 15 + 7 * 17 + 10 * 19 + 10 * 131 + 40 * 151 + 70 * 171 + 100 * 191, 2 * 13 + 5 * 15 + 8 * 17 + 11 * 19 + 20 * 131 + 50 * 151 + 80 * 171 + 110 * 191, 3 * 13 + 6 * 15 + 9 * 17 + 12 * 19 + 30 * 131 + 60 * 151 + 90 * 171 + 120 * 191,
    1095
    1
        1 * 14 + 4 * 16 + 7 * 18 + 10 * 20 + 10 * 141 + 40 * 161 + 70 * 181 + 100 * 201, 2 * 14 + 5 * 16 + 8 * 18 + 11 * 20 + 20 * 141 + 50 * 161 + 80 * 181 + 110 * 201, 3 * 14 + 6 * 16 + 9 * 18 + 12 * 20 + 30 * 141 + 60 * 161 + 90 * 181 + 120 * 201,
    1096
    1
      };
    1097
    1
      ccv_nnc_tensor_t dbt = ccv_nnc_tensor(dbtp, CPU_TENSOR_NHWC(32F, 2, 3), 0);
    1098
    1
      REQUIRE_TENSOR_EQ(db, &dbt, "db should be equal");
    1099
    1
      ccv_nnc_tensor_free(g);
    1100
    1
      ccv_nnc_tensor_free(a);
    1101
    1
      ccv_nnc_tensor_free(b);
    1102
    1
      ccv_nnc_tensor_free(h);
    1103
    1
      ccv_nnc_tensor_free(db);
    1104
    1
      ccv_nnc_tensor_free(dbias);
    1105
    1
      ccv_nnc_tensor_free(gg);
    1106
    1
      ccv_nnc_tensor_free(ga);
    1107
    1
      ccv_nnc_tensor_free(gb);
    1108
    1
      ccv_nnc_tensor_free(gh);
    1109
    1
      ccv_nnc_tensor_free(gdb);
    1110
    1
      ccv_nnc_tensor_free(gdbias);
    1111
    1
    }
    1112
    1113
    TEST_CASE("backward gemm with transpose b batch 2, batched b")
    1114
    1
    {
    1115
    1
      GUARD_ELSE_RETURN(ccv_nnc_cmd_ok(CCV_NNC_GEMM_FORWARD, CCV_NNC_BACKEND_GPU_CUBLAS) &&
    1116
    1
        ccv_nnc_cmd_ok(CCV_NNC_GEMM_BACKWARD, CCV_NNC_BACKEND_GPU_CUBLAS));
    1117
    1
      float gp[] = {
    1118
    1
        1, 2, 3,
    1119
    1
        4, 5, 6,
    1120
    1
        7, 8, 9,
    1121
    1
        10, 11, 12,
    1122
    1
        10, 20, 30,
    1123
    1
        40, 50, 60,
    1124
    1
        70, 80, 90,
    1125
    1
        100, 110, 120,
    1126
    1
      };
    1127
    1
      ccv_nnc_tensor_t* const g = ccv_nnc_tensor_new(gp, CPU_TENSOR_NHWC(32F, 2, 4, 3), 0);
    1128
    1
      float ap[] = {
    1129
    1
        13, 14,
    1130
    1
        15, 16,
    1131
    1
        17, 18,
    1132
    1
        19, 20,
    1133
    1
        131, 141,
    1134
    1
        151, 161,
    1135
    1
        171, 181,
    1136
    1
        191, 201,
    1137
    1
      };
    1138
    1
      ccv_nnc_tensor_t* const a = ccv_nnc_tensor_new(ap, CPU_TENSOR_NHWC(32F, 2, 4, 2), 0);
    1139
    1
      float bp[] = {
    1140
    1
        21, 24,
    1141
    1
        22, 25,
    1142
    1
        23, 26,
    1143
    1
        212, 242,
    1144
    1
        222, 252,
    1145
    1
        232, 262,
    1146
    1
      };
    1147
    1
      ccv_nnc_tensor_t* const b = ccv_nnc_tensor_new(bp, CPU_TENSOR_NHWC(32F, 2, 3, 2), 0);
    1148
    1
      ccv_nnc_tensor_t* const h = ccv_nnc_tensor_new(0, CPU_TENSOR_NHWC(32F, 2, 4, 2), 0);
    1149
    1
      ccv_nnc_tensor_t* const db = ccv_nnc_tensor_new(0, CPU_TENSOR_NHWC(32F, 2, 3, 2), 0);
    1150
    1
      ccv_nnc_tensor_t* const dbias = ccv_nnc_tensor_new(0, CPU_TENSOR_NHWC(32F, 2, 1, 3), 0);
    1151
    1
      ccv_nnc_tensor_t* const gg = ccv_nnc_tensor_new(0, GPU_TENSOR_NHWC(000, 32F, 2, 4, 3), 0);
    1152
    1
      ccv_nnc_tensor_t* const ga = ccv_nnc_tensor_new(0, GPU_TENSOR_NHWC(000, 32F, 2, 4, 2), 0);
    1153
    1
      ccv_nnc_tensor_t* const gb = ccv_nnc_tensor_new(0, GPU_TENSOR_NHWC(000, 32F, 2, 3, 2), 0);
    1154
    1
      ccv_nnc_tensor_t* const gh = ccv_nnc_tensor_new(0, GPU_TENSOR_NHWC(000, 32F, 2, 4, 2), 0);
    1155
    1
      ccv_nnc_tensor_t* const gdb = ccv_nnc_tensor_new(0, GPU_TENSOR_NHWC(000, 32F, 2, 3, 2), 0);
    1156
    1
      ccv_nnc_tensor_t* const gdbias = ccv_nnc_tensor_new(0, GPU_TENSOR_NHWC(000, 32F, 2, 1, 3), 0);
    1157
    1
      ccv_nnc_cmd_exec(CMD_DATA_TRANSFER_FORWARD(), ccv_nnc_no_hint, 0, TENSOR_LIST(g, a, b), TENSOR_LIST(gg, ga, gb), 0);
    1158
    1
      ccv_nnc_cmd_exec(CMD_GEMM_BACKWARD(NO_TRANSPOSE, TRANSPOSE(1, 2)), ccv_nnc_no_hint, 0, TENSOR_LIST(gg, ga, gb), TENSOR_LIST(gh, gdb, gdbias), 0);
    1159
    1
      ccv_nnc_cmd_exec(CMD_DATA_TRANSFER_FORWARD(), ccv_nnc_no_hint, 0, TENSOR_LIST(gh, gdb, gdbias), TENSOR_LIST(h, db, dbias), 0);
    1160
    1
      float dbiastp[] = {
    1161
    1
        22, 26, 30,
    1162
    1
        220, 260, 300,
    1163
    1
      };
    1164
    1
      ccv_nnc_tensor_t dbiast = ccv_nnc_tensor(dbiastp, CPU_TENSOR_NHWC(32F, 2, 1, 3), 0);
    1165
    1
      REQUIRE_TENSOR_EQ(dbias, &dbiast, "bias should be equal");
    1166
    1
      float htp[] = {
    1167
    1
        1 * 21 + 2 * 22 + 3 * 23, 1 * 24 + 2 * 25 + 3 * 26,
    1168
    1
        4 * 21 + 5 * 22 + 6 * 23, 4 * 24 + 5 * 25 + 6 * 26,
    1169
    1
        7 * 21 + 8 * 22 + 9 * 23, 7 * 24 + 8 * 25 + 9 * 26,
    1170
    1
        10 * 21 + 11 * 22 + 12 * 23, 10 * 24 + 11 * 25 + 12 * 26,
    1171
    1
        10 * 212 + 20 * 222 + 30 * 232, 10 * 242 + 20 * 252 + 30 * 262,
    1172
    1
        40 * 212 + 50 * 222 + 60 * 232, 40 * 242 + 50 * 252 + 60 * 262,
    1173
    1
        70 * 212 + 80 * 222 + 90 * 232, 70 * 242 + 80 * 252 + 90 * 262,
    1174
    1
        100 * 212 + 110 * 222 + 120 * 232, 100 * 242 + 110 * 252 + 120 * 262,
    1175
    1
      };
    1176
    1
      ccv_nnc_tensor_t ht = ccv_nnc_tensor(htp, CPU_TENSOR_NHWC(32F, 2, 4, 2), 0);
    1177
    1
      REQUIRE_TENSOR_EQ(h, &ht, "h should be equal");
    1178
    1
      float dbtp[] = {
    1179
    1
        1 * 13 + 4 * 15 + 7 * 17 + 10 * 19, 1 * 14 + 4 * 16 + 7 * 18 + 10 * 20,
    1180
    1
        2 * 13 + 5 * 15 + 8 * 17 + 11 * 19, 2 * 14 + 5 * 16 + 8 * 18 + 11 * 20,
    1181
    1
        3 * 13 + 6 * 15 + 9 * 17 + 12 * 19, 3 * 14 + 6 * 16 + 9 * 18 + 12 * 20,
    1182
    1
        10 * 131 + 40 * 151 + 70 * 171 + 100 * 191, 10 * 141 + 40 * 161 + 70 * 181 + 100 * 201,
    1183
    1
        20 * 131 + 50 * 151 + 80 * 171 + 110 * 191, 20 * 141 + 50 * 161 + 80 * 181 + 110 * 201,
    1184
    1
        30 * 131 + 60 * 151 + 90 * 171 + 120 * 191, 30 * 141 + 60 * 161 + 90 * 181 + 120 * 201,
    1185
    1
      };
    1186
    1
      ccv_nnc_tensor_t dbt = ccv_nnc_tensor(dbtp, CPU_TENSOR_NHWC(32F, 2, 3, 2), 0);
    1187
    1
      REQUIRE_TENSOR_EQ(db, &dbt, "db should be equal");
    1188
    1
      ccv_nnc_tensor_free(g);
    1189
    1
      ccv_nnc_tensor_free(a);
    1190
    1
      ccv_nnc_tensor_free(b);
    1191
    1
      ccv_nnc_tensor_free(h);
    1192
    1
      ccv_nnc_tensor_free(db);
    1193
    1
      ccv_nnc_tensor_free(dbias);
    1194
    1
      ccv_nnc_tensor_free(gg);
    1195
    1
      ccv_nnc_tensor_free(ga);
    1196
    1
      ccv_nnc_tensor_free(gb);
    1197
    1
      ccv_nnc_tensor_free(gh);
    1198
    1
      ccv_nnc_tensor_free(gdb);
    1199
    1
      ccv_nnc_tensor_free(gdbias);
    1200
    1
    }
    1201
    1202
    TEST_CASE("backward gemm with transpose a and b batch 2, same b")
    1203
    1
    {
    1204
    1
      GUARD_ELSE_RETURN(ccv_nnc_cmd_ok(CCV_NNC_GEMM_FORWARD, CCV_NNC_BACKEND_GPU_CUBLAS) &&
    1205
    1
        ccv_nnc_cmd_ok(CCV_NNC_GEMM_BACKWARD, CCV_NNC_BACKEND_GPU_CUBLAS));
    1206
    1
      float gp[] = {
    1207
    1
        1, 2, 3,
    1208
    1
        4, 5, 6,
    1209
    1
        7, 8, 9,
    1210
    1
        10, 11, 12,
    1211
    1
        10, 20, 30,
    1212
    1
        40, 50, 60,
    1213
    1
        70, 80, 90,
    1214
    1
        100, 110, 120,
    1215
    1
      };
    1216
    1
      ccv_nnc_tensor_t* const g = ccv_nnc_tensor_new(gp, CPU_TENSOR_NHWC(32F, 2, 4, 3), 0);
    1217
    1
      float ap[] = {
    1218
    1
        13, 15, 17, 19,
    1219
    1
        14, 16, 18, 20,
    1220
    1
        131, 151, 171, 191,
    1221
    1
        141, 161, 181, 201,
    1222
    1
      };
    1223
    1
      ccv_nnc_tensor_t* const a = ccv_nnc_tensor_new(ap, CPU_TENSOR_NHWC(32F, 2, 2, 4), 0);
    1224
    1
      float bp[] = {
    1225
    1
        21, 24,
    1226
    1
        22, 25,
    1227
    1
        23, 26,
    1228
    1
      };
    1229
    1
      ccv_nnc_tensor_t* const b = ccv_nnc_tensor_new(bp, CPU_TENSOR_NHWC(32F, 3, 2), 0);
    1230
    1
      ccv_nnc_tensor_t* const h = ccv_nnc_tensor_new(0, CPU_TENSOR_NHWC(32F, 2, 2, 4), 0);
    1231
    1
      ccv_nnc_tensor_t* const db = ccv_nnc_tensor_new(0, CPU_TENSOR_NHWC(32F, 3, 2), 0);
    1232
    1
      ccv_nnc_tensor_t* const dbias = ccv_nnc_tensor_new(0, CPU_TENSOR_NHWC(32F, 3), 0);
    1233
    1
      ccv_nnc_tensor_t* const gg = ccv_nnc_tensor_new(0, GPU_TENSOR_NHWC(000, 32F, 2, 4, 3), 0);
    1234
    1
      ccv_nnc_tensor_t* const ga = ccv_nnc_tensor_new(0, GPU_TENSOR_NHWC(000, 32F, 2, 2, 4), 0);
    1235
    1
      ccv_nnc_tensor_t* const gb = ccv_nnc_tensor_new(0, GPU_TENSOR_NHWC(000, 32F, 3, 2), 0);
    1236
    1
      ccv_nnc_tensor_t* const gh = ccv_nnc_tensor_new(0, GPU_TENSOR_NHWC(000, 32F, 2, 2, 4), 0);
    1237
    1
      ccv_nnc_tensor_t* const gdb = ccv_nnc_tensor_new(0, GPU_TENSOR_NHWC(000, 32F, 3, 2), 0);
    1238
    1
      ccv_nnc_tensor_t* const gdbias = ccv_nnc_tensor_new(0, GPU_TENSOR_NHWC(000, 32F, 3), 0);
    1239
    1
      ccv_nnc_cmd_exec(CMD_DATA_TRANSFER_FORWARD(), ccv_nnc_no_hint, 0, TENSOR_LIST(g, a, b), TENSOR_LIST(gg, ga, gb), 0);
    1240
    1
      ccv_nnc_cmd_exec(CMD_GEMM_BACKWARD(TRANSPOSE(1, 2), TRANSPOSE(0, 1)), ccv_nnc_no_hint, 0, TENSOR_LIST(gg, ga, gb), TENSOR_LIST(gh, gdb, gdbias), 0);
    1241
    1
      ccv_nnc_cmd_exec(CMD_DATA_TRANSFER_FORWARD(), ccv_nnc_no_hint, 0, TENSOR_LIST(gh, gdb, gdbias), TENSOR_LIST(h, db, dbias), 0);
    1242
    1
      float dbiastp[] = {
    1243
    1
        22 + 220, 26 + 260, 30 + 300,
    1244
    1
      };
    1245
    1
      ccv_nnc_tensor_t dbiast = ccv_nnc_tensor(dbiastp, CPU_TENSOR_NHWC(32F, 3), 0);
    1246
    1
      REQUIRE_TENSOR_EQ(dbias, &dbiast, "bias should be equal");
    1247
    1
      float htp[] = {
    1248
    1
        1 * 21 + 2 * 22 + 3 * 23, 4 * 21 + 5 * 22 + 6 * 23, 7 * 21 + 8 * 22 + 9 * 23, 10 * 21 + 11 * 22 + 12 * 23,
    1249
    1
        1 * 24 + 2 * 25 + 3 * 26, 4 * 24 + 5 * 25 + 6 * 26, 7 * 24 + 8 * 25 + 9 * 26, 10 * 24 + 11 * 25 + 12 * 26,
    1250
    1
        10 * 21 + 20 * 22 + 30 * 23, 40 * 21 + 50 * 22 + 60 * 23, 70 * 21 + 80 * 22 + 90 * 23, 100 * 21 + 110 * 22 + 120 * 23,
    1251
    1
        10 * 24 + 20 * 25 + 30 * 26, 40 * 24 + 50 * 25 + 60 * 26, 70 * 24 + 80 * 25 + 90 * 26, 100 * 24 + 110 * 25 + 120 * 26,
    1252
    1
      };
    1253
    1
      ccv_nnc_tensor_t ht = ccv_nnc_tensor(htp, CPU_TENSOR_NHWC(32F, 2, 2, 4), 0);
    1254
    1
      REQUIRE_TENSOR_EQ(h, &ht, "h should be equal");
    1255
    1
      float dbtp[] = {
    1256
    1
        1 * 13 + 4 * 15 + 7 * 17 + 10 * 19 + 10 * 131 + 40 * 151 + 70 * 171 + 100 * 191, 1 * 14 + 4 * 16 + 7 * 18 + 10 * 20 + 10 * 141 + 40 * 161 + 70 * 181 + 100 * 201,
    1257
    1
        2 * 13 + 5 * 15 + 8 * 17 + 11 * 19 + 20 * 131 + 50 * 151 + 80 * 171 + 110 * 191, 2 * 14 + 5 * 16 + 8 * 18 + 11 * 20 + 20 * 141 + 50 * 161 + 80 * 181 + 110 * 201,
    1258
    1
        3 * 13 + 6 * 15 + 9 * 17 + 12 * 19 + 30 * 131 + 60 * 151 + 90 * 171 + 120 * 191, 3 * 14 + 6 * 16 + 9 * 18 + 12 * 20 + 30 * 141 + 60 * 161 + 90 * 181 + 120 * 201,
    1259
    1
      };
    1260
    1
      ccv_nnc_tensor_t dbt = ccv_nnc_tensor(dbtp, CPU_TENSOR_NHWC(32F, 3, 2), 0);
    1261
    1
      REQUIRE_TENSOR_EQ(db, &dbt, "db should be equal");
    1262
    1
      ccv_nnc_tensor_free(g);
    1263
    1
      ccv_nnc_tensor_free(a);
    1264
    1
      ccv_nnc_tensor_free(b);
    1265
    1
      ccv_nnc_tensor_free(h);
    1266
    1
      ccv_nnc_tensor_free(db);
    1267
    1
      ccv_nnc_tensor_free(dbias);
    1268
    1
      ccv_nnc_tensor_free(gg);
    1269
    1
      ccv_nnc_tensor_free(ga);
    1270
    1
      ccv_nnc_tensor_free(gb);
    1271
    1
      ccv_nnc_tensor_free(gh);
    1272
    1
      ccv_nnc_tensor_free(gdb);
    1273
    1
      ccv_nnc_tensor_free(gdbias);
    1274
    1
    }
    1275
    1276
    TEST_CASE("cublas forward gemm")
    1277
    1
    {
    1278
    1
      GUARD_ELSE_RETURN(ccv_nnc_cmd_ok(CCV_NNC_GEMM_FORWARD, CCV_NNC_BACKEND_GPU_CUBLAS));
    1279
    1
      dsfmt_t dsfmt;
    1280
    1
      dsfmt_init_gen_rand(&dsfmt, 0);
    1281
    1
      ccv_nnc_tensor_t* a = ccv_nnc_tensor_new(0, GPU_TENSOR_NHWC(000, 32F, 10, 128), 0);
    1282
    1
      ccv_nnc_tensor_t* w = ccv_nnc_tensor_new(0, GPU_TENSOR_NHWC(000, 32F, 64, 128), 0);
    1283
    1
      ccv_nnc_tensor_t* bias = ccv_nnc_tensor_new(0, GPU_TENSOR_NHWC(000, 32F, 64), 0);
    1284
    1
      ccv_nnc_tensor_t* b = ccv_nnc_tensor_new(0, GPU_TENSOR_NHWC(000, 32F, 10, 64), 0);
    1285
    1286
    1
      ccv_nnc_tensor_t* ha = ccv_nnc_tensor_new(0, CPU_TENSOR_NHWC(32F, 1, 128), 0);
    1287
    1
      ccv_nnc_tensor_t* hw = ccv_nnc_tensor_new(0, CPU_TENSOR_NHWC(32F, 64, 128), 0);
    1288
    1
      ccv_nnc_tensor_t* hbias = ccv_nnc_tensor_new(0, CPU_TENSOR_NHWC(32F, 64), 0);
    1289
    1
      ccv_nnc_tensor_t* hb = ccv_nnc_tensor_new(0, CPU_TENSOR_NHWC(32F, 1, 64), 0);
    1290
    1
      int i;
    1291
    8.19k
      for (i = 0; i < 64 * 128; i++8.19k)
    1292
    8.19k
        hw->data.f32[i] = dsfmt_genrand_open_close(&dsfmt) / (64 * 128);
    1293
    65
      for (i = 0; i < 64; i++64)
    1294
    64
        hbias->data.f32[i] = dsfmt_genrand_open_close(&dsfmt);
    1295
    1
      ccv_nnc_tensor_t* ha1 = ccv_nnc_tensor_new(0, CPU_TENSOR_NHWC(32F, 10, 128), 0);
    1296
    1.28k
      for (i = 0; i < 10 * 128; i++1.28k)
    1297
    1.28k
        ha1->data.f32[i] = dsfmt_genrand_open_close(&dsfmt);
    1298
    129
      for (i = 0; i < 128; i++128)
    1299
    128
        ha->data.f32[i] = ha1->data.f32[i];
    1300
    1
      ccv_nnc_cmd_exec(CMD_DATA_TRANSFER_FORWARD(), ccv_nnc_no_hint, 0, TENSOR_LIST(ha1, hw, hbias), TENSOR_LIST(a, w, bias), 0);
    1301
    1
      ccv_nnc_cmd_exec(CMD_GEMM_FORWARD(NO_TRANSPOSE, TRANSPOSE(0, 1)), ccv_nnc_no_hint, 0, TENSOR_LIST(ha, hw, hbias), TENSOR_LIST(hb), 0);
    1302
    1
      ccv_nnc_cmd_exec(CMD_GEMM_FORWARD(NO_TRANSPOSE, TRANSPOSE(0, 1)), ccv_nnc_no_hint, 0, TENSOR_LIST(a, w, bias), TENSOR_LIST(b), 0);
    1303
    1
      ccv_nnc_tensor_t* tb = ccv_nnc_tensor_new(0, CPU_TENSOR_NHWC(32F, 10, 64), 0);
    1304
    1
      ccv_nnc_cmd_exec(CMD_DATA_TRANSFER_FORWARD(), ccv_nnc_no_hint, 0, TENSOR_LIST(b), TENSOR_LIST(tb), 0);
    1305
    1
      ccv_nnc_tensor_t* tb1 = ccv_nnc_tensor_new(0, CPU_TENSOR_NHWC(32F, 1, 64), 0);
    1306
    65
      for (i = 0; i < 64; i++64)
    1307
    64
        tb1->data.f32[i] = tb->data.f32[i];
    1308
    1
      REQUIRE_TENSOR_EQ(tb1, hb, "GPU computed output should be the same as CPU computed ones");
    1309
    1
      ccv_nnc_tensor_free(a);
    1310
    1
      ccv_nnc_tensor_free(w);
    1311
    1
      ccv_nnc_tensor_free(bias);
    1312
    1
      ccv_nnc_tensor_free(tb);
    1313
    1
      ccv_nnc_tensor_free(b);
    1314
    1
      ccv_nnc_tensor_free(ha);
    1315
    1
      ccv_nnc_tensor_free(ha1);
    1316
    1
      ccv_nnc_tensor_free(tb1);
    1317
    1
      ccv_nnc_tensor_free(hw);
    1318
    1
      ccv_nnc_tensor_free(hbias);
    1319
    1
      ccv_nnc_tensor_free(hb);
    1320
    1
    }
    1321
    1322
    TEST_CASE("cublas forward gemm in half precision")
    1323
    1
    {
    1324
    1
      GUARD_ELSE_RETURN(ccv_nnc_cmd_ok(CCV_NNC_GEMM_FORWARD, CCV_NNC_BACKEND_GPU_CUBLAS));
    1325
    1
      dsfmt_t dsfmt;
    1326
    1
      dsfmt_init_gen_rand(&dsfmt, 0);
    1327
    1
      ccv_nnc_tensor_t* a = ccv_nnc_tensor_new(0, GPU_TENSOR_NHWC(000, 16F, 10, 128), 0);
    1328
    1
      ccv_nnc_tensor_t* w = ccv_nnc_tensor_new(0, GPU_TENSOR_NHWC(000, 16F, 64, 128), 0);
    1329
    1
      ccv_nnc_tensor_t* bias = ccv_nnc_tensor_new(0, GPU_TENSOR_NHWC(000, 16F, 64), 0);
    1330
    1
      ccv_nnc_tensor_t* b = ccv_nnc_tensor_new(0, GPU_TENSOR_NHWC(000, 16F, 10, 64), 0);
    1331
    1332
    1
      ccv_nnc_tensor_t* ha = ccv_nnc_tensor_new(0, CPU_TENSOR_NHWC(32F, 1, 128), 0);
    1333
    1
      ccv_nnc_tensor_t* hw = ccv_nnc_tensor_new(0, CPU_TENSOR_NHWC(32F, 64, 128), 0);
    1334
    1
      ccv_nnc_tensor_t* hbias = ccv_nnc_tensor_new(0, CPU_TENSOR_NHWC(32F, 64), 0);
    1335
    1
      ccv_nnc_tensor_t* hb = ccv_nnc_tensor_new(0, CPU_TENSOR_NHWC(32F, 1, 64), 0);
    1336
    1
      int i;
    1337
    8.19k
      for (i = 0; i < 64 * 128; i++8.19k)
    1338
    8.19k
        hw->data.f32[i] = dsfmt_genrand_open_close(&dsfmt) / (64 * 128);
    1339
    65
      for (i = 0; i < 64; i++64)
    1340
    64
        hbias->data.f32[i] = dsfmt_genrand_open_close(&dsfmt);
    1341
    1
      ccv_nnc_tensor_t* ha1 = ccv_nnc_tensor_new(0, CPU_TENSOR_NHWC(32F, 10, 128), 0);
    1342
    1.28k
      for (i = 0; i < 10 * 128; i++1.28k)
    1343
    1.28k
        ha1->data.f32[i] = dsfmt_genrand_open_close(&dsfmt);
    1344
    129
      for (i = 0; i < 128; i++128)
    1345
    128
        ha->data.f32[i] = ha1->data.f32[i];
    1346
    1
      ccv_nnc_tensor_t* ha2 = ccv_nnc_tensor_new(0, CPU_TENSOR_NHWC(16F, 10, 128), 0);
    1347
    1
      ccv_nnc_tensor_t* hw2 = ccv_nnc_tensor_new(0, CPU_TENSOR_NHWC(16F, 64, 128), 0);
    1348
    1
      ccv_nnc_tensor_t* hbias2 = ccv_nnc_tensor_new(0, CPU_TENSOR_NHWC(16F, 64), 0);
    1349
    1
      ccv_nnc_cmd_exec(CMD_DATATYPE_CONVERSION_FORWARD(), ccv_nnc_no_hint, 0, TENSOR_LIST(ha1, hw, hbias), TENSOR_LIST(ha2, hw2, hbias2), 0);
    1350
    1
      ccv_nnc_cmd_exec(CMD_DATA_TRANSFER_FORWARD(), ccv_nnc_no_hint, 0, TENSOR_LIST(ha2, hw2, hbias2), TENSOR_LIST(a, w, bias), 0);
    1351
    1
      ccv_nnc_cmd_exec(CMD_GEMM_FORWARD(NO_TRANSPOSE, TRANSPOSE(0, 1)), ccv_nnc_no_hint, 0, TENSOR_LIST(ha, hw, hbias), TENSOR_LIST(hb), 0);
    1352
    1
      ccv_nnc_cmd_exec(CMD_GEMM_FORWARD(NO_TRANSPOSE, TRANSPOSE(0, 1)), ccv_nnc_no_hint, 0, TENSOR_LIST(a, w, bias), TENSOR_LIST(b), 0);
    1353
    1
      ccv_nnc_tensor_t* tb = ccv_nnc_tensor_new(0, CPU_TENSOR_NHWC(16F, 10, 64), 0);
    1354
    1
      ccv_nnc_cmd_exec(CMD_DATA_TRANSFER_FORWARD(), ccv_nnc_no_hint, 0, TENSOR_LIST(b), TENSOR_LIST(tb), 0);
    1355
    1
      ccv_nnc_tensor_t* tb1 = ccv_nnc_tensor_new(0, CPU_TENSOR_NHWC(32F, 10, 64), 0);
    1356
    1
      ccv_nnc_cmd_exec(CMD_DATATYPE_CONVERSION_FORWARD(), ccv_nnc_no_hint, 0, TENSOR_LIST(tb), TENSOR_LIST(tb1), 0);
    1357
    1
      REQUIRE_ARRAY_EQ_WITH_TOLERANCE(float, tb1->data.f32, hb->data.f32, 64, 1e-3, "GPU computed output should be the same as CPU computed ones");
    1358
    1
      ccv_nnc_tensor_free(a);
    1359
    1
      ccv_nnc_tensor_free(w);
    1360
    1
      ccv_nnc_tensor_free(bias);
    1361
    1
      ccv_nnc_tensor_free(b);
    1362
    1
      ccv_nnc_tensor_free(tb);
    1363
    1
      ccv_nnc_tensor_free(ha);
    1364
    1
      ccv_nnc_tensor_free(ha1);
    1365
    1
      ccv_nnc_tensor_free(tb1);
    1366
    1
      ccv_nnc_tensor_free(hw);
    1367
    1
      ccv_nnc_tensor_free(hbias);
    1368
    1
      ccv_nnc_tensor_free(hb);
    1369
    1
      ccv_nnc_tensor_free(ha2);
    1370
    1
      ccv_nnc_tensor_free(hw2);
    1371
    1
      ccv_nnc_tensor_free(hbias2);
    1372
    1
    }
    1373
    1374
    TEST_CASE("cublas forward gemm in bfloat precision")
    1375
    1
    {
    1376
    1
      GUARD_ELSE_RETURN(ccv_nnc_cmd_ok(CCV_NNC_GEMM_FORWARD, CCV_NNC_BACKEND_GPU_CUBLAS));
    1377
    1
      dsfmt_t dsfmt;
    1378
    1
      dsfmt_init_gen_rand(&dsfmt, 0);
    1379
    1
      ccv_nnc_tensor_t* a = ccv_nnc_tensor_new(0, GPU_TENSOR_NHWC(000, 16BF, 10, 128), 0);
    1380
    1
      ccv_nnc_tensor_t* w = ccv_nnc_tensor_new(0, GPU_TENSOR_NHWC(000, 16BF, 64, 128), 0);
    1381
    1
      ccv_nnc_tensor_t* bias = ccv_nnc_tensor_new(0, GPU_TENSOR_NHWC(000, 16BF, 64), 0);
    1382
    1
      ccv_nnc_tensor_t* b = ccv_nnc_tensor_new(0, GPU_TENSOR_NHWC(000, 16BF, 10, 64), 0);
    1383
    1384
    1
      ccv_nnc_tensor_t* ha = ccv_nnc_tensor_new(0, CPU_TENSOR_NHWC(32F, 1, 128), 0);
    1385
    1
      ccv_nnc_tensor_t* hw = ccv_nnc_tensor_new(0, CPU_TENSOR_NHWC(32F, 64, 128), 0);
    1386
    1
      ccv_nnc_tensor_t* hbias = ccv_nnc_tensor_new(0, CPU_TENSOR_NHWC(32F, 64), 0);
    1387
    1
      ccv_nnc_tensor_t* hb = ccv_nnc_tensor_new(0, CPU_TENSOR_NHWC(32F, 1, 64), 0);
    1388
    1
      int i;
    1389
    8.19k
      for (i = 0; i < 64 * 128; i++8.19k)
    1390
    8.19k
        hw->data.f32[i] = dsfmt_genrand_open_close(&dsfmt) / (64 * 128);
    1391
    65
      for (i = 0; i < 64; i++64)
    1392
    64
        hbias->data.f32[i] = dsfmt_genrand_open_close(&dsfmt);
    1393
    1
      ccv_nnc_tensor_t* ha1 = ccv_nnc_tensor_new(0, CPU_TENSOR_NHWC(32F, 10, 128), 0);
    1394
    1.28k
      for (i = 0; i < 10 * 128; i++1.28k)
    1395
    1.28k
        ha1->data.f32[i] = dsfmt_genrand_open_close(&dsfmt);
    1396
    129
      for (i = 0; i < 128; i++128)
    1397
    128
        ha->data.f32[i] = ha1->data.f32[i];
    1398
    1
      ccv_nnc_tensor_t* ha2 = ccv_nnc_tensor_new(0, CPU_TENSOR_NHWC(16BF, 10, 128), 0);
    1399
    1
      ccv_nnc_tensor_t* hw2 = ccv_nnc_tensor_new(0, CPU_TENSOR_NHWC(16BF, 64, 128), 0);
    1400
    1
      ccv_nnc_tensor_t* hbias2 = ccv_nnc_tensor_new(0, CPU_TENSOR_NHWC(16BF, 64), 0);
    1401
    1
      ccv_nnc_cmd_exec(CMD_DATATYPE_CONVERSION_FORWARD(), ccv_nnc_no_hint, 0, TENSOR_LIST(ha1, hw, hbias), TENSOR_LIST(ha2, hw2, hbias2), 0);
    1402
    1
      ccv_nnc_cmd_exec(CMD_DATA_TRANSFER_FORWARD(), ccv_nnc_no_hint, 0, TENSOR_LIST(ha2, hw2, hbias2), TENSOR_LIST(a, w, bias), 0);
    1403
    1
      ccv_nnc_cmd_exec(CMD_GEMM_FORWARD(NO_TRANSPOSE, TRANSPOSE(0, 1)), ccv_nnc_no_hint, 0, TENSOR_LIST(ha, hw, hbias), TENSOR_LIST(hb), 0);
    1404
    1
      ccv_nnc_cmd_exec(CMD_GEMM_FORWARD(NO_TRANSPOSE, TRANSPOSE(0, 1)), ccv_nnc_no_hint, 0, TENSOR_LIST(a, w, bias), TENSOR_LIST(b), 0);
    1405
    1
      ccv_nnc_tensor_t* tb = ccv_nnc_tensor_new(0, CPU_TENSOR_NHWC(16BF, 10, 64), 0);
    1406
    1
      ccv_nnc_cmd_exec(CMD_DATA_TRANSFER_FORWARD(), ccv_nnc_no_hint, 0, TENSOR_LIST(b), TENSOR_LIST(tb), 0);
    1407
    1
      ccv_nnc_tensor_t* tb1 = ccv_nnc_tensor_new(0, CPU_TENSOR_NHWC(32F, 10, 64), 0);
    1408
    1
      ccv_nnc_cmd_exec(CMD_DATATYPE_CONVERSION_FORWARD(), ccv_nnc_no_hint, 0, TENSOR_LIST(tb), TENSOR_LIST(tb1), 0);
    1409
    1
      REQUIRE_ARRAY_EQ_WITH_TOLERANCE(float, tb1->data.f32, hb->data.f32, 64, 1e-2, "GPU computed output should be the same as CPU computed ones");
    1410
    1
      ccv_nnc_tensor_free(a);
    1411
    1
      ccv_nnc_tensor_free(w);
    1412
    1
      ccv_nnc_tensor_free(bias);
    1413
    1
      ccv_nnc_tensor_free(b);
    1414
    1
      ccv_nnc_tensor_free(tb);
    1415
    1
      ccv_nnc_tensor_free(ha);
    1416
    1
      ccv_nnc_tensor_free(ha1);
    1417
    1
      ccv_nnc_tensor_free(tb1);
    1418
    1
      ccv_nnc_tensor_free(hw);
    1419
    1
      ccv_nnc_tensor_free(hbias);
    1420
    1
      ccv_nnc_tensor_free(hb);
    1421
    1
      ccv_nnc_tensor_free(ha2);
    1422
    1
      ccv_nnc_tensor_free(hw2);
    1423
    1
      ccv_nnc_tensor_free(hbias2);
    1424
    1
    }
    1425
    1426
    TEST_CASE("cublas forward gemv in half precision, variant 1")
    1427
    1
    {
    1428
    1
      GUARD_ELSE_RETURN(ccv_nnc_cmd_ok(CCV_NNC_GEMM_FORWARD, CCV_NNC_BACKEND_GPU_CUBLAS));
    1429
    1
      dsfmt_t dsfmt;
    1430
    1
      dsfmt_init_gen_rand(&dsfmt, 0);
    1431
    1
      ccv_nnc_tensor_t* a = ccv_nnc_tensor_new(0, GPU_TENSOR_NHWC(000, 16F, 1, 128), 0);
    1432
    1
      ccv_nnc_tensor_t* w = ccv_nnc_tensor_new(0, GPU_TENSOR_NHWC(000, 16F, 64, 128), 0);
    1433
    1
      ccv_nnc_tensor_t* bias = ccv_nnc_tensor_new(0, GPU_TENSOR_NHWC(000, 16F, 64), 0);
    1434
    1
      ccv_nnc_tensor_t* b = ccv_nnc_tensor_new(0, GPU_TENSOR_NHWC(000, 16F, 1, 64), 0);
    1435
    1436
    1
      ccv_nnc_tensor_t* ha = ccv_nnc_tensor_new(0, CPU_TENSOR_NHWC(32F, 1, 128), 0);
    1437
    1
      ccv_nnc_tensor_t* hw = ccv_nnc_tensor_new(0, CPU_TENSOR_NHWC(32F, 64, 128), 0);
    1438
    1
      ccv_nnc_tensor_t* hbias = ccv_nnc_tensor_new(0, CPU_TENSOR_NHWC(32F, 64), 0);
    1439
    1
      ccv_nnc_tensor_t* hb = ccv_nnc_tensor_new(0, CPU_TENSOR_NHWC(32F, 1, 64), 0);
    1440
    1
      int i;
    1441
    8.19k
      for (i = 0; i < 64 * 128; i++8.19k)
    1442
    8.19k
        hw->data.f32[i] = dsfmt_genrand_open_close(&dsfmt) / (64 * 128);
    1443
    65
      for (i = 0; i < 64; i++64)
    1444
    64
        hbias->data.f32[i] = dsfmt_genrand_open_close(&dsfmt);
    1445
    1
      ccv_nnc_tensor_t* ha1 = ccv_nnc_tensor_new(0, CPU_TENSOR_NHWC(32F, 1, 128), 0);
    1446
    129
      for (i = 0; i < 128; i++128)
    1447
    128
        ha1->data.f32[i] = dsfmt_genrand_open_close(&dsfmt);
    1448
    129
      for (i = 0; i < 128; i++128)
    1449
    128
        ha->data.f32[i] = ha1->data.f32[i];
    1450
    1
      ccv_nnc_tensor_t* ha2 = ccv_nnc_tensor_new(0, CPU_TENSOR_NHWC(16F, 1, 128), 0);
    1451
    1
      ccv_nnc_tensor_t* hw2 = ccv_nnc_tensor_new(0, CPU_TENSOR_NHWC(16F, 64, 128), 0);
    1452
    1
      ccv_nnc_tensor_t* hbias2 = ccv_nnc_tensor_new(0, CPU_TENSOR_NHWC(16F, 64), 0);
    1453
    1
      ccv_nnc_cmd_exec(CMD_DATATYPE_CONVERSION_FORWARD(), ccv_nnc_no_hint, 0, TENSOR_LIST(ha1, hw, hbias), TENSOR_LIST(ha2, hw2, hbias2), 0);
    1454
    1
      ccv_nnc_cmd_exec(CMD_DATA_TRANSFER_FORWARD(), ccv_nnc_no_hint, 0, TENSOR_LIST(ha2, hw2, hbias2), TENSOR_LIST(a, w, bias), 0);
    1455
    1
      ccv_nnc_cmd_exec(CMD_GEMM_FORWARD(NO_TRANSPOSE, TRANSPOSE(0, 1)), ccv_nnc_no_hint, 0, TENSOR_LIST(ha, hw, hbias), TENSOR_LIST(hb), 0);
    1456
    1
      ccv_nnc_cmd_exec(CMD_GEMM_FORWARD(NO_TRANSPOSE, TRANSPOSE(0, 1)), ccv_nnc_no_hint, 0, TENSOR_LIST(a, w, bias), TENSOR_LIST(b), 0);
    1457
    1
      ccv_nnc_tensor_t* tb = ccv_nnc_tensor_new(0, CPU_TENSOR_NHWC(16F, 1, 64), 0);
    1458
    1
      ccv_nnc_cmd_exec(CMD_DATA_TRANSFER_FORWARD(), ccv_nnc_no_hint, 0, TENSOR_LIST(b), TENSOR_LIST(tb), 0);
    1459
    1
      ccv_nnc_tensor_t* tb1 = ccv_nnc_tensor_new(0, CPU_TENSOR_NHWC(32F, 1, 64), 0);
    1460
    1
      ccv_nnc_cmd_exec(CMD_DATATYPE_CONVERSION_FORWARD(), ccv_nnc_no_hint, 0, TENSOR_LIST(tb), TENSOR_LIST(tb1), 0);
    1461
    1
      REQUIRE_ARRAY_EQ_WITH_TOLERANCE(float, tb1->data.f32, hb->data.f32, 64, 1e-3, "GPU computed output should be the same as CPU computed ones");
    1462
    1
      ccv_nnc_tensor_free(a);
    1463
    1
      ccv_nnc_tensor_free(w);
    1464
    1
      ccv_nnc_tensor_free(bias);
    1465
    1
      ccv_nnc_tensor_free(b);
    1466
    1
      ccv_nnc_tensor_free(tb);
    1467
    1
      ccv_nnc_tensor_free(ha);
    1468
    1
      ccv_nnc_tensor_free(ha1);
    1469
    1
      ccv_nnc_tensor_free(tb1);
    1470
    1
      ccv_nnc_tensor_free(hw);
    1471
    1
      ccv_nnc_tensor_free(hbias);
    1472
    1
      ccv_nnc_tensor_free(hb);
    1473
    1
      ccv_nnc_tensor_free(ha2);
    1474
    1
      ccv_nnc_tensor_free(hw2);
    1475
    1
      ccv_nnc_tensor_free(hbias2);
    1476
    1
    }
    1477
    1478
    TEST_CASE("cublas forward gemv in bfloat precision, variant 1")
    1479
    1
    {
    1480
    1
      GUARD_ELSE_RETURN(ccv_nnc_cmd_ok(CCV_NNC_GEMM_FORWARD, CCV_NNC_BACKEND_GPU_CUBLAS));
    1481
    1
      dsfmt_t dsfmt;
    1482
    1
      dsfmt_init_gen_rand(&dsfmt, 0);
    1483
    1
      ccv_nnc_tensor_t* a = ccv_nnc_tensor_new(0, GPU_TENSOR_NHWC(000, 16BF, 1, 128), 0);
    1484
    1
      ccv_nnc_tensor_t* w = ccv_nnc_tensor_new(0, GPU_TENSOR_NHWC(000, 16BF, 64, 128), 0);
    1485
    1
      ccv_nnc_tensor_t* bias = ccv_nnc_tensor_new(0, GPU_TENSOR_NHWC(000, 16BF, 64), 0);
    1486
    1
      ccv_nnc_tensor_t* b = ccv_nnc_tensor_new(0, GPU_TENSOR_NHWC(000, 16BF, 1, 64), 0);
    1487
    1488
    1
      ccv_nnc_tensor_t* ha = ccv_nnc_tensor_new(0, CPU_TENSOR_NHWC(32F, 1, 128), 0);
    1489
    1
      ccv_nnc_tensor_t* hw = ccv_nnc_tensor_new(0, CPU_TENSOR_NHWC(32F, 64, 128), 0);
    1490
    1
      ccv_nnc_tensor_t* hbias = ccv_nnc_tensor_new(0, CPU_TENSOR_NHWC(32F, 64), 0);
    1491
    1
      ccv_nnc_tensor_t* hb = ccv_nnc_tensor_new(0, CPU_TENSOR_NHWC(32F, 1, 64), 0);
    1492
    1
      int i;
    1493
    8.19k
      for (i = 0; i < 64 * 128; i++8.19k)
    1494
    8.19k
        hw->data.f32[i] = dsfmt_genrand_open_close(&dsfmt) / (64 * 128);
    1495
    65
      for (i = 0; i < 64; i++64)
    1496
    64
        hbias->data.f32[i] = dsfmt_genrand_open_close(&dsfmt);
    1497
    1
      ccv_nnc_tensor_t* ha1 = ccv_nnc_tensor_new(0, CPU_TENSOR_NHWC(32F, 1, 128), 0);
    1498
    129
      for (i = 0; i < 128; i++128)
    1499
    128
        ha1->data.f32[i] = dsfmt_genrand_open_close(&dsfmt);
    1500
    129
      for (i = 0; i < 128; i++128)
    1501
    128
        ha->data.f32[i] = ha1->data.f32[i];
    1502
    1
      ccv_nnc_tensor_t* ha2 = ccv_nnc_tensor_new(0, CPU_TENSOR_NHWC(16BF, 1, 128), 0);
    1503
    1
      ccv_nnc_tensor_t* hw2 = ccv_nnc_tensor_new(0, CPU_TENSOR_NHWC(16BF, 64, 128), 0);
    1504
    1
      ccv_nnc_tensor_t* hbias2 = ccv_nnc_tensor_new(0, CPU_TENSOR_NHWC(16BF, 64), 0);
    1505
    1
      ccv_nnc_cmd_exec(CMD_DATATYPE_CONVERSION_FORWARD(), ccv_nnc_no_hint, 0, TENSOR_LIST(ha1, hw, hbias), TENSOR_LIST(ha2, hw2, hbias2), 0);
    1506
    1
      ccv_nnc_cmd_exec(CMD_DATA_TRANSFER_FORWARD(), ccv_nnc_no_hint, 0, TENSOR_LIST(ha2, hw2, hbias2), TENSOR_LIST(a, w, bias), 0);
    1507
    1
      ccv_nnc_cmd_exec(CMD_GEMM_FORWARD(NO_TRANSPOSE, TRANSPOSE(0, 1)), ccv_nnc_no_hint, 0, TENSOR_LIST(ha, hw, hbias), TENSOR_LIST(hb), 0);
    1508
    1
      ccv_nnc_cmd_exec(CMD_GEMM_FORWARD(NO_TRANSPOSE, TRANSPOSE(0, 1)), ccv_nnc_no_hint, 0, TENSOR_LIST(a, w, bias), TENSOR_LIST(b), 0);
    1509
    1
      ccv_nnc_tensor_t* tb = ccv_nnc_tensor_new(0, CPU_TENSOR_NHWC(16BF, 1, 64), 0);
    1510
    1
      ccv_nnc_cmd_exec(CMD_DATA_TRANSFER_FORWARD(), ccv_nnc_no_hint, 0, TENSOR_LIST(b), TENSOR_LIST(tb), 0);
    1511
    1
      ccv_nnc_tensor_t* tb1 = ccv_nnc_tensor_new(0, CPU_TENSOR_NHWC(32F, 1, 64), 0);
    1512
    1
      ccv_nnc_cmd_exec(CMD_DATATYPE_CONVERSION_FORWARD(), ccv_nnc_no_hint, 0, TENSOR_LIST(tb), TENSOR_LIST(tb1), 0);
    1513
    1
      REQUIRE_ARRAY_EQ_WITH_TOLERANCE(float, tb1->data.f32, hb->data.f32, 64, 1e-2, "GPU computed output should be the same as CPU computed ones");
    1514
    1
      ccv_nnc_tensor_free(a);
    1515
    1
      ccv_nnc_tensor_free(w);
    1516
    1
      ccv_nnc_tensor_free(bias);
    1517
    1
      ccv_nnc_tensor_free(b);
    1518
    1
      ccv_nnc_tensor_free(tb);
    1519
    1
      ccv_nnc_tensor_free(ha);
    1520
    1
      ccv_nnc_tensor_free(ha1);
    1521
    1
      ccv_nnc_tensor_free(tb1);
    1522
    1
      ccv_nnc_tensor_free(hw);
    1523
    1
      ccv_nnc_tensor_free(hbias);
    1524
    1
      ccv_nnc_tensor_free(hb);
    1525
    1
      ccv_nnc_tensor_free(ha2);
    1526
    1
      ccv_nnc_tensor_free(hw2);
    1527
    1
      ccv_nnc_tensor_free(hbias2);
    1528
    1
    }
    1529
    1530
    TEST_CASE("cublas forward gemm no bias")
    1531
    1
    {
    1532
    1
      GUARD_ELSE_RETURN(ccv_nnc_cmd_ok(CCV_NNC_GEMM_FORWARD, CCV_NNC_BACKEND_GPU_CUBLAS));
    1533
    1
      dsfmt_t dsfmt;
    1534
    1
      dsfmt_init_gen_rand(&dsfmt, 0);
    1535
    1
      ccv_nnc_tensor_t* a = ccv_nnc_tensor_new(0, GPU_TENSOR_NHWC(000, 32F, 10, 128), 0);
    1536
    1
      ccv_nnc_tensor_t* w = ccv_nnc_tensor_new(0, GPU_TENSOR_NHWC(000, 32F, 64, 128), 0);
    1537
    1
      ccv_nnc_tensor_t* b = ccv_nnc_tensor_new(0, GPU_TENSOR_NHWC(000, 32F, 10, 64), 0);
    1538
    1539
    1
      ccv_nnc_tensor_t* ha = ccv_nnc_tensor_new(0, CPU_TENSOR_NHWC(32F, 1, 128), 0);
    1540
    1
      ccv_nnc_tensor_t* hw = ccv_nnc_tensor_new(0, CPU_TENSOR_NHWC(32F, 64, 128), 0);
    1541
    1
      ccv_nnc_tensor_t* hb = ccv_nnc_tensor_new(0, CPU_TENSOR_NHWC(32F, 1, 64), 0);
    1542
    1
      int i;
    1543
    8.19k
      for (i = 0; i < 64 * 128; i++8.19k)
    1544
    8.19k
        hw->data.f32[i] = dsfmt_genrand_open_close(&dsfmt) / (64 * 128);
    1545
    1
      ccv_nnc_tensor_t* ha1 = ccv_nnc_tensor_new(0, CPU_TENSOR_NHWC(32F, 10, 128), 0);
    1546
    1.28k
      for (i = 0; i < 10 * 128; i++1.28k)
    1547
    1.28k
        ha1->data.f32[i] = dsfmt_genrand_open_close(&dsfmt);
    1548
    129
      for (i = 0; i < 128; i++128)
    1549
    128
        ha->data.f32[i] = ha1->data.f32[i];
    1550
    1
      ccv_nnc_cmd_exec(CMD_DATA_TRANSFER_FORWARD(), ccv_nnc_no_hint, 0, TENSOR_LIST(ha1, hw), TENSOR_LIST(a, w), 0);
    1551
    1
      ccv_nnc_cmd_exec(CMD_GEMM_FORWARD(NO_TRANSPOSE, TRANSPOSE(0, 1)), ccv_nnc_no_hint, 0, TENSOR_LIST(ha, hw), TENSOR_LIST(hb), 0);
    1552
    1
      ccv_nnc_cmd_exec(CMD_GEMM_FORWARD(NO_TRANSPOSE, TRANSPOSE(0, 1)), ccv_nnc_no_hint, 0, TENSOR_LIST(a, w), TENSOR_LIST(b), 0);
    1553
    1
      ccv_nnc_tensor_t* tb = ccv_nnc_tensor_new(0, CPU_TENSOR_NHWC(32F, 10, 64), 0);
    1554
    1
      ccv_nnc_cmd_exec(CMD_DATA_TRANSFER_FORWARD(), ccv_nnc_no_hint, 0, TENSOR_LIST(b), TENSOR_LIST(tb), 0);
    1555
    1
      ccv_nnc_tensor_t* tb1 = ccv_nnc_tensor_new(0, CPU_TENSOR_NHWC(32F, 1, 64), 0);
    1556
    65
      for (i = 0; i < 64; i++64)
    1557
    64
        tb1->data.f32[i] = tb->data.f32[i];
    1558
    1
      REQUIRE_TENSOR_EQ(tb1, hb, "GPU computed output should be the same as CPU computed ones");
    1559
    1
      ccv_nnc_tensor_free(a);
    1560
    1
      ccv_nnc_tensor_free(w);
    1561
    1
      ccv_nnc_tensor_free(b);
    1562
    1
      ccv_nnc_tensor_free(tb);
    1563
    1
      ccv_nnc_tensor_free(ha);
    1564
    1
      ccv_nnc_tensor_free(ha1);
    1565
    1
      ccv_nnc_tensor_free(tb1);
    1566
    1
      ccv_nnc_tensor_free(hw);
    1567
    1
      ccv_nnc_tensor_free(hb);
    1568
    1
    }
    1569
    1570
    TEST_CASE("cublas forward gemm no bias in half precision")
    1571
    1
    {
    1572
    1
      GUARD_ELSE_RETURN(ccv_nnc_cmd_ok(CCV_NNC_GEMM_FORWARD, CCV_NNC_BACKEND_GPU_CUBLAS));
    1573
    1
      dsfmt_t dsfmt;
    1574
    1
      dsfmt_init_gen_rand(&dsfmt, 0);
    1575
    1
      ccv_nnc_tensor_t* a = ccv_nnc_tensor_new(0, GPU_TENSOR_NHWC(000, 16F, 10, 128), 0);
    1576
    1
      ccv_nnc_tensor_t* w = ccv_nnc_tensor_new(0, GPU_TENSOR_NHWC(000, 16F, 64, 128), 0);
    1577
    1
      ccv_nnc_tensor_t* b = ccv_nnc_tensor_new(0, GPU_TENSOR_NHWC(000, 16F, 10, 64), 0);
    1578
    1579
    1
      ccv_nnc_tensor_t* ha = ccv_nnc_tensor_new(0, CPU_TENSOR_NHWC(32F, 1, 128), 0);
    1580
    1
      ccv_nnc_tensor_t* hw = ccv_nnc_tensor_new(0, CPU_TENSOR_NHWC(32F, 64, 128), 0);
    1581
    1
      ccv_nnc_tensor_t* hb = ccv_nnc_tensor_new(0, CPU_TENSOR_NHWC(32F, 1, 64), 0);
    1582
    1
      int i;
    1583
    8.19k
      for (i = 0; i < 64 * 128; i++8.19k)
    1584
    8.19k
        hw->data.f32[i] = dsfmt_genrand_open_close(&dsfmt) / (64 * 128);
    1585
    1
      ccv_nnc_tensor_t* ha1 = ccv_nnc_tensor_new(0, CPU_TENSOR_NHWC(32F, 10, 128), 0);
    1586
    1.28k
      for (i = 0; i < 10 * 128; i++1.28k)
    1587
    1.28k
        ha1->data.f32[i] = dsfmt_genrand_open_close(&dsfmt);
    1588
    129
      for (i = 0; i < 128; i++128)
    1589
    128
        ha->data.f32[i] = ha1->data.f32[i];
    1590
    1
      ccv_nnc_tensor_t* ha2 = ccv_nnc_tensor_new(0, CPU_TENSOR_NHWC(16F, 10, 128), 0);
    1591
    1
      ccv_nnc_tensor_t* hw2 = ccv_nnc_tensor_new(0, CPU_TENSOR_NHWC(16F, 64, 128), 0);
    1592
    1
      ccv_nnc_cmd_exec(CMD_DATATYPE_CONVERSION_FORWARD(), ccv_nnc_no_hint, 0, TENSOR_LIST(ha1, hw), TENSOR_LIST(ha2, hw2), 0);
    1593
    1
      ccv_nnc_cmd_exec(CMD_DATA_TRANSFER_FORWARD(), ccv_nnc_no_hint, 0, TENSOR_LIST(ha2, hw2), TENSOR_LIST(a, w), 0);
    1594
    1
      ccv_nnc_cmd_exec(CMD_GEMM_FORWARD(NO_TRANSPOSE, TRANSPOSE(0, 1)), ccv_nnc_no_hint, 0, TENSOR_LIST(ha, hw), TENSOR_LIST(hb), 0);
    1595
    1
      ccv_nnc_cmd_exec(CMD_GEMM_FORWARD(NO_TRANSPOSE, TRANSPOSE(0, 1)), ccv_nnc_no_hint, 0, TENSOR_LIST(a, w), TENSOR_LIST(b), 0);
    1596
    1
      ccv_nnc_tensor_t* tb = ccv_nnc_tensor_new(0, CPU_TENSOR_NHWC(16F, 10, 64), 0);
    1597
    1
      ccv_nnc_cmd_exec(CMD_DATA_TRANSFER_FORWARD(), ccv_nnc_no_hint, 0, TENSOR_LIST(b), TENSOR_LIST(tb), 0);
    1598
    1
      ccv_nnc_tensor_t* tb1 = ccv_nnc_tensor_new(0, CPU_TENSOR_NHWC(32F, 10, 64), 0);
    1599
    1
      ccv_nnc_cmd_exec(CMD_DATATYPE_CONVERSION_FORWARD(), ccv_nnc_no_hint, 0, TENSOR_LIST(tb), TENSOR_LIST(tb1), 0);
    1600
    1
      REQUIRE_ARRAY_EQ_WITH_TOLERANCE(float, tb1->data.f32, hb->data.f32, 64, 1e-3, "GPU computed output should be the same as CPU computed ones");
    1601
    1
      ccv_nnc_tensor_free(a);
    1602
    1
      ccv_nnc_tensor_free(w);
    1603
    1
      ccv_nnc_tensor_free(b);
    1604
    1
      ccv_nnc_tensor_free(tb);
    1605
    1
      ccv_nnc_tensor_free(ha);
    1606
    1
      ccv_nnc_tensor_free(ha1);
    1607
    1
      ccv_nnc_tensor_free(tb1);
    1608
    1
      ccv_nnc_tensor_free(hw);
    1609
    1
      ccv_nnc_tensor_free(hb);
    1610
    1
      ccv_nnc_tensor_free(ha2);
    1611
    1
      ccv_nnc_tensor_free(hw2);
    1612
    1
    }
    1613
    1614
    TEST_CASE("cublas forward gemv in half precision no bias, variant 1")
    1615
    1
    {
    1616
    1
      GUARD_ELSE_RETURN(ccv_nnc_cmd_ok(CCV_NNC_GEMM_FORWARD, CCV_NNC_BACKEND_GPU_CUBLAS));
    1617
    1
      dsfmt_t dsfmt;
    1618
    1
      dsfmt_init_gen_rand(&dsfmt, 0);
    1619
    1
      ccv_nnc_tensor_t* a = ccv_nnc_tensor_new(0, GPU_TENSOR_NHWC(000, 16F, 1, 128), 0);
    1620
    1
      ccv_nnc_tensor_t* w = ccv_nnc_tensor_new(0, GPU_TENSOR_NHWC(000, 16F, 64, 128), 0);
    1621
    1
      ccv_nnc_tensor_t* b = ccv_nnc_tensor_new(0, GPU_TENSOR_NHWC(000, 16F, 1, 64), 0);
    1622
    1623
    1
      ccv_nnc_tensor_t* ha = ccv_nnc_tensor_new(0, CPU_TENSOR_NHWC(32F, 1, 128), 0);
    1624
    1
      ccv_nnc_tensor_t* hw = ccv_nnc_tensor_new(0, CPU_TENSOR_NHWC(32F, 64, 128), 0);
    1625
    1
      ccv_nnc_tensor_t* hb = ccv_nnc_tensor_new(0, CPU_TENSOR_NHWC(32F, 1, 64), 0);
    1626
    1
      int i;
    1627
    8.19k
      for (i = 0; i < 64 * 128; i++8.19k)
    1628
    8.19k
        hw->data.f32[i] = dsfmt_genrand_open_close(&dsfmt) / (64 * 128);
    1629
    1
      ccv_nnc_tensor_t* ha1 = ccv_nnc_tensor_new(0, CPU_TENSOR_NHWC(32F, 1, 128), 0);
    1630
    129
      for (i = 0; i < 128; i++128)
    1631
    128
        ha1->data.f32[i] = dsfmt_genrand_open_close(&dsfmt);
    1632
    129
      for (i = 0; i < 128; i++128)
    1633
    128
        ha->data.f32[i] = ha1->data.f32[i];
    1634
    1
      ccv_nnc_tensor_t* ha2 = ccv_nnc_tensor_new(0, CPU_TENSOR_NHWC(16F, 1, 128), 0);
    1635
    1
      ccv_nnc_tensor_t* hw2 = ccv_nnc_tensor_new(0, CPU_TENSOR_NHWC(16F, 64, 128), 0);
    1636
    1
      ccv_nnc_cmd_exec(CMD_DATATYPE_CONVERSION_FORWARD(), ccv_nnc_no_hint, 0, TENSOR_LIST(ha1, hw), TENSOR_LIST(ha2, hw2), 0);
    1637
    1
      ccv_nnc_cmd_exec(CMD_DATA_TRANSFER_FORWARD(), ccv_nnc_no_hint, 0, TENSOR_LIST(ha2, hw2), TENSOR_LIST(a, w), 0);
    1638
    1
      ccv_nnc_cmd_exec(CMD_GEMM_FORWARD(NO_TRANSPOSE, TRANSPOSE(0, 1)), ccv_nnc_no_hint, 0, TENSOR_LIST(ha, hw), TENSOR_LIST(hb), 0);
    1639
    1
      ccv_nnc_cmd_exec(CMD_GEMM_FORWARD(NO_TRANSPOSE, TRANSPOSE(0, 1)), ccv_nnc_no_hint, 0, TENSOR_LIST(a, w), TENSOR_LIST(b), 0);
    1640
    1
      ccv_nnc_tensor_t* tb = ccv_nnc_tensor_new(0, CPU_TENSOR_NHWC(16F, 1, 64), 0);
    1641
    1
      ccv_nnc_cmd_exec(CMD_DATA_TRANSFER_FORWARD(), ccv_nnc_no_hint, 0, TENSOR_LIST(b), TENSOR_LIST(tb), 0);
    1642
    1
      ccv_nnc_tensor_t* tb1 = ccv_nnc_tensor_new(0, CPU_TENSOR_NHWC(32F, 1, 64), 0);
    1643
    1
      ccv_nnc_cmd_exec(CMD_DATATYPE_CONVERSION_FORWARD(), ccv_nnc_no_hint, 0, TENSOR_LIST(tb), TENSOR_LIST(tb1), 0);
    1644
    1
      REQUIRE_ARRAY_EQ_WITH_TOLERANCE(float, tb1->data.f32, hb->data.f32, 64, 1e-3, "GPU computed output should be the same as CPU computed ones");
    1645
    1
      ccv_nnc_tensor_free(a);
    1646
    1
      ccv_nnc_tensor_free(w);
    1647
    1
      ccv_nnc_tensor_free(b);
    1648
    1
      ccv_nnc_tensor_free(tb);
    1649
    1
      ccv_nnc_tensor_free(ha);
    1650
    1
      ccv_nnc_tensor_free(ha1);
    1651
    1
      ccv_nnc_tensor_free(tb1);
    1652
    1
      ccv_nnc_tensor_free(hw);
    1653
    1
      ccv_nnc_tensor_free(hb);
    1654
    1
      ccv_nnc_tensor_free(ha2);
    1655
    1
      ccv_nnc_tensor_free(hw2);
    1656
    1
    }
    1657
    1658
    TEST_CASE("cublas forward gemv in half precision no bias, variant 2")
    1659
    1
    {
    1660
    1
      GUARD_ELSE_RETURN(ccv_nnc_cmd_ok(CCV_NNC_GEMM_FORWARD, CCV_NNC_BACKEND_GPU_CUBLAS));
    1661
    1
      dsfmt_t dsfmt;
    1662
    1
      dsfmt_init_gen_rand(&dsfmt, 0);
    1663
    1
      ccv_nnc_tensor_t* w = ccv_nnc_tensor_new(0, GPU_TENSOR_NHWC(000, 16F, 64, 128), 0);
    1664
    1
      ccv_nnc_tensor_t* a = ccv_nnc_tensor_new(0, GPU_TENSOR_NHWC(000, 16F, 128, 1), 0);
    1665
    1
      ccv_nnc_tensor_t* b = ccv_nnc_tensor_new(0, GPU_TENSOR_NHWC(000, 16F, 64, 1), 0);
    1666
    1667
    1
      ccv_nnc_tensor_t* hw = ccv_nnc_tensor_new(0, CPU_TENSOR_NHWC(32F, 64, 128), 0);
    1668
    1
      ccv_nnc_tensor_t* ha = ccv_nnc_tensor_new(0, CPU_TENSOR_NHWC(32F, 128, 1), 0);
    1669
    1
      ccv_nnc_tensor_t* hb = ccv_nnc_tensor_new(0, CPU_TENSOR_NHWC(32F, 64, 1), 0);
    1670
    1
      int i;
    1671
    8.19k
      for (i = 0; i < 64 * 128; i++8.19k)
    1672
    8.19k
        hw->data.f32[i] = dsfmt_genrand_open_close(&dsfmt) / (64 * 128);
    1673
    1
      ccv_nnc_tensor_t* ha1 = ccv_nnc_tensor_new(0, CPU_TENSOR_NHWC(32F, 128, 1), 0);
    1674
    129
      for (i = 0; i < 128; i++128)
    1675
    128
        ha1->data.f32[i] = dsfmt_genrand_open_close(&dsfmt);
    1676
    129
      for (i = 0; i < 128; i++128)
    1677
    128
        ha->data.f32[i] = ha1->data.f32[i];
    1678
    1
      ccv_nnc_tensor_t* hw2 = ccv_nnc_tensor_new(0, CPU_TENSOR_NHWC(16F, 64, 128), 0);
    1679
    1
      ccv_nnc_tensor_t* ha2 = ccv_nnc_tensor_new(0, CPU_TENSOR_NHWC(16F, 128, 1), 0);
    1680
    1
      ccv_nnc_cmd_exec(CMD_DATATYPE_CONVERSION_FORWARD(), ccv_nnc_no_hint, 0, TENSOR_LIST(ha1, hw), TENSOR_LIST(ha2, hw2), 0);
    1681
    1
      ccv_nnc_cmd_exec(CMD_DATA_TRANSFER_FORWARD(), ccv_nnc_no_hint, 0, TENSOR_LIST(ha2, hw2), TENSOR_LIST(a, w), 0);
    1682
    1
      ccv_nnc_cmd_exec(CMD_GEMM_FORWARD(NO_TRANSPOSE, NO_TRANSPOSE), ccv_nnc_no_hint, 0, TENSOR_LIST(hw, ha), TENSOR_LIST(hb), 0);
    1683
    1
      ccv_nnc_cmd_exec(CMD_GEMM_FORWARD(NO_TRANSPOSE, NO_TRANSPOSE), ccv_nnc_no_hint, 0, TENSOR_LIST(w, a), TENSOR_LIST(b), 0);
    1684
    1
      ccv_nnc_tensor_t* tb = ccv_nnc_tensor_new(0, CPU_TENSOR_NHWC(16F, 64, 1), 0);
    1685
    1
      ccv_nnc_cmd_exec(CMD_DATA_TRANSFER_FORWARD(), ccv_nnc_no_hint, 0, TENSOR_LIST(b), TENSOR_LIST(tb), 0);
    1686
    1
      ccv_nnc_tensor_t* tb1 = ccv_nnc_tensor_new(0, CPU_TENSOR_NHWC(32F, 64, 1), 0);
    1687
    1
      ccv_nnc_cmd_exec(CMD_DATATYPE_CONVERSION_FORWARD(), ccv_nnc_no_hint, 0, TENSOR_LIST(tb), TENSOR_LIST(tb1), 0);
    1688
    1
      REQUIRE_ARRAY_EQ_WITH_TOLERANCE(float, tb1->data.f32, hb->data.f32, 64, 1e-3, "GPU computed output should be the same as CPU computed ones");
    1689
    1
      ccv_nnc_tensor_free(a);
    1690
    1
      ccv_nnc_tensor_free(w);
    1691
    1
      ccv_nnc_tensor_free(b);
    1692
    1
      ccv_nnc_tensor_free(tb);
    1693
    1
      ccv_nnc_tensor_free(ha);
    1694
    1
      ccv_nnc_tensor_free(ha1);
    1695
    1
      ccv_nnc_tensor_free(tb1);
    1696
    1
      ccv_nnc_tensor_free(hw);
    1697
    1
      ccv_nnc_tensor_free(hb);
    1698
    1
      ccv_nnc_tensor_free(ha2);
    1699
    1
      ccv_nnc_tensor_free(hw2);
    1700
    1
    }
    1701
    1702
    TEST_CASE("cublas backward gemm")
    1703
    1
    {
    1704
    1
      GUARD_ELSE_RETURN(ccv_nnc_cmd_ok(CCV_NNC_GEMM_FORWARD, CCV_NNC_BACKEND_GPU_CUBLAS) &&
    1705
    1
        ccv_nnc_cmd_ok(CCV_NNC_GEMM_BACKWARD, CCV_NNC_BACKEND_GPU_CUBLAS));
    1706
    1
      dsfmt_t dsfmt;
    1707
    1
      dsfmt_init_gen_rand(&dsfmt, 0);
    1708
    1
      ccv_nnc_tensor_t* a = ccv_nnc_tensor_new(0, GPU_TENSOR_NHWC(000, 32F, 10, 128), 0);
    1709
    1
      ccv_nnc_tensor_t* w = ccv_nnc_tensor_new(0, GPU_TENSOR_NHWC(000, 32F, 64, 128), 0);
    1710
    1
      ccv_nnc_tensor_t* bias = ccv_nnc_tensor_new(0, GPU_TENSOR_NHWC(000, 32F, 64), 0);
    1711
    1
      ccv_nnc_tensor_t* b = ccv_nnc_tensor_new(0, GPU_TENSOR_NHWC(000, 32F, 10, 64), 0);
    1712
    1
      ccv_nnc_tensor_t* g = ccv_nnc_tensor_new(0, GPU_TENSOR_NHWC(000, 32F, 10, 64), 0);
    1713
    1
      ccv_nnc_tensor_t* dw = ccv_nnc_tensor_new(0, GPU_TENSOR_NHWC(000, 32F, 64, 128), 0);
    1714
    1
      ccv_nnc_tensor_t* dbias = ccv_nnc_tensor_new(0, GPU_TENSOR_NHWC(000, 32F, 64), 0);
    1715
    1
      ccv_nnc_tensor_t* h = ccv_nnc_tensor_new(0, GPU_TENSOR_NHWC(000, 32F, 10, 128), 0);
    1716
    1717
    1
      ccv_nnc_tensor_t* ha = ccv_nnc_tensor_new(0, CPU_TENSOR_NHWC(32F, 10, 128), 0);
    1718
    1
      ccv_nnc_tensor_t* hw = ccv_nnc_tensor_new(0, CPU_TENSOR_NHWC(32F, 64, 128), 0);
    1719
    1
      ccv_nnc_tensor_t* hbias = ccv_nnc_tensor_new(0, CPU_TENSOR_NHWC(32F, 64), 0);
    1720
    1
      ccv_nnc_tensor_t* hb = ccv_nnc_tensor_new(0, CPU_TENSOR_NHWC(32F, 10, 64), 0);
    1721
    1
      ccv_nnc_tensor_t* hg = ccv_nnc_tensor_new(0, CPU_TENSOR_NHWC(32F, 10, 64), 0);
    1722
    1
      ccv_nnc_tensor_t* hdw = ccv_nnc_tensor_new(0, CPU_TENSOR_NHWC(32F, 64, 128), 0);
    1723
    1
      ccv_nnc_tensor_t* hdbias = ccv_nnc_tensor_new(0, CPU_TENSOR_NHWC(32F, 64), 0);
    1724
    1
      ccv_nnc_tensor_t* hh = ccv_nnc_tensor_new(0, CPU_TENSOR_NHWC(32F, 10, 128), 0);
    1725
    1
      int i;
    1726
    8.19k
      for (i = 0; i < 64 * 128; i++8.19k)
    1727
    8.19k
        hw->data.f32[i] = dsfmt_genrand_open_close(&dsfmt) / (64 * 128);
    1728
    65
      for (i = 0; i < 64; i++64)
    1729
    64
        hbias->data.f32[i] = dsfmt_genrand_open_close(&dsfmt);
    1730
    1.28k
      for (i = 0; i < 10 * 128; i++1.28k)
    1731
    1.28k
        ha->data.f32[i] = dsfmt_genrand_open_close(&dsfmt);
    1732
    641
      for (i = 0; i < 10 * 64; i++640)
    1733
    640
        hg->data.f32[i] = dsfmt_genrand_open_close(&dsfmt);
    1734
    1
      ccv_nnc_cmd_exec(CMD_DATA_TRANSFER_FORWARD(), ccv_nnc_no_hint, 0, TENSOR_LIST(ha, hw, hbias, hg), TENSOR_LIST(a, w, bias, g), 0);
    1735
    1
      ccv_nnc_cmd_exec(CMD_GEMM_FORWARD(NO_TRANSPOSE, TRANSPOSE(0, 1)), ccv_nnc_no_hint, 0, TENSOR_LIST(ha, hw, hbias), TENSOR_LIST(hb), 0);
    1736
    1
      ccv_nnc_cmd_exec(CMD_GEMM_BACKWARD(NO_TRANSPOSE, TRANSPOSE(0, 1)), ccv_nnc_no_hint, 0, TENSOR_LIST(hg, ha, hw, 0), TENSOR_LIST(hh, hdw, hdbias), 0);
    1737
    1
      ccv_nnc_cmd_exec(CMD_GEMM_FORWARD(NO_TRANSPOSE, TRANSPOSE(0, 1)), ccv_nnc_no_hint, 0, TENSOR_LIST(a, w, bias), TENSOR_LIST(b), 0);
    1738
    1
      ccv_nnc_cmd_exec(CMD_GEMM_BACKWARD(NO_TRANSPOSE, TRANSPOSE(0, 1)), ccv_nnc_no_hint, 0, TENSOR_LIST(g, a, w, 0), TENSOR_LIST(h, dw, dbias), 0);
    1739
    1
      ccv_nnc_tensor_t* tb = ccv_nnc_tensor_new(0, CPU_TENSOR_NHWC(32F, 10, 64), 0);
    1740
    1
      ccv_nnc_tensor_t* tdw = ccv_nnc_tensor_new(0, CPU_TENSOR_NHWC(32F, 64, 128), 0);
    1741
    1
      ccv_nnc_tensor_t* tdbias = ccv_nnc_tensor_new(0, CPU_TENSOR_NHWC(32F, 64), 0);
    1742
    1
      ccv_nnc_tensor_t* th = ccv_nnc_tensor_new(0, CPU_TENSOR_NHWC(32F, 10, 128), 0);
    1743
    1
      ccv_nnc_cmd_exec(CMD_DATA_TRANSFER_FORWARD(), ccv_nnc_no_hint, 0, TENSOR_LIST(b, dw, dbias, h), TENSOR_LIST(tb, tdw, tdbias, th), 0);
    1744
    1
      REQUIRE_TENSOR_EQ(tb, hb, "GPU computed output should be the same as CPU computed ones");
    1745
    1
      REQUIRE_TENSOR_EQ(tdw, hdw, "GPU computed output should be the same as CPU computed ones");
    1746
    1
      REQUIRE_TENSOR_EQ(tdbias, hdbias, "GPU computed output should be the same as CPU computed ones");
    1747
    1
      REQUIRE_TENSOR_EQ(th, hh, "GPU computed output should be the same as CPU computed ones");
    1748
    1
      ccv_nnc_tensor_free(a);
    1749
    1
      ccv_nnc_tensor_free(w);
    1750
    1
      ccv_nnc_tensor_free(bias);
    1751
    1
      ccv_nnc_tensor_free(b);
    1752
    1
      ccv_nnc_tensor_free(g);
    1753
    1
      ccv_nnc_tensor_free(dw);
    1754
    1
      ccv_nnc_tensor_free(dbias);
    1755
    1
      ccv_nnc_tensor_free(h);
    1756
    1
      ccv_nnc_tensor_free(ha);
    1757
    1
      ccv_nnc_tensor_free(hw);
    1758
    1
      ccv_nnc_tensor_free(hbias);
    1759
    1
      ccv_nnc_tensor_free(hb);
    1760
    1
      ccv_nnc_tensor_free(hg);
    1761
    1
      ccv_nnc_tensor_free(hdw);
    1762
    1
      ccv_nnc_tensor_free(hdbias);
    1763
    1
      ccv_nnc_tensor_free(hh);
    1764
    1
      ccv_nnc_tensor_free(tb);
    1765
    1
      ccv_nnc_tensor_free(th);
    1766
    1
      ccv_nnc_tensor_free(tdw);
    1767
    1
      ccv_nnc_tensor_free(tdbias);
    1768
    1
    }
    1769
    1770
    TEST_CASE("cublas backward gemm in half precision")
    1771
    1
    {
    1772
    1
      GUARD_ELSE_RETURN(ccv_nnc_cmd_ok(CCV_NNC_GEMM_FORWARD, CCV_NNC_BACKEND_GPU_CUBLAS) &&
    1773
    1
        ccv_nnc_cmd_ok(CCV_NNC_GEMM_BACKWARD, CCV_NNC_BACKEND_GPU_CUBLAS));
    1774
    1
      dsfmt_t dsfmt;
    1775
    1
      dsfmt_init_gen_rand(&dsfmt, 0);
    1776
    1
      ccv_nnc_tensor_t* a = ccv_nnc_tensor_new(0, GPU_TENSOR_NHWC(000, 16F, 10, 128), 0);
    1777
    1
      ccv_nnc_tensor_t* w = ccv_nnc_tensor_new(0, GPU_TENSOR_NHWC(000, 16F, 64, 128), 0);
    1778
    1
      ccv_nnc_tensor_t* bias = ccv_nnc_tensor_new(0, GPU_TENSOR_NHWC(000, 16F, 64), 0);
    1779
    1
      ccv_nnc_tensor_t* b = ccv_nnc_tensor_new(0, GPU_TENSOR_NHWC(000, 16F, 10, 64), 0);
    1780
    1
      ccv_nnc_tensor_t* g = ccv_nnc_tensor_new(0, GPU_TENSOR_NHWC(000, 16F, 10, 64), 0);
    1781
    1
      ccv_nnc_tensor_t* dw = ccv_nnc_tensor_new(0, GPU_TENSOR_NHWC(000, 16F, 64, 128), 0);
    1782
    1
      ccv_nnc_tensor_t* dbias = ccv_nnc_tensor_new(0, GPU_TENSOR_NHWC(000, 16F, 64), 0);
    1783
    1
      ccv_nnc_tensor_t* h = ccv_nnc_tensor_new(0, GPU_TENSOR_NHWC(000, 16F, 10, 128), 0);
    1784
    1785
    1
      ccv_nnc_tensor_t* ha = ccv_nnc_tensor_new(0, CPU_TENSOR_NHWC(32F, 10, 128), 0);
    1786
    1
      ccv_nnc_tensor_t* hw = ccv_nnc_tensor_new(0, CPU_TENSOR_NHWC(32F, 64, 128), 0);
    1787
    1
      ccv_nnc_tensor_t* hbias = ccv_nnc_tensor_new(0, CPU_TENSOR_NHWC(32F, 64), 0);
    1788
    1
      ccv_nnc_tensor_t* hb = ccv_nnc_tensor_new(0, CPU_TENSOR_NHWC(32F, 10, 64), 0);
    1789
    1
      ccv_nnc_tensor_t* hg = ccv_nnc_tensor_new(0, CPU_TENSOR_NHWC(32F, 10, 64), 0);
    1790
    1
      ccv_nnc_tensor_t* hdw = ccv_nnc_tensor_new(0, CPU_TENSOR_NHWC(32F, 64, 128), 0);
    1791
    1
      ccv_nnc_tensor_t* hdbias = ccv_nnc_tensor_new(0, CPU_TENSOR_NHWC(32F, 64), 0);
    1792
    1
      ccv_nnc_tensor_t* hh = ccv_nnc_tensor_new(0, CPU_TENSOR_NHWC(32F, 10, 128), 0);
    1793
    1
      int i;
    1794
    8.19k
      for (i = 0; i < 64 * 128; i++8.19k)
    1795
    8.19k
        hw->data.f32[i] = dsfmt_genrand_open_close(&dsfmt) / (64 * 128);
    1796
    65
      for (i = 0; i < 64; i++64)
    1797
    64
        hbias->data.f32[i] = dsfmt_genrand_open_close(&dsfmt);
    1798
    1.28k
      for (i = 0; i < 10 * 128; i++1.28k)
    1799
    1.28k
        ha->data.f32[i] = dsfmt_genrand_open_close(&dsfmt);
    1800
    641
      for (i = 0; i < 10 * 64; i++640)
    1801
    640
        hg->data.f32[i] = dsfmt_genrand_open_close(&dsfmt);
    1802
    1
      ccv_nnc_tensor_t* ha2 = ccv_nnc_tensor_new(0, CPU_TENSOR_NHWC(16F, 10, 128), 0);
    1803
    1
      ccv_nnc_tensor_t* hw2 = ccv_nnc_tensor_new(0, CPU_TENSOR_NHWC(16F, 64, 128), 0);
    1804
    1
      ccv_nnc_tensor_t* hbias2 = ccv_nnc_tensor_new(0, CPU_TENSOR_NHWC(16F, 64), 0);
    1805
    1
      ccv_nnc_tensor_t* hg2 = ccv_nnc_tensor_new(0, CPU_TENSOR_NHWC(16F, 10, 64), 0);
    1806
    1
      ccv_nnc_cmd_exec(CMD_DATATYPE_CONVERSION_FORWARD(), ccv_nnc_no_hint, 0, TENSOR_LIST(ha, hw, hbias, hg), TENSOR_LIST(ha2, hw2, hbias2, hg2), 0);
    1807
    1
      ccv_nnc_cmd_exec(CMD_DATA_TRANSFER_FORWARD(), ccv_nnc_no_hint, 0, TENSOR_LIST(ha2, hw2, hbias2, hg2), TENSOR_LIST(a, w, bias, g), 0);
    1808
    1
      ccv_nnc_cmd_exec(CMD_GEMM_FORWARD(NO_TRANSPOSE, TRANSPOSE(0, 1)), ccv_nnc_no_hint, 0, TENSOR_LIST(ha, hw, hbias), TENSOR_LIST(hb), 0);
    1809
    1
      ccv_nnc_cmd_exec(CMD_GEMM_BACKWARD(NO_TRANSPOSE, TRANSPOSE(0, 1)), ccv_nnc_no_hint, 0, TENSOR_LIST(hg, ha, hw, 0), TENSOR_LIST(hh, hdw, hdbias), 0);
    1810
    1
      ccv_nnc_cmd_exec(CMD_GEMM_FORWARD(NO_TRANSPOSE, TRANSPOSE(0, 1)), ccv_nnc_no_hint, 0, TENSOR_LIST(a, w, bias), TENSOR_LIST(b), 0);
    1811
    1
      ccv_nnc_cmd_exec(CMD_GEMM_BACKWARD(NO_TRANSPOSE, TRANSPOSE(0, 1)), ccv_nnc_no_hint, 0, TENSOR_LIST(g, a, w, 0), TENSOR_LIST(h, dw, dbias), 0);
    1812
    1
      ccv_nnc_tensor_t* tb = ccv_nnc_tensor_new(0, CPU_TENSOR_NHWC(16F, 10, 64), 0);
    1813
    1
      ccv_nnc_tensor_t* tdw = ccv_nnc_tensor_new(0, CPU_TENSOR_NHWC(16F, 64, 128), 0);
    1814
    1
      ccv_nnc_tensor_t* tdbias = ccv_nnc_tensor_new(0, CPU_TENSOR_NHWC(16F, 64), 0);
    1815
    1
      ccv_nnc_tensor_t* th = ccv_nnc_tensor_new(0, CPU_TENSOR_NHWC(16F, 10, 128), 0);
    1816
    1
      ccv_nnc_cmd_exec(CMD_DATA_TRANSFER_FORWARD(), ccv_nnc_no_hint, 0, TENSOR_LIST(b, dw, dbias, h), TENSOR_LIST(tb, tdw, tdbias, th), 0);
    1817
    1
      ccv_nnc_tensor_t* tb1 = ccv_nnc_tensor_new(0, CPU_TENSOR_NHWC(32F, 10, 64), 0);
    1818
    1
      ccv_nnc_tensor_t* tdw1 = ccv_nnc_tensor_new(0, CPU_TENSOR_NHWC(32F, 64, 128), 0);
    1819
    1
      ccv_nnc_tensor_t* tdbias1 = ccv_nnc_tensor_new(0, CPU_TENSOR_NHWC(32F, 64), 0);
    1820
    1
      ccv_nnc_tensor_t* th1 = ccv_nnc_tensor_new(0, CPU_TENSOR_NHWC(32F, 10, 128), 0);
    1821
    1
      ccv_nnc_cmd_exec(CMD_DATATYPE_CONVERSION_FORWARD(), ccv_nnc_no_hint, 0, TENSOR_LIST(tb, tdw, tdbias, th), TENSOR_LIST(tb1, tdw1, tdbias1, th1), 0);
    1822
    1
      REQUIRE_ARRAY_EQ_WITH_TOLERANCE(float, tb1->data.f32, hb->data.f32, 10 * 64, 1e-3, "GPU computed output should be the same as CPU computed ones");
    1823
    1
      REQUIRE_ARRAY_EQ_WITH_TOLERANCE(float, tdw1->data.f32, hdw->data.f32, 64 * 128, 1e-2, "GPU computed output should be the same as CPU computed ones");
    1824
    1
      REQUIRE_ARRAY_EQ_WITH_TOLERANCE(float, tdbias1->data.f32, hdbias->data.f32, 64, 1e-2, "GPU computed output should be the same as CPU computed ones");
    1825
    1
      REQUIRE_ARRAY_EQ_WITH_TOLERANCE(float, th1->data.f32, hh->data.f32, 10 * 128, 1e-3, "GPU computed output should be the same as CPU computed ones");
    1826
    1
      ccv_nnc_tensor_free(a);
    1827
    1
      ccv_nnc_tensor_free(w);
    1828
    1
      ccv_nnc_tensor_free(bias);
    1829
    1
      ccv_nnc_tensor_free(b);
    1830
    1
      ccv_nnc_tensor_free(g);
    1831
    1
      ccv_nnc_tensor_free(dw);
    1832
    1
      ccv_nnc_tensor_free(dbias);
    1833
    1
      ccv_nnc_tensor_free(h);
    1834
    1
      ccv_nnc_tensor_free(ha);
    1835
    1
      ccv_nnc_tensor_free(hw);
    1836
    1
      ccv_nnc_tensor_free(hbias);
    1837
    1
      ccv_nnc_tensor_free(hb);
    1838
    1
      ccv_nnc_tensor_free(hg);
    1839
    1
      ccv_nnc_tensor_free(hdw);
    1840
    1
      ccv_nnc_tensor_free(hdbias);
    1841
    1
      ccv_nnc_tensor_free(hh);
    1842
    1
      ccv_nnc_tensor_free(tb);
    1843
    1
      ccv_nnc_tensor_free(th);
    1844
    1
      ccv_nnc_tensor_free(tdw);
    1845
    1
      ccv_nnc_tensor_free(tdbias);
    1846
    1
      ccv_nnc_tensor_free(ha2);
    1847
    1
      ccv_nnc_tensor_free(hw2);
    1848
    1
      ccv_nnc_tensor_free(hbias2);
    1849
    1
      ccv_nnc_tensor_free(hg2);
    1850
    1
      ccv_nnc_tensor_free(tb1);
    1851
    1
      ccv_nnc_tensor_free(tdw1);
    1852
    1
      ccv_nnc_tensor_free(tdbias1);
    1853
    1
      ccv_nnc_tensor_free(th1);
    1854
    1
    }
    1855
    1856
    TEST_CASE("cublas backward gemm no bias")
    1857
    1
    {
    1858
    1
      GUARD_ELSE_RETURN(ccv_nnc_cmd_ok(CCV_NNC_GEMM_FORWARD, CCV_NNC_BACKEND_GPU_CUBLAS) &&
    1859
    1
        ccv_nnc_cmd_ok(CCV_NNC_GEMM_BACKWARD, CCV_NNC_BACKEND_GPU_CUBLAS));
    1860
    1
      dsfmt_t dsfmt;
    1861
    1
      dsfmt_init_gen_rand(&dsfmt, 0);
    1862
    1
      ccv_nnc_tensor_t* a = ccv_nnc_tensor_new(0, GPU_TENSOR_NHWC(000, 32F, 10, 128), 0);
    1863
    1
      ccv_nnc_tensor_t* w = ccv_nnc_tensor_new(0, GPU_TENSOR_NHWC(000, 32F, 64, 128), 0);
    1864
    1
      ccv_nnc_tensor_t* b = ccv_nnc_tensor_new(0, GPU_TENSOR_NHWC(000, 32F, 10, 64), 0);
    1865
    1
      ccv_nnc_tensor_t* g = ccv_nnc_tensor_new(0, GPU_TENSOR_NHWC(000, 32F, 10, 64), 0);
    1866
    1
      ccv_nnc_tensor_t* dw = ccv_nnc_tensor_new(0, GPU_TENSOR_NHWC(000, 32F, 64, 128), 0);
    1867
    1
      ccv_nnc_tensor_t* h = ccv_nnc_tensor_new(0, GPU_TENSOR_NHWC(000, 32F, 10, 128), 0);
    1868
    1869
    1
      ccv_nnc_tensor_t* ha = ccv_nnc_tensor_new(0, CPU_TENSOR_NHWC(32F, 10, 128), 0);
    1870
    1
      ccv_nnc_tensor_t* hw = ccv_nnc_tensor_new(0, CPU_TENSOR_NHWC(32F, 64, 128), 0);
    1871
    1
      ccv_nnc_tensor_t* hb = ccv_nnc_tensor_new(0, CPU_TENSOR_NHWC(32F, 10, 64), 0);
    1872
    1
      ccv_nnc_tensor_t* hg = ccv_nnc_tensor_new(0, CPU_TENSOR_NHWC(32F, 10, 64), 0);
    1873
    1
      ccv_nnc_tensor_t* hdw = ccv_nnc_tensor_new(0, CPU_TENSOR_NHWC(32F, 64, 128), 0);
    1874
    1
      ccv_nnc_tensor_t* hh = ccv_nnc_tensor_new(0, CPU_TENSOR_NHWC(32F, 10, 128), 0);
    1875
    1
      int i;
    1876
    8.19k
      for (i = 0; i < 64 * 128; i++8.19k)
    1877
    8.19k
        hw->data.f32[i] = dsfmt_genrand_open_close(&dsfmt) / (64 * 128);
    1878
    1.28k
      for (i = 0; i < 10 * 128; i++1.28k)
    1879
    1.28k
        ha->data.f32[i] = dsfmt_genrand_open_close(&dsfmt);
    1880
    641
      for (i = 0; i < 10 * 64; i++640)
    1881
    640
        hg->data.f32[i] = dsfmt_genrand_open_close(&dsfmt);
    1882
    1
      ccv_nnc_cmd_exec(CMD_DATA_TRANSFER_FORWARD(), ccv_nnc_no_hint, 0, TENSOR_LIST(ha, hw, hg), TENSOR_LIST(a, w, g), 0);
    1883
    1
      ccv_nnc_cmd_exec(CMD_GEMM_FORWARD(NO_TRANSPOSE, TRANSPOSE(0, 1)), ccv_nnc_no_hint, 0, TENSOR_LIST(ha, hw), TENSOR_LIST(hb), 0);
    1884
    1
      ccv_nnc_cmd_exec(CMD_GEMM_BACKWARD(NO_TRANSPOSE, TRANSPOSE(0, 1)), ccv_nnc_no_hint, 0, TENSOR_LIST(hg, ha, hw, 0), TENSOR_LIST(hh, hdw, 0), 0);
    1885
    1
      ccv_nnc_cmd_exec(CMD_GEMM_FORWARD(NO_TRANSPOSE, TRANSPOSE(0, 1)), ccv_nnc_no_hint, 0, TENSOR_LIST(a, w), TENSOR_LIST(b), 0);
    1886
    1
      ccv_nnc_cmd_exec(CMD_GEMM_BACKWARD(NO_TRANSPOSE, TRANSPOSE(0, 1)), ccv_nnc_no_hint, 0, TENSOR_LIST(g, a, w, 0), TENSOR_LIST(h, dw, 0), 0);
    1887
    1
      ccv_nnc_tensor_t* tb = ccv_nnc_tensor_new(0, CPU_TENSOR_NHWC(32F, 10, 64), 0);
    1888
    1
      ccv_nnc_tensor_t* tdw = ccv_nnc_tensor_new(0, CPU_TENSOR_NHWC(32F, 64, 128), 0);
    1889
    1
      ccv_nnc_tensor_t* th = ccv_nnc_tensor_new(0, CPU_TENSOR_NHWC(32F, 10, 128), 0);
    1890
    1
      ccv_nnc_cmd_exec(CMD_DATA_TRANSFER_FORWARD(), ccv_nnc_no_hint, 0, TENSOR_LIST(b, dw, h), TENSOR_LIST(tb, tdw, th), 0);
    1891
    1
      REQUIRE_TENSOR_EQ(tb, hb, "GPU computed output should be the same as CPU computed ones");
    1892
    1
      REQUIRE_TENSOR_EQ(tdw, hdw, "GPU computed output should be the same as CPU computed ones");
    1893
    1
      REQUIRE_TENSOR_EQ(th, hh, "GPU computed output should be the same as CPU computed ones");
    1894
    1
      ccv_nnc_tensor_free(a);
    1895
    1
      ccv_nnc_tensor_free(w);
    1896
    1
      ccv_nnc_tensor_free(b);
    1897
    1
      ccv_nnc_tensor_free(g);
    1898
    1
      ccv_nnc_tensor_free(dw);
    1899
    1
      ccv_nnc_tensor_free(h);
    1900
    1
      ccv_nnc_tensor_free(ha);
    1901
    1
      ccv_nnc_tensor_free(hw);
    1902
    1
      ccv_nnc_tensor_free(hb);
    1903
    1
      ccv_nnc_tensor_free(hg);
    1904
    1
      ccv_nnc_tensor_free(hdw);
    1905
    1
      ccv_nnc_tensor_free(hh);
    1906
    1
      ccv_nnc_tensor_free(tb);
    1907
    1
      ccv_nnc_tensor_free(th);
    1908
    1
      ccv_nnc_tensor_free(tdw);
    1909
    1
    }
    1910
    1911
    TEST_CASE("cublas backward gemm no bias in half precision")
    1912
    1
    {
    1913
    1
      GUARD_ELSE_RETURN(ccv_nnc_cmd_ok(CCV_NNC_GEMM_FORWARD, CCV_NNC_BACKEND_GPU_CUBLAS) &&
    1914
    1
        ccv_nnc_cmd_ok(CCV_NNC_GEMM_BACKWARD, CCV_NNC_BACKEND_GPU_CUBLAS));
    1915
    1
      dsfmt_t dsfmt;
    1916
    1
      dsfmt_init_gen_rand(&dsfmt, 0);
    1917
    1
      ccv_nnc_tensor_t* a = ccv_nnc_tensor_new(0, GPU_TENSOR_NHWC(000, 16F, 10, 128), 0);
    1918
    1
      ccv_nnc_tensor_t* w = ccv_nnc_tensor_new(0, GPU_TENSOR_NHWC(000, 16F, 64, 128), 0);
    1919
    1
      ccv_nnc_tensor_t* b = ccv_nnc_tensor_new(0, GPU_TENSOR_NHWC(000, 16F, 10, 64), 0);
    1920
    1
      ccv_nnc_tensor_t* g = ccv_nnc_tensor_new(0, GPU_TENSOR_NHWC(000, 16F, 10, 64), 0);
    1921
    1
      ccv_nnc_tensor_t* dw = ccv_nnc_tensor_new(0, GPU_TENSOR_NHWC(000, 16F, 64, 128), 0);
    1922
    1
      ccv_nnc_tensor_t* h = ccv_nnc_tensor_new(0, GPU_TENSOR_NHWC(000, 16F, 10, 128), 0);
    1923
    1924
    1
      ccv_nnc_tensor_t* ha = ccv_nnc_tensor_new(0, CPU_TENSOR_NHWC(32F, 10, 128), 0);
    1925
    1
      ccv_nnc_tensor_t* hw = ccv_nnc_tensor_new(0, CPU_TENSOR_NHWC(32F, 64, 128), 0);
    1926
    1
      ccv_nnc_tensor_t* hb = ccv_nnc_tensor_new(0, CPU_TENSOR_NHWC(32F, 10, 64), 0);
    1927
    1
      ccv_nnc_tensor_t* hg = ccv_nnc_tensor_new(0, CPU_TENSOR_NHWC(32F, 10, 64), 0);
    1928
    1
      ccv_nnc_tensor_t* hdw = ccv_nnc_tensor_new(0, CPU_TENSOR_NHWC(32F, 64, 128), 0);
    1929
    1
      ccv_nnc_tensor_t* hh = ccv_nnc_tensor_new(0, CPU_TENSOR_NHWC(32F, 10, 128), 0);
    1930
    1
      int i;
    1931
    8.19k
      for (i = 0; i < 64 * 128; i++8.19k)
    1932
    8.19k
        hw->data.f32[i] = dsfmt_genrand_open_close(&dsfmt) / (64 * 128);
    1933
    1.28k
      for (i = 0; i < 10 * 128; i++1.28k)
    1934
    1.28k
        ha->data.f32[i] = dsfmt_genrand_open_close(&dsfmt);
    1935
    641
      for (i = 0; i < 10 * 64; i++640)
    1936
    640
        hg->data.f32[i] = dsfmt_genrand_open_close(&dsfmt);
    1937
    1
      ccv_nnc_tensor_t* ha2 = ccv_nnc_tensor_new(0, CPU_TENSOR_NHWC(16F, 10, 128), 0);
    1938
    1
      ccv_nnc_tensor_t* hw2 = ccv_nnc_tensor_new(0, CPU_TENSOR_NHWC(16F, 64, 128), 0);
    1939
    1
      ccv_nnc_tensor_t* hg2 = ccv_nnc_tensor_new(0, CPU_TENSOR_NHWC(16F, 10, 64), 0);
    1940
    1
      ccv_nnc_cmd_exec(CMD_DATATYPE_CONVERSION_FORWARD(), ccv_nnc_no_hint, 0, TENSOR_LIST(ha, hw, hg), TENSOR_LIST(ha2, hw2, hg2), 0);
    1941
    1
      ccv_nnc_cmd_exec(CMD_DATA_TRANSFER_FORWARD(), ccv_nnc_no_hint, 0, TENSOR_LIST(ha2, hw2, hg2), TENSOR_LIST(a, w, g), 0);
    1942
    1
      ccv_nnc_cmd_exec(CMD_GEMM_FORWARD(NO_TRANSPOSE, TRANSPOSE(0, 1)), ccv_nnc_no_hint, 0, TENSOR_LIST(ha, hw), TENSOR_LIST(hb), 0);
    1943
    1
      ccv_nnc_cmd_exec(CMD_GEMM_BACKWARD(NO_TRANSPOSE, TRANSPOSE(0, 1)), ccv_nnc_no_hint, 0, TENSOR_LIST(hg, ha, hw, 0), TENSOR_LIST(hh, hdw, 0), 0);
    1944
    1
      ccv_nnc_cmd_exec(CMD_GEMM_FORWARD(NO_TRANSPOSE, TRANSPOSE(0, 1)), ccv_nnc_no_hint, 0, TENSOR_LIST(a, w), TENSOR_LIST(b), 0);
    1945
    1
      ccv_nnc_cmd_exec(CMD_GEMM_BACKWARD(NO_TRANSPOSE, TRANSPOSE(0, 1)), ccv_nnc_no_hint, 0, TENSOR_LIST(g, a, w, 0), TENSOR_LIST(h, dw, 0), 0);
    1946
    1
      ccv_nnc_tensor_t* tb = ccv_nnc_tensor_new(0, CPU_TENSOR_NHWC(16F, 10, 64), 0);
    1947
    1
      ccv_nnc_tensor_t* tdw = ccv_nnc_tensor_new(0, CPU_TENSOR_NHWC(16F, 64, 128), 0);
    1948
    1
      ccv_nnc_tensor_t* th = ccv_nnc_tensor_new(0, CPU_TENSOR_NHWC(16F, 10, 128), 0);
    1949
    1
      ccv_nnc_tensor_t* tb1 = ccv_nnc_tensor_new(0, CPU_TENSOR_NHWC(32F, 10, 64), 0);
    1950
    1
      ccv_nnc_tensor_t* tdw1 = ccv_nnc_tensor_new(0, CPU_TENSOR_NHWC(32F, 64, 128), 0);
    1951
    1
      ccv_nnc_tensor_t* th1 = ccv_nnc_tensor_new(0, CPU_TENSOR_NHWC(32F, 10, 128), 0);
    1952
    1
      ccv_nnc_cmd_exec(CMD_DATA_TRANSFER_FORWARD(), ccv_nnc_no_hint, 0, TENSOR_LIST(b, dw, h), TENSOR_LIST(tb, tdw, th), 0);
    1953
    1
      ccv_nnc_cmd_exec(CMD_DATATYPE_CONVERSION_FORWARD(), ccv_nnc_no_hint, 0, TENSOR_LIST(tb, tdw, th), TENSOR_LIST(tb1, tdw1, th1), 0);
    1954
    1
      REQUIRE_ARRAY_EQ_WITH_TOLERANCE(float, tb1->data.f32, hb->data.f32, 10 * 64, 1e-3, "GPU computed output should be the same as CPU computed ones");
    1955
    1
      REQUIRE_ARRAY_EQ_WITH_TOLERANCE(float, tdw1->data.f32, hdw->data.f32, 64 * 128, 1e-2, "GPU computed output should be the same as CPU computed ones");
    1956
    1
      REQUIRE_ARRAY_EQ_WITH_TOLERANCE(float, th1->data.f32, hh->data.f32, 10 * 128, 1e-3, "GPU computed output should be the same as CPU computed ones");
    1957
    1
      ccv_nnc_tensor_free(a);
    1958
    1
      ccv_nnc_tensor_free(w);
    1959
    1
      ccv_nnc_tensor_free(b);
    1960
    1
      ccv_nnc_tensor_free(g);
    1961
    1
      ccv_nnc_tensor_free(dw);
    1962
    1
      ccv_nnc_tensor_free(h);
    1963
    1
      ccv_nnc_tensor_free(ha);
    1964
    1
      ccv_nnc_tensor_free(hw);
    1965
    1
      ccv_nnc_tensor_free(hb);
    1966
    1
      ccv_nnc_tensor_free(hg);
    1967
    1
      ccv_nnc_tensor_free(hdw);
    1968
    1
      ccv_nnc_tensor_free(hh);
    1969
    1
      ccv_nnc_tensor_free(tb);
    1970
    1
      ccv_nnc_tensor_free(th);
    1971
    1
      ccv_nnc_tensor_free(tdw);
    1972
    1
      ccv_nnc_tensor_free(ha2);
    1973
    1
      ccv_nnc_tensor_free(hw2);
    1974
    1
      ccv_nnc_tensor_free(hg2);
    1975
    1
      ccv_nnc_tensor_free(tb1);
    1976
    1
      ccv_nnc_tensor_free(tdw1);
    1977
    1
      ccv_nnc_tensor_free(th1);
    1978
    1
    }
    1979
    1980
    TEST_CASE("cublas handle permute")
    1981
    1
    {
    1982
    1
      GUARD_ELSE_RETURN(ccv_nnc_cmd_ok(CCV_NNC_GEMM_FORWARD, CCV_NNC_BACKEND_GPU_CUBLAS));
    1983
    1
      dsfmt_t dsfmt;
    1984
    1
      dsfmt_init_gen_rand(&dsfmt, 0);
    1985
    1
      ccv_nnc_tensor_t* a = ccv_nnc_tensor_new(0, GPU_TENSOR_NHWC(000, 32F, 10, 2, 128), 0);
    1986
    1
      ccv_nnc_tensor_t* w = ccv_nnc_tensor_new(0, GPU_TENSOR_NHWC(000, 32F, 64, 2, 128), 0);
    1987
    1
      ccv_nnc_tensor_t* ha = ccv_nnc_tensor_new(0, CPU_TENSOR_NHWC(32F, 10, 2, 128), 0);
    1988
    1
      ccv_nnc_tensor_t* hw = ccv_nnc_tensor_new(0, CPU_TENSOR_NHWC(32F, 64, 2, 128), 0);
    1989
    1
      ccv_nnc_tensor_t* b = ccv_nnc_tensor_new(0, GPU_TENSOR_NHWC(000, 32F, 2, 10, 64), 0);
    1990
    1991
    1
      ccv_nnc_tensor_t* at = ccv_nnc_tensor_new(0, GPU_TENSOR_NHWC(000, 32F, 2, 10, 128), 0);
    1992
    1
      ccv_nnc_tensor_t* wt = ccv_nnc_tensor_new(0, GPU_TENSOR_NHWC(000, 32F, 2, 64, 128), 0);
    1993
    1
      ccv_nnc_tensor_t* bt = ccv_nnc_tensor_new(0, GPU_TENSOR_NHWC(000, 32F, 2, 10, 64), 0);
    1994
    1
      int i;
    1995
    16.3k
      for (i = 0; i < 2 * 64 * 128; i++16.3k)
    1996
    16.3k
        hw->data.f32[i] = dsfmt_genrand_open_close(&dsfmt) / (64 * 128);
    1997
    2.56k
      for (i = 0; i < 2 * 10 * 128; i++2.56k)
    1998
    2.56k
        ha->data.f32[i] = dsfmt_genrand_open_close(&dsfmt);
    1999
    1
      ccv_nnc_cmd_exec(CMD_DATA_TRANSFER_FORWARD(), ccv_nnc_no_hint, 0, TENSOR_LIST(ha, hw), TENSOR_LIST(a, w), 0);
    2000
    1
      ccv_nnc_cmd_exec(CMD_TRANSPOSE_FORWARD(0, 1), ccv_nnc_no_hint, 0, TENSOR_LIST(a), TENSOR_LIST(at), 0);
    2001
    1
      ccv_nnc_cmd_exec(CMD_TRANSPOSE_FORWARD(0, 1), ccv_nnc_no_hint, 0, TENSOR_LIST(w), TENSOR_LIST(wt), 0);
    2002
    1
      ccv_nnc_cmd_exec(CMD_GEMM_FORWARD(NO_TRANSPOSE, TRANSPOSE(1, 2)), ccv_nnc_no_hint, 0, TENSOR_LIST(at, wt), TENSOR_LIST(bt), 0);
    2003
    1
      ccv_nnc_tensor_view_t* av = ccv_nnc_tensor_view_new(a, GPU_TENSOR_NHWC(000, 32F, 2, 10, 128), ccv_nnc_no_ofs, DIM_ALLOC(128, 2 * 128, 1));
    2004
    1
      ccv_nnc_tensor_view_t* wv = ccv_nnc_tensor_view_new(w, GPU_TENSOR_NHWC(000, 32F, 2, 64, 128), ccv_nnc_no_ofs, DIM_ALLOC(128, 2 * 128, 1));
    2005
    1
      ccv_nnc_cmd_exec(CMD_GEMM_FORWARD(NO_TRANSPOSE, TRANSPOSE(1, 2)), ccv_nnc_no_hint, 0, TENSOR_LIST((ccv_nnc_tensor_t*)av, (ccv_nnc_tensor_t*)wv), TENSOR_LIST(b), 0);
    2006
    1
      ccv_nnc_tensor_t* hb = ccv_nnc_tensor_new(0, CPU_TENSOR_NHWC(32F, 2, 10, 64), 0);
    2007
    1
      ccv_nnc_tensor_t* hbt = ccv_nnc_tensor_new(0, CPU_TENSOR_NHWC(32F, 2, 10, 64), 0);
    2008
    1
      ccv_nnc_cmd_exec(CMD_DATA_TRANSFER_FORWARD(), ccv_nnc_no_hint, 0, TENSOR_LIST(b, bt), TENSOR_LIST(hb, hbt), 0);
    2009
    1
      REQUIRE_TENSOR_EQ(hb, hbt, "permute computed output should be the same as non-permute computed ones");
    2010
    1
      ccv_nnc_tensor_free(ha);
    2011
    1
      ccv_nnc_tensor_free(hw);
    2012
    1
      ccv_nnc_tensor_free(a);
    2013
    1
      ccv_nnc_tensor_free(w);
    2014
    1
      ccv_nnc_tensor_free(b);
    2015
    1
      ccv_nnc_tensor_view_free(av);
    2016
    1
      ccv_nnc_tensor_view_free(wv);
    2017
    1
      ccv_nnc_tensor_free(at);
    2018
    1
      ccv_nnc_tensor_free(wt);
    2019
    1
      ccv_nnc_tensor_free(bt);
    2020
    1
      ccv_nnc_tensor_free(hb);
    2021
    1
      ccv_nnc_tensor_free(hbt);
    2022
    1
    }
    2023
    2024
    TEST_CASE("generalized batched gemm with batch (2, 4) compare cublas")
    2025
    1
    {
    2026
    1
      GUARD_ELSE_RETURN(ccv_nnc_cmd_ok(CCV_NNC_GEMM_FORWARD, CCV_NNC_BACKEND_GPU_CUBLAS));
    2027
      // This is a particular batched gemm which treat every dimensions other than the last two as batching.
    2028
    1
      dsfmt_t dsfmt;
    2029
    1
      dsfmt_init_gen_rand(&dsfmt, 0);
    2030
    1
      ccv_nnc_tensor_t* ha = ccv_nnc_tensor_new(0, CPU_TENSOR_NHWC(32F, 2, 10, 4, 128), 0);
    2031
    1
      ccv_nnc_tensor_t* hw = ccv_nnc_tensor_new(0, CPU_TENSOR_NHWC(32F, 2, 64, 4, 128), 0);
    2032
    1
      ccv_nnc_tensor_t* hb = ccv_nnc_tensor_new(0, CPU_TENSOR_NHWC(32F, 2, 4, 10, 64), 0);
    2033
    1
      ccv_nnc_tensor_t* a = ccv_nnc_tensor_new(0, GPU_TENSOR_NHWC(000, 32F, 2, 10, 4, 128), 0);
    2034
    1
      ccv_nnc_tensor_t* w = ccv_nnc_tensor_new(0, GPU_TENSOR_NHWC(000, 32F, 2, 64, 4, 128), 0);
    2035
    1
      ccv_nnc_tensor_t* b = ccv_nnc_tensor_new(0, GPU_TENSOR_NHWC(000, 32F, 2, 4, 10, 64), 0);
    2036
    2037
    1
      ccv_nnc_tensor_t* at = ccv_nnc_tensor_new(0, CPU_TENSOR_NHWC(32F, 2, 4, 10, 128), 0);
    2038
    1
      ccv_nnc_tensor_t* wt = ccv_nnc_tensor_new(0, CPU_TENSOR_NHWC(32F, 2, 4, 64, 128), 0);
    2039
    1
      ccv_nnc_tensor_t* bt = ccv_nnc_tensor_new(0, CPU_TENSOR_NHWC(32F, 2, 4, 10, 64), 0);
    2040
    1
      int i;
    2041
    65.5k
      for (i = 0; i < 8 * 64 * 128; i++65.5k)
    2042
    65.5k
        hw->data.f32[i] = dsfmt_genrand_open_close(&dsfmt) / (64 * 128);
    2043
    10.2k
      for (i = 0; i < 8 * 10 * 128; i++10.2k)
    2044
    10.2k
        ha->data.f32[i] = dsfmt_genrand_open_close(&dsfmt);
    2045
    1
      ccv_nnc_cmd_exec(CMD_TRANSPOSE_FORWARD(1, 2), ccv_nnc_no_hint, 0, TENSOR_LIST(ha), TENSOR_LIST(at), 0);
    2046
    1
      ccv_nnc_cmd_exec(CMD_TRANSPOSE_FORWARD(1, 2), ccv_nnc_no_hint, 0, TENSOR_LIST(hw), TENSOR_LIST(wt), 0);
    2047
    1
      ccv_nnc_cmd_exec(CMD_DATA_TRANSFER_FORWARD(), ccv_nnc_no_hint, 0, TENSOR_LIST(ha, hw), TENSOR_LIST(a, w), 0);
    2048
    1
      ccv_nnc_tensor_view_t* av = ccv_nnc_tensor_view_new(a, GPU_TENSOR_NHWC(000, 32F, 2, 4, 10, 128), ccv_nnc_no_ofs, DIM_ALLOC(10 * 4 * 128, 128, 4 * 128, 1));
    2049
    1
      ccv_nnc_tensor_view_t* wv = ccv_nnc_tensor_view_new(w, GPU_TENSOR_NHWC(000, 32F, 2, 4, 64, 128), ccv_nnc_no_ofs, DIM_ALLOC(64 * 4 * 128, 128, 4 * 128, 1));
    2050
    1
      ccv_nnc_cmd_exec(CMD_GEMM_FORWARD(NO_TRANSPOSE, TRANSPOSE(2, 3)), ccv_nnc_no_hint, 0, TENSOR_LIST((ccv_nnc_tensor_t*)av, (ccv_nnc_tensor_t*)wv), TENSOR_LIST(b), 0);
    2051
    1
      ccv_nnc_cmd_exec(CMD_GEMM_FORWARD(NO_TRANSPOSE, TRANSPOSE(2, 3)), ccv_nnc_no_hint, 0, TENSOR_LIST(at, wt), TENSOR_LIST(bt), 0);
    2052
    1
      ccv_nnc_cmd_exec(CMD_DATA_TRANSFER_FORWARD(), ccv_nnc_no_hint, 0, TENSOR_LIST(b), TENSOR_LIST(hb), 0);
    2053
    1
      REQUIRE_TENSOR_EQ(hb, bt, "permute computed output should be the same as non-permute computed ones");
    2054
    1
      ccv_nnc_tensor_free(ha);
    2055
    1
      ccv_nnc_tensor_free(hw);
    2056
    1
      ccv_nnc_tensor_free(hb);
    2057
    1
      ccv_nnc_tensor_free(a);
    2058
    1
      ccv_nnc_tensor_free(w);
    2059
    1
      ccv_nnc_tensor_free(b);
    2060
    1
      ccv_nnc_tensor_view_free(av);
    2061
    1
      ccv_nnc_tensor_view_free(wv);
    2062
    1
      ccv_nnc_tensor_free(at);
    2063
    1
      ccv_nnc_tensor_free(wt);
    2064
    1
      ccv_nnc_tensor_free(bt);
    2065
    1
    }
    2066
    2067
    TEST_CASE("generalized batched gemm with batch (2, 4) and broadcast compare cublas")
    2068
    1
    {
    2069
    1
      GUARD_ELSE_RETURN(ccv_nnc_cmd_ok(CCV_NNC_GEMM_FORWARD, CCV_NNC_BACKEND_GPU_CUBLAS));
    2070
      // This is a particular batched gemm which treat every dimensions other than the last two as batching.
    2071
    1
      dsfmt_t dsfmt;
    2072
    1
      dsfmt_init_gen_rand(&dsfmt, 0);
    2073
    1
      ccv_nnc_tensor_t* ha = ccv_nnc_tensor_new(0, CPU_TENSOR_NHWC(32F, 2, 10, 4, 128), 0);
    2074
    1
      ccv_nnc_tensor_t* hw = ccv_nnc_tensor_new(0, CPU_TENSOR_NHWC(32F, 64, 128), 0);
    2075
    1
      ccv_nnc_tensor_t* hb = ccv_nnc_tensor_new(0, CPU_TENSOR_NHWC(32F, 2, 4, 10, 64), 0);
    2076
    1
      ccv_nnc_tensor_t* a = ccv_nnc_tensor_new(0, GPU_TENSOR_NHWC(000, 32F, 2, 10, 4, 128), 0);
    2077
    1
      ccv_nnc_tensor_t* w = ccv_nnc_tensor_new(0, GPU_TENSOR_NHWC(000, 32F, 64, 128), 0);
    2078
    1
      ccv_nnc_tensor_t* b = ccv_nnc_tensor_new(0, GPU_TENSOR_NHWC(000, 32F, 2, 4, 10, 64), 0);
    2079
    2080
    1
      ccv_nnc_tensor_t* at = ccv_nnc_tensor_new(0, CPU_TENSOR_NHWC(32F, 2, 4, 10, 128), 0);
    2081
    1
      ccv_nnc_tensor_t* bt = ccv_nnc_tensor_new(0, CPU_TENSOR_NHWC(32F, 2, 4, 10, 64), 0);
    2082
    1
      int i;
    2083
    8.19k
      for (i = 0; i < 64 * 128; i++8.19k)
    2084
    8.19k
        hw->data.f32[i] = dsfmt_genrand_open_close(&dsfmt) / (64 * 128);
    2085
    10.2k
      for (i = 0; i < 8 * 10 * 128; i++10.2k)
    2086
    10.2k
        ha->data.f32[i] = dsfmt_genrand_open_close(&dsfmt);
    2087
    1
      ccv_nnc_cmd_exec(CMD_TRANSPOSE_FORWARD(1, 2), ccv_nnc_no_hint, 0, TENSOR_LIST(ha), TENSOR_LIST(at), 0);
    2088
    1
      ccv_nnc_cmd_exec(CMD_DATA_TRANSFER_FORWARD(), ccv_nnc_no_hint, 0, TENSOR_LIST(ha, hw), TENSOR_LIST(a, w), 0);
    2089
    1
      ccv_nnc_tensor_view_t* av = ccv_nnc_tensor_view_new(a, GPU_TENSOR_NHWC(000, 32F, 2, 4, 10, 128), ccv_nnc_no_ofs, DIM_ALLOC(10 * 4 * 128, 128, 4 * 128, 1));
    2090
    1
      ccv_nnc_cmd_exec(CMD_GEMM_FORWARD(NO_TRANSPOSE, TRANSPOSE(0, 1)), ccv_nnc_no_hint, 0, TENSOR_LIST((ccv_nnc_tensor_t*)av, w), TENSOR_LIST(b), 0);
    2091
    1
      ccv_nnc_cmd_exec(CMD_GEMM_FORWARD(NO_TRANSPOSE, TRANSPOSE(0, 1)), ccv_nnc_no_hint, 0, TENSOR_LIST(at, hw), TENSOR_LIST(bt), 0);
    2092
    1
      ccv_nnc_cmd_exec(CMD_DATA_TRANSFER_FORWARD(), ccv_nnc_no_hint, 0, TENSOR_LIST(b), TENSOR_LIST(hb), 0);
    2093
    1
      REQUIRE_TENSOR_EQ(hb, bt, "permute computed output should be the same as non-permute computed ones");
    2094
    1
      ccv_nnc_tensor_free(ha);
    2095
    1
      ccv_nnc_tensor_free(hw);
    2096
    1
      ccv_nnc_tensor_free(hb);
    2097
    1
      ccv_nnc_tensor_free(a);
    2098
    1
      ccv_nnc_tensor_free(w);
    2099
    1
      ccv_nnc_tensor_free(b);
    2100
    1
      ccv_nnc_tensor_view_free(av);
    2101
    1
      ccv_nnc_tensor_free(at);
    2102
    1
      ccv_nnc_tensor_free(bt);
    2103
    1
    }
    2104
    2105
    TEST_CASE("generalized batched gemm with batch (2, 4) with bias compare cublas")
    2106
    1
    {
    2107
    1
      GUARD_ELSE_RETURN(ccv_nnc_cmd_ok(CCV_NNC_GEMM_FORWARD, CCV_NNC_BACKEND_GPU_CUBLAS));
    2108
      // This is a particular batched gemm which treat every dimensions other than the last two as batching.
    2109
    1
      dsfmt_t dsfmt;
    2110
    1
      dsfmt_init_gen_rand(&dsfmt, 0);
    2111
    1
      ccv_nnc_tensor_t* ha = ccv_nnc_tensor_new(0, CPU_TENSOR_NHWC(32F, 2, 10, 4, 128), 0);
    2112
    1
      ccv_nnc_tensor_t* hw = ccv_nnc_tensor_new(0, CPU_TENSOR_NHWC(32F, 2, 64, 4, 128), 0);
    2113
    1
      ccv_nnc_tensor_t* hbias = ccv_nnc_tensor_new(0, CPU_TENSOR_NHWC(32F, 64), 0);
    2114
    1
      ccv_nnc_tensor_t* hb = ccv_nnc_tensor_new(0, CPU_TENSOR_NHWC(32F, 2, 4, 10, 64), 0);
    2115
    1
      ccv_nnc_tensor_t* a = ccv_nnc_tensor_new(0, GPU_TENSOR_NHWC(000, 32F, 2, 10, 4, 128), 0);
    2116
    1
      ccv_nnc_tensor_t* w = ccv_nnc_tensor_new(0, GPU_TENSOR_NHWC(000, 32F, 2, 64, 4, 128), 0);
    2117
    1
      ccv_nnc_tensor_t* bias = ccv_nnc_tensor_new(0, GPU_TENSOR_NHWC(000, 32F, 64), 0);
    2118
    1
      ccv_nnc_tensor_t* b = ccv_nnc_tensor_new(0, GPU_TENSOR_NHWC(000, 32F, 2, 4, 10, 64), 0);
    2119
    2120
    1
      ccv_nnc_tensor_t* at = ccv_nnc_tensor_new(0, CPU_TENSOR_NHWC(32F, 2, 4, 10, 128), 0);
    2121
    1
      ccv_nnc_tensor_t* wt = ccv_nnc_tensor_new(0, CPU_TENSOR_NHWC(32F, 2, 4, 64, 128), 0);
    2122
    1
      ccv_nnc_tensor_t* bt = ccv_nnc_tensor_new(0, CPU_TENSOR_NHWC(32F, 2, 4, 10, 64), 0);
    2123
    1
      int i;
    2124
    65.5k
      for (i = 0; i < 8 * 64 * 128; i++65.5k)
    2125
    65.5k
        hw->data.f32[i] = dsfmt_genrand_open_close(&dsfmt) / (64 * 128);
    2126
    65
      for (i = 0; i < 64; i++64)
    2127
    64
        hbias->data.f32[i] = dsfmt_genrand_open_close(&dsfmt) / 64;
    2128
    10.2k
      for (i = 0; i < 8 * 10 * 128; i++10.2k)
    2129
    10.2k
        ha->data.f32[i] = dsfmt_genrand_open_close(&dsfmt);
    2130
    1
      ccv_nnc_cmd_exec(CMD_TRANSPOSE_FORWARD(1, 2), ccv_nnc_no_hint, 0, TENSOR_LIST(ha), TENSOR_LIST(at), 0);
    2131
    1
      ccv_nnc_cmd_exec(CMD_TRANSPOSE_FORWARD(1, 2), ccv_nnc_no_hint, 0, TENSOR_LIST(hw), TENSOR_LIST(wt), 0);
    2132
    1
      ccv_nnc_cmd_exec(CMD_DATA_TRANSFER_FORWARD(), ccv_nnc_no_hint, 0, TENSOR_LIST(ha, hw, hbias), TENSOR_LIST(a, w, bias), 0);
    2133
    1
      ccv_nnc_tensor_view_t* av = ccv_nnc_tensor_view_new(a, GPU_TENSOR_NHWC(000, 32F, 2, 4, 10, 128), ccv_nnc_no_ofs, DIM_ALLOC(10 * 4 * 128, 128, 4 * 128, 1));
    2134
    1
      ccv_nnc_tensor_view_t* wv = ccv_nnc_tensor_view_new(w, GPU_TENSOR_NHWC(000, 32F, 2, 4, 64, 128), ccv_nnc_no_ofs, DIM_ALLOC(64 * 4 * 128, 128, 4 * 128, 1));
    2135
    1
      ccv_nnc_cmd_exec(CMD_GEMM_FORWARD(NO_TRANSPOSE, TRANSPOSE(2, 3)), ccv_nnc_no_hint, 0, TENSOR_LIST((ccv_nnc_tensor_t*)av, (ccv_nnc_tensor_t*)wv, bias), TENSOR_LIST(b), 0);
    2136
    1
      ccv_nnc_cmd_exec(CMD_GEMM_FORWARD(NO_TRANSPOSE, TRANSPOSE(2, 3)), ccv_nnc_no_hint, 0, TENSOR_LIST(at, wt, hbias), TENSOR_LIST(bt), 0);
    2137
    1
      ccv_nnc_cmd_exec(CMD_DATA_TRANSFER_FORWARD(), ccv_nnc_no_hint, 0, TENSOR_LIST(b), TENSOR_LIST(hb), 0);
    2138
    1
      REQUIRE_TENSOR_EQ(hb, bt, "permute computed output should be the same as non-permute computed ones");
    2139
    1
      ccv_nnc_tensor_free(ha);
    2140
    1
      ccv_nnc_tensor_free(hw);
    2141
    1
      ccv_nnc_tensor_free(hbias);
    2142
    1
      ccv_nnc_tensor_free(hb);
    2143
    1
      ccv_nnc_tensor_free(a);
    2144
    1
      ccv_nnc_tensor_free(w);
    2145
    1
      ccv_nnc_tensor_free(bias);
    2146
    1
      ccv_nnc_tensor_free(b);
    2147
    1
      ccv_nnc_tensor_view_free(av);
    2148
    1
      ccv_nnc_tensor_view_free(wv);
    2149
    1
      ccv_nnc_tensor_free(at);
    2150
    1
      ccv_nnc_tensor_free(wt);
    2151
    1
      ccv_nnc_tensor_free(bt);
    2152
    1
    }
    2153
    2154
    TEST_CASE("generalized batched gemm with batch (2, 4) with bias and broadcast compare cublas")
    2155
    1
    {
    2156
    1
      GUARD_ELSE_RETURN(ccv_nnc_cmd_ok(CCV_NNC_GEMM_FORWARD, CCV_NNC_BACKEND_GPU_CUBLAS));
    2157
      // This is a particular batched gemm which treat every dimensions other than the last two as batching.
    2158
    1
      dsfmt_t dsfmt;
    2159
    1
      dsfmt_init_gen_rand(&dsfmt, 0);
    2160
    1
      ccv_nnc_tensor_t* ha = ccv_nnc_tensor_new(0, CPU_TENSOR_NHWC(32F, 2, 10, 4, 128), 0);
    2161
    1
      ccv_nnc_tensor_t* hw = ccv_nnc_tensor_new(0, CPU_TENSOR_NHWC(32F, 64, 128), 0);
    2162
    1
      ccv_nnc_tensor_t* hbias = ccv_nnc_tensor_new(0, CPU_TENSOR_NHWC(32F, 64), 0);
    2163
    1
      ccv_nnc_tensor_t* hb = ccv_nnc_tensor_new(0, CPU_TENSOR_NHWC(32F, 2, 4, 10, 64), 0);
    2164
    1
      ccv_nnc_tensor_t* a = ccv_nnc_tensor_new(0, GPU_TENSOR_NHWC(000, 32F, 2, 10, 4, 128), 0);
    2165
    1
      ccv_nnc_tensor_t* w = ccv_nnc_tensor_new(0, GPU_TENSOR_NHWC(000, 32F, 64, 128), 0);
    2166
    1
      ccv_nnc_tensor_t* bias = ccv_nnc_tensor_new(0, GPU_TENSOR_NHWC(000, 32F, 64), 0);
    2167
    1
      ccv_nnc_tensor_t* b = ccv_nnc_tensor_new(0, GPU_TENSOR_NHWC(000, 32F, 2, 4, 10, 64), 0);
    2168
    2169
    1
      ccv_nnc_tensor_t* at = ccv_nnc_tensor_new(0, CPU_TENSOR_NHWC(32F, 2, 4, 10, 128), 0);
    2170
    1
      ccv_nnc_tensor_t* bt = ccv_nnc_tensor_new(0, CPU_TENSOR_NHWC(32F, 2, 4, 10, 64), 0);
    2171
    1
      int i;
    2172
    8.19k
      for (i = 0; i < 64 * 128; i++8.19k)
    2173
    8.19k
        hw->data.f32[i] = dsfmt_genrand_open_close(&dsfmt) / (64 * 128);
    2174
    65
      for (i = 0; i < 64; i++64)
    2175
    64
        hbias->data.f32[i] = dsfmt_genrand_open_close(&dsfmt) / 64;
    2176
    10.2k
      for (i = 0; i < 8 * 10 * 128; i++10.2k)
    2177
    10.2k
        ha->data.f32[i] = dsfmt_genrand_open_close(&dsfmt);
    2178
    1
      ccv_nnc_cmd_exec(CMD_TRANSPOSE_FORWARD(1, 2), ccv_nnc_no_hint, 0, TENSOR_LIST(ha), TENSOR_LIST(at), 0);
    2179
    1
      ccv_nnc_cmd_exec(CMD_DATA_TRANSFER_FORWARD(), ccv_nnc_no_hint, 0, TENSOR_LIST(ha, hw, hbias), TENSOR_LIST(a, w, bias), 0);
    2180
    1
      ccv_nnc_tensor_view_t* av = ccv_nnc_tensor_view_new(a, GPU_TENSOR_NHWC(000, 32F, 2, 4, 10, 128), ccv_nnc_no_ofs, DIM_ALLOC(10 * 4 * 128, 128, 4 * 128, 1));
    2181
    1
      ccv_nnc_cmd_exec(CMD_GEMM_FORWARD(NO_TRANSPOSE, TRANSPOSE(0, 1)), ccv_nnc_no_hint, 0, TENSOR_LIST((ccv_nnc_tensor_t*)av, w, bias), TENSOR_LIST(b), 0);
    2182
    1
      ccv_nnc_cmd_exec(CMD_GEMM_FORWARD(NO_TRANSPOSE, TRANSPOSE(0, 1)), ccv_nnc_no_hint, 0, TENSOR_LIST(at, hw, hbias), TENSOR_LIST(bt), 0);
    2183
    1
      ccv_nnc_cmd_exec(CMD_DATA_TRANSFER_FORWARD(), ccv_nnc_no_hint, 0, TENSOR_LIST(b), TENSOR_LIST(hb), 0);
    2184
    1
      REQUIRE_TENSOR_EQ(hb, bt, "permute computed output should be the same as non-permute computed ones");
    2185
    1
      ccv_nnc_tensor_free(ha);
    2186
    1
      ccv_nnc_tensor_free(hw);
    2187
    1
      ccv_nnc_tensor_free(hbias);
    2188
    1
      ccv_nnc_tensor_free(hb);
    2189
    1
      ccv_nnc_tensor_free(a);
    2190
    1
      ccv_nnc_tensor_free(w);
    2191
    1
      ccv_nnc_tensor_free(bias);
    2192
    1
      ccv_nnc_tensor_free(b);
    2193
    1
      ccv_nnc_tensor_view_free(av);
    2194
    1
      ccv_nnc_tensor_free(at);
    2195
    1
      ccv_nnc_tensor_free(bt);
    2196
    1
    }
    2197
    2198
    TEST_CASE("generalized batched backward gemm with batch (2, 4) compare cublas")
    2199
    1
    {
    2200
    1
      GUARD_ELSE_RETURN(ccv_nnc_cmd_ok(CCV_NNC_GEMM_BACKWARD, CCV_NNC_BACKEND_GPU_CUBLAS));
    2201
      // This is a particular batched gemm which treat every dimensions other than the last two as batching.
    2202
    1
      dsfmt_t dsfmt;
    2203
    1
      dsfmt_init_gen_rand(&dsfmt, 0);
    2204
    1
      ccv_nnc_tensor_t* ha = ccv_nnc_tensor_new(0, CPU_TENSOR_NHWC(32F, 2, 10, 4, 128), 0);
    2205
    1
      ccv_nnc_tensor_t* hw = ccv_nnc_tensor_new(0, CPU_TENSOR_NHWC(32F, 2, 64, 4, 128), 0);
    2206
    1
      ccv_nnc_tensor_t* hda = ccv_nnc_tensor_new(0, CPU_TENSOR_NHWC(32F, 2, 10, 4, 128), 0);
    2207
    1
      ccv_nnc_tensor_t* hdw = ccv_nnc_tensor_new(0, CPU_TENSOR_NHWC(32F, 2, 64, 4, 128), 0);
    2208
    1
      ccv_nnc_tensor_t* hb = ccv_nnc_tensor_new(0, CPU_TENSOR_NHWC(32F, 2, 4, 10, 64), 0);
    2209
    1
      ccv_nnc_tensor_t* a = ccv_nnc_tensor_new(0, GPU_TENSOR_NHWC(000, 32F, 2, 10, 4, 128), 0);
    2210
    1
      ccv_nnc_tensor_t* w = ccv_nnc_tensor_new(0, GPU_TENSOR_NHWC(000, 32F, 2, 64, 4, 128), 0);
    2211
    1
      ccv_nnc_tensor_t* da = ccv_nnc_tensor_new(0, GPU_TENSOR_NHWC(000, 32F, 2, 10, 4, 128), 0);
    2212
    1
      ccv_nnc_tensor_t* dw = ccv_nnc_tensor_new(0, GPU_TENSOR_NHWC(000, 32F, 2, 64, 4, 128), 0);
    2213
    1
      ccv_nnc_tensor_t* b = ccv_nnc_tensor_new(0, GPU_TENSOR_NHWC(000, 32F, 2, 4, 10, 64), 0);
    2214
    2215
    1
      ccv_nnc_tensor_t* at = ccv_nnc_tensor_new(0, CPU_TENSOR_NHWC(32F, 2, 4, 10, 128), 0);
    2216
    1
      ccv_nnc_tensor_t* wt = ccv_nnc_tensor_new(0, CPU_TENSOR_NHWC(32F, 2, 4, 64, 128), 0);
    2217
    1
      ccv_nnc_tensor_t* dat = ccv_nnc_tensor_new(0, CPU_TENSOR_NHWC(32F, 2, 4, 10, 128), 0);
    2218
    1
      ccv_nnc_tensor_t* dwt = ccv_nnc_tensor_new(0, CPU_TENSOR_NHWC(32F, 2, 4, 64, 128), 0);
    2219
    1
      ccv_nnc_tensor_t* tda = ccv_nnc_tensor_new(0, CPU_TENSOR_NHWC(32F, 2, 10, 4, 128), 0);
    2220
    1
      ccv_nnc_tensor_t* tdw = ccv_nnc_tensor_new(0, CPU_TENSOR_NHWC(32F, 2, 64, 4, 128), 0);
    2221
    1
      int i;
    2222
    65.5k
      for (i = 0; i < 8 * 64 * 128; i++65.5k)
    2223
    65.5k
        hw->data.f32[i] = dsfmt_genrand_open_close(&dsfmt) / (64 * 128);
    2224
    10.2k
      for (i = 0; i < 8 * 10 * 128; i++10.2k)
    2225
    10.2k
        ha->data.f32[i] = dsfmt_genrand_open_close(&dsfmt);
    2226
    5.12k
      for (i = 0; i < 2 * 4 * 10 * 64; i++5.12k)
    2227
    5.12k
        hb->data.f32[i] = dsfmt_genrand_open_close(&dsfmt);
    2228
    1
      ccv_nnc_cmd_exec(CMD_TRANSPOSE_FORWARD(1, 2), ccv_nnc_no_hint, 0, TENSOR_LIST(ha), TENSOR_LIST(at), 0);
    2229
    1
      ccv_nnc_cmd_exec(CMD_TRANSPOSE_FORWARD(1, 2), ccv_nnc_no_hint, 0, TENSOR_LIST(hw), TENSOR_LIST(wt), 0);
    2230
    1
      ccv_nnc_cmd_exec(CMD_DATA_TRANSFER_FORWARD(), ccv_nnc_no_hint, 0, TENSOR_LIST(ha, hw, hb), TENSOR_LIST(a, w, b), 0);
    2231
    1
      ccv_nnc_tensor_view_t* av = ccv_nnc_tensor_view_new(a, GPU_TENSOR_NHWC(000, 32F, 2, 4, 10, 128), ccv_nnc_no_ofs, DIM_ALLOC(10 * 4 * 128, 128, 4 * 128, 1));
    2232
    1
      ccv_nnc_tensor_view_t* wv = ccv_nnc_tensor_view_new(w, GPU_TENSOR_NHWC(000, 32F, 2, 4, 64, 128), ccv_nnc_no_ofs, DIM_ALLOC(64 * 4 * 128, 128, 4 * 128, 1));
    2233
    1
      ccv_nnc_tensor_view_t* dav = ccv_nnc_tensor_view_new(da, GPU_TENSOR_NHWC(000, 32F, 2, 4, 10, 128), ccv_nnc_no_ofs, DIM_ALLOC(10 * 4 * 128, 128, 4 * 128, 1));
    2234
    1
      ccv_nnc_tensor_view_t* dwv = ccv_nnc_tensor_view_new(dw, GPU_TENSOR_NHWC(000, 32F, 2, 4, 64, 128), ccv_nnc_no_ofs, DIM_ALLOC(64 * 4 * 128, 128, 4 * 128, 1));
    2235
    1
      ccv_nnc_cmd_exec(CMD_GEMM_BACKWARD(NO_TRANSPOSE, TRANSPOSE(2, 3)), ccv_nnc_no_hint, 0, TENSOR_LIST(b, (ccv_nnc_tensor_t*)av, (ccv_nnc_tensor_t*)wv), TENSOR_LIST((ccv_nnc_tensor_t*)dav, (ccv_nnc_tensor_t*)dwv), 0);
    2236
    1
      ccv_nnc_cmd_exec(CMD_GEMM_BACKWARD(NO_TRANSPOSE, TRANSPOSE(2, 3)), ccv_nnc_no_hint, 0, TENSOR_LIST(hb, at, wt), TENSOR_LIST(dat, dwt), 0);
    2237
    1
      ccv_nnc_cmd_exec(CMD_TRANSPOSE_FORWARD(1, 2), ccv_nnc_no_hint, 0, TENSOR_LIST(dat), TENSOR_LIST(tda), 0);
    2238
    1
      ccv_nnc_cmd_exec(CMD_TRANSPOSE_FORWARD(1, 2), ccv_nnc_no_hint, 0, TENSOR_LIST(dwt), TENSOR_LIST(tdw), 0);
    2239
    1
      ccv_nnc_cmd_exec(CMD_DATA_TRANSFER_FORWARD(), ccv_nnc_no_hint, 0, TENSOR_LIST(da, dw), TENSOR_LIST(hda, hdw), 0);
    2240
    1
      REQUIRE_TENSOR_EQ(hda, tda, "permute computed output should be the same as non-permute computed ones");
    2241
    1
      REQUIRE_TENSOR_EQ(hdw, tdw, "permute computed output should be the same as non-permute computed ones");
    2242
    1
      ccv_nnc_tensor_free(ha);
    2243
    1
      ccv_nnc_tensor_free(hw);
    2244
    1
      ccv_nnc_tensor_free(hda);
    2245
    1
      ccv_nnc_tensor_free(hdw);
    2246
    1
      ccv_nnc_tensor_free(hb);
    2247
    1
      ccv_nnc_tensor_free(a);
    2248
    1
      ccv_nnc_tensor_free(w);
    2249
    1
      ccv_nnc_tensor_free(da);
    2250
    1
      ccv_nnc_tensor_free(dw);
    2251
    1
      ccv_nnc_tensor_free(b);
    2252
    1
      ccv_nnc_tensor_view_free(av);
    2253
    1
      ccv_nnc_tensor_view_free(wv);
    2254
    1
      ccv_nnc_tensor_view_free(dav);
    2255
    1
      ccv_nnc_tensor_view_free(dwv);
    2256
    1
      ccv_nnc_tensor_free(at);
    2257
    1
      ccv_nnc_tensor_free(wt);
    2258
    1
      ccv_nnc_tensor_free(dat);
    2259
    1
      ccv_nnc_tensor_free(tda);
    2260
    1
      ccv_nnc_tensor_free(dwt);
    2261
    1
      ccv_nnc_tensor_free(tdw);
    2262
    1
    }
    2263
    2264
    TEST_CASE("generalized batched backward gemm with batch (2, 4) and broadcast compare cublas")
    2265
    1
    {
    2266
    1
      GUARD_ELSE_RETURN(ccv_nnc_cmd_ok(CCV_NNC_GEMM_BACKWARD, CCV_NNC_BACKEND_GPU_CUBLAS));
    2267
      // This is a particular batched gemm which treat every dimensions other than the last two as batching.
    2268
    1
      dsfmt_t dsfmt;
    2269
    1
      dsfmt_init_gen_rand(&dsfmt, 0);
    2270
    1
      ccv_nnc_tensor_t* ha = ccv_nnc_tensor_new(0, CPU_TENSOR_NHWC(32F, 2, 10, 4, 128), 0);
    2271
    1
      ccv_nnc_tensor_t* hw = ccv_nnc_tensor_new(0, CPU_TENSOR_NHWC(32F, 64, 128), 0);
    2272
    1
      ccv_nnc_tensor_t* hda = ccv_nnc_tensor_new(0, CPU_TENSOR_NHWC(32F, 2, 10, 4, 128), 0);
    2273
    1
      ccv_nnc_tensor_t* hdw = ccv_nnc_tensor_new(0, CPU_TENSOR_NHWC(32F, 64, 128), 0);
    2274
    1
      ccv_nnc_tensor_t* hb = ccv_nnc_tensor_new(0, CPU_TENSOR_NHWC(32F, 2, 4, 10, 64), 0);
    2275
    1
      ccv_nnc_tensor_t* a = ccv_nnc_tensor_new(0, GPU_TENSOR_NHWC(000, 32F, 2, 10, 4, 128), 0);
    2276
    1
      ccv_nnc_tensor_t* w = ccv_nnc_tensor_new(0, GPU_TENSOR_NHWC(000, 32F, 64, 128), 0);
    2277
    1
      ccv_nnc_tensor_t* da = ccv_nnc_tensor_new(0, GPU_TENSOR_NHWC(000, 32F, 2, 10, 4, 128), 0);
    2278
    1
      ccv_nnc_tensor_t* dw = ccv_nnc_tensor_new(0, GPU_TENSOR_NHWC(000, 32F, 64, 128), 0);
    2279
    1
      ccv_nnc_tensor_t* b = ccv_nnc_tensor_new(0, GPU_TENSOR_NHWC(000, 32F, 2, 4, 10, 64), 0);
    2280
    2281
    1
      ccv_nnc_tensor_t* at = ccv_nnc_tensor_new(0, CPU_TENSOR_NHWC(32F, 2, 4, 10, 128), 0);
    2282
    1
      ccv_nnc_tensor_t* dat = ccv_nnc_tensor_new(0, CPU_TENSOR_NHWC(32F, 2, 4, 10, 128), 0);
    2283
    1
      ccv_nnc_tensor_t* tda = ccv_nnc_tensor_new(0, CPU_TENSOR_NHWC(32F, 2, 10, 4, 128), 0);
    2284
    1
      ccv_nnc_tensor_t* tdw = ccv_nnc_tensor_new(0, CPU_TENSOR_NHWC(32F, 64, 128), 0);
    2285
    1
      int i;
    2286
    8.19k
      for (i = 0; i < 64 * 128; i++8.19k)
    2287
    8.19k
        hw->data.f32[i] = dsfmt_genrand_open_close(&dsfmt) / (64 * 128);
    2288
    10.2k
      for (i = 0; i < 8 * 10 * 128; i++10.2k)
    2289
    10.2k
        ha->data.f32[i] = dsfmt_genrand_open_close(&dsfmt);
    2290
    5.12k
      for (i = 0; i < 2 * 4 * 10 * 64; i++5.12k)
    2291
    5.12k
        hb->data.f32[i] = dsfmt_genrand_open_close(&dsfmt);
    2292
    1
      ccv_nnc_cmd_exec(CMD_TRANSPOSE_FORWARD(1, 2), ccv_nnc_no_hint, 0, TENSOR_LIST(ha), TENSOR_LIST(at), 0);
    2293
    1
      ccv_nnc_cmd_exec(CMD_DATA_TRANSFER_FORWARD(), ccv_nnc_no_hint, 0, TENSOR_LIST(ha, hw, hb), TENSOR_LIST(a, w, b), 0);
    2294
    1
      ccv_nnc_tensor_view_t* av = ccv_nnc_tensor_view_new(a, GPU_TENSOR_NHWC(000, 32F, 2, 4, 10, 128), ccv_nnc_no_ofs, DIM_ALLOC(10 * 4 * 128, 128, 4 * 128, 1));
    2295
    1
      ccv_nnc_tensor_view_t* dav = ccv_nnc_tensor_view_new(da, GPU_TENSOR_NHWC(000, 32F, 2, 4, 10, 128), ccv_nnc_no_ofs, DIM_ALLOC(10 * 4 * 128, 128, 4 * 128, 1));
    2296
    1
      ccv_nnc_cmd_exec(CMD_GEMM_BACKWARD(NO_TRANSPOSE, TRANSPOSE(0, 1)), ccv_nnc_no_hint, 0, TENSOR_LIST(b, (ccv_nnc_tensor_t*)av, w), TENSOR_LIST((ccv_nnc_tensor_t*)dav, dw), 0);
    2297
    1
      ccv_nnc_cmd_exec(CMD_GEMM_BACKWARD(NO_TRANSPOSE, TRANSPOSE(0, 1)), ccv_nnc_no_hint, 0, TENSOR_LIST(hb, at, hw), TENSOR_LIST(dat, tdw), 0);
    2298
    1
      ccv_nnc_cmd_exec(CMD_TRANSPOSE_FORWARD(1, 2), ccv_nnc_no_hint, 0, TENSOR_LIST(dat), TENSOR_LIST(tda), 0);
    2299
    1
      ccv_nnc_cmd_exec(CMD_DATA_TRANSFER_FORWARD(), ccv_nnc_no_hint, 0, TENSOR_LIST(da, dw), TENSOR_LIST(hda, hdw), 0);
    2300
    1
      REQUIRE_TENSOR_EQ(hda, tda, "permute computed output should be the same as non-permute computed ones");
    2301
    1
      REQUIRE_TENSOR_EQ(hdw, tdw, "permute computed output should be the same as non-permute computed ones");
    2302
    1
      ccv_nnc_tensor_free(ha);
    2303
    1
      ccv_nnc_tensor_free(hw);
    2304
    1
      ccv_nnc_tensor_free(hda);
    2305
    1
      ccv_nnc_tensor_free(hdw);
    2306
    1
      ccv_nnc_tensor_free(hb);
    2307
    1
      ccv_nnc_tensor_free(a);
    2308
    1
      ccv_nnc_tensor_free(w);
    2309
    1
      ccv_nnc_tensor_free(da);
    2310
    1
      ccv_nnc_tensor_free(dw);
    2311
    1
      ccv_nnc_tensor_free(b);
    2312
    1
      ccv_nnc_tensor_view_free(av);
    2313
    1
      ccv_nnc_tensor_view_free(dav);
    2314
    1
      ccv_nnc_tensor_free(at);
    2315
    1
      ccv_nnc_tensor_free(dat);
    2316
    1
      ccv_nnc_tensor_free(tda);
    2317
    1
      ccv_nnc_tensor_free(tdw);
    2318
    1
    }
    2319
    2320
    TEST_CASE("generalized batched backward gemm with batch (2, 4) with bias compare cublas")
    2321
    1
    {
    2322
    1
      GUARD_ELSE_RETURN(ccv_nnc_cmd_ok(CCV_NNC_GEMM_BACKWARD, CCV_NNC_BACKEND_GPU_CUBLAS));
    2323
      // This is a particular batched gemm which treat every dimensions other than the last two as batching.
    2324
    1
      dsfmt_t dsfmt;
    2325
    1
      dsfmt_init_gen_rand(&dsfmt, 0);
    2326
    1
      ccv_nnc_tensor_t* ha = ccv_nnc_tensor_new(0, CPU_TENSOR_NHWC(32F, 2, 10, 4, 128), 0);
    2327
    1
      ccv_nnc_tensor_t* hw = ccv_nnc_tensor_new(0, CPU_TENSOR_NHWC(32F, 2, 64, 4, 128), 0);
    2328
    1
      ccv_nnc_tensor_t* hda = ccv_nnc_tensor_new(0, CPU_TENSOR_NHWC(32F, 2, 10, 4, 128), 0);
    2329
    1
      ccv_nnc_tensor_t* hdw = ccv_nnc_tensor_new(0, CPU_TENSOR_NHWC(32F, 2, 64, 4, 128), 0);
    2330
    1
      ccv_nnc_tensor_t* hdbias = ccv_nnc_tensor_new(0, CPU_TENSOR_NHWC(32F, 64), 0);
    2331
    1
      ccv_nnc_tensor_t* hb = ccv_nnc_tensor_new(0, CPU_TENSOR_NHWC(32F, 2, 4, 10, 64), 0);
    2332
    1
      ccv_nnc_tensor_t* a = ccv_nnc_tensor_new(0, GPU_TENSOR_NHWC(000, 32F, 2, 10, 4, 128), 0);
    2333
    1
      ccv_nnc_tensor_t* w = ccv_nnc_tensor_new(0, GPU_TENSOR_NHWC(000, 32F, 2, 64, 4, 128), 0);
    2334
    1
      ccv_nnc_tensor_t* da = ccv_nnc_tensor_new(0, GPU_TENSOR_NHWC(000, 32F, 2, 10, 4, 128), 0);
    2335
    1
      ccv_nnc_tensor_t* dw = ccv_nnc_tensor_new(0, GPU_TENSOR_NHWC(000, 32F, 2, 64, 4, 128), 0);
    2336
    1
      ccv_nnc_tensor_t* dbias = ccv_nnc_tensor_new(0, GPU_TENSOR_NHWC(000, 32F, 64), 0);
    2337
    1
      ccv_nnc_tensor_t* b = ccv_nnc_tensor_new(0, GPU_TENSOR_NHWC(000, 32F, 2, 4, 10, 64), 0);
    2338
    2339
    1
      ccv_nnc_tensor_t* at = ccv_nnc_tensor_new(0, CPU_TENSOR_NHWC(32F, 2, 4, 10, 128), 0);
    2340
    1
      ccv_nnc_tensor_t* wt = ccv_nnc_tensor_new(0, CPU_TENSOR_NHWC(32F, 2, 4, 64, 128), 0);
    2341
    1
      ccv_nnc_tensor_t* dat = ccv_nnc_tensor_new(0, CPU_TENSOR_NHWC(32F, 2, 4, 10, 128), 0);
    2342
    1
      ccv_nnc_tensor_t* dwt = ccv_nnc_tensor_new(0, CPU_TENSOR_NHWC(32F, 2, 4, 64, 128), 0);
    2343
    1
      ccv_nnc_tensor_t* tda = ccv_nnc_tensor_new(0, CPU_TENSOR_NHWC(32F, 2, 10, 4, 128), 0);
    2344
    1
      ccv_nnc_tensor_t* tdw = ccv_nnc_tensor_new(0, CPU_TENSOR_NHWC(32F, 2, 64, 4, 128), 0);
    2345
    1
      ccv_nnc_tensor_t* tdbias = ccv_nnc_tensor_new(0, CPU_TENSOR_NHWC(32F, 64), 0);
    2346
    1
      int i;
    2347
    65.5k
      for (i = 0; i < 8 * 64 * 128; i++65.5k)
    2348
    65.5k
        hw->data.f32[i] = dsfmt_genrand_open_close(&dsfmt) / (64 * 128);
    2349
    10.2k
      for (i = 0; i < 8 * 10 * 128; i++10.2k)
    2350
    10.2k
        ha->data.f32[i] = dsfmt_genrand_open_close(&dsfmt);
    2351
    5.12k
      for (i = 0; i < 2 * 4 * 10 * 64; i++5.12k)
    2352
    5.12k
        hb->data.f32[i] = dsfmt_genrand_open_close(&dsfmt);
    2353
    1
      ccv_nnc_cmd_exec(CMD_TRANSPOSE_FORWARD(1, 2), ccv_nnc_no_hint, 0, TENSOR_LIST(ha), TENSOR_LIST(at), 0);
    2354
    1
      ccv_nnc_cmd_exec(CMD_TRANSPOSE_FORWARD(1, 2), ccv_nnc_no_hint, 0, TENSOR_LIST(hw), TENSOR_LIST(wt), 0);
    2355
    1
      ccv_nnc_cmd_exec(CMD_DATA_TRANSFER_FORWARD(), ccv_nnc_no_hint, 0, TENSOR_LIST(ha, hw, hb), TENSOR_LIST(a, w, b), 0);
    2356
    1
      ccv_nnc_tensor_view_t* av = ccv_nnc_tensor_view_new(a, GPU_TENSOR_NHWC(000, 32F, 2, 4, 10, 128), ccv_nnc_no_ofs, DIM_ALLOC(10 * 4 * 128, 128, 4 * 128, 1));
    2357
    1
      ccv_nnc_tensor_view_t* wv = ccv_nnc_tensor_view_new(w, GPU_TENSOR_NHWC(000, 32F, 2, 4, 64, 128), ccv_nnc_no_ofs, DIM_ALLOC(64 * 4 * 128, 128, 4 * 128, 1));
    2358
    1
      ccv_nnc_tensor_view_t* dav = ccv_nnc_tensor_view_new(da, GPU_TENSOR_NHWC(000, 32F, 2, 4, 10, 128), ccv_nnc_no_ofs, DIM_ALLOC(10 * 4 * 128, 128, 4 * 128, 1));
    2359
    1
      ccv_nnc_tensor_view_t* dwv = ccv_nnc_tensor_view_new(dw, GPU_TENSOR_NHWC(000, 32F, 2, 4, 64, 128), ccv_nnc_no_ofs, DIM_ALLOC(64 * 4 * 128, 128, 4 * 128, 1));
    2360
    1
      ccv_nnc_cmd_exec(CMD_GEMM_BACKWARD(NO_TRANSPOSE, TRANSPOSE(2, 3)), ccv_nnc_no_hint, 0, TENSOR_LIST(b, (ccv_nnc_tensor_t*)av, (ccv_nnc_tensor_t*)wv), TENSOR_LIST((ccv_nnc_tensor_t*)dav, (ccv_nnc_tensor_t*)dwv, dbias), 0);
    2361
    1
      ccv_nnc_cmd_exec(CMD_GEMM_BACKWARD(NO_TRANSPOSE, TRANSPOSE(2, 3)), ccv_nnc_no_hint, 0, TENSOR_LIST(hb, at, wt), TENSOR_LIST(dat, dwt, tdbias), 0);
    2362
    1
      ccv_nnc_cmd_exec(CMD_DATA_TRANSFER_FORWARD(), ccv_nnc_no_hint, 0, TENSOR_LIST(da, dw, dbias), TENSOR_LIST(hda, hdw, hdbias), 0);
    2363
    1
      ccv_nnc_cmd_exec(CMD_TRANSPOSE_FORWARD(1, 2), ccv_nnc_no_hint, 0, TENSOR_LIST(dat), TENSOR_LIST(tda), 0);
    2364
    1
      ccv_nnc_cmd_exec(CMD_TRANSPOSE_FORWARD(1, 2), ccv_nnc_no_hint, 0, TENSOR_LIST(dwt), TENSOR_LIST(tdw), 0);
    2365
    1
      REQUIRE_TENSOR_EQ(hda, tda, "permute computed output should be the same as non-permute computed ones");
    2366
    1
      REQUIRE_TENSOR_EQ(hdw, tdw, "permute computed output should be the same as non-permute computed ones");
    2367
    1
      REQUIRE_TENSOR_EQ(hdbias, tdbias, "permute computed output should be the same as non-permute computed ones");
    2368
    1
      ccv_nnc_tensor_free(ha);
    2369
    1
      ccv_nnc_tensor_free(hw);
    2370
    1
      ccv_nnc_tensor_free(hda);
    2371
    1
      ccv_nnc_tensor_free(hdw);
    2372
    1
      ccv_nnc_tensor_free(hdbias);
    2373
    1
      ccv_nnc_tensor_free(hb);
    2374
    1
      ccv_nnc_tensor_free(a);
    2375
    1
      ccv_nnc_tensor_free(w);
    2376
    1
      ccv_nnc_tensor_free(da);
    2377
    1
      ccv_nnc_tensor_free(dw);
    2378
    1
      ccv_nnc_tensor_free(dbias);
    2379
    1
      ccv_nnc_tensor_free(b);
    2380
    1
      ccv_nnc_tensor_view_free(av);
    2381
    1
      ccv_nnc_tensor_view_free(wv);
    2382
    1
      ccv_nnc_tensor_view_free(dav);
    2383
    1
      ccv_nnc_tensor_view_free(dwv);
    2384
    1
      ccv_nnc_tensor_free(at);
    2385
    1
      ccv_nnc_tensor_free(wt);
    2386
    1
      ccv_nnc_tensor_free(dat);
    2387
    1
      ccv_nnc_tensor_free(dwt);
    2388
    1
      ccv_nnc_tensor_free(tda);
    2389
    1
      ccv_nnc_tensor_free(tdw);
    2390
    1
      ccv_nnc_tensor_free(tdbias);
    2391
    1
    }
    2392
    2393
    TEST_CASE("generalized batched backward gemm with batch (2, 4) with bias and broadcast compare cublas")
    2394
    1
    {
    2395
    1
      GUARD_ELSE_RETURN(ccv_nnc_cmd_ok(CCV_NNC_GEMM_BACKWARD, CCV_NNC_BACKEND_GPU_CUBLAS));
    2396
      // This is a particular batched gemm which treat every dimensions other than the last two as batching.
    2397
    1
      dsfmt_t dsfmt;
    2398
    1
      dsfmt_init_gen_rand(&dsfmt, 0);
    2399
    1
      ccv_nnc_tensor_t* ha = ccv_nnc_tensor_new(0, CPU_TENSOR_NHWC(32F, 2, 10, 4, 128), 0);
    2400
    1
      ccv_nnc_tensor_t* hw = ccv_nnc_tensor_new(0, CPU_TENSOR_NHWC(32F, 64, 128), 0);
    2401
    1
      ccv_nnc_tensor_t* hda = ccv_nnc_tensor_new(0, CPU_TENSOR_NHWC(32F, 2, 10, 4, 128), 0);
    2402
    1
      ccv_nnc_tensor_t* hdw = ccv_nnc_tensor_new(0, CPU_TENSOR_NHWC(32F, 64, 128), 0);
    2403
    1
      ccv_nnc_tensor_t* hdbias = ccv_nnc_tensor_new(0, CPU_TENSOR_NHWC(32F, 64), 0);
    2404
    1
      ccv_nnc_tensor_t* hb = ccv_nnc_tensor_new(0, CPU_TENSOR_NHWC(32F, 2, 4, 10, 64), 0);
    2405
    1
      ccv_nnc_tensor_t* a = ccv_nnc_tensor_new(0, GPU_TENSOR_NHWC(000, 32F, 2, 10, 4, 128), 0);
    2406
    1
      ccv_nnc_tensor_t* w = ccv_nnc_tensor_new(0, GPU_TENSOR_NHWC(000, 32F, 64, 128), 0);
    2407
    1
      ccv_nnc_tensor_t* da = ccv_nnc_tensor_new(0, GPU_TENSOR_NHWC(000, 32F, 2, 10, 4, 128), 0);
    2408
    1
      ccv_nnc_tensor_t* dw = ccv_nnc_tensor_new(0, GPU_TENSOR_NHWC(000, 32F, 64, 128), 0);
    2409
    1
      ccv_nnc_tensor_t* dbias = ccv_nnc_tensor_new(0, GPU_TENSOR_NHWC(000, 32F, 64), 0);
    2410
    1
      ccv_nnc_tensor_t* b = ccv_nnc_tensor_new(0, GPU_TENSOR_NHWC(000, 32F, 2, 4, 10, 64), 0);
    2411
    2412
    1
      ccv_nnc_tensor_t* at = ccv_nnc_tensor_new(0, CPU_TENSOR_NHWC(32F, 2, 4, 10, 128), 0);
    2413
    1
      ccv_nnc_tensor_t* dat = ccv_nnc_tensor_new(0, CPU_TENSOR_NHWC(32F, 2, 4, 10, 128), 0);
    2414
    1
      ccv_nnc_tensor_t* tda = ccv_nnc_tensor_new(0, CPU_TENSOR_NHWC(32F, 2, 10, 4, 128), 0);
    2415
    1
      ccv_nnc_tensor_t* tdw = ccv_nnc_tensor_new(0, CPU_TENSOR_NHWC(32F, 64, 128), 0);
    2416
    1
      ccv_nnc_tensor_t* tdbias = ccv_nnc_tensor_new(0, CPU_TENSOR_NHWC(32F, 64), 0);
    2417
    1
      int i;
    2418
    8.19k
      for (i = 0; i < 64 * 128; i++8.19k)
    2419
    8.19k
        hw->data.f32[i] = dsfmt_genrand_open_close(&dsfmt) / (64 * 128);
    2420
    10.2k
      for (i = 0; i < 8 * 10 * 128; i++10.2k)
    2421
    10.2k
        ha->data.f32[i] = dsfmt_genrand_open_close(&dsfmt);
    2422
    5.12k
      for (i = 0; i < 2 * 4 * 10 * 64; i++5.12k)
    2423
    5.12k
        hb->data.f32[i] = dsfmt_genrand_open_close(&dsfmt);
    2424
    1
      ccv_nnc_cmd_exec(CMD_TRANSPOSE_FORWARD(1, 2), ccv_nnc_no_hint, 0, TENSOR_LIST(ha), TENSOR_LIST(at), 0);
    2425
    1
      ccv_nnc_cmd_exec(CMD_DATA_TRANSFER_FORWARD(), ccv_nnc_no_hint, 0, TENSOR_LIST(ha, hw, hb), TENSOR_LIST(a, w, b), 0);
    2426
    1
      ccv_nnc_tensor_view_t* av = ccv_nnc_tensor_view_new(a, GPU_TENSOR_NHWC(000, 32F, 2, 4, 10, 128), ccv_nnc_no_ofs, DIM_ALLOC(10 * 4 * 128, 128, 4 * 128, 1));
    2427
    1
      ccv_nnc_tensor_view_t* dav = ccv_nnc_tensor_view_new(da, GPU_TENSOR_NHWC(000, 32F, 2, 4, 10, 128), ccv_nnc_no_ofs, DIM_ALLOC(10 * 4 * 128, 128, 4 * 128, 1));
    2428
    1
      ccv_nnc_cmd_exec(CMD_GEMM_BACKWARD(NO_TRANSPOSE, TRANSPOSE(0, 1)), ccv_nnc_no_hint, 0, TENSOR_LIST(b, (ccv_nnc_tensor_t*)av, w, dbias), TENSOR_LIST((ccv_nnc_tensor_t*)dav, dw, dbias), 0);
    2429
    1
      ccv_nnc_cmd_exec(CMD_GEMM_BACKWARD(NO_TRANSPOSE, TRANSPOSE(0, 1)), ccv_nnc_no_hint, 0, TENSOR_LIST(hb, at, hw, hdbias), TENSOR_LIST(dat, tdw, tdbias), 0);
    2430
    1
      ccv_nnc_cmd_exec(CMD_DATA_TRANSFER_FORWARD(), ccv_nnc_no_hint, 0, TENSOR_LIST(da, dw, dbias), TENSOR_LIST(hda, hdw, hdbias), 0);
    2431
    1
      ccv_nnc_cmd_exec(CMD_TRANSPOSE_FORWARD(1, 2), ccv_nnc_no_hint, 0, TENSOR_LIST(dat), TENSOR_LIST(tda), 0);
    2432
    1
      REQUIRE_TENSOR_EQ(hda, tda, "permute computed output should be the same as non-permute computed ones");
    2433
    1
      REQUIRE_TENSOR_EQ(hdw, tdw, "permute computed output should be the same as non-permute computed ones");
    2434
    1
      REQUIRE_TENSOR_EQ(hdbias, tdbias, "permute computed output should be the same as non-permute computed ones");
    2435
    1
      ccv_nnc_tensor_free(ha);
    2436
    1
      ccv_nnc_tensor_free(hw);
    2437
    1
      ccv_nnc_tensor_free(hda);
    2438
    1
      ccv_nnc_tensor_free(hdw);
    2439
    1
      ccv_nnc_tensor_free(hdbias);
    2440
    1
      ccv_nnc_tensor_free(hb);
    2441
    1
      ccv_nnc_tensor_free(a);
    2442
    1
      ccv_nnc_tensor_free(w);
    2443
    1
      ccv_nnc_tensor_free(da);
    2444
    1
      ccv_nnc_tensor_free(dw);
    2445
    1
      ccv_nnc_tensor_free(dbias);
    2446
    1
      ccv_nnc_tensor_free(b);
    2447
    1
      ccv_nnc_tensor_view_free(av);
    2448
    1
      ccv_nnc_tensor_view_free(dav);
    2449
    1
      ccv_nnc_tensor_free(at);
    2450
    1
      ccv_nnc_tensor_free(dat);
    2451
    1
      ccv_nnc_tensor_free(tdw);
    2452
    1
      ccv_nnc_tensor_free(tdbias);
    2453
    1
    }
    2454
    2455
    TEST_CASE("ewdiv forward with reciprocal")
    2456
    1
    {
    2457
    1
      GUARD_ELSE_RETURN(ccv_nnc_cmd_ok(CCV_NNC_EWDIV_FORWARD, CCV_NNC_BACKEND_GPU_REF));
    2458
    1
      ccv_nnc_tensor_t* a = ccv_nnc_tensor_new(0, GPU_TENSOR_NCHW(000, 32F, 10, 100), 0);
    2459
    1
      ccv_nnc_tensor_t* b = ccv_nnc_tensor_new(0, GPU_TENSOR_NCHW(000, 32F, 10, 100), 0);
    2460
    1
      ccv_nnc_tensor_t* ha = ccv_nnc_tensor_new(0, CPU_TENSOR_NCHW(32F, 10, 100), 0);
    2461
    1
      ccv_nnc_tensor_t* hb = ccv_nnc_tensor_new(0, CPU_TENSOR_NCHW(32F, 10, 100), 0);
    2462
    1
      ccv_nnc_tensor_t* bt = ccv_nnc_tensor_new(0, CPU_TENSOR_NCHW(32F, 10, 100), 0);
    2463
    1
      dsfmt_t dsfmt;
    2464
    1
      dsfmt_init_gen_rand(&dsfmt, 0);
    2465
    1
      int i;
    2466
    1.00k
      for (i = 0; i < 1000; i++1.00k)
    2467
    1.00k
        ha->data.f32[i] = dsfmt_genrand_open_close(&dsfmt) * 0.01;
    2468
    1
      ccv_nnc_cmd_exec(CMD_DATA_TRANSFER_FORWARD(), ccv_nnc_no_hint, 0, TENSOR_LIST(ha), TENSOR_LIST(a), 0);
    2469
    1
      ccv_nnc_cmd_exec(CMD_EWDIV_FORWARD(), ccv_nnc_no_hint, 0, TENSOR_LIST(0, a), TENSOR_LIST(b), 0);
    2470
    1
      ccv_nnc_cmd_exec(CMD_EWDIV_FORWARD(), ccv_nnc_no_hint, 0, TENSOR_LIST(0, ha), TENSOR_LIST(bt), 0);
    2471
    1
      ccv_nnc_cmd_exec(CMD_DATA_TRANSFER_FORWARD(), ccv_nnc_no_hint, 0, TENSOR_LIST(b), TENSOR_LIST(hb), 0);
    2472
    1
      REQUIRE_TENSOR_EQ(bt, hb, "GPU computed output should be the same as CPU computed ones");
    2473
    1
      ccv_nnc_tensor_free(a);
    2474
    1
      ccv_nnc_tensor_free(b);
    2475
    1
      ccv_nnc_tensor_free(ha);
    2476
    1
      ccv_nnc_tensor_free(hb);
    2477
    1
      ccv_nnc_tensor_free(bt);
    2478
    1
    }
    2479
    2480
    TEST_CASE("ewdiv forward")
    2481
    1
    {
    2482
    1
      GUARD_ELSE_RETURN(ccv_nnc_cmd_ok(CCV_NNC_EWDIV_FORWARD, CCV_NNC_BACKEND_GPU_REF));
    2483
    1
      ccv_nnc_tensor_t* a = ccv_nnc_tensor_new(0, GPU_TENSOR_NCHW(000, 32F, 10, 100), 0);
    2484
    1
      ccv_nnc_tensor_t* b = ccv_nnc_tensor_new(0, GPU_TENSOR_NCHW(000, 32F, 10, 100), 0);
    2485
    1
      ccv_nnc_tensor_t* c = ccv_nnc_tensor_new(0, GPU_TENSOR_NCHW(000, 32F, 10, 100), 0);
    2486
    1
      ccv_nnc_tensor_t* ha = ccv_nnc_tensor_new(0, CPU_TENSOR_NCHW(32F, 10, 100), 0);
    2487
    1
      ccv_nnc_tensor_t* hb = ccv_nnc_tensor_new(0, CPU_TENSOR_NCHW(32F, 10, 100), 0);
    2488
    1
      ccv_nnc_tensor_t* hc = ccv_nnc_tensor_new(0, CPU_TENSOR_NCHW(32F, 10, 100), 0);
    2489
    1
      ccv_nnc_tensor_t* ct = ccv_nnc_tensor_new(0, CPU_TENSOR_NCHW(32F, 10, 100), 0);
    2490
    1
      dsfmt_t dsfmt;
    2491
    1
      dsfmt_init_gen_rand(&dsfmt, 0);
    2492
    1
      int i;
    2493
    1.00k
      for (i = 0; i < 1000; i++1.00k)
    2494
    1.00k
        ha->data.f32[i] = dsfmt_genrand_open_close(&dsfmt) * 0.01;
    2495
    1.00k
      for (i = 0; i < 1000; i++1.00k)
    2496
    1.00k
        hb->data.f32[i] = dsfmt_genrand_open_close(&dsfmt) * 0.01;
    2497
    1
      ccv_nnc_cmd_exec(CMD_DATA_TRANSFER_FORWARD(), ccv_nnc_no_hint, 0, TENSOR_LIST(ha, hb), TENSOR_LIST(a, b), 0);
    2498
    1
      ccv_nnc_cmd_exec(CMD_EWDIV_FORWARD(), ccv_nnc_no_hint, 0, TENSOR_LIST(a, b), TENSOR_LIST(c), 0);
    2499
    1
      ccv_nnc_cmd_exec(CMD_EWDIV_FORWARD(), ccv_nnc_no_hint, 0, TENSOR_LIST(ha, hb), TENSOR_LIST(ct), 0);
    2500
    1
      ccv_nnc_cmd_exec(CMD_DATA_TRANSFER_FORWARD(), ccv_nnc_no_hint, 0, TENSOR_LIST(c), TENSOR_LIST(hc), 0);
    2501
    1
      REQUIRE_TENSOR_EQ(ct, hc, "GPU computed output should be the same as CPU computed ones");
    2502
    1
      ccv_nnc_tensor_free(a);
    2503
    1
      ccv_nnc_tensor_free(b);
    2504
    1
      ccv_nnc_tensor_free(c);
    2505
    1
      ccv_nnc_tensor_free(ha);
    2506
    1
      ccv_nnc_tensor_free(hb);
    2507
    1
      ccv_nnc_tensor_free(hc);
    2508
    1
      ccv_nnc_tensor_free(ct);
    2509
    1
    }
    2510
    2511
    TEST_CASE("ewdiv backward with output 1")
    2512
    1
    {
    2513
    1
      GUARD_ELSE_RETURN(ccv_nnc_cmd_ok(CCV_NNC_EWDIV_FORWARD, CCV_NNC_BACKEND_GPU_REF) &&
    2514
    1
        ccv_nnc_cmd_ok(CCV_NNC_EWDIV_BACKWARD, CCV_NNC_BACKEND_GPU_REF));
    2515
    1
      ccv_nnc_tensor_t* a = ccv_nnc_tensor_new(0, GPU_TENSOR_NCHW(000, 32F, 10, 100), 0);
    2516
    1
      ccv_nnc_tensor_t* b = ccv_nnc_tensor_new(0, GPU_TENSOR_NCHW(000, 32F, 10, 100), 0);
    2517
    1
      ccv_nnc_tensor_t* c = ccv_nnc_tensor_new(0, GPU_TENSOR_NCHW(000, 32F, 10, 100), 0);
    2518
    1
      ccv_nnc_tensor_t* g = ccv_nnc_tensor_new(0, GPU_TENSOR_NCHW(000, 32F, 10, 100), 0);
    2519
    1
      ccv_nnc_tensor_t* da = ccv_nnc_tensor_new(0, GPU_TENSOR_NCHW(000, 32F, 10, 100), 0);
    2520
    1
      ccv_nnc_tensor_t* ha = ccv_nnc_tensor_new(0, CPU_TENSOR_NCHW(32F, 10, 100), 0);
    2521
    1
      ccv_nnc_tensor_t* hda = ccv_nnc_tensor_new(0, CPU_TENSOR_NCHW(32F, 10, 100), 0);
    2522
    1
      ccv_nnc_tensor_t* hb = ccv_nnc_tensor_new(0, CPU_TENSOR_NCHW(32F, 10, 100), 0);
    2523
    1
      ccv_nnc_tensor_t* hc = ccv_nnc_tensor_new(0, CPU_TENSOR_NCHW(32F, 10, 100), 0);
    2524
    1
      ccv_nnc_tensor_t* hg = ccv_nnc_tensor_new(0, CPU_TENSOR_NCHW(32F, 10, 100), 0);
    2525
    1
      ccv_nnc_tensor_t* dat = ccv_nnc_tensor_new(0, CPU_TENSOR_NCHW(32F, 10, 100), 0);
    2526
    1
      dsfmt_t dsfmt;
    2527
    1
      dsfmt_init_gen_rand(&dsfmt, 0);
    2528
    1
      int i;
    2529
    1.00k
      for (i = 0; i < 1000; i++1.00k)
    2530
    1.00k
        ha->data.f32[i] = dsfmt_genrand_open_close(&dsfmt) * 0.01;
    2531
    1.00k
      for (i = 0; i < 1000; i++1.00k)
    2532
    1.00k
        hb->data.f32[i] = dsfmt_genrand_open_close(&dsfmt) * 0.01;
    2533
    1.00k
      for (i = 0; i < 1000; i++1.00k)
    2534
    1.00k
        hg->data.f32[i] = dsfmt_genrand_open_close(&dsfmt);
    2535
    1
      ccv_nnc_cmd_exec(CMD_DATA_TRANSFER_FORWARD(), ccv_nnc_no_hint, 0, TENSOR_LIST(ha, hb, hg), TENSOR_LIST(a, b, g), 0);
    2536
    1
      ccv_nnc_cmd_exec(CMD_EWDIV_FORWARD(), ccv_nnc_no_hint, 0, TENSOR_LIST(a, b), TENSOR_LIST(c), 0);
    2537
    1
      ccv_nnc_cmd_exec(CMD_EWDIV_BACKWARD(), ccv_nnc_no_hint, 0, TENSOR_LIST(g, 0, b), TENSOR_LIST(da), 0);
    2538
    1
      ccv_nnc_cmd_exec(CMD_EWDIV_FORWARD(), ccv_nnc_no_hint, 0, TENSOR_LIST(ha, hb), TENSOR_LIST(hc), 0);
    2539
    1
      ccv_nnc_cmd_exec(CMD_EWDIV_BACKWARD(), ccv_nnc_no_hint, 0, TENSOR_LIST(hg, 0, hb), TENSOR_LIST(dat), 0);
    2540
    1
      ccv_nnc_cmd_exec(CMD_DATA_TRANSFER_FORWARD(), ccv_nnc_no_hint, 0, TENSOR_LIST(da), TENSOR_LIST(hda), 0);
    2541
    1
      REQUIRE_TENSOR_EQ(dat, hda, "GPU computed output should be the same as CPU computed ones");
    2542
    1
      ccv_nnc_tensor_free(a);
    2543
    1
      ccv_nnc_tensor_free(b);
    2544
    1
      ccv_nnc_tensor_free(c);
    2545
    1
      ccv_nnc_tensor_free(g);
    2546
    1
      ccv_nnc_tensor_free(da);
    2547
    1
      ccv_nnc_tensor_free(ha);
    2548
    1
      ccv_nnc_tensor_free(hb);
    2549
    1
      ccv_nnc_tensor_free(hc);
    2550
    1
      ccv_nnc_tensor_free(hg);
    2551
    1
      ccv_nnc_tensor_free(hda);
    2552
    1
      ccv_nnc_tensor_free(dat);
    2553
    1
    }
    2554
    2555
    TEST_CASE("ewdiv backward with output 2")
    2556
    1
    {
    2557
    1
      GUARD_ELSE_RETURN(ccv_nnc_cmd_ok(CCV_NNC_EWDIV_FORWARD, CCV_NNC_BACKEND_GPU_REF) &&
    2558
    1
        ccv_nnc_cmd_ok(CCV_NNC_EWDIV_BACKWARD, CCV_NNC_BACKEND_GPU_REF));
    2559
    1
      ccv_nnc_tensor_t* a = ccv_nnc_tensor_new(0, GPU_TENSOR_NCHW(000, 32F, 10, 100), 0);
    2560
    1
      ccv_nnc_tensor_t* b = ccv_nnc_tensor_new(0, GPU_TENSOR_NCHW(000, 32F, 10, 100), 0);
    2561
    1
      ccv_nnc_tensor_t* c = ccv_nnc_tensor_new(0, GPU_TENSOR_NCHW(000, 32F, 10, 100), 0);
    2562
    1
      ccv_nnc_tensor_t* g = ccv_nnc_tensor_new(0, GPU_TENSOR_NCHW(000, 32F, 10, 100), 0);
    2563
    1
      ccv_nnc_tensor_t* db = ccv_nnc_tensor_new(0, GPU_TENSOR_NCHW(000, 32F, 10, 100), 0);
    2564
    1
      ccv_nnc_tensor_t* ha = ccv_nnc_tensor_new(0, CPU_TENSOR_NCHW(32F, 10, 100), 0);
    2565
    1
      ccv_nnc_tensor_t* hdb = ccv_nnc_tensor_new(0, CPU_TENSOR_NCHW(32F, 10, 100), 0);
    2566
    1
      ccv_nnc_tensor_t* hb = ccv_nnc_tensor_new(0, CPU_TENSOR_NCHW(32F, 10, 100), 0);
    2567
    1
      ccv_nnc_tensor_t* hc = ccv_nnc_tensor_new(0, CPU_TENSOR_NCHW(32F, 10, 100), 0);
    2568
    1
      ccv_nnc_tensor_t* hg = ccv_nnc_tensor_new(0, CPU_TENSOR_NCHW(32F, 10, 100), 0);
    2569
    1
      ccv_nnc_tensor_t* dbt = ccv_nnc_tensor_new(0, CPU_TENSOR_NCHW(32F, 10, 100), 0);
    2570
    1
      dsfmt_t dsfmt;
    2571
    1
      dsfmt_init_gen_rand(&dsfmt, 0);
    2572
    1
      int i;
    2573
    1.00k
      for (i = 0; i < 1000; i++1.00k)
    2574
    1.00k
        ha->data.f32[i] = dsfmt_genrand_open_close(&dsfmt) * 0.01;
    2575
    1.00k
      for (i = 0; i < 1000; i++1.00k)
    2576
    1.00k
        hb->data.f32[i] = dsfmt_genrand_open_close(&dsfmt) * 0.01;
    2577
    1.00k
      for (i = 0; i < 1000; i++1.00k)
    2578
    1.00k
        hg->data.f32[i] = dsfmt_genrand_open_close(&dsfmt);
    2579
    1
      ccv_nnc_cmd_exec(CMD_DATA_TRANSFER_FORWARD(), ccv_nnc_no_hint, 0, TENSOR_LIST(ha, hb, hg), TENSOR_LIST(a, b, g), 0);
    2580
    1
      ccv_nnc_cmd_exec(CMD_EWDIV_FORWARD(), ccv_nnc_no_hint, 0, TENSOR_LIST(a, b), TENSOR_LIST(c), 0);
    2581
    1
      ccv_nnc_cmd_exec(CMD_EWDIV_BACKWARD(), ccv_nnc_no_hint, 0, TENSOR_LIST(g, 0, b, c), TENSOR_LIST(0, db), 0);
    2582
    1
      ccv_nnc_cmd_exec(CMD_EWDIV_FORWARD(), ccv_nnc_no_hint, 0, TENSOR_LIST(ha, hb), TENSOR_LIST(hc), 0);
    2583
    1
      ccv_nnc_cmd_exec(CMD_EWDIV_BACKWARD(), ccv_nnc_no_hint, 0, TENSOR_LIST(hg, 0, hb, hc), TENSOR_LIST(0, dbt), 0);
    2584
    1
      ccv_nnc_cmd_exec(CMD_DATA_TRANSFER_FORWARD(), ccv_nnc_no_hint, 0, TENSOR_LIST(db), TENSOR_LIST(hdb), 0);
    2585
    1
      REQUIRE_TENSOR_EQ(dbt, hdb, "GPU computed output should be the same as CPU computed ones");
    2586
    1
      ccv_nnc_tensor_free(a);
    2587
    1
      ccv_nnc_tensor_free(b);
    2588
    1
      ccv_nnc_tensor_free(c);
    2589
    1
      ccv_nnc_tensor_free(g);
    2590
    1
      ccv_nnc_tensor_free(db);
    2591
    1
      ccv_nnc_tensor_free(ha);
    2592
    1
      ccv_nnc_tensor_free(hb);
    2593
    1
      ccv_nnc_tensor_free(hc);
    2594
    1
      ccv_nnc_tensor_free(hg);
    2595
    1
      ccv_nnc_tensor_free(hdb);
    2596
    1
      ccv_nnc_tensor_free(dbt);
    2597
    1
    }
    2598
    2599
    static ccv_nnc_tensor_param_t _ccv_nnc_ew_cpu_nchw_datatype(const int datatype)
    2600
    20
    {
    2601
    20
      ccv_nnc_tensor_param_t params = CPU_TENSOR_NCHW(32F, 10, 100);
    2602
    20
      params.datatype = datatype;
    2603
    20
      return params;
    2604
    20
    }
    2605
    2606
    static ccv_nnc_tensor_param_t _ccv_nnc_ew_gpu_nchw_datatype(const int datatype)
    2607
    16
    {
    2608
    16
      ccv_nnc_tensor_param_t params = GPU_TENSOR_NCHW(000, 32F, 10, 100);
    2609
    16
      params.datatype = datatype;
    2610
    16
      return params;
    2611
    16
    }
    2612
    2613
    static void _ccv_nnc_ewexp_gpu_ref_datatype(const int datatype, float* const forward_max_diff, float* const backward_max_diff)
    2614
    2
    {
    2615
    2
      ccv_nnc_tensor_t* const ha32 = ccv_nnc_tensor_new(0, CPU_TENSOR_NCHW(32F, 10, 100), 0);
    2616
    2
      ccv_nnc_tensor_t* const hg32 = ccv_nnc_tensor_new(0, CPU_TENSOR_NCHW(32F, 10, 100), 0);
    2617
    2
      ccv_nnc_tensor_t* const ha = ccv_nnc_tensor_new(0, _ccv_nnc_ew_cpu_nchw_datatype(datatype), 0);
    2618
    2
      ccv_nnc_tensor_t* const hg = ccv_nnc_tensor_new(0, _ccv_nnc_ew_cpu_nchw_datatype(datatype), 0);
    2619
    2
      ccv_nnc_tensor_t* const har = ccv_nnc_tensor_new(0, CPU_TENSOR_NCHW(32F, 10, 100), 0);
    2620
    2
      ccv_nnc_tensor_t* const hgr = ccv_nnc_tensor_new(0, CPU_TENSOR_NCHW(32F, 10, 100), 0);
    2621
    2
      ccv_nnc_tensor_t* const hb = ccv_nnc_tensor_new(0, _ccv_nnc_ew_cpu_nchw_datatype(datatype), 0);
    2622
    2
      ccv_nnc_tensor_t* const hb32 = ccv_nnc_tensor_new(0, CPU_TENSOR_NCHW(32F, 10, 100), 0);
    2623
    2
      ccv_nnc_tensor_t* const hbr = ccv_nnc_tensor_new(0, CPU_TENSOR_NCHW(32F, 10, 100), 0);
    2624
    2
      ccv_nnc_tensor_t* const hda = ccv_nnc_tensor_new(0, _ccv_nnc_ew_cpu_nchw_datatype(datatype), 0);
    2625
    2
      ccv_nnc_tensor_t* const hda32 = ccv_nnc_tensor_new(0, CPU_TENSOR_NCHW(32F, 10, 100), 0);
    2626
    2
      ccv_nnc_tensor_t* const dat = ccv_nnc_tensor_new(0, _ccv_nnc_ew_cpu_nchw_datatype(datatype), 0);
    2627
    2
      ccv_nnc_tensor_t* const dat32 = ccv_nnc_tensor_new(0, CPU_TENSOR_NCHW(32F, 10, 100), 0);
    2628
    2
      ccv_nnc_tensor_t* const datr = ccv_nnc_tensor_new(0, CPU_TENSOR_NCHW(32F, 10, 100), 0);
    2629
    2
      ccv_nnc_tensor_t* const a = ccv_nnc_tensor_new(0, _ccv_nnc_ew_gpu_nchw_datatype(datatype), 0);
    2630
    2
      ccv_nnc_tensor_t* const b = ccv_nnc_tensor_new(0, _ccv_nnc_ew_gpu_nchw_datatype(datatype), 0);
    2631
    2
      ccv_nnc_tensor_t* const g = ccv_nnc_tensor_new(0, _ccv_nnc_ew_gpu_nchw_datatype(datatype), 0);
    2632
    2
      ccv_nnc_tensor_t* const da = ccv_nnc_tensor_new(0, _ccv_nnc_ew_gpu_nchw_datatype(datatype), 0);
    2633
    2
      dsfmt_t dsfmt;
    2634
    2
      dsfmt_init_gen_rand(&dsfmt, 0);
    2635
    2
      int i;
    2636
    2.00k
      for (i = 0; i < 1000; i++2.00k)
    2637
    2.00k
        ha32->data.f32[i] = dsfmt_genrand_open_close(&dsfmt) * 6 - 3;
    2638
    2.00k
      for (i = 0; i < 1000; i++2.00k)
    2639
    2.00k
        hg32->data.f32[i] = dsfmt_genrand_open_close(&dsfmt);
    2640
    2
      ccv_nnc_cmd_exec(CMD_DATATYPE_CONVERSION_FORWARD(), ccv_nnc_no_hint, 0, TENSOR_LIST(ha32, hg32), TENSOR_LIST(ha, hg), 0);
    2641
    2
      ccv_nnc_cmd_exec(CMD_DATATYPE_CONVERSION_FORWARD(), ccv_nnc_no_hint, 0, TENSOR_LIST(ha, hg), TENSOR_LIST(har, hgr), 0);
    2642
    2
      ccv_nnc_cmd_exec(CMD_DATA_TRANSFER_FORWARD(), ccv_nnc_no_hint, 0, TENSOR_LIST(ha, hg), TENSOR_LIST(a, g), 0);
    2643
    2
      ccv_nnc_cmd_exec(CMD_EWEXP_FORWARD(), ccv_nnc_no_hint, 0, TENSOR_LIST(a), TENSOR_LIST(b), 0);
    2644
    2
      ccv_nnc_cmd_exec(CMD_EWEXP_FORWARD(), ccv_nnc_no_hint, 0, TENSOR_LIST(har), TENSOR_LIST(hb32), 0);
    2645
    2
      ccv_nnc_cmd_exec(CMD_DATATYPE_CONVERSION_FORWARD(), ccv_nnc_no_hint, 0, TENSOR_LIST(hb32), TENSOR_LIST(hb), 0);
    2646
    2
      ccv_nnc_cmd_exec(CMD_DATATYPE_CONVERSION_FORWARD(), ccv_nnc_no_hint, 0, TENSOR_LIST(hb), TENSOR_LIST(hbr), 0);
    2647
    2
      ccv_nnc_cmd_exec(CMD_DATA_TRANSFER_FORWARD(), ccv_nnc_no_hint, 0, TENSOR_LIST(b), TENSOR_LIST(hb), 0);
    2648
    2
      ccv_nnc_cmd_exec(CMD_DATATYPE_CONVERSION_FORWARD(), ccv_nnc_no_hint, 0, TENSOR_LIST(hb), TENSOR_LIST(hb32), 0);
    2649
    2
      *forward_max_diff = 0;
    2650
    2.00k
      for (i = 0; i < 1000; i++2.00k)
    2651
    2.00k
      {
    2652
    2.00k
        const float max_val = ccv_max(ccv_max(fabsf(hbr->data.f32[i]), fabsf(hb32->data.f32[i])), 1);
    2653
    2.00k
        *forward_max_diff = ccv_max(*forward_max_diff, fabsf(hbr->data.f32[i] - hb32->data.f32[i]) / max_val);
    2654
    2.00k
      }
    2655
    2
      ccv_nnc_cmd_exec(CMD_EWEXP_BACKWARD(), ccv_nnc_no_hint, 0, TENSOR_LIST(g, 0, b), TENSOR_LIST(da), 0);
    2656
    2
      ccv_nnc_cmd_exec(CMD_EWEXP_BACKWARD(), ccv_nnc_no_hint, 0, TENSOR_LIST(hgr, 0, hbr), TENSOR_LIST(dat32), 0);
    2657
    2
      ccv_nnc_cmd_exec(CMD_DATATYPE_CONVERSION_FORWARD(), ccv_nnc_no_hint, 0, TENSOR_LIST(dat32), TENSOR_LIST(dat), 0);
    2658
    2
      ccv_nnc_cmd_exec(CMD_DATATYPE_CONVERSION_FORWARD(), ccv_nnc_no_hint, 0, TENSOR_LIST(dat), TENSOR_LIST(datr), 0);
    2659
    2
      ccv_nnc_cmd_exec(CMD_DATA_TRANSFER_FORWARD(), ccv_nnc_no_hint, 0, TENSOR_LIST(da), TENSOR_LIST(hda), 0);
    2660
    2
      ccv_nnc_cmd_exec(CMD_DATATYPE_CONVERSION_FORWARD(), ccv_nnc_no_hint, 0, TENSOR_LIST(hda), TENSOR_LIST(hda32), 0);
    2661
    2
      *backward_max_diff = 0;
    2662
    2.00k
      for (i = 0; i < 1000; i++2.00k)
    2663
    2.00k
      {
    2664
    2.00k
        const float max_val = ccv_max(ccv_max(fabsf(datr->data.f32[i]), fabsf(hda32->data.f32[i])), 1);
    2665
    2.00k
        *backward_max_diff = ccv_max(*backward_max_diff, fabsf(datr->data.f32[i] - hda32->data.f32[i]) / max_val);
    2666
    2.00k
      }
    2667
    2
      ccv_nnc_tensor_free(ha32);
    2668
    2
      ccv_nnc_tensor_free(hg32);
    2669
    2
      ccv_nnc_tensor_free(ha);
    2670
    2
      ccv_nnc_tensor_free(hg);
    2671
    2
      ccv_nnc_tensor_free(har);
    2672
    2
      ccv_nnc_tensor_free(hgr);
    2673
    2
      ccv_nnc_tensor_free(hb);
    2674
    2
      ccv_nnc_tensor_free(hb32);
    2675
    2
      ccv_nnc_tensor_free(hbr);
    2676
    2
      ccv_nnc_tensor_free(hda);
    2677
    2
      ccv_nnc_tensor_free(hda32);
    2678
    2
      ccv_nnc_tensor_free(dat);
    2679
    2
      ccv_nnc_tensor_free(dat32);
    2680
    2
      ccv_nnc_tensor_free(datr);
    2681
    2
      ccv_nnc_tensor_free(a);
    2682
    2
      ccv_nnc_tensor_free(b);
    2683
    2
      ccv_nnc_tensor_free(g);
    2684
    2
      ccv_nnc_tensor_free(da);
    2685
    2
    }
    2686
    2687
    static void _ccv_nnc_ewsoftplus_gpu_ref_datatype(const int datatype, float* const forward_max_diff, float* const backward_max_diff)
    2688
    2
    {
    2689
    2
      ccv_nnc_tensor_t* const ha32 = ccv_nnc_tensor_new(0, CPU_TENSOR_NCHW(32F, 10, 100), 0);
    2690
    2
      ccv_nnc_tensor_t* const hg32 = ccv_nnc_tensor_new(0, CPU_TENSOR_NCHW(32F, 10, 100), 0);
    2691
    2
      ccv_nnc_tensor_t* const ha = ccv_nnc_tensor_new(0, _ccv_nnc_ew_cpu_nchw_datatype(datatype), 0);
    2692
    2
      ccv_nnc_tensor_t* const hg = ccv_nnc_tensor_new(0, _ccv_nnc_ew_cpu_nchw_datatype(datatype), 0);
    2693
    2
      ccv_nnc_tensor_t* const har = ccv_nnc_tensor_new(0, CPU_TENSOR_NCHW(32F, 10, 100), 0);
    2694
    2
      ccv_nnc_tensor_t* const hgr = ccv_nnc_tensor_new(0, CPU_TENSOR_NCHW(32F, 10, 100), 0);
    2695
    2
      ccv_nnc_tensor_t* const hb = ccv_nnc_tensor_new(0, _ccv_nnc_ew_cpu_nchw_datatype(datatype), 0);
    2696
    2
      ccv_nnc_tensor_t* const hb32 = ccv_nnc_tensor_new(0, CPU_TENSOR_NCHW(32F, 10, 100), 0);
    2697
    2
      ccv_nnc_tensor_t* const hbr = ccv_nnc_tensor_new(0, CPU_TENSOR_NCHW(32F, 10, 100), 0);
    2698
    2
      ccv_nnc_tensor_t* const hda = ccv_nnc_tensor_new(0, _ccv_nnc_ew_cpu_nchw_datatype(datatype), 0);
    2699
    2
      ccv_nnc_tensor_t* const hda32 = ccv_nnc_tensor_new(0, CPU_TENSOR_NCHW(32F, 10, 100), 0);
    2700
    2
      ccv_nnc_tensor_t* const dat = ccv_nnc_tensor_new(0, _ccv_nnc_ew_cpu_nchw_datatype(datatype), 0);
    2701
    2
      ccv_nnc_tensor_t* const dat32 = ccv_nnc_tensor_new(0, CPU_TENSOR_NCHW(32F, 10, 100), 0);
    2702
    2
      ccv_nnc_tensor_t* const datr = ccv_nnc_tensor_new(0, CPU_TENSOR_NCHW(32F, 10, 100), 0);
    2703
    2
      ccv_nnc_tensor_t* const a = ccv_nnc_tensor_new(0, _ccv_nnc_ew_gpu_nchw_datatype(datatype), 0);
    2704
    2
      ccv_nnc_tensor_t* const b = ccv_nnc_tensor_new(0, _ccv_nnc_ew_gpu_nchw_datatype(datatype), 0);
    2705
    2
      ccv_nnc_tensor_t* const g = ccv_nnc_tensor_new(0, _ccv_nnc_ew_gpu_nchw_datatype(datatype), 0);
    2706
    2
      ccv_nnc_tensor_t* const da = ccv_nnc_tensor_new(0, _ccv_nnc_ew_gpu_nchw_datatype(datatype), 0);
    2707
    2
      dsfmt_t dsfmt;
    2708
    2
      dsfmt_init_gen_rand(&dsfmt, 0);
    2709
    2
      int i;
    2710
    2.00k
      for (i = 0; i < 1000; i++2.00k)
    2711
    2.00k
        ha32->data.f32[i] = dsfmt_genrand_open_close(&dsfmt) * 16 - 8;
    2712
    2.00k
      for (i = 0; i < 1000; i++2.00k)
    2713
    2.00k
        hg32->data.f32[i] = dsfmt_genrand_open_close(&dsfmt);
    2714
    2
      ccv_nnc_cmd_exec(CMD_DATATYPE_CONVERSION_FORWARD(), ccv_nnc_no_hint, 0, TENSOR_LIST(ha32, hg32), TENSOR_LIST(ha, hg), 0);
    2715
    2
      ccv_nnc_cmd_exec(CMD_DATATYPE_CONVERSION_FORWARD(), ccv_nnc_no_hint, 0, TENSOR_LIST(ha, hg), TENSOR_LIST(har, hgr), 0);
    2716
    2
      ccv_nnc_cmd_exec(CMD_DATA_TRANSFER_FORWARD(), ccv_nnc_no_hint, 0, TENSOR_LIST(ha, hg), TENSOR_LIST(a, g), 0);
    2717
    2
      ccv_nnc_cmd_exec(CMD_EWSOFTPLUS_FORWARD(), ccv_nnc_no_hint, 0, TENSOR_LIST(a), TENSOR_LIST(b), 0);
    2718
    2
      ccv_nnc_cmd_exec(CMD_EWSOFTPLUS_FORWARD(), ccv_nnc_no_hint, 0, TENSOR_LIST(har), TENSOR_LIST(hb32), 0);
    2719
    2
      ccv_nnc_cmd_exec(CMD_DATATYPE_CONVERSION_FORWARD(), ccv_nnc_no_hint, 0, TENSOR_LIST(hb32), TENSOR_LIST(hb), 0);
    2720
    2
      ccv_nnc_cmd_exec(CMD_DATATYPE_CONVERSION_FORWARD(), ccv_nnc_no_hint, 0, TENSOR_LIST(hb), TENSOR_LIST(hbr), 0);
    2721
    2
      ccv_nnc_cmd_exec(CMD_DATA_TRANSFER_FORWARD(), ccv_nnc_no_hint, 0, TENSOR_LIST(b), TENSOR_LIST(hb), 0);
    2722
    2
      ccv_nnc_cmd_exec(CMD_DATATYPE_CONVERSION_FORWARD(), ccv_nnc_no_hint, 0, TENSOR_LIST(hb), TENSOR_LIST(hb32), 0);
    2723
    2
      *forward_max_diff = 0;
    2724
    2.00k
      for (i = 0; i < 1000; i++2.00k)
    2725
    2.00k
      {
    2726
    2.00k
        const float max_val = ccv_max(ccv_max(fabsf(hbr->data.f32[i]), fabsf(hb32->data.f32[i])), 1);
    2727
    2.00k
        *forward_max_diff = ccv_max(*forward_max_diff, fabsf(hbr->data.f32[i] - hb32->data.f32[i]) / max_val);
    2728
    2.00k
      }
    2729
    2
      ccv_nnc_cmd_exec(CMD_EWSOFTPLUS_BACKWARD(), ccv_nnc_no_hint, 0, TENSOR_LIST(g, a), TENSOR_LIST(da), 0);
    2730
    2
      ccv_nnc_cmd_exec(CMD_EWSOFTPLUS_BACKWARD(), ccv_nnc_no_hint, 0, TENSOR_LIST(hgr, har), TENSOR_LIST(dat32), 0);
    2731
    2
      ccv_nnc_cmd_exec(CMD_DATATYPE_CONVERSION_FORWARD(), ccv_nnc_no_hint, 0, TENSOR_LIST(dat32), TENSOR_LIST(dat), 0);
    2732
    2
      ccv_nnc_cmd_exec(CMD_DATATYPE_CONVERSION_FORWARD(), ccv_nnc_no_hint, 0, TENSOR_LIST(dat), TENSOR_LIST(datr), 0);
    2733
    2
      ccv_nnc_cmd_exec(CMD_DATA_TRANSFER_FORWARD(), ccv_nnc_no_hint, 0, TENSOR_LIST(da), TENSOR_LIST(hda), 0);
    2734
    2
      ccv_nnc_cmd_exec(CMD_DATATYPE_CONVERSION_FORWARD(), ccv_nnc_no_hint, 0, TENSOR_LIST(hda), TENSOR_LIST(hda32), 0);
    2735
    2
      *backward_max_diff = 0;
    2736
    2.00k
      for (i = 0; i < 1000; i++2.00k)
    2737
    2.00k
      {
    2738
    2.00k
        const float max_val = ccv_max(ccv_max(fabsf(datr->data.f32[i]), fabsf(hda32->data.f32[i])), 1);
    2739
    2.00k
        *backward_max_diff = ccv_max(*backward_max_diff, fabsf(datr->data.f32[i] - hda32->data.f32[i]) / max_val);
    2740
    2.00k
      }
    2741
    2
      ccv_nnc_tensor_free(ha32);
    2742
    2
      ccv_nnc_tensor_free(hg32);
    2743
    2
      ccv_nnc_tensor_free(ha);
    2744
    2
      ccv_nnc_tensor_free(hg);
    2745
    2
      ccv_nnc_tensor_free(har);
    2746
    2
      ccv_nnc_tensor_free(hgr);
    2747
    2
      ccv_nnc_tensor_free(hb);
    2748
    2
      ccv_nnc_tensor_free(hb32);
    2749
    2
      ccv_nnc_tensor_free(hbr);
    2750
    2
      ccv_nnc_tensor_free(hda);
    2751
    2
      ccv_nnc_tensor_free(hda32);
    2752
    2
      ccv_nnc_tensor_free(dat);
    2753
    2
      ccv_nnc_tensor_free(dat32);
    2754
    2
      ccv_nnc_tensor_free(datr);
    2755
    2
      ccv_nnc_tensor_free(a);
    2756
    2
      ccv_nnc_tensor_free(b);
    2757
    2
      ccv_nnc_tensor_free(g);
    2758
    2
      ccv_nnc_tensor_free(da);
    2759
    2
    }
    2760
    2761
    TEST_CASE("exp forward")
    2762
    1
    {
    2763
    1
      GUARD_ELSE_RETURN(ccv_nnc_cmd_ok(CCV_NNC_EWEXP_FORWARD, CCV_NNC_BACKEND_GPU_REF));
    2764
    1
      ccv_nnc_tensor_t* a = ccv_nnc_tensor_new(0, GPU_TENSOR_NCHW(000, 32F, 10, 100), 0);
    2765
    1
      ccv_nnc_tensor_t* b = ccv_nnc_tensor_new(0, GPU_TENSOR_NCHW(000, 32F, 10, 100), 0);
    2766
    1
      ccv_nnc_tensor_t* ha = ccv_nnc_tensor_new(0, CPU_TENSOR_NCHW(32F, 10, 100), 0);
    2767
    1
      ccv_nnc_tensor_t* hb = ccv_nnc_tensor_new(0, CPU_TENSOR_NCHW(32F, 10, 100), 0);
    2768
    1
      ccv_nnc_tensor_t* bt = ccv_nnc_tensor_new(0, CPU_TENSOR_NCHW(32F, 10, 100), 0);
    2769
    1
      dsfmt_t dsfmt;
    2770
    1
      dsfmt_init_gen_rand(&dsfmt, 0);
    2771
    1
      int i;
    2772
    1.00k
      for (i = 0; i < 1000; i++1.00k)
    2773
    1.00k
        ha->data.f32[i] = dsfmt_genrand_open_close(&dsfmt) * 10 - 1;
    2774
    1
      ccv_nnc_cmd_exec(CMD_DATA_TRANSFER_FORWARD(), ccv_nnc_no_hint, 0, TENSOR_LIST(ha), TENSOR_LIST(a), 0);
    2775
    1
      ccv_nnc_cmd_exec(CMD_EWEXP_FORWARD(), ccv_nnc_no_hint, 0, TENSOR_LIST(a), TENSOR_LIST(b), 0);
    2776
    1
      ccv_nnc_cmd_exec(CMD_EWEXP_FORWARD(), ccv_nnc_no_hint, 0, TENSOR_LIST(ha), TENSOR_LIST(bt), 0);
    2777
    1
      ccv_nnc_cmd_exec(CMD_DATA_TRANSFER_FORWARD(), ccv_nnc_no_hint, 0, TENSOR_LIST(b), TENSOR_LIST(hb), 0);
    2778
    1
      REQUIRE_TENSOR_EQ(bt, hb, "GPU computed output should be the same as CPU computed ones");
    2779
    1
      ccv_nnc_tensor_free(a);
    2780
    1
      ccv_nnc_tensor_free(b);
    2781
    1
      ccv_nnc_tensor_free(ha);
    2782
    1
      ccv_nnc_tensor_free(hb);
    2783
    1
      ccv_nnc_tensor_free(bt);
    2784
    1
    }
    2785
    2786
    TEST_CASE("ewexp backward")
    2787
    1
    {
    2788
    1
      GUARD_ELSE_RETURN(ccv_nnc_cmd_ok(CCV_NNC_EWEXP_FORWARD, CCV_NNC_BACKEND_GPU_REF) &&
    2789
    1
        ccv_nnc_cmd_ok(CCV_NNC_EWEXP_BACKWARD, CCV_NNC_BACKEND_GPU_REF));
    2790
    1
      ccv_nnc_tensor_t* a = ccv_nnc_tensor_new(0, GPU_TENSOR_NCHW(000, 32F, 10, 100), 0);
    2791
    1
      ccv_nnc_tensor_t* b = ccv_nnc_tensor_new(0, GPU_TENSOR_NCHW(000, 32F, 10, 100), 0);
    2792
    1
      ccv_nnc_tensor_t* g = ccv_nnc_tensor_new(0, GPU_TENSOR_NCHW(000, 32F, 10, 100), 0);
    2793
    1
      ccv_nnc_tensor_t* da = ccv_nnc_tensor_new(0, GPU_TENSOR_NCHW(000, 32F, 10, 100), 0);
    2794
    1
      ccv_nnc_tensor_t* ha = ccv_nnc_tensor_new(0, CPU_TENSOR_NCHW(32F, 10, 100), 0);
    2795
    1
      ccv_nnc_tensor_t* hda = ccv_nnc_tensor_new(0, CPU_TENSOR_NCHW(32F, 10, 100), 0);
    2796
    1
      ccv_nnc_tensor_t* hb = ccv_nnc_tensor_new(0, CPU_TENSOR_NCHW(32F, 10, 100), 0);
    2797
    1
      ccv_nnc_tensor_t* hg = ccv_nnc_tensor_new(0, CPU_TENSOR_NCHW(32F, 10, 100), 0);
    2798
    1
      ccv_nnc_tensor_t* dat = ccv_nnc_tensor_new(0, CPU_TENSOR_NCHW(32F, 10, 100), 0);
    2799
    1
      dsfmt_t dsfmt;
    2800
    1
      dsfmt_init_gen_rand(&dsfmt, 0);
    2801
    1
      int i;
    2802
    1.00k
      for (i = 0; i < 1000; i++1.00k)
    2803
    1.00k
        ha->data.f32[i] = dsfmt_genrand_open_close(&dsfmt) * 10;
    2804
    1.00k
      for (i = 0; i < 1000; i++1.00k)
    2805
    1.00k
        hg->data.f32[i] = dsfmt_genrand_open_close(&dsfmt);
    2806
    1
      ccv_nnc_cmd_exec(CMD_DATA_TRANSFER_FORWARD(), ccv_nnc_no_hint, 0, TENSOR_LIST(ha, hg), TENSOR_LIST(a, g), 0);
    2807
    1
      ccv_nnc_cmd_exec(CMD_EWEXP_FORWARD(), ccv_nnc_no_hint, 0, TENSOR_LIST(a), TENSOR_LIST(b), 0);
    2808
    1
      ccv_nnc_cmd_exec(CMD_EWEXP_BACKWARD(), ccv_nnc_no_hint, 0, TENSOR_LIST(g, 0, b), TENSOR_LIST(da), 0);
    2809
    1
      ccv_nnc_cmd_exec(CMD_EWEXP_FORWARD(), ccv_nnc_no_hint, 0, TENSOR_LIST(ha), TENSOR_LIST(hb), 0);
    2810
    1
      ccv_nnc_cmd_exec(CMD_EWEXP_BACKWARD(), ccv_nnc_no_hint, 0, TENSOR_LIST(hg, 0, hb), TENSOR_LIST(dat), 0);
    2811
    1
      ccv_nnc_cmd_exec(CMD_DATA_TRANSFER_FORWARD(), ccv_nnc_no_hint, 0, TENSOR_LIST(da), TENSOR_LIST(hda), 0);
    2812
    1
      REQUIRE_TENSOR_EQ(dat, hda, "GPU computed output should be the same as CPU computed ones");
    2813
    1
      ccv_nnc_tensor_free(a);
    2814
    1
      ccv_nnc_tensor_free(b);
    2815
    1
      ccv_nnc_tensor_free(g);
    2816
    1
      ccv_nnc_tensor_free(da);
    2817
    1
      ccv_nnc_tensor_free(ha);
    2818
    1
      ccv_nnc_tensor_free(hb);
    2819
    1
      ccv_nnc_tensor_free(hg);
    2820
    1
      ccv_nnc_tensor_free(hda);
    2821
    1
      ccv_nnc_tensor_free(dat);
    2822
    1
    }
    2823
    2824
    TEST_CASE("ewexp half precision")
    2825
    1
    {
    2826
    1
      GUARD_ELSE_RETURN(ccv_nnc_cmd_ok(CCV_NNC_EWEXP_FORWARD, CCV_NNC_BACKEND_GPU_REF) &&
    2827
    1
        ccv_nnc_cmd_ok(CCV_NNC_EWEXP_BACKWARD, CCV_NNC_BACKEND_GPU_REF));
    2828
    1
      float forward_max_diff, backward_max_diff;
    2829
    1
      _ccv_nnc_ewexp_gpu_ref_datatype(CCV_16F, &forward_max_diff, &backward_max_diff);
    2830
    1
      REQUIRE(forward_max_diff <= 5e-3, "GPU computed output should be close to CPU computed ones after 16F rounding");
    2831
    1
      REQUIRE(backward_max_diff <= 5e-3, "GPU computed gradient should be close to CPU computed ones after 16F rounding");
    2832
    1
    }
    2833
    2834
    TEST_CASE("ewexp bfloat precision")
    2835
    1
    {
    2836
    1
      GUARD_ELSE_RETURN(ccv_nnc_cmd_ok(CCV_NNC_EWEXP_FORWARD, CCV_NNC_BACKEND_GPU_REF) &&
    2837
    1
        ccv_nnc_cmd_ok(CCV_NNC_EWEXP_BACKWARD, CCV_NNC_BACKEND_GPU_REF));
    2838
    1
      float forward_max_diff, backward_max_diff;
    2839
    1
      _ccv_nnc_ewexp_gpu_ref_datatype(CCV_16BF, &forward_max_diff, &backward_max_diff);
    2840
    1
      REQUIRE(forward_max_diff <= 2e-2, "GPU computed output should be close to CPU computed ones after 16BF rounding");
    2841
    1
      REQUIRE(backward_max_diff <= 2e-2, "GPU computed gradient should be close to CPU computed ones after 16BF rounding");
    2842
    1
    }
    2843
    2844
    TEST_CASE("ewsoftplus forward")
    2845
    1
    {
    2846
    1
      GUARD_ELSE_RETURN(ccv_nnc_cmd_ok(CCV_NNC_EWSOFTPLUS_FORWARD, CCV_NNC_BACKEND_GPU_REF));
    2847
    1
      ccv_nnc_tensor_t* a = ccv_nnc_tensor_new(0, GPU_TENSOR_NCHW(000, 32F, 10, 100), 0);
    2848
    1
      ccv_nnc_tensor_t* b = ccv_nnc_tensor_new(0, GPU_TENSOR_NCHW(000, 32F, 10, 100), 0);
    2849
    1
      ccv_nnc_tensor_t* ha = ccv_nnc_tensor_new(0, CPU_TENSOR_NCHW(32F, 10, 100), 0);
    2850
    1
      ccv_nnc_tensor_t* hb = ccv_nnc_tensor_new(0, CPU_TENSOR_NCHW(32F, 10, 100), 0);
    2851
    1
      ccv_nnc_tensor_t* bt = ccv_nnc_tensor_new(0, CPU_TENSOR_NCHW(32F, 10, 100), 0);
    2852
    1
      dsfmt_t dsfmt;
    2853
    1
      dsfmt_init_gen_rand(&dsfmt, 0);
    2854
    1
      int i;
    2855
    1.00k
      for (i = 0; i < 1000; i++1.00k)
    2856
    1.00k
        ha->data.f32[i] = dsfmt_genrand_open_close(&dsfmt) * 40 - 20;
    2857
    1
      ccv_nnc_cmd_exec(CMD_DATA_TRANSFER_FORWARD(), ccv_nnc_no_hint, 0, TENSOR_LIST(ha), TENSOR_LIST(a), 0);
    2858
    1
      ccv_nnc_cmd_exec(CMD_EWSOFTPLUS_FORWARD(), ccv_nnc_no_hint, 0, TENSOR_LIST(a), TENSOR_LIST(b), 0);
    2859
    1
      ccv_nnc_cmd_exec(CMD_EWSOFTPLUS_FORWARD(), ccv_nnc_no_hint, 0, TENSOR_LIST(ha), TENSOR_LIST(bt), 0);
    2860
    1
      ccv_nnc_cmd_exec(CMD_DATA_TRANSFER_FORWARD(), ccv_nnc_no_hint, 0, TENSOR_LIST(b), TENSOR_LIST(hb), 0);
    2861
    1
      REQUIRE_ARRAY_EQ_WITH_TOLERANCE(float, bt->data.f32, hb->data.f32, 1000, 1e-5, "GPU computed output should be the same as CPU computed ones");
    2862
    1
      ccv_nnc_tensor_free(a);
    2863
    1
      ccv_nnc_tensor_free(b);
    2864
    1
      ccv_nnc_tensor_free(ha);
    2865
    1
      ccv_nnc_tensor_free(hb);
    2866
    1
      ccv_nnc_tensor_free(bt);
    2867
    1
    }
    2868
    2869
    TEST_CASE("ewsoftplus backward")
    2870
    1
    {
    2871
    1
      GUARD_ELSE_RETURN(ccv_nnc_cmd_ok(CCV_NNC_EWSOFTPLUS_FORWARD, CCV_NNC_BACKEND_GPU_REF) &&
    2872
    1
        ccv_nnc_cmd_ok(CCV_NNC_EWSOFTPLUS_BACKWARD, CCV_NNC_BACKEND_GPU_REF));
    2873
    1
      ccv_nnc_tensor_t* a = ccv_nnc_tensor_new(0, GPU_TENSOR_NCHW(000, 32F, 10, 100), 0);
    2874
    1
      ccv_nnc_tensor_t* b = ccv_nnc_tensor_new(0, GPU_TENSOR_NCHW(000, 32F, 10, 100), 0);
    2875
    1
      ccv_nnc_tensor_t* g = ccv_nnc_tensor_new(0, GPU_TENSOR_NCHW(000, 32F, 10, 100), 0);
    2876
    1
      ccv_nnc_tensor_t* da = ccv_nnc_tensor_new(0, GPU_TENSOR_NCHW(000, 32F, 10, 100), 0);
    2877
    1
      ccv_nnc_tensor_t* ha = ccv_nnc_tensor_new(0, CPU_TENSOR_NCHW(32F, 10, 100), 0);
    2878
    1
      ccv_nnc_tensor_t* hda = ccv_nnc_tensor_new(0, CPU_TENSOR_NCHW(32F, 10, 100), 0);
    2879
    1
      ccv_nnc_tensor_t* hb = ccv_nnc_tensor_new(0, CPU_TENSOR_NCHW(32F, 10, 100), 0);
    2880
    1
      ccv_nnc_tensor_t* hg = ccv_nnc_tensor_new(0, CPU_TENSOR_NCHW(32F, 10, 100), 0);
    2881
    1
      ccv_nnc_tensor_t* dat = ccv_nnc_tensor_new(0, CPU_TENSOR_NCHW(32F, 10, 100), 0);
    2882
    1
      dsfmt_t dsfmt;
    2883
    1
      dsfmt_init_gen_rand(&dsfmt, 0);
    2884
    1
      int i;
    2885
    1.00k
      for (i = 0; i < 1000; i++1.00k)
    2886
    1.00k
        ha->data.f32[i] = dsfmt_genrand_open_close(&dsfmt) * 40 - 20;
    2887
    1.00k
      for (i = 0; i < 1000; i++1.00k)
    2888
    1.00k
        hg->data.f32[i] = dsfmt_genrand_open_close(&dsfmt);
    2889
    1
      ccv_nnc_cmd_exec(CMD_DATA_TRANSFER_FORWARD(), ccv_nnc_no_hint, 0, TENSOR_LIST(ha, hg), TENSOR_LIST(a, g), 0);
    2890
    1
      ccv_nnc_cmd_exec(CMD_EWSOFTPLUS_FORWARD(), ccv_nnc_no_hint, 0, TENSOR_LIST(a), TENSOR_LIST(b), 0);
    2891
    1
      ccv_nnc_cmd_exec(CMD_EWSOFTPLUS_BACKWARD(), ccv_nnc_no_hint, 0, TENSOR_LIST(g, a), TENSOR_LIST(da), 0);
    2892
    1
      ccv_nnc_cmd_exec(CMD_EWSOFTPLUS_FORWARD(), ccv_nnc_no_hint, 0, TENSOR_LIST(ha), TENSOR_LIST(hb), 0);
    2893
    1
      ccv_nnc_cmd_exec(CMD_EWSOFTPLUS_BACKWARD(), ccv_nnc_no_hint, 0, TENSOR_LIST(hg, ha), TENSOR_LIST(dat), 0);
    2894
    1
      ccv_nnc_cmd_exec(CMD_DATA_TRANSFER_FORWARD(), ccv_nnc_no_hint, 0, TENSOR_LIST(da), TENSOR_LIST(hda), 0);
    2895
    1
      REQUIRE_ARRAY_EQ_WITH_TOLERANCE(float, dat->data.f32, hda->data.f32, 1000, 1e-5, "GPU computed output should be the same as CPU computed ones");
    2896
    1
      ccv_nnc_tensor_free(a);
    2897
    1
      ccv_nnc_tensor_free(b);
    2898
    1
      ccv_nnc_tensor_free(g);
    2899
    1
      ccv_nnc_tensor_free(da);
    2900
    1
      ccv_nnc_tensor_free(ha);
    2901
    1
      ccv_nnc_tensor_free(hb);
    2902
    1
      ccv_nnc_tensor_free(hg);
    2903
    1
      ccv_nnc_tensor_free(hda);
    2904
    1
      ccv_nnc_tensor_free(dat);
    2905
    1
    }
    2906
    2907
    TEST_CASE("ewsoftplus half precision")
    2908
    1
    {
    2909
    1
      GUARD_ELSE_RETURN(ccv_nnc_cmd_ok(CCV_NNC_EWSOFTPLUS_FORWARD, CCV_NNC_BACKEND_GPU_REF) &&
    2910
    1
        ccv_nnc_cmd_ok(CCV_NNC_EWSOFTPLUS_BACKWARD, CCV_NNC_BACKEND_GPU_REF));
    2911
    1
      float forward_max_diff, backward_max_diff;
    2912
    1
      _ccv_nnc_ewsoftplus_gpu_ref_datatype(CCV_16F, &forward_max_diff, &backward_max_diff);
    2913
    1
      REQUIRE(forward_max_diff <= 5e-3, "GPU computed output should be close to CPU computed ones after 16F rounding");
    2914
    1
      REQUIRE(backward_max_diff <= 5e-3, "GPU computed gradient should be close to CPU computed ones after 16F rounding");
    2915
    1
    }
    2916
    2917
    TEST_CASE("ewsoftplus bfloat precision")
    2918
    1
    {
    2919
    1
      GUARD_ELSE_RETURN(ccv_nnc_cmd_ok(CCV_NNC_EWSOFTPLUS_FORWARD, CCV_NNC_BACKEND_GPU_REF) &&
    2920
    1
        ccv_nnc_cmd_ok(CCV_NNC_EWSOFTPLUS_BACKWARD, CCV_NNC_BACKEND_GPU_REF));
    2921
    1
      float forward_max_diff, backward_max_diff;
    2922
    1
      _ccv_nnc_ewsoftplus_gpu_ref_datatype(CCV_16BF, &forward_max_diff, &backward_max_diff);
    2923
    1
      REQUIRE(forward_max_diff <= 2e-2, "GPU computed output should be close to CPU computed ones after 16BF rounding");
    2924
    1
      REQUIRE(backward_max_diff <= 2e-2, "GPU computed gradient should be close to CPU computed ones after 16BF rounding");
    2925
    1
    }
    2926
    2927
    TEST_CASE("ewpow forward")
    2928
    1
    {
    2929
    1
      GUARD_ELSE_RETURN(ccv_nnc_cmd_ok(CCV_NNC_EWPOW_FORWARD, CCV_NNC_BACKEND_GPU_REF));
    2930
    1
      ccv_nnc_tensor_t* a = ccv_nnc_tensor_new(0, GPU_TENSOR_NCHW(000, 32F, 10, 100), 0);
    2931
    1
      ccv_nnc_tensor_t* c = ccv_nnc_tensor_new(0, GPU_TENSOR_NCHW(000, 32F, 10, 100), 0);
    2932
    1
      ccv_nnc_tensor_t* ha = ccv_nnc_tensor_new(0, CPU_TENSOR_NCHW(32F, 10, 100), 0);
    2933
    1
      ccv_nnc_tensor_t* hc = ccv_nnc_tensor_new(0, CPU_TENSOR_NCHW(32F, 10, 100), 0);
    2934
    1
      ccv_nnc_tensor_t* ct = ccv_nnc_tensor_new(0, CPU_TENSOR_NCHW(32F, 10, 100), 0);
    2935
    1
      dsfmt_t dsfmt;
    2936
    1
      dsfmt_init_gen_rand(&dsfmt, 0);
    2937
    1
      int i;
    2938
    1.00k
      for (i = 0; i < 1000; i++1.00k)
    2939
    1.00k
        ha->data.f32[i] = dsfmt_genrand_open_close(&dsfmt) * 2 + 0.1;
    2940
    1
      ccv_nnc_cmd_exec(CMD_DATA_TRANSFER_FORWARD(), ccv_nnc_no_hint, 0, TENSOR_LIST(ha), TENSOR_LIST(a), 0);
    2941
    1
      ccv_nnc_cmd_exec(CMD_EWPOW_FORWARD(3), ccv_nnc_no_hint, 0, TENSOR_LIST(a), TENSOR_LIST(c), 0);
    2942
    1
      ccv_nnc_cmd_exec(CMD_EWPOW_FORWARD(3), ccv_nnc_no_hint, 0, TENSOR_LIST(ha), TENSOR_LIST(ct), 0);
    2943
    1
      ccv_nnc_cmd_exec(CMD_DATA_TRANSFER_FORWARD(), ccv_nnc_no_hint, 0, TENSOR_LIST(c), TENSOR_LIST(hc), 0);
    2944
    1
      REQUIRE_TENSOR_EQ(ct, hc, "GPU computed output should be the same as CPU computed ones");
    2945
    1
      ccv_nnc_tensor_free(a);
    2946
    1
      ccv_nnc_tensor_free(c);
    2947
    1
      ccv_nnc_tensor_free(ha);
    2948
    1
      ccv_nnc_tensor_free(hc);
    2949
    1
      ccv_nnc_tensor_free(ct);
    2950
    1
    }
    2951
    2952
    TEST_CASE("ewpow backward")
    2953
    1
    {
    2954
    1
      GUARD_ELSE_RETURN(ccv_nnc_cmd_ok(CCV_NNC_EWPOW_FORWARD, CCV_NNC_BACKEND_GPU_REF) &&
    2955
    1
        ccv_nnc_cmd_ok(CCV_NNC_EWPOW_BACKWARD, CCV_NNC_BACKEND_GPU_REF));
    2956
    1
      ccv_nnc_tensor_t* a = ccv_nnc_tensor_new(0, GPU_TENSOR_NCHW(000, 32F, 10, 100), 0);
    2957
    1
      ccv_nnc_tensor_t* c = ccv_nnc_tensor_new(0, GPU_TENSOR_NCHW(000, 32F, 10, 100), 0);
    2958
    1
      ccv_nnc_tensor_t* g = ccv_nnc_tensor_new(0, GPU_TENSOR_NCHW(000, 32F, 10, 100), 0);
    2959
    1
      ccv_nnc_tensor_t* da = ccv_nnc_tensor_new(0, GPU_TENSOR_NCHW(000, 32F, 10, 100), 0);
    2960
    1
      ccv_nnc_tensor_t* ha = ccv_nnc_tensor_new(0, CPU_TENSOR_NCHW(32F, 10, 100), 0);
    2961
    1
      ccv_nnc_tensor_t* hc = ccv_nnc_tensor_new(0, CPU_TENSOR_NCHW(32F, 10, 100), 0);
    2962
    1
      ccv_nnc_tensor_t* hg = ccv_nnc_tensor_new(0, CPU_TENSOR_NCHW(32F, 10, 100), 0);
    2963
    1
      ccv_nnc_tensor_t* hda = ccv_nnc_tensor_new(0, CPU_TENSOR_NCHW(32F, 10, 100), 0);
    2964
    1
      ccv_nnc_tensor_t* dat = ccv_nnc_tensor_new(0, CPU_TENSOR_NCHW(32F, 10, 100), 0);
    2965
    1
      dsfmt_t dsfmt;
    2966
    1
      dsfmt_init_gen_rand(&dsfmt, 0);
    2967
    1
      int i;
    2968
    1.00k
      for (i = 0; i < 1000; i++1.00k)
    2969
    1.00k
        ha->data.f32[i] = dsfmt_genrand_open_close(&dsfmt) * 2 + 0.1;
    2970
    1.00k
      for (i = 0; i < 1000; i++1.00k)
    2971
    1.00k
        hg->data.f32[i] = dsfmt_genrand_open_close(&dsfmt) * 2 - 1;
    2972
    1
      ccv_nnc_cmd_exec(CMD_DATA_TRANSFER_FORWARD(), ccv_nnc_no_hint, 0, TENSOR_LIST(ha, hg), TENSOR_LIST(a, g), 0);
    2973
    1
      ccv_nnc_cmd_exec(CMD_EWPOW_FORWARD(3), ccv_nnc_no_hint, 0, TENSOR_LIST(a), TENSOR_LIST(c), 0);
    2974
    1
      ccv_nnc_cmd_exec(CMD_EWPOW_BACKWARD(3), ccv_nnc_no_hint, 0, TENSOR_LIST(g, a), TENSOR_LIST(da), 0);
    2975
    1
      ccv_nnc_cmd_exec(CMD_EWPOW_FORWARD(3), ccv_nnc_no_hint, 0, TENSOR_LIST(ha), TENSOR_LIST(hc), 0);
    2976
    1
      ccv_nnc_cmd_exec(CMD_EWPOW_BACKWARD(3), ccv_nnc_no_hint, 0, TENSOR_LIST(hg, ha), TENSOR_LIST(dat), 0);
    2977
    1
      ccv_nnc_cmd_exec(CMD_DATA_TRANSFER_FORWARD(), ccv_nnc_no_hint, 0, TENSOR_LIST(da), TENSOR_LIST(hda), 0);
    2978
    1
      REQUIRE_TENSOR_EQ(dat, hda, "GPU computed dA should be the same as CPU computed ones");
    2979
    1
      ccv_nnc_tensor_free(a);
    2980
    1
      ccv_nnc_tensor_free(c);
    2981
    1
      ccv_nnc_tensor_free(g);
    2982
    1
      ccv_nnc_tensor_free(da);
    2983
    1
      ccv_nnc_tensor_free(ha);
    2984
    1
      ccv_nnc_tensor_free(hc);
    2985
    1
      ccv_nnc_tensor_free(hg);
    2986
    1
      ccv_nnc_tensor_free(hda);
    2987
    1
      ccv_nnc_tensor_free(dat);
    2988
    1
    }
    2989
    2990
    TEST_CASE("ewsin forward")
    2991
    1
    {
    2992
    1
      GUARD_ELSE_RETURN(ccv_nnc_cmd_ok(CCV_NNC_EWSIN_FORWARD, CCV_NNC_BACKEND_GPU_REF));
    2993
    1
      ccv_nnc_tensor_t* a = ccv_nnc_tensor_new(0, GPU_TENSOR_NCHW(000, 32F, 10, 100), 0);
    2994
    1
      ccv_nnc_tensor_t* b = ccv_nnc_tensor_new(0, GPU_TENSOR_NCHW(000, 32F, 10, 100), 0);
    2995
    1
      ccv_nnc_tensor_t* ha = ccv_nnc_tensor_new(0, CPU_TENSOR_NCHW(32F, 10, 100), 0);
    2996
    1
      ccv_nnc_tensor_t* hb = ccv_nnc_tensor_new(0, CPU_TENSOR_NCHW(32F, 10, 100), 0);
    2997
    1
      ccv_nnc_tensor_t* bt = ccv_nnc_tensor_new(0, CPU_TENSOR_NCHW(32F, 10, 100), 0);
    2998
    1
      dsfmt_t dsfmt;
    2999
    1
      dsfmt_init_gen_rand(&dsfmt, 0);
    3000
    1
      int i;
    3001
    1.00k
      for (i = 0; i < 1000; i++1.00k)
    3002
    1.00k
        ha->data.f32[i] = dsfmt_genrand_open_close(&dsfmt) * 10 - 5;
    3003
    1
      ccv_nnc_cmd_exec(CMD_DATA_TRANSFER_FORWARD(), ccv_nnc_no_hint, 0, TENSOR_LIST(ha), TENSOR_LIST(a), 0);
    3004
    1
      ccv_nnc_cmd_exec(CMD_EWSIN_FORWARD(), ccv_nnc_no_hint, 0, TENSOR_LIST(a), TENSOR_LIST(b), 0);
    3005
    1
      ccv_nnc_cmd_exec(CMD_EWSIN_FORWARD(), ccv_nnc_no_hint, 0, TENSOR_LIST(ha), TENSOR_LIST(bt), 0);
    3006
    1
      ccv_nnc_cmd_exec(CMD_DATA_TRANSFER_FORWARD(), ccv_nnc_no_hint, 0, TENSOR_LIST(b), TENSOR_LIST(hb), 0);
    3007
    1
      REQUIRE_ARRAY_EQ_WITH_TOLERANCE(float, bt->data.f32, hb->data.f32, 10 * 100, 1e-3, "GPU computed output should be the same as CPU computed ones");
    3008
    1
      ccv_nnc_tensor_free(a);
    3009
    1
      ccv_nnc_tensor_free(b);
    3010
    1
      ccv_nnc_tensor_free(ha);
    3011
    1
      ccv_nnc_tensor_free(hb);
    3012
    1
      ccv_nnc_tensor_free(bt);
    3013
    1
    }
    3014
    3015
    TEST_CASE("ewsin backward")
    3016
    1
    {
    3017
    1
      GUARD_ELSE_RETURN(ccv_nnc_cmd_ok(CCV_NNC_EWSIN_FORWARD, CCV_NNC_BACKEND_GPU_REF) &&
    3018
    1
        ccv_nnc_cmd_ok(CCV_NNC_EWSIN_BACKWARD, CCV_NNC_BACKEND_GPU_REF));
    3019
    1
      ccv_nnc_tensor_t* a = ccv_nnc_tensor_new(0, GPU_TENSOR_NCHW(000, 32F, 10, 100), 0);
    3020
    1
      ccv_nnc_tensor_t* b = ccv_nnc_tensor_new(0, GPU_TENSOR_NCHW(000, 32F, 10, 100), 0);
    3021
    1
      ccv_nnc_tensor_t* g = ccv_nnc_tensor_new(0, GPU_TENSOR_NCHW(000, 32F, 10, 100), 0);
    3022
    1
      ccv_nnc_tensor_t* da = ccv_nnc_tensor_new(0, GPU_TENSOR_NCHW(000, 32F, 10, 100), 0);
    3023
    1
      ccv_nnc_tensor_t* ha = ccv_nnc_tensor_new(0, CPU_TENSOR_NCHW(32F, 10, 100), 0);
    3024
    1
      ccv_nnc_tensor_t* hda = ccv_nnc_tensor_new(0, CPU_TENSOR_NCHW(32F, 10, 100), 0);
    3025
    1
      ccv_nnc_tensor_t* hb = ccv_nnc_tensor_new(0, CPU_TENSOR_NCHW(32F, 10, 100), 0);
    3026
    1
      ccv_nnc_tensor_t* hg = ccv_nnc_tensor_new(0, CPU_TENSOR_NCHW(32F, 10, 100), 0);
    3027
    1
      ccv_nnc_tensor_t* dat = ccv_nnc_tensor_new(0, CPU_TENSOR_NCHW(32F, 10, 100), 0);
    3028
    1
      dsfmt_t dsfmt;
    3029
    1
      dsfmt_init_gen_rand(&dsfmt, 0);
    3030
    1
      int i;
    3031
    1.00k
      for (i = 0; i < 1000; i++1.00k)
    3032
    1.00k
        ha->data.f32[i] = dsfmt_genrand_open_close(&dsfmt) * 10 - 5;
    3033
    1.00k
      for (i = 0; i < 1000; i++1.00k)
    3034
    1.00k
        hg->data.f32[i] = dsfmt_genrand_open_close(&dsfmt) * 2 - 1;
    3035
    1
      ccv_nnc_cmd_exec(CMD_DATA_TRANSFER_FORWARD(), ccv_nnc_no_hint, 0, TENSOR_LIST(ha, hg), TENSOR_LIST(a, g), 0);
    3036
    1
      ccv_nnc_cmd_exec(CMD_EWSIN_FORWARD(), ccv_nnc_no_hint, 0, TENSOR_LIST(a), TENSOR_LIST(b), 0);
    3037
    1
      ccv_nnc_cmd_exec(CMD_EWSIN_BACKWARD(), ccv_nnc_no_hint, 0, TENSOR_LIST(g, a), TENSOR_LIST(da), 0);
    3038
    1
      ccv_nnc_cmd_exec(CMD_EWSIN_FORWARD(), ccv_nnc_no_hint, 0, TENSOR_LIST(ha), TENSOR_LIST(hb), 0);
    3039
    1
      ccv_nnc_cmd_exec(CMD_EWSIN_BACKWARD(), ccv_nnc_no_hint, 0, TENSOR_LIST(hg, ha), TENSOR_LIST(dat), 0);
    3040
    1
      ccv_nnc_cmd_exec(CMD_DATA_TRANSFER_FORWARD(), ccv_nnc_no_hint, 0, TENSOR_LIST(da), TENSOR_LIST(hda), 0);
    3041
    1
      REQUIRE_ARRAY_EQ_WITH_TOLERANCE(float, dat->data.f32, hda->data.f32, 10 * 100, 1e-3, "GPU computed output should be the same as CPU computed ones");
    3042
    1
      ccv_nnc_tensor_free(a);
    3043
    1
      ccv_nnc_tensor_free(b);
    3044
    1
      ccv_nnc_tensor_free(g);
    3045
    1
      ccv_nnc_tensor_free(da);
    3046
    1
      ccv_nnc_tensor_free(ha);
    3047
    1
      ccv_nnc_tensor_free(hb);
    3048
    1
      ccv_nnc_tensor_free(hg);
    3049
    1
      ccv_nnc_tensor_free(hda);
    3050
    1
      ccv_nnc_tensor_free(dat);
    3051
    1
    }
    3052
    3053
    TEST_CASE("ewcos forward")
    3054
    1
    {
    3055
    1
      GUARD_ELSE_RETURN(ccv_nnc_cmd_ok(CCV_NNC_EWCOS_FORWARD, CCV_NNC_BACKEND_GPU_REF));
    3056
    1
      ccv_nnc_tensor_t* a = ccv_nnc_tensor_new(0, GPU_TENSOR_NCHW(000, 32F, 10, 100), 0);
    3057
    1
      ccv_nnc_tensor_t* b = ccv_nnc_tensor_new(0, GPU_TENSOR_NCHW(000, 32F, 10, 100), 0);
    3058
    1
      ccv_nnc_tensor_t* ha = ccv_nnc_tensor_new(0, CPU_TENSOR_NCHW(32F, 10, 100), 0);
    3059
    1
      ccv_nnc_tensor_t* hb = ccv_nnc_tensor_new(0, CPU_TENSOR_NCHW(32F, 10, 100), 0);
    3060
    1
      ccv_nnc_tensor_t* bt = ccv_nnc_tensor_new(0, CPU_TENSOR_NCHW(32F, 10, 100), 0);
    3061
    1
      dsfmt_t dsfmt;
    3062
    1
      dsfmt_init_gen_rand(&dsfmt, 0);
    3063
    1
      int i;
    3064
    1.00k
      for (i = 0; i < 1000; i++1.00k)
    3065
    1.00k
        ha->data.f32[i] = dsfmt_genrand_open_close(&dsfmt) * 10 - 5;
    3066
    1
      ccv_nnc_cmd_exec(CMD_DATA_TRANSFER_FORWARD(), ccv_nnc_no_hint, 0, TENSOR_LIST(ha), TENSOR_LIST(a), 0);
    3067
    1
      ccv_nnc_cmd_exec(CMD_EWCOS_FORWARD(), ccv_nnc_no_hint, 0, TENSOR_LIST(a), TENSOR_LIST(b), 0);
    3068
    1
      ccv_nnc_cmd_exec(CMD_EWCOS_FORWARD(), ccv_nnc_no_hint, 0, TENSOR_LIST(ha), TENSOR_LIST(bt), 0);
    3069
    1
      ccv_nnc_cmd_exec(CMD_DATA_TRANSFER_FORWARD(), ccv_nnc_no_hint, 0, TENSOR_LIST(b), TENSOR_LIST(hb), 0);
    3070
    1
      REQUIRE_ARRAY_EQ_WITH_TOLERANCE(float, bt->data.f32, hb->data.f32, 10 * 100, 1e-3, "GPU computed output should be the same as CPU computed ones");
    3071
    1
      ccv_nnc_tensor_free(a);
    3072
    1
      ccv_nnc_tensor_free(b);
    3073
    1
      ccv_nnc_tensor_free(ha);
    3074
    1
      ccv_nnc_tensor_free(hb);
    3075
    1
      ccv_nnc_tensor_free(bt);
    3076
    1
    }
    3077
    3078
    TEST_CASE("ewcos backward")
    3079
    1
    {
    3080
    1
      GUARD_ELSE_RETURN(ccv_nnc_cmd_ok(CCV_NNC_EWCOS_FORWARD, CCV_NNC_BACKEND_GPU_REF) &&
    3081
    1
        ccv_nnc_cmd_ok(CCV_NNC_EWCOS_BACKWARD, CCV_NNC_BACKEND_GPU_REF));
    3082
    1
      ccv_nnc_tensor_t* a = ccv_nnc_tensor_new(0, GPU_TENSOR_NCHW(000, 32F, 10, 100), 0);
    3083
    1
      ccv_nnc_tensor_t* b = ccv_nnc_tensor_new(0, GPU_TENSOR_NCHW(000, 32F, 10, 100), 0);
    3084
    1
      ccv_nnc_tensor_t* g = ccv_nnc_tensor_new(0, GPU_TENSOR_NCHW(000, 32F, 10, 100), 0);
    3085
    1
      ccv_nnc_tensor_t* da = ccv_nnc_tensor_new(0, GPU_TENSOR_NCHW(000, 32F, 10, 100), 0);
    3086
    1
      ccv_nnc_tensor_t* ha = ccv_nnc_tensor_new(0, CPU_TENSOR_NCHW(32F, 10, 100), 0);
    3087
    1
      ccv_nnc_tensor_t* hda = ccv_nnc_tensor_new(0, CPU_TENSOR_NCHW(32F, 10, 100), 0);
    3088
    1
      ccv_nnc_tensor_t* hb = ccv_nnc_tensor_new(0, CPU_TENSOR_NCHW(32F, 10, 100), 0);
    3089
    1
      ccv_nnc_tensor_t* hg = ccv_nnc_tensor_new(0, CPU_TENSOR_NCHW(32F, 10, 100), 0);
    3090
    1
      ccv_nnc_tensor_t* dat = ccv_nnc_tensor_new(0, CPU_TENSOR_NCHW(32F, 10, 100), 0);
    3091
    1
      dsfmt_t dsfmt;
    3092
    1
      dsfmt_init_gen_rand(&dsfmt, 0);
    3093
    1
      int i;
    3094
    1.00k
      for (i = 0; i < 1000; i++1.00k)
    3095
    1.00k
        ha->data.f32[i] = dsfmt_genrand_open_close(&dsfmt) * 10 - 5;
    3096
    1.00k
      for (i = 0; i < 1000; i++1.00k)
    3097
    1.00k
        hg->data.f32[i] = dsfmt_genrand_open_close(&dsfmt) * 2 - 1;
    3098
    1
      ccv_nnc_cmd_exec(CMD_DATA_TRANSFER_FORWARD(), ccv_nnc_no_hint, 0, TENSOR_LIST(ha, hg), TENSOR_LIST(a, g), 0);
    3099
    1
      ccv_nnc_cmd_exec(CMD_EWCOS_FORWARD(), ccv_nnc_no_hint, 0, TENSOR_LIST(a), TENSOR_LIST(b), 0);
    3100
    1
      ccv_nnc_cmd_exec(CMD_EWCOS_BACKWARD(), ccv_nnc_no_hint, 0, TENSOR_LIST(g, a), TENSOR_LIST(da), 0);
    3101
    1
      ccv_nnc_cmd_exec(CMD_EWCOS_FORWARD(), ccv_nnc_no_hint, 0, TENSOR_LIST(ha), TENSOR_LIST(hb), 0);
    3102
    1
      ccv_nnc_cmd_exec(CMD_EWCOS_BACKWARD(), ccv_nnc_no_hint, 0, TENSOR_LIST(hg, ha), TENSOR_LIST(dat), 0);
    3103
    1
      ccv_nnc_cmd_exec(CMD_DATA_TRANSFER_FORWARD(), ccv_nnc_no_hint, 0, TENSOR_LIST(da), TENSOR_LIST(hda), 0);
    3104
    1
      REQUIRE_ARRAY_EQ_WITH_TOLERANCE(float, dat->data.f32, hda->data.f32, 10 * 100, 1e-3, "GPU computed output should be the same as CPU computed ones");
    3105
    1
      ccv_nnc_tensor_free(a);
    3106
    1
      ccv_nnc_tensor_free(b);
    3107
    1
      ccv_nnc_tensor_free(g);
    3108
    1
      ccv_nnc_tensor_free(da);
    3109
    1
      ccv_nnc_tensor_free(ha);
    3110
    1
      ccv_nnc_tensor_free(hb);
    3111
    1
      ccv_nnc_tensor_free(hg);
    3112
    1
      ccv_nnc_tensor_free(hda);
    3113
    1
      ccv_nnc_tensor_free(dat);
    3114
    1
    }
    3115
    3116
    TEST_CASE("ewlog forward")
    3117
    1
    {
    3118
    1
      GUARD_ELSE_RETURN(ccv_nnc_cmd_ok(CCV_NNC_EWLOG_FORWARD, CCV_NNC_BACKEND_GPU_REF));
    3119
    1
      ccv_nnc_tensor_t* a = ccv_nnc_tensor_new(0, GPU_TENSOR_NCHW(000, 32F, 10, 100), 0);
    3120
    1
      ccv_nnc_tensor_t* b = ccv_nnc_tensor_new(0, GPU_TENSOR_NCHW(000, 32F, 10, 100), 0);
    3121
    1
      ccv_nnc_tensor_t* ha = ccv_nnc_tensor_new(0, CPU_TENSOR_NCHW(32F, 10, 100), 0);
    3122
    1
      ccv_nnc_tensor_t* hb = ccv_nnc_tensor_new(0, CPU_TENSOR_NCHW(32F, 10, 100), 0);
    3123
    1
      ccv_nnc_tensor_t* bt = ccv_nnc_tensor_new(0, CPU_TENSOR_NCHW(32F, 10, 100), 0);
    3124
    1
      dsfmt_t dsfmt;
    3125
    1
      dsfmt_init_gen_rand(&dsfmt, 0);
    3126
    1
      int i;
    3127
    1.00k
      for (i = 0; i < 1000; i++1.00k)
    3128
    1.00k
        ha->data.f32[i] = dsfmt_genrand_open_close(&dsfmt) * 10 + 0.0001;
    3129
    1
      ccv_nnc_cmd_exec(CMD_DATA_TRANSFER_FORWARD(), ccv_nnc_no_hint, 0, TENSOR_LIST(ha), TENSOR_LIST(a), 0);
    3130
    1
      ccv_nnc_cmd_exec(CMD_EWLOG_FORWARD(), ccv_nnc_no_hint, 0, TENSOR_LIST(a), TENSOR_LIST(b), 0);
    3131
    1
      ccv_nnc_cmd_exec(CMD_EWLOG_FORWARD(), ccv_nnc_no_hint, 0, TENSOR_LIST(ha), TENSOR_LIST(bt), 0);
    3132
    1
      ccv_nnc_cmd_exec(CMD_DATA_TRANSFER_FORWARD(), ccv_nnc_no_hint, 0, TENSOR_LIST(b), TENSOR_LIST(hb), 0);
    3133
    1
      REQUIRE_TENSOR_EQ(bt, hb, "GPU computed output should be the same as CPU computed ones");
    3134
    1
      ccv_nnc_tensor_free(a);
    3135
    1
      ccv_nnc_tensor_free(b);
    3136
    1
      ccv_nnc_tensor_free(ha);
    3137
    1
      ccv_nnc_tensor_free(hb);
    3138
    1
      ccv_nnc_tensor_free(bt);
    3139
    1
    }
    3140
    3141
    TEST_CASE("ewlog backward")
    3142
    1
    {
    3143
    1
      GUARD_ELSE_RETURN(ccv_nnc_cmd_ok(CCV_NNC_EWLOG_FORWARD, CCV_NNC_BACKEND_GPU_REF) &&
    3144
    1
        ccv_nnc_cmd_ok(CCV_NNC_EWLOG_BACKWARD, CCV_NNC_BACKEND_GPU_REF));
    3145
    1
      ccv_nnc_tensor_t* a = ccv_nnc_tensor_new(0, GPU_TENSOR_NCHW(000, 32F, 10, 100), 0);
    3146
    1
      ccv_nnc_tensor_t* b = ccv_nnc_tensor_new(0, GPU_TENSOR_NCHW(000, 32F, 10, 100), 0);
    3147
    1
      ccv_nnc_tensor_t* g = ccv_nnc_tensor_new(0, GPU_TENSOR_NCHW(000, 32F, 10, 100), 0);
    3148
    1
      ccv_nnc_tensor_t* da = ccv_nnc_tensor_new(0, GPU_TENSOR_NCHW(000, 32F, 10, 100), 0);
    3149
    1
      ccv_nnc_tensor_t* ha = ccv_nnc_tensor_new(0, CPU_TENSOR_NCHW(32F, 10, 100), 0);
    3150
    1
      ccv_nnc_tensor_t* hda = ccv_nnc_tensor_new(0, CPU_TENSOR_NCHW(32F, 10, 100), 0);
    3151
    1
      ccv_nnc_tensor_t* hb = ccv_nnc_tensor_new(0, CPU_TENSOR_NCHW(32F, 10, 100), 0);
    3152
    1
      ccv_nnc_tensor_t* hg = ccv_nnc_tensor_new(0, CPU_TENSOR_NCHW(32F, 10, 100), 0);
    3153
    1
      ccv_nnc_tensor_t* dat = ccv_nnc_tensor_new(0, CPU_TENSOR_NCHW(32F, 10, 100), 0);
    3154
    1
      dsfmt_t dsfmt;
    3155
    1
      dsfmt_init_gen_rand(&dsfmt, 0);
    3156
    1
      int i;
    3157
    1.00k
      for (i = 0; i < 1000; i++1.00k)
    3158
    1.00k
        ha->data.f32[i] = dsfmt_genrand_open_close(&dsfmt) * 10;
    3159
    1.00k
      for (i = 0; i < 1000; i++1.00k)
    3160
    1.00k
        hg->data.f32[i] = dsfmt_genrand_open_close(&dsfmt);
    3161
    1
      ccv_nnc_cmd_exec(CMD_DATA_TRANSFER_FORWARD(), ccv_nnc_no_hint, 0, TENSOR_LIST(ha, hg), TENSOR_LIST(a, g), 0);
    3162
    1
      ccv_nnc_cmd_exec(CMD_EWLOG_FORWARD(), ccv_nnc_no_hint, 0, TENSOR_LIST(a), TENSOR_LIST(b), 0);
    3163
    1
      ccv_nnc_cmd_exec(CMD_EWLOG_BACKWARD(), ccv_nnc_no_hint, 0, TENSOR_LIST(g, a), TENSOR_LIST(da), 0);
    3164
    1
      ccv_nnc_cmd_exec(CMD_EWLOG_FORWARD(), ccv_nnc_no_hint, 0, TENSOR_LIST(ha), TENSOR_LIST(hb), 0);
    3165
    1
      ccv_nnc_cmd_exec(CMD_EWLOG_BACKWARD(), ccv_nnc_no_hint, 0, TENSOR_LIST(hg, ha), TENSOR_LIST(dat), 0);
    3166
    1
      ccv_nnc_cmd_exec(CMD_DATA_TRANSFER_FORWARD(), ccv_nnc_no_hint, 0, TENSOR_LIST(da), TENSOR_LIST(hda), 0);
    3167
    1
      REQUIRE_TENSOR_EQ(dat, hda, "GPU computed output should be the same as CPU computed ones");
    3168
    1
      ccv_nnc_tensor_free(a);
    3169
    1
      ccv_nnc_tensor_free(b);
    3170
    1
      ccv_nnc_tensor_free(g);
    3171
    1
      ccv_nnc_tensor_free(da);
    3172
    1
      ccv_nnc_tensor_free(ha);
    3173
    1
      ccv_nnc_tensor_free(hb);
    3174
    1
      ccv_nnc_tensor_free(hg);
    3175
    1
      ccv_nnc_tensor_free(hda);
    3176
    1
      ccv_nnc_tensor_free(dat);
    3177
    1
    }
    3178
    3179
    TEST_CASE("ewsqrt forward")
    3180
    1
    {
    3181
    1
      GUARD_ELSE_RETURN(ccv_nnc_cmd_ok(CCV_NNC_EWSQRT_FORWARD, CCV_NNC_BACKEND_GPU_REF));
    3182
    1
      ccv_nnc_tensor_t* a = ccv_nnc_tensor_new(0, GPU_TENSOR_NCHW(000, 32F, 10, 100), 0);
    3183
    1
      ccv_nnc_tensor_t* b = ccv_nnc_tensor_new(0, GPU_TENSOR_NCHW(000, 32F, 10, 100), 0);
    3184
    1
      ccv_nnc_tensor_t* ha = ccv_nnc_tensor_new(0, CPU_TENSOR_NCHW(32F, 10, 100), 0);
    3185
    1
      ccv_nnc_tensor_t* hb = ccv_nnc_tensor_new(0, CPU_TENSOR_NCHW(32F, 10, 100), 0);
    3186
    1
      ccv_nnc_tensor_t* bt = ccv_nnc_tensor_new(0, CPU_TENSOR_NCHW(32F, 10, 100), 0);
    3187
    1
      dsfmt_t dsfmt;
    3188
    1
      dsfmt_init_gen_rand(&dsfmt, 0);
    3189
    1
      int i;
    3190
    1.00k
      for (i = 0; i < 1000; i++1.00k)
    3191
    1.00k
        ha->data.f32[i] = dsfmt_genrand_open_close(&dsfmt) * 10 + 0.0001;
    3192
    1
      ccv_nnc_cmd_exec(CMD_DATA_TRANSFER_FORWARD(), ccv_nnc_no_hint, 0, TENSOR_LIST(ha), TENSOR_LIST(a), 0);
    3193
    1
      ccv_nnc_cmd_exec(CMD_EWSQRT_FORWARD(), ccv_nnc_no_hint, 0, TENSOR_LIST(a), TENSOR_LIST(b), 0);
    3194
    1
      ccv_nnc_cmd_exec(CMD_EWSQRT_FORWARD(), ccv_nnc_no_hint, 0, TENSOR_LIST(ha), TENSOR_LIST(bt), 0);
    3195
    1
      ccv_nnc_cmd_exec(CMD_DATA_TRANSFER_FORWARD(), ccv_nnc_no_hint, 0, TENSOR_LIST(b), TENSOR_LIST(hb), 0);
    3196
    1
      REQUIRE_TENSOR_EQ(bt, hb, "GPU computed output should be the same as CPU computed ones");
    3197
    1
      ccv_nnc_tensor_free(a);
    3198
    1
      ccv_nnc_tensor_free(b);
    3199
    1
      ccv_nnc_tensor_free(ha);
    3200
    1
      ccv_nnc_tensor_free(hb);
    3201
    1
      ccv_nnc_tensor_free(bt);
    3202
    1
    }
    3203
    3204
    TEST_CASE("ewsqrt backward")
    3205
    1
    {
    3206
    1
      GUARD_ELSE_RETURN(ccv_nnc_cmd_ok(CCV_NNC_EWSQRT_FORWARD, CCV_NNC_BACKEND_GPU_REF) &&
    3207
    1
        ccv_nnc_cmd_ok(CCV_NNC_EWSQRT_BACKWARD, CCV_NNC_BACKEND_GPU_REF));
    3208
    1
      ccv_nnc_tensor_t* a = ccv_nnc_tensor_new(0, GPU_TENSOR_NCHW(000, 32F, 10, 100), 0);
    3209
    1
      ccv_nnc_tensor_t* b = ccv_nnc_tensor_new(0, GPU_TENSOR_NCHW(000, 32F, 10, 100), 0);
    3210
    1
      ccv_nnc_tensor_t* g = ccv_nnc_tensor_new(0, GPU_TENSOR_NCHW(000, 32F, 10, 100), 0);
    3211
    1
      ccv_nnc_tensor_t* da = ccv_nnc_tensor_new(0, GPU_TENSOR_NCHW(000, 32F, 10, 100), 0);
    3212
    1
      ccv_nnc_tensor_t* ha = ccv_nnc_tensor_new(0, CPU_TENSOR_NCHW(32F, 10, 100), 0);
    3213
    1
      ccv_nnc_tensor_t* hda = ccv_nnc_tensor_new(0, CPU_TENSOR_NCHW(32F, 10, 100), 0);
    3214
    1
      ccv_nnc_tensor_t* hb = ccv_nnc_tensor_new(0, CPU_TENSOR_NCHW(32F, 10, 100), 0);
    3215
    1
      ccv_nnc_tensor_t* hg = ccv_nnc_tensor_new(0, CPU_TENSOR_NCHW(32F, 10, 100), 0);
    3216
    1
      ccv_nnc_tensor_t* dat = ccv_nnc_tensor_new(0, CPU_TENSOR_NCHW(32F, 10, 100), 0);
    3217
    1
      dsfmt_t dsfmt;
    3218
    1
      dsfmt_init_gen_rand(&dsfmt, 0);
    3219
    1
      int i;
    3220
    1.00k
      for (i = 0; i < 1000; i++1.00k)
    3221
    1.00k
        ha->data.f32[i] = dsfmt_genrand_open_close(&dsfmt) * 10;
    3222
    1.00k
      for (i = 0; i < 1000; i++1.00k)
    3223
    1.00k
        hg->data.f32[i] = dsfmt_genrand_open_close(&dsfmt);
    3224
    1
      ccv_nnc_cmd_exec(CMD_DATA_TRANSFER_FORWARD(), ccv_nnc_no_hint, 0, TENSOR_LIST(ha, hg), TENSOR_LIST(a, g), 0);
    3225
    1
      ccv_nnc_cmd_exec(CMD_EWSQRT_FORWARD(), ccv_nnc_no_hint, 0, TENSOR_LIST(a), TENSOR_LIST(b), 0);
    3226
    1
      ccv_nnc_cmd_exec(CMD_EWSQRT_BACKWARD(), ccv_nnc_no_hint, 0, TENSOR_LIST(g, 0, b), TENSOR_LIST(da), 0);
    3227
    1
      ccv_nnc_cmd_exec(CMD_EWSQRT_FORWARD(), ccv_nnc_no_hint, 0, TENSOR_LIST(ha), TENSOR_LIST(hb), 0);
    3228
    1
      ccv_nnc_cmd_exec(CMD_EWSQRT_BACKWARD(), ccv_nnc_no_hint, 0, TENSOR_LIST(hg, 0, hb), TENSOR_LIST(dat), 0);
    3229
    1
      ccv_nnc_cmd_exec(CMD_DATA_TRANSFER_FORWARD(), ccv_nnc_no_hint, 0, TENSOR_LIST(da), TENSOR_LIST(hda), 0);
    3230
    1
      REQUIRE_TENSOR_EQ(dat, hda, "GPU computed output should be the same as CPU computed ones");
    3231
    1
      ccv_nnc_tensor_free(a);
    3232
    1
      ccv_nnc_tensor_free(b);
    3233
    1
      ccv_nnc_tensor_free(g);
    3234
    1
      ccv_nnc_tensor_free(da);
    3235
    1
      ccv_nnc_tensor_free(ha);
    3236
    1
      ccv_nnc_tensor_free(hb);
    3237
    1
      ccv_nnc_tensor_free(hg);
    3238
    1
      ccv_nnc_tensor_free(hda);
    3239
    1
      ccv_nnc_tensor_free(dat);
    3240
    1
    }
    3241
    3242
    TEST_CASE("ewabs forward")
    3243
    1
    {
    3244
    1
      GUARD_ELSE_RETURN(ccv_nnc_cmd_ok(CCV_NNC_EWABS_FORWARD, CCV_NNC_BACKEND_GPU_REF));
    3245
    1
      ccv_nnc_tensor_t* a = ccv_nnc_tensor_new(0, GPU_TENSOR_NCHW(000, 32F, 10, 100), 0);
    3246
    1
      ccv_nnc_tensor_t* b = ccv_nnc_tensor_new(0, GPU_TENSOR_NCHW(000, 32F, 10, 100), 0);
    3247
    1
      ccv_nnc_tensor_t* ha = ccv_nnc_tensor_new(0, CPU_TENSOR_NCHW(32F, 10, 100), 0);
    3248
    1
      ccv_nnc_tensor_t* hb = ccv_nnc_tensor_new(0, CPU_TENSOR_NCHW(32F, 10, 100), 0);
    3249
    1
      ccv_nnc_tensor_t* bt = ccv_nnc_tensor_new(0, CPU_TENSOR_NCHW(32F, 10, 100), 0);
    3250
    1
      dsfmt_t dsfmt;
    3251
    1
      dsfmt_init_gen_rand(&dsfmt, 0);
    3252
    1
      int i;
    3253
    1.00k
      for (i = 0; i < 1000; i++1.00k)
    3254
    1.00k
        ha->data.f32[i] = dsfmt_genrand_open_close(&dsfmt) * 10 - 5 + 0.0001;
    3255
    1
      ccv_nnc_cmd_exec(CMD_DATA_TRANSFER_FORWARD(), ccv_nnc_no_hint, 0, TENSOR_LIST(ha), TENSOR_LIST(a), 0);
    3256
    1
      ccv_nnc_cmd_exec(CMD_EWABS_FORWARD(), ccv_nnc_no_hint, 0, TENSOR_LIST(a), TENSOR_LIST(b), 0);
    3257
    1
      ccv_nnc_cmd_exec(CMD_EWABS_FORWARD(), ccv_nnc_no_hint, 0, TENSOR_LIST(ha), TENSOR_LIST(bt), 0);
    3258
    1
      ccv_nnc_cmd_exec(CMD_DATA_TRANSFER_FORWARD(), ccv_nnc_no_hint, 0, TENSOR_LIST(b), TENSOR_LIST(hb), 0);
    3259
    1
      REQUIRE_TENSOR_EQ(bt, hb, "GPU computed output should be the same as CPU computed ones");
    3260
    1
      ccv_nnc_tensor_free(a);
    3261
    1
      ccv_nnc_tensor_free(b);
    3262
    1
      ccv_nnc_tensor_free(ha);
    3263
    1
      ccv_nnc_tensor_free(hb);
    3264
    1
      ccv_nnc_tensor_free(bt);
    3265
    1
    }
    3266
    3267
    TEST_CASE("ewabs backward")
    3268
    1
    {
    3269
    1
      GUARD_ELSE_RETURN(ccv_nnc_cmd_ok(CCV_NNC_EWABS_FORWARD, CCV_NNC_BACKEND_GPU_REF) &&
    3270
    1
        ccv_nnc_cmd_ok(CCV_NNC_EWABS_BACKWARD, CCV_NNC_BACKEND_GPU_REF));
    3271
    1
      ccv_nnc_tensor_t* a = ccv_nnc_tensor_new(0, GPU_TENSOR_NCHW(000, 32F, 10, 100), 0);
    3272
    1
      ccv_nnc_tensor_t* b = ccv_nnc_tensor_new(0, GPU_TENSOR_NCHW(000, 32F, 10, 100), 0);
    3273
    1
      ccv_nnc_tensor_t* g = ccv_nnc_tensor_new(0, GPU_TENSOR_NCHW(000, 32F, 10, 100), 0);
    3274
    1
      ccv_nnc_tensor_t* da = ccv_nnc_tensor_new(0, GPU_TENSOR_NCHW(000, 32F, 10, 100), 0);
    3275
    1
      ccv_nnc_tensor_t* ha = ccv_nnc_tensor_new(0, CPU_TENSOR_NCHW(32F, 10, 100), 0);
    3276
    1
      ccv_nnc_tensor_t* hda = ccv_nnc_tensor_new(0, CPU_TENSOR_NCHW(32F, 10, 100), 0);
    3277
    1
      ccv_nnc_tensor_t* hb = ccv_nnc_tensor_new(0, CPU_TENSOR_NCHW(32F, 10, 100), 0);
    3278
    1
      ccv_nnc_tensor_t* hg = ccv_nnc_tensor_new(0, CPU_TENSOR_NCHW(32F, 10, 100), 0);
    3279
    1
      ccv_nnc_tensor_t* dat = ccv_nnc_tensor_new(0, CPU_TENSOR_NCHW(32F, 10, 100), 0);
    3280
    1
      dsfmt_t dsfmt;
    3281
    1
      dsfmt_init_gen_rand(&dsfmt, 0);
    3282
    1
      int i;
    3283
    1.00k
      for (i = 0; i < 1000; i++1.00k)
    3284
    1.00k
        ha->data.f32[i] = dsfmt_genrand_open_close(&dsfmt) * 10;
    3285
    1.00k
      for (i = 0; i < 1000; i++1.00k)
    3286
    1.00k
        hg->data.f32[i] = dsfmt_genrand_open_close(&dsfmt);
    3287
    1
      ccv_nnc_cmd_exec(CMD_DATA_TRANSFER_FORWARD(), ccv_nnc_no_hint, 0, TENSOR_LIST(ha, hg), TENSOR_LIST(a, g), 0);
    3288
    1
      ccv_nnc_cmd_exec(CMD_EWABS_FORWARD(), ccv_nnc_no_hint, 0, TENSOR_LIST(a), TENSOR_LIST(b), 0);
    3289
    1
      ccv_nnc_cmd_exec(CMD_EWABS_BACKWARD(), ccv_nnc_no_hint, 0, TENSOR_LIST(g, a), TENSOR_LIST(da), 0);
    3290
    1
      ccv_nnc_cmd_exec(CMD_EWABS_FORWARD(), ccv_nnc_no_hint, 0, TENSOR_LIST(ha), TENSOR_LIST(hb), 0);
    3291
    1
      ccv_nnc_cmd_exec(CMD_EWABS_BACKWARD(), ccv_nnc_no_hint, 0, TENSOR_LIST(hg, ha), TENSOR_LIST(dat), 0);
    3292
    1
      ccv_nnc_cmd_exec(CMD_DATA_TRANSFER_FORWARD(), ccv_nnc_no_hint, 0, TENSOR_LIST(da), TENSOR_LIST(hda), 0);
    3293
    1
      REQUIRE_TENSOR_EQ(dat, hda, "GPU computed output should be the same as CPU computed ones");
    3294
    1
      ccv_nnc_tensor_free(a);
    3295
    1
      ccv_nnc_tensor_free(b);
    3296
    1
      ccv_nnc_tensor_free(g);
    3297
    1
      ccv_nnc_tensor_free(da);
    3298
    1
      ccv_nnc_tensor_free(ha);
    3299
    1
      ccv_nnc_tensor_free(hb);
    3300
    1
      ccv_nnc_tensor_free(hg);
    3301
    1
      ccv_nnc_tensor_free(hda);
    3302
    1
      ccv_nnc_tensor_free(dat);
    3303
    1
    }
    3304
    3305
    TEST_CASE("clamp forward")
    3306
    1
    {
    3307
    1
      GUARD_ELSE_RETURN(ccv_nnc_cmd_ok(CCV_NNC_CLAMP_FORWARD, CCV_NNC_BACKEND_GPU_REF));
    3308
    1
      ccv_nnc_tensor_t* a = ccv_nnc_tensor_new(0, GPU_TENSOR_NCHW(000, 32F, 10, 100), 0);
    3309
    1
      ccv_nnc_tensor_t* b = ccv_nnc_tensor_new(0, GPU_TENSOR_NCHW(000, 32F, 10, 100), 0);
    3310
    1
      ccv_nnc_tensor_t* ha = ccv_nnc_tensor_new(0, CPU_TENSOR_NCHW(32F, 10, 100), 0);
    3311
    1
      ccv_nnc_tensor_t* hb = ccv_nnc_tensor_new(0, CPU_TENSOR_NCHW(32F, 10, 100), 0);
    3312
    1
      ccv_nnc_tensor_t* bt = ccv_nnc_tensor_new(0, CPU_TENSOR_NCHW(32F, 10, 100), 0);
    3313
    1
      dsfmt_t dsfmt;
    3314
    1
      dsfmt_init_gen_rand(&dsfmt, 0);
    3315
    1
      int i;
    3316
    1.00k
      for (i = 0; i < 1000; i++1.00k)
    3317
    1.00k
        ha->data.f32[i] = dsfmt_genrand_open_close(&dsfmt) * 10 - 1;
    3318
    1
      ccv_nnc_cmd_exec(CMD_DATA_TRANSFER_FORWARD(), ccv_nnc_no_hint, 0, TENSOR_LIST(ha), TENSOR_LIST(a), 0);
    3319
    1
      ccv_nnc_cmd_exec(CMD_CLAMP_FORWARD(0, 6), ccv_nnc_no_hint, 0, TENSOR_LIST(a), TENSOR_LIST(b), 0);
    3320
    1
      ccv_nnc_cmd_exec(CMD_CLAMP_FORWARD(0, 6), ccv_nnc_no_hint, 0, TENSOR_LIST(ha), TENSOR_LIST(bt), 0);
    3321
    1
      ccv_nnc_cmd_exec(CMD_DATA_TRANSFER_FORWARD(), ccv_nnc_no_hint, 0, TENSOR_LIST(b), TENSOR_LIST(hb), 0);
    3322
    1
      REQUIRE_TENSOR_EQ(bt, hb, "GPU computed output should be the same as CPU computed ones");
    3323
    1
      ccv_nnc_tensor_free(a);
    3324
    1
      ccv_nnc_tensor_free(b);
    3325
    1
      ccv_nnc_tensor_free(ha);
    3326
    1
      ccv_nnc_tensor_free(hb);
    3327
    1
      ccv_nnc_tensor_free(bt);
    3328
    1
    }
    3329
    3330
    TEST_CASE("clamp backward")
    3331
    1
    {
    3332
    1
      GUARD_ELSE_RETURN(ccv_nnc_cmd_ok(CCV_NNC_CLAMP_FORWARD, CCV_NNC_BACKEND_GPU_REF) &&
    3333
    1
        ccv_nnc_cmd_ok(CCV_NNC_CLAMP_BACKWARD, CCV_NNC_BACKEND_GPU_REF));
    3334
    1
      ccv_nnc_tensor_t* a = ccv_nnc_tensor_new(0, GPU_TENSOR_NCHW(000, 32F, 10, 100), 0);
    3335
    1
      ccv_nnc_tensor_t* b = ccv_nnc_tensor_new(0, GPU_TENSOR_NCHW(000, 32F, 10, 100), 0);
    3336
    1
      ccv_nnc_tensor_t* g = ccv_nnc_tensor_new(0, GPU_TENSOR_NCHW(000, 32F, 10, 100), 0);
    3337
    1
      ccv_nnc_tensor_t* da = ccv_nnc_tensor_new(0, GPU_TENSOR_NCHW(000, 32F, 10, 100), 0);
    3338
    1
      ccv_nnc_tensor_t* ha = ccv_nnc_tensor_new(0, CPU_TENSOR_NCHW(32F, 10, 100), 0);
    3339
    1
      ccv_nnc_tensor_t* hda = ccv_nnc_tensor_new(0, CPU_TENSOR_NCHW(32F, 10, 100), 0);
    3340
    1
      ccv_nnc_tensor_t* hb = ccv_nnc_tensor_new(0, CPU_TENSOR_NCHW(32F, 10, 100), 0);
    3341
    1
      ccv_nnc_tensor_t* hg = ccv_nnc_tensor_new(0, CPU_TENSOR_NCHW(32F, 10, 100), 0);
    3342
    1
      ccv_nnc_tensor_t* dat = ccv_nnc_tensor_new(0, CPU_TENSOR_NCHW(32F, 10, 100), 0);
    3343
    1
      dsfmt_t dsfmt;
    3344
    1
      dsfmt_init_gen_rand(&dsfmt, 0);
    3345
    1
      int i;
    3346
    1.00k
      for (i = 0; i < 1000; i++1.00k)
    3347
    1.00k
        ha->data.f32[i] = dsfmt_genrand_open_close(&dsfmt) * 10;
    3348
    1.00k
      for (i = 0; i < 1000; i++1.00k)
    3349
    1.00k
        hg->data.f32[i] = dsfmt_genrand_open_close(&dsfmt);
    3350
    1
      ccv_nnc_cmd_exec(CMD_DATA_TRANSFER_FORWARD(), ccv_nnc_no_hint, 0, TENSOR_LIST(ha, hg), TENSOR_LIST(a, g), 0);
    3351
    1
      ccv_nnc_cmd_exec(CMD_CLAMP_FORWARD(0, 5), ccv_nnc_no_hint, 0, TENSOR_LIST(a), TENSOR_LIST(b), 0);
    3352
    1
      ccv_nnc_cmd_exec(CMD_CLAMP_BACKWARD(0, 5), ccv_nnc_no_hint, 0, TENSOR_LIST(g, 0, b), TENSOR_LIST(da), 0);
    3353
    1
      ccv_nnc_cmd_exec(CMD_CLAMP_FORWARD(0, 5), ccv_nnc_no_hint, 0, TENSOR_LIST(ha), TENSOR_LIST(hb), 0);
    3354
    1
      ccv_nnc_cmd_exec(CMD_CLAMP_BACKWARD(0, 5), ccv_nnc_no_hint, 0, TENSOR_LIST(hg, 0, hb), TENSOR_LIST(dat), 0);
    3355
    1
      ccv_nnc_cmd_exec(CMD_DATA_TRANSFER_FORWARD(), ccv_nnc_no_hint, 0, TENSOR_LIST(da), TENSOR_LIST(hda), 0);
    3356
    1
      REQUIRE_TENSOR_EQ(dat, hda, "GPU computed output should be the same as CPU computed ones");
    3357
    1
      ccv_nnc_tensor_free(a);
    3358
    1
      ccv_nnc_tensor_free(b);
    3359
    1
      ccv_nnc_tensor_free(g);
    3360
    1
      ccv_nnc_tensor_free(da);
    3361
    1
      ccv_nnc_tensor_free(ha);
    3362
    1
      ccv_nnc_tensor_free(hb);
    3363
    1
      ccv_nnc_tensor_free(hg);
    3364
    1
      ccv_nnc_tensor_free(hda);
    3365
    1
      ccv_nnc_tensor_free(dat);
    3366
    1
    }
    3367
    3368
    TEST_CASE("clamp forward with only max")
    3369
    1
    {
    3370
    1
      GUARD_ELSE_RETURN(ccv_nnc_cmd_ok(CCV_NNC_CLAMP_FORWARD, CCV_NNC_BACKEND_GPU_REF));
    3371
    1
      ccv_nnc_tensor_t* a = ccv_nnc_tensor_new(0, GPU_TENSOR_NCHW(000, 32F, 10, 100), 0);
    3372
    1
      ccv_nnc_tensor_t* b = ccv_nnc_tensor_new(0, GPU_TENSOR_NCHW(000, 32F, 10, 100), 0);
    3373
    1
      ccv_nnc_tensor_t* ha = ccv_nnc_tensor_new(0, CPU_TENSOR_NCHW(32F, 10, 100), 0);
    3374
    1
      ccv_nnc_tensor_t* hb = ccv_nnc_tensor_new(0, CPU_TENSOR_NCHW(32F, 10, 100), 0);
    3375
    1
      ccv_nnc_tensor_t* bt = ccv_nnc_tensor_new(0, CPU_TENSOR_NCHW(32F, 10, 100), 0);
    3376
    1
      dsfmt_t dsfmt;
    3377
    1
      dsfmt_init_gen_rand(&dsfmt, 0);
    3378
    1
      int i;
    3379
    1.00k
      for (i = 0; i < 1000; i++1.00k)
    3380
    1.00k
        ha->data.f32[i] = dsfmt_genrand_open_close(&dsfmt) * 10 - 1;
    3381
    1
      ccv_nnc_cmd_exec(CMD_DATA_TRANSFER_FORWARD(), ccv_nnc_no_hint, 0, TENSOR_LIST(ha), TENSOR_LIST(a), 0);
    3382
    1
      ccv_nnc_cmd_exec(CMD_CLAMP_FORWARD(NAN, 6), ccv_nnc_no_hint, 0, TENSOR_LIST(a), TENSOR_LIST(b), 0);
    3383
    1
      ccv_nnc_cmd_exec(CMD_CLAMP_FORWARD(NAN, 6), ccv_nnc_no_hint, 0, TENSOR_LIST(ha), TENSOR_LIST(bt), 0);
    3384
    1
      ccv_nnc_cmd_exec(CMD_DATA_TRANSFER_FORWARD(), ccv_nnc_no_hint, 0, TENSOR_LIST(b), TENSOR_LIST(hb), 0);
    3385
    1
      REQUIRE_TENSOR_EQ(bt, hb, "GPU computed output should be the same as CPU computed ones");
    3386
    1
      ccv_nnc_tensor_free(a);
    3387
    1
      ccv_nnc_tensor_free(b);
    3388
    1
      ccv_nnc_tensor_free(ha);
    3389
    1
      ccv_nnc_tensor_free(hb);
    3390
    1
      ccv_nnc_tensor_free(bt);
    3391
    1
    }
    3392
    3393
    TEST_CASE("clamp backward with only max")
    3394
    1
    {
    3395
    1
      GUARD_ELSE_RETURN(ccv_nnc_cmd_ok(CCV_NNC_CLAMP_FORWARD, CCV_NNC_BACKEND_GPU_REF) &&
    3396
    1
        ccv_nnc_cmd_ok(CCV_NNC_CLAMP_BACKWARD, CCV_NNC_BACKEND_GPU_REF));
    3397
    1
      ccv_nnc_tensor_t* a = ccv_nnc_tensor_new(0, GPU_TENSOR_NCHW(000, 32F, 10, 100), 0);
    3398
    1
      ccv_nnc_tensor_t* b = ccv_nnc_tensor_new(0, GPU_TENSOR_NCHW(000, 32F, 10, 100), 0);
    3399
    1
      ccv_nnc_tensor_t* g = ccv_nnc_tensor_new(0, GPU_TENSOR_NCHW(000, 32F, 10, 100), 0);
    3400
    1
      ccv_nnc_tensor_t* da = ccv_nnc_tensor_new(0, GPU_TENSOR_NCHW(000, 32F, 10, 100), 0);
    3401
    1
      ccv_nnc_tensor_t* ha = ccv_nnc_tensor_new(0, CPU_TENSOR_NCHW(32F, 10, 100), 0);
    3402
    1
      ccv_nnc_tensor_t* hda = ccv_nnc_tensor_new(0, CPU_TENSOR_NCHW(32F, 10, 100), 0);
    3403
    1
      ccv_nnc_tensor_t* hb = ccv_nnc_tensor_new(0, CPU_TENSOR_NCHW(32F, 10, 100), 0);
    3404
    1
      ccv_nnc_tensor_t* hg = ccv_nnc_tensor_new(0, CPU_TENSOR_NCHW(32F, 10, 100), 0);
    3405
    1
      ccv_nnc_tensor_t* dat = ccv_nnc_tensor_new(0, CPU_TENSOR_NCHW(32F, 10, 100), 0);
    3406
    1
      dsfmt_t dsfmt;
    3407
    1
      dsfmt_init_gen_rand(&dsfmt, 0);
    3408
    1
      int i;
    3409
    1.00k
      for (i = 0; i < 1000; i++1.00k)
    3410
    1.00k
        ha->data.f32[i] = dsfmt_genrand_open_close(&dsfmt) * 10;
    3411
    1.00k
      for (i = 0; i < 1000; i++1.00k)
    3412
    1.00k
        hg->data.f32[i] = dsfmt_genrand_open_close(&dsfmt);
    3413
    1
      ccv_nnc_cmd_exec(CMD_DATA_TRANSFER_FORWARD(), ccv_nnc_no_hint, 0, TENSOR_LIST(ha, hg), TENSOR_LIST(a, g), 0);
    3414
    1
      ccv_nnc_cmd_exec(CMD_CLAMP_FORWARD(NAN, 5), ccv_nnc_no_hint, 0, TENSOR_LIST(a), TENSOR_LIST(b), 0);
    3415
    1
      ccv_nnc_cmd_exec(CMD_CLAMP_BACKWARD(NAN, 5), ccv_nnc_no_hint, 0, TENSOR_LIST(g, 0, b), TENSOR_LIST(da), 0);
    3416
    1
      ccv_nnc_cmd_exec(CMD_CLAMP_FORWARD(NAN, 5), ccv_nnc_no_hint, 0, TENSOR_LIST(ha), TENSOR_LIST(hb), 0);
    3417
    1
      ccv_nnc_cmd_exec(CMD_CLAMP_BACKWARD(NAN, 5), ccv_nnc_no_hint, 0, TENSOR_LIST(hg, 0, hb), TENSOR_LIST(dat), 0);
    3418
    1
      ccv_nnc_cmd_exec(CMD_DATA_TRANSFER_FORWARD(), ccv_nnc_no_hint, 0, TENSOR_LIST(da), TENSOR_LIST(hda), 0);
    3419
    1
      REQUIRE_TENSOR_EQ(dat, hda, "GPU computed output should be the same as CPU computed ones");
    3420
    1
      ccv_nnc_tensor_free(a);
    3421
    1
      ccv_nnc_tensor_free(b);
    3422
    1
      ccv_nnc_tensor_free(g);
    3423
    1
      ccv_nnc_tensor_free(da);
    3424
    1
      ccv_nnc_tensor_free(ha);
    3425
    1
      ccv_nnc_tensor_free(hb);
    3426
    1
      ccv_nnc_tensor_free(hg);
    3427
    1
      ccv_nnc_tensor_free(hda);
    3428
    1
      ccv_nnc_tensor_free(dat);
    3429
    1
    }
    3430
    3431
    TEST_CASE("clamp forward with only min")
    3432
    1
    {
    3433
    1
      GUARD_ELSE_RETURN(ccv_nnc_cmd_ok(CCV_NNC_CLAMP_FORWARD, CCV_NNC_BACKEND_GPU_REF));
    3434
    1
      ccv_nnc_tensor_t* a = ccv_nnc_tensor_new(0, GPU_TENSOR_NCHW(000, 32F, 10, 100), 0);
    3435
    1
      ccv_nnc_tensor_t* b = ccv_nnc_tensor_new(0, GPU_TENSOR_NCHW(000, 32F, 10, 100), 0);
    3436
    1
      ccv_nnc_tensor_t* ha = ccv_nnc_tensor_new(0, CPU_TENSOR_NCHW(32F, 10, 100), 0);
    3437
    1
      ccv_nnc_tensor_t* hb = ccv_nnc_tensor_new(0, CPU_TENSOR_NCHW(32F, 10, 100), 0);
    3438
    1
      ccv_nnc_tensor_t* bt = ccv_nnc_tensor_new(0, CPU_TENSOR_NCHW(32F, 10, 100), 0);
    3439
    1
      dsfmt_t dsfmt;
    3440
    1
      dsfmt_init_gen_rand(&dsfmt, 0);
    3441
    1
      int i;
    3442
    1.00k
      for (i = 0; i < 1000; i++1.00k)
    3443
    1.00k
        ha->data.f32[i] = dsfmt_genrand_open_close(&dsfmt) * 10 - 1;
    3444
    1
      ccv_nnc_cmd_exec(CMD_DATA_TRANSFER_FORWARD(), ccv_nnc_no_hint, 0, TENSOR_LIST(ha), TENSOR_LIST(a), 0);
    3445
    1
      ccv_nnc_cmd_exec(CMD_CLAMP_FORWARD(0, NAN), ccv_nnc_no_hint, 0, TENSOR_LIST(a), TENSOR_LIST(b), 0);
    3446
    1
      ccv_nnc_cmd_exec(CMD_CLAMP_FORWARD(0, NAN), ccv_nnc_no_hint, 0, TENSOR_LIST(ha), TENSOR_LIST(bt), 0);
    3447
    1
      ccv_nnc_cmd_exec(CMD_DATA_TRANSFER_FORWARD(), ccv_nnc_no_hint, 0, TENSOR_LIST(b), TENSOR_LIST(hb), 0);
    3448
    1
      REQUIRE_TENSOR_EQ(bt, hb, "GPU computed output should be the same as CPU computed ones");
    3449
    1
      ccv_nnc_tensor_free(a);
    3450
    1
      ccv_nnc_tensor_free(b);
    3451
    1
      ccv_nnc_tensor_free(ha);
    3452
    1
      ccv_nnc_tensor_free(hb);
    3453
    1
      ccv_nnc_tensor_free(bt);
    3454
    1
    }
    3455
    3456
    TEST_CASE("clamp backward with only min")
    3457
    1
    {
    3458
    1
      GUARD_ELSE_RETURN(ccv_nnc_cmd_ok(CCV_NNC_CLAMP_FORWARD, CCV_NNC_BACKEND_GPU_REF) &&
    3459
    1
        ccv_nnc_cmd_ok(CCV_NNC_CLAMP_BACKWARD, CCV_NNC_BACKEND_GPU_REF));
    3460
    1
      ccv_nnc_tensor_t* a = ccv_nnc_tensor_new(0, GPU_TENSOR_NCHW(000, 32F, 10, 100), 0);
    3461
    1
      ccv_nnc_tensor_t* b = ccv_nnc_tensor_new(0, GPU_TENSOR_NCHW(000, 32F, 10, 100), 0);
    3462
    1
      ccv_nnc_tensor_t* g = ccv_nnc_tensor_new(0, GPU_TENSOR_NCHW(000, 32F, 10, 100), 0);
    3463
    1
      ccv_nnc_tensor_t* da = ccv_nnc_tensor_new(0, GPU_TENSOR_NCHW(000, 32F, 10, 100), 0);
    3464
    1
      ccv_nnc_tensor_t* ha = ccv_nnc_tensor_new(0, CPU_TENSOR_NCHW(32F, 10, 100), 0);
    3465
    1
      ccv_nnc_tensor_t* hda = ccv_nnc_tensor_new(0, CPU_TENSOR_NCHW(32F, 10, 100), 0);
    3466
    1
      ccv_nnc_tensor_t* hb = ccv_nnc_tensor_new(0, CPU_TENSOR_NCHW(32F, 10, 100), 0);
    3467
    1
      ccv_nnc_tensor_t* hg = ccv_nnc_tensor_new(0, CPU_TENSOR_NCHW(32F, 10, 100), 0);
    3468
    1
      ccv_nnc_tensor_t* dat = ccv_nnc_tensor_new(0, CPU_TENSOR_NCHW(32F, 10, 100), 0);
    3469
    1
      dsfmt_t dsfmt;
    3470
    1
      dsfmt_init_gen_rand(&dsfmt, 0);
    3471
    1
      int i;
    3472
    1.00k
      for (i = 0; i < 1000; i++1.00k)
    3473
    1.00k
        ha->data.f32[i] = dsfmt_genrand_open_close(&dsfmt) * 10;
    3474
    1.00k
      for (i = 0; i < 1000; i++1.00k)
    3475
    1.00k
        hg->data.f32[i] = dsfmt_genrand_open_close(&dsfmt);
    3476
    1
      ccv_nnc_cmd_exec(CMD_DATA_TRANSFER_FORWARD(), ccv_nnc_no_hint, 0, TENSOR_LIST(ha, hg), TENSOR_LIST(a, g), 0);
    3477
    1
      ccv_nnc_cmd_exec(CMD_CLAMP_FORWARD(0, NAN), ccv_nnc_no_hint, 0, TENSOR_LIST(a), TENSOR_LIST(b), 0);
    3478
    1
      ccv_nnc_cmd_exec(CMD_CLAMP_BACKWARD(0, NAN), ccv_nnc_no_hint, 0, TENSOR_LIST(g, 0, b), TENSOR_LIST(da), 0);
    3479
    1
      ccv_nnc_cmd_exec(CMD_CLAMP_FORWARD(0, NAN), ccv_nnc_no_hint, 0, TENSOR_LIST(ha), TENSOR_LIST(hb), 0);
    3480
    1
      ccv_nnc_cmd_exec(CMD_CLAMP_BACKWARD(0, NAN), ccv_nnc_no_hint, 0, TENSOR_LIST(hg, 0, hb), TENSOR_LIST(dat), 0);
    3481
    1
      ccv_nnc_cmd_exec(CMD_DATA_TRANSFER_FORWARD(), ccv_nnc_no_hint, 0, TENSOR_LIST(da), TENSOR_LIST(hda), 0);
    3482
    1
      REQUIRE_TENSOR_EQ(dat, hda, "GPU computed output should be the same as CPU computed ones");
    3483
    1
      ccv_nnc_tensor_free(a);
    3484
    1
      ccv_nnc_tensor_free(b);
    3485
    1
      ccv_nnc_tensor_free(g);
    3486
    1
      ccv_nnc_tensor_free(da);
    3487
    1
      ccv_nnc_tensor_free(ha);
    3488
    1
      ccv_nnc_tensor_free(hb);
    3489
    1
      ccv_nnc_tensor_free(hg);
    3490
    1
      ccv_nnc_tensor_free(hda);
    3491
    1
      ccv_nnc_tensor_free(dat);
    3492
    1
    }
    3493
    3494
    TEST_CASE("scaled dot product attention with flash_attn")
    3495
    1
    {
    3496
    1
      GUARD_ELSE_RETURN(ccv_nnc_cmd_ok(CCV_NNC_SCALED_DOT_PRODUCT_ATTENTION_FORWARD, CCV_NNC_BACKEND_GPU_REF));
    3497
      // Bypass error: variable-sized object may not be initialized
    3498
    0
    #define num_long_trials 4
    3499
    0
    #define num_short_trials 2
    3500
    0
    #define num_trials (num_long_trials + num_short_trials)
    3501
    3502
    0
      for (int trial = 0; trial < num_trials; ++trial) {
    3503
    0
        int B_candidates[num_trials] = {  32,   12, 16, 1, 2, 1 };
    3504
    0
        int R_candidates[num_trials] = { 160,  256, 128, 77, 77, 5 };
    3505
    0
        int C_candidates[num_trials] = { 128,  128, 128, 128, 128, 5 };
    3506
    0
        int Hq_candidates[num_trials] = {   8,  8, 8, 8, 8, 32 };
    3507
    0
        int Hk_candidates[num_trials] = {   8,  8, 8, 8, 2, 8 };
    3508
    0
        int D_candidates[num_trials] = {  64, 40, 160, 224, 224, 128 };
    3509
    0
        int is_causal_candidates[num_trials] = {  1, 0, 1, 1, 0, 1 };
    3510
    3511
    0
        int B = B_candidates[trial];
    3512
    0
        int R = R_candidates[trial];
    3513
    0
        int C = C_candidates[trial];
    3514
    0
        int Hq = Hq_candidates[trial];
    3515
    0
        int Hk = Hk_candidates[trial];
    3516
    0
        int D = D_candidates[trial];
    3517
    0
        int is_causal = is_causal_candidates[trial];
    3518
    0
        float scale = 1.0 / sqrt((float)D);
    3519
    3520
    0
        GUARD_ELSE_RETURN(ccv_nnc_cmd_ok(CCV_NNC_SCALED_DOT_PRODUCT_ATTENTION_FORWARD, CCV_NNC_BACKEND_GPU_REF));
    3521
    0
        ccv_nnc_tensor_t* const q_tensor = ccv_nnc_tensor_new(0, CPU_TENSOR_NHWC(32F, B, R, Hq, D), 0);
    3522
    0
        ccv_nnc_tensor_t* const k_tensor = ccv_nnc_tensor_new(0, CPU_TENSOR_NHWC(32F, B, C, Hk, D), 0);
    3523
    0
        ccv_nnc_tensor_t* const v_tensor = ccv_nnc_tensor_new(0, CPU_TENSOR_NHWC(32F, B, C, Hk, D), 0);
    3524
    3525
    0
        for (int i = 0; i < B * R * Hq * D; ++i) {
    3526
    0
          q_tensor->data.f32[i] = (float)(i) / (float)(B * R * Hq * D);
    3527
    0
        }
    3528
    0
        for (int i = 0; i < B * C * Hk * D; ++i) {
    3529
    0
          k_tensor->data.f32[i] = (float)(i) / (float)(B * C * Hk * D);
    3530
    0
        }
    3531
    0
        for (int i = 0; i < B * C * Hk * D; ++i) {
    3532
    0
          v_tensor->data.f32[i] = (float)(i) / (float)(B * C * Hk * D);
    3533
    0
        }
    3534
    3535
    0
        ccv_nnc_tensor_t* const o_tensor = ccv_nnc_tensor_new(0, CPU_TENSOR_NHWC(32F, B, R, Hq, D), 0);
    3536
    0
        ccv_nnc_cmd_exec(CMD_SCALED_DOT_PRODUCT_ATTENTION_FORWARD(scale, is_causal), ccv_nnc_no_hint, 0, TENSOR_LIST(q_tensor, k_tensor, v_tensor, NULL, NULL, NULL), TENSOR_LIST(o_tensor, NULL), 0);
    3537
    0
        ccv_nnc_tensor_t* const q_tensor_f16 = ccv_nnc_tensor_new(0, CPU_TENSOR_NHWC(16F, B, R, Hq, D), 0);
    3538
    0
        ccv_nnc_tensor_t* const k_tensor_f16 = ccv_nnc_tensor_new(0, CPU_TENSOR_NHWC(16F, B, C, Hk, D), 0);
    3539
    0
        ccv_nnc_tensor_t* const v_tensor_f16 = ccv_nnc_tensor_new(0, CPU_TENSOR_NHWC(16F, B, C, Hk, D), 0);
    3540
    0
        ccv_nnc_cmd_exec(CMD_DATATYPE_CONVERSION_FORWARD(), ccv_nnc_no_hint, 0, TENSOR_LIST(q_tensor, k_tensor, v_tensor), TENSOR_LIST(q_tensor_f16, k_tensor_f16, v_tensor_f16), 0);
    3541
    3542
        // Why it there 000 in the beginning of the argument list for GPU_TENSOR_NHWC?
    3543
    0
        ccv_nnc_tensor_t* const gpu_q_tensor = ccv_nnc_tensor_new(0, GPU_TENSOR_NHWC(000, 16F, B, R, Hq, D), 0);
    3544
    0
        ccv_nnc_tensor_t* const gpu_k_tensor = ccv_nnc_tensor_new(0, GPU_TENSOR_NHWC(000, 16F, B, C, Hk, D), 0);
    3545
    0
        ccv_nnc_tensor_t* const gpu_v_tensor = ccv_nnc_tensor_new(0, GPU_TENSOR_NHWC(000, 16F, B, C, Hk, D), 0);
    3546
    0
        ccv_nnc_tensor_t* const gpu_o_tensor = ccv_nnc_tensor_new(0, GPU_TENSOR_NHWC(000, 16F, B, R, Hq, D), 0);
    3547
    0
        ccv_nnc_cmd_exec(CMD_DATA_TRANSFER_FORWARD(), ccv_nnc_no_hint, 0, TENSOR_LIST(q_tensor_f16, k_tensor_f16, v_tensor_f16), TENSOR_LIST(gpu_q_tensor, gpu_k_tensor, gpu_v_tensor), 0);
    3548
    3549
    0
        ccv_nnc_cmd_exec(CMD_SCALED_DOT_PRODUCT_ATTENTION_FORWARD(scale, is_causal), ccv_nnc_no_hint, 0, TENSOR_LIST(gpu_q_tensor, gpu_k_tensor, gpu_v_tensor, NULL, NULL, NULL), TENSOR_LIST(gpu_o_tensor, NULL), 0);
    3550
    3551
    0
        ccv_nnc_tensor_t* const copy_of_gpu_o_tensor_f16 = ccv_nnc_tensor_new(0, CPU_TENSOR_NHWC(16F, B, R, Hq, D), 0);
    3552
    0
        ccv_nnc_cmd_exec(CMD_DATA_TRANSFER_FORWARD(), ccv_nnc_no_hint, 0, TENSOR_LIST(gpu_o_tensor), TENSOR_LIST(copy_of_gpu_o_tensor_f16), 0);
    3553
    0
        ccv_nnc_tensor_t* const copy_of_gpu_o_tensor = ccv_nnc_tensor_new(0, CPU_TENSOR_NHWC(32F, B, R, Hq, D), 0);
    3554
    0
        ccv_nnc_cmd_exec(CMD_DATATYPE_CONVERSION_FORWARD(), ccv_nnc_no_hint, 0, TENSOR_LIST(copy_of_gpu_o_tensor_f16), TENSOR_LIST(copy_of_gpu_o_tensor), 0);
    3555
    3556
    0
        REQUIRE_ARRAY_EQ_WITH_TOLERANCE(float, copy_of_gpu_o_tensor->data.f32, o_tensor->data.f32, B * R * Hq * D, 3e-3, "GPU computed output should be the same as CPU computed ones");
    3557
    3558
    0
        ccv_nnc_tensor_free(o_tensor);
    3559
    0
        ccv_nnc_tensor_free(gpu_o_tensor);
    3560
    0
        ccv_nnc_tensor_free(copy_of_gpu_o_tensor);
    3561
    0
        ccv_nnc_tensor_free(copy_of_gpu_o_tensor_f16);
    3562
    0
        ccv_nnc_tensor_free(q_tensor);
    3563
    0
        ccv_nnc_tensor_free(k_tensor);
    3564
    0
        ccv_nnc_tensor_free(v_tensor);
    3565
    0
        ccv_nnc_tensor_free(q_tensor_f16);
    3566
    0
        ccv_nnc_tensor_free(k_tensor_f16);
    3567
    0
        ccv_nnc_tensor_free(v_tensor_f16);
    3568
    0
        ccv_nnc_tensor_free(gpu_q_tensor);
    3569
    0
        ccv_nnc_tensor_free(gpu_k_tensor);
    3570
    0
        ccv_nnc_tensor_free(gpu_v_tensor);
    3571
    0
      }
    3572
    0
    #undef num_long_trials
    3573
    0
    #undef num_short_trials
    3574
    0
    #undef num_trials
    3575
    0
    }
    3576
    3577
    TEST_CASE("scaled dot product attention with flash_attn varlen")
    3578
    1
    {
    3579
    1
      GUARD_ELSE_RETURN(ccv_nnc_cmd_ok(CCV_NNC_SCALED_DOT_PRODUCT_ATTENTION_FORWARD, CCV_NNC_BACKEND_GPU_REF));
    3580
    0
      const int B = 4;
    3581
    0
      const int Hq = 4;
    3582
    0
      const int Hk = 2;
    3583
    0
      const int D = 64;
    3584
    0
      const int q_offsets[2][5] = {
    3585
    0
        { 0, 5, 8, 15, 17 },
    3586
    0
        { 0, 4, 10, 13, 20 },
    3587
    0
      };
    3588
    0
      const int kv_offsets[2][5] = {
    3589
    0
        { 0, 6, 13, 16, 22 },
    3590
    0
        { 0, 4, 11, 15, 21 },
    3591
    0
      };
    3592
    0
      const int max_seqlen_q[2] = { 7, 7 };
    3593
    0
      const int max_seqlen_kv[2] = { 7, 7 };
    3594
    0
      for (int trial = 0; trial < 2; ++trial)
    3595
    0
      {
    3596
    0
        const int is_causal = trial;
    3597
    0
        const int total_q = q_offsets[trial][B];
    3598
    0
        const int total_k = kv_offsets[trial][B];
    3599
    0
        const float scale = 1.0 / sqrt((float)D);
    3600
    0
        ccv_nnc_tensor_t* const q_tensor = ccv_nnc_tensor_new(0, CPU_TENSOR_NHWC(32F, 1, total_q, Hq, D), 0);
    3601
    0
        ccv_nnc_tensor_t* const k_tensor = ccv_nnc_tensor_new(0, CPU_TENSOR_NHWC(32F, 1, total_k, Hk, D), 0);
    3602
    0
        ccv_nnc_tensor_t* const v_tensor = ccv_nnc_tensor_new(0, CPU_TENSOR_NHWC(32F, 1, total_k, Hk, D), 0);
    3603
    0
        for (int i = 0; i < total_q * Hq * D; ++i)
    3604
    0
          q_tensor->data.f32[i] = (float)((i * 17) % 97 - 48) / 256;
    3605
    0
        for (int i = 0; i < total_k * Hk * D; ++i)
    3606
    0
        {
    3607
    0
          k_tensor->data.f32[i] = (float)((i * 13) % 89 - 44) / 256;
    3608
    0
          v_tensor->data.f32[i] = (float)((i * 19) % 101 - 50) / 128;
    3609
    0
        }
    3610
    0
        ccv_nnc_tensor_t* const o_tensor_ref = ccv_nnc_tensor_new(0, CPU_TENSOR_NHWC(32F, 1, total_q, Hq, D), 0);
    3611
    0
        for (int b = 0; b < B; ++b)
    3612
    0
        {
    3613
    0
          const int q_start = q_offsets[trial][b];
    3614
    0
          const int k_start = kv_offsets[trial][b];
    3615
    0
          const int R = q_offsets[trial][b + 1] - q_start;
    3616
    0
          const int C = kv_offsets[trial][b + 1] - k_start;
    3617
    0
          ccv_nnc_tensor_t* const q_seq = ccv_nnc_tensor_new(0, CPU_TENSOR_NHWC(32F, 1, R, Hq, D), 0);
    3618
    0
          ccv_nnc_tensor_t* const k_seq = ccv_nnc_tensor_new(0, CPU_TENSOR_NHWC(32F, 1, C, Hk, D), 0);
    3619
    0
          ccv_nnc_tensor_t* const v_seq = ccv_nnc_tensor_new(0, CPU_TENSOR_NHWC(32F, 1, C, Hk, D), 0);
    3620
    0
          ccv_nnc_tensor_t* const o_seq = ccv_nnc_tensor_new(0, CPU_TENSOR_NHWC(32F, 1, R, Hq, D), 0);
    3621
    0
          memcpy(q_seq->data.f32, q_tensor->data.f32 + q_start * Hq * D, sizeof(float) * R * Hq * D);
    3622
    0
          memcpy(k_seq->data.f32, k_tensor->data.f32 + k_start * Hk * D, sizeof(float) * C * Hk * D);
    3623
    0
          memcpy(v_seq->data.f32, v_tensor->data.f32 + k_start * Hk * D, sizeof(float) * C * Hk * D);
    3624
    0
          ccv_nnc_cmd_exec(CMD_SCALED_DOT_PRODUCT_ATTENTION_FORWARD(scale, is_causal), ccv_nnc_no_hint, 0, TENSOR_LIST(q_seq, k_seq, v_seq), TENSOR_LIST(o_seq), 0);
    3625
    0
          memcpy(o_tensor_ref->data.f32 + q_start * Hq * D, o_seq->data.f32, sizeof(float) * R * Hq * D);
    3626
    0
          ccv_nnc_tensor_free(q_seq);
    3627
    0
          ccv_nnc_tensor_free(k_seq);
    3628
    0
          ccv_nnc_tensor_free(v_seq);
    3629
    0
          ccv_nnc_tensor_free(o_seq);
    3630
    0
        }
    3631
    0
        ccv_nnc_tensor_t* const h_q_seq_offsets = ccv_nnc_tensor_new(0, CPU_TENSOR_NHWC(32S, B + 1), 0);
    3632
    0
        ccv_nnc_tensor_t* const h_kv_seq_offsets = ccv_nnc_tensor_new(0, CPU_TENSOR_NHWC(32S, B + 1), 0);
    3633
    0
        for (int i = 0; i < B + 1; ++i)
    3634
    0
        {
    3635
    0
          h_q_seq_offsets->data.i32[i] = q_offsets[trial][i];
    3636
    0
          h_kv_seq_offsets->data.i32[i] = kv_offsets[trial][i];
    3637
    0
        }
    3638
    0
        ccv_nnc_cmd_t cmd = CMD_SCALED_DOT_PRODUCT_ATTENTION_FORWARD(scale, is_causal);
    3639
    0
        cmd.info.scaled_dot_product_attention.is_varlen = 1;
    3640
    0
        cmd.info.scaled_dot_product_attention.max_seqlen_q = max_seqlen_q[trial];
    3641
    0
        cmd.info.scaled_dot_product_attention.max_seqlen_kv = max_seqlen_kv[trial];
    3642
    0
        ccv_nnc_tensor_t* const o_tensor = ccv_nnc_tensor_new(0, CPU_TENSOR_NHWC(32F, 1, total_q, Hq, D), 0);
    3643
    0
        ccv_nnc_cmd_exec(cmd, ccv_nnc_no_hint, 0, TENSOR_LIST(q_tensor, k_tensor, v_tensor, NULL, NULL, NULL, h_q_seq_offsets, h_kv_seq_offsets), TENSOR_LIST(o_tensor, NULL), 0);
    3644
    0
        REQUIRE_ARRAY_EQ_WITH_TOLERANCE(float, o_tensor->data.f32, o_tensor_ref->data.f32, total_q * Hq * D, 1e-5, "varlen CPU computed output should match per-sequence CPU output when causal=%d", is_causal);
    3645
    0
        ccv_nnc_tensor_t* const q_tensor_f16 = ccv_nnc_tensor_new(0, CPU_TENSOR_NHWC(16F, 1, total_q, Hq, D), 0);
    3646
    0
        ccv_nnc_tensor_t* const k_tensor_f16 = ccv_nnc_tensor_new(0, CPU_TENSOR_NHWC(16F, 1, total_k, Hk, D), 0);
    3647
    0
        ccv_nnc_tensor_t* const v_tensor_f16 = ccv_nnc_tensor_new(0, CPU_TENSOR_NHWC(16F, 1, total_k, Hk, D), 0);
    3648
    0
        ccv_nnc_cmd_exec(CMD_DATATYPE_CONVERSION_FORWARD(), ccv_nnc_no_hint, 0, TENSOR_LIST(q_tensor, k_tensor, v_tensor), TENSOR_LIST(q_tensor_f16, k_tensor_f16, v_tensor_f16), 0);
    3649
    0
        ccv_nnc_tensor_t* const gpu_q_tensor = ccv_nnc_tensor_new(0, GPU_TENSOR_NHWC(000, 16F, 1, total_q, Hq, D), 0);
    3650
    0
        ccv_nnc_tensor_t* const gpu_k_tensor = ccv_nnc_tensor_new(0, GPU_TENSOR_NHWC(000, 16F, 1, total_k, Hk, D), 0);
    3651
    0
        ccv_nnc_tensor_t* const gpu_v_tensor = ccv_nnc_tensor_new(0, GPU_TENSOR_NHWC(000, 16F, 1, total_k, Hk, D), 0);
    3652
    0
        ccv_nnc_tensor_t* const gpu_q_seq_offsets = ccv_nnc_tensor_new(0, GPU_TENSOR_NHWC(000, 32S, B + 1), 0);
    3653
    0
        ccv_nnc_tensor_t* const gpu_kv_seq_offsets = ccv_nnc_tensor_new(0, GPU_TENSOR_NHWC(000, 32S, B + 1), 0);
    3654
    0
        ccv_nnc_tensor_t* const gpu_o_tensor = ccv_nnc_tensor_new(0, GPU_TENSOR_NHWC(000, 16F, 1, total_q, Hq, D), 0);
    3655
    0
        ccv_nnc_cmd_exec(CMD_DATA_TRANSFER_FORWARD(), ccv_nnc_no_hint, 0, TENSOR_LIST(q_tensor_f16, k_tensor_f16, v_tensor_f16, h_q_seq_offsets, h_kv_seq_offsets), TENSOR_LIST(gpu_q_tensor, gpu_k_tensor, gpu_v_tensor, gpu_q_seq_offsets, gpu_kv_seq_offsets), 0);
    3656
    0
        ccv_nnc_cmd_exec(cmd, ccv_nnc_no_hint, 0, TENSOR_LIST(gpu_q_tensor, gpu_k_tensor, gpu_v_tensor, NULL, NULL, NULL, gpu_q_seq_offsets, gpu_kv_seq_offsets), TENSOR_LIST(gpu_o_tensor, NULL), 0);
    3657
    0
        ccv_nnc_tensor_t* const copy_of_gpu_o_tensor_f16 = ccv_nnc_tensor_new(0, CPU_TENSOR_NHWC(16F, 1, total_q, Hq, D), 0);
    3658
    0
        ccv_nnc_cmd_exec(CMD_DATA_TRANSFER_FORWARD(), ccv_nnc_no_hint, 0, TENSOR_LIST(gpu_o_tensor), TENSOR_LIST(copy_of_gpu_o_tensor_f16), 0);
    3659
    0
        ccv_nnc_tensor_t* const copy_of_gpu_o_tensor = ccv_nnc_tensor_new(0, CPU_TENSOR_NHWC(32F, 1, total_q, Hq, D), 0);
    3660
    0
        ccv_nnc_cmd_exec(CMD_DATATYPE_CONVERSION_FORWARD(), ccv_nnc_no_hint, 0, TENSOR_LIST(copy_of_gpu_o_tensor_f16), TENSOR_LIST(copy_of_gpu_o_tensor), 0);
    3661
    0
        REQUIRE_ARRAY_EQ_WITH_TOLERANCE(float, copy_of_gpu_o_tensor->data.f32, o_tensor->data.f32, total_q * Hq * D, 5e-3, "varlen GPU computed output should match packed CPU output when causal=%d", is_causal);
    3662
    0
        ccv_nnc_tensor_free(q_tensor);
    3663
    0
        ccv_nnc_tensor_free(k_tensor);
    3664
    0
        ccv_nnc_tensor_free(v_tensor);
    3665
    0
        ccv_nnc_tensor_free(o_tensor_ref);
    3666
    0
        ccv_nnc_tensor_free(o_tensor);
    3667
    0
        ccv_nnc_tensor_free(q_tensor_f16);
    3668
    0
        ccv_nnc_tensor_free(k_tensor_f16);
    3669
    0
        ccv_nnc_tensor_free(v_tensor_f16);
    3670
    0
        ccv_nnc_tensor_free(h_q_seq_offsets);
    3671
    0
        ccv_nnc_tensor_free(h_kv_seq_offsets);
    3672
    0
        ccv_nnc_tensor_free(gpu_q_tensor);
    3673
    0
        ccv_nnc_tensor_free(gpu_k_tensor);
    3674
    0
        ccv_nnc_tensor_free(gpu_v_tensor);
    3675
    0
        ccv_nnc_tensor_free(gpu_q_seq_offsets);
    3676
    0
        ccv_nnc_tensor_free(gpu_kv_seq_offsets);
    3677
    0
        ccv_nnc_tensor_free(gpu_o_tensor);
    3678
    0
        ccv_nnc_tensor_free(copy_of_gpu_o_tensor_f16);
    3679
    0
        ccv_nnc_tensor_free(copy_of_gpu_o_tensor);
    3680
    0
      }
    3681
    0
    }
    3682
    3683
    TEST_CASE("scaled dot product attention with flash_attn in bfloat")
    3684
    1
    {
    3685
    1
      GUARD_ELSE_RETURN(ccv_nnc_cmd_ok(CCV_NNC_SCALED_DOT_PRODUCT_ATTENTION_FORWARD, CCV_NNC_BACKEND_GPU_REF));
    3686
      // Bypass error: variable-sized object may not be initialized
    3687
    0
    #define num_long_trials 4
    3688
    0
    #define num_short_trials 2
    3689
    0
    #define num_trials (num_long_trials + num_short_trials)
    3690
    3691
    0
      for (int trial = 0; trial < num_trials; ++trial) {
    3692
    0
        int B_candidates[num_trials] = {  32,   12, 16, 1, 2, 1 };
    3693
    0
        int R_candidates[num_trials] = { 160,  256, 128, 77, 77, 5 };
    3694
    0
        int C_candidates[num_trials] = { 128,  128, 128, 128, 128, 5 };
    3695
    0
        int Hq_candidates[num_trials] = {   8,  8, 8, 8, 8, 32 };
    3696
    0
        int Hk_candidates[num_trials] = {   8,  8, 8, 8, 2, 8 };
    3697
    0
        int D_candidates[num_trials] = {  64, 40, 160, 224, 224, 128 };
    3698
    0
        int is_causal_candidates[num_trials] = {  1, 0, 1, 1, 0, 1 };
    3699
    3700
    0
        int B = B_candidates[trial];
    3701
    0
        int R = R_candidates[trial];
    3702
    0
        int C = C_candidates[trial];
    3703
    0
        int Hq = Hq_candidates[trial];
    3704
    0
        int Hk = Hk_candidates[trial];
    3705
    0
        int D = D_candidates[trial];
    3706
    0
        int is_causal = is_causal_candidates[trial];
    3707
    0
        float scale = 1.0 / sqrt((float)D);
    3708
    3709
    0
        GUARD_ELSE_RETURN(ccv_nnc_cmd_ok(CCV_NNC_SCALED_DOT_PRODUCT_ATTENTION_FORWARD, CCV_NNC_BACKEND_GPU_REF));
    3710
    0
        ccv_nnc_tensor_t* const q_tensor = ccv_nnc_tensor_new(0, CPU_TENSOR_NHWC(32F, B, R, Hq, D), 0);
    3711
    0
        ccv_nnc_tensor_t* const k_tensor = ccv_nnc_tensor_new(0, CPU_TENSOR_NHWC(32F, B, C, Hk, D), 0);
    3712
    0
        ccv_nnc_tensor_t* const v_tensor = ccv_nnc_tensor_new(0, CPU_TENSOR_NHWC(32F, B, C, Hk, D), 0);
    3713
    3714
    0
        for (int i = 0; i < B * R * Hq * D; ++i) {
    3715
    0
          q_tensor->data.f32[i] = (float)(i) / (float)(B * R * Hq * D);
    3716
    0
        }
    3717
    0
        for (int i = 0; i < B * C * Hk * D; ++i) {
    3718
    0
          k_tensor->data.f32[i] = (float)(i) / (float)(B * C * Hk * D);
    3719
    0
        }
    3720
    0
        for (int i = 0; i < B * C * Hk * D; ++i) {
    3721
    0
          v_tensor->data.f32[i] = (float)(i) / (float)(B * C * Hk * D);
    3722
    0
        }
    3723
    3724
    0
        ccv_nnc_tensor_t* const o_tensor = ccv_nnc_tensor_new(0, CPU_TENSOR_NHWC(32F, B, R, Hq, D), 0);
    3725
    0
        ccv_nnc_cmd_exec(CMD_SCALED_DOT_PRODUCT_ATTENTION_FORWARD(scale, is_causal), ccv_nnc_no_hint, 0, TENSOR_LIST(q_tensor, k_tensor, v_tensor, NULL, NULL, NULL), TENSOR_LIST(o_tensor, NULL), 0);
    3726
    0
        ccv_nnc_tensor_t* const q_tensor_f16 = ccv_nnc_tensor_new(0, CPU_TENSOR_NHWC(16BF, B, R, Hq, D), 0);
    3727
    0
        ccv_nnc_tensor_t* const k_tensor_f16 = ccv_nnc_tensor_new(0, CPU_TENSOR_NHWC(16BF, B, C, Hk, D), 0);
    3728
    0
        ccv_nnc_tensor_t* const v_tensor_f16 = ccv_nnc_tensor_new(0, CPU_TENSOR_NHWC(16BF, B, C, Hk, D), 0);
    3729
    0
        ccv_nnc_cmd_exec(CMD_DATATYPE_CONVERSION_FORWARD(), ccv_nnc_no_hint, 0, TENSOR_LIST(q_tensor, k_tensor, v_tensor), TENSOR_LIST(q_tensor_f16, k_tensor_f16, v_tensor_f16), 0);
    3730
    3731
        // Why it there 000 in the beginning of the argument list for GPU_TENSOR_NHWC?
    3732
    0
        ccv_nnc_tensor_t* const gpu_q_tensor = ccv_nnc_tensor_new(0, GPU_TENSOR_NHWC(000, 16BF, B, R, Hq, D), 0);
    3733
    0
        ccv_nnc_tensor_t* const gpu_k_tensor = ccv_nnc_tensor_new(0, GPU_TENSOR_NHWC(000, 16BF, B, C, Hk, D), 0);
    3734
    0
        ccv_nnc_tensor_t* const gpu_v_tensor = ccv_nnc_tensor_new(0, GPU_TENSOR_NHWC(000, 16BF, B, C, Hk, D), 0);
    3735
    0
        ccv_nnc_tensor_t* const gpu_o_tensor = ccv_nnc_tensor_new(0, GPU_TENSOR_NHWC(000, 16BF, B, R, Hq, D), 0);
    3736
    0
        ccv_nnc_cmd_exec(CMD_DATA_TRANSFER_FORWARD(), ccv_nnc_no_hint, 0, TENSOR_LIST(q_tensor_f16, k_tensor_f16, v_tensor_f16), TENSOR_LIST(gpu_q_tensor, gpu_k_tensor, gpu_v_tensor), 0);
    3737
    3738
    0
        ccv_nnc_cmd_exec(CMD_SCALED_DOT_PRODUCT_ATTENTION_FORWARD(scale, is_causal), ccv_nnc_no_hint, 0, TENSOR_LIST(gpu_q_tensor, gpu_k_tensor, gpu_v_tensor, NULL, NULL, NULL), TENSOR_LIST(gpu_o_tensor, NULL), 0);
    3739
    3740
    0
        ccv_nnc_tensor_t* const copy_of_gpu_o_tensor_f16 = ccv_nnc_tensor_new(0, CPU_TENSOR_NHWC(16BF, B, R, Hq, D), 0);
    3741
    0
        ccv_nnc_cmd_exec(CMD_DATA_TRANSFER_FORWARD(), ccv_nnc_no_hint, 0, TENSOR_LIST(gpu_o_tensor), TENSOR_LIST(copy_of_gpu_o_tensor_f16), 0);
    3742
    0
        ccv_nnc_tensor_t* const copy_of_gpu_o_tensor = ccv_nnc_tensor_new(0, CPU_TENSOR_NHWC(32F, B, R, Hq, D), 0);
    3743
    0
        ccv_nnc_cmd_exec(CMD_DATATYPE_CONVERSION_FORWARD(), ccv_nnc_no_hint, 0, TENSOR_LIST(copy_of_gpu_o_tensor_f16), TENSOR_LIST(copy_of_gpu_o_tensor), 0);
    3744
    3745
    0
        REQUIRE_ARRAY_EQ_WITH_TOLERANCE(float, copy_of_gpu_o_tensor->data.f32, o_tensor->data.f32, B * R * Hq * D, 1e-2, "GPU computed output should be the same as CPU computed ones");
    3746
    3747
    0
        ccv_nnc_tensor_free(o_tensor);
    3748
    0
        ccv_nnc_tensor_free(gpu_o_tensor);
    3749
    0
        ccv_nnc_tensor_free(copy_of_gpu_o_tensor);
    3750
    0
        ccv_nnc_tensor_free(copy_of_gpu_o_tensor_f16);
    3751
    0
        ccv_nnc_tensor_free(q_tensor);
    3752
    0
        ccv_nnc_tensor_free(k_tensor);
    3753
    0
        ccv_nnc_tensor_free(v_tensor);
    3754
    0
        ccv_nnc_tensor_free(q_tensor_f16);
    3755
    0
        ccv_nnc_tensor_free(k_tensor_f16);
    3756
    0
        ccv_nnc_tensor_free(v_tensor_f16);
    3757
    0
        ccv_nnc_tensor_free(gpu_q_tensor);
    3758
    0
        ccv_nnc_tensor_free(gpu_k_tensor);
    3759
    0
        ccv_nnc_tensor_free(gpu_v_tensor);
    3760
    0
      }
    3761
    0
    #undef num_long_trials
    3762
    0
    #undef num_short_trials
    3763
    0
    #undef num_trials
    3764
    0
    }
    3765
    3766
    TEST_CASE("scaled dot product attention + unify head with flash_attn")
    3767
    1
    {
    3768
    1
      GUARD_ELSE_RETURN(ccv_nnc_cmd_ok(CCV_NNC_SCALED_DOT_PRODUCT_ATTENTION_FORWARD, CCV_NNC_BACKEND_GPU_REF));
    3769
    0
      ccv_nnc_symbolic_graph_t* const sdp_symbolic_graph = ccv_nnc_symbolic_graph_new();
    3770
    0
      ccv_nnc_tensor_symbol_t q = ccv_nnc_tensor_symbol_new(sdp_symbolic_graph, CPU_TENSOR_NHWC(32F, 32, 128, 8, 64), "q");
    3771
    0
      ccv_nnc_tensor_symbol_t k = ccv_nnc_tensor_symbol_new(sdp_symbolic_graph, CPU_TENSOR_NHWC(32F, 32, 128, 8, 64), "k");
    3772
    0
      ccv_nnc_tensor_symbol_t v = ccv_nnc_tensor_symbol_new(sdp_symbolic_graph, CPU_TENSOR_NHWC(32F, 32, 128, 8, 64), "v");
    3773
    0
      ccv_nnc_tensor_symbol_t w = ccv_nnc_tensor_symbol_new(sdp_symbolic_graph, CPU_TENSOR_NHWC(32F, 512, 512), "w");
    3774
    0
      ccv_nnc_tensor_symbol_t bias = ccv_nnc_tensor_symbol_new(sdp_symbolic_graph, CPU_TENSOR_NHWC(32F, 512), "bias");
    3775
    0
      ccv_nnc_tensor_symbol_t c = ccv_nnc_tensor_symbol_new(sdp_symbolic_graph, CPU_TENSOR_NHWC(32F, 32, 128, 8, 64), "c");
    3776
    0
      ccv_nnc_tensor_symbol_t r = ccv_nnc_tensor_symbol_new(sdp_symbolic_graph, CPU_TENSOR_NHWC(32F, 32, 128, 512), "r");
    3777
    0
      ccv_nnc_graph_exec_symbol_new(sdp_symbolic_graph, CMD_SCALED_DOT_PRODUCT_ATTENTION_FORWARD(1.0 / 8, 0), TENSOR_SYMBOL_LIST(q, k, v, NO_TENSOR_SYMBOL, w, bias), TENSOR_SYMBOL_LIST(r, NO_TENSOR_SYMBOL, c), "scaled_dot_product_attention");
    3778
    0
      ccv_nnc_graph_exec_symbol_autogen(sdp_symbolic_graph, 0, 0, CCV_NNC_AUTOGEN_ALL_EXECS | CCV_NNC_AUTOGEN_SOURCES_AND_DESTINATIONS);
    3779
    0
      ccv_nnc_graph_t* sdp_graph = 0;
    3780
    0
      ccv_nnc_tensor_arena_t* sdp_tensor_arena = 0;
    3781
    0
      ccv_nnc_graph_exec_arena_t* sdp_graph_exec_arena = 0;
    3782
    0
      ccv_nnc_symbolic_graph_compile(sdp_symbolic_graph, ccv_nnc_default_compile_params, 0, 0, 0, 0, SYMBOLIC_GRAPH_SOURCES(sdp_symbolic_graph), SYMBOLIC_GRAPH_DESTINATIONS(sdp_symbolic_graph), &sdp_graph, &sdp_tensor_arena, &sdp_graph_exec_arena);
    3783
    0
      ccv_nnc_tensor_t* const q_tensor = ccv_nnc_tensor_from_symbol(sdp_tensor_arena, q);
    3784
    0
      ccv_nnc_tensor_t* const k_tensor = ccv_nnc_tensor_from_symbol(sdp_tensor_arena, k);
    3785
    0
      ccv_nnc_tensor_t* const v_tensor = ccv_nnc_tensor_from_symbol(sdp_tensor_arena, v);
    3786
    0
      ccv_nnc_tensor_t* const w_tensor = ccv_nnc_tensor_from_symbol(sdp_tensor_arena, w);
    3787
    0
      ccv_nnc_tensor_t* const bias_tensor = ccv_nnc_tensor_from_symbol(sdp_tensor_arena, bias);
    3788
    0
      dsfmt_t dsfmt;
    3789
    0
      int i;
    3790
    0
      dsfmt_init_gen_rand(&dsfmt, 1);
    3791
    0
      for (i = 0; i < 32 * 8 * 128 * 64; i++)
    3792
    0
        q_tensor->data.f32[i] = dsfmt_genrand_open_close(&dsfmt);
    3793
    0
      for (i = 0; i < 32 * 8 * 128 * 64; i++)
    3794
    0
        k_tensor->data.f32[i] = dsfmt_genrand_open_close(&dsfmt);
    3795
    0
      for (i = 0; i < 32 * 8 * 128 * 64; i++)
    3796
    0
        v_tensor->data.f32[i] = dsfmt_genrand_open_close(&dsfmt);
    3797
    0
      for (i = 0; i < 512 * 512; i++)
    3798
    0
        w_tensor->data.f32[i] = dsfmt_genrand_open_close(&dsfmt) / sqrtf(512);
    3799
    0
      for (i = 0; i < 512; i++)
    3800
    0
        bias_tensor->data.f32[i] = dsfmt_genrand_open_close(&dsfmt);
    3801
    0
      ccv_nnc_tensor_t* const q_tensor_f16 = ccv_nnc_tensor_new(0, CPU_TENSOR_NHWC(16F, 32, 128, 8, 64), 0);
    3802
    0
      ccv_nnc_tensor_t* const k_tensor_f16 = ccv_nnc_tensor_new(0, CPU_TENSOR_NHWC(16F, 32, 128, 8, 64), 0);
    3803
    0
      ccv_nnc_tensor_t* const v_tensor_f16 = ccv_nnc_tensor_new(0, CPU_TENSOR_NHWC(16F, 32, 128, 8, 64), 0);
    3804
    0
      ccv_nnc_tensor_t* const w_tensor_f16 = ccv_nnc_tensor_new(0, CPU_TENSOR_NHWC(16F, 512, 512), 0);
    3805
    0
      ccv_nnc_tensor_t* const bias_tensor_f16 = ccv_nnc_tensor_new(0, CPU_TENSOR_NHWC(16F, 512), 0);
    3806
    0
      ccv_nnc_cmd_exec(CMD_DATATYPE_CONVERSION_FORWARD(), ccv_nnc_no_hint, 0, TENSOR_LIST(q_tensor, k_tensor, v_tensor, w_tensor, bias_tensor), TENSOR_LIST(q_tensor_f16, k_tensor_f16, v_tensor_f16, w_tensor_f16, bias_tensor_f16), 0);
    3807
    0
      ccv_nnc_symbolic_graph_t* const g_symbolic_graph = ccv_nnc_symbolic_graph_new();
    3808
    0
      ccv_nnc_tensor_symbol_t gq = ccv_nnc_tensor_symbol_new(g_symbolic_graph, GPU_TENSOR_NHWC(000, 16F, 32, 128, 8, 64), "q");
    3809
    0
      ccv_nnc_tensor_symbol_t gk = ccv_nnc_tensor_symbol_new(g_symbolic_graph, GPU_TENSOR_NHWC(000, 16F, 32, 128, 8, 64), "k");
    3810
    0
      ccv_nnc_tensor_symbol_t gv = ccv_nnc_tensor_symbol_new(g_symbolic_graph, GPU_TENSOR_NHWC(000, 16F, 32, 128, 8, 64), "v");
    3811
    0
      ccv_nnc_tensor_symbol_t gw = ccv_nnc_tensor_symbol_new(g_symbolic_graph, GPU_TENSOR_NHWC(000, 16F, 512, 512), "w");
    3812
    0
      ccv_nnc_tensor_symbol_t gbias = ccv_nnc_tensor_symbol_new(g_symbolic_graph, GPU_TENSOR_NHWC(000, 16F, 512), "bias");
    3813
    0
      ccv_nnc_tensor_symbol_t gc = ccv_nnc_tensor_symbol_new(g_symbolic_graph, GPU_TENSOR_NHWC(000, 16F, 32, 128, 8, 64), "c");
    3814
    0
      ccv_nnc_tensor_symbol_t gr = ccv_nnc_tensor_symbol_new(g_symbolic_graph, GPU_TENSOR_NHWC(000, 16F, 32, 128, 512), "r");
    3815
    0
      ccv_nnc_graph_exec_symbol_new(g_symbolic_graph, CMD_SCALED_DOT_PRODUCT_ATTENTION_FORWARD(1.0 / 8, 0), TENSOR_SYMBOL_LIST(gq, gk, gv, NO_TENSOR_SYMBOL, gw, gbias), TENSOR_SYMBOL_LIST(gr, NO_TENSOR_SYMBOL, gc), "scaled_dot_product_attention");
    3816
    0
      ccv_nnc_graph_exec_symbol_autogen(g_symbolic_graph, 0, 0, CCV_NNC_AUTOGEN_ALL_EXECS | CCV_NNC_AUTOGEN_SOURCES_AND_DESTINATIONS);
    3817
    0
      ccv_nnc_graph_t* g_graph = 0;
    3818
    0
      ccv_nnc_tensor_arena_t* g_tensor_arena = 0;
    3819
    0
      ccv_nnc_graph_exec_arena_t* g_graph_exec_arena = 0;
    3820
    0
      ccv_nnc_symbolic_graph_compile(g_symbolic_graph, ccv_nnc_default_compile_params, 0, 0, 0, 0, SYMBOLIC_GRAPH_SOURCES(g_symbolic_graph), SYMBOLIC_GRAPH_DESTINATIONS(g_symbolic_graph), &g_graph, &g_tensor_arena, &g_graph_exec_arena);
    3821
    0
      ccv_nnc_tensor_t* const gq_tensor = ccv_nnc_tensor_from_symbol(g_tensor_arena, gq);
    3822
    0
      ccv_nnc_tensor_t* const gk_tensor = ccv_nnc_tensor_from_symbol(g_tensor_arena, gk);
    3823
    0
      ccv_nnc_tensor_t* const gv_tensor = ccv_nnc_tensor_from_symbol(g_tensor_arena, gv);
    3824
    0
      ccv_nnc_tensor_t* const gw_tensor = ccv_nnc_tensor_from_symbol(g_tensor_arena, gw);
    3825
    0
      ccv_nnc_tensor_t* const gbias_tensor = ccv_nnc_tensor_from_symbol(g_tensor_arena, gbias);
    3826
    0
      ccv_nnc_cmd_exec(CMD_DATA_TRANSFER_FORWARD(), ccv_nnc_no_hint, 0, TENSOR_LIST(q_tensor_f16, k_tensor_f16, v_tensor_f16, w_tensor_f16, bias_tensor_f16), TENSOR_LIST(gq_tensor, gk_tensor, gv_tensor, gw_tensor, gbias_tensor), 0);
    3827
    0
      ccv_nnc_graph_run(sdp_graph, 0, TRAVERSE_FULL, 0, 0);
    3828
    0
      ccv_nnc_graph_run(g_graph, 0, TRAVERSE_FULL, 0, 0);
    3829
    0
      ccv_nnc_tensor_t* const r_tensor = ccv_nnc_tensor_from_symbol(sdp_tensor_arena, r);
    3830
    0
      ccv_nnc_tensor_t* const o_tensor = ccv_nnc_tensor_from_symbol(sdp_tensor_arena, c);
    3831
    0
      ccv_nnc_tensor_t* const gc_tensor = ccv_nnc_tensor_from_symbol(g_tensor_arena, gc);
    3832
    0
      ccv_nnc_tensor_t* const gr_tensor = ccv_nnc_tensor_from_symbol(g_tensor_arena, gr);
    3833
    0
      ccv_nnc_tensor_t* const ho_f16 = ccv_nnc_tensor_new(0, CPU_TENSOR_NHWC(16F, 32, 128, 8, 64), 0);
    3834
    0
      ccv_nnc_tensor_t* const hr_f16 = ccv_nnc_tensor_new(0, CPU_TENSOR_NHWC(16F, 32, 128, 512), 0);
    3835
    0
      ccv_nnc_cmd_exec(CMD_DATA_TRANSFER_FORWARD(), ccv_nnc_no_hint, 0, TENSOR_LIST(gc_tensor, gr_tensor), TENSOR_LIST(ho_f16, hr_f16), 0);
    3836
    0
      ccv_nnc_tensor_t* const ho = ccv_nnc_tensor_new(0, CPU_TENSOR_NHWC(32F, 32, 128, 8, 64), 0);
    3837
    0
      ccv_nnc_tensor_t* const hr = ccv_nnc_tensor_new(0, CPU_TENSOR_NHWC(32F, 32, 128, 512), 0);
    3838
    0
      ccv_nnc_cmd_exec(CMD_DATATYPE_CONVERSION_FORWARD(), ccv_nnc_no_hint, 0, TENSOR_LIST(ho_f16, hr_f16), TENSOR_LIST(ho, hr), 0);
    3839
    0
      REQUIRE_ARRAY_EQ_WITH_TOLERANCE(float, o_tensor->data.f32, ho->data.f32, 32 * 128 * 8 * 64, 3e-3, "graph computed result should match scaled dot product attention op result");
    3840
    0
      REQUIRE_ARRAY_EQ_WITH_TOLERANCE(float, r_tensor->data.f32, hr->data.f32, 32 * 128 * 512, 3e-2, "graph computed result should match scaled dot product attention op result");
    3841
    0
      ccv_nnc_symbolic_graph_free(sdp_symbolic_graph);
    3842
    0
      ccv_nnc_tensor_arena_free(sdp_tensor_arena);
    3843
    0
      ccv_nnc_graph_exec_arena_free(sdp_graph_exec_arena);
    3844
    0
      ccv_nnc_graph_free(sdp_graph);
    3845
    0
      ccv_nnc_symbolic_graph_free(g_symbolic_graph);
    3846
    0
      ccv_nnc_tensor_arena_free(g_tensor_arena);
    3847
    0
      ccv_nnc_graph_exec_arena_free(g_graph_exec_arena);
    3848
    0
      ccv_nnc_graph_free(g_graph);
    3849
    0
      ccv_nnc_tensor_free(ho);
    3850
    0
      ccv_nnc_tensor_free(hr);
    3851
    0
      ccv_nnc_tensor_free(ho_f16);
    3852
    0
      ccv_nnc_tensor_free(hr_f16);
    3853
    0
      ccv_nnc_tensor_free(q_tensor_f16);
    3854
    0
      ccv_nnc_tensor_free(k_tensor_f16);
    3855
    0
      ccv_nnc_tensor_free(v_tensor_f16);
    3856
    0
      ccv_nnc_tensor_free(w_tensor_f16);
    3857
    0
      ccv_nnc_tensor_free(bias_tensor_f16);
    3858
    0
    }
    3859
    3860
    TEST_CASE("scaled dot product attention + unify head with flash_attn in bfloat")
    3861
    1
    {
    3862
    1
      GUARD_ELSE_RETURN(ccv_nnc_cmd_ok(CCV_NNC_SCALED_DOT_PRODUCT_ATTENTION_FORWARD, CCV_NNC_BACKEND_GPU_REF));
    3863
    0
      ccv_nnc_symbolic_graph_t* const sdp_symbolic_graph = ccv_nnc_symbolic_graph_new();
    3864
    0
      ccv_nnc_tensor_symbol_t q = ccv_nnc_tensor_symbol_new(sdp_symbolic_graph, CPU_TENSOR_NHWC(32F, 32, 128, 8, 64), "q");
    3865
    0
      ccv_nnc_tensor_symbol_t k = ccv_nnc_tensor_symbol_new(sdp_symbolic_graph, CPU_TENSOR_NHWC(32F, 32, 128, 8, 64), "k");
    3866
    0
      ccv_nnc_tensor_symbol_t v = ccv_nnc_tensor_symbol_new(sdp_symbolic_graph, CPU_TENSOR_NHWC(32F, 32, 128, 8, 64), "v");
    3867
    0
      ccv_nnc_tensor_symbol_t w = ccv_nnc_tensor_symbol_new(sdp_symbolic_graph, CPU_TENSOR_NHWC(32F, 512, 512), "w");
    3868
    0
      ccv_nnc_tensor_symbol_t bias = ccv_nnc_tensor_symbol_new(sdp_symbolic_graph, CPU_TENSOR_NHWC(32F, 512), "bias");
    3869
    0
      ccv_nnc_tensor_symbol_t c = ccv_nnc_tensor_symbol_new(sdp_symbolic_graph, CPU_TENSOR_NHWC(32F, 32, 128, 8, 64), "c");
    3870
    0
      ccv_nnc_tensor_symbol_t r = ccv_nnc_tensor_symbol_new(sdp_symbolic_graph, CPU_TENSOR_NHWC(32F, 32, 128, 512), "r");
    3871
    0
      ccv_nnc_graph_exec_symbol_new(sdp_symbolic_graph, CMD_SCALED_DOT_PRODUCT_ATTENTION_FORWARD(1.0 / 8, 0), TENSOR_SYMBOL_LIST(q, k, v, NO_TENSOR_SYMBOL, w, bias), TENSOR_SYMBOL_LIST(r, NO_TENSOR_SYMBOL, c), "scaled_dot_product_attention");
    3872
    0
      ccv_nnc_graph_exec_symbol_autogen(sdp_symbolic_graph, 0, 0, CCV_NNC_AUTOGEN_ALL_EXECS | CCV_NNC_AUTOGEN_SOURCES_AND_DESTINATIONS);
    3873
    0
      ccv_nnc_graph_t* sdp_graph = 0;
    3874
    0
      ccv_nnc_tensor_arena_t* sdp_tensor_arena = 0;
    3875
    0
      ccv_nnc_graph_exec_arena_t* sdp_graph_exec_arena = 0;
    3876
    0
      ccv_nnc_symbolic_graph_compile(sdp_symbolic_graph, ccv_nnc_default_compile_params, 0, 0, 0, 0, SYMBOLIC_GRAPH_SOURCES(sdp_symbolic_graph), SYMBOLIC_GRAPH_DESTINATIONS(sdp_symbolic_graph), &sdp_graph, &sdp_tensor_arena, &sdp_graph_exec_arena);
    3877
    0
      ccv_nnc_tensor_t* const q_tensor = ccv_nnc_tensor_from_symbol(sdp_tensor_arena, q);
    3878
    0
      ccv_nnc_tensor_t* const k_tensor = ccv_nnc_tensor_from_symbol(sdp_tensor_arena, k);
    3879
    0
      ccv_nnc_tensor_t* const v_tensor = ccv_nnc_tensor_from_symbol(sdp_tensor_arena, v);
    3880
    0
      ccv_nnc_tensor_t* const w_tensor = ccv_nnc_tensor_from_symbol(sdp_tensor_arena, w);
    3881
    0
      ccv_nnc_tensor_t* const bias_tensor = ccv_nnc_tensor_from_symbol(sdp_tensor_arena, bias);
    3882
    0
      dsfmt_t dsfmt;
    3883
    0
      int i;
    3884
    0
      dsfmt_init_gen_rand(&dsfmt, 1);
    3885
    0
      for (i = 0; i < 32 * 8 * 128 * 64; i++)
    3886
    0
        q_tensor->data.f32[i] = dsfmt_genrand_open_close(&dsfmt);
    3887
    0
      for (i = 0; i < 32 * 8 * 128 * 64; i++)
    3888
    0
        k_tensor->data.f32[i] = dsfmt_genrand_open_close(&dsfmt);
    3889
    0
      for (i = 0; i < 32 * 8 * 128 * 64; i++)
    3890
    0
        v_tensor->data.f32[i] = dsfmt_genrand_open_close(&dsfmt);
    3891
    0
      for (i = 0; i < 512 * 512; i++)
    3892
    0
        w_tensor->data.f32[i] = dsfmt_genrand_open_close(&dsfmt) / sqrtf(512);
    3893
    0
      for (i = 0; i < 512; i++)
    3894
    0
        bias_tensor->data.f32[i] = dsfmt_genrand_open_close(&dsfmt);
    3895
    0
      ccv_nnc_tensor_t* const q_tensor_f16 = ccv_nnc_tensor_new(0, CPU_TENSOR_NHWC(16BF, 32, 128, 8, 64), 0);
    3896
    0
      ccv_nnc_tensor_t* const k_tensor_f16 = ccv_nnc_tensor_new(0, CPU_TENSOR_NHWC(16BF, 32, 128, 8, 64), 0);
    3897
    0
      ccv_nnc_tensor_t* const v_tensor_f16 = ccv_nnc_tensor_new(0, CPU_TENSOR_NHWC(16BF, 32, 128, 8, 64), 0);
    3898
    0
      ccv_nnc_tensor_t* const w_tensor_f16 = ccv_nnc_tensor_new(0, CPU_TENSOR_NHWC(16BF, 512, 512), 0);
    3899
    0
      ccv_nnc_tensor_t* const bias_tensor_f16 = ccv_nnc_tensor_new(0, CPU_TENSOR_NHWC(16BF, 512), 0);
    3900
    0
      ccv_nnc_cmd_exec(CMD_DATATYPE_CONVERSION_FORWARD(), ccv_nnc_no_hint, 0, TENSOR_LIST(q_tensor, k_tensor, v_tensor, w_tensor, bias_tensor), TENSOR_LIST(q_tensor_f16, k_tensor_f16, v_tensor_f16, w_tensor_f16, bias_tensor_f16), 0);
    3901
    0
      ccv_nnc_symbolic_graph_t* const g_symbolic_graph = ccv_nnc_symbolic_graph_new();
    3902
    0
      ccv_nnc_tensor_symbol_t gq = ccv_nnc_tensor_symbol_new(g_symbolic_graph, GPU_TENSOR_NHWC(000, 16BF, 32, 128, 8, 64), "q");
    3903
    0
      ccv_nnc_tensor_symbol_t gk = ccv_nnc_tensor_symbol_new(g_symbolic_graph, GPU_TENSOR_NHWC(000, 16BF, 32, 128, 8, 64), "k");
    3904
    0
      ccv_nnc_tensor_symbol_t gv = ccv_nnc_tensor_symbol_new(g_symbolic_graph, GPU_TENSOR_NHWC(000, 16BF, 32, 128, 8, 64), "v");
    3905
    0
      ccv_nnc_tensor_symbol_t gw = ccv_nnc_tensor_symbol_new(g_symbolic_graph, GPU_TENSOR_NHWC(000, 16BF, 512, 512), "w");
    3906
    0
      ccv_nnc_tensor_symbol_t gbias = ccv_nnc_tensor_symbol_new(g_symbolic_graph, GPU_TENSOR_NHWC(000, 16BF, 512), "bias");
    3907
    0
      ccv_nnc_tensor_symbol_t gc = ccv_nnc_tensor_symbol_new(g_symbolic_graph, GPU_TENSOR_NHWC(000, 16BF, 32, 128, 8, 64), "c");
    3908
    0
      ccv_nnc_tensor_symbol_t gr = ccv_nnc_tensor_symbol_new(g_symbolic_graph, GPU_TENSOR_NHWC(000, 16BF, 32, 128, 512), "r");
    3909
    0
      ccv_nnc_graph_exec_symbol_new(g_symbolic_graph, CMD_SCALED_DOT_PRODUCT_ATTENTION_FORWARD(1.0 / 8, 0), TENSOR_SYMBOL_LIST(gq, gk, gv, NO_TENSOR_SYMBOL, gw, gbias), TENSOR_SYMBOL_LIST(gr, NO_TENSOR_SYMBOL, gc), "scaled_dot_product_attention");
    3910
    0
      ccv_nnc_graph_exec_symbol_autogen(g_symbolic_graph, 0, 0, CCV_NNC_AUTOGEN_ALL_EXECS | CCV_NNC_AUTOGEN_SOURCES_AND_DESTINATIONS);
    3911
    0
      ccv_nnc_graph_t* g_graph = 0;
    3912
    0
      ccv_nnc_tensor_arena_t* g_tensor_arena = 0;
    3913
    0
      ccv_nnc_graph_exec_arena_t* g_graph_exec_arena = 0;
    3914
    0
      ccv_nnc_symbolic_graph_compile(g_symbolic_graph, ccv_nnc_default_compile_params, 0, 0, 0, 0, SYMBOLIC_GRAPH_SOURCES(g_symbolic_graph), SYMBOLIC_GRAPH_DESTINATIONS(g_symbolic_graph), &g_graph, &g_tensor_arena, &g_graph_exec_arena);
    3915
    0
      ccv_nnc_tensor_t* const gq_tensor = ccv_nnc_tensor_from_symbol(g_tensor_arena, gq);
    3916
    0
      ccv_nnc_tensor_t* const gk_tensor = ccv_nnc_tensor_from_symbol(g_tensor_arena, gk);
    3917
    0
      ccv_nnc_tensor_t* const gv_tensor = ccv_nnc_tensor_from_symbol(g_tensor_arena, gv);
    3918
    0
      ccv_nnc_tensor_t* const gw_tensor = ccv_nnc_tensor_from_symbol(g_tensor_arena, gw);
    3919
    0
      ccv_nnc_tensor_t* const gbias_tensor = ccv_nnc_tensor_from_symbol(g_tensor_arena, gbias);
    3920
    0
      ccv_nnc_cmd_exec(CMD_DATA_TRANSFER_FORWARD(), ccv_nnc_no_hint, 0, TENSOR_LIST(q_tensor_f16, k_tensor_f16, v_tensor_f16, w_tensor_f16, bias_tensor_f16), TENSOR_LIST(gq_tensor, gk_tensor, gv_tensor, gw_tensor, gbias_tensor), 0);
    3921
    0
      ccv_nnc_graph_run(sdp_graph, 0, TRAVERSE_FULL, 0, 0);
    3922
    0
      ccv_nnc_graph_run(g_graph, 0, TRAVERSE_FULL, 0, 0);
    3923
    0
      ccv_nnc_tensor_t* const r_tensor = ccv_nnc_tensor_from_symbol(sdp_tensor_arena, r);
    3924
    0
      ccv_nnc_tensor_t* const o_tensor = ccv_nnc_tensor_from_symbol(sdp_tensor_arena, c);
    3925
    0
      ccv_nnc_tensor_t* const gc_tensor = ccv_nnc_tensor_from_symbol(g_tensor_arena, gc);
    3926
    0
      ccv_nnc_tensor_t* const gr_tensor = ccv_nnc_tensor_from_symbol(g_tensor_arena, gr);
    3927
    0
      ccv_nnc_tensor_t* const ho_f16 = ccv_nnc_tensor_new(0, CPU_TENSOR_NHWC(16BF, 32, 128, 8, 64), 0);
    3928
    0
      ccv_nnc_tensor_t* const hr_f16 = ccv_nnc_tensor_new(0, CPU_TENSOR_NHWC(16BF, 32, 128, 512), 0);
    3929
    0
      ccv_nnc_cmd_exec(CMD_DATA_TRANSFER_FORWARD(), ccv_nnc_no_hint, 0, TENSOR_LIST(gc_tensor, gr_tensor), TENSOR_LIST(ho_f16, hr_f16), 0);
    3930
    0
      ccv_nnc_tensor_t* const ho = ccv_nnc_tensor_new(0, CPU_TENSOR_NHWC(32F, 32, 128, 8, 64), 0);
    3931
    0
      ccv_nnc_tensor_t* const hr = ccv_nnc_tensor_new(0, CPU_TENSOR_NHWC(32F, 32, 128, 512), 0);
    3932
    0
      ccv_nnc_cmd_exec(CMD_DATATYPE_CONVERSION_FORWARD(), ccv_nnc_no_hint, 0, TENSOR_LIST(ho_f16, hr_f16), TENSOR_LIST(ho, hr), 0);
    3933
    0
      REQUIRE_ARRAY_EQ_WITH_TOLERANCE(float, o_tensor->data.f32, ho->data.f32, 32 * 128 * 8 * 64, 1e-2, "graph computed result should match scaled dot product attention op result");
    3934
    0
      REQUIRE_ARRAY_EQ_WITH_TOLERANCE(float, r_tensor->data.f32, hr->data.f32, 32 * 128 * 512, 1e-1, "graph computed result should match scaled dot product attention op result");
    3935
    0
      ccv_nnc_symbolic_graph_free(sdp_symbolic_graph);
    3936
    0
      ccv_nnc_tensor_arena_free(sdp_tensor_arena);
    3937
    0
      ccv_nnc_graph_exec_arena_free(sdp_graph_exec_arena);
    3938
    0
      ccv_nnc_graph_free(sdp_graph);
    3939
    0
      ccv_nnc_symbolic_graph_free(g_symbolic_graph);
    3940
    0
      ccv_nnc_tensor_arena_free(g_tensor_arena);
    3941
    0
      ccv_nnc_graph_exec_arena_free(g_graph_exec_arena);
    3942
    0
      ccv_nnc_graph_free(g_graph);
    3943
    0
      ccv_nnc_tensor_free(ho);
    3944
    0
      ccv_nnc_tensor_free(hr);
    3945
    0
      ccv_nnc_tensor_free(ho_f16);
    3946
    0
      ccv_nnc_tensor_free(hr_f16);
    3947
    0
      ccv_nnc_tensor_free(q_tensor_f16);
    3948
    0
      ccv_nnc_tensor_free(k_tensor_f16);
    3949
    0
      ccv_nnc_tensor_free(v_tensor_f16);
    3950
    0
      ccv_nnc_tensor_free(w_tensor_f16);
    3951
    0
      ccv_nnc_tensor_free(bias_tensor_f16);
    3952
    0
    }
    3953
    3954
    TEST_CASE("scaled dot product attention gradient with flash_attn")
    3955
    1
    {
    3956
    1
      GUARD_ELSE_RETURN(ccv_nnc_cmd_ok(CCV_NNC_SCALED_DOT_PRODUCT_ATTENTION_FORWARD, CCV_NNC_BACKEND_GPU_REF) &&
    3957
    1
        ccv_nnc_cmd_ok(CCV_NNC_SCALED_DOT_PRODUCT_ATTENTION_BACKWARD, CCV_NNC_BACKEND_GPU_REF));
    3958
    0
    #define num_long_trials 8
    3959
    0
    #define num_short_trials 4
    3960
    0
    #define num_trials (num_long_trials + num_short_trials)
    3961
    3962
    0
      dsfmt_t dsfmt;
    3963
    0
      dsfmt_init_gen_rand(&dsfmt, 10);
    3964
    0
      for (int trial = 0; trial < num_trials; ++trial) {
    3965
    0
        const int B_candidates[num_trials] = {  32,   12, 16, 1, 2, 1, 32,   12, 16, 1, 2, 1 };
    3966
    0
        const int R_candidates[num_trials] = { 160,  256, 128, 77, 77, 5, 160,  256, 128, 77, 77, 5 };
    3967
    0
        const int C_candidates[num_trials] = { 128,  128, 128, 128, 128, 5, 128,  128, 128, 128, 128, 5 };
    3968
    0
        const int Hq_candidates[num_trials] = {   8,  8, 8, 8, 8, 32, 8,  8, 8, 8, 8, 32 };
    3969
    0
        const int Hk_candidates[num_trials] = {   8,  8, 8, 8, 2, 8, 8,  8, 8, 8, 2, 8 };
    3970
    0
        const int D_candidates[num_trials] = {  64, 40, 160, 192, 256, 128, 64, 40, 160, 192, 256, 128 };
    3971
    0
        const int is_causal_candidates[num_trials] = {  1, 0, 1, 1, 0, 1, 1, 0, 1, 1, 0, 1 };
    3972
    0
        const int deterministic_candidates[num_trials] = {  0, 0, 0, 0, 0, 0, 1, 1, 1, 1, 1, 1 };
    3973
    3974
    0
        const int B = B_candidates[trial];
    3975
    0
        const int R = R_candidates[trial];
    3976
    0
        const int C = C_candidates[trial];
    3977
    0
        const int Hq = Hq_candidates[trial];
    3978
    0
        const int Hk = Hk_candidates[trial];
    3979
    0
        const int D = D_candidates[trial];
    3980
    0
        const int is_causal = is_causal_candidates[trial];
    3981
    0
        const int deterministic = deterministic_candidates[trial];
    3982
    0
        const float scale = 1.0 / sqrt((float)D);
    3983
    3984
    0
        ccv_nnc_tensor_t* const q_tensor = ccv_nnc_tensor_new(0, CPU_TENSOR_NHWC(32F, B, R, Hq, D), 0);
    3985
    0
        ccv_nnc_tensor_t* const k_tensor = ccv_nnc_tensor_new(0, CPU_TENSOR_NHWC(32F, B, C, Hk, D), 0);
    3986
    0
        ccv_nnc_tensor_t* const v_tensor = ccv_nnc_tensor_new(0, CPU_TENSOR_NHWC(32F, B, C, Hk, D), 0);
    3987
    0
        ccv_nnc_tensor_t* const dq_tensor = ccv_nnc_tensor_new(0, CPU_TENSOR_NHWC(32F, B, R, Hq, D), 0);
    3988
    0
        ccv_nnc_tensor_t* const dk_tensor = ccv_nnc_tensor_new(0, CPU_TENSOR_NHWC(32F, B, C, Hk, D), 0);
    3989
    0
        ccv_nnc_tensor_t* const dv_tensor = ccv_nnc_tensor_new(0, CPU_TENSOR_NHWC(32F, B, C, Hk, D), 0);
    3990
    3991
    0
        for (int i = 0; i < B * R * Hq * D; ++i) {
    3992
    0
          q_tensor->data.f32[i] = dsfmt_genrand_open_close(&dsfmt);
    3993
    0
        }
    3994
    0
        for (int i = 0; i < B * C * Hk * D; ++i) {
    3995
    0
          k_tensor->data.f32[i] = dsfmt_genrand_open_close(&dsfmt);
    3996
    0
        }
    3997
    0
        for (int i = 0; i < B * C * Hk * D; ++i) {
    3998
    0
          v_tensor->data.f32[i] = dsfmt_genrand_open_close(&dsfmt);
    3999
    0
        }
    4000
    4001
    0
        ccv_nnc_tensor_t* const do_tensor = ccv_nnc_tensor_new(0, CPU_TENSOR_NHWC(32F, B, R, Hq, D), 0);
    4002
    0
        for (int i = 0; i < B * R * Hq * D; ++i) {
    4003
    0
          do_tensor->data.f32[i] = dsfmt_genrand_open_close(&dsfmt);
    4004
    0
        }
    4005
    0
        ccv_nnc_cmd_exec(CMD_SCALED_DOT_PRODUCT_ATTENTION_BACKWARD(scale, is_causal), ccv_nnc_no_hint, 0, TENSOR_LIST(do_tensor, 0, 0, q_tensor, k_tensor, v_tensor), TENSOR_LIST(dq_tensor, dk_tensor, dv_tensor), 0);
    4006
    0
        ccv_nnc_tensor_t* const q_tensor_f16 = ccv_nnc_tensor_new(0, CPU_TENSOR_NHWC(16F, B, R, Hq, D), 0);
    4007
    0
        ccv_nnc_tensor_t* const k_tensor_f16 = ccv_nnc_tensor_new(0, CPU_TENSOR_NHWC(16F, B, C, Hk, D), 0);
    4008
    0
        ccv_nnc_tensor_t* const v_tensor_f16 = ccv_nnc_tensor_new(0, CPU_TENSOR_NHWC(16F, B, C, Hk, D), 0);
    4009
    0
        ccv_nnc_tensor_t* const do_tensor_f16 = ccv_nnc_tensor_new(0, CPU_TENSOR_NHWC(16F, B, R, Hq, D), 0);
    4010
    0
        ccv_nnc_cmd_exec(CMD_DATATYPE_CONVERSION_FORWARD(), ccv_nnc_no_hint, 0, TENSOR_LIST(q_tensor, k_tensor, v_tensor, do_tensor), TENSOR_LIST(q_tensor_f16, k_tensor_f16, v_tensor_f16, do_tensor_f16), 0);
    4011
    4012
    0
        ccv_nnc_tensor_t* const gpu_q_tensor = ccv_nnc_tensor_new(0, GPU_TENSOR_NHWC(000, 16F, B, R, Hq, D), 0);
    4013
    0
        ccv_nnc_tensor_t* const gpu_k_tensor = ccv_nnc_tensor_new(0, GPU_TENSOR_NHWC(000, 16F, B, C, Hk, D), 0);
    4014
    0
        ccv_nnc_tensor_t* const gpu_v_tensor = ccv_nnc_tensor_new(0, GPU_TENSOR_NHWC(000, 16F, B, C, Hk, D), 0);
    4015
    0
        ccv_nnc_tensor_t* const gpu_o_tensor = ccv_nnc_tensor_new(0, GPU_TENSOR_NHWC(000, 16F, B, R, Hq, D), 0);
    4016
    0
        ccv_nnc_tensor_t* const gpu_do_tensor = ccv_nnc_tensor_new(0, GPU_TENSOR_NHWC(000, 16F, B, R, Hq, D), 0);
    4017
    0
        ccv_nnc_tensor_t* const gpu_dq_tensor = ccv_nnc_tensor_new(0, GPU_TENSOR_NHWC(000, 16F, B, R, Hq, D), 0);
    4018
    0
        ccv_nnc_tensor_t* const gpu_dk_tensor = ccv_nnc_tensor_new(0, GPU_TENSOR_NHWC(000, 16F, B, C, Hk, D), 0);
    4019
    0
        ccv_nnc_tensor_t* const gpu_dv_tensor = ccv_nnc_tensor_new(0, GPU_TENSOR_NHWC(000, 16F, B, C, Hk, D), 0);
    4020
    0
        ccv_nnc_cmd_exec(CMD_DATA_TRANSFER_FORWARD(), ccv_nnc_no_hint, 0, TENSOR_LIST(q_tensor_f16, k_tensor_f16, v_tensor_f16, do_tensor_f16), TENSOR_LIST(gpu_q_tensor, gpu_k_tensor, gpu_v_tensor, gpu_do_tensor), 0);
    4021
    4022
    0
        ccv_nnc_tensor_t* const gpu_softmax_lse = ccv_nnc_tensor_new(0, GPU_TENSOR_NHWC(000, 32F, B, Hq, R), 0);
    4023
    0
        ccv_nnc_cmd_exec(CMD_SCALED_DOT_PRODUCT_ATTENTION_FORWARD(scale, is_causal), ccv_nnc_no_hint, 0, TENSOR_LIST(gpu_q_tensor, gpu_k_tensor, gpu_v_tensor, NULL, NULL, NULL), TENSOR_LIST(gpu_o_tensor, gpu_softmax_lse), 0);
    4024
    4025
    0
        ccv_nnc_cmd_t cmd = CMD_SCALED_DOT_PRODUCT_ATTENTION_BACKWARD(scale, is_causal);
    4026
    0
        cmd.info.scaled_dot_product_attention.deterministic = deterministic;
    4027
    0
        ccv_nnc_cmd_exec(cmd, ccv_nnc_no_hint, 0, TENSOR_LIST(gpu_do_tensor, 0, 0, gpu_q_tensor, gpu_k_tensor, gpu_v_tensor, 0, 0, 0, gpu_o_tensor, gpu_softmax_lse), TENSOR_LIST(gpu_dq_tensor, gpu_dk_tensor, gpu_dv_tensor), 0);
    4028
    4029
    0
        ccv_nnc_tensor_t* const copy_of_gpu_dq_tensor_f16 = ccv_nnc_tensor_new(0, CPU_TENSOR_NHWC(16F, B, R, Hq, D), 0);
    4030
    0
        ccv_nnc_tensor_t* const copy_of_gpu_dk_tensor_f16 = ccv_nnc_tensor_new(0, CPU_TENSOR_NHWC(16F, B, C, Hk, D), 0);
    4031
    0
        ccv_nnc_tensor_t* const copy_of_gpu_dv_tensor_f16 = ccv_nnc_tensor_new(0, CPU_TENSOR_NHWC(16F, B, C, Hk, D), 0);
    4032
    0
        ccv_nnc_cmd_exec(CMD_DATA_TRANSFER_FORWARD(), ccv_nnc_no_hint, 0, TENSOR_LIST(gpu_dq_tensor, gpu_dk_tensor, gpu_dv_tensor), TENSOR_LIST(copy_of_gpu_dq_tensor_f16, copy_of_gpu_dk_tensor_f16, copy_of_gpu_dv_tensor_f16), 0);
    4033
    4034
    0
        ccv_nnc_tensor_t* const copy_of_gpu_dq_tensor = ccv_nnc_tensor_new(0, CPU_TENSOR_NHWC(32F, B, R, Hq, D), 0);
    4035
    0
        ccv_nnc_tensor_t* const copy_of_gpu_dk_tensor = ccv_nnc_tensor_new(0, CPU_TENSOR_NHWC(32F, B, C, Hk, D), 0);
    4036
    0
        ccv_nnc_tensor_t* const copy_of_gpu_dv_tensor = ccv_nnc_tensor_new(0, CPU_TENSOR_NHWC(32F, B, C, Hk, D), 0);
    4037
    0
        ccv_nnc_cmd_exec(CMD_DATATYPE_CONVERSION_FORWARD(), ccv_nnc_no_hint, 0, TENSOR_LIST(copy_of_gpu_dq_tensor_f16, copy_of_gpu_dk_tensor_f16, copy_of_gpu_dv_tensor_f16), TENSOR_LIST(copy_of_gpu_dq_tensor, copy_of_gpu_dk_tensor, copy_of_gpu_dv_tensor), 0);
    4038
    4039
    0
        REQUIRE_ARRAY_EQ_WITH_TOLERANCE(float, copy_of_gpu_dq_tensor->data.f32, dq_tensor->data.f32, B * R * Hq * D, 1e-3, "scaled dot product attention result should be the same");
    4040
    0
        REQUIRE_ARRAY_EQ_WITH_TOLERANCE(float, copy_of_gpu_dk_tensor->data.f32, dk_tensor->data.f32, B * C * Hk * D, 3e-3, "scaled dot product attention result should be the same");
    4041
    0
        REQUIRE_ARRAY_EQ_WITH_TOLERANCE(float, copy_of_gpu_dv_tensor->data.f32, dv_tensor->data.f32, B * C * Hk * D, 6e-3, "GPU computed output should be the same as CPU computed ones");
    4042
    4043
    0
        ccv_nnc_tensor_free(do_tensor);
    4044
    0
        ccv_nnc_tensor_free(gpu_do_tensor);
    4045
    0
        ccv_nnc_tensor_free(gpu_o_tensor);
    4046
    0
        ccv_nnc_tensor_free(copy_of_gpu_dq_tensor_f16);
    4047
    0
        ccv_nnc_tensor_free(copy_of_gpu_dk_tensor_f16);
    4048
    0
        ccv_nnc_tensor_free(copy_of_gpu_dv_tensor_f16);
    4049
    0
        ccv_nnc_tensor_free(copy_of_gpu_dq_tensor);
    4050
    0
        ccv_nnc_tensor_free(copy_of_gpu_dk_tensor);
    4051
    0
        ccv_nnc_tensor_free(copy_of_gpu_dv_tensor);
    4052
    0
        ccv_nnc_tensor_free(q_tensor);
    4053
    0
        ccv_nnc_tensor_free(k_tensor);
    4054
    0
        ccv_nnc_tensor_free(v_tensor);
    4055
    0
        ccv_nnc_tensor_free(q_tensor_f16);
    4056
    0
        ccv_nnc_tensor_free(k_tensor_f16);
    4057
    0
        ccv_nnc_tensor_free(v_tensor_f16);
    4058
    0
        ccv_nnc_tensor_free(do_tensor_f16);
    4059
    0
        ccv_nnc_tensor_free(gpu_q_tensor);
    4060
    0
        ccv_nnc_tensor_free(gpu_k_tensor);
    4061
    0
        ccv_nnc_tensor_free(gpu_v_tensor);
    4062
    0
        ccv_nnc_tensor_free(dq_tensor);
    4063
    0
        ccv_nnc_tensor_free(dk_tensor);
    4064
    0
        ccv_nnc_tensor_free(dv_tensor);
    4065
    0
        ccv_nnc_tensor_free(gpu_dq_tensor);
    4066
    0
        ccv_nnc_tensor_free(gpu_dk_tensor);
    4067
    0
        ccv_nnc_tensor_free(gpu_dv_tensor);
    4068
    0
        ccv_nnc_tensor_free(gpu_softmax_lse);
    4069
    0
      }
    4070
    0
    #undef num_long_trials
    4071
    0
    #undef num_short_trials
    4072
    0
    #undef num_trials
    4073
    0
    }
    4074
    4075
    static ccv_nnc_tensor_param_t _ccv_nnc_cmul_mixed_params(const int memory, const int datatype)
    4076
    38
    {
    4077
    38
      return (ccv_nnc_tensor_param_t){
    4078
    38
        .type = memory,
    4079
    38
        .format = CCV_TENSOR_FORMAT_NCHW,
    4080
    38
        .datatype = datatype,
    4081
    38
        .dim = { 2, 3, 10, 0 },
    4082
    38
      };
    4083
    38
    }
    4084
    4085
    static int _ccv_nnc_cmul_round_to_datatype(const ccv_nnc_tensor_t* const source, ccv_nnc_tensor_t* const typed, ccv_nnc_tensor_t* const rounded, const int count)
    4086
    14
    {
    4087
    14
      int i;
    4088
    14
      switch (typed->info.datatype)
    4089
    14
      {
    4090
    1
        case CCV_32F:
    4091
    61
          for (i = 0; i < count; i++60)
    4092
    60
          {
    4093
    60
            typed->data.f32[i] = source->data.f32[i];
    4094
    60
            rounded->data.f32[i] = source->data.f32[i];
    4095
    60
          }
    4096
    1
          return CCV_NNC_EXEC_SUCCESS;
    4097
    7
        case CCV_16F: {
    4098
    7
          int status = ccv_nnc_cmd_exec(CMD_DATATYPE_CONVERSION_FORWARD(), ccv_nnc_no_hint, 0, TENSOR_LIST((ccv_nnc_tensor_t*)source), TENSOR_LIST(typed), 0);
    4099
    7
          if (status != CCV_NNC_EXEC_SUCCESS)
    4100
    0
            return status;
    4101
    7
          return ccv_nnc_cmd_exec(CMD_DATATYPE_CONVERSION_FORWARD(), ccv_nnc_no_hint, 0, TENSOR_LIST(typed), TENSOR_LIST(rounded), 0);
    4102
    7
        }
    4103
    6
        case CCV_16BF:
    4104
    6
          ccv_float_to_bfloat(source->data.f32, (uint16_t*)typed->data.f16, count);
    4105
    6
          ccv_bfloat_to_float((uint16_t*)typed->data.f16, rounded->data.f32, count);
    4106
    6
          return CCV_NNC_EXEC_SUCCESS;
    4107
    14
      }
    4108
    0
      return CCV_NNC_EXEC_INVALID;
    4109
    14
    }
    4110
    4111
    static int _ccv_nnc_cmul_convert_to_float(const ccv_nnc_tensor_t* const typed, ccv_nnc_tensor_t* const rounded, const int count)
    4112
    5
    {
    4113
    5
      int i;
    4114
    5
      switch (typed->info.datatype)
    4115
    5
      {
    4116
    1
        case CCV_32F:
    4117
    61
          for (i = 0; i < count; i++60)
    4118
    60
            rounded->data.f32[i] = typed->data.f32[i];
    4119
    1
          return CCV_NNC_EXEC_SUCCESS;
    4120
    2
        case CCV_16F:
    4121
    2
          return ccv_nnc_cmd_exec(CMD_DATATYPE_CONVERSION_FORWARD(), ccv_nnc_no_hint, 0, TENSOR_LIST((ccv_nnc_tensor_t*)typed), TENSOR_LIST(rounded), 0);
    4122
    2
        case CCV_16BF:
    4123
    2
          ccv_bfloat_to_float((uint16_t*)typed->data.f16, rounded->data.f32, count);
    4124
    2
          return CCV_NNC_EXEC_SUCCESS;
    4125
    5
      }
    4126
    0
      return CCV_NNC_EXEC_INVALID;
    4127
    5
    }
    4128
    4129
    static int _ccv_nnc_cmul_mixed_precision_case(const int backend, const int a_datatype, const int b_datatype, const int c_datatype, float* const max_diff)
    4130
    3
    {
    4131
    3
      const int count = 2 * 3 * 10;
    4132
    3
      ccv_nnc_tensor_param_t ha32_params = _ccv_nnc_cmul_mixed_params(CCV_TENSOR_CPU_MEMORY, CCV_32F);
    4133
    3
      ccv_nnc_tensor_param_t hb32_params = _ccv_nnc_cmul_mixed_params(CCV_TENSOR_CPU_MEMORY, CCV_32F);
    4134
    3
      ccv_nnc_tensor_param_t hc32_params = _ccv_nnc_cmul_mixed_params(CCV_TENSOR_CPU_MEMORY, CCV_32F);
    4135
    3
      ccv_nnc_tensor_param_t ha_params = _ccv_nnc_cmul_mixed_params(CCV_TENSOR_CPU_MEMORY, a_datatype);
    4136
    3
      ccv_nnc_tensor_param_t hb_params = _ccv_nnc_cmul_mixed_params(CCV_TENSOR_CPU_MEMORY, b_datatype);
    4137
    3
      ccv_nnc_tensor_param_t hc_params = _ccv_nnc_cmul_mixed_params(CCV_TENSOR_CPU_MEMORY, c_datatype);
    4138
    3
      ccv_nnc_tensor_param_t ga_params = _ccv_nnc_cmul_mixed_params(CCV_TENSOR_GPU_MEMORY | 000, a_datatype);
    4139
    3
      ccv_nnc_tensor_param_t gb_params = _ccv_nnc_cmul_mixed_params(CCV_TENSOR_GPU_MEMORY | 000, b_datatype);
    4140
    3
      ccv_nnc_tensor_param_t gc_params = _ccv_nnc_cmul_mixed_params(CCV_TENSOR_GPU_MEMORY | 000, c_datatype);
    4141
    3
      ccv_nnc_tensor_t* const ha32 = ccv_nnc_tensor_new(0, ha32_params, 0);
    4142
    3
      ccv_nnc_tensor_t* const hb32 = ccv_nnc_tensor_new(0, hb32_params, 0);
    4143
    3
      ccv_nnc_tensor_t* const ha = ccv_nnc_tensor_new(0, ha_params, 0);
    4144
    3
      ccv_nnc_tensor_t* const hb = ccv_nnc_tensor_new(0, hb_params, 0);
    4145
    3
      ccv_nnc_tensor_t* const hc = ccv_nnc_tensor_new(0, hc_params, 0);
    4146
    3
      ccv_nnc_tensor_t* const ha_rounded = ccv_nnc_tensor_new(0, ha32_params, 0);
    4147
    3
      ccv_nnc_tensor_t* const hb_rounded = ccv_nnc_tensor_new(0, hb32_params, 0);
    4148
    3
      ccv_nnc_tensor_t* const expected = ccv_nnc_tensor_new(0, hc32_params, 0);
    4149
    3
      ccv_nnc_tensor_t* const expected_rounded = ccv_nnc_tensor_new(0, hc32_params, 0);
    4150
    3
      ccv_nnc_tensor_t* const actual = ccv_nnc_tensor_new(0, hc32_params, 0);
    4151
    3
      ccv_nnc_tensor_t* const expected_typed = ccv_nnc_tensor_new(0, hc_params, 0);
    4152
    3
      ccv_nnc_tensor_t* const ga = ccv_nnc_tensor_new(0, ga_params, 0);
    4153
    3
      ccv_nnc_tensor_t* const gb = ccv_nnc_tensor_new(0, gb_params, 0);
    4154
    3
      ccv_nnc_tensor_t* const gc = ccv_nnc_tensor_new(0, gc_params, 0);
    4155
    3
      int i;
    4156
    183
      for (i = 0; i < count; i++180)
    4157
    180
      {
    4158
    180
        ha32->data.f32[i] = (float)((i % 13) - 6) * 0.07f;
    4159
    180
        hb32->data.f32[i] = (float)(((i * 5 + 3) % 17) - 8) * 0.05f;
    4160
    180
      }
    4161
    3
      int status = _ccv_nnc_cmul_round_to_datatype(ha32, ha, ha_rounded, count);
    4162
    3
      if (status == CCV_NNC_EXEC_SUCCESS)
    4163
    3
        status = _ccv_nnc_cmul_round_to_datatype(hb32, hb, hb_rounded, count);
    4164
    3
      if (status == CCV_NNC_EXEC_SUCCESS)
    4165
    3
        status = ccv_nnc_cmd_exec(CMD_DATA_TRANSFER_FORWARD(), ccv_nnc_no_hint, 0, TENSOR_LIST(ha, hb), TENSOR_LIST(ga, gb), 0);
    4166
    3
      if (status == CCV_NNC_EXEC_SUCCESS)
    4167
    3
      {
    4168
    3
        ccv_nnc_cmd_t cmd = CMD_CMUL_FORWARD();
    4169
    3
        cmd.backend = backend;
    4170
    3
        status = ccv_nnc_cmd_exec(cmd, ccv_nnc_no_hint, 0, TENSOR_LIST(ga, gb), TENSOR_LIST(gc), 0);
    4171
    3
      }
    4172
    3
      if (status == CCV_NNC_EXEC_SUCCESS)
    4173
    3
        status = ccv_nnc_cmd_exec(CMD_DATA_TRANSFER_FORWARD(), ccv_nnc_no_hint, 0, TENSOR_LIST(gc), TENSOR_LIST(hc), 0);
    4174
    3
      if (status == CCV_NNC_EXEC_SUCCESS)
    4175
    3
        status = _ccv_nnc_cmul_convert_to_float(hc, actual, count);
    4176
    3
      if (status == CCV_NNC_EXEC_SUCCESS)
    4177
    3
      {
    4178
    93
        for (i = 0; i < count; i += 290)
    4179
    90
        {
    4180
    90
          const float a0 = ha_rounded->data.f32[i];
    4181
    90
          const float a1 = ha_rounded->data.f32[i + 1];
    4182
    90
          const float b0 = hb_rounded->data.f32[i];
    4183
    90
          const float b1 = hb_rounded->data.f32[i + 1];
    4184
    90
          expected->data.f32[i] = a0 * b0 - a1 * b1;
    4185
    90
          expected->data.f32[i + 1] = a0 * b1 + a1 * b0;
    4186
    90
        }
    4187
    3
        status = _ccv_nnc_cmul_round_to_datatype(expected, expected_typed, expected_rounded, count);
    4188
    3
      }
    4189
    3
      *max_diff = 0;
    4190
    3
      if (status == CCV_NNC_EXEC_SUCCESS)
    4191
    183
        for (i = 0; 3i < count; i++180)
    4192
    180
          *max_diff = ccv_max(*max_diff, fabsf(actual->data.f32[i] - expected_rounded->data.f32[i]));
    4193
    3
      ccv_nnc_tensor_free(ha32);
    4194
    3
      ccv_nnc_tensor_free(hb32);
    4195
    3
      ccv_nnc_tensor_free(ha);
    4196
    3
      ccv_nnc_tensor_free(hb);
    4197
    3
      ccv_nnc_tensor_free(hc);
    4198
    3
      ccv_nnc_tensor_free(ha_rounded);
    4199
    3
      ccv_nnc_tensor_free(hb_rounded);
    4200
    3
      ccv_nnc_tensor_free(expected);
    4201
    3
      ccv_nnc_tensor_free(expected_rounded);
    4202
    3
      ccv_nnc_tensor_free(actual);
    4203
    3
      ccv_nnc_tensor_free(expected_typed);
    4204
    3
      ccv_nnc_tensor_free(ga);
    4205
    3
      ccv_nnc_tensor_free(gb);
    4206
    3
      ccv_nnc_tensor_free(gc);
    4207
    3
      return status;
    4208
    3
    }
    4209
    4210
    static int _ccv_nnc_cmul_mixed_precision_backward_case(const int backend, const int g_datatype, const int a_datatype, const int b_datatype, const int c_datatype, const int d_datatype, float* const max_diff_c, float* const max_diff_d)
    4211
    1
    {
    4212
    1
      const int count = 2 * 3 * 10;
    4213
    1
      ccv_nnc_tensor_param_t h32_params = _ccv_nnc_cmul_mixed_params(CCV_TENSOR_CPU_MEMORY, CCV_32F);
    4214
    1
      ccv_nnc_tensor_param_t hg_params = _ccv_nnc_cmul_mixed_params(CCV_TENSOR_CPU_MEMORY, g_datatype);
    4215
    1
      ccv_nnc_tensor_param_t ha_params = _ccv_nnc_cmul_mixed_params(CCV_TENSOR_CPU_MEMORY, a_datatype);
    4216
    1
      ccv_nnc_tensor_param_t hb_params = _ccv_nnc_cmul_mixed_params(CCV_TENSOR_CPU_MEMORY, b_datatype);
    4217
    1
      ccv_nnc_tensor_param_t hc_params = _ccv_nnc_cmul_mixed_params(CCV_TENSOR_CPU_MEMORY, c_datatype);
    4218
    1
      ccv_nnc_tensor_param_t hd_params = _ccv_nnc_cmul_mixed_params(CCV_TENSOR_CPU_MEMORY, d_datatype);
    4219
    1
      ccv_nnc_tensor_param_t gg_params = _ccv_nnc_cmul_mixed_params(CCV_TENSOR_GPU_MEMORY | 000, g_datatype);
    4220
    1
      ccv_nnc_tensor_param_t ga_params = _ccv_nnc_cmul_mixed_params(CCV_TENSOR_GPU_MEMORY | 000, a_datatype);
    4221
    1
      ccv_nnc_tensor_param_t gb_params = _ccv_nnc_cmul_mixed_params(CCV_TENSOR_GPU_MEMORY | 000, b_datatype);
    4222
    1
      ccv_nnc_tensor_param_t gc_params = _ccv_nnc_cmul_mixed_params(CCV_TENSOR_GPU_MEMORY | 000, c_datatype);
    4223
    1
      ccv_nnc_tensor_param_t gd_params = _ccv_nnc_cmul_mixed_params(CCV_TENSOR_GPU_MEMORY | 000, d_datatype);
    4224
    1
      ccv_nnc_tensor_t* const hg32 = ccv_nnc_tensor_new(0, h32_params, 0);
    4225
    1
      ccv_nnc_tensor_t* const ha32 = ccv_nnc_tensor_new(0, h32_params, 0);
    4226
    1
      ccv_nnc_tensor_t* const hb32 = ccv_nnc_tensor_new(0, h32_params, 0);
    4227
    1
      ccv_nnc_tensor_t* const hg = ccv_nnc_tensor_new(0, hg_params, 0);
    4228
    1
      ccv_nnc_tensor_t* const ha = ccv_nnc_tensor_new(0, ha_params, 0);
    4229
    1
      ccv_nnc_tensor_t* const hb = ccv_nnc_tensor_new(0, hb_params, 0);
    4230
    1
      ccv_nnc_tensor_t* const hc = ccv_nnc_tensor_new(0, hc_params, 0);
    4231
    1
      ccv_nnc_tensor_t* const hd = ccv_nnc_tensor_new(0, hd_params, 0);
    4232
    1
      ccv_nnc_tensor_t* const hg_rounded = ccv_nnc_tensor_new(0, h32_params, 0);
    4233
    1
      ccv_nnc_tensor_t* const ha_rounded = ccv_nnc_tensor_new(0, h32_params, 0);
    4234
    1
      ccv_nnc_tensor_t* const hb_rounded = ccv_nnc_tensor_new(0, h32_params, 0);
    4235
    1
      ccv_nnc_tensor_t* const expected_c = ccv_nnc_tensor_new(0, h32_params, 0);
    4236
    1
      ccv_nnc_tensor_t* const expected_d = ccv_nnc_tensor_new(0, h32_params, 0);
    4237
    1
      ccv_nnc_tensor_t* const expected_c_rounded = ccv_nnc_tensor_new(0, h32_params, 0);
    4238
    1
      ccv_nnc_tensor_t* const expected_d_rounded = ccv_nnc_tensor_new(0, h32_params, 0);
    4239
    1
      ccv_nnc_tensor_t* const actual_c = ccv_nnc_tensor_new(0, h32_params, 0);
    4240
    1
      ccv_nnc_tensor_t* const actual_d = ccv_nnc_tensor_new(0, h32_params, 0);
    4241
    1
      ccv_nnc_tensor_t* const expected_c_typed = ccv_nnc_tensor_new(0, hc_params, 0);
    4242
    1
      ccv_nnc_tensor_t* const expected_d_typed = ccv_nnc_tensor_new(0, hd_params, 0);
    4243
    1
      ccv_nnc_tensor_t* const gg = ccv_nnc_tensor_new(0, gg_params, 0);
    4244
    1
      ccv_nnc_tensor_t* const ga = ccv_nnc_tensor_new(0, ga_params, 0);
    4245
    1
      ccv_nnc_tensor_t* const gb = ccv_nnc_tensor_new(0, gb_params, 0);
    4246
    1
      ccv_nnc_tensor_t* const gc = ccv_nnc_tensor_new(0, gc_params, 0);
    4247
    1
      ccv_nnc_tensor_t* const gd = ccv_nnc_tensor_new(0, gd_params, 0);
    4248
    1
      int i;
    4249
    61
      for (i = 0; i < count; i++60)
    4250
    60
      {
    4251
    60
        hg32->data.f32[i] = (float)(((i * 7 + 1) % 19) - 9) * 0.04f;
    4252
    60
        ha32->data.f32[i] = (float)((i % 13) - 6) * 0.07f;
    4253
    60
        hb32->data.f32[i] = (float)(((i * 5 + 3) % 17) - 8) * 0.05f;
    4254
    60
      }
    4255
    1
      int status = _ccv_nnc_cmul_round_to_datatype(hg32, hg, hg_rounded, count);
    4256
    1
      if (status == CCV_NNC_EXEC_SUCCESS)
    4257
    1
        status = _ccv_nnc_cmul_round_to_datatype(ha32, ha, ha_rounded, count);
    4258
    1
      if (status == CCV_NNC_EXEC_SUCCESS)
    4259
    1
        status = _ccv_nnc_cmul_round_to_datatype(hb32, hb, hb_rounded, count);
    4260
    1
      if (status == CCV_NNC_EXEC_SUCCESS)
    4261
    1
        status = ccv_nnc_cmd_exec(CMD_DATA_TRANSFER_FORWARD(), ccv_nnc_no_hint, 0, TENSOR_LIST(hg, ha, hb), TENSOR_LIST(gg, ga, gb), 0);
    4262
    1
      if (status == CCV_NNC_EXEC_SUCCESS)
    4263
    1
      {
    4264
    1
        ccv_nnc_cmd_t cmd = CMD_CMUL_BACKWARD();
    4265
    1
        cmd.backend = backend;
    4266
    1
        status = ccv_nnc_cmd_exec(cmd, ccv_nnc_no_hint, 0, TENSOR_LIST(gg, ga, gb), TENSOR_LIST(gc, gd), 0);
    4267
    1
      }
    4268
    1
      if (status == CCV_NNC_EXEC_SUCCESS)
    4269
    1
        status = ccv_nnc_cmd_exec(CMD_DATA_TRANSFER_FORWARD(), ccv_nnc_no_hint, 0, TENSOR_LIST(gc, gd), TENSOR_LIST(hc, hd), 0);
    4270
    1
      if (status == CCV_NNC_EXEC_SUCCESS)
    4271
    1
        status = _ccv_nnc_cmul_convert_to_float(hc, actual_c, count);
    4272
    1
      if (status == CCV_NNC_EXEC_SUCCESS)
    4273
    1
        status = _ccv_nnc_cmul_convert_to_float(hd, actual_d, count);
    4274
    1
      if (status == CCV_NNC_EXEC_SUCCESS)
    4275
    1
      {
    4276
    31
        for (i = 0; i < count; i += 230)
    4277
    30
        {
    4278
    30
          const float g0 = hg_rounded->data.f32[i];
    4279
    30
          const float g1 = hg_rounded->data.f32[i + 1];
    4280
    30
          const float a0 = ha_rounded->data.f32[i];
    4281
    30
          const float a1 = ha_rounded->data.f32[i + 1];
    4282
    30
          const float b0 = hb_rounded->data.f32[i];
    4283
    30
          const float b1 = hb_rounded->data.f32[i + 1];
    4284
    30
          expected_c->data.f32[i] = g0 * b0 + g1 * b1;
    4285
    30
          expected_c->data.f32[i + 1] = -g0 * b1 + g1 * b0;
    4286
    30
          expected_d->data.f32[i] = g0 * a0 + g1 * a1;
    4287
    30
          expected_d->data.f32[i + 1] = -g0 * a1 + g1 * a0;
    4288
    30
        }
    4289
    1
        status = _ccv_nnc_cmul_round_to_datatype(expected_c, expected_c_typed, expected_c_rounded, count);
    4290
    1
      }
    4291
    1
      if (status == CCV_NNC_EXEC_SUCCESS)
    4292
    1
        status = _ccv_nnc_cmul_round_to_datatype(expected_d, expected_d_typed, expected_d_rounded, count);
    4293
    1
      *max_diff_c = 0;
    4294
    1
      *max_diff_d = 0;
    4295
    1
      if (status == CCV_NNC_EXEC_SUCCESS)
    4296
    61
        for (i = 0; 1i < count; i++60)
    4297
    60
        {
    4298
    60
          *max_diff_c = ccv_max(*max_diff_c, fabsf(actual_c->data.f32[i] - expected_c_rounded->data.f32[i]));
    4299
    60
          *max_diff_d = ccv_max(*max_diff_d, fabsf(actual_d->data.f32[i] - expected_d_rounded->data.f32[i]));
    4300
    60
        }
    4301
    1
      ccv_nnc_tensor_free(hg32);
    4302
    1
      ccv_nnc_tensor_free(ha32);
    4303
    1
      ccv_nnc_tensor_free(hb32);
    4304
    1
      ccv_nnc_tensor_free(hg);
    4305
    1
      ccv_nnc_tensor_free(ha);
    4306
    1
      ccv_nnc_tensor_free(hb);
    4307
    1
      ccv_nnc_tensor_free(hc);
    4308
    1
      ccv_nnc_tensor_free(hd);
    4309
    1
      ccv_nnc_tensor_free(hg_rounded);
    4310
    1
      ccv_nnc_tensor_free(ha_rounded);
    4311
    1
      ccv_nnc_tensor_free(hb_rounded);
    4312
    1
      ccv_nnc_tensor_free(expected_c);
    4313
    1
      ccv_nnc_tensor_free(expected_d);
    4314
    1
      ccv_nnc_tensor_free(expected_c_rounded);
    4315
    1
      ccv_nnc_tensor_free(expected_d_rounded);
    4316
    1
      ccv_nnc_tensor_free(actual_c);
    4317
    1
      ccv_nnc_tensor_free(actual_d);
    4318
    1
      ccv_nnc_tensor_free(expected_c_typed);
    4319
    1
      ccv_nnc_tensor_free(expected_d_typed);
    4320
    1
      ccv_nnc_tensor_free(gg);
    4321
    1
      ccv_nnc_tensor_free(ga);
    4322
    1
      ccv_nnc_tensor_free(gb);
    4323
    1
      ccv_nnc_tensor_free(gc);
    4324
    1
      ccv_nnc_tensor_free(gd);
    4325
    1
      return status;
    4326
    1
    }
    4327
    4328
    TEST_CASE("cmul mixed half and bfloat precision with mps")
    4329
    1
    {
    4330
    1
      GUARD_ELSE_RETURN(ccv_nnc_cmd_ok(CCV_NNC_CMUL_FORWARD, CCV_NNC_BACKEND_MPS));
    4331
    0
      const uint64_t old_flags = ccv_nnc_flags();
    4332
    0
      if (old_flags & CCV_NNC_DISABLE_MFA)
    4333
    0
        ccv_nnc_disable_flag(CCV_NNC_DISABLE_MFA);
    4334
    0
      float max_diff_mfa_0 = 1e30f;
    4335
    0
      float max_diff_mfa_1 = 1e30f;
    4336
    0
      float max_diff_mfa_2 = 1e30f;
    4337
    0
      const int status_mfa_0 = _ccv_nnc_cmul_mixed_precision_case(CCV_NNC_BACKEND_MPS, CCV_16F, CCV_16BF, CCV_16F, &max_diff_mfa_0);
    4338
    0
      const int status_mfa_1 = _ccv_nnc_cmul_mixed_precision_case(CCV_NNC_BACKEND_MPS, CCV_16BF, CCV_16F, CCV_16BF, &max_diff_mfa_1);
    4339
    0
      const int status_mfa_2 = _ccv_nnc_cmul_mixed_precision_case(CCV_NNC_BACKEND_MPS, CCV_16F, CCV_16BF, CCV_32F, &max_diff_mfa_2);
    4340
    0
      ccv_nnc_enable_flag(CCV_NNC_DISABLE_MFA);
    4341
    0
      float max_diff_graph_0 = 1e30f;
    4342
    0
      float max_diff_graph_1 = 1e30f;
    4343
    0
      float max_diff_graph_2 = 1e30f;
    4344
    0
      const int status_graph_0 = _ccv_nnc_cmul_mixed_precision_case(CCV_NNC_BACKEND_MPS, CCV_16F, CCV_16BF, CCV_16F, &max_diff_graph_0);
    4345
    0
      const int status_graph_1 = _ccv_nnc_cmul_mixed_precision_case(CCV_NNC_BACKEND_MPS, CCV_16BF, CCV_16F, CCV_16BF, &max_diff_graph_1);
    4346
    0
      const int status_graph_2 = _ccv_nnc_cmul_mixed_precision_case(CCV_NNC_BACKEND_MPS, CCV_16F, CCV_16BF, CCV_32F, &max_diff_graph_2);
    4347
    0
      if (old_flags & CCV_NNC_DISABLE_MFA)
    4348
    0
        ccv_nnc_enable_flag(CCV_NNC_DISABLE_MFA);
    4349
    0
      else
    4350
    0
        ccv_nnc_disable_flag(CCV_NNC_DISABLE_MFA);
    4351
    0
      REQUIRE_EQ(CCV_NNC_EXEC_SUCCESS, status_mfa_0, "mixed half / bfloat cmul through mfa should run");
    4352
    0
      REQUIRE_EQ(CCV_NNC_EXEC_SUCCESS, status_mfa_1, "mixed bfloat / half cmul through mfa should run");
    4353
    0
      REQUIRE_EQ(CCV_NNC_EXEC_SUCCESS, status_mfa_2, "mixed half / bfloat cmul to float through mfa should run");
    4354
    0
      REQUIRE_EQ(CCV_NNC_EXEC_SUCCESS, status_graph_0, "mixed half / bfloat cmul through graph fallback should run");
    4355
    0
      REQUIRE_EQ(CCV_NNC_EXEC_SUCCESS, status_graph_1, "mixed bfloat / half cmul through graph fallback should run");
    4356
    0
      REQUIRE_EQ(CCV_NNC_EXEC_SUCCESS, status_graph_2, "mixed half / bfloat cmul to float through graph fallback should run");
    4357
    0
      REQUIRE(max_diff_mfa_0 <= 3e-2, "mixed half / bfloat cmul through mfa should match fp32 reference rounded to output dtype");
    4358
    0
      REQUIRE(max_diff_mfa_1 <= 3e-2, "mixed bfloat / half cmul through mfa should match fp32 reference rounded to output dtype");
    4359
    0
      REQUIRE(max_diff_mfa_2 <= 3e-2, "mixed half / bfloat cmul to float through mfa should match fp32 reference");
    4360
    0
      REQUIRE(max_diff_graph_0 <= 3e-2, "mixed half / bfloat cmul through graph fallback should match fp32 reference rounded to output dtype");
    4361
    0
      REQUIRE(max_diff_graph_1 <= 3e-2, "mixed bfloat / half cmul through graph fallback should match fp32 reference rounded to output dtype");
    4362
    0
      REQUIRE(max_diff_graph_2 <= 3e-2, "mixed half / bfloat cmul to float through graph fallback should match fp32 reference");
    4363
    0
    }
    4364
    4365
    TEST_CASE("cmul gradient mixed half and bfloat precision with mps")
    4366
    1
    {
    4367
    1
      GUARD_ELSE_RETURN(ccv_nnc_cmd_ok(CCV_NNC_CMUL_BACKWARD, CCV_NNC_BACKEND_MPS));
    4368
    0
      const uint64_t old_flags = ccv_nnc_flags();
    4369
    0
      if (old_flags & CCV_NNC_DISABLE_MFA)
    4370
    0
        ccv_nnc_disable_flag(CCV_NNC_DISABLE_MFA);
    4371
    0
      float max_diff_mfa_c = 1e30f;
    4372
    0
      float max_diff_mfa_d = 1e30f;
    4373
    0
      const int status_mfa = _ccv_nnc_cmul_mixed_precision_backward_case(CCV_NNC_BACKEND_MPS, CCV_16F, CCV_16BF, CCV_16F, CCV_16BF, CCV_16F, &max_diff_mfa_c, &max_diff_mfa_d);
    4374
    0
      ccv_nnc_enable_flag(CCV_NNC_DISABLE_MFA);
    4375
    0
      float max_diff_graph_c = 1e30f;
    4376
    0
      float max_diff_graph_d = 1e30f;
    4377
    0
      const int status_graph = _ccv_nnc_cmul_mixed_precision_backward_case(CCV_NNC_BACKEND_MPS, CCV_16F, CCV_16BF, CCV_16F, CCV_16BF, CCV_16F, &max_diff_graph_c, &max_diff_graph_d);
    4378
    0
      if (old_flags & CCV_NNC_DISABLE_MFA)
    4379
    0
        ccv_nnc_enable_flag(CCV_NNC_DISABLE_MFA);
    4380
    0
      else
    4381
    0
        ccv_nnc_disable_flag(CCV_NNC_DISABLE_MFA);
    4382
    0
      REQUIRE_EQ(CCV_NNC_EXEC_SUCCESS, status_mfa, "mixed half / bfloat cmul gradient through mfa should run");
    4383
    0
      REQUIRE_EQ(CCV_NNC_EXEC_SUCCESS, status_graph, "mixed half / bfloat cmul gradient through graph fallback should run");
    4384
    0
      REQUIRE(max_diff_mfa_c <= 3e-2, "mixed half / bfloat cmul gradient first output through mfa should match fp32 reference rounded to output dtype");
    4385
    0
      REQUIRE(max_diff_mfa_d <= 3e-2, "mixed half / bfloat cmul gradient second output through mfa should match fp32 reference rounded to output dtype");
    4386
    0
      REQUIRE(max_diff_graph_c <= 3e-2, "mixed half / bfloat cmul gradient first output through graph fallback should match fp32 reference rounded to output dtype");
    4387
    0
      REQUIRE(max_diff_graph_d <= 3e-2, "mixed half / bfloat cmul gradient second output through graph fallback should match fp32 reference rounded to output dtype");
    4388
    0
    }
    4389
    4390
    TEST_CASE("cmul mixed half and bfloat precision with cuda")
    4391
    1
    {
    4392
    1
      GUARD_ELSE_RETURN(ccv_nnc_cmd_ok(CCV_NNC_CMUL_FORWARD, CCV_NNC_BACKEND_GPU_REF));
    4393
    1
      float max_diff_0 = 1e30f;
    4394
    1
      float max_diff_1 = 1e30f;
    4395
    1
      float max_diff_2 = 1e30f;
    4396
    1
      const int status_0 = _ccv_nnc_cmul_mixed_precision_case(CCV_NNC_BACKEND_GPU_REF, CCV_16F, CCV_16BF, CCV_16F, &max_diff_0);
    4397
    1
      const int status_1 = _ccv_nnc_cmul_mixed_precision_case(CCV_NNC_BACKEND_GPU_REF, CCV_16BF, CCV_16F, CCV_16BF, &max_diff_1);
    4398
    1
      const int status_2 = _ccv_nnc_cmul_mixed_precision_case(CCV_NNC_BACKEND_GPU_REF, CCV_16F, CCV_16BF, CCV_32F, &max_diff_2);
    4399
    1
      REQUIRE_EQ(CCV_NNC_EXEC_SUCCESS, status_0, "mixed half / bfloat cmul through cuda should run");
    4400
    1
      REQUIRE_EQ(CCV_NNC_EXEC_SUCCESS, status_1, "mixed bfloat / half cmul through cuda should run");
    4401
    1
      REQUIRE_EQ(CCV_NNC_EXEC_SUCCESS, status_2, "mixed half / bfloat cmul to float through cuda should run");
    4402
    1
      REQUIRE(max_diff_0 <= 3e-2, "mixed half / bfloat cmul through cuda should match fp32 reference rounded to output dtype");
    4403
    1
      REQUIRE(max_diff_1 <= 3e-2, "mixed bfloat / half cmul through cuda should match fp32 reference rounded to output dtype");
    4404
    1
      REQUIRE(max_diff_2 <= 3e-2, "mixed half / bfloat cmul to float through cuda should match fp32 reference");
    4405
    1
    }
    4406
    4407
    TEST_CASE("cmul gradient mixed half and bfloat precision with cuda")
    4408
    1
    {
    4409
    1
      GUARD_ELSE_RETURN(ccv_nnc_cmd_ok(CCV_NNC_CMUL_BACKWARD, CCV_NNC_BACKEND_GPU_REF));
    4410
    1
      float max_diff_c = 1e30f;
    4411
    1
      float max_diff_d = 1e30f;
    4412
    1
      const int status = _ccv_nnc_cmul_mixed_precision_backward_case(CCV_NNC_BACKEND_GPU_REF, CCV_16F, CCV_16BF, CCV_16F, CCV_16BF, CCV_16F, &max_diff_c, &max_diff_d);
    4413
    1
      REQUIRE_EQ(CCV_NNC_EXEC_SUCCESS, status, "mixed half / bfloat cmul gradient through cuda should run");
    4414
    1
      REQUIRE(max_diff_c <= 3e-2, "mixed half / bfloat cmul gradient first output through cuda should match fp32 reference rounded to output dtype");
    4415
    1
      REQUIRE(max_diff_d <= 3e-2, "mixed half / bfloat cmul gradient second output through cuda should match fp32 reference rounded to output dtype");
    4416
    1
    }
    4417
    4418
    TEST_CASE("cmul in float")
    4419
    1
    {
    4420
    1
      GUARD_ELSE_RETURN(ccv_nnc_cmd_ok(CCV_NNC_CMUL_FORWARD, CCV_NNC_BACKEND_GPU_REF) || ccv_nnc_cmd_ok(CCV_NNC_CMUL_FORWARD, CCV_NNC_BACKEND_MPS));
    4421
    1
      ccv_nnc_symbolic_graph_t* const symbolic_graph = ccv_nnc_symbolic_graph_new();
    4422
    1
      ccv_nnc_tensor_symbol_t a = ccv_nnc_tensor_symbol_new(symbolic_graph, GPU_TENSOR_NCHW(000, 32F, 20, 10), "a");
    4423
    1
      ccv_nnc_tensor_symbol_t b = ccv_nnc_tensor_symbol_new(symbolic_graph, GPU_TENSOR_NCHW(000, 32F, 20, 10), "b");
    4424
    1
      ccv_nnc_tensor_symbol_t c = ccv_nnc_tensor_symbol_new(symbolic_graph, GPU_TENSOR_NCHW(000, 32F, 20, 10), "c");
    4425
    1
      ccv_nnc_graph_exec_symbol_new(symbolic_graph, CMD_CMUL_FORWARD(), TENSOR_SYMBOL_LIST(a, b), TENSOR_SYMBOL_LIST(c), "cmul");
    4426
    1
      ccv_nnc_graph_exec_symbol_autogen(symbolic_graph, 0, 0, CCV_NNC_AUTOGEN_ALL_EXECS | CCV_NNC_AUTOGEN_SOURCES_AND_DESTINATIONS);
    4427
    1
      SYMBOLIC_GRAPH_GEN(symbolic_graph, CCV_NNC_LONG_DOT_GRAPH);
    4428
    1
      ccv_nnc_graph_t* graph = 0;
    4429
    1
      ccv_nnc_tensor_arena_t* tensor_arena = 0;
    4430
    1
      ccv_nnc_graph_exec_arena_t* graph_exec_arena = 0;
    4431
    1
      ccv_nnc_symbolic_graph_compile(symbolic_graph, ccv_nnc_default_compile_params, 0, 0, 0, 0, SYMBOLIC_GRAPH_SOURCES(symbolic_graph), SYMBOLIC_GRAPH_DESTINATIONS(symbolic_graph), &graph, &tensor_arena, &graph_exec_arena);
    4432
    1
      GRAPH_GEN(graph, CCV_NNC_LONG_DOT_GRAPH);
    4433
    1
      ccv_nnc_tensor_t* const x_tensor = ccv_nnc_tensor_new(0, CPU_TENSOR_NCHW(32F, 20, 10), 0);
    4434
    1
      ccv_nnc_tensor_t* const y_tensor = ccv_nnc_tensor_new(0, CPU_TENSOR_NCHW(32F, 20, 10), 0);
    4435
    1
      dsfmt_t dsfmt;
    4436
    1
      dsfmt_init_gen_rand(&dsfmt, 0);
    4437
    1
      int i;
    4438
    201
      for (i = 0; i < 20 * 10; i++200)
    4439
    200
        x_tensor->data.f32[i] = dsfmt_genrand_open_close(&dsfmt);
    4440
    201
      for (i = 0; i < 20 * 10; i++200)
    4441
    200
        y_tensor->data.f32[i] = dsfmt_genrand_open_close(&dsfmt);
    4442
    1
      ccv_nnc_tensor_t* const a_tensor = ccv_nnc_tensor_from_symbol(tensor_arena, a);
    4443
    1
      ccv_nnc_cmd_exec(CMD_DATA_TRANSFER_FORWARD(), ccv_nnc_no_hint, 0, TENSOR_LIST(x_tensor), TENSOR_LIST(a_tensor), 0);
    4444
    1
      ccv_nnc_tensor_t* const b_tensor = ccv_nnc_tensor_from_symbol(tensor_arena, b);
    4445
    1
      ccv_nnc_cmd_exec(CMD_DATA_TRANSFER_FORWARD(), ccv_nnc_no_hint, 0, TENSOR_LIST(y_tensor), TENSOR_LIST(b_tensor), 0);
    4446
    1
      ccv_nnc_graph_run(graph, 0, TRAVERSE_FULL, 0, 0);
    4447
    1
      ccv_nnc_tensor_t* const z_tensor = ccv_nnc_tensor_new(0, CPU_TENSOR_NCHW(32F, 20, 10), 0);
    4448
    1
      ccv_nnc_tensor_t* const c_tensor = ccv_nnc_tensor_from_symbol(tensor_arena, c);
    4449
    1
      ccv_nnc_cmd_exec(CMD_DATA_TRANSFER_FORWARD(), ccv_nnc_no_hint, 0, TENSOR_LIST(c_tensor), TENSOR_LIST(z_tensor), 0);
    4450
    1
      ccv_nnc_tensor_t* const tz = ccv_nnc_tensor_new(0, CPU_TENSOR_NCHW(32F, 20, 10), 0);
    4451
    1
      ccv_nnc_cmd_exec(CMD_CMUL_FORWARD(), ccv_nnc_no_hint, 0, TENSOR_LIST(x_tensor, y_tensor), TENSOR_LIST(tz), 0);
    4452
    1
      REQUIRE_TENSOR_EQ(tz, z_tensor, "gelu from cudnn should match from CPU");
    4453
    1
      ccv_nnc_tensor_free(x_tensor);
    4454
    1
      ccv_nnc_tensor_free(y_tensor);
    4455
    1
      ccv_nnc_tensor_free(z_tensor);
    4456
    1
      ccv_nnc_tensor_free(tz);
    4457
    1
      ccv_nnc_graph_free(graph);
    4458
    1
      ccv_nnc_tensor_arena_free(tensor_arena);
    4459
    1
      ccv_nnc_graph_exec_arena_free(graph_exec_arena);
    4460
    1
      ccv_nnc_symbolic_graph_free(symbolic_graph);
    4461
    1
    }
    4462
    4463
    TEST_CASE("cmul in half precision")
    4464
    1
    {
    4465
    1
      GUARD_ELSE_RETURN(ccv_nnc_cmd_ok(CCV_NNC_CMUL_FORWARD, CCV_NNC_BACKEND_GPU_REF) || ccv_nnc_cmd_ok(CCV_NNC_CMUL_FORWARD, CCV_NNC_BACKEND_MPS));
    4466
    1
      ccv_nnc_symbolic_graph_t* const symbolic_graph = ccv_nnc_symbolic_graph_new();
    4467
    1
      ccv_nnc_tensor_symbol_t a = ccv_nnc_tensor_symbol_new(symbolic_graph, GPU_TENSOR_NCHW(000, 16F, 20, 10), "a");
    4468
    1
      ccv_nnc_tensor_symbol_t b = ccv_nnc_tensor_symbol_new(symbolic_graph, GPU_TENSOR_NCHW(000, 16F, 20, 10), "b");
    4469
    1
      ccv_nnc_tensor_symbol_t c = ccv_nnc_tensor_symbol_new(symbolic_graph, GPU_TENSOR_NCHW(000, 16F, 20, 10), "c");
    4470
    1
      ccv_nnc_graph_exec_symbol_new(symbolic_graph, CMD_CMUL_FORWARD(), TENSOR_SYMBOL_LIST(a, b), TENSOR_SYMBOL_LIST(c), "cmul");
    4471
    1
      ccv_nnc_graph_exec_symbol_autogen(symbolic_graph, 0, 0, CCV_NNC_AUTOGEN_ALL_EXECS | CCV_NNC_AUTOGEN_SOURCES_AND_DESTINATIONS);
    4472
    1
      SYMBOLIC_GRAPH_GEN(symbolic_graph, CCV_NNC_LONG_DOT_GRAPH);
    4473
    1
      ccv_nnc_graph_t* graph = 0;
    4474
    1
      ccv_nnc_tensor_arena_t* tensor_arena = 0;
    4475
    1
      ccv_nnc_graph_exec_arena_t* graph_exec_arena = 0;
    4476
    1
      ccv_nnc_symbolic_graph_compile(symbolic_graph, ccv_nnc_default_compile_params, 0, 0, 0, 0, SYMBOLIC_GRAPH_SOURCES(symbolic_graph), SYMBOLIC_GRAPH_DESTINATIONS(symbolic_graph), &graph, &tensor_arena, &graph_exec_arena);
    4477
    1
      GRAPH_GEN(graph, CCV_NNC_LONG_DOT_GRAPH);
    4478
    1
      ccv_nnc_tensor_t* const x_tensor = ccv_nnc_tensor_new(0, CPU_TENSOR_NCHW(32F, 20, 10), 0);
    4479
    1
      ccv_nnc_tensor_t* const y_tensor = ccv_nnc_tensor_new(0, CPU_TENSOR_NCHW(32F, 20, 10), 0);
    4480
    1
      dsfmt_t dsfmt;
    4481
    1
      dsfmt_init_gen_rand(&dsfmt, 0);
    4482
    1
      int i;
    4483
    201
      for (i = 0; i < 20 * 10; i++200)
    4484
    200
        x_tensor->data.f32[i] = dsfmt_genrand_open_close(&dsfmt);
    4485
    201
      for (i = 0; i < 20 * 10; i++200)
    4486
    200
        y_tensor->data.f32[i] = dsfmt_genrand_open_close(&dsfmt);
    4487
    1
      ccv_nnc_tensor_t* const a_tensor = ccv_nnc_tensor_from_symbol(tensor_arena, a);
    4488
    1
      ccv_nnc_tensor_t* const x16_tensor = ccv_nnc_tensor_new(0, CPU_TENSOR_NCHW(16F, 20, 10), 0);
    4489
    1
      ccv_nnc_cmd_exec(CMD_DATATYPE_CONVERSION_FORWARD(), ccv_nnc_no_hint, 0, TENSOR_LIST(x_tensor), TENSOR_LIST(x16_tensor), 0);
    4490
    1
      ccv_nnc_cmd_exec(CMD_DATA_TRANSFER_FORWARD(), ccv_nnc_no_hint, 0, TENSOR_LIST(x16_tensor), TENSOR_LIST(a_tensor), 0);
    4491
    1
      ccv_nnc_tensor_t* const b_tensor = ccv_nnc_tensor_from_symbol(tensor_arena, b);
    4492
    1
      ccv_nnc_tensor_t* const y16_tensor = ccv_nnc_tensor_new(0, CPU_TENSOR_NCHW(16F, 20, 10), 0);
    4493
    1
      ccv_nnc_cmd_exec(CMD_DATATYPE_CONVERSION_FORWARD(), ccv_nnc_no_hint, 0, TENSOR_LIST(y_tensor), TENSOR_LIST(y16_tensor), 0);
    4494
    1
      ccv_nnc_cmd_exec(CMD_DATA_TRANSFER_FORWARD(), ccv_nnc_no_hint, 0, TENSOR_LIST(y16_tensor), TENSOR_LIST(b_tensor), 0);
    4495
    1
      ccv_nnc_graph_run(graph, 0, TRAVERSE_FULL, 0, 0);
    4496
    1
      ccv_nnc_tensor_t* const z16_tensor = ccv_nnc_tensor_new(0, CPU_TENSOR_NCHW(16F, 20, 10), 0);
    4497
    1
      ccv_nnc_tensor_t* const z_tensor = ccv_nnc_tensor_new(0, CPU_TENSOR_NCHW(32F, 20, 10), 0);
    4498
    1
      ccv_nnc_tensor_t* const c_tensor = ccv_nnc_tensor_from_symbol(tensor_arena, c);
    4499
    1
      ccv_nnc_cmd_exec(CMD_DATA_TRANSFER_FORWARD(), ccv_nnc_no_hint, 0, TENSOR_LIST(c_tensor), TENSOR_LIST(z16_tensor), 0);
    4500
    1
      ccv_nnc_cmd_exec(CMD_DATATYPE_CONVERSION_FORWARD(), ccv_nnc_no_hint, 0, TENSOR_LIST(z16_tensor), TENSOR_LIST(z_tensor), 0);
    4501
    1
      ccv_nnc_tensor_t* const tz = ccv_nnc_tensor_new(0, CPU_TENSOR_NCHW(32F, 20, 10), 0);
    4502
    1
      ccv_nnc_cmd_exec(CMD_CMUL_FORWARD(), ccv_nnc_no_hint, 0, TENSOR_LIST(x_tensor, y_tensor), TENSOR_LIST(tz), 0);
    4503
    1
      REQUIRE_ARRAY_EQ_WITH_TOLERANCE(float, tz->data.f32, z_tensor->data.f32, 20 * 10, 2e-3, "gelu from cudnn should match from CPU");
    4504
    1
      ccv_nnc_tensor_free(x_tensor);
    4505
    1
      ccv_nnc_tensor_free(x16_tensor);
    4506
    1
      ccv_nnc_tensor_free(y16_tensor);
    4507
    1
      ccv_nnc_tensor_free(y_tensor);
    4508
    1
      ccv_nnc_tensor_free(z16_tensor);
    4509
    1
      ccv_nnc_tensor_free(z_tensor);
    4510
    1
      ccv_nnc_tensor_free(tz);
    4511
    1
      ccv_nnc_graph_free(graph);
    4512
    1
      ccv_nnc_tensor_arena_free(tensor_arena);
    4513
    1
      ccv_nnc_graph_exec_arena_free(graph_exec_arena);
    4514
    1
      ccv_nnc_symbolic_graph_free(symbolic_graph);
    4515
    1
    }
    4516
    4517
    TEST_CASE("cmul in float, broadcast semantics")
    4518
    1
    {
    4519
    1
      GUARD_ELSE_RETURN(ccv_nnc_cmd_ok(CCV_NNC_CMUL_FORWARD, CCV_NNC_BACKEND_GPU_REF) || ccv_nnc_cmd_ok(CCV_NNC_CMUL_FORWARD, CCV_NNC_BACKEND_MPS));
    4520
    1
      ccv_nnc_symbolic_graph_t* const symbolic_graph = ccv_nnc_symbolic_graph_new();
    4521
    1
      ccv_nnc_tensor_symbol_t a = ccv_nnc_tensor_symbol_new(symbolic_graph, GPU_TENSOR_NCHW(000, 32F, 1, 5, 8, 128), "a");
    4522
    1
      ccv_nnc_tensor_symbol_t b = ccv_nnc_tensor_symbol_new(symbolic_graph, GPU_TENSOR_NCHW(000, 32F, 1, 5, 1, 128), "b");
    4523
    1
      ccv_nnc_tensor_symbol_t c = ccv_nnc_tensor_symbol_new(symbolic_graph, GPU_TENSOR_NCHW(000, 32F, 1, 5, 8, 128), "c");
    4524
    1
      ccv_nnc_graph_exec_symbol_new(symbolic_graph, CMD_CMUL_FORWARD(), TENSOR_SYMBOL_LIST(a, b), TENSOR_SYMBOL_LIST(c), "cmul");
    4525
    1
      ccv_nnc_graph_exec_symbol_autogen(symbolic_graph, 0, 0, CCV_NNC_AUTOGEN_ALL_EXECS | CCV_NNC_AUTOGEN_SOURCES_AND_DESTINATIONS);
    4526
    1
      SYMBOLIC_GRAPH_GEN(symbolic_graph, CCV_NNC_LONG_DOT_GRAPH);
    4527
    1
      ccv_nnc_graph_t* graph = 0;
    4528
    1
      ccv_nnc_tensor_arena_t* tensor_arena = 0;
    4529
    1
      ccv_nnc_graph_exec_arena_t* graph_exec_arena = 0;
    4530
    1
      ccv_nnc_symbolic_graph_compile(symbolic_graph, ccv_nnc_default_compile_params, 0, 0, 0, 0, SYMBOLIC_GRAPH_SOURCES(symbolic_graph), SYMBOLIC_GRAPH_DESTINATIONS(symbolic_graph), &graph, &tensor_arena, &graph_exec_arena);
    4531
    1
      GRAPH_GEN(graph, CCV_NNC_LONG_DOT_GRAPH);
    4532
    1
      ccv_nnc_tensor_t* const x_tensor = ccv_nnc_tensor_new(0, CPU_TENSOR_NCHW(32F, 1, 5, 8, 128), 0);
    4533
    1
      ccv_nnc_tensor_t* const y_tensor = ccv_nnc_tensor_new(0, CPU_TENSOR_NCHW(32F, 1, 5, 1, 128), 0);
    4534
    1
      dsfmt_t dsfmt;
    4535
    1
      dsfmt_init_gen_rand(&dsfmt, 0);
    4536
    1
      int i;
    4537
    5.12k
      for (i = 0; i < 1 * 5 * 8 * 128; i++5.12k)
    4538
    5.12k
        x_tensor->data.f32[i] = dsfmt_genrand_open_close(&dsfmt);
    4539
    641
      for (i = 0; i < 1 * 5 * 1 * 128; i++640)
    4540
    640
        y_tensor->data.f32[i] = dsfmt_genrand_open_close(&dsfmt);
    4541
    1
      ccv_nnc_tensor_t* const a_tensor = ccv_nnc_tensor_from_symbol(tensor_arena, a);
    4542
    1
      ccv_nnc_cmd_exec(CMD_DATA_TRANSFER_FORWARD(), ccv_nnc_no_hint, 0, TENSOR_LIST(x_tensor), TENSOR_LIST(a_tensor), 0);
    4543
    1
      ccv_nnc_tensor_t* const b_tensor = ccv_nnc_tensor_from_symbol(tensor_arena, b);
    4544
    1
      ccv_nnc_cmd_exec(CMD_DATA_TRANSFER_FORWARD(), ccv_nnc_no_hint, 0, TENSOR_LIST(y_tensor), TENSOR_LIST(b_tensor), 0);
    4545
    1
      ccv_nnc_graph_run(graph, 0, TRAVERSE_FULL, 0, 0);
    4546
    1
      ccv_nnc_tensor_t* const z_tensor = ccv_nnc_tensor_new(0, CPU_TENSOR_NCHW(32F, 1, 5, 8, 128), 0);
    4547
    1
      ccv_nnc_tensor_t* const c_tensor = ccv_nnc_tensor_from_symbol(tensor_arena, c);
    4548
    1
      ccv_nnc_cmd_exec(CMD_DATA_TRANSFER_FORWARD(), ccv_nnc_no_hint, 0, TENSOR_LIST(c_tensor), TENSOR_LIST(z_tensor), 0);
    4549
    1
      ccv_nnc_tensor_t* const tz = ccv_nnc_tensor_new(0, CPU_TENSOR_NCHW(32F, 1, 5, 8, 128), 0);
    4550
    1
      ccv_nnc_cmd_exec(CMD_CMUL_FORWARD(), ccv_nnc_no_hint, 0, TENSOR_LIST(x_tensor, y_tensor), TENSOR_LIST(tz), 0);
    4551
    1
      REQUIRE_TENSOR_EQ(tz, z_tensor, "gelu from cudnn should match from CPU");
    4552
    1
      ccv_nnc_tensor_free(x_tensor);
    4553
    1
      ccv_nnc_tensor_free(y_tensor);
    4554
    1
      ccv_nnc_tensor_free(z_tensor);
    4555
    1
      ccv_nnc_tensor_free(tz);
    4556
    1
      ccv_nnc_graph_free(graph);
    4557
    1
      ccv_nnc_tensor_arena_free(tensor_arena);
    4558
    1
      ccv_nnc_graph_exec_arena_free(graph_exec_arena);
    4559
    1
      ccv_nnc_symbolic_graph_free(symbolic_graph);
    4560
    1
    }
    4561
    4562
    TEST_CASE("cmul in float, broadcast semantics with longer than 65535 sequence")
    4563
    1
    {
    4564
    1
      GUARD_ELSE_RETURN(ccv_nnc_cmd_ok(CCV_NNC_CMUL_FORWARD, CCV_NNC_BACKEND_GPU_REF) || ccv_nnc_cmd_ok(CCV_NNC_CMUL_FORWARD, CCV_NNC_BACKEND_MPS));
    4565
    1
      ccv_nnc_symbolic_graph_t* const symbolic_graph = ccv_nnc_symbolic_graph_new();
    4566
    1
      ccv_nnc_tensor_symbol_t a = ccv_nnc_tensor_symbol_new(symbolic_graph, GPU_TENSOR_NCHW(000, 32F, 1, 70000, 8, 16), "a");
    4567
    1
      ccv_nnc_tensor_symbol_t b = ccv_nnc_tensor_symbol_new(symbolic_graph, GPU_TENSOR_NCHW(000, 32F, 1, 70000, 1, 16), "b");
    4568
    1
      ccv_nnc_tensor_symbol_t c = ccv_nnc_tensor_symbol_new(symbolic_graph, GPU_TENSOR_NCHW(000, 32F, 1, 70000, 8, 16), "c");
    4569
    1
      ccv_nnc_graph_exec_symbol_new(symbolic_graph, CMD_CMUL_FORWARD(), TENSOR_SYMBOL_LIST(a, b), TENSOR_SYMBOL_LIST(c), "cmul");
    4570
    1
      ccv_nnc_graph_exec_symbol_autogen(symbolic_graph, 0, 0, CCV_NNC_AUTOGEN_ALL_EXECS | CCV_NNC_AUTOGEN_SOURCES_AND_DESTINATIONS);
    4571
    1
      SYMBOLIC_GRAPH_GEN(symbolic_graph, CCV_NNC_LONG_DOT_GRAPH);
    4572
    1
      ccv_nnc_graph_t* graph = 0;
    4573
    1
      ccv_nnc_tensor_arena_t* tensor_arena = 0;
    4574
    1
      ccv_nnc_graph_exec_arena_t* graph_exec_arena = 0;
    4575
    1
      ccv_nnc_symbolic_graph_compile(symbolic_graph, ccv_nnc_default_compile_params, 0, 0, 0, 0, SYMBOLIC_GRAPH_SOURCES(symbolic_graph), SYMBOLIC_GRAPH_DESTINATIONS(symbolic_graph), &graph, &tensor_arena, &graph_exec_arena);
    4576
    1
      GRAPH_GEN(graph, CCV_NNC_LONG_DOT_GRAPH);
    4577
    1
      ccv_nnc_tensor_t* const x_tensor = ccv_nnc_tensor_new(0, CPU_TENSOR_NCHW(32F, 1, 70000, 8, 16), 0);
    4578
    1
      ccv_nnc_tensor_t* const y_tensor = ccv_nnc_tensor_new(0, CPU_TENSOR_NCHW(32F, 1, 70000, 1, 16), 0);
    4579
    1
      dsfmt_t dsfmt;
    4580
    1
      dsfmt_init_gen_rand(&dsfmt, 0);
    4581
    1
      int i;
    4582
    8.96M
      for (i = 0; i < 1 * 70000 * 8 * 16; i++8.96M)
    4583
    8.96M
        x_tensor->data.f32[i] = dsfmt_genrand_open_close(&dsfmt);
    4584
    1.12M
      for (i = 0; i < 1 * 70000 * 1 * 16; i++1.12M)
    4585
    1.12M
        y_tensor->data.f32[i] = dsfmt_genrand_open_close(&dsfmt);
    4586
    1
      ccv_nnc_tensor_t* const a_tensor = ccv_nnc_tensor_from_symbol(tensor_arena, a);
    4587
    1
      ccv_nnc_cmd_exec(CMD_DATA_TRANSFER_FORWARD(), ccv_nnc_no_hint, 0, TENSOR_LIST(x_tensor), TENSOR_LIST(a_tensor), 0);
    4588
    1
      ccv_nnc_tensor_t* const b_tensor = ccv_nnc_tensor_from_symbol(tensor_arena, b);
    4589
    1
      ccv_nnc_cmd_exec(CMD_DATA_TRANSFER_FORWARD(), ccv_nnc_no_hint, 0, TENSOR_LIST(y_tensor), TENSOR_LIST(b_tensor), 0);
    4590
    1
      ccv_nnc_graph_run(graph, 0, TRAVERSE_FULL, 0, 0);
    4591
    1
      ccv_nnc_tensor_t* const z_tensor = ccv_nnc_tensor_new(0, CPU_TENSOR_NCHW(32F, 1, 70000, 8, 16), 0);
    4592
    1
      ccv_nnc_tensor_t* const c_tensor = ccv_nnc_tensor_from_symbol(tensor_arena, c);
    4593
    1
      ccv_nnc_cmd_exec(CMD_DATA_TRANSFER_FORWARD(), ccv_nnc_no_hint, 0, TENSOR_LIST(c_tensor), TENSOR_LIST(z_tensor), 0);
    4594
    1
      ccv_nnc_tensor_t* const tz = ccv_nnc_tensor_new(0, CPU_TENSOR_NCHW(32F, 1, 70000, 8, 16), 0);
    4595
    1
      ccv_nnc_cmd_exec(CMD_CMUL_FORWARD(), ccv_nnc_no_hint, 0, TENSOR_LIST(x_tensor, y_tensor), TENSOR_LIST(tz), 0);
    4596
    1
      REQUIRE_TENSOR_EQ(tz, z_tensor, "gelu from cudnn should match from CPU");
    4597
    1
      ccv_nnc_tensor_free(x_tensor);
    4598
    1
      ccv_nnc_tensor_free(y_tensor);
    4599
    1
      ccv_nnc_tensor_free(z_tensor);
    4600
    1
      ccv_nnc_tensor_free(tz);
    4601
    1
      ccv_nnc_graph_free(graph);
    4602
    1
      ccv_nnc_tensor_arena_free(tensor_arena);
    4603
    1
      ccv_nnc_graph_exec_arena_free(graph_exec_arena);
    4604
    1
      ccv_nnc_symbolic_graph_free(symbolic_graph);
    4605
    1
    }
    4606
    4607
    TEST_CASE("cmul in float, broadcast semantics with longer than 65535 sequence and more than 1 batch size")
    4608
    1
    {
    4609
    1
      GUARD_ELSE_RETURN(ccv_nnc_cmd_ok(CCV_NNC_CMUL_FORWARD, CCV_NNC_BACKEND_GPU_REF) || ccv_nnc_cmd_ok(CCV_NNC_CMUL_FORWARD, CCV_NNC_BACKEND_MPS));
    4610
    1
      ccv_nnc_symbolic_graph_t* const symbolic_graph = ccv_nnc_symbolic_graph_new();
    4611
    1
      ccv_nnc_tensor_symbol_t a = ccv_nnc_tensor_symbol_new(symbolic_graph, GPU_TENSOR_NCHW(000, 32F, 2, 40000, 8, 16), "a");
    4612
    1
      ccv_nnc_tensor_symbol_t b = ccv_nnc_tensor_symbol_new(symbolic_graph, GPU_TENSOR_NCHW(000, 32F, 1, 40000, 1, 16), "b");
    4613
    1
      ccv_nnc_tensor_symbol_t c = ccv_nnc_tensor_symbol_new(symbolic_graph, GPU_TENSOR_NCHW(000, 32F, 2, 40000, 8, 16), "c");
    4614
    1
      ccv_nnc_graph_exec_symbol_new(symbolic_graph, CMD_CMUL_FORWARD(), TENSOR_SYMBOL_LIST(a, b), TENSOR_SYMBOL_LIST(c), "cmul");
    4615
    1
      ccv_nnc_graph_exec_symbol_autogen(symbolic_graph, 0, 0, CCV_NNC_AUTOGEN_ALL_EXECS | CCV_NNC_AUTOGEN_SOURCES_AND_DESTINATIONS);
    4616
    1
      SYMBOLIC_GRAPH_GEN(symbolic_graph, CCV_NNC_LONG_DOT_GRAPH);
    4617
    1
      ccv_nnc_graph_t* graph = 0;
    4618
    1
      ccv_nnc_tensor_arena_t* tensor_arena = 0;
    4619
    1
      ccv_nnc_graph_exec_arena_t* graph_exec_arena = 0;
    4620
    1
      ccv_nnc_symbolic_graph_compile(symbolic_graph, ccv_nnc_default_compile_params, 0, 0, 0, 0, SYMBOLIC_GRAPH_SOURCES(symbolic_graph), SYMBOLIC_GRAPH_DESTINATIONS(symbolic_graph), &graph, &tensor_arena, &graph_exec_arena);
    4621
    1
      GRAPH_GEN(graph, CCV_NNC_LONG_DOT_GRAPH);
    4622
    1
      ccv_nnc_tensor_t* const x_tensor = ccv_nnc_tensor_new(0, CPU_TENSOR_NCHW(32F, 2, 40000, 8, 16), 0);
    4623
    1
      ccv_nnc_tensor_t* const y_tensor = ccv_nnc_tensor_new(0, CPU_TENSOR_NCHW(32F, 1, 40000, 1, 16), 0);
    4624
    1
      dsfmt_t dsfmt;
    4625
    1
      dsfmt_init_gen_rand(&dsfmt, 0);
    4626
    1
      int i;
    4627
    10.2M
      for (i = 0; i < 2 * 40000 * 8 * 16; i++10.2M)
    4628
    10.2M
        x_tensor->data.f32[i] = dsfmt_genrand_open_close(&dsfmt);
    4629
    640k
      for (i = 0; i < 1 * 40000 * 1 * 16; i++640k)
    4630
    640k
        y_tensor->data.f32[i] = dsfmt_genrand_open_close(&dsfmt);
    4631
    1
      ccv_nnc_tensor_t* const a_tensor = ccv_nnc_tensor_from_symbol(tensor_arena, a);
    4632
    1
      ccv_nnc_cmd_exec(CMD_DATA_TRANSFER_FORWARD(), ccv_nnc_no_hint, 0, TENSOR_LIST(x_tensor), TENSOR_LIST(a_tensor), 0);
    4633
    1
      ccv_nnc_tensor_t* const b_tensor = ccv_nnc_tensor_from_symbol(tensor_arena, b);
    4634
    1
      ccv_nnc_cmd_exec(CMD_DATA_TRANSFER_FORWARD(), ccv_nnc_no_hint, 0, TENSOR_LIST(y_tensor), TENSOR_LIST(b_tensor), 0);
    4635
    1
      ccv_nnc_graph_run(graph, 0, TRAVERSE_FULL, 0, 0);
    4636
    1
      ccv_nnc_tensor_t* const z_tensor = ccv_nnc_tensor_new(0, CPU_TENSOR_NCHW(32F, 2, 40000, 8, 16), 0);
    4637
    1
      ccv_nnc_tensor_t* const c_tensor = ccv_nnc_tensor_from_symbol(tensor_arena, c);
    4638
    1
      ccv_nnc_cmd_exec(CMD_DATA_TRANSFER_FORWARD(), ccv_nnc_no_hint, 0, TENSOR_LIST(c_tensor), TENSOR_LIST(z_tensor), 0);
    4639
    1
      ccv_nnc_tensor_t* const tz = ccv_nnc_tensor_new(0, CPU_TENSOR_NCHW(32F, 2, 40000, 8, 16), 0);
    4640
    1
      ccv_nnc_cmd_exec(CMD_CMUL_FORWARD(), ccv_nnc_no_hint, 0, TENSOR_LIST(x_tensor, y_tensor), TENSOR_LIST(tz), 0);
    4641
    1
      REQUIRE_TENSOR_EQ(tz, z_tensor, "gelu from cudnn should match from CPU");
    4642
    1
      ccv_nnc_tensor_free(x_tensor);
    4643
    1
      ccv_nnc_tensor_free(y_tensor);
    4644
    1
      ccv_nnc_tensor_free(z_tensor);
    4645
    1
      ccv_nnc_tensor_free(tz);
    4646
    1
      ccv_nnc_graph_free(graph);
    4647
    1
      ccv_nnc_tensor_arena_free(tensor_arena);
    4648
    1
      ccv_nnc_graph_exec_arena_free(graph_exec_arena);
    4649
    1
      ccv_nnc_symbolic_graph_free(symbolic_graph);
    4650
    1
    }
    4651
    4652
    TEST_CASE("cmul gradient in float")
    4653
    1
    {
    4654
    1
      GUARD_ELSE_RETURN(ccv_nnc_cmd_ok(CCV_NNC_CMUL_BACKWARD, CCV_NNC_BACKEND_GPU_REF) || ccv_nnc_cmd_ok(CCV_NNC_CMUL_BACKWARD, CCV_NNC_BACKEND_MPS));
    4655
    1
      ccv_nnc_symbolic_graph_t* const symbolic_graph = ccv_nnc_symbolic_graph_new();
    4656
    1
      ccv_nnc_tensor_symbol_t a = ccv_nnc_tensor_symbol_new(symbolic_graph, GPU_TENSOR_NCHW(000, 32F, 20, 10), "a");
    4657
    1
      ccv_nnc_tensor_symbol_t b = ccv_nnc_tensor_symbol_new(symbolic_graph, GPU_TENSOR_NCHW(000, 32F, 20, 10), "b");
    4658
    1
      ccv_nnc_tensor_symbol_t c = ccv_nnc_tensor_symbol_new(symbolic_graph, GPU_TENSOR_NCHW(000, 32F, 20, 10), "c");
    4659
    1
      ccv_nnc_tensor_symbol_t d = ccv_nnc_tensor_symbol_new(symbolic_graph, GPU_TENSOR_NCHW(000, 32F, 20, 10), "d");
    4660
    1
      ccv_nnc_tensor_symbol_t e = ccv_nnc_tensor_symbol_new(symbolic_graph, GPU_TENSOR_NCHW(000, 32F, 20, 10), "e");
    4661
    1
      ccv_nnc_graph_exec_symbol_new(symbolic_graph, CMD_CMUL_BACKWARD(), TENSOR_SYMBOL_LIST(a, b, c), TENSOR_SYMBOL_LIST(d, e), "cmul");
    4662
    1
      ccv_nnc_graph_exec_symbol_autogen(symbolic_graph, 0, 0, CCV_NNC_AUTOGEN_ALL_EXECS | CCV_NNC_AUTOGEN_SOURCES_AND_DESTINATIONS);
    4663
    1
      SYMBOLIC_GRAPH_GEN(symbolic_graph, CCV_NNC_LONG_DOT_GRAPH);
    4664
    1
      ccv_nnc_graph_t* graph = 0;
    4665
    1
      ccv_nnc_tensor_arena_t* tensor_arena = 0;
    4666
    1
      ccv_nnc_graph_exec_arena_t* graph_exec_arena = 0;
    4667
    1
      ccv_nnc_symbolic_graph_compile(symbolic_graph, ccv_nnc_default_compile_params, 0, 0, 0, 0, SYMBOLIC_GRAPH_SOURCES(symbolic_graph), SYMBOLIC_GRAPH_DESTINATIONS(symbolic_graph), &graph, &tensor_arena, &graph_exec_arena);
    4668
    1
      GRAPH_GEN(graph, CCV_NNC_LONG_DOT_GRAPH);
    4669
    1
      ccv_nnc_tensor_t* const x_tensor = ccv_nnc_tensor_new(0, CPU_TENSOR_NCHW(32F, 20, 10), 0);
    4670
    1
      ccv_nnc_tensor_t* const y_tensor = ccv_nnc_tensor_new(0, CPU_TENSOR_NCHW(32F, 20, 10), 0);
    4671
    1
      ccv_nnc_tensor_t* const z_tensor = ccv_nnc_tensor_new(0, CPU_TENSOR_NCHW(32F, 20, 10), 0);
    4672
    1
      dsfmt_t dsfmt;
    4673
    1
      dsfmt_init_gen_rand(&dsfmt, 0);
    4674
    1
      int i;
    4675
    201
      for (i = 0; i < 20 * 10; i++200)
    4676
    200
        x_tensor->data.f32[i] = dsfmt_genrand_open_close(&dsfmt);
    4677
    201
      for (i = 0; i < 20 * 10; i++200)
    4678
    200
        y_tensor->data.f32[i] = dsfmt_genrand_open_close(&dsfmt);
    4679
    201
      for (i = 0; i < 20 * 10; i++200)
    4680
    200
        z_tensor->data.f32[i] = dsfmt_genrand_open_close(&dsfmt);
    4681
    1
      ccv_nnc_tensor_t* const a_tensor = ccv_nnc_tensor_from_symbol(tensor_arena, a);
    4682
    1
      ccv_nnc_cmd_exec(CMD_DATA_TRANSFER_FORWARD(), ccv_nnc_no_hint, 0, TENSOR_LIST(x_tensor), TENSOR_LIST(a_tensor), 0);
    4683
    1
      ccv_nnc_tensor_t* const b_tensor = ccv_nnc_tensor_from_symbol(tensor_arena, b);
    4684
    1
      ccv_nnc_cmd_exec(CMD_DATA_TRANSFER_FORWARD(), ccv_nnc_no_hint, 0, TENSOR_LIST(y_tensor), TENSOR_LIST(b_tensor), 0);
    4685
    1
      ccv_nnc_tensor_t* const c_tensor = ccv_nnc_tensor_from_symbol(tensor_arena, c);
    4686
    1
      ccv_nnc_cmd_exec(CMD_DATA_TRANSFER_FORWARD(), ccv_nnc_no_hint, 0, TENSOR_LIST(z_tensor), TENSOR_LIST(c_tensor), 0);
    4687
    1
      ccv_nnc_graph_run(graph, 0, TRAVERSE_FULL, 0, 0);
    4688
    1
      ccv_nnc_tensor_t* const od_tensor = ccv_nnc_tensor_new(0, CPU_TENSOR_NCHW(32F, 20, 10), 0);
    4689
    1
      ccv_nnc_tensor_t* const d_tensor = ccv_nnc_tensor_from_symbol(tensor_arena, d);
    4690
    1
      ccv_nnc_tensor_t* const oe_tensor = ccv_nnc_tensor_new(0, CPU_TENSOR_NCHW(32F, 20, 10), 0);
    4691
    1
      ccv_nnc_tensor_t* const e_tensor = ccv_nnc_tensor_from_symbol(tensor_arena, e);
    4692
    1
      ccv_nnc_cmd_exec(CMD_DATA_TRANSFER_FORWARD(), ccv_nnc_no_hint, 0, TENSOR_LIST(d_tensor, e_tensor), TENSOR_LIST(od_tensor, oe_tensor), 0);
    4693
    1
      ccv_nnc_tensor_t* const td = ccv_nnc_tensor_new(0, CPU_TENSOR_NCHW(32F, 20, 10), 0);
    4694
    1
      ccv_nnc_tensor_t* const te = ccv_nnc_tensor_new(0, CPU_TENSOR_NCHW(32F, 20, 10), 0);
    4695
    1
      ccv_nnc_cmd_exec(CMD_CMUL_BACKWARD(), ccv_nnc_no_hint, 0, TENSOR_LIST(x_tensor, y_tensor, z_tensor), TENSOR_LIST(td, te), 0);
    4696
    1
      REQUIRE_TENSOR_EQ(td, od_tensor, "cmul gradient from cudnn should match from CPU");
    4697
    1
      REQUIRE_TENSOR_EQ(te, oe_tensor, "cmul gradient from cudnn should match from CPU");
    4698
    1
      ccv_nnc_tensor_free(x_tensor);
    4699
    1
      ccv_nnc_tensor_free(y_tensor);
    4700
    1
      ccv_nnc_tensor_free(z_tensor);
    4701
    1
      ccv_nnc_tensor_free(od_tensor);
    4702
    1
      ccv_nnc_tensor_free(oe_tensor);
    4703
    1
      ccv_nnc_tensor_free(td);
    4704
    1
      ccv_nnc_tensor_free(te);
    4705
    1
      ccv_nnc_graph_free(graph);
    4706
    1
      ccv_nnc_tensor_arena_free(tensor_arena);
    4707
    1
      ccv_nnc_graph_exec_arena_free(graph_exec_arena);
    4708
    1
      ccv_nnc_symbolic_graph_free(symbolic_graph);
    4709
    1
    }
    4710
    4711
    TEST_CASE("cmul gradient in half precision")
    4712
    1
    {
    4713
    1
      GUARD_ELSE_RETURN(ccv_nnc_cmd_ok(CCV_NNC_CMUL_BACKWARD, CCV_NNC_BACKEND_GPU_REF) || ccv_nnc_cmd_ok(CCV_NNC_CMUL_BACKWARD, CCV_NNC_BACKEND_MPS));
    4714
    1
      ccv_nnc_symbolic_graph_t* const symbolic_graph = ccv_nnc_symbolic_graph_new();
    4715
    1
      ccv_nnc_tensor_symbol_t a = ccv_nnc_tensor_symbol_new(symbolic_graph, GPU_TENSOR_NCHW(000, 16F, 20, 10), "a");
    4716
    1
      ccv_nnc_tensor_symbol_t b = ccv_nnc_tensor_symbol_new(symbolic_graph, GPU_TENSOR_NCHW(000, 16F, 20, 10), "b");
    4717
    1
      ccv_nnc_tensor_symbol_t c = ccv_nnc_tensor_symbol_new(symbolic_graph, GPU_TENSOR_NCHW(000, 16F, 20, 10), "c");
    4718
    1
      ccv_nnc_tensor_symbol_t d = ccv_nnc_tensor_symbol_new(symbolic_graph, GPU_TENSOR_NCHW(000, 16F, 20, 10), "c");
    4719
    1
      ccv_nnc_tensor_symbol_t e = ccv_nnc_tensor_symbol_new(symbolic_graph, GPU_TENSOR_NCHW(000, 16F, 20, 10), "c");
    4720
    1
      ccv_nnc_graph_exec_symbol_new(symbolic_graph, CMD_CMUL_BACKWARD(), TENSOR_SYMBOL_LIST(a, b, c), TENSOR_SYMBOL_LIST(d, e), "cmul");
    4721
    1
      ccv_nnc_graph_exec_symbol_autogen(symbolic_graph, 0, 0, CCV_NNC_AUTOGEN_ALL_EXECS | CCV_NNC_AUTOGEN_SOURCES_AND_DESTINATIONS);
    4722
    1
      SYMBOLIC_GRAPH_GEN(symbolic_graph, CCV_NNC_LONG_DOT_GRAPH);
    4723
    1
      ccv_nnc_graph_t* graph = 0;
    4724
    1
      ccv_nnc_tensor_arena_t* tensor_arena = 0;
    4725
    1
      ccv_nnc_graph_exec_arena_t* graph_exec_arena = 0;
    4726
    1
      ccv_nnc_symbolic_graph_compile(symbolic_graph, ccv_nnc_default_compile_params, 0, 0, 0, 0, SYMBOLIC_GRAPH_SOURCES(symbolic_graph), SYMBOLIC_GRAPH_DESTINATIONS(symbolic_graph), &graph, &tensor_arena, &graph_exec_arena);
    4727
    1
      GRAPH_GEN(graph, CCV_NNC_LONG_DOT_GRAPH);
    4728
    1
      ccv_nnc_tensor_t* const x_tensor = ccv_nnc_tensor_new(0, CPU_TENSOR_NCHW(32F, 20, 10), 0);
    4729
    1
      ccv_nnc_tensor_t* const y_tensor = ccv_nnc_tensor_new(0, CPU_TENSOR_NCHW(32F, 20, 10), 0);
    4730
    1
      ccv_nnc_tensor_t* const z_tensor = ccv_nnc_tensor_new(0, CPU_TENSOR_NCHW(32F, 20, 10), 0);
    4731
    1
      dsfmt_t dsfmt;
    4732
    1
      dsfmt_init_gen_rand(&dsfmt, 0);
    4733
    1
      int i;
    4734
    201
      for (i = 0; i < 20 * 10; i++200)
    4735
    200
        x_tensor->data.f32[i] = dsfmt_genrand_open_close(&dsfmt);
    4736
    201
      for (i = 0; i < 20 * 10; i++200)
    4737
    200
        y_tensor->data.f32[i] = dsfmt_genrand_open_close(&dsfmt);
    4738
    201
      for (i = 0; i < 20 * 10; i++200)
    4739
    200
        z_tensor->data.f32[i] = dsfmt_genrand_open_close(&dsfmt);
    4740
    1
      ccv_nnc_tensor_t* const a_tensor = ccv_nnc_tensor_from_symbol(tensor_arena, a);
    4741
    1
      ccv_nnc_tensor_t* const x16_tensor = ccv_nnc_tensor_new(0, CPU_TENSOR_NCHW(16F, 20, 10), 0);
    4742
    1
      ccv_nnc_cmd_exec(CMD_DATATYPE_CONVERSION_FORWARD(), ccv_nnc_no_hint, 0, TENSOR_LIST(x_tensor), TENSOR_LIST(x16_tensor), 0);
    4743
    1
      ccv_nnc_cmd_exec(CMD_DATA_TRANSFER_FORWARD(), ccv_nnc_no_hint, 0, TENSOR_LIST(x16_tensor), TENSOR_LIST(a_tensor), 0);
    4744
    1
      ccv_nnc_tensor_t* const b_tensor = ccv_nnc_tensor_from_symbol(tensor_arena, b);
    4745
    1
      ccv_nnc_tensor_t* const y16_tensor = ccv_nnc_tensor_new(0, CPU_TENSOR_NCHW(16F, 20, 10), 0);
    4746
    1
      ccv_nnc_cmd_exec(CMD_DATATYPE_CONVERSION_FORWARD(), ccv_nnc_no_hint, 0, TENSOR_LIST(y_tensor), TENSOR_LIST(y16_tensor), 0);
    4747
    1
      ccv_nnc_cmd_exec(CMD_DATA_TRANSFER_FORWARD(), ccv_nnc_no_hint, 0, TENSOR_LIST(y16_tensor), TENSOR_LIST(b_tensor), 0);
    4748
    1
      ccv_nnc_tensor_t* const c_tensor = ccv_nnc_tensor_from_symbol(tensor_arena, c);
    4749
    1
      ccv_nnc_tensor_t* const z16_tensor = ccv_nnc_tensor_new(0, CPU_TENSOR_NCHW(16F, 20, 10), 0);
    4750
    1
      ccv_nnc_cmd_exec(CMD_DATATYPE_CONVERSION_FORWARD(), ccv_nnc_no_hint, 0, TENSOR_LIST(z_tensor), TENSOR_LIST(z16_tensor), 0);
    4751
    1
      ccv_nnc_cmd_exec(CMD_DATA_TRANSFER_FORWARD(), ccv_nnc_no_hint, 0, TENSOR_LIST(z16_tensor), TENSOR_LIST(c_tensor), 0);
    4752
    1
      ccv_nnc_graph_run(graph, 0, TRAVERSE_FULL, 0, 0);
    4753
    1
      ccv_nnc_tensor_t* const od16_tensor = ccv_nnc_tensor_new(0, CPU_TENSOR_NCHW(16F, 20, 10), 0);
    4754
    1
      ccv_nnc_tensor_t* const od_tensor = ccv_nnc_tensor_new(0, CPU_TENSOR_NCHW(32F, 20, 10), 0);
    4755
    1
      ccv_nnc_tensor_t* const d_tensor = ccv_nnc_tensor_from_symbol(tensor_arena, d);
    4756
    1
      ccv_nnc_cmd_exec(CMD_DATA_TRANSFER_FORWARD(), ccv_nnc_no_hint, 0, TENSOR_LIST(d_tensor), TENSOR_LIST(od16_tensor), 0);
    4757
    1
      ccv_nnc_cmd_exec(CMD_DATATYPE_CONVERSION_FORWARD(), ccv_nnc_no_hint, 0, TENSOR_LIST(od16_tensor), TENSOR_LIST(od_tensor), 0);
    4758
    1
      ccv_nnc_tensor_t* const oe16_tensor = ccv_nnc_tensor_new(0, CPU_TENSOR_NCHW(16F, 20, 10), 0);
    4759
    1
      ccv_nnc_tensor_t* const oe_tensor = ccv_nnc_tensor_new(0, CPU_TENSOR_NCHW(32F, 20, 10), 0);
    4760
    1
      ccv_nnc_tensor_t* const e_tensor = ccv_nnc_tensor_from_symbol(tensor_arena, e);
    4761
    1
      ccv_nnc_cmd_exec(CMD_DATA_TRANSFER_FORWARD(), ccv_nnc_no_hint, 0, TENSOR_LIST(e_tensor), TENSOR_LIST(oe16_tensor), 0);
    4762
    1
      ccv_nnc_cmd_exec(CMD_DATATYPE_CONVERSION_FORWARD(), ccv_nnc_no_hint, 0, TENSOR_LIST(oe16_tensor), TENSOR_LIST(oe_tensor), 0);
    4763
    1
      ccv_nnc_tensor_t* const td = ccv_nnc_tensor_new(0, CPU_TENSOR_NCHW(32F, 20, 10), 0);
    4764
    1
      ccv_nnc_tensor_t* const te = ccv_nnc_tensor_new(0, CPU_TENSOR_NCHW(32F, 20, 10), 0);
    4765
    1
      ccv_nnc_cmd_exec(CMD_CMUL_BACKWARD(), ccv_nnc_no_hint, 0, TENSOR_LIST(x_tensor, y_tensor, z_tensor), TENSOR_LIST(td, te), 0);
    4766
    1
      REQUIRE_ARRAY_EQ_WITH_TOLERANCE(float, td->data.f32, od_tensor->data.f32, 20 * 10, 2e-3, "gelu from cudnn should match from CPU");
    4767
    1
      REQUIRE_ARRAY_EQ_WITH_TOLERANCE(float, te->data.f32, oe_tensor->data.f32, 20 * 10, 2e-3, "gelu from cudnn should match from CPU");
    4768
    1
      ccv_nnc_tensor_free(x_tensor);
    4769
    1
      ccv_nnc_tensor_free(x16_tensor);
    4770
    1
      ccv_nnc_tensor_free(y_tensor);
    4771
    1
      ccv_nnc_tensor_free(y16_tensor);
    4772
    1
      ccv_nnc_tensor_free(z_tensor);
    4773
    1
      ccv_nnc_tensor_free(z16_tensor);
    4774
    1
      ccv_nnc_tensor_free(od_tensor);
    4775
    1
      ccv_nnc_tensor_free(od16_tensor);
    4776
    1
      ccv_nnc_tensor_free(td);
    4777
    1
      ccv_nnc_tensor_free(oe_tensor);
    4778
    1
      ccv_nnc_tensor_free(oe16_tensor);
    4779
    1
      ccv_nnc_tensor_free(te);
    4780
    1
      ccv_nnc_graph_free(graph);
    4781
    1
      ccv_nnc_tensor_arena_free(tensor_arena);
    4782
    1
      ccv_nnc_graph_exec_arena_free(graph_exec_arena);
    4783
    1
      ccv_nnc_symbolic_graph_free(symbolic_graph);
    4784
    1
    }
    4785
    4786
    TEST_CASE("segmented gemm")
    4787
    1
    {
    4788
    1
      GUARD_ELSE_RETURN(ccv_nnc_cmd_ok(CCV_NNC_SEGMENTED_GEMM_FORWARD, CCV_NNC_BACKEND_GPU_CUBLAS) || ccv_nnc_cmd_ok(CCV_NNC_SEGMENTED_GEMM_FORWARD, CCV_NNC_BACKEND_MPS));
    4789
    1
      dsfmt_t dsfmt;
    4790
    1
      dsfmt_init_gen_rand(&dsfmt, 0);
    4791
    1
      ccv_nnc_tensor_t* ha = ccv_nnc_tensor_new(0, CPU_TENSOR_NHWC(32F, 80, 128), 0);
    4792
    1
      ccv_nnc_tensor_t* hindices = ccv_nnc_tensor_new(0, CPU_TENSOR_NHWC(32S, 3), 0);
    4793
    1
      hindices->data.i32[0] = 0;
    4794
    1
      hindices->data.i32[1] = 2;
    4795
    1
      hindices->data.i32[2] = 1;
    4796
    1
      ccv_nnc_tensor_t* hcounts = ccv_nnc_tensor_new(0, CPU_TENSOR_NHWC(32S, 3), 0);
    4797
    1
      hcounts->data.i32[0] = 20;
    4798
    1
      hcounts->data.i32[1] = 25;
    4799
    1
      hcounts->data.i32[2] = 35;
    4800
    1
      ccv_nnc_tensor_t* hw = ccv_nnc_tensor_new(0, CPU_TENSOR_NHWC(32F, 3, 64, 128), 0);
    4801
    1
      ccv_nnc_tensor_t* hb = ccv_nnc_tensor_new(0, CPU_TENSOR_NHWC(32F, 80, 64), 0);
    4802
    1
      ccv_nnc_tensor_t* bt = ccv_nnc_tensor_new(0, CPU_TENSOR_NHWC(32F, 80, 64), 0);
    4803
    1
      int i;
    4804
    24.5k
      for (i = 0; i < 3 * 64 * 128; i++24.5k)
    4805
    24.5k
        hw->data.f32[i] = dsfmt_genrand_open_close(&dsfmt) / (64 * 128);
    4806
    10.2k
      for (i = 0; i < 80 * 128; i++10.2k)
    4807
    10.2k
        ha->data.f32[i] = dsfmt_genrand_open_close(&dsfmt);
    4808
    1
      ccv_nnc_tensor_t* a = ccv_nnc_tensor_new(0, GPU_TENSOR_NHWC(000, 32F, 80, 128), 0);
    4809
    1
      ccv_nnc_tensor_t* indices = ccv_nnc_tensor_new(0, GPU_TENSOR_NHWC(000, 32S, 3), 0);
    4810
    1
      ccv_nnc_tensor_t* counts = ccv_nnc_tensor_new(0, GPU_TENSOR_NHWC(000, 32S, 3), 0);
    4811
    1
      ccv_nnc_tensor_t* w = ccv_nnc_tensor_new(0, GPU_TENSOR_NHWC(000, 32F, 3, 64, 128), 0);
    4812
    1
      ccv_nnc_tensor_t* b = ccv_nnc_tensor_new(0, GPU_TENSOR_NHWC(000, 32F, 80, 64), 0);
    4813
    1
      ccv_nnc_cmd_exec(CMD_DATA_TRANSFER_FORWARD(), ccv_nnc_no_hint, 0, TENSOR_LIST(ha, hindices, hcounts, hw), TENSOR_LIST(a, indices, counts, w), 0);
    4814
    1
      ccv_nnc_cmd_exec(CMD_SEGMENTED_GEMM_FORWARD(NO_TRANSPOSE, TRANSPOSE(1, 2)), ccv_nnc_no_hint, 0, TENSOR_LIST(a, indices, counts, w), TENSOR_LIST(b), 0);
    4815
    1
      ccv_nnc_cmd_exec(CMD_DATA_TRANSFER_FORWARD(), ccv_nnc_no_hint, 0, TENSOR_LIST(b), TENSOR_LIST(hb), 0);
    4816
    1
      ccv_nnc_cmd_exec(CMD_SEGMENTED_GEMM_FORWARD(NO_TRANSPOSE, TRANSPOSE(1, 2)), ccv_nnc_no_hint, 0, TENSOR_LIST(ha, hindices, hcounts, hw), TENSOR_LIST(bt), 0);
    4817
    1
      REQUIRE_TENSOR_EQ(hb, bt, "should match from CPU");
    4818
    1
      ccv_nnc_tensor_free(a);
    4819
    1
      ccv_nnc_tensor_free(indices);
    4820
    1
      ccv_nnc_tensor_free(counts);
    4821
    1
      ccv_nnc_tensor_free(w);
    4822
    1
      ccv_nnc_tensor_free(b);
    4823
    1
      ccv_nnc_tensor_free(ha);
    4824
    1
      ccv_nnc_tensor_free(hindices);
    4825
    1
      ccv_nnc_tensor_free(hcounts);
    4826
    1
      ccv_nnc_tensor_free(hw);
    4827
    1
      ccv_nnc_tensor_free(hb);
    4828
    1
      ccv_nnc_tensor_free(bt);
    4829
    1
    }
    4830
    4831
    TEST_CASE("segmented gemm with bias")
    4832
    1
    {
    4833
    1
      GUARD_ELSE_RETURN(ccv_nnc_cmd_ok(CCV_NNC_SEGMENTED_GEMM_FORWARD, CCV_NNC_BACKEND_GPU_CUBLAS) || ccv_nnc_cmd_ok(CCV_NNC_SEGMENTED_GEMM_FORWARD, CCV_NNC_BACKEND_MPS));
    4834
    1
      dsfmt_t dsfmt;
    4835
    1
      dsfmt_init_gen_rand(&dsfmt, 0);
    4836
    1
      ccv_nnc_tensor_t* ha = ccv_nnc_tensor_new(0, CPU_TENSOR_NHWC(32F, 80, 128), 0);
    4837
    1
      ccv_nnc_tensor_t* hindices = ccv_nnc_tensor_new(0, CPU_TENSOR_NHWC(32S, 3), 0);
    4838
    1
      hindices->data.i32[0] = 0;
    4839
    1
      hindices->data.i32[1] = 1;
    4840
    1
      hindices->data.i32[2] = 2;
    4841
    1
      ccv_nnc_tensor_t* hcounts = ccv_nnc_tensor_new(0, CPU_TENSOR_NHWC(32S, 3), 0);
    4842
    1
      hcounts->data.i32[0] = 20;
    4843
    1
      hcounts->data.i32[1] = 25;
    4844
    1
      hcounts->data.i32[2] = 35;
    4845
    1
      ccv_nnc_tensor_t* hw = ccv_nnc_tensor_new(0, CPU_TENSOR_NHWC(32F, 3, 64, 128), 0);
    4846
    1
      ccv_nnc_tensor_t* hbias = ccv_nnc_tensor_new(0, CPU_TENSOR_NHWC(32F, 3, 64), 0);
    4847
    1
      ccv_nnc_tensor_t* hb = ccv_nnc_tensor_new(0, CPU_TENSOR_NHWC(32F, 80, 64), 0);
    4848
    1
      ccv_nnc_tensor_t* bt = ccv_nnc_tensor_new(0, CPU_TENSOR_NHWC(32F, 80, 64), 0);
    4849
    1
      int i;
    4850
    24.5k
      for (i = 0; i < 3 * 64 * 128; i++24.5k)
    4851
    24.5k
        hw->data.f32[i] = dsfmt_genrand_open_close(&dsfmt) / (64 * 128);
    4852
    193
      for (i = 0; i < 3 * 64; i++192)
    4853
    192
        hbias->data.f32[i] = dsfmt_genrand_open_close(&dsfmt) / 64;
    4854
    10.2k
      for (i = 0; i < 80 * 128; i++10.2k)
    4855
    10.2k
        ha->data.f32[i] = dsfmt_genrand_open_close(&dsfmt);
    4856
    1
      ccv_nnc_tensor_t* a = ccv_nnc_tensor_new(0, GPU_TENSOR_NHWC(000, 32F, 80, 128), 0);
    4857
    1
      ccv_nnc_tensor_t* indices = ccv_nnc_tensor_new(0, GPU_TENSOR_NHWC(000, 32S, 3), 0);
    4858
    1
      ccv_nnc_tensor_t* counts = ccv_nnc_tensor_new(0, GPU_TENSOR_NHWC(000, 32S, 3), 0);
    4859
    1
      ccv_nnc_tensor_t* w = ccv_nnc_tensor_new(0, GPU_TENSOR_NHWC(000, 32F, 3, 64, 128), 0);
    4860
    1
      ccv_nnc_tensor_t* bias = ccv_nnc_tensor_new(0, GPU_TENSOR_NHWC(000, 32F, 3, 64), 0);
    4861
    1
      ccv_nnc_tensor_t* b = ccv_nnc_tensor_new(0, GPU_TENSOR_NHWC(000, 32F, 80, 64), 0);
    4862
    1
      ccv_nnc_cmd_exec(CMD_DATA_TRANSFER_FORWARD(), ccv_nnc_no_hint, 0, TENSOR_LIST(ha, hindices, hcounts, hw, hbias), TENSOR_LIST(a, indices, counts, w, bias), 0);
    4863
    1
      ccv_nnc_cmd_exec(CMD_SEGMENTED_GEMM_FORWARD(NO_TRANSPOSE, TRANSPOSE(1, 2)), ccv_nnc_no_hint, 0, TENSOR_LIST(a, indices, counts, w, bias), TENSOR_LIST(b), 0);
    4864
    1
      ccv_nnc_cmd_exec(CMD_DATA_TRANSFER_FORWARD(), ccv_nnc_no_hint, 0, TENSOR_LIST(b), TENSOR_LIST(hb), 0);
    4865
    1
      ccv_nnc_cmd_exec(CMD_SEGMENTED_GEMM_FORWARD(NO_TRANSPOSE, TRANSPOSE(1, 2)), ccv_nnc_no_hint, 0, TENSOR_LIST(ha, hindices, hcounts, hw, hbias), TENSOR_LIST(bt), 0);
    4866
    1
      REQUIRE_TENSOR_EQ(hb, bt, "should match from CPU");
    4867
    1
      ccv_nnc_tensor_free(a);
    4868
    1
      ccv_nnc_tensor_free(indices);
    4869
    1
      ccv_nnc_tensor_free(counts);
    4870
    1
      ccv_nnc_tensor_free(w);
    4871
    1
      ccv_nnc_tensor_free(bias);
    4872
    1
      ccv_nnc_tensor_free(b);
    4873
    1
      ccv_nnc_tensor_free(ha);
    4874
    1
      ccv_nnc_tensor_free(hindices);
    4875
    1
      ccv_nnc_tensor_free(hcounts);
    4876
    1
      ccv_nnc_tensor_free(hw);
    4877
    1
      ccv_nnc_tensor_free(hbias);
    4878
    1
      ccv_nnc_tensor_free(hb);
    4879
    1
      ccv_nnc_tensor_free(bt);
    4880
    1
    }
    4881
    4882
    TEST_CASE("segmented gemm in half precision")
    4883
    1
    {
    4884
    1
      GUARD_ELSE_RETURN(ccv_nnc_cmd_ok(CCV_NNC_SEGMENTED_GEMM_FORWARD, CCV_NNC_BACKEND_GPU_CUBLAS) || ccv_nnc_cmd_ok(CCV_NNC_SEGMENTED_GEMM_FORWARD, CCV_NNC_BACKEND_MPS));
    4885
    1
      dsfmt_t dsfmt;
    4886
    1
      dsfmt_init_gen_rand(&dsfmt, 0);
    4887
    1
      ccv_nnc_tensor_t* ha = ccv_nnc_tensor_new(0, CPU_TENSOR_NHWC(32F, 80, 128), 0);
    4888
    1
      ccv_nnc_tensor_t* hindices = ccv_nnc_tensor_new(0, CPU_TENSOR_NHWC(32S, 3), 0);
    4889
    1
      hindices->data.i32[0] = 0;
    4890
    1
      hindices->data.i32[1] = 2;
    4891
    1
      hindices->data.i32[2] = 1;
    4892
    1
      ccv_nnc_tensor_t* hcounts = ccv_nnc_tensor_new(0, CPU_TENSOR_NHWC(32S, 3), 0);
    4893
    1
      hcounts->data.i32[0] = 20;
    4894
    1
      hcounts->data.i32[1] = 25;
    4895
    1
      hcounts->data.i32[2] = 35;
    4896
    1
      ccv_nnc_tensor_t* hw = ccv_nnc_tensor_new(0, CPU_TENSOR_NHWC(32F, 3, 64, 128), 0);
    4897
    1
      ccv_nnc_tensor_t* hb = ccv_nnc_tensor_new(0, CPU_TENSOR_NHWC(32F, 80, 64), 0);
    4898
    1
      ccv_nnc_tensor_t* hb16 = ccv_nnc_tensor_new(0, CPU_TENSOR_NHWC(16F, 80, 64), 0);
    4899
    1
      ccv_nnc_tensor_t* bt = ccv_nnc_tensor_new(0, CPU_TENSOR_NHWC(32F, 80, 64), 0);
    4900
    1
      int i;
    4901
    24.5k
      for (i = 0; i < 3 * 64 * 128; i++24.5k)
    4902
    24.5k
        hw->data.f32[i] = dsfmt_genrand_open_close(&dsfmt) / (64 * 128);
    4903
    10.2k
      for (i = 0; i < 80 * 128; i++10.2k)
    4904
    10.2k
        ha->data.f32[i] = dsfmt_genrand_open_close(&dsfmt);
    4905
    1
      ccv_nnc_tensor_t* ha16 = ccv_nnc_tensor_new(0, CPU_TENSOR_NHWC(16F, 80, 128), 0);
    4906
    1
      ccv_nnc_tensor_t* hw16 = ccv_nnc_tensor_new(0, CPU_TENSOR_NHWC(16F, 3, 64, 128), 0);
    4907
    1
      ccv_nnc_cmd_exec(CMD_DATATYPE_CONVERSION_FORWARD(), ccv_nnc_no_hint, 0, TENSOR_LIST(ha, hw), TENSOR_LIST(ha16, hw16), 0);
    4908
    1
      ccv_nnc_tensor_t* a = ccv_nnc_tensor_new(0, GPU_TENSOR_NHWC(000, 16F, 80, 128), 0);
    4909
    1
      ccv_nnc_tensor_t* indices = ccv_nnc_tensor_new(0, GPU_TENSOR_NHWC(000, 32S, 3), 0);
    4910
    1
      ccv_nnc_tensor_t* counts = ccv_nnc_tensor_new(0, GPU_TENSOR_NHWC(000, 32S, 3), 0);
    4911
    1
      ccv_nnc_tensor_t* w = ccv_nnc_tensor_new(0, GPU_TENSOR_NHWC(000, 16F, 3, 64, 128), 0);
    4912
    1
      ccv_nnc_tensor_t* b = ccv_nnc_tensor_new(0, GPU_TENSOR_NHWC(000, 16F, 80, 64), 0);
    4913
    1
      ccv_nnc_cmd_exec(CMD_DATA_TRANSFER_FORWARD(), ccv_nnc_no_hint, 0, TENSOR_LIST(ha16, hindices, hcounts, hw16), TENSOR_LIST(a, indices, counts, w), 0);
    4914
    1
      ccv_nnc_cmd_exec(CMD_SEGMENTED_GEMM_FORWARD(NO_TRANSPOSE, TRANSPOSE(1, 2)), ccv_nnc_no_hint, 0, TENSOR_LIST(a, indices, counts, w), TENSOR_LIST(b), 0);
    4915
    1
      ccv_nnc_cmd_exec(CMD_DATA_TRANSFER_FORWARD(), ccv_nnc_no_hint, 0, TENSOR_LIST(b), TENSOR_LIST(hb16), 0);
    4916
    1
      ccv_nnc_cmd_exec(CMD_DATATYPE_CONVERSION_FORWARD(), ccv_nnc_no_hint, 0, TENSOR_LIST(hb16), TENSOR_LIST(hb), 0);
    4917
    1
      ccv_nnc_cmd_exec(CMD_SEGMENTED_GEMM_FORWARD(NO_TRANSPOSE, TRANSPOSE(1, 2)), ccv_nnc_no_hint, 0, TENSOR_LIST(ha, hindices, hcounts, hw), TENSOR_LIST(bt), 0);
    4918
    1
      REQUIRE_ARRAY_EQ_WITH_TOLERANCE(float, hb->data.f32, bt->data.f32, 80 * 64, 1e-3, "should match from CPU");
    4919
    1
      ccv_nnc_tensor_free(a);
    4920
    1
      ccv_nnc_tensor_free(indices);
    4921
    1
      ccv_nnc_tensor_free(counts);
    4922
    1
      ccv_nnc_tensor_free(w);
    4923
    1
      ccv_nnc_tensor_free(b);
    4924
    1
      ccv_nnc_tensor_free(ha);
    4925
    1
      ccv_nnc_tensor_free(hindices);
    4926
    1
      ccv_nnc_tensor_free(hcounts);
    4927
    1
      ccv_nnc_tensor_free(hw);
    4928
    1
      ccv_nnc_tensor_free(hb);
    4929
    1
      ccv_nnc_tensor_free(ha16);
    4930
    1
      ccv_nnc_tensor_free(hw16);
    4931
    1
      ccv_nnc_tensor_free(hb16);
    4932
    1
      ccv_nnc_tensor_free(bt);
    4933
    1
    }
    4934
    4935
    TEST_CASE("segmented gemm with bias in half precision")
    4936
    1
    {
    4937
    1
      GUARD_ELSE_RETURN(ccv_nnc_cmd_ok(CCV_NNC_SEGMENTED_GEMM_FORWARD, CCV_NNC_BACKEND_GPU_CUBLAS) || ccv_nnc_cmd_ok(CCV_NNC_SEGMENTED_GEMM_FORWARD, CCV_NNC_BACKEND_MPS));
    4938
    1
      dsfmt_t dsfmt;
    4939
    1
      dsfmt_init_gen_rand(&dsfmt, 0);
    4940
    1
      ccv_nnc_tensor_t* ha = ccv_nnc_tensor_new(0, CPU_TENSOR_NHWC(32F, 80, 128), 0);
    4941
    1
      ccv_nnc_tensor_t* hindices = ccv_nnc_tensor_new(0, CPU_TENSOR_NHWC(32S, 3), 0);
    4942
    1
      hindices->data.i32[0] = 0;
    4943
    1
      hindices->data.i32[1] = 1;
    4944
    1
      hindices->data.i32[2] = 2;
    4945
    1
      ccv_nnc_tensor_t* hcounts = ccv_nnc_tensor_new(0, CPU_TENSOR_NHWC(32S, 3), 0);
    4946
    1
      hcounts->data.i32[0] = 20;
    4947
    1
      hcounts->data.i32[1] = 25;
    4948
    1
      hcounts->data.i32[2] = 35;
    4949
    1
      ccv_nnc_tensor_t* hw = ccv_nnc_tensor_new(0, CPU_TENSOR_NHWC(32F, 3, 64, 128), 0);
    4950
    1
      ccv_nnc_tensor_t* hbias = ccv_nnc_tensor_new(0, CPU_TENSOR_NHWC(32F, 3, 64), 0);
    4951
    1
      ccv_nnc_tensor_t* hb = ccv_nnc_tensor_new(0, CPU_TENSOR_NHWC(32F, 80, 64), 0);
    4952
    1
      ccv_nnc_tensor_t* hb16 = ccv_nnc_tensor_new(0, CPU_TENSOR_NHWC(16F, 80, 64), 0);
    4953
    1
      ccv_nnc_tensor_t* bt = ccv_nnc_tensor_new(0, CPU_TENSOR_NHWC(32F, 80, 64), 0);
    4954
    1
      int i;
    4955
    24.5k
      for (i = 0; i < 3 * 64 * 128; i++24.5k)
    4956
    24.5k
        hw->data.f32[i] = dsfmt_genrand_open_close(&dsfmt) / (64 * 128);
    4957
    193
      for (i = 0; i < 3 * 64; i++192)
    4958
    192
        hbias->data.f32[i] = dsfmt_genrand_open_close(&dsfmt) / 64;
    4959
    10.2k
      for (i = 0; i < 80 * 128; i++10.2k)
    4960
    10.2k
        ha->data.f32[i] = dsfmt_genrand_open_close(&dsfmt);
    4961
    1
      ccv_nnc_tensor_t* ha16 = ccv_nnc_tensor_new(0, CPU_TENSOR_NHWC(16F, 80, 128), 0);
    4962
    1
      ccv_nnc_tensor_t* hw16 = ccv_nnc_tensor_new(0, CPU_TENSOR_NHWC(16F, 3, 64, 128), 0);
    4963
    1
      ccv_nnc_tensor_t* hbias16 = ccv_nnc_tensor_new(0, CPU_TENSOR_NHWC(16F, 3, 64), 0);
    4964
    1
      ccv_nnc_cmd_exec(CMD_DATATYPE_CONVERSION_FORWARD(), ccv_nnc_no_hint, 0, TENSOR_LIST(ha, hw, hbias), TENSOR_LIST(ha16, hw16, hbias16), 0);
    4965
    1
      ccv_nnc_tensor_t* a = ccv_nnc_tensor_new(0, GPU_TENSOR_NHWC(000, 16F, 80, 128), 0);
    4966
    1
      ccv_nnc_tensor_t* indices = ccv_nnc_tensor_new(0, GPU_TENSOR_NHWC(000, 32S, 3), 0);
    4967
    1
      ccv_nnc_tensor_t* counts = ccv_nnc_tensor_new(0, GPU_TENSOR_NHWC(000, 32S, 3), 0);
    4968
    1
      ccv_nnc_tensor_t* w = ccv_nnc_tensor_new(0, GPU_TENSOR_NHWC(000, 16F, 3, 64, 128), 0);
    4969
    1
      ccv_nnc_tensor_t* bias = ccv_nnc_tensor_new(0, GPU_TENSOR_NHWC(000, 16F, 3, 64), 0);
    4970
    1
      ccv_nnc_tensor_t* b = ccv_nnc_tensor_new(0, GPU_TENSOR_NHWC(000, 16F, 80, 64), 0);
    4971
    1
      ccv_nnc_cmd_exec(CMD_DATA_TRANSFER_FORWARD(), ccv_nnc_no_hint, 0, TENSOR_LIST(ha16, hindices, hcounts, hw16, hbias16), TENSOR_LIST(a, indices, counts, w, bias), 0);
    4972
    1
      ccv_nnc_cmd_exec(CMD_SEGMENTED_GEMM_FORWARD(NO_TRANSPOSE, TRANSPOSE(1, 2)), ccv_nnc_no_hint, 0, TENSOR_LIST(a, indices, counts, w, bias), TENSOR_LIST(b), 0);
    4973
    1
      ccv_nnc_cmd_exec(CMD_DATA_TRANSFER_FORWARD(), ccv_nnc_no_hint, 0, TENSOR_LIST(b), TENSOR_LIST(hb16), 0);
    4974
    1
      ccv_nnc_cmd_exec(CMD_DATATYPE_CONVERSION_FORWARD(), ccv_nnc_no_hint, 0, TENSOR_LIST(hb16), TENSOR_LIST(hb), 0);
    4975
    1
      ccv_nnc_cmd_exec(CMD_SEGMENTED_GEMM_FORWARD(NO_TRANSPOSE, TRANSPOSE(1, 2)), ccv_nnc_no_hint, 0, TENSOR_LIST(ha, hindices, hcounts, hw, hbias), TENSOR_LIST(bt), 0);
    4976
    1
      REQUIRE_ARRAY_EQ_WITH_TOLERANCE(float, hb->data.f32, bt->data.f32, 80 * 64, 1e-3, "should match from CPU");
    4977
    1
      ccv_nnc_tensor_free(a);
    4978
    1
      ccv_nnc_tensor_free(indices);
    4979
    1
      ccv_nnc_tensor_free(counts);
    4980
    1
      ccv_nnc_tensor_free(w);
    4981
    1
      ccv_nnc_tensor_free(bias);
    4982
    1
      ccv_nnc_tensor_free(b);
    4983
    1
      ccv_nnc_tensor_free(ha);
    4984
    1
      ccv_nnc_tensor_free(hindices);
    4985
    1
      ccv_nnc_tensor_free(hcounts);
    4986
    1
      ccv_nnc_tensor_free(hw);
    4987
    1
      ccv_nnc_tensor_free(hbias);
    4988
    1
      ccv_nnc_tensor_free(hb);
    4989
    1
      ccv_nnc_tensor_free(ha16);
    4990
    1
      ccv_nnc_tensor_free(hw16);
    4991
    1
      ccv_nnc_tensor_free(hbias16);
    4992
    1
      ccv_nnc_tensor_free(hb16);
    4993
    1
      ccv_nnc_tensor_free(bt);
    4994
    1
    }
    4995
    4996
    TEST_CASE("segmented gemm, reuse")
    4997
    1
    {
    4998
    1
      GUARD_ELSE_RETURN(ccv_nnc_cmd_ok(CCV_NNC_SEGMENTED_GEMM_FORWARD, CCV_NNC_BACKEND_GPU_CUBLAS) || ccv_nnc_cmd_ok(CCV_NNC_SEGMENTED_GEMM_FORWARD, CCV_NNC_BACKEND_MPS));
    4999
    1
      dsfmt_t dsfmt;
    5000
    1
      dsfmt_init_gen_rand(&dsfmt, 0);
    5001
    1
      ccv_nnc_tensor_t* ha = ccv_nnc_tensor_new(0, CPU_TENSOR_NHWC(32F, 80, 128), 0);
    5002
    1
      ccv_nnc_tensor_t* hindices = ccv_nnc_tensor_new(0, CPU_TENSOR_NHWC(32S, 3), 0);
    5003
    1
      hindices->data.i32[0] = 0;
    5004
    1
      hindices->data.i32[1] = 1;
    5005
    1
      hindices->data.i32[2] = 2;
    5006
    1
      ccv_nnc_tensor_t* hcounts = ccv_nnc_tensor_new(0, CPU_TENSOR_NHWC(32S, 3), 0);
    5007
    1
      hcounts->data.i32[0] = 20;
    5008
    1
      hcounts->data.i32[1] = 30;
    5009
    1
      hcounts->data.i32[2] = 30;
    5010
    1
      ccv_nnc_tensor_t* hw = ccv_nnc_tensor_new(0, CPU_TENSOR_NHWC(32F, 3, 64, 128), 0);
    5011
    1
      ccv_nnc_tensor_t* hb = ccv_nnc_tensor_new(0, CPU_TENSOR_NHWC(32F, 80, 64), 0);
    5012
    1
      ccv_nnc_tensor_t* bt = ccv_nnc_tensor_new(0, CPU_TENSOR_NHWC(32F, 80, 64), 0);
    5013
    1
      int i;
    5014
    24.5k
      for (i = 0; i < 3 * 64 * 128; i++24.5k)
    5015
    24.5k
        hw->data.f32[i] = dsfmt_genrand_open_close(&dsfmt) / (64 * 128);
    5016
    10.2k
      for (i = 0; i < 80 * 128; i++10.2k)
    5017
    10.2k
        ha->data.f32[i] = dsfmt_genrand_open_close(&dsfmt);
    5018
    1
      ccv_nnc_tensor_t* a = ccv_nnc_tensor_new(0, GPU_TENSOR_NHWC(000, 32F, 80, 128), 0);
    5019
    1
      ccv_nnc_tensor_t* indices = ccv_nnc_tensor_new(0, GPU_TENSOR_NHWC(000, 32S, 3), 0);
    5020
    1
      ccv_nnc_tensor_t* counts = ccv_nnc_tensor_new(0, GPU_TENSOR_NHWC(000, 32S, 3), 0);
    5021
    1
      ccv_nnc_tensor_t* w = ccv_nnc_tensor_new(0, GPU_TENSOR_NHWC(000, 32F, 3, 64, 128), 0);
    5022
    1
      ccv_nnc_tensor_t* b = ccv_nnc_tensor_new(0, GPU_TENSOR_NHWC(000, 32F, 80, 64), 0);
    5023
    1
      ccv_nnc_cmd_exec(CMD_DATA_TRANSFER_FORWARD(), ccv_nnc_no_hint, 0, TENSOR_LIST(ha, hindices, hcounts, hw), TENSOR_LIST(a, indices, counts, w), 0);
    5024
    1
      ccv_nnc_cmd_exec(CMD_SEGMENTED_GEMM_FORWARD(NO_TRANSPOSE, TRANSPOSE(1, 2)), ccv_nnc_no_hint, 0, TENSOR_LIST(a, indices, counts, w), TENSOR_LIST(b), 0);
    5025
    1
      ccv_nnc_cmd_exec(CMD_DATA_TRANSFER_FORWARD(), ccv_nnc_no_hint, 0, TENSOR_LIST(b), TENSOR_LIST(hb), 0);
    5026
    1
      ccv_nnc_cmd_exec(CMD_SEGMENTED_GEMM_FORWARD(NO_TRANSPOSE, TRANSPOSE(1, 2)), ccv_nnc_no_hint, 0, TENSOR_LIST(ha, hindices, hcounts, hw), TENSOR_LIST(bt), 0);
    5027
    1
      REQUIRE_TENSOR_EQ(hb, bt, "should match from CPU");
    5028
    1
      ccv_nnc_tensor_free(a);
    5029
    1
      ccv_nnc_tensor_free(indices);
    5030
    1
      ccv_nnc_tensor_free(counts);
    5031
    1
      ccv_nnc_tensor_free(w);
    5032
    1
      ccv_nnc_tensor_free(b);
    5033
    1
      ccv_nnc_tensor_free(ha);
    5034
    1
      ccv_nnc_tensor_free(hindices);
    5035
    1
      ccv_nnc_tensor_free(hcounts);
    5036
    1
      ccv_nnc_tensor_free(hw);
    5037
    1
      ccv_nnc_tensor_free(hb);
    5038
    1
      ccv_nnc_tensor_free(bt);
    5039
    1
    }
    5040
    5041
    TEST_CASE("segmented gemm, large k")
    5042
    1
    {
    5043
    1
      GUARD_ELSE_RETURN(ccv_nnc_cmd_ok(CCV_NNC_SEGMENTED_GEMM_FORWARD, CCV_NNC_BACKEND_GPU_CUBLAS) || ccv_nnc_cmd_ok(CCV_NNC_SEGMENTED_GEMM_FORWARD, CCV_NNC_BACKEND_MPS));
    5044
    1
      dsfmt_t dsfmt;
    5045
    1
      dsfmt_init_gen_rand(&dsfmt, 0);
    5046
    1
      ccv_nnc_tensor_t* ha = ccv_nnc_tensor_new(0, CPU_TENSOR_NHWC(32F, 80, 5120), 0);
    5047
    1
      ccv_nnc_tensor_t* hindices = ccv_nnc_tensor_new(0, CPU_TENSOR_NHWC(32S, 3), 0);
    5048
    1
      hindices->data.i32[0] = 0;
    5049
    1
      hindices->data.i32[1] = 1;
    5050
    1
      hindices->data.i32[2] = 2;
    5051
    1
      ccv_nnc_tensor_t* hcounts = ccv_nnc_tensor_new(0, CPU_TENSOR_NHWC(32S, 3), 0);
    5052
    1
      hcounts->data.i32[0] = 20;
    5053
    1
      hcounts->data.i32[1] = 30;
    5054
    1
      hcounts->data.i32[2] = 30;
    5055
    1
      ccv_nnc_tensor_t* hw = ccv_nnc_tensor_new(0, CPU_TENSOR_NHWC(32F, 3, 2560, 5120), 0);
    5056
    1
      ccv_nnc_tensor_t* hb = ccv_nnc_tensor_new(0, CPU_TENSOR_NHWC(32F, 80, 2560), 0);
    5057
    1
      ccv_nnc_tensor_t* bt = ccv_nnc_tensor_new(0, CPU_TENSOR_NHWC(32F, 80, 2560), 0);
    5058
    1
      int i;
    5059
    39.3M
      for (i = 0; i < 3 * 2560 * 5120; i++39.3M)
    5060
    39.3M
        hw->data.f32[i] = dsfmt_genrand_open_close(&dsfmt) / 5120;
    5061
    409k
      for (i = 0; i < 80 * 5120; i++409k)
    5062
    409k
        ha->data.f32[i] = dsfmt_genrand_open_close(&dsfmt);
    5063
    1
      ccv_nnc_tensor_t* a = ccv_nnc_tensor_new(0, GPU_TENSOR_NHWC(000, 32F, 80, 5120), 0);
    5064
    1
      ccv_nnc_tensor_t* indices = ccv_nnc_tensor_new(0, GPU_TENSOR_NHWC(000, 32S, 3), 0);
    5065
    1
      ccv_nnc_tensor_t* counts = ccv_nnc_tensor_new(0, GPU_TENSOR_NHWC(000, 32S, 3), 0);
    5066
    1
      ccv_nnc_tensor_t* w = ccv_nnc_tensor_new(0, GPU_TENSOR_NHWC(000, 32F, 3, 2560, 5120), 0);
    5067
    1
      ccv_nnc_tensor_t* b = ccv_nnc_tensor_new(0, GPU_TENSOR_NHWC(000, 32F, 80, 2560), 0);
    5068
    1
      ccv_nnc_cmd_exec(CMD_DATA_TRANSFER_FORWARD(), ccv_nnc_no_hint, 0, TENSOR_LIST(ha, hindices, hcounts, hw), TENSOR_LIST(a, indices, counts, w), 0);
    5069
    1
      ccv_nnc_cmd_exec(CMD_SEGMENTED_GEMM_FORWARD(NO_TRANSPOSE, TRANSPOSE(1, 2)), ccv_nnc_no_hint, 0, TENSOR_LIST(a, indices, counts, w), TENSOR_LIST(b), 0);
    5070
    1
      ccv_nnc_cmd_exec(CMD_DATA_TRANSFER_FORWARD(), ccv_nnc_no_hint, 0, TENSOR_LIST(b), TENSOR_LIST(hb), 0);
    5071
    1
      ccv_nnc_cmd_exec(CMD_SEGMENTED_GEMM_FORWARD(NO_TRANSPOSE, TRANSPOSE(1, 2)), ccv_nnc_no_hint, 0, TENSOR_LIST(ha, hindices, hcounts, hw), TENSOR_LIST(bt), 0);
    5072
    1
      REQUIRE_TENSOR_EQ(hb, bt, "should match from CPU");
    5073
    1
      ccv_nnc_tensor_free(a);
    5074
    1
      ccv_nnc_tensor_free(indices);
    5075
    1
      ccv_nnc_tensor_free(counts);
    5076
    1
      ccv_nnc_tensor_free(w);
    5077
    1
      ccv_nnc_tensor_free(b);
    5078
    1
      ccv_nnc_tensor_free(ha);
    5079
    1
      ccv_nnc_tensor_free(hindices);
    5080
    1
      ccv_nnc_tensor_free(hcounts);
    5081
    1
      ccv_nnc_tensor_free(hw);
    5082
    1
      ccv_nnc_tensor_free(hb);
    5083
    1
      ccv_nnc_tensor_free(bt);
    5084
    1
    }
    5085
    5086
    #include "case_main.h"