## Coverage Report

#### Created: 2026-05-04 15:30

next uncovered line (L), next uncovered region (R), next uncovered branch (B)

    /home/liu/actions-runner/_work/ccv/ccv/test/int/nnc/swish.tests.c

    Line
    Count
    Source
    1
    #include "case.h"
    2
    #include "ccv_case.h"
    3
    #include "ccv_nnc_case.h"
    4
    #include <ccv.h>
    5
    #include <nnc/ccv_nnc.h>
    6
    #include <nnc/ccv_nnc_easy.h>
    7
    #include <3rdparty/dsfmt/dSFMT.h>
    8
    9
    TEST_SETUP()
    10
    {
    11
      ccv_nnc_init();
    12
    }
    13
    14
    TEST_CASE("swish in float")
    15
    1
    {
    16
    1
      GUARD_ELSE_RETURN(ccv_nnc_cmd_ok(CCV_NNC_SWISH_FORWARD, CCV_NNC_BACKEND_GPU_REF) || ccv_nnc_cmd_ok(CCV_NNC_SWISH_FORWARD, CCV_NNC_BACKEND_MPS));
    17
    1
      ccv_nnc_symbolic_graph_t* const symbolic_graph = ccv_nnc_symbolic_graph_new();
    18
    1
      ccv_nnc_tensor_symbol_t a = ccv_nnc_tensor_symbol_new(symbolic_graph, GPU_TENSOR_NCHW(000, 32F, 20, 10), "a");
    19
    1
      ccv_nnc_tensor_symbol_t b = ccv_nnc_tensor_symbol_new(symbolic_graph, GPU_TENSOR_NCHW(000, 32F, 20, 10), "b");
    20
    1
      ccv_nnc_graph_exec_symbol_new(symbolic_graph, CMD_SWISH_FORWARD(1), TENSOR_SYMBOL_LIST(a), TENSOR_SYMBOL_LIST(b), "swish");
    21
    1
      ccv_nnc_graph_exec_symbol_autogen(symbolic_graph, 0, 0, CCV_NNC_AUTOGEN_ALL_EXECS | CCV_NNC_AUTOGEN_SOURCES_AND_DESTINATIONS);
    22
    1
      SYMBOLIC_GRAPH_GEN(symbolic_graph, CCV_NNC_LONG_DOT_GRAPH);
    23
    1
      ccv_nnc_graph_t* graph = 0;
    24
    1
      ccv_nnc_tensor_arena_t* tensor_arena = 0;
    25
    1
      ccv_nnc_graph_exec_arena_t* graph_exec_arena = 0;
    26
    1
      ccv_nnc_symbolic_graph_compile(symbolic_graph, ccv_nnc_default_compile_params, 0, 0, 0, 0, SYMBOLIC_GRAPH_SOURCES(symbolic_graph), SYMBOLIC_GRAPH_DESTINATIONS(symbolic_graph), &graph, &tensor_arena, &graph_exec_arena);
    27
    1
      GRAPH_GEN(graph, CCV_NNC_LONG_DOT_GRAPH);
    28
    1
      ccv_nnc_tensor_t* const x_tensor = ccv_nnc_tensor_new(0, CPU_TENSOR_NCHW(32F, 20, 10), 0);
    29
    1
      dsfmt_t dsfmt;
    30
    1
      dsfmt_init_gen_rand(&dsfmt, 0);
    31
    1
      int i;
    32
    201
      for (i = 0; i < 20 * 10; i++200)
    33
    200
        x_tensor->data.f32[i] = dsfmt_genrand_open_close(&dsfmt);
    34
    1
      ccv_nnc_tensor_t* const a_tensor = ccv_nnc_tensor_from_symbol(tensor_arena, a);
    35
    1
      ccv_nnc_cmd_exec(CMD_DATA_TRANSFER_FORWARD(), ccv_nnc_no_hint, 0, TENSOR_LIST(x_tensor), TENSOR_LIST(a_tensor), 0);
    36
    1
      ccv_nnc_graph_run(graph, 0, TRAVERSE_FULL, 0, 0);
    37
    1
      ccv_nnc_tensor_t* const y_tensor = ccv_nnc_tensor_new(0, CPU_TENSOR_NCHW(32F, 20, 10), 0);
    38
    1
      ccv_nnc_tensor_t* const b_tensor = ccv_nnc_tensor_from_symbol(tensor_arena, b);
    39
    1
      ccv_nnc_cmd_exec(CMD_DATA_TRANSFER_FORWARD(), ccv_nnc_no_hint, 0, TENSOR_LIST(b_tensor), TENSOR_LIST(y_tensor), 0);
    40
    1
      ccv_nnc_tensor_t* const ty = ccv_nnc_tensor_new(0, CPU_TENSOR_NCHW(32F, 20, 10), 0);
    41
    1
      ccv_nnc_cmd_exec(CMD_SWISH_FORWARD(1), ccv_nnc_no_hint, 0, TENSOR_LIST(x_tensor), TENSOR_LIST(ty), 0);
    42
    1
      REQUIRE_TENSOR_EQ(ty, y_tensor, "swish from cudnn should match from CPU");
    43
    1
      ccv_nnc_tensor_free(x_tensor);
    44
    1
      ccv_nnc_tensor_free(y_tensor);
    45
    1
      ccv_nnc_tensor_free(ty);
    46
    1
      ccv_nnc_graph_free(graph);
    47
    1
      ccv_nnc_tensor_arena_free(tensor_arena);
    48
    1
      ccv_nnc_graph_exec_arena_free(graph_exec_arena);
    49
    1
      ccv_nnc_symbolic_graph_free(symbolic_graph);
    50
    1
    }
    51
    52
    TEST_CASE("swish in half precision")
    53
    1
    {
    54
    1
      GUARD_ELSE_RETURN(ccv_nnc_cmd_ok(CCV_NNC_SWISH_FORWARD, CCV_NNC_BACKEND_GPU_REF) || ccv_nnc_cmd_ok(CCV_NNC_SWISH_FORWARD, CCV_NNC_BACKEND_MPS));
    55
    1
      ccv_nnc_symbolic_graph_t* const symbolic_graph = ccv_nnc_symbolic_graph_new();
    56
    1
      ccv_nnc_tensor_symbol_t a = ccv_nnc_tensor_symbol_new(symbolic_graph, GPU_TENSOR_NCHW(000, 16F, 20, 10), "a");
    57
    1
      ccv_nnc_tensor_symbol_t b = ccv_nnc_tensor_symbol_new(symbolic_graph, GPU_TENSOR_NCHW(000, 16F, 20, 10), "b");
    58
    1
      ccv_nnc_graph_exec_symbol_new(symbolic_graph, CMD_SWISH_FORWARD(1), TENSOR_SYMBOL_LIST(a), TENSOR_SYMBOL_LIST(b), "swish");
    59
    1
      ccv_nnc_graph_exec_symbol_autogen(symbolic_graph, 0, 0, CCV_NNC_AUTOGEN_ALL_EXECS | CCV_NNC_AUTOGEN_SOURCES_AND_DESTINATIONS);
    60
    1
      SYMBOLIC_GRAPH_GEN(symbolic_graph, CCV_NNC_LONG_DOT_GRAPH);
    61
    1
      ccv_nnc_graph_t* graph = 0;
    62
    1
      ccv_nnc_tensor_arena_t* tensor_arena = 0;
    63
    1
      ccv_nnc_graph_exec_arena_t* graph_exec_arena = 0;
    64
    1
      ccv_nnc_symbolic_graph_compile(symbolic_graph, ccv_nnc_default_compile_params, 0, 0, 0, 0, SYMBOLIC_GRAPH_SOURCES(symbolic_graph), SYMBOLIC_GRAPH_DESTINATIONS(symbolic_graph), &graph, &tensor_arena, &graph_exec_arena);
    65
    1
      GRAPH_GEN(graph, CCV_NNC_LONG_DOT_GRAPH);
    66
    1
      ccv_nnc_tensor_t* const x_tensor = ccv_nnc_tensor_new(0, CPU_TENSOR_NCHW(32F, 20, 10), 0);
    67
    1
      dsfmt_t dsfmt;
    68
    1
      dsfmt_init_gen_rand(&dsfmt, 0);
    69
    1
      int i;
    70
    201
      for (i = 0; i < 20 * 10; i++200)
    71
    200
        x_tensor->data.f32[i] = dsfmt_genrand_open_close(&dsfmt);
    72
    1
      ccv_nnc_tensor_t* const a_tensor = ccv_nnc_tensor_from_symbol(tensor_arena, a);
    73
    1
      ccv_nnc_tensor_t* const x16_tensor = ccv_nnc_tensor_new(0, CPU_TENSOR_NCHW(16F, 20, 10), 0);
    74
    1
      ccv_nnc_cmd_exec(CMD_DATATYPE_CONVERSION_FORWARD(), ccv_nnc_no_hint, 0, TENSOR_LIST(x_tensor), TENSOR_LIST(x16_tensor), 0);
    75
    1
      ccv_nnc_cmd_exec(CMD_DATA_TRANSFER_FORWARD(), ccv_nnc_no_hint, 0, TENSOR_LIST(x16_tensor), TENSOR_LIST(a_tensor), 0);
    76
    1
      ccv_nnc_graph_run(graph, 0, TRAVERSE_FULL, 0, 0);
    77
    1
      ccv_nnc_tensor_t* const y16_tensor = ccv_nnc_tensor_new(0, CPU_TENSOR_NCHW(16F, 20, 10), 0);
    78
    1
      ccv_nnc_tensor_t* const y_tensor = ccv_nnc_tensor_new(0, CPU_TENSOR_NCHW(32F, 20, 10), 0);
    79
    1
      ccv_nnc_tensor_t* const b_tensor = ccv_nnc_tensor_from_symbol(tensor_arena, b);
    80
    1
      ccv_nnc_cmd_exec(CMD_DATA_TRANSFER_FORWARD(), ccv_nnc_no_hint, 0, TENSOR_LIST(b_tensor), TENSOR_LIST(y16_tensor), 0);
    81
    1
      ccv_nnc_cmd_exec(CMD_DATATYPE_CONVERSION_FORWARD(), ccv_nnc_no_hint, 0, TENSOR_LIST(y16_tensor), TENSOR_LIST(y_tensor), 0);
    82
    1
      ccv_nnc_tensor_t* const ty = ccv_nnc_tensor_new(0, CPU_TENSOR_NCHW(32F, 20, 10), 0);
    83
    1
      ccv_nnc_cmd_exec(CMD_SWISH_FORWARD(1), ccv_nnc_no_hint, 0, TENSOR_LIST(x_tensor), TENSOR_LIST(ty), 0);
    84
    1
      REQUIRE_ARRAY_EQ_WITH_TOLERANCE(float, ty->data.f32, y_tensor->data.f32, 20 * 10, 1e-3, "swish from cudnn should match from CPU");
    85
    1
      ccv_nnc_tensor_free(x_tensor);
    86
    1
      ccv_nnc_tensor_free(x16_tensor);
    87
    1
      ccv_nnc_tensor_free(y16_tensor);
    88
    1
      ccv_nnc_tensor_free(y_tensor);
    89
    1
      ccv_nnc_tensor_free(ty);
    90
    1
      ccv_nnc_graph_free(graph);
    91
    1
      ccv_nnc_tensor_arena_free(tensor_arena);
    92
    1
      ccv_nnc_graph_exec_arena_free(graph_exec_arena);
    93
    1
      ccv_nnc_symbolic_graph_free(symbolic_graph);
    94
    1
    }
    95
    96
    TEST_CASE("swish gradient in float")
    97
    1
    {
    98
    1
      GUARD_ELSE_RETURN(ccv_nnc_cmd_ok(CCV_NNC_SWISH_FORWARD, CCV_NNC_BACKEND_GPU_REF) &&
    99
    1
        ccv_nnc_cmd_ok(CCV_NNC_SWISH_BACKWARD, CCV_NNC_BACKEND_GPU_REF));
    100
    1
      ccv_nnc_symbolic_graph_t* const symbolic_graph = ccv_nnc_symbolic_graph_new();
    101
    1
      ccv_nnc_tensor_symbol_t x = ccv_nnc_tensor_symbol_new(symbolic_graph, GPU_TENSOR_NHWC(000, 32F, 10, 100), "x");
    102
    1
      ccv_nnc_tensor_symbol_t y = ccv_nnc_tensor_symbol_new(symbolic_graph, GPU_TENSOR_NHWC(000, 32F, 10, 100), "y");
    103
    1
      ccv_nnc_graph_exec_symbol_new(symbolic_graph, CMD_SWISH_FORWARD(1), TENSOR_SYMBOL_LIST(x), TENSOR_SYMBOL_LIST(y), "swish");
    104
    1
      ccv_nnc_graph_exec_symbol_autogen(symbolic_graph, 0, 0, CCV_NNC_AUTOGEN_ALL_EXECS | CCV_NNC_AUTOGEN_SOURCES_AND_DESTINATIONS);
    105
    1
      ccv_nnc_symbolic_graph_backward(symbolic_graph, TENSOR_SYMBOL_LIST(y), TENSOR_SYMBOL_LIST(x), SYMBOLIC_GRAPH_SOURCES(symbolic_graph), SYMBOLIC_GRAPH_DESTINATIONS(symbolic_graph));
    106
    1
      ccv_nnc_graph_exec_symbol_autogen(symbolic_graph, 0, 0, CCV_NNC_AUTOGEN_ALL_EXECS | CCV_NNC_AUTOGEN_SOURCES_AND_DESTINATIONS);
    107
    1
      SYMBOLIC_GRAPH_GEN(symbolic_graph, CCV_NNC_LONG_DOT_GRAPH);
    108
    1
      ccv_nnc_tensor_symbol_t dy = ccv_nnc_tensor_symbol_for_backward(symbolic_graph, y);
    109
    1
      ccv_nnc_tensor_symbol_t dx = ccv_nnc_tensor_symbol_for_backward(symbolic_graph, x);
    110
    1
      ccv_nnc_tensor_t* const x_tensor = ccv_nnc_tensor_new(0, CPU_TENSOR_NHWC(32F, 10, 100), 0);
    111
    1
      dsfmt_t dsfmt;
    112
    1
      dsfmt_init_gen_rand(&dsfmt, 0);
    113
    1
      int i;
    114
    1.00k
      for (i = 0; i < 10 * 100; i++1.00k)
    115
    1.00k
        x_tensor->data.f32[i] = dsfmt_genrand_open_close(&dsfmt);
    116
    1
      ccv_nnc_tensor_t* const dy_tensor = ccv_nnc_tensor_new(0, CPU_TENSOR_NHWC(32F, 10, 100), 0);
    117
    1.00k
      for (i = 0; i < 10 * 100; i++1.00k)
    118
    1.00k
        dy_tensor->data.f32[i] = 0;
    119
    11
      for (i = 0; i < 10; i++10)
    120
    10
        dy_tensor->data.f32[i * 100 + i] = 1;
    121
    1
      ccv_nnc_tensor_t* const dyt = ccv_nnc_tensor_new(0, GPU_TENSOR_NHWC(000, 32F, 10, 100), 0);
    122
    1
      ccv_nnc_cmd_exec(CMD_DATA_TRANSFER_FORWARD(), ccv_nnc_no_hint, 0, TENSOR_LIST(dy_tensor), TENSOR_LIST(dyt), 0);
    123
    1
      ccv_nnc_graph_t* graph = 0;
    124
    1
      ccv_nnc_tensor_arena_t* tensor_arena = 0;
    125
    1
      ccv_nnc_graph_exec_arena_t* graph_exec_arena = 0;
    126
    1
      ccv_nnc_symbolic_graph_compile(symbolic_graph, ccv_nnc_default_compile_params, TENSOR_BIND_MAP(KV(dy, dyt)), TENSOR_SYMBOL_LIST(y), SYMBOLIC_GRAPH_SOURCES(symbolic_graph), SYMBOLIC_GRAPH_DESTINATIONS(symbolic_graph), &graph, &tensor_arena, &graph_exec_arena);
    127
    1
      GRAPH_GEN(graph, CCV_NNC_LONG_DOT_GRAPH);
    128
    1
      ccv_nnc_tensor_t* const xt = ccv_nnc_tensor_from_symbol(tensor_arena, x);
    129
    1
      ccv_nnc_cmd_exec(CMD_DATA_TRANSFER_FORWARD(), ccv_nnc_no_hint, 0, TENSOR_LIST(x_tensor), TENSOR_LIST(xt), 0);
    130
    1
      ccv_nnc_graph_run(graph, 0, TRAVERSE_FULL, 0, 0);
    131
    1
      ccv_nnc_tensor_t* const dx_tensor = ccv_nnc_tensor_new(0, CPU_TENSOR_NHWC(32F, 10, 100), 0);
    132
    1
      ccv_nnc_tensor_t* const dxt = ccv_nnc_tensor_from_symbol(tensor_arena, dx);
    133
    1
      ccv_nnc_tensor_t* const y_tensor = ccv_nnc_tensor_new(0, CPU_TENSOR_NHWC(32F, 10, 100), 0);
    134
    1
      ccv_nnc_tensor_t* const yt = ccv_nnc_tensor_from_symbol(tensor_arena, y);
    135
    1
      ccv_nnc_cmd_exec(CMD_DATA_TRANSFER_FORWARD(), ccv_nnc_no_hint, 0, TENSOR_LIST(dxt), TENSOR_LIST(dx_tensor), 0);
    136
    1
      ccv_nnc_cmd_exec(CMD_DATA_TRANSFER_FORWARD(), ccv_nnc_no_hint, 0, TENSOR_LIST(yt), TENSOR_LIST(y_tensor), 0);
    137
    1
      ccv_nnc_tensor_t* const ty_tensor = ccv_nnc_tensor_new(0, CPU_TENSOR_NHWC(32F, 10, 100), 0);
    138
    1
      ccv_nnc_cmd_exec(CMD_SWISH_FORWARD(1), ccv_nnc_no_hint, 0, TENSOR_LIST(x_tensor), TENSOR_LIST(ty_tensor), 0);
    139
    1
      REQUIRE_TENSOR_EQ(ty_tensor, y_tensor, "forward pass should match");
    140
    1
      ccv_nnc_tensor_t* const tdx_tensor = ccv_nnc_tensor_new(0, CPU_TENSOR_NHWC(32F, 10, 100), 0);
    141
    1
      ccv_nnc_cmd_exec(CMD_SWISH_BACKWARD(1), ccv_nnc_no_hint, 0, TENSOR_LIST(dy_tensor, x_tensor, 0), TENSOR_LIST(tdx_tensor), 0);
    142
    1
      REQUIRE_TENSOR_EQ(tdx_tensor, dx_tensor, "backward pass should match");
    143
    1
      ccv_nnc_tensor_free(x_tensor);
    144
    1
      ccv_nnc_tensor_free(y_tensor);
    145
    1
      ccv_nnc_tensor_free(dx_tensor);
    146
    1
      ccv_nnc_tensor_free(dy_tensor);
    147
    1
      ccv_nnc_tensor_free(ty_tensor);
    148
    1
      ccv_nnc_tensor_free(tdx_tensor);
    149
    1
      ccv_nnc_tensor_free(dyt);
    150
    1
      ccv_nnc_graph_free(graph);
    151
    1
      ccv_nnc_tensor_arena_free(tensor_arena);
    152
    1
      ccv_nnc_graph_exec_arena_free(graph_exec_arena);
    153
    1
      ccv_nnc_symbolic_graph_free(symbolic_graph);
    154
    1
    }
    155
    156
    TEST_CASE("mps swish gradient in float")
    157
    1
    {
    158
    1
      GUARD_ELSE_RETURN(ccv_nnc_cmd_ok(CCV_NNC_SWISH_FORWARD, CCV_NNC_BACKEND_MPS) &&
    159
    1
        ccv_nnc_cmd_ok(CCV_NNC_SWISH_BACKWARD, CCV_NNC_BACKEND_MPS));
    160
    0
      ccv_nnc_symbolic_graph_t* const symbolic_graph = ccv_nnc_symbolic_graph_new();
    161
    0
      ccv_nnc_tensor_symbol_t x = ccv_nnc_tensor_symbol_new(symbolic_graph, GPU_TENSOR_NHWC(000, 32F, 10, 100), "x");
    162
    0
      ccv_nnc_tensor_symbol_t y = ccv_nnc_tensor_symbol_new(symbolic_graph, GPU_TENSOR_NHWC(000, 32F, 10, 100), "y");
    163
    0
      ccv_nnc_graph_exec_symbol_new(symbolic_graph, CMD_SWISH_FORWARD(1), TENSOR_SYMBOL_LIST(x), TENSOR_SYMBOL_LIST(y), "swish");
    164
    0
      ccv_nnc_graph_exec_symbol_autogen(symbolic_graph, 0, 0, CCV_NNC_AUTOGEN_ALL_EXECS | CCV_NNC_AUTOGEN_SOURCES_AND_DESTINATIONS);
    165
    0
      ccv_nnc_symbolic_graph_backward(symbolic_graph, TENSOR_SYMBOL_LIST(y), TENSOR_SYMBOL_LIST(x), SYMBOLIC_GRAPH_SOURCES(symbolic_graph), SYMBOLIC_GRAPH_DESTINATIONS(symbolic_graph));
    166
    0
      ccv_nnc_graph_exec_symbol_autogen(symbolic_graph, 0, 0, CCV_NNC_AUTOGEN_ALL_EXECS | CCV_NNC_AUTOGEN_SOURCES_AND_DESTINATIONS);
    167
    0
      SYMBOLIC_GRAPH_GEN(symbolic_graph, CCV_NNC_LONG_DOT_GRAPH);
    168
    0
      ccv_nnc_tensor_symbol_t dy = ccv_nnc_tensor_symbol_for_backward(symbolic_graph, y);
    169
    0
      ccv_nnc_tensor_symbol_t dx = ccv_nnc_tensor_symbol_for_backward(symbolic_graph, x);
    170
    0
      ccv_nnc_tensor_t* const x_tensor = ccv_nnc_tensor_new(0, CPU_TENSOR_NHWC(32F, 10, 100), 0);
    171
    0
      dsfmt_t dsfmt;
    172
    0
      dsfmt_init_gen_rand(&dsfmt, 0);
    173
    0
      int i;
    174
    0
      for (i = 0; i < 10 * 100; i++)
    175
    0
        x_tensor->data.f32[i] = dsfmt_genrand_open_close(&dsfmt);
    176
    0
      ccv_nnc_tensor_t* const dy_tensor = ccv_nnc_tensor_new(0, CPU_TENSOR_NHWC(32F, 10, 100), 0);
    177
    0
      for (i = 0; i < 10 * 100; i++)
    178
    0
        dy_tensor->data.f32[i] = 0;
    179
    0
      for (i = 0; i < 10; i++)
    180
    0
        dy_tensor->data.f32[i * 100 + i] = 1;
    181
    0
      ccv_nnc_tensor_t* const dyt = ccv_nnc_tensor_new(0, GPU_TENSOR_NHWC(000, 32F, 10, 100), 0);
    182
    0
      ccv_nnc_cmd_exec(CMD_DATA_TRANSFER_FORWARD(), ccv_nnc_no_hint, 0, TENSOR_LIST(dy_tensor), TENSOR_LIST(dyt), 0);
    183
    0
      ccv_nnc_graph_t* graph = 0;
    184
    0
      ccv_nnc_tensor_arena_t* tensor_arena = 0;
    185
    0
      ccv_nnc_graph_exec_arena_t* graph_exec_arena = 0;
    186
    0
      ccv_nnc_symbolic_graph_compile(symbolic_graph, ccv_nnc_default_compile_params, TENSOR_BIND_MAP(KV(dy, dyt)), TENSOR_SYMBOL_LIST(y), SYMBOLIC_GRAPH_SOURCES(symbolic_graph), SYMBOLIC_GRAPH_DESTINATIONS(symbolic_graph), &graph, &tensor_arena, &graph_exec_arena);
    187
    0
      GRAPH_GEN(graph, CCV_NNC_LONG_DOT_GRAPH);
    188
    0
      ccv_nnc_tensor_t* const xt = ccv_nnc_tensor_from_symbol(tensor_arena, x);
    189
    0
      ccv_nnc_cmd_exec(CMD_DATA_TRANSFER_FORWARD(), ccv_nnc_no_hint, 0, TENSOR_LIST(x_tensor), TENSOR_LIST(xt), 0);
    190
    0
      ccv_nnc_graph_run(graph, 0, TRAVERSE_FULL, 0, 0);
    191
    0
      ccv_nnc_tensor_t* const dx_tensor = ccv_nnc_tensor_new(0, CPU_TENSOR_NHWC(32F, 10, 100), 0);
    192
    0
      ccv_nnc_tensor_t* const dxt = ccv_nnc_tensor_from_symbol(tensor_arena, dx);
    193
    0
      ccv_nnc_tensor_t* const y_tensor = ccv_nnc_tensor_new(0, CPU_TENSOR_NHWC(32F, 10, 100), 0);
    194
    0
      ccv_nnc_tensor_t* const yt = ccv_nnc_tensor_from_symbol(tensor_arena, y);
    195
    0
      ccv_nnc_cmd_exec(CMD_DATA_TRANSFER_FORWARD(), ccv_nnc_no_hint, 0, TENSOR_LIST(dxt), TENSOR_LIST(dx_tensor), 0);
    196
    0
      ccv_nnc_cmd_exec(CMD_DATA_TRANSFER_FORWARD(), ccv_nnc_no_hint, 0, TENSOR_LIST(yt), TENSOR_LIST(y_tensor), 0);
    197
    0
      ccv_nnc_tensor_t* const ty_tensor = ccv_nnc_tensor_new(0, CPU_TENSOR_NHWC(32F, 10, 100), 0);
    198
    0
      ccv_nnc_cmd_exec(CMD_SWISH_FORWARD(1), ccv_nnc_no_hint, 0, TENSOR_LIST(x_tensor), TENSOR_LIST(ty_tensor), 0);
    199
    0
      REQUIRE_TENSOR_EQ(ty_tensor, y_tensor, "forward pass should match");
    200
    0
      ccv_nnc_tensor_t* const tdx_tensor = ccv_nnc_tensor_new(0, CPU_TENSOR_NHWC(32F, 10, 100), 0);
    201
    0
      ccv_nnc_cmd_exec(CMD_SWISH_BACKWARD(1), ccv_nnc_no_hint, 0, TENSOR_LIST(dy_tensor, x_tensor, 0), TENSOR_LIST(tdx_tensor), 0);
    202
    0
      REQUIRE_TENSOR_EQ(tdx_tensor, dx_tensor, "backward pass should match");
    203
    0
      ccv_nnc_tensor_free(x_tensor);
    204
    0
      ccv_nnc_tensor_free(y_tensor);
    205
    0
      ccv_nnc_tensor_free(dx_tensor);
    206
    0
      ccv_nnc_tensor_free(dy_tensor);
    207
    0
      ccv_nnc_tensor_free(ty_tensor);
    208
    0
      ccv_nnc_tensor_free(tdx_tensor);
    209
    0
      ccv_nnc_tensor_free(dyt);
    210
    0
      ccv_nnc_graph_free(graph);
    211
    0
      ccv_nnc_tensor_arena_free(tensor_arena);
    212
    0
      ccv_nnc_graph_exec_arena_free(graph_exec_arena);
    213
    0
      ccv_nnc_symbolic_graph_free(symbolic_graph);
    214
    0
    }
    215
    216
    TEST_CASE("swish gradient in half precision")
    217
    1
    {
    218
    1
      GUARD_ELSE_RETURN(ccv_nnc_cmd_ok(CCV_NNC_SWISH_FORWARD, CCV_NNC_BACKEND_GPU_REF) &&
    219
    1
        ccv_nnc_cmd_ok(CCV_NNC_SWISH_BACKWARD, CCV_NNC_BACKEND_GPU_REF));
    220
    1
      ccv_nnc_symbolic_graph_t* const symbolic_graph = ccv_nnc_symbolic_graph_new();
    221
    1
      ccv_nnc_tensor_symbol_t x = ccv_nnc_tensor_symbol_new(symbolic_graph, GPU_TENSOR_NHWC(000, 16F, 10, 100), "x");
    222
    1
      ccv_nnc_tensor_symbol_t y = ccv_nnc_tensor_symbol_new(symbolic_graph, GPU_TENSOR_NHWC(000, 16F, 10, 100), "y");
    223
    1
      ccv_nnc_graph_exec_symbol_new(symbolic_graph, CMD_SWISH_FORWARD(1), TENSOR_SYMBOL_LIST(x), TENSOR_SYMBOL_LIST(y), "swish");
    224
    1
      ccv_nnc_graph_exec_symbol_autogen(symbolic_graph, 0, 0, CCV_NNC_AUTOGEN_ALL_EXECS | CCV_NNC_AUTOGEN_SOURCES_AND_DESTINATIONS);
    225
    1
      ccv_nnc_symbolic_graph_backward(symbolic_graph, TENSOR_SYMBOL_LIST(y), TENSOR_SYMBOL_LIST(x), SYMBOLIC_GRAPH_SOURCES(symbolic_graph), SYMBOLIC_GRAPH_DESTINATIONS(symbolic_graph));
    226
    1
      ccv_nnc_graph_exec_symbol_autogen(symbolic_graph, 0, 0, CCV_NNC_AUTOGEN_ALL_EXECS | CCV_NNC_AUTOGEN_SOURCES_AND_DESTINATIONS);
    227
    1
      SYMBOLIC_GRAPH_GEN(symbolic_graph, CCV_NNC_LONG_DOT_GRAPH);
    228
    1
      ccv_nnc_tensor_symbol_t dy = ccv_nnc_tensor_symbol_for_backward(symbolic_graph, y);
    229
    1
      ccv_nnc_tensor_symbol_t dx = ccv_nnc_tensor_symbol_for_backward(symbolic_graph, x);
    230
    1
      ccv_nnc_tensor_t* const x_tensor = ccv_nnc_tensor_new(0, CPU_TENSOR_NHWC(32F, 10, 100), 0);
    231
    1
      dsfmt_t dsfmt;
    232
    1
      dsfmt_init_gen_rand(&dsfmt, 0);
    233
    1
      int i;
    234
    1.00k
      for (i = 0; i < 10 * 100; i++1.00k)
    235
    1.00k
        x_tensor->data.f32[i] = dsfmt_genrand_open_close(&dsfmt);
    236
    1
      ccv_nnc_tensor_t* const dy_tensor = ccv_nnc_tensor_new(0, CPU_TENSOR_NHWC(32F, 10, 100), 0);
    237
    1.00k
      for (i = 0; i < 10 * 100; i++1.00k)
    238
    1.00k
        dy_tensor->data.f32[i] = 0;
    239
    11
      for (i = 0; i < 10; i++10)
    240
    10
        dy_tensor->data.f32[i * 100 + i] = 1;
    241
    1
      ccv_nnc_tensor_t* const dy16_tensor = ccv_nnc_tensor_new(0, CPU_TENSOR_NHWC(16F, 10, 100), 0);
    242
    1
      ccv_nnc_tensor_t* const dyt = ccv_nnc_tensor_new(0, GPU_TENSOR_NHWC(000, 16F, 10, 100), 0);
    243
    1
      ccv_nnc_cmd_exec(CMD_DATATYPE_CONVERSION_FORWARD(), ccv_nnc_no_hint, 0, TENSOR_LIST(dy_tensor), TENSOR_LIST(dy16_tensor), 0);
    244
    1
      ccv_nnc_cmd_exec(CMD_DATA_TRANSFER_FORWARD(), ccv_nnc_no_hint, 0, TENSOR_LIST(dy16_tensor), TENSOR_LIST(dyt), 0);
    245
    1
      ccv_nnc_graph_t* graph = 0;
    246
    1
      ccv_nnc_tensor_arena_t* tensor_arena = 0;
    247
    1
      ccv_nnc_graph_exec_arena_t* graph_exec_arena = 0;
    248
    1
      ccv_nnc_symbolic_graph_compile(symbolic_graph, ccv_nnc_default_compile_params, TENSOR_BIND_MAP(KV(dy, dyt)), TENSOR_SYMBOL_LIST(y), SYMBOLIC_GRAPH_SOURCES(symbolic_graph), SYMBOLIC_GRAPH_DESTINATIONS(symbolic_graph), &graph, &tensor_arena, &graph_exec_arena);
    249
    1
      GRAPH_GEN(graph, CCV_NNC_LONG_DOT_GRAPH);
    250
    1
      ccv_nnc_tensor_t* const xt = ccv_nnc_tensor_from_symbol(tensor_arena, x);
    251
    1
      ccv_nnc_tensor_t* const x16_tensor = ccv_nnc_tensor_new(0, CPU_TENSOR_NHWC(16F, 10, 100), 0);
    252
    1
      ccv_nnc_cmd_exec(CMD_DATATYPE_CONVERSION_FORWARD(), ccv_nnc_no_hint, 0, TENSOR_LIST(x_tensor), TENSOR_LIST(x16_tensor), 0);
    253
    1
      ccv_nnc_cmd_exec(CMD_DATA_TRANSFER_FORWARD(), ccv_nnc_no_hint, 0, TENSOR_LIST(x16_tensor), TENSOR_LIST(xt), 0);
    254
    1
      ccv_nnc_graph_run(graph, 0, TRAVERSE_FULL, 0, 0);
    255
    1
      ccv_nnc_tensor_t* const dx16_tensor = ccv_nnc_tensor_new(0, CPU_TENSOR_NHWC(16F, 10, 100), 0);
    256
    1
      ccv_nnc_tensor_t* const dx_tensor = ccv_nnc_tensor_new(0, CPU_TENSOR_NHWC(32F, 10, 100), 0);
    257
    1
      ccv_nnc_tensor_t* const dxt = ccv_nnc_tensor_from_symbol(tensor_arena, dx);
    258
    1
      ccv_nnc_tensor_t* const y16_tensor = ccv_nnc_tensor_new(0, CPU_TENSOR_NHWC(16F, 10, 100), 0);
    259
    1
      ccv_nnc_tensor_t* const y_tensor = ccv_nnc_tensor_new(0, CPU_TENSOR_NHWC(32F, 10, 100), 0);
    260
    1
      ccv_nnc_tensor_t* const yt = ccv_nnc_tensor_from_symbol(tensor_arena, y);
    261
    1
      ccv_nnc_cmd_exec(CMD_DATA_TRANSFER_FORWARD(), ccv_nnc_no_hint, 0, TENSOR_LIST(dxt), TENSOR_LIST(dx16_tensor), 0);
    262
    1
      ccv_nnc_cmd_exec(CMD_DATATYPE_CONVERSION_FORWARD(), ccv_nnc_no_hint, 0, TENSOR_LIST(dx16_tensor), TENSOR_LIST(dx_tensor), 0);
    263
    1
      ccv_nnc_cmd_exec(CMD_DATA_TRANSFER_FORWARD(), ccv_nnc_no_hint, 0, TENSOR_LIST(yt), TENSOR_LIST(y16_tensor), 0);
    264
    1
      ccv_nnc_cmd_exec(CMD_DATATYPE_CONVERSION_FORWARD(), ccv_nnc_no_hint, 0, TENSOR_LIST(y16_tensor), TENSOR_LIST(y_tensor), 0);
    265
    1
      ccv_nnc_tensor_t* const ty_tensor = ccv_nnc_tensor_new(0, CPU_TENSOR_NHWC(32F, 10, 100), 0);
    266
    1
      ccv_nnc_cmd_exec(CMD_SWISH_FORWARD(1), ccv_nnc_no_hint, 0, TENSOR_LIST(x_tensor), TENSOR_LIST(ty_tensor), 0);
    267
    1
      REQUIRE_ARRAY_EQ_WITH_TOLERANCE(float, ty_tensor->data.f32, y_tensor->data.f32, 10 * 100, 1e-3, "forward pass should match");
    268
    1
      ccv_nnc_tensor_t* const tdx_tensor = ccv_nnc_tensor_new(0, CPU_TENSOR_NHWC(32F, 10, 100), 0);
    269
    1
      ccv_nnc_cmd_exec(CMD_SWISH_BACKWARD(1), ccv_nnc_no_hint, 0, TENSOR_LIST(dy_tensor, x_tensor, 0), TENSOR_LIST(tdx_tensor), 0);
    270
    1
      REQUIRE_ARRAY_EQ_WITH_TOLERANCE(float, tdx_tensor->data.f32, dx_tensor->data.f32, 10 * 100, 1e-3, "backward pass should match");
    271
    1
      ccv_nnc_tensor_free(x_tensor);
    272
    1
      ccv_nnc_tensor_free(x16_tensor);
    273
    1
      ccv_nnc_tensor_free(y_tensor);
    274
    1
      ccv_nnc_tensor_free(y16_tensor);
    275
    1
      ccv_nnc_tensor_free(dx_tensor);
    276
    1
      ccv_nnc_tensor_free(dx16_tensor);
    277
    1
      ccv_nnc_tensor_free(dy_tensor);
    278
    1
      ccv_nnc_tensor_free(dy16_tensor);
    279
    1
      ccv_nnc_tensor_free(ty_tensor);
    280
    1
      ccv_nnc_tensor_free(tdx_tensor);
    281
    1
      ccv_nnc_tensor_free(dyt);
    282
    1
      ccv_nnc_graph_free(graph);
    283
    1
      ccv_nnc_tensor_arena_free(tensor_arena);
    284
    1
      ccv_nnc_graph_exec_arena_free(graph_exec_arena);
    285
    1
      ccv_nnc_symbolic_graph_free(symbolic_graph);
    286
    1
    }
    287
    288
    TEST_CASE("mps swish gradient in half precision")
    289
    1
    {
    290
    1
      GUARD_ELSE_RETURN(ccv_nnc_cmd_ok(CCV_NNC_SWISH_FORWARD, CCV_NNC_BACKEND_MPS) &&
    291
    1
        ccv_nnc_cmd_ok(CCV_NNC_SWISH_BACKWARD, CCV_NNC_BACKEND_MPS));
    292
    0
      ccv_nnc_symbolic_graph_t* const symbolic_graph = ccv_nnc_symbolic_graph_new();
    293
    0
      ccv_nnc_tensor_symbol_t x = ccv_nnc_tensor_symbol_new(symbolic_graph, GPU_TENSOR_NHWC(000, 16F, 10, 100), "x");
    294
    0
      ccv_nnc_tensor_symbol_t y = ccv_nnc_tensor_symbol_new(symbolic_graph, GPU_TENSOR_NHWC(000, 16F, 10, 100), "y");
    295
    0
      ccv_nnc_graph_exec_symbol_new(symbolic_graph, CMD_SWISH_FORWARD(1), TENSOR_SYMBOL_LIST(x), TENSOR_SYMBOL_LIST(y), "swish");
    296
    0
      ccv_nnc_graph_exec_symbol_autogen(symbolic_graph, 0, 0, CCV_NNC_AUTOGEN_ALL_EXECS | CCV_NNC_AUTOGEN_SOURCES_AND_DESTINATIONS);
    297
    0
      ccv_nnc_symbolic_graph_backward(symbolic_graph, TENSOR_SYMBOL_LIST(y), TENSOR_SYMBOL_LIST(x), SYMBOLIC_GRAPH_SOURCES(symbolic_graph), SYMBOLIC_GRAPH_DESTINATIONS(symbolic_graph));
    298
    0
      ccv_nnc_graph_exec_symbol_autogen(symbolic_graph, 0, 0, CCV_NNC_AUTOGEN_ALL_EXECS | CCV_NNC_AUTOGEN_SOURCES_AND_DESTINATIONS);
    299
    0
      SYMBOLIC_GRAPH_GEN(symbolic_graph, CCV_NNC_LONG_DOT_GRAPH);
    300
    0
      ccv_nnc_tensor_symbol_t dy = ccv_nnc_tensor_symbol_for_backward(symbolic_graph, y);
    301
    0
      ccv_nnc_tensor_symbol_t dx = ccv_nnc_tensor_symbol_for_backward(symbolic_graph, x);
    302
    0
      ccv_nnc_tensor_t* const x_tensor = ccv_nnc_tensor_new(0, CPU_TENSOR_NHWC(32F, 10, 100), 0);
    303
    0
      dsfmt_t dsfmt;
    304
    0
      dsfmt_init_gen_rand(&dsfmt, 0);
    305
    0
      int i;
    306
    0
      for (i = 0; i < 10 * 100; i++)
    307
    0
        x_tensor->data.f32[i] = dsfmt_genrand_open_close(&dsfmt);
    308
    0
      ccv_nnc_tensor_t* const dy_tensor = ccv_nnc_tensor_new(0, CPU_TENSOR_NHWC(32F, 10, 100), 0);
    309
    0
      for (i = 0; i < 10 * 100; i++)
    310
    0
        dy_tensor->data.f32[i] = 0;
    311
    0
      for (i = 0; i < 10; i++)
    312
    0
        dy_tensor->data.f32[i * 100 + i] = 1;
    313
    0
      ccv_nnc_tensor_t* const dy16_tensor = ccv_nnc_tensor_new(0, CPU_TENSOR_NHWC(16F, 10, 100), 0);
    314
    0
      ccv_nnc_tensor_t* const dyt = ccv_nnc_tensor_new(0, GPU_TENSOR_NHWC(000, 16F, 10, 100), 0);
    315
    0
      ccv_nnc_cmd_exec(CMD_DATATYPE_CONVERSION_FORWARD(), ccv_nnc_no_hint, 0, TENSOR_LIST(dy_tensor), TENSOR_LIST(dy16_tensor), 0);
    316
    0
      ccv_nnc_cmd_exec(CMD_DATA_TRANSFER_FORWARD(), ccv_nnc_no_hint, 0, TENSOR_LIST(dy16_tensor), TENSOR_LIST(dyt), 0);
    317
    0
      ccv_nnc_graph_t* graph = 0;
    318
    0
      ccv_nnc_tensor_arena_t* tensor_arena = 0;
    319
    0
      ccv_nnc_graph_exec_arena_t* graph_exec_arena = 0;
    320
    0
      ccv_nnc_symbolic_graph_compile(symbolic_graph, ccv_nnc_default_compile_params, TENSOR_BIND_MAP(KV(dy, dyt)), TENSOR_SYMBOL_LIST(y), SYMBOLIC_GRAPH_SOURCES(symbolic_graph), SYMBOLIC_GRAPH_DESTINATIONS(symbolic_graph), &graph, &tensor_arena, &graph_exec_arena);
    321
    0
      GRAPH_GEN(graph, CCV_NNC_LONG_DOT_GRAPH);
    322
    0
      ccv_nnc_tensor_t* const xt = ccv_nnc_tensor_from_symbol(tensor_arena, x);
    323
    0
      ccv_nnc_tensor_t* const x16_tensor = ccv_nnc_tensor_new(0, CPU_TENSOR_NHWC(16F, 10, 100), 0);
    324
    0
      ccv_nnc_cmd_exec(CMD_DATATYPE_CONVERSION_FORWARD(), ccv_nnc_no_hint, 0, TENSOR_LIST(x_tensor), TENSOR_LIST(x16_tensor), 0);
    325
    0
      ccv_nnc_cmd_exec(CMD_DATA_TRANSFER_FORWARD(), ccv_nnc_no_hint, 0, TENSOR_LIST(x16_tensor), TENSOR_LIST(xt), 0);
    326
    0
      ccv_nnc_graph_run(graph, 0, TRAVERSE_FULL, 0, 0);
    327
    0
      ccv_nnc_tensor_t* const dx16_tensor = ccv_nnc_tensor_new(0, CPU_TENSOR_NHWC(16F, 10, 100), 0);
    328
    0
      ccv_nnc_tensor_t* const dx_tensor = ccv_nnc_tensor_new(0, CPU_TENSOR_NHWC(32F, 10, 100), 0);
    329
    0
      ccv_nnc_tensor_t* const dxt = ccv_nnc_tensor_from_symbol(tensor_arena, dx);
    330
    0
      ccv_nnc_tensor_t* const y16_tensor = ccv_nnc_tensor_new(0, CPU_TENSOR_NHWC(16F, 10, 100), 0);
    331
    0
      ccv_nnc_tensor_t* const y_tensor = ccv_nnc_tensor_new(0, CPU_TENSOR_NHWC(32F, 10, 100), 0);
    332
    0
      ccv_nnc_tensor_t* const yt = ccv_nnc_tensor_from_symbol(tensor_arena, y);
    333
    0
      ccv_nnc_cmd_exec(CMD_DATA_TRANSFER_FORWARD(), ccv_nnc_no_hint, 0, TENSOR_LIST(dxt), TENSOR_LIST(dx16_tensor), 0);
    334
    0
      ccv_nnc_cmd_exec(CMD_DATATYPE_CONVERSION_FORWARD(), ccv_nnc_no_hint, 0, TENSOR_LIST(dx16_tensor), TENSOR_LIST(dx_tensor), 0);
    335
    0
      ccv_nnc_cmd_exec(CMD_DATA_TRANSFER_FORWARD(), ccv_nnc_no_hint, 0, TENSOR_LIST(yt), TENSOR_LIST(y16_tensor), 0);
    336
    0
      ccv_nnc_cmd_exec(CMD_DATATYPE_CONVERSION_FORWARD(), ccv_nnc_no_hint, 0, TENSOR_LIST(y16_tensor), TENSOR_LIST(y_tensor), 0);
    337
    0
      ccv_nnc_tensor_t* const ty_tensor = ccv_nnc_tensor_new(0, CPU_TENSOR_NHWC(32F, 10, 100), 0);
    338
    0
      ccv_nnc_cmd_exec(CMD_SWISH_FORWARD(1), ccv_nnc_no_hint, 0, TENSOR_LIST(x_tensor), TENSOR_LIST(ty_tensor), 0);
    339
    0
      REQUIRE_ARRAY_EQ_WITH_TOLERANCE(float, ty_tensor->data.f32, y_tensor->data.f32, 10 * 100, 1e-3, "forward pass should match");
    340
    0
      ccv_nnc_tensor_t* const tdx_tensor = ccv_nnc_tensor_new(0, CPU_TENSOR_NHWC(32F, 10, 100), 0);
    341
    0
      ccv_nnc_cmd_exec(CMD_SWISH_BACKWARD(1), ccv_nnc_no_hint, 0, TENSOR_LIST(dy_tensor, x_tensor, 0), TENSOR_LIST(tdx_tensor), 0);
    342
    0
      REQUIRE_ARRAY_EQ_WITH_TOLERANCE(float, tdx_tensor->data.f32, dx_tensor->data.f32, 10 * 100, 1e-3, "backward pass should match");
    343
    0
      ccv_nnc_tensor_free(x_tensor);
    344
    0
      ccv_nnc_tensor_free(x16_tensor);
    345
    0
      ccv_nnc_tensor_free(y_tensor);
    346
    0
      ccv_nnc_tensor_free(y16_tensor);
    347
    0
      ccv_nnc_tensor_free(dx_tensor);
    348
    0
      ccv_nnc_tensor_free(dx16_tensor);
    349
    0
      ccv_nnc_tensor_free(dy_tensor);
    350
    0
      ccv_nnc_tensor_free(dy16_tensor);
    351
    0
      ccv_nnc_tensor_free(ty_tensor);
    352
    0
      ccv_nnc_tensor_free(tdx_tensor);
    353
    0
      ccv_nnc_tensor_free(dyt);
    354
    0
      ccv_nnc_graph_free(graph);
    355
    0
      ccv_nnc_tensor_arena_free(tensor_arena);
    356
    0
      ccv_nnc_graph_exec_arena_free(graph_exec_arena);
    357
    0
      ccv_nnc_symbolic_graph_free(symbolic_graph);
    358
    0
    }
    359
    360
    TEST_CASE("swish gradient with non-one beta in half precision")
    361
    1
    {
    362
    1
      GUARD_ELSE_RETURN(ccv_nnc_cmd_ok(CCV_NNC_SWISH_FORWARD, CCV_NNC_BACKEND_GPU_REF) &&
    363
    1
        ccv_nnc_cmd_ok(CCV_NNC_SWISH_BACKWARD, CCV_NNC_BACKEND_GPU_REF));
    364
    1
      const float beta = 1.7;
    365
    1
      ccv_nnc_symbolic_graph_t* const symbolic_graph = ccv_nnc_symbolic_graph_new();
    366
    1
      ccv_nnc_tensor_symbol_t x = ccv_nnc_tensor_symbol_new(symbolic_graph, GPU_TENSOR_NHWC(000, 16F, 10, 100), "x");
    367
    1
      ccv_nnc_tensor_symbol_t y = ccv_nnc_tensor_symbol_new(symbolic_graph, GPU_TENSOR_NHWC(000, 16F, 10, 100), "y");
    368
    1
      ccv_nnc_graph_exec_symbol_new(symbolic_graph, CMD_SWISH_FORWARD(beta), TENSOR_SYMBOL_LIST(x), TENSOR_SYMBOL_LIST(y), "swish");
    369
    1
      ccv_nnc_graph_exec_symbol_autogen(symbolic_graph, 0, 0, CCV_NNC_AUTOGEN_ALL_EXECS | CCV_NNC_AUTOGEN_SOURCES_AND_DESTINATIONS);
    370
    1
      ccv_nnc_symbolic_graph_backward(symbolic_graph, TENSOR_SYMBOL_LIST(y), TENSOR_SYMBOL_LIST(x), SYMBOLIC_GRAPH_SOURCES(symbolic_graph), SYMBOLIC_GRAPH_DESTINATIONS(symbolic_graph));
    371
    1
      ccv_nnc_graph_exec_symbol_autogen(symbolic_graph, 0, 0, CCV_NNC_AUTOGEN_ALL_EXECS | CCV_NNC_AUTOGEN_SOURCES_AND_DESTINATIONS);
    372
    1
      SYMBOLIC_GRAPH_GEN(symbolic_graph, CCV_NNC_LONG_DOT_GRAPH);
    373
    1
      ccv_nnc_tensor_symbol_t dy = ccv_nnc_tensor_symbol_for_backward(symbolic_graph, y);
    374
    1
      ccv_nnc_tensor_symbol_t dx = ccv_nnc_tensor_symbol_for_backward(symbolic_graph, x);
    375
    1
      ccv_nnc_tensor_t* const x_tensor = ccv_nnc_tensor_new(0, CPU_TENSOR_NHWC(32F, 10, 100), 0);
    376
    1
      dsfmt_t dsfmt;
    377
    1
      dsfmt_init_gen_rand(&dsfmt, 0);
    378
    1
      int i;
    379
    1.00k
      for (i = 0; i < 10 * 100; i++1.00k)
    380
    1.00k
        x_tensor->data.f32[i] = dsfmt_genrand_open_close(&dsfmt);
    381
    1
      ccv_nnc_tensor_t* const dy_tensor = ccv_nnc_tensor_new(0, CPU_TENSOR_NHWC(32F, 10, 100), 0);
    382
    1.00k
      for (i = 0; i < 10 * 100; i++1.00k)
    383
    1.00k
        dy_tensor->data.f32[i] = 0;
    384
    11
      for (i = 0; i < 10; i++10)
    385
    10
        dy_tensor->data.f32[i * 100 + i] = 1;
    386
    1
      ccv_nnc_tensor_t* const dy16_tensor = ccv_nnc_tensor_new(0, CPU_TENSOR_NHWC(16F, 10, 100), 0);
    387
    1
      ccv_nnc_tensor_t* const dyt = ccv_nnc_tensor_new(0, GPU_TENSOR_NHWC(000, 16F, 10, 100), 0);
    388
    1
      ccv_nnc_cmd_exec(CMD_DATATYPE_CONVERSION_FORWARD(), ccv_nnc_no_hint, 0, TENSOR_LIST(dy_tensor), TENSOR_LIST(dy16_tensor), 0);
    389
    1
      ccv_nnc_cmd_exec(CMD_DATA_TRANSFER_FORWARD(), ccv_nnc_no_hint, 0, TENSOR_LIST(dy16_tensor), TENSOR_LIST(dyt), 0);
    390
    1
      ccv_nnc_graph_t* graph = 0;
    391
    1
      ccv_nnc_tensor_arena_t* tensor_arena = 0;
    392
    1
      ccv_nnc_graph_exec_arena_t* graph_exec_arena = 0;
    393
    1
      ccv_nnc_symbolic_graph_compile(symbolic_graph, ccv_nnc_default_compile_params, TENSOR_BIND_MAP(KV(dy, dyt)), TENSOR_SYMBOL_LIST(y), SYMBOLIC_GRAPH_SOURCES(symbolic_graph), SYMBOLIC_GRAPH_DESTINATIONS(symbolic_graph), &graph, &tensor_arena, &graph_exec_arena);
    394
    1
      GRAPH_GEN(graph, CCV_NNC_LONG_DOT_GRAPH);
    395
    1
      ccv_nnc_tensor_t* const xt = ccv_nnc_tensor_from_symbol(tensor_arena, x);
    396
    1
      ccv_nnc_tensor_t* const x16_tensor = ccv_nnc_tensor_new(0, CPU_TENSOR_NHWC(16F, 10, 100), 0);
    397
    1
      ccv_nnc_cmd_exec(CMD_DATATYPE_CONVERSION_FORWARD(), ccv_nnc_no_hint, 0, TENSOR_LIST(x_tensor), TENSOR_LIST(x16_tensor), 0);
    398
    1
      ccv_nnc_cmd_exec(CMD_DATA_TRANSFER_FORWARD(), ccv_nnc_no_hint, 0, TENSOR_LIST(x16_tensor), TENSOR_LIST(xt), 0);
    399
    1
      ccv_nnc_graph_run(graph, 0, TRAVERSE_FULL, 0, 0);
    400
    1
      ccv_nnc_tensor_t* const dx16_tensor = ccv_nnc_tensor_new(0, CPU_TENSOR_NHWC(16F, 10, 100), 0);
    401
    1
      ccv_nnc_tensor_t* const dx_tensor = ccv_nnc_tensor_new(0, CPU_TENSOR_NHWC(32F, 10, 100), 0);
    402
    1
      ccv_nnc_tensor_t* const dxt = ccv_nnc_tensor_from_symbol(tensor_arena, dx);
    403
    1
      ccv_nnc_tensor_t* const y16_tensor = ccv_nnc_tensor_new(0, CPU_TENSOR_NHWC(16F, 10, 100), 0);
    404
    1
      ccv_nnc_tensor_t* const y_tensor = ccv_nnc_tensor_new(0, CPU_TENSOR_NHWC(32F, 10, 100), 0);
    405
    1
      ccv_nnc_tensor_t* const yt = ccv_nnc_tensor_from_symbol(tensor_arena, y);
    406
    1
      ccv_nnc_cmd_exec(CMD_DATA_TRANSFER_FORWARD(), ccv_nnc_no_hint, 0, TENSOR_LIST(dxt), TENSOR_LIST(dx16_tensor), 0);
    407
    1
      ccv_nnc_cmd_exec(CMD_DATATYPE_CONVERSION_FORWARD(), ccv_nnc_no_hint, 0, TENSOR_LIST(dx16_tensor), TENSOR_LIST(dx_tensor), 0);
    408
    1
      ccv_nnc_cmd_exec(CMD_DATA_TRANSFER_FORWARD(), ccv_nnc_no_hint, 0, TENSOR_LIST(yt), TENSOR_LIST(y16_tensor), 0);
    409
    1
      ccv_nnc_cmd_exec(CMD_DATATYPE_CONVERSION_FORWARD(), ccv_nnc_no_hint, 0, TENSOR_LIST(y16_tensor), TENSOR_LIST(y_tensor), 0);
    410
    1
      ccv_nnc_tensor_t* const ty_tensor = ccv_nnc_tensor_new(0, CPU_TENSOR_NHWC(32F, 10, 100), 0);
    411
    1
      ccv_nnc_cmd_exec(CMD_SWISH_FORWARD(beta), ccv_nnc_no_hint, 0, TENSOR_LIST(x_tensor), TENSOR_LIST(ty_tensor), 0);
    412
    1
      REQUIRE_ARRAY_EQ_WITH_TOLERANCE(float, ty_tensor->data.f32, y_tensor->data.f32, 10 * 100, 1e-3, "forward pass should match");
    413
    1
      ccv_nnc_tensor_t* const tdx_tensor = ccv_nnc_tensor_new(0, CPU_TENSOR_NHWC(32F, 10, 100), 0);
    414
    1
      ccv_nnc_cmd_exec(CMD_SWISH_BACKWARD(beta), ccv_nnc_no_hint, 0, TENSOR_LIST(dy_tensor, x_tensor, 0), TENSOR_LIST(tdx_tensor), 0);
    415
    1
      REQUIRE_ARRAY_EQ_WITH_TOLERANCE(float, tdx_tensor->data.f32, dx_tensor->data.f32, 10 * 100, 1e-3, "backward pass should match");
    416
    1
      ccv_nnc_tensor_free(x_tensor);
    417
    1
      ccv_nnc_tensor_free(x16_tensor);
    418
    1
      ccv_nnc_tensor_free(y_tensor);
    419
    1
      ccv_nnc_tensor_free(y16_tensor);
    420
    1
      ccv_nnc_tensor_free(dx_tensor);
    421
    1
      ccv_nnc_tensor_free(dx16_tensor);
    422
    1
      ccv_nnc_tensor_free(dy_tensor);
    423
    1
      ccv_nnc_tensor_free(dy16_tensor);
    424
    1
      ccv_nnc_tensor_free(ty_tensor);
    425
    1
      ccv_nnc_tensor_free(tdx_tensor);
    426
    1
      ccv_nnc_tensor_free(dyt);
    427
    1
      ccv_nnc_graph_free(graph);
    428
    1
      ccv_nnc_tensor_arena_free(tensor_arena);
    429
    1
      ccv_nnc_graph_exec_arena_free(graph_exec_arena);
    430
    1
      ccv_nnc_symbolic_graph_free(symbolic_graph);
    431
    1
    }
    432
    433
    TEST_CASE("mps swish gradient with non-one beta in half precision")
    434
    1
    {
    435
    1
      GUARD_ELSE_RETURN(ccv_nnc_cmd_ok(CCV_NNC_SWISH_FORWARD, CCV_NNC_BACKEND_MPS) &&
    436
    1
        ccv_nnc_cmd_ok(CCV_NNC_SWISH_BACKWARD, CCV_NNC_BACKEND_MPS));
    437
    0
      const float beta = 1.7;
    438
    0
      ccv_nnc_symbolic_graph_t* const symbolic_graph = ccv_nnc_symbolic_graph_new();
    439
    0
      ccv_nnc_tensor_symbol_t x = ccv_nnc_tensor_symbol_new(symbolic_graph, GPU_TENSOR_NHWC(000, 16F, 10, 100), "x");
    440
    0
      ccv_nnc_tensor_symbol_t y = ccv_nnc_tensor_symbol_new(symbolic_graph, GPU_TENSOR_NHWC(000, 16F, 10, 100), "y");
    441
    0
      ccv_nnc_graph_exec_symbol_new(symbolic_graph, CMD_SWISH_FORWARD(beta), TENSOR_SYMBOL_LIST(x), TENSOR_SYMBOL_LIST(y), "swish");
    442
    0
      ccv_nnc_graph_exec_symbol_autogen(symbolic_graph, 0, 0, CCV_NNC_AUTOGEN_ALL_EXECS | CCV_NNC_AUTOGEN_SOURCES_AND_DESTINATIONS);
    443
    0
      ccv_nnc_symbolic_graph_backward(symbolic_graph, TENSOR_SYMBOL_LIST(y), TENSOR_SYMBOL_LIST(x), SYMBOLIC_GRAPH_SOURCES(symbolic_graph), SYMBOLIC_GRAPH_DESTINATIONS(symbolic_graph));
    444
    0
      ccv_nnc_graph_exec_symbol_autogen(symbolic_graph, 0, 0, CCV_NNC_AUTOGEN_ALL_EXECS | CCV_NNC_AUTOGEN_SOURCES_AND_DESTINATIONS);
    445
    0
      SYMBOLIC_GRAPH_GEN(symbolic_graph, CCV_NNC_LONG_DOT_GRAPH);
    446
    0
      ccv_nnc_tensor_symbol_t dy = ccv_nnc_tensor_symbol_for_backward(symbolic_graph, y);
    447
    0
      ccv_nnc_tensor_symbol_t dx = ccv_nnc_tensor_symbol_for_backward(symbolic_graph, x);
    448
    0
      ccv_nnc_tensor_t* const x_tensor = ccv_nnc_tensor_new(0, CPU_TENSOR_NHWC(32F, 10, 100), 0);
    449
    0
      dsfmt_t dsfmt;
    450
    0
      dsfmt_init_gen_rand(&dsfmt, 0);
    451
    0
      int i;
    452
    0
      for (i = 0; i < 10 * 100; i++)
    453
    0
        x_tensor->data.f32[i] = dsfmt_genrand_open_close(&dsfmt);
    454
    0
      ccv_nnc_tensor_t* const dy_tensor = ccv_nnc_tensor_new(0, CPU_TENSOR_NHWC(32F, 10, 100), 0);
    455
    0
      for (i = 0; i < 10 * 100; i++)
    456
    0
        dy_tensor->data.f32[i] = 0;
    457
    0
      for (i = 0; i < 10; i++)
    458
    0
        dy_tensor->data.f32[i * 100 + i] = 1;
    459
    0
      ccv_nnc_tensor_t* const dy16_tensor = ccv_nnc_tensor_new(0, CPU_TENSOR_NHWC(16F, 10, 100), 0);
    460
    0
      ccv_nnc_tensor_t* const dyt = ccv_nnc_tensor_new(0, GPU_TENSOR_NHWC(000, 16F, 10, 100), 0);
    461
    0
      ccv_nnc_cmd_exec(CMD_DATATYPE_CONVERSION_FORWARD(), ccv_nnc_no_hint, 0, TENSOR_LIST(dy_tensor), TENSOR_LIST(dy16_tensor), 0);
    462
    0
      ccv_nnc_cmd_exec(CMD_DATA_TRANSFER_FORWARD(), ccv_nnc_no_hint, 0, TENSOR_LIST(dy16_tensor), TENSOR_LIST(dyt), 0);
    463
    0
      ccv_nnc_graph_t* graph = 0;
    464
    0
      ccv_nnc_tensor_arena_t* tensor_arena = 0;
    465
    0
      ccv_nnc_graph_exec_arena_t* graph_exec_arena = 0;
    466
    0
      ccv_nnc_symbolic_graph_compile(symbolic_graph, ccv_nnc_default_compile_params, TENSOR_BIND_MAP(KV(dy, dyt)), TENSOR_SYMBOL_LIST(y), SYMBOLIC_GRAPH_SOURCES(symbolic_graph), SYMBOLIC_GRAPH_DESTINATIONS(symbolic_graph), &graph, &tensor_arena, &graph_exec_arena);
    467
    0
      GRAPH_GEN(graph, CCV_NNC_LONG_DOT_GRAPH);
    468
    0
      ccv_nnc_tensor_t* const xt = ccv_nnc_tensor_from_symbol(tensor_arena, x);
    469
    0
      ccv_nnc_tensor_t* const x16_tensor = ccv_nnc_tensor_new(0, CPU_TENSOR_NHWC(16F, 10, 100), 0);
    470
    0
      ccv_nnc_cmd_exec(CMD_DATATYPE_CONVERSION_FORWARD(), ccv_nnc_no_hint, 0, TENSOR_LIST(x_tensor), TENSOR_LIST(x16_tensor), 0);
    471
    0
      ccv_nnc_cmd_exec(CMD_DATA_TRANSFER_FORWARD(), ccv_nnc_no_hint, 0, TENSOR_LIST(x16_tensor), TENSOR_LIST(xt), 0);
    472
    0
      ccv_nnc_graph_run(graph, 0, TRAVERSE_FULL, 0, 0);
    473
    0
      ccv_nnc_tensor_t* const dx16_tensor = ccv_nnc_tensor_new(0, CPU_TENSOR_NHWC(16F, 10, 100), 0);
    474
    0
      ccv_nnc_tensor_t* const dx_tensor = ccv_nnc_tensor_new(0, CPU_TENSOR_NHWC(32F, 10, 100), 0);
    475
    0
      ccv_nnc_tensor_t* const dxt = ccv_nnc_tensor_from_symbol(tensor_arena, dx);
    476
    0
      ccv_nnc_tensor_t* const y16_tensor = ccv_nnc_tensor_new(0, CPU_TENSOR_NHWC(16F, 10, 100), 0);
    477
    0
      ccv_nnc_tensor_t* const y_tensor = ccv_nnc_tensor_new(0, CPU_TENSOR_NHWC(32F, 10, 100), 0);
    478
    0
      ccv_nnc_tensor_t* const yt = ccv_nnc_tensor_from_symbol(tensor_arena, y);
    479
    0
      ccv_nnc_cmd_exec(CMD_DATA_TRANSFER_FORWARD(), ccv_nnc_no_hint, 0, TENSOR_LIST(dxt), TENSOR_LIST(dx16_tensor), 0);
    480
    0
      ccv_nnc_cmd_exec(CMD_DATATYPE_CONVERSION_FORWARD(), ccv_nnc_no_hint, 0, TENSOR_LIST(dx16_tensor), TENSOR_LIST(dx_tensor), 0);
    481
    0
      ccv_nnc_cmd_exec(CMD_DATA_TRANSFER_FORWARD(), ccv_nnc_no_hint, 0, TENSOR_LIST(yt), TENSOR_LIST(y16_tensor), 0);
    482
    0
      ccv_nnc_cmd_exec(CMD_DATATYPE_CONVERSION_FORWARD(), ccv_nnc_no_hint, 0, TENSOR_LIST(y16_tensor), TENSOR_LIST(y_tensor), 0);
    483
    0
      ccv_nnc_tensor_t* const ty_tensor = ccv_nnc_tensor_new(0, CPU_TENSOR_NHWC(32F, 10, 100), 0);
    484
    0
      ccv_nnc_cmd_exec(CMD_SWISH_FORWARD(beta), ccv_nnc_no_hint, 0, TENSOR_LIST(x_tensor), TENSOR_LIST(ty_tensor), 0);
    485
    0
      REQUIRE_ARRAY_EQ_WITH_TOLERANCE(float, ty_tensor->data.f32, y_tensor->data.f32, 10 * 100, 1e-3, "forward pass should match");
    486
    0
      ccv_nnc_tensor_t* const tdx_tensor = ccv_nnc_tensor_new(0, CPU_TENSOR_NHWC(32F, 10, 100), 0);
    487
    0
      ccv_nnc_cmd_exec(CMD_SWISH_BACKWARD(beta), ccv_nnc_no_hint, 0, TENSOR_LIST(dy_tensor, x_tensor, 0), TENSOR_LIST(tdx_tensor), 0);
    488
    0
      REQUIRE_ARRAY_EQ_WITH_TOLERANCE(float, tdx_tensor->data.f32, dx_tensor->data.f32, 10 * 100, 1e-3, "backward pass should match");
    489
    0
      ccv_nnc_tensor_free(x_tensor);
    490
    0
      ccv_nnc_tensor_free(x16_tensor);
    491
    0
      ccv_nnc_tensor_free(y_tensor);
    492
    0
      ccv_nnc_tensor_free(y16_tensor);
    493
    0
      ccv_nnc_tensor_free(dx_tensor);
    494
    0
      ccv_nnc_tensor_free(dx16_tensor);
    495
    0
      ccv_nnc_tensor_free(dy_tensor);
    496
    0
      ccv_nnc_tensor_free(dy16_tensor);
    497
    0
      ccv_nnc_tensor_free(ty_tensor);
    498
    0
      ccv_nnc_tensor_free(tdx_tensor);
    499
    0
      ccv_nnc_tensor_free(dyt);
    500
    0
      ccv_nnc_graph_free(graph);
    501
    0
      ccv_nnc_tensor_arena_free(tensor_arena);
    502
    0
      ccv_nnc_graph_exec_arena_free(graph_exec_arena);
    503
    0
      ccv_nnc_symbolic_graph_free(symbolic_graph);
    504
    0
    }
    505
    506
    TEST_CASE("mps swish with extreme beta stays finite")
    507
    1
    {
    508
    1
      GUARD_ELSE_RETURN(ccv_nnc_cmd_ok(CCV_NNC_SWISH_FORWARD, CCV_NNC_BACKEND_MPS) &&
    509
    1
        ccv_nnc_cmd_ok(CCV_NNC_SWISH_BACKWARD, CCV_NNC_BACKEND_MPS));
    510
    0
      const float beta = 1e20f;
    511
    0
      static const float xv[] = {-1e20f, -1.0f, 0.0f, 1.0f, 1e20f};
    512
    0
      static const float expected_y[] = {0.0f, 0.0f, 0.0f, 1.0f, 1e20f};
    513
    0
      static const float expected_dx[] = {0.0f, 0.0f, 0.5f, 1.0f, 1.0f};
    514
    0
      ccv_nnc_tensor_t* const x_tensor = ccv_nnc_tensor_new(0, CPU_TENSOR_NHWC(32F, 5), 0);
    515
    0
      ccv_nnc_tensor_t* const y_tensor = ccv_nnc_tensor_new(0, CPU_TENSOR_NHWC(32F, 5), 0);
    516
    0
      ccv_nnc_tensor_t* const dy_tensor = ccv_nnc_tensor_new(0, CPU_TENSOR_NHWC(32F, 5), 0);
    517
    0
      ccv_nnc_tensor_t* const dx_tensor = ccv_nnc_tensor_new(0, CPU_TENSOR_NHWC(32F, 5), 0);
    518
    0
      ccv_nnc_tensor_t* const xt = ccv_nnc_tensor_new(0, GPU_TENSOR_NHWC(000, 32F, 5), 0);
    519
    0
      ccv_nnc_tensor_t* const yt = ccv_nnc_tensor_new(0, GPU_TENSOR_NHWC(000, 32F, 5), 0);
    520
    0
      ccv_nnc_tensor_t* const dyt = ccv_nnc_tensor_new(0, GPU_TENSOR_NHWC(000, 32F, 5), 0);
    521
    0
      ccv_nnc_tensor_t* const dxt = ccv_nnc_tensor_new(0, GPU_TENSOR_NHWC(000, 32F, 5), 0);
    522
    0
      int i;
    523
    0
      for (i = 0; i < 5; i++)
    524
    0
      {
    525
    0
        x_tensor->data.f32[i] = xv[i];
    526
    0
        dy_tensor->data.f32[i] = 1.0f;
    527
    0
      }
    528
    0
      ccv_nnc_cmd_exec(CMD_DATA_TRANSFER_FORWARD(), ccv_nnc_no_hint, 0, TENSOR_LIST(x_tensor), TENSOR_LIST(xt), 0);
    529
    0
      ccv_nnc_cmd_exec(CMD_DATA_TRANSFER_FORWARD(), ccv_nnc_no_hint, 0, TENSOR_LIST(dy_tensor), TENSOR_LIST(dyt), 0);
    530
    0
      ccv_nnc_cmd_exec(CMD_SWISH_FORWARD(beta), ccv_nnc_no_hint, 0, TENSOR_LIST(xt), TENSOR_LIST(yt), 0);
    531
    0
      ccv_nnc_cmd_exec(CMD_SWISH_BACKWARD(beta), ccv_nnc_no_hint, 0, TENSOR_LIST(dyt, xt, 0), TENSOR_LIST(dxt), 0);
    532
    0
      ccv_nnc_cmd_exec(CMD_DATA_TRANSFER_FORWARD(), ccv_nnc_no_hint, 0, TENSOR_LIST(yt), TENSOR_LIST(y_tensor), 0);
    533
    0
      ccv_nnc_cmd_exec(CMD_DATA_TRANSFER_FORWARD(), ccv_nnc_no_hint, 0, TENSOR_LIST(dxt), TENSOR_LIST(dx_tensor), 0);
    534
    0
      for (i = 0; i < 5; i++)
    535
    0
      {
    536
    0
        REQUIRE(isfinite(y_tensor->data.f32[i]), "forward pass should stay finite at %d", i);
    537
    0
        REQUIRE(isfinite(dx_tensor->data.f32[i]), "backward pass should stay finite at %d", i);
    538
    0
        const float y_tolerance = fmaxf(fabsf(expected_y[i]) * 1e-6f, 1e-6f);
    539
    0
        REQUIRE_EQ_WITH_TOLERANCE(y_tensor->data.f32[i], expected_y[i], y_tolerance, "forward pass should saturate cleanly at %d", i);
    540
    0
        REQUIRE_EQ_WITH_TOLERANCE(dx_tensor->data.f32[i], expected_dx[i], 1e-6f, "backward pass should saturate cleanly at %d", i);
    541
    0
      }
    542
    0
      ccv_nnc_tensor_free(x_tensor);
    543
    0
      ccv_nnc_tensor_free(y_tensor);
    544
    0
      ccv_nnc_tensor_free(dy_tensor);
    545
    0
      ccv_nnc_tensor_free(dx_tensor);
    546
    0
      ccv_nnc_tensor_free(xt);
    547
    0
      ccv_nnc_tensor_free(yt);
    548
    0
      ccv_nnc_tensor_free(dyt);
    549
    0
      ccv_nnc_tensor_free(dxt);
    550
    0
    }
    551
    552
    #include "case_main.h"