## Coverage Report

#### Created: 2026-05-04 15:30

next uncovered line (L), next uncovered region (R), next uncovered branch (B)

    /home/liu/actions-runner/_work/ccv/ccv/test/unit/nnc/group.norm.tests.c

    Line
    Count
    Source
    1
    #include "case.h"
    2
    #include "ccv_case.h"
    3
    #include "ccv_nnc_case.h"
    4
    #include <ccv.h>
    5
    #include <nnc/ccv_nnc.h>
    6
    #include <nnc/ccv_nnc_easy.h>
    7
    #include "3rdparty/dsfmt/dSFMT.h"
    8
    9
    TEST_SETUP()
    10
    {
    11
      ccv_nnc_init();
    12
    }
    13
    14
    TEST_CASE("implement simple group norm (like layer norm) with other symbolic graph")
    15
    1
    {
    16
    1
      ccv_nnc_symbolic_graph_t* const symbolic_graph = ccv_nnc_symbolic_graph_new();
    17
    1
      ccv_nnc_tensor_symbol_t x = ccv_nnc_tensor_symbol_new(symbolic_graph, CPU_TENSOR_NHWC(32F, 8, 4, 4, 10), "x");
    18
    1
      ccv_nnc_tensor_symbol_t sum = ccv_nnc_tensor_symbol_new(symbolic_graph, CPU_TENSOR_NHWC(32F, 8, 4, 4, 1), "sum");
    19
    1
      ccv_nnc_tensor_symbol_t mean = ccv_nnc_tensor_symbol_new(symbolic_graph, CPU_TENSOR_NHWC(32F, 8, 4, 4, 1), "mean");
    20
    1
      ccv_nnc_tensor_symbol_t whitening = ccv_nnc_tensor_symbol_new(symbolic_graph, CPU_TENSOR_NHWC(32F, 8, 4, 4, 10), "whitening");
    21
    1
      ccv_nnc_tensor_symbol_t sqr = ccv_nnc_tensor_symbol_new(symbolic_graph, CPU_TENSOR_NHWC(32F, 8, 4, 4, 10), "sqr");
    22
    1
      ccv_nnc_tensor_symbol_t varsum = ccv_nnc_tensor_symbol_new(symbolic_graph, CPU_TENSOR_NHWC(32F, 8, 4, 4, 1), "varsum");
    23
    1
      ccv_nnc_tensor_symbol_t var = ccv_nnc_tensor_symbol_new(symbolic_graph, CPU_TENSOR_NHWC(32F, 8, 4, 4, 1), "var");
    24
    1
      ccv_nnc_tensor_symbol_t logvar = ccv_nnc_tensor_symbol_new(symbolic_graph, CPU_TENSOR_NHWC(32F, 8, 4, 4, 1), "logvar");
    25
    1
      ccv_nnc_tensor_symbol_t logvar_2 = ccv_nnc_tensor_symbol_new(symbolic_graph, CPU_TENSOR_NHWC(32F, 8, 4, 4, 1), "logvar");
    26
    1
      ccv_nnc_tensor_symbol_t std = ccv_nnc_tensor_symbol_new(symbolic_graph, CPU_TENSOR_NHWC(32F, 8, 4, 4, 1), "std");
    27
    1
      ccv_nnc_tensor_symbol_t inv_std = ccv_nnc_tensor_symbol_new(symbolic_graph, CPU_TENSOR_NHWC(32F, 8, 4, 4, 1), "inv_std");
    28
    1
      ccv_nnc_tensor_symbol_t y = ccv_nnc_tensor_symbol_new(symbolic_graph, CPU_TENSOR_NHWC(32F, 8, 4, 4, 10), "y");
    29
    1
      ccv_nnc_graph_exec_symbol_new(symbolic_graph, CMD_REDUCE_SUM_FORWARD(3), TENSOR_SYMBOL_LIST(x), TENSOR_SYMBOL_LIST(sum), "sum");
    30
    1
      ccv_nnc_graph_exec_symbol_new(symbolic_graph, CMD_SCALAR_MUL_FORWARD(1.0 / 10), TENSOR_SYMBOL_LIST(sum), TENSOR_SYMBOL_LIST(mean), "mean");
    31
    1
      ccv_nnc_graph_exec_symbol_new(symbolic_graph, CMD_ADD_FORWARD(1, -1), TENSOR_SYMBOL_LIST(x, mean), TENSOR_SYMBOL_LIST(whitening), "whitening");
    32
    1
      ccv_nnc_graph_exec_symbol_new(symbolic_graph, CMD_EWPROD_FORWARD(), TENSOR_SYMBOL_LIST(whitening, whitening), TENSOR_SYMBOL_LIST(sqr), "sqr");
    33
    1
      ccv_nnc_graph_exec_symbol_new(symbolic_graph, CMD_REDUCE_SUM_FORWARD(3), TENSOR_SYMBOL_LIST(sqr), TENSOR_SYMBOL_LIST(varsum), "varsum");
    34
    1
      ccv_nnc_graph_exec_symbol_new(symbolic_graph, CMD_SCALAR_MUL_FORWARD(1.0 / 10), TENSOR_SYMBOL_LIST(varsum), TENSOR_SYMBOL_LIST(var), "var");
    35
    1
      ccv_nnc_graph_exec_symbol_new(symbolic_graph, CMD_EWLOG_FORWARD(), TENSOR_SYMBOL_LIST(var), TENSOR_SYMBOL_LIST(logvar), "log(var)");
    36
    1
      ccv_nnc_graph_exec_symbol_new(symbolic_graph, CMD_SCALAR_MUL_FORWARD(0.5), TENSOR_SYMBOL_LIST(logvar), TENSOR_SYMBOL_LIST(logvar_2), "log(var)/2");
    37
    1
      ccv_nnc_graph_exec_symbol_new(symbolic_graph, CMD_EWEXP_FORWARD(), TENSOR_SYMBOL_LIST(logvar_2), TENSOR_SYMBOL_LIST(std), "std");
    38
    1
      ccv_nnc_graph_exec_symbol_new(symbolic_graph, CMD_EWDIV_FORWARD(), TENSOR_SYMBOL_LIST(NO_TENSOR_SYMBOL, std), TENSOR_SYMBOL_LIST(inv_std), "1/std");
    39
    1
      ccv_nnc_graph_exec_symbol_new(symbolic_graph, CMD_MUL_FORWARD(1), TENSOR_SYMBOL_LIST(whitening, inv_std), TENSOR_SYMBOL_LIST(y), "y");
    40
    1
      ccv_nnc_graph_exec_symbol_autogen(symbolic_graph, 0, 0, CCV_NNC_AUTOGEN_ALL_EXECS | CCV_NNC_AUTOGEN_SOURCES_AND_DESTINATIONS);
    41
    1
      SYMBOLIC_GRAPH_GEN(symbolic_graph, CCV_NNC_LONG_DOT_GRAPH);
    42
    1
      ccv_nnc_graph_t* graph = 0;
    43
    1
      ccv_nnc_tensor_arena_t* tensor_arena = 0;
    44
    1
      ccv_nnc_graph_exec_arena_t* graph_exec_arena = 0;
    45
    1
      ccv_nnc_symbolic_graph_compile(symbolic_graph, ccv_nnc_default_compile_params, 0, 0, 0, 0, SYMBOLIC_GRAPH_SOURCES(symbolic_graph), SYMBOLIC_GRAPH_DESTINATIONS(symbolic_graph), &graph, &tensor_arena, &graph_exec_arena);
    46
    1
      GRAPH_GEN(graph, CCV_NNC_LONG_DOT_GRAPH);
    47
    1
      ccv_nnc_tensor_t* const x_tensor = ccv_nnc_tensor_from_symbol(tensor_arena, x);
    48
    1
      dsfmt_t dsfmt;
    49
    1
      int i;
    50
    1
      dsfmt_init_gen_rand(&dsfmt, 1);
    51
    1.28k
      for (i = 0; i < 8 * 4 * 4 * 10; i++1.28k)
    52
    1.28k
        x_tensor->data.f32[i] = dsfmt_genrand_open_close(&dsfmt);
    53
    1
      ccv_nnc_symbolic_graph_t* const group_norm_symbolic_graph = ccv_nnc_symbolic_graph_new();
    54
    1
      ccv_nnc_tensor_symbol_t bx = ccv_nnc_tensor_symbol_new(group_norm_symbolic_graph, CPU_TENSOR_NHWC(32F, 8, 4, 4, 10), "x");
    55
    1
      ccv_nnc_tensor_symbol_t by = ccv_nnc_tensor_symbol_new(group_norm_symbolic_graph, CPU_TENSOR_NHWC(32F, 8, 4, 4, 10), "y");
    56
    1
      ccv_nnc_tensor_symbol_t scale = ccv_nnc_tensor_symbol_new(group_norm_symbolic_graph, CPU_TENSOR_NHWC(32F, 1, 4, 4, 10), "scale");
    57
    1
      ccv_nnc_tensor_symbol_t bias = ccv_nnc_tensor_symbol_new(group_norm_symbolic_graph, CPU_TENSOR_NHWC(32F, 1, 4, 4, 10), "bias");
    58
    1
      ccv_nnc_tensor_symbol_t saved_mean = ccv_nnc_tensor_symbol_new(group_norm_symbolic_graph, CPU_TENSOR_NHWC(32F, 8, 4, 4, 1), "saved_mean");
    59
    1
      ccv_nnc_tensor_symbol_t saved_inv_std = ccv_nnc_tensor_symbol_new(group_norm_symbolic_graph, CPU_TENSOR_NHWC(32F, 8, 4, 4, 1), "saved_inv_std");
    60
    1
      ccv_nnc_graph_exec_symbol_new(group_norm_symbolic_graph, CMD_SET_FORWARD(1), 0, 0, TENSOR_SYMBOL_LIST(scale), "set_scale");
    61
    1
      ccv_nnc_graph_exec_symbol_new(group_norm_symbolic_graph, CMD_SET_FORWARD(0), 0, 0, TENSOR_SYMBOL_LIST(bias), "set_bias");
    62
    1
      ccv_nnc_graph_exec_symbol_new(group_norm_symbolic_graph, CMD_GROUP_NORM_FORWARD(3, 1, 0, 1), TENSOR_SYMBOL_LIST(bx, scale, bias), TENSOR_SYMBOL_LIST(by, saved_mean, saved_inv_std), "group_norm");
    63
    1
      ccv_nnc_graph_exec_symbol_autogen(group_norm_symbolic_graph, 0, 0, CCV_NNC_AUTOGEN_ALL_EXECS | CCV_NNC_AUTOGEN_SOURCES_AND_DESTINATIONS);
    64
    1
      ccv_nnc_graph_t* group_norm_graph = 0;
    65
    1
      ccv_nnc_tensor_arena_t* group_norm_tensor_arena = 0;
    66
    1
      ccv_nnc_graph_exec_arena_t* group_norm_graph_exec_arena = 0;
    67
    1
      ccv_nnc_symbolic_graph_compile(group_norm_symbolic_graph, ccv_nnc_default_compile_params, 0, 0, 0, 0, SYMBOLIC_GRAPH_SOURCES(group_norm_symbolic_graph), SYMBOLIC_GRAPH_DESTINATIONS(group_norm_symbolic_graph), &group_norm_graph, &group_norm_tensor_arena, &group_norm_graph_exec_arena);
    68
    1
      ccv_nnc_tensor_t* const bx_tensor = ccv_nnc_tensor_from_symbol(group_norm_tensor_arena, bx);
    69
    1
      memcpy(bx_tensor->data.f32, x_tensor->data.f32, sizeof(float) * 8 * 4 * 4 * 10);
    70
    1
      ccv_nnc_graph_run(graph, 0, TRAVERSE_FULL, 0, 0);
    71
    1
      ccv_nnc_graph_run(group_norm_graph, 0, TRAVERSE_FULL, 0, 0);
    72
    1
      ccv_nnc_tensor_t* const y_tensor = ccv_nnc_tensor_from_symbol(tensor_arena, y);
    73
    1
      ccv_nnc_tensor_t* const by_tensor = ccv_nnc_tensor_from_symbol(group_norm_tensor_arena, by);
    74
    1
      REQUIRE_TENSOR_EQ(y_tensor, by_tensor, "graph computed result should match batch norm op result");
    75
    1
      ccv_nnc_symbolic_graph_free(symbolic_graph);
    76
    1
      ccv_nnc_tensor_arena_free(tensor_arena);
    77
    1
      ccv_nnc_graph_exec_arena_free(graph_exec_arena);
    78
    1
      ccv_nnc_graph_free(graph);
    79
    1
      ccv_nnc_symbolic_graph_free(group_norm_symbolic_graph);
    80
    1
      ccv_nnc_tensor_arena_free(group_norm_tensor_arena);
    81
    1
      ccv_nnc_graph_exec_arena_free(group_norm_graph_exec_arena);
    82
    1
      ccv_nnc_graph_free(group_norm_graph);
    83
    1
    }
    84
    85
    TEST_CASE("compare simple group norm (like layer norm) gradient with other symbolic graph")
    86
    1
    {
    87
    1
      ccv_nnc_symbolic_graph_t* const symbolic_graph = ccv_nnc_symbolic_graph_new();
    88
    1
      ccv_nnc_tensor_symbol_t x = ccv_nnc_tensor_symbol_new(symbolic_graph, CPU_TENSOR_NHWC(32F, 8, 4, 4, 10), "x");
    89
    1
      ccv_nnc_tensor_symbol_t sum = ccv_nnc_tensor_symbol_new(symbolic_graph, CPU_TENSOR_NHWC(32F, 8, 4, 4, 1), "sum");
    90
    1
      ccv_nnc_tensor_symbol_t mean = ccv_nnc_tensor_symbol_new(symbolic_graph, CPU_TENSOR_NHWC(32F, 8, 4, 4, 1), "mean");
    91
    1
      ccv_nnc_tensor_symbol_t whitening = ccv_nnc_tensor_symbol_new(symbolic_graph, CPU_TENSOR_NHWC(32F, 8, 4, 4, 10), "whitening");
    92
    1
      ccv_nnc_tensor_symbol_t sqr = ccv_nnc_tensor_symbol_new(symbolic_graph, CPU_TENSOR_NHWC(32F, 8, 4, 4, 10), "sqr");
    93
    1
      ccv_nnc_tensor_symbol_t varsum = ccv_nnc_tensor_symbol_new(symbolic_graph, CPU_TENSOR_NHWC(32F, 8, 4, 4, 1), "varsum");
    94
    1
      ccv_nnc_tensor_symbol_t var = ccv_nnc_tensor_symbol_new(symbolic_graph, CPU_TENSOR_NHWC(32F, 8, 4, 4, 1), "var");
    95
    1
      ccv_nnc_tensor_symbol_t logvar = ccv_nnc_tensor_symbol_new(symbolic_graph, CPU_TENSOR_NHWC(32F, 8, 4, 4, 1), "logvar");
    96
    1
      ccv_nnc_tensor_symbol_t logvar_2 = ccv_nnc_tensor_symbol_new(symbolic_graph, CPU_TENSOR_NHWC(32F, 8, 4, 4, 1), "logvar");
    97
    1
      ccv_nnc_tensor_symbol_t std = ccv_nnc_tensor_symbol_new(symbolic_graph, CPU_TENSOR_NHWC(32F, 8, 4, 4, 1), "std");
    98
    1
      ccv_nnc_tensor_symbol_t inv_std = ccv_nnc_tensor_symbol_new(symbolic_graph, CPU_TENSOR_NHWC(32F, 8, 4, 4, 1), "inv_std");
    99
    1
      ccv_nnc_tensor_symbol_t y = ccv_nnc_tensor_symbol_new(symbolic_graph, CPU_TENSOR_NHWC(32F, 8, 4, 4, 10), "y");
    100
    1
      ccv_nnc_graph_exec_symbol_new(symbolic_graph, CMD_REDUCE_SUM_FORWARD(3), TENSOR_SYMBOL_LIST(x), TENSOR_SYMBOL_LIST(sum), "sum");
    101
    1
      ccv_nnc_graph_exec_symbol_new(symbolic_graph, CMD_SCALAR_MUL_FORWARD(1.0 / 10), TENSOR_SYMBOL_LIST(sum), TENSOR_SYMBOL_LIST(mean), "mean");
    102
    1
      ccv_nnc_graph_exec_symbol_new(symbolic_graph, CMD_ADD_FORWARD(1, -1), TENSOR_SYMBOL_LIST(x, mean), TENSOR_SYMBOL_LIST(whitening), "whitening");
    103
    1
      ccv_nnc_graph_exec_symbol_new(symbolic_graph, CMD_EWPROD_FORWARD(), TENSOR_SYMBOL_LIST(whitening, whitening), TENSOR_SYMBOL_LIST(sqr), "sqr");
    104
    1
      ccv_nnc_graph_exec_symbol_new(symbolic_graph, CMD_REDUCE_SUM_FORWARD(3), TENSOR_SYMBOL_LIST(sqr), TENSOR_SYMBOL_LIST(varsum), "varsum");
    105
    1
      ccv_nnc_graph_exec_symbol_new(symbolic_graph, CMD_SCALAR_MUL_FORWARD(1.0 / 10), TENSOR_SYMBOL_LIST(varsum), TENSOR_SYMBOL_LIST(var), "var");
    106
    1
      ccv_nnc_graph_exec_symbol_new(symbolic_graph, CMD_EWLOG_FORWARD(), TENSOR_SYMBOL_LIST(var), TENSOR_SYMBOL_LIST(logvar), "log(var)");
    107
    1
      ccv_nnc_graph_exec_symbol_new(symbolic_graph, CMD_SCALAR_MUL_FORWARD(0.5), TENSOR_SYMBOL_LIST(logvar), TENSOR_SYMBOL_LIST(logvar_2), "log(var)/2");
    108
    1
      ccv_nnc_graph_exec_symbol_new(symbolic_graph, CMD_EWEXP_FORWARD(), TENSOR_SYMBOL_LIST(logvar_2), TENSOR_SYMBOL_LIST(std), "std");
    109
    1
      ccv_nnc_graph_exec_symbol_new(symbolic_graph, CMD_EWDIV_FORWARD(), TENSOR_SYMBOL_LIST(NO_TENSOR_SYMBOL, std), TENSOR_SYMBOL_LIST(inv_std), "1/std");
    110
    1
      ccv_nnc_graph_exec_symbol_new(symbolic_graph, CMD_MUL_FORWARD(1), TENSOR_SYMBOL_LIST(whitening, inv_std), TENSOR_SYMBOL_LIST(y), "y");
    111
    1
      ccv_nnc_graph_exec_symbol_autogen(symbolic_graph, 0, 0, CCV_NNC_AUTOGEN_ALL_EXECS | CCV_NNC_AUTOGEN_SOURCES_AND_DESTINATIONS);
    112
    1
      ccv_nnc_symbolic_graph_backward(symbolic_graph, TENSOR_SYMBOL_LIST(y), TENSOR_SYMBOL_LIST(x), SYMBOLIC_GRAPH_SOURCES(symbolic_graph), SYMBOLIC_GRAPH_DESTINATIONS(symbolic_graph));
    113
    1
      ccv_nnc_graph_exec_symbol_autogen(symbolic_graph, 0, 0, CCV_NNC_AUTOGEN_ALL_EXECS | CCV_NNC_AUTOGEN_SOURCES_AND_DESTINATIONS);
    114
    1
      ccv_nnc_tensor_symbol_t dy = ccv_nnc_tensor_symbol_for_backward(symbolic_graph, y);
    115
    1
      ccv_nnc_tensor_symbol_t dx = ccv_nnc_tensor_symbol_for_backward(symbolic_graph, x);
    116
    1
      SYMBOLIC_GRAPH_GEN(symbolic_graph, CCV_NNC_LONG_DOT_GRAPH);
    117
    1
      ccv_nnc_graph_t* graph = 0;
    118
    1
      ccv_nnc_tensor_arena_t* tensor_arena = 0;
    119
    1
      ccv_nnc_graph_exec_arena_t* graph_exec_arena = 0;
    120
    1
      ccv_nnc_symbolic_graph_compile(symbolic_graph, ccv_nnc_default_compile_params, 0, 0, 0, 0, SYMBOLIC_GRAPH_SOURCES(symbolic_graph), SYMBOLIC_GRAPH_DESTINATIONS(symbolic_graph), &graph, &tensor_arena, &graph_exec_arena);
    121
    1
      GRAPH_GEN(graph, CCV_NNC_LONG_DOT_GRAPH);
    122
    1
      ccv_nnc_tensor_t* const x_tensor = ccv_nnc_tensor_from_symbol(tensor_arena, x);
    123
    1
      dsfmt_t dsfmt;
    124
    1
      int i;
    125
    1
      dsfmt_init_gen_rand(&dsfmt, 1);
    126
    1.28k
      for (i = 0; i < 8 * 4 * 4 * 10; i++1.28k)
    127
    1.28k
        x_tensor->data.f32[i] = dsfmt_genrand_open_close(&dsfmt);
    128
    1
      ccv_nnc_symbolic_graph_t* const group_norm_symbolic_graph = ccv_nnc_symbolic_graph_new();
    129
    1
      ccv_nnc_tensor_symbol_t bx = ccv_nnc_tensor_symbol_new(group_norm_symbolic_graph, CPU_TENSOR_NHWC(32F, 8, 4, 4, 10), "x");
    130
    1
      ccv_nnc_tensor_symbol_t by = ccv_nnc_tensor_symbol_new(group_norm_symbolic_graph, CPU_TENSOR_NHWC(32F, 8, 4, 4, 10), "y");
    131
    1
      ccv_nnc_tensor_symbol_t scale = ccv_nnc_tensor_symbol_new(group_norm_symbolic_graph, CPU_TENSOR_NHWC(32F, 1, 4, 4, 10), "scale");
    132
    1
      ccv_nnc_tensor_symbol_t bias = ccv_nnc_tensor_symbol_new(group_norm_symbolic_graph, CPU_TENSOR_NHWC(32F, 1, 4, 4, 10), "bias");
    133
    1
      ccv_nnc_tensor_symbol_t saved_mean = ccv_nnc_tensor_symbol_new(group_norm_symbolic_graph, CPU_TENSOR_NHWC(32F, 8, 4, 4, 1), "saved_mean");
    134
    1
      ccv_nnc_tensor_symbol_t saved_inv_std = ccv_nnc_tensor_symbol_new(group_norm_symbolic_graph, CPU_TENSOR_NHWC(32F, 8, 4, 4, 1), "saved_inv_std");
    135
    1
      ccv_nnc_graph_exec_symbol_new(group_norm_symbolic_graph, CMD_SET_FORWARD(1), 0, 0, TENSOR_SYMBOL_LIST(scale), "set_scale");
    136
    1
      ccv_nnc_graph_exec_symbol_new(group_norm_symbolic_graph, CMD_SET_FORWARD(0), 0, 0, TENSOR_SYMBOL_LIST(bias), "set_bias");
    137
    1
      ccv_nnc_graph_exec_symbol_new(group_norm_symbolic_graph, CMD_GROUP_NORM_FORWARD(3, 1, 0, 1), TENSOR_SYMBOL_LIST(bx, scale, bias), TENSOR_SYMBOL_LIST(by, saved_mean, saved_inv_std), "group_norm");
    138
    1
      ccv_nnc_graph_exec_symbol_autogen(group_norm_symbolic_graph, 0, 0, CCV_NNC_AUTOGEN_ALL_EXECS | CCV_NNC_AUTOGEN_SOURCES_AND_DESTINATIONS);
    139
    1
      ccv_nnc_symbolic_graph_backward(group_norm_symbolic_graph, TENSOR_SYMBOL_LIST(by), TENSOR_SYMBOL_LIST(bx, scale, bias), SYMBOLIC_GRAPH_SOURCES(group_norm_symbolic_graph), SYMBOLIC_GRAPH_DESTINATIONS(group_norm_symbolic_graph));
    140
    1
      ccv_nnc_graph_exec_symbol_autogen(group_norm_symbolic_graph, 0, 0, CCV_NNC_AUTOGEN_ALL_EXECS | CCV_NNC_AUTOGEN_SOURCES_AND_DESTINATIONS);
    141
    1
      ccv_nnc_tensor_symbol_t dby = ccv_nnc_tensor_symbol_for_backward(group_norm_symbolic_graph, by);
    142
    1
      ccv_nnc_tensor_symbol_t dbx = ccv_nnc_tensor_symbol_for_backward(group_norm_symbolic_graph, bx);
    143
    1
      ccv_nnc_graph_t* group_norm_graph = 0;
    144
    1
      ccv_nnc_tensor_arena_t* group_norm_tensor_arena = 0;
    145
    1
      ccv_nnc_graph_exec_arena_t* group_norm_graph_exec_arena = 0;
    146
    1
      ccv_nnc_symbolic_graph_compile(group_norm_symbolic_graph, ccv_nnc_default_compile_params, 0, 0, 0, 0, SYMBOLIC_GRAPH_SOURCES(group_norm_symbolic_graph), SYMBOLIC_GRAPH_DESTINATIONS(group_norm_symbolic_graph), &group_norm_graph, &group_norm_tensor_arena, &group_norm_graph_exec_arena);
    147
    1
      ccv_nnc_tensor_t* const bx_tensor = ccv_nnc_tensor_from_symbol(group_norm_tensor_arena, bx);
    148
    1
      ccv_nnc_tensor_t* const dy_tensor = ccv_nnc_tensor_from_symbol(tensor_arena, dy);
    149
    1
      ccv_nnc_tensor_t* const dby_tensor = ccv_nnc_tensor_from_symbol(group_norm_tensor_arena, dby);
    150
    1.28k
      for (i = 0; i < 8 * 4 * 4 * 10; i++1.28k)
    151
    1.28k
        dby_tensor->data.f32[i] = dy_tensor->data.f32[i] = dsfmt_genrand_open_close(&dsfmt) * 2 - 1;
    152
    1
      memcpy(bx_tensor->data.f32, x_tensor->data.f32, sizeof(float) * 8 * 4 * 4 * 10);
    153
    1
      ccv_nnc_graph_run(graph, 0, TRAVERSE_FULL, 0, 0);
    154
    1
      ccv_nnc_graph_run(group_norm_graph, 0, TRAVERSE_FULL, 0, 0);
    155
    1
      ccv_nnc_tensor_t* const dx_tensor = ccv_nnc_tensor_from_symbol(tensor_arena, dx);
    156
    1
      ccv_nnc_tensor_t* const dbx_tensor = ccv_nnc_tensor_from_symbol(group_norm_tensor_arena, dbx);
    157
    1
      REQUIRE_ARRAY_EQ_WITH_TOLERANCE(float, dx_tensor->data.f32, dbx_tensor->data.f32, 8 * 4 * 4 * 10, 1e-5, "graph computed result should match layer norm op result");
    158
    1
      ccv_nnc_symbolic_graph_free(symbolic_graph);
    159
    1
      ccv_nnc_tensor_arena_free(tensor_arena);
    160
    1
      ccv_nnc_graph_exec_arena_free(graph_exec_arena);
    161
    1
      ccv_nnc_graph_free(graph);
    162
    1
      ccv_nnc_symbolic_graph_free(group_norm_symbolic_graph);
    163
    1
      ccv_nnc_tensor_arena_free(group_norm_tensor_arena);
    164
    1
      ccv_nnc_graph_exec_arena_free(group_norm_graph_exec_arena);
    165
    1
      ccv_nnc_graph_free(group_norm_graph);
    166
    1
    }
    167
    168
    TEST_CASE("implement group norm with other symbolic graph")
    169
    1
    {
    170
    1
      ccv_nnc_symbolic_graph_t* const symbolic_graph = ccv_nnc_symbolic_graph_new();
    171
    1
      ccv_nnc_tensor_symbol_t x = ccv_nnc_tensor_symbol_new(symbolic_graph, CPU_TENSOR_NHWC(32F, 8, 4, 4, 5), "x");
    172
    1
      ccv_nnc_tensor_symbol_t sum = ccv_nnc_tensor_symbol_new(symbolic_graph, CPU_TENSOR_NHWC(32F, 8, 4, 1, 5), "sum");
    173
    1
      ccv_nnc_tensor_symbol_t mean = ccv_nnc_tensor_symbol_new(symbolic_graph, CPU_TENSOR_NHWC(32F, 8, 4, 1, 5), "mean");
    174
    1
      ccv_nnc_tensor_symbol_t whitening = ccv_nnc_tensor_symbol_new(symbolic_graph, CPU_TENSOR_NHWC(32F, 8, 4, 4, 5), "whitening");
    175
    1
      ccv_nnc_tensor_symbol_t sqr = ccv_nnc_tensor_symbol_new(symbolic_graph, CPU_TENSOR_NHWC(32F, 8, 4, 4, 5), "sqr");
    176
    1
      ccv_nnc_tensor_symbol_t varsum = ccv_nnc_tensor_symbol_new(symbolic_graph, CPU_TENSOR_NHWC(32F, 8, 4, 1, 5), "varsum");
    177
    1
      ccv_nnc_tensor_symbol_t var = ccv_nnc_tensor_symbol_new(symbolic_graph, CPU_TENSOR_NHWC(32F, 8, 4, 1, 5), "var");
    178
    1
      ccv_nnc_tensor_symbol_t logvar = ccv_nnc_tensor_symbol_new(symbolic_graph, CPU_TENSOR_NHWC(32F, 8, 4, 1, 5), "logvar");
    179
    1
      ccv_nnc_tensor_symbol_t logvar_2 = ccv_nnc_tensor_symbol_new(symbolic_graph, CPU_TENSOR_NHWC(32F, 8, 4, 1, 5), "logvar");
    180
    1
      ccv_nnc_tensor_symbol_t std = ccv_nnc_tensor_symbol_new(symbolic_graph, CPU_TENSOR_NHWC(32F, 8, 4, 1, 5), "std");
    181
    1
      ccv_nnc_tensor_symbol_t inv_std = ccv_nnc_tensor_symbol_new(symbolic_graph, CPU_TENSOR_NHWC(32F, 8, 4, 1, 5), "inv_std");
    182
    1
      ccv_nnc_tensor_symbol_t y = ccv_nnc_tensor_symbol_new(symbolic_graph, CPU_TENSOR_NHWC(32F, 8, 4, 4, 5), "y");
    183
    1
      ccv_nnc_graph_exec_symbol_new(symbolic_graph, CMD_REDUCE_SUM_FORWARD(1), TENSOR_SYMBOL_LIST(x), TENSOR_SYMBOL_LIST(sum), "sum");
    184
    1
      ccv_nnc_graph_exec_symbol_new(symbolic_graph, CMD_SCALAR_MUL_FORWARD(1.0 / 4), TENSOR_SYMBOL_LIST(sum), TENSOR_SYMBOL_LIST(mean), "mean");
    185
    1
      ccv_nnc_graph_exec_symbol_new(symbolic_graph, CMD_ADD_FORWARD(1, -1), TENSOR_SYMBOL_LIST(x, mean), TENSOR_SYMBOL_LIST(whitening), "whitening");
    186
    1
      ccv_nnc_graph_exec_symbol_new(symbolic_graph, CMD_EWPROD_FORWARD(), TENSOR_SYMBOL_LIST(whitening, whitening), TENSOR_SYMBOL_LIST(sqr), "sqr");
    187
    1
      ccv_nnc_graph_exec_symbol_new(symbolic_graph, CMD_REDUCE_SUM_FORWARD(1), TENSOR_SYMBOL_LIST(sqr), TENSOR_SYMBOL_LIST(varsum), "varsum");
    188
    1
      ccv_nnc_graph_exec_symbol_new(symbolic_graph, CMD_SCALAR_MUL_FORWARD(1.0 / 4), TENSOR_SYMBOL_LIST(varsum), TENSOR_SYMBOL_LIST(var), "var");
    189
    1
      ccv_nnc_graph_exec_symbol_new(symbolic_graph, CMD_EWLOG_FORWARD(), TENSOR_SYMBOL_LIST(var), TENSOR_SYMBOL_LIST(logvar), "log(var)");
    190
    1
      ccv_nnc_graph_exec_symbol_new(symbolic_graph, CMD_SCALAR_MUL_FORWARD(0.5), TENSOR_SYMBOL_LIST(logvar), TENSOR_SYMBOL_LIST(logvar_2), "log(var)/2");
    191
    1
      ccv_nnc_graph_exec_symbol_new(symbolic_graph, CMD_EWEXP_FORWARD(), TENSOR_SYMBOL_LIST(logvar_2), TENSOR_SYMBOL_LIST(std), "std");
    192
    1
      ccv_nnc_graph_exec_symbol_new(symbolic_graph, CMD_EWDIV_FORWARD(), TENSOR_SYMBOL_LIST(NO_TENSOR_SYMBOL, std), TENSOR_SYMBOL_LIST(inv_std), "1/std");
    193
    1
      ccv_nnc_graph_exec_symbol_new(symbolic_graph, CMD_MUL_FORWARD(1), TENSOR_SYMBOL_LIST(whitening, inv_std), TENSOR_SYMBOL_LIST(y), "y");
    194
    1
      ccv_nnc_graph_exec_symbol_autogen(symbolic_graph, 0, 0, CCV_NNC_AUTOGEN_ALL_EXECS | CCV_NNC_AUTOGEN_SOURCES_AND_DESTINATIONS);
    195
    1
      SYMBOLIC_GRAPH_GEN(symbolic_graph, CCV_NNC_LONG_DOT_GRAPH);
    196
    1
      ccv_nnc_graph_t* graph = 0;
    197
    1
      ccv_nnc_tensor_arena_t* tensor_arena = 0;
    198
    1
      ccv_nnc_graph_exec_arena_t* graph_exec_arena = 0;
    199
    1
      ccv_nnc_symbolic_graph_compile(symbolic_graph, ccv_nnc_default_compile_params, 0, 0, 0, 0, SYMBOLIC_GRAPH_SOURCES(symbolic_graph), SYMBOLIC_GRAPH_DESTINATIONS(symbolic_graph), &graph, &tensor_arena, &graph_exec_arena);
    200
    1
      GRAPH_GEN(graph, CCV_NNC_LONG_DOT_GRAPH);
    201
    1
      ccv_nnc_tensor_t* const x_tensor = ccv_nnc_tensor_from_symbol(tensor_arena, x);
    202
    1
      dsfmt_t dsfmt;
    203
    1
      int i;
    204
    1
      dsfmt_init_gen_rand(&dsfmt, 1);
    205
    641
      for (i = 0; i < 8 * 4 * 4 * 5; i++640)
    206
    640
        x_tensor->data.f32[i] = dsfmt_genrand_open_close(&dsfmt);
    207
    1
      ccv_nnc_symbolic_graph_t* const group_norm_symbolic_graph = ccv_nnc_symbolic_graph_new();
    208
    1
      ccv_nnc_tensor_symbol_t bx = ccv_nnc_tensor_symbol_new(group_norm_symbolic_graph, CPU_TENSOR_NHWC(32F, 8, 16, 5), "x");
    209
    1
      ccv_nnc_tensor_symbol_t by = ccv_nnc_tensor_symbol_new(group_norm_symbolic_graph, CPU_TENSOR_NHWC(32F, 8, 16, 5), "y");
    210
    1
      ccv_nnc_tensor_symbol_t scale = ccv_nnc_tensor_symbol_new(group_norm_symbolic_graph, CPU_TENSOR_NHWC(32F, 1, 16, 5), "scale");
    211
    1
      ccv_nnc_tensor_symbol_t bias = ccv_nnc_tensor_symbol_new(group_norm_symbolic_graph, CPU_TENSOR_NHWC(32F, 1, 16, 5), "bias");
    212
    1
      ccv_nnc_tensor_symbol_t saved_mean = ccv_nnc_tensor_symbol_new(group_norm_symbolic_graph, CPU_TENSOR_NHWC(32F, 8, 4, 5), "saved_mean");
    213
    1
      ccv_nnc_tensor_symbol_t saved_inv_std = ccv_nnc_tensor_symbol_new(group_norm_symbolic_graph, CPU_TENSOR_NHWC(32F, 8, 4, 5), "saved_inv_std");
    214
    1
      ccv_nnc_graph_exec_symbol_new(group_norm_symbolic_graph, CMD_SET_FORWARD(1), 0, 0, TENSOR_SYMBOL_LIST(scale), "set_scale");
    215
    1
      ccv_nnc_graph_exec_symbol_new(group_norm_symbolic_graph, CMD_SET_FORWARD(0), 0, 0, TENSOR_SYMBOL_LIST(bias), "set_bias");
    216
    1
      ccv_nnc_graph_exec_symbol_new(group_norm_symbolic_graph, CMD_GROUP_NORM_FORWARD(1, 4, 0, 1), TENSOR_SYMBOL_LIST(bx, scale, bias), TENSOR_SYMBOL_LIST(by, saved_mean, saved_inv_std), "group_norm");
    217
    1
      ccv_nnc_graph_exec_symbol_autogen(group_norm_symbolic_graph, 0, 0, CCV_NNC_AUTOGEN_ALL_EXECS | CCV_NNC_AUTOGEN_SOURCES_AND_DESTINATIONS);
    218
    1
      ccv_nnc_graph_t* group_norm_graph = 0;
    219
    1
      ccv_nnc_tensor_arena_t* group_norm_tensor_arena = 0;
    220
    1
      ccv_nnc_graph_exec_arena_t* group_norm_graph_exec_arena = 0;
    221
    1
      ccv_nnc_symbolic_graph_compile(group_norm_symbolic_graph, ccv_nnc_default_compile_params, 0, 0, 0, 0, SYMBOLIC_GRAPH_SOURCES(group_norm_symbolic_graph), SYMBOLIC_GRAPH_DESTINATIONS(group_norm_symbolic_graph), &group_norm_graph, &group_norm_tensor_arena, &group_norm_graph_exec_arena);
    222
    1
      ccv_nnc_tensor_t* const bx_tensor = ccv_nnc_tensor_from_symbol(group_norm_tensor_arena, bx);
    223
    1
      memcpy(bx_tensor->data.f32, x_tensor->data.f32, sizeof(float) * 8 * 16 * 5);
    224
    1
      ccv_nnc_graph_run(graph, 0, TRAVERSE_FULL, 0, 0);
    225
    1
      ccv_nnc_graph_run(group_norm_graph, 0, TRAVERSE_FULL, 0, 0);
    226
    1
      ccv_nnc_tensor_t* const y_tensor = ccv_nnc_tensor_from_symbol(tensor_arena, y);
    227
    1
      ccv_nnc_tensor_t* const by_tensor = ccv_nnc_tensor_from_symbol(group_norm_tensor_arena, by);
    228
    1
      REQUIRE_ARRAY_EQ_WITH_TOLERANCE(float, y_tensor->data.f32, by_tensor->data.f32, 8 * 16 * 5, 1e-5, "graph computed result should match batch norm op result");
    229
    1
      ccv_nnc_symbolic_graph_free(symbolic_graph);
    230
    1
      ccv_nnc_tensor_arena_free(tensor_arena);
    231
    1
      ccv_nnc_graph_exec_arena_free(graph_exec_arena);
    232
    1
      ccv_nnc_graph_free(graph);
    233
    1
      ccv_nnc_symbolic_graph_free(group_norm_symbolic_graph);
    234
    1
      ccv_nnc_tensor_arena_free(group_norm_tensor_arena);
    235
    1
      ccv_nnc_graph_exec_arena_free(group_norm_graph_exec_arena);
    236
    1
      ccv_nnc_graph_free(group_norm_graph);
    237
    1
    }
    238
    239
    TEST_CASE("compare group norm gradient with other symbolic graph")
    240
    1
    {
    241
    1
      ccv_nnc_symbolic_graph_t* const symbolic_graph = ccv_nnc_symbolic_graph_new();
    242
    1
      ccv_nnc_tensor_symbol_t x = ccv_nnc_tensor_symbol_new(symbolic_graph, CPU_TENSOR_NHWC(32F, 8, 4, 4, 5), "x");
    243
    1
      ccv_nnc_tensor_symbol_t sum = ccv_nnc_tensor_symbol_new(symbolic_graph, CPU_TENSOR_NHWC(32F, 8, 4, 1, 5), "sum");
    244
    1
      ccv_nnc_tensor_symbol_t mean = ccv_nnc_tensor_symbol_new(symbolic_graph, CPU_TENSOR_NHWC(32F, 8, 4, 1, 5), "mean");
    245
    1
      ccv_nnc_tensor_symbol_t whitening = ccv_nnc_tensor_symbol_new(symbolic_graph, CPU_TENSOR_NHWC(32F, 8, 4, 4, 5), "whitening");
    246
    1
      ccv_nnc_tensor_symbol_t sqr = ccv_nnc_tensor_symbol_new(symbolic_graph, CPU_TENSOR_NHWC(32F, 8, 4, 4, 5), "sqr");
    247
    1
      ccv_nnc_tensor_symbol_t varsum = ccv_nnc_tensor_symbol_new(symbolic_graph, CPU_TENSOR_NHWC(32F, 8, 4, 1, 5), "varsum");
    248
    1
      ccv_nnc_tensor_symbol_t var = ccv_nnc_tensor_symbol_new(symbolic_graph, CPU_TENSOR_NHWC(32F, 8, 4, 1, 5), "var");
    249
    1
      ccv_nnc_tensor_symbol_t logvar = ccv_nnc_tensor_symbol_new(symbolic_graph, CPU_TENSOR_NHWC(32F, 8, 4, 1, 5), "logvar");
    250
    1
      ccv_nnc_tensor_symbol_t logvar_2 = ccv_nnc_tensor_symbol_new(symbolic_graph, CPU_TENSOR_NHWC(32F, 8, 4, 1, 5), "logvar");
    251
    1
      ccv_nnc_tensor_symbol_t std = ccv_nnc_tensor_symbol_new(symbolic_graph, CPU_TENSOR_NHWC(32F, 8, 4, 1, 5), "std");
    252
    1
      ccv_nnc_tensor_symbol_t inv_std = ccv_nnc_tensor_symbol_new(symbolic_graph, CPU_TENSOR_NHWC(32F, 8, 4, 1, 5), "inv_std");
    253
    1
      ccv_nnc_tensor_symbol_t y = ccv_nnc_tensor_symbol_new(symbolic_graph, CPU_TENSOR_NHWC(32F, 8, 4, 4, 5), "y");
    254
    1
      ccv_nnc_graph_exec_symbol_new(symbolic_graph, CMD_REDUCE_SUM_FORWARD(1), TENSOR_SYMBOL_LIST(x), TENSOR_SYMBOL_LIST(sum), "sum");
    255
    1
      ccv_nnc_graph_exec_symbol_new(symbolic_graph, CMD_SCALAR_MUL_FORWARD(1.0 / 4), TENSOR_SYMBOL_LIST(sum), TENSOR_SYMBOL_LIST(mean), "mean");
    256
    1
      ccv_nnc_graph_exec_symbol_new(symbolic_graph, CMD_ADD_FORWARD(1, -1), TENSOR_SYMBOL_LIST(x, mean), TENSOR_SYMBOL_LIST(whitening), "whitening");
    257
    1
      ccv_nnc_graph_exec_symbol_new(symbolic_graph, CMD_EWPROD_FORWARD(), TENSOR_SYMBOL_LIST(whitening, whitening), TENSOR_SYMBOL_LIST(sqr), "sqr");
    258
    1
      ccv_nnc_graph_exec_symbol_new(symbolic_graph, CMD_REDUCE_SUM_FORWARD(1), TENSOR_SYMBOL_LIST(sqr), TENSOR_SYMBOL_LIST(varsum), "varsum");
    259
    1
      ccv_nnc_graph_exec_symbol_new(symbolic_graph, CMD_SCALAR_MUL_FORWARD(1.0 / 4), TENSOR_SYMBOL_LIST(varsum), TENSOR_SYMBOL_LIST(var), "var");
    260
    1
      ccv_nnc_graph_exec_symbol_new(symbolic_graph, CMD_EWLOG_FORWARD(), TENSOR_SYMBOL_LIST(var), TENSOR_SYMBOL_LIST(logvar), "log(var)");
    261
    1
      ccv_nnc_graph_exec_symbol_new(symbolic_graph, CMD_SCALAR_MUL_FORWARD(0.5), TENSOR_SYMBOL_LIST(logvar), TENSOR_SYMBOL_LIST(logvar_2), "log(var)/2");
    262
    1
      ccv_nnc_graph_exec_symbol_new(symbolic_graph, CMD_EWEXP_FORWARD(), TENSOR_SYMBOL_LIST(logvar_2), TENSOR_SYMBOL_LIST(std), "std");
    263
    1
      ccv_nnc_graph_exec_symbol_new(symbolic_graph, CMD_EWDIV_FORWARD(), TENSOR_SYMBOL_LIST(NO_TENSOR_SYMBOL, std), TENSOR_SYMBOL_LIST(inv_std), "1/std");
    264
    1
      ccv_nnc_graph_exec_symbol_new(symbolic_graph, CMD_MUL_FORWARD(1), TENSOR_SYMBOL_LIST(whitening, inv_std), TENSOR_SYMBOL_LIST(y), "y");
    265
    1
      ccv_nnc_graph_exec_symbol_autogen(symbolic_graph, 0, 0, CCV_NNC_AUTOGEN_ALL_EXECS | CCV_NNC_AUTOGEN_SOURCES_AND_DESTINATIONS);
    266
    1
      ccv_nnc_symbolic_graph_backward(symbolic_graph, TENSOR_SYMBOL_LIST(y), TENSOR_SYMBOL_LIST(x), SYMBOLIC_GRAPH_SOURCES(symbolic_graph), SYMBOLIC_GRAPH_DESTINATIONS(symbolic_graph));
    267
    1
      ccv_nnc_graph_exec_symbol_autogen(symbolic_graph, 0, 0, CCV_NNC_AUTOGEN_ALL_EXECS | CCV_NNC_AUTOGEN_SOURCES_AND_DESTINATIONS);
    268
    1
      ccv_nnc_tensor_symbol_t dy = ccv_nnc_tensor_symbol_for_backward(symbolic_graph, y);
    269
    1
      ccv_nnc_tensor_symbol_t dx = ccv_nnc_tensor_symbol_for_backward(symbolic_graph, x);
    270
    1
      SYMBOLIC_GRAPH_GEN(symbolic_graph, CCV_NNC_LONG_DOT_GRAPH);
    271
    1
      ccv_nnc_graph_t* graph = 0;
    272
    1
      ccv_nnc_tensor_arena_t* tensor_arena = 0;
    273
    1
      ccv_nnc_graph_exec_arena_t* graph_exec_arena = 0;
    274
    1
      ccv_nnc_symbolic_graph_compile(symbolic_graph, ccv_nnc_default_compile_params, 0, 0, 0, 0, SYMBOLIC_GRAPH_SOURCES(symbolic_graph), SYMBOLIC_GRAPH_DESTINATIONS(symbolic_graph), &graph, &tensor_arena, &graph_exec_arena);
    275
    1
      GRAPH_GEN(graph, CCV_NNC_LONG_DOT_GRAPH);
    276
    1
      ccv_nnc_tensor_t* const x_tensor = ccv_nnc_tensor_from_symbol(tensor_arena, x);
    277
    1
      dsfmt_t dsfmt;
    278
    1
      int i;
    279
    1
      dsfmt_init_gen_rand(&dsfmt, 1);
    280
    641
      for (i = 0; i < 8 * 4 * 4 * 5; i++640)
    281
    640
        x_tensor->data.f32[i] = dsfmt_genrand_open_close(&dsfmt);
    282
    1
      ccv_nnc_symbolic_graph_t* const group_norm_symbolic_graph = ccv_nnc_symbolic_graph_new();
    283
    1
      ccv_nnc_tensor_symbol_t bx = ccv_nnc_tensor_symbol_new(group_norm_symbolic_graph, CPU_TENSOR_NHWC(32F, 8, 16, 5), "x");
    284
    1
      ccv_nnc_tensor_symbol_t by = ccv_nnc_tensor_symbol_new(group_norm_symbolic_graph, CPU_TENSOR_NHWC(32F, 8, 16, 5), "y");
    285
    1
      ccv_nnc_tensor_symbol_t scale = ccv_nnc_tensor_symbol_new(group_norm_symbolic_graph, CPU_TENSOR_NHWC(32F, 1, 16, 5), "scale");
    286
    1
      ccv_nnc_tensor_symbol_t bias = ccv_nnc_tensor_symbol_new(group_norm_symbolic_graph, CPU_TENSOR_NHWC(32F, 1, 16, 5), "bias");
    287
    1
      ccv_nnc_tensor_symbol_t saved_mean = ccv_nnc_tensor_symbol_new(group_norm_symbolic_graph, CPU_TENSOR_NHWC(32F, 8, 4, 5), "saved_mean");
    288
    1
      ccv_nnc_tensor_symbol_t saved_inv_std = ccv_nnc_tensor_symbol_new(group_norm_symbolic_graph, CPU_TENSOR_NHWC(32F, 8, 4, 5), "saved_inv_std");
    289
    1
      ccv_nnc_graph_exec_symbol_new(group_norm_symbolic_graph, CMD_SET_FORWARD(1), 0, 0, TENSOR_SYMBOL_LIST(scale), "set_scale");
    290
    1
      ccv_nnc_graph_exec_symbol_new(group_norm_symbolic_graph, CMD_SET_FORWARD(0), 0, 0, TENSOR_SYMBOL_LIST(bias), "set_bias");
    291
    1
      ccv_nnc_graph_exec_symbol_new(group_norm_symbolic_graph, CMD_GROUP_NORM_FORWARD(1, 4, 0, 1), TENSOR_SYMBOL_LIST(bx, scale, bias), TENSOR_SYMBOL_LIST(by, saved_mean, saved_inv_std), "group_norm");
    292
    1
      ccv_nnc_graph_exec_symbol_autogen(group_norm_symbolic_graph, 0, 0, CCV_NNC_AUTOGEN_ALL_EXECS | CCV_NNC_AUTOGEN_SOURCES_AND_DESTINATIONS);
    293
    1
      ccv_nnc_symbolic_graph_backward(group_norm_symbolic_graph, TENSOR_SYMBOL_LIST(by), TENSOR_SYMBOL_LIST(bx, scale, bias), SYMBOLIC_GRAPH_SOURCES(group_norm_symbolic_graph), SYMBOLIC_GRAPH_DESTINATIONS(group_norm_symbolic_graph));
    294
    1
      ccv_nnc_graph_exec_symbol_autogen(group_norm_symbolic_graph, 0, 0, CCV_NNC_AUTOGEN_ALL_EXECS | CCV_NNC_AUTOGEN_SOURCES_AND_DESTINATIONS);
    295
    1
      ccv_nnc_tensor_symbol_t dby = ccv_nnc_tensor_symbol_for_backward(group_norm_symbolic_graph, by);
    296
    1
      ccv_nnc_tensor_symbol_t dbx = ccv_nnc_tensor_symbol_for_backward(group_norm_symbolic_graph, bx);
    297
    1
      ccv_nnc_graph_t* group_norm_graph = 0;
    298
    1
      ccv_nnc_tensor_arena_t* group_norm_tensor_arena = 0;
    299
    1
      ccv_nnc_graph_exec_arena_t* group_norm_graph_exec_arena = 0;
    300
    1
      ccv_nnc_symbolic_graph_compile(group_norm_symbolic_graph, ccv_nnc_default_compile_params, 0, 0, 0, 0, SYMBOLIC_GRAPH_SOURCES(group_norm_symbolic_graph), SYMBOLIC_GRAPH_DESTINATIONS(group_norm_symbolic_graph), &group_norm_graph, &group_norm_tensor_arena, &group_norm_graph_exec_arena);
    301
    1
      ccv_nnc_tensor_t* const bx_tensor = ccv_nnc_tensor_from_symbol(group_norm_tensor_arena, bx);
    302
    1
      ccv_nnc_tensor_t* const dy_tensor = ccv_nnc_tensor_from_symbol(tensor_arena, dy);
    303
    1
      ccv_nnc_tensor_t* const dby_tensor = ccv_nnc_tensor_from_symbol(group_norm_tensor_arena, dby);
    304
    641
      for (i = 0; i < 8 * 16 * 5; i++640)
    305
    640
        dby_tensor->data.f32[i] = dy_tensor->data.f32[i] = dsfmt_genrand_open_close(&dsfmt) * 2 - 1;
    306
    1
      memcpy(bx_tensor->data.f32, x_tensor->data.f32, sizeof(float) * 8 * 16 * 5);
    307
    1
      ccv_nnc_graph_run(graph, 0, TRAVERSE_FULL, 0, 0);
    308
    1
      ccv_nnc_graph_run(group_norm_graph, 0, TRAVERSE_FULL, 0, 0);
    309
    1
      ccv_nnc_tensor_t* const dx_tensor = ccv_nnc_tensor_from_symbol(tensor_arena, dx);
    310
    1
      ccv_nnc_tensor_t* const dbx_tensor = ccv_nnc_tensor_from_symbol(group_norm_tensor_arena, dbx);
    311
    1
      REQUIRE_ARRAY_EQ_WITH_TOLERANCE(float, dx_tensor->data.f32, dbx_tensor->data.f32, 8 * 16 * 5, 1e-5, "graph computed result should match layer norm op result");
    312
    1
      ccv_nnc_symbolic_graph_free(symbolic_graph);
    313
    1
      ccv_nnc_tensor_arena_free(tensor_arena);
    314
    1
      ccv_nnc_graph_exec_arena_free(graph_exec_arena);
    315
    1
      ccv_nnc_graph_free(graph);
    316
    1
      ccv_nnc_symbolic_graph_free(group_norm_symbolic_graph);
    317
    1
      ccv_nnc_tensor_arena_free(group_norm_tensor_arena);
    318
    1
      ccv_nnc_graph_exec_arena_free(group_norm_graph_exec_arena);
    319
    1
      ccv_nnc_graph_free(group_norm_graph);
    320
    1
    }
    321
    322
    TEST_CASE("implement group norm with other symbolic graph without scale / bias")
    323
    1
    {
    324
    1
      ccv_nnc_symbolic_graph_t* const symbolic_graph = ccv_nnc_symbolic_graph_new();
    325
    1
      ccv_nnc_tensor_symbol_t x = ccv_nnc_tensor_symbol_new(symbolic_graph, CPU_TENSOR_NHWC(32F, 8, 4, 4, 5), "x");
    326
    1
      ccv_nnc_tensor_symbol_t sum = ccv_nnc_tensor_symbol_new(symbolic_graph, CPU_TENSOR_NHWC(32F, 8, 4, 1, 5), "sum");
    327
    1
      ccv_nnc_tensor_symbol_t mean = ccv_nnc_tensor_symbol_new(symbolic_graph, CPU_TENSOR_NHWC(32F, 8, 4, 1, 5), "mean");
    328
    1
      ccv_nnc_tensor_symbol_t whitening = ccv_nnc_tensor_symbol_new(symbolic_graph, CPU_TENSOR_NHWC(32F, 8, 4, 4, 5), "whitening");
    329
    1
      ccv_nnc_tensor_symbol_t sqr = ccv_nnc_tensor_symbol_new(symbolic_graph, CPU_TENSOR_NHWC(32F, 8, 4, 4, 5), "sqr");
    330
    1
      ccv_nnc_tensor_symbol_t varsum = ccv_nnc_tensor_symbol_new(symbolic_graph, CPU_TENSOR_NHWC(32F, 8, 4, 1, 5), "varsum");
    331
    1
      ccv_nnc_tensor_symbol_t var = ccv_nnc_tensor_symbol_new(symbolic_graph, CPU_TENSOR_NHWC(32F, 8, 4, 1, 5), "var");
    332
    1
      ccv_nnc_tensor_symbol_t logvar = ccv_nnc_tensor_symbol_new(symbolic_graph, CPU_TENSOR_NHWC(32F, 8, 4, 1, 5), "logvar");
    333
    1
      ccv_nnc_tensor_symbol_t logvar_2 = ccv_nnc_tensor_symbol_new(symbolic_graph, CPU_TENSOR_NHWC(32F, 8, 4, 1, 5), "logvar");
    334
    1
      ccv_nnc_tensor_symbol_t std = ccv_nnc_tensor_symbol_new(symbolic_graph, CPU_TENSOR_NHWC(32F, 8, 4, 1, 5), "std");
    335
    1
      ccv_nnc_tensor_symbol_t inv_std = ccv_nnc_tensor_symbol_new(symbolic_graph, CPU_TENSOR_NHWC(32F, 8, 4, 1, 5), "inv_std");
    336
    1
      ccv_nnc_tensor_symbol_t y = ccv_nnc_tensor_symbol_new(symbolic_graph, CPU_TENSOR_NHWC(32F, 8, 4, 4, 5), "y");
    337
    1
      ccv_nnc_graph_exec_symbol_new(symbolic_graph, CMD_REDUCE_SUM_FORWARD(1), TENSOR_SYMBOL_LIST(x), TENSOR_SYMBOL_LIST(sum), "sum");
    338
    1
      ccv_nnc_graph_exec_symbol_new(symbolic_graph, CMD_SCALAR_MUL_FORWARD(1.0 / 4), TENSOR_SYMBOL_LIST(sum), TENSOR_SYMBOL_LIST(mean), "mean");
    339
    1
      ccv_nnc_graph_exec_symbol_new(symbolic_graph, CMD_ADD_FORWARD(1, -1), TENSOR_SYMBOL_LIST(x, mean), TENSOR_SYMBOL_LIST(whitening), "whitening");
    340
    1
      ccv_nnc_graph_exec_symbol_new(symbolic_graph, CMD_EWPROD_FORWARD(), TENSOR_SYMBOL_LIST(whitening, whitening), TENSOR_SYMBOL_LIST(sqr), "sqr");
    341
    1
      ccv_nnc_graph_exec_symbol_new(symbolic_graph, CMD_REDUCE_SUM_FORWARD(1), TENSOR_SYMBOL_LIST(sqr), TENSOR_SYMBOL_LIST(varsum), "varsum");
    342
    1
      ccv_nnc_graph_exec_symbol_new(symbolic_graph, CMD_SCALAR_MUL_FORWARD(1.0 / 4), TENSOR_SYMBOL_LIST(varsum), TENSOR_SYMBOL_LIST(var), "var");
    343
    1
      ccv_nnc_graph_exec_symbol_new(symbolic_graph, CMD_EWLOG_FORWARD(), TENSOR_SYMBOL_LIST(var), TENSOR_SYMBOL_LIST(logvar), "log(var)");
    344
    1
      ccv_nnc_graph_exec_symbol_new(symbolic_graph, CMD_SCALAR_MUL_FORWARD(0.5), TENSOR_SYMBOL_LIST(logvar), TENSOR_SYMBOL_LIST(logvar_2), "log(var)/2");
    345
    1
      ccv_nnc_graph_exec_symbol_new(symbolic_graph, CMD_EWEXP_FORWARD(), TENSOR_SYMBOL_LIST(logvar_2), TENSOR_SYMBOL_LIST(std), "std");
    346
    1
      ccv_nnc_graph_exec_symbol_new(symbolic_graph, CMD_EWDIV_FORWARD(), TENSOR_SYMBOL_LIST(NO_TENSOR_SYMBOL, std), TENSOR_SYMBOL_LIST(inv_std), "1/std");
    347
    1
      ccv_nnc_graph_exec_symbol_new(symbolic_graph, CMD_MUL_FORWARD(1), TENSOR_SYMBOL_LIST(whitening, inv_std), TENSOR_SYMBOL_LIST(y), "y");
    348
    1
      ccv_nnc_graph_exec_symbol_autogen(symbolic_graph, 0, 0, CCV_NNC_AUTOGEN_ALL_EXECS | CCV_NNC_AUTOGEN_SOURCES_AND_DESTINATIONS);
    349
    1
      SYMBOLIC_GRAPH_GEN(symbolic_graph, CCV_NNC_LONG_DOT_GRAPH);
    350
    1
      ccv_nnc_graph_t* graph = 0;
    351
    1
      ccv_nnc_tensor_arena_t* tensor_arena = 0;
    352
    1
      ccv_nnc_graph_exec_arena_t* graph_exec_arena = 0;
    353
    1
      ccv_nnc_symbolic_graph_compile(symbolic_graph, ccv_nnc_default_compile_params, 0, 0, 0, 0, SYMBOLIC_GRAPH_SOURCES(symbolic_graph), SYMBOLIC_GRAPH_DESTINATIONS(symbolic_graph), &graph, &tensor_arena, &graph_exec_arena);
    354
    1
      GRAPH_GEN(graph, CCV_NNC_LONG_DOT_GRAPH);
    355
    1
      ccv_nnc_tensor_t* const x_tensor = ccv_nnc_tensor_from_symbol(tensor_arena, x);
    356
    1
      dsfmt_t dsfmt;
    357
    1
      int i;
    358
    1
      dsfmt_init_gen_rand(&dsfmt, 1);
    359
    641
      for (i = 0; i < 8 * 4 * 4 * 5; i++640)
    360
    640
        x_tensor->data.f32[i] = dsfmt_genrand_open_close(&dsfmt);
    361
    1
      ccv_nnc_symbolic_graph_t* const group_norm_symbolic_graph = ccv_nnc_symbolic_graph_new();
    362
    1
      ccv_nnc_tensor_symbol_t bx = ccv_nnc_tensor_symbol_new(group_norm_symbolic_graph, CPU_TENSOR_NHWC(32F, 8, 16, 5), "x");
    363
    1
      ccv_nnc_tensor_symbol_t by = ccv_nnc_tensor_symbol_new(group_norm_symbolic_graph, CPU_TENSOR_NHWC(32F, 8, 16, 5), "y");
    364
    1
      ccv_nnc_tensor_symbol_t saved_mean = ccv_nnc_tensor_symbol_new(group_norm_symbolic_graph, CPU_TENSOR_NHWC(32F, 8, 4, 5), "saved_mean");
    365
    1
      ccv_nnc_tensor_symbol_t saved_inv_std = ccv_nnc_tensor_symbol_new(group_norm_symbolic_graph, CPU_TENSOR_NHWC(32F, 8, 4, 5), "saved_inv_std");
    366
    1
      ccv_nnc_graph_exec_symbol_new(group_norm_symbolic_graph, CMD_GROUP_NORM_FORWARD(1, 4, 0, 0), TENSOR_SYMBOL_LIST(bx), TENSOR_SYMBOL_LIST(by, saved_mean, saved_inv_std), "group_norm");
    367
    1
      ccv_nnc_graph_exec_symbol_autogen(group_norm_symbolic_graph, 0, 0, CCV_NNC_AUTOGEN_ALL_EXECS | CCV_NNC_AUTOGEN_SOURCES_AND_DESTINATIONS);
    368
    1
      ccv_nnc_graph_t* group_norm_graph = 0;
    369
    1
      ccv_nnc_tensor_arena_t* group_norm_tensor_arena = 0;
    370
    1
      ccv_nnc_graph_exec_arena_t* group_norm_graph_exec_arena = 0;
    371
    1
      ccv_nnc_symbolic_graph_compile(group_norm_symbolic_graph, ccv_nnc_default_compile_params, 0, 0, 0, 0, SYMBOLIC_GRAPH_SOURCES(group_norm_symbolic_graph), SYMBOLIC_GRAPH_DESTINATIONS(group_norm_symbolic_graph), &group_norm_graph, &group_norm_tensor_arena, &group_norm_graph_exec_arena);
    372
    1
      ccv_nnc_tensor_t* const bx_tensor = ccv_nnc_tensor_from_symbol(group_norm_tensor_arena, bx);
    373
    1
      memcpy(bx_tensor->data.f32, x_tensor->data.f32, sizeof(float) * 8 * 16 * 5);
    374
    1
      ccv_nnc_graph_run(graph, 0, TRAVERSE_FULL, 0, 0);
    375
    1
      ccv_nnc_graph_run(group_norm_graph, 0, TRAVERSE_FULL, 0, 0);
    376
    1
      ccv_nnc_tensor_t* const y_tensor = ccv_nnc_tensor_from_symbol(tensor_arena, y);
    377
    1
      ccv_nnc_tensor_t* const by_tensor = ccv_nnc_tensor_from_symbol(group_norm_tensor_arena, by);
    378
    1
      REQUIRE_ARRAY_EQ_WITH_TOLERANCE(float, y_tensor->data.f32, by_tensor->data.f32, 8 * 16 * 5, 1e-5, "graph computed result should match batch norm op result");
    379
    1
      ccv_nnc_symbolic_graph_free(symbolic_graph);
    380
    1
      ccv_nnc_tensor_arena_free(tensor_arena);
    381
    1
      ccv_nnc_graph_exec_arena_free(graph_exec_arena);
    382
    1
      ccv_nnc_graph_free(graph);
    383
    1
      ccv_nnc_symbolic_graph_free(group_norm_symbolic_graph);
    384
    1
      ccv_nnc_tensor_arena_free(group_norm_tensor_arena);
    385
    1
      ccv_nnc_graph_exec_arena_free(group_norm_graph_exec_arena);
    386
    1
      ccv_nnc_graph_free(group_norm_graph);
    387
    1
    }
    388
    389
    TEST_CASE("compare group norm gradient with other symbolic graph without scale / bias")
    390
    1
    {
    391
    1
      ccv_nnc_symbolic_graph_t* const symbolic_graph = ccv_nnc_symbolic_graph_new();
    392
    1
      ccv_nnc_tensor_symbol_t x = ccv_nnc_tensor_symbol_new(symbolic_graph, CPU_TENSOR_NHWC(32F, 8, 4, 4, 5), "x");
    393
    1
      ccv_nnc_tensor_symbol_t sum = ccv_nnc_tensor_symbol_new(symbolic_graph, CPU_TENSOR_NHWC(32F, 8, 4, 1, 5), "sum");
    394
    1
      ccv_nnc_tensor_symbol_t mean = ccv_nnc_tensor_symbol_new(symbolic_graph, CPU_TENSOR_NHWC(32F, 8, 4, 1, 5), "mean");
    395
    1
      ccv_nnc_tensor_symbol_t whitening = ccv_nnc_tensor_symbol_new(symbolic_graph, CPU_TENSOR_NHWC(32F, 8, 4, 4, 5), "whitening");
    396
    1
      ccv_nnc_tensor_symbol_t sqr = ccv_nnc_tensor_symbol_new(symbolic_graph, CPU_TENSOR_NHWC(32F, 8, 4, 4, 5), "sqr");
    397
    1
      ccv_nnc_tensor_symbol_t varsum = ccv_nnc_tensor_symbol_new(symbolic_graph, CPU_TENSOR_NHWC(32F, 8, 4, 1, 5), "varsum");
    398
    1
      ccv_nnc_tensor_symbol_t var = ccv_nnc_tensor_symbol_new(symbolic_graph, CPU_TENSOR_NHWC(32F, 8, 4, 1, 5), "var");
    399
    1
      ccv_nnc_tensor_symbol_t logvar = ccv_nnc_tensor_symbol_new(symbolic_graph, CPU_TENSOR_NHWC(32F, 8, 4, 1, 5), "logvar");
    400
    1
      ccv_nnc_tensor_symbol_t logvar_2 = ccv_nnc_tensor_symbol_new(symbolic_graph, CPU_TENSOR_NHWC(32F, 8, 4, 1, 5), "logvar");
    401
    1
      ccv_nnc_tensor_symbol_t std = ccv_nnc_tensor_symbol_new(symbolic_graph, CPU_TENSOR_NHWC(32F, 8, 4, 1, 5), "std");
    402
    1
      ccv_nnc_tensor_symbol_t inv_std = ccv_nnc_tensor_symbol_new(symbolic_graph, CPU_TENSOR_NHWC(32F, 8, 4, 1, 5), "inv_std");
    403
    1
      ccv_nnc_tensor_symbol_t y = ccv_nnc_tensor_symbol_new(symbolic_graph, CPU_TENSOR_NHWC(32F, 8, 4, 4, 5), "y");
    404
    1
      ccv_nnc_graph_exec_symbol_new(symbolic_graph, CMD_REDUCE_SUM_FORWARD(1), TENSOR_SYMBOL_LIST(x), TENSOR_SYMBOL_LIST(sum), "sum");
    405
    1
      ccv_nnc_graph_exec_symbol_new(symbolic_graph, CMD_SCALAR_MUL_FORWARD(1.0 / 4), TENSOR_SYMBOL_LIST(sum), TENSOR_SYMBOL_LIST(mean), "mean");
    406
    1
      ccv_nnc_graph_exec_symbol_new(symbolic_graph, CMD_ADD_FORWARD(1, -1), TENSOR_SYMBOL_LIST(x, mean), TENSOR_SYMBOL_LIST(whitening), "whitening");
    407
    1
      ccv_nnc_graph_exec_symbol_new(symbolic_graph, CMD_EWPROD_FORWARD(), TENSOR_SYMBOL_LIST(whitening, whitening), TENSOR_SYMBOL_LIST(sqr), "sqr");
    408
    1
      ccv_nnc_graph_exec_symbol_new(symbolic_graph, CMD_REDUCE_SUM_FORWARD(1), TENSOR_SYMBOL_LIST(sqr), TENSOR_SYMBOL_LIST(varsum), "varsum");
    409
    1
      ccv_nnc_graph_exec_symbol_new(symbolic_graph, CMD_SCALAR_MUL_FORWARD(1.0 / 4), TENSOR_SYMBOL_LIST(varsum), TENSOR_SYMBOL_LIST(var), "var");
    410
    1
      ccv_nnc_graph_exec_symbol_new(symbolic_graph, CMD_EWLOG_FORWARD(), TENSOR_SYMBOL_LIST(var), TENSOR_SYMBOL_LIST(logvar), "log(var)");
    411
    1
      ccv_nnc_graph_exec_symbol_new(symbolic_graph, CMD_SCALAR_MUL_FORWARD(0.5), TENSOR_SYMBOL_LIST(logvar), TENSOR_SYMBOL_LIST(logvar_2), "log(var)/2");
    412
    1
      ccv_nnc_graph_exec_symbol_new(symbolic_graph, CMD_EWEXP_FORWARD(), TENSOR_SYMBOL_LIST(logvar_2), TENSOR_SYMBOL_LIST(std), "std");
    413
    1
      ccv_nnc_graph_exec_symbol_new(symbolic_graph, CMD_EWDIV_FORWARD(), TENSOR_SYMBOL_LIST(NO_TENSOR_SYMBOL, std), TENSOR_SYMBOL_LIST(inv_std), "1/std");
    414
    1
      ccv_nnc_graph_exec_symbol_new(symbolic_graph, CMD_MUL_FORWARD(1), TENSOR_SYMBOL_LIST(whitening, inv_std), TENSOR_SYMBOL_LIST(y), "y");
    415
    1
      ccv_nnc_graph_exec_symbol_autogen(symbolic_graph, 0, 0, CCV_NNC_AUTOGEN_ALL_EXECS | CCV_NNC_AUTOGEN_SOURCES_AND_DESTINATIONS);
    416
    1
      ccv_nnc_symbolic_graph_backward(symbolic_graph, TENSOR_SYMBOL_LIST(y), TENSOR_SYMBOL_LIST(x), SYMBOLIC_GRAPH_SOURCES(symbolic_graph), SYMBOLIC_GRAPH_DESTINATIONS(symbolic_graph));
    417
    1
      ccv_nnc_graph_exec_symbol_autogen(symbolic_graph, 0, 0, CCV_NNC_AUTOGEN_ALL_EXECS | CCV_NNC_AUTOGEN_SOURCES_AND_DESTINATIONS);
    418
    1
      ccv_nnc_tensor_symbol_t dy = ccv_nnc_tensor_symbol_for_backward(symbolic_graph, y);
    419
    1
      ccv_nnc_tensor_symbol_t dx = ccv_nnc_tensor_symbol_for_backward(symbolic_graph, x);
    420
    1
      SYMBOLIC_GRAPH_GEN(symbolic_graph, CCV_NNC_LONG_DOT_GRAPH);
    421
    1
      ccv_nnc_graph_t* graph = 0;
    422
    1
      ccv_nnc_tensor_arena_t* tensor_arena = 0;
    423
    1
      ccv_nnc_graph_exec_arena_t* graph_exec_arena = 0;
    424
    1
      ccv_nnc_symbolic_graph_compile(symbolic_graph, ccv_nnc_default_compile_params, 0, 0, 0, 0, SYMBOLIC_GRAPH_SOURCES(symbolic_graph), SYMBOLIC_GRAPH_DESTINATIONS(symbolic_graph), &graph, &tensor_arena, &graph_exec_arena);
    425
    1
      GRAPH_GEN(graph, CCV_NNC_LONG_DOT_GRAPH);
    426
    1
      ccv_nnc_tensor_t* const x_tensor = ccv_nnc_tensor_from_symbol(tensor_arena, x);
    427
    1
      dsfmt_t dsfmt;
    428
    1
      int i;
    429
    1
      dsfmt_init_gen_rand(&dsfmt, 1);
    430
    641
      for (i = 0; i < 8 * 4 * 4 * 5; i++640)
    431
    640
        x_tensor->data.f32[i] = dsfmt_genrand_open_close(&dsfmt);
    432
    1
      ccv_nnc_symbolic_graph_t* const group_norm_symbolic_graph = ccv_nnc_symbolic_graph_new();
    433
    1
      ccv_nnc_tensor_symbol_t bx = ccv_nnc_tensor_symbol_new(group_norm_symbolic_graph, CPU_TENSOR_NHWC(32F, 8, 16, 5), "x");
    434
    1
      ccv_nnc_tensor_symbol_t by = ccv_nnc_tensor_symbol_new(group_norm_symbolic_graph, CPU_TENSOR_NHWC(32F, 8, 16, 5), "y");
    435
    1
      ccv_nnc_tensor_symbol_t saved_mean = ccv_nnc_tensor_symbol_new(group_norm_symbolic_graph, CPU_TENSOR_NHWC(32F, 8, 4, 5), "saved_mean");
    436
    1
      ccv_nnc_tensor_symbol_t saved_inv_std = ccv_nnc_tensor_symbol_new(group_norm_symbolic_graph, CPU_TENSOR_NHWC(32F, 8, 4, 5), "saved_inv_std");
    437
    1
      ccv_nnc_graph_exec_symbol_new(group_norm_symbolic_graph, CMD_GROUP_NORM_FORWARD(1, 4, 0, 0), TENSOR_SYMBOL_LIST(bx), TENSOR_SYMBOL_LIST(by, saved_mean, saved_inv_std), "group_norm");
    438
    1
      ccv_nnc_graph_exec_symbol_autogen(group_norm_symbolic_graph, 0, 0, CCV_NNC_AUTOGEN_ALL_EXECS | CCV_NNC_AUTOGEN_SOURCES_AND_DESTINATIONS);
    439
    1
      ccv_nnc_symbolic_graph_backward(group_norm_symbolic_graph, TENSOR_SYMBOL_LIST(by), TENSOR_SYMBOL_LIST(bx), SYMBOLIC_GRAPH_SOURCES(group_norm_symbolic_graph), SYMBOLIC_GRAPH_DESTINATIONS(group_norm_symbolic_graph));
    440
    1
      ccv_nnc_graph_exec_symbol_autogen(group_norm_symbolic_graph, 0, 0, CCV_NNC_AUTOGEN_ALL_EXECS | CCV_NNC_AUTOGEN_SOURCES_AND_DESTINATIONS);
    441
    1
      ccv_nnc_tensor_symbol_t dby = ccv_nnc_tensor_symbol_for_backward(group_norm_symbolic_graph, by);
    442
    1
      ccv_nnc_tensor_symbol_t dbx = ccv_nnc_tensor_symbol_for_backward(group_norm_symbolic_graph, bx);
    443
    1
      ccv_nnc_graph_t* group_norm_graph = 0;
    444
    1
      ccv_nnc_tensor_arena_t* group_norm_tensor_arena = 0;
    445
    1
      ccv_nnc_graph_exec_arena_t* group_norm_graph_exec_arena = 0;
    446
    1
      ccv_nnc_symbolic_graph_compile(group_norm_symbolic_graph, ccv_nnc_default_compile_params, 0, 0, 0, 0, SYMBOLIC_GRAPH_SOURCES(group_norm_symbolic_graph), SYMBOLIC_GRAPH_DESTINATIONS(group_norm_symbolic_graph), &group_norm_graph, &group_norm_tensor_arena, &group_norm_graph_exec_arena);
    447
    1
      ccv_nnc_tensor_t* const bx_tensor = ccv_nnc_tensor_from_symbol(group_norm_tensor_arena, bx);
    448
    1
      ccv_nnc_tensor_t* const dy_tensor = ccv_nnc_tensor_from_symbol(tensor_arena, dy);
    449
    1
      ccv_nnc_tensor_t* const dby_tensor = ccv_nnc_tensor_from_symbol(group_norm_tensor_arena, dby);
    450
    641
      for (i = 0; i < 8 * 16 * 5; i++640)
    451
    640
        dby_tensor->data.f32[i] = dy_tensor->data.f32[i] = dsfmt_genrand_open_close(&dsfmt) * 2 - 1;
    452
    1
      memcpy(bx_tensor->data.f32, x_tensor->data.f32, sizeof(float) * 8 * 16 * 5);
    453
    1
      ccv_nnc_graph_run(graph, 0, TRAVERSE_FULL, 0, 0);
    454
    1
      ccv_nnc_graph_run(group_norm_graph, 0, TRAVERSE_FULL, 0, 0);
    455
    1
      ccv_nnc_tensor_t* const dx_tensor = ccv_nnc_tensor_from_symbol(tensor_arena, dx);
    456
    1
      ccv_nnc_tensor_t* const dbx_tensor = ccv_nnc_tensor_from_symbol(group_norm_tensor_arena, dbx);
    457
    1
      REQUIRE_ARRAY_EQ_WITH_TOLERANCE(float, dx_tensor->data.f32, dbx_tensor->data.f32, 8 * 16 * 5, 1e-5, "graph computed result should match layer norm op result");
    458
    1
      ccv_nnc_symbolic_graph_free(symbolic_graph);
    459
    1
      ccv_nnc_tensor_arena_free(tensor_arena);
    460
    1
      ccv_nnc_graph_exec_arena_free(graph_exec_arena);
    461
    1
      ccv_nnc_graph_free(graph);
    462
    1
      ccv_nnc_symbolic_graph_free(group_norm_symbolic_graph);
    463
    1
      ccv_nnc_tensor_arena_free(group_norm_tensor_arena);
    464
    1
      ccv_nnc_graph_exec_arena_free(group_norm_graph_exec_arena);
    465
    1
      ccv_nnc_graph_free(group_norm_graph);
    466
    1
    }
    467
    468
    #include "case_main.h"