Coverage Report

Created: 2025-02-24 17:43

next uncovered line (L), next uncovered region (R), next uncovered branch (B)
/home/liu/actions-runner/_work/ccv/ccv/test/unit/nnc/group.norm.tests.c
Line
Count
Source
1
#include "case.h"
2
#include "ccv_case.h"
3
#include "ccv_nnc_case.h"
4
#include <ccv.h>
5
#include <nnc/ccv_nnc.h>
6
#include <nnc/ccv_nnc_easy.h>
7
#include "3rdparty/dsfmt/dSFMT.h"
8
9
TEST_SETUP()
10
{
11
  ccv_nnc_init();
12
}
13
14
TEST_CASE("implement simple group norm (like layer norm) with other symbolic graph")
15
1
{
16
1
  ccv_nnc_symbolic_graph_t* const symbolic_graph = ccv_nnc_symbolic_graph_new();
17
1
  ccv_nnc_tensor_symbol_t x = ccv_nnc_tensor_symbol_new(symbolic_graph, CPU_TENSOR_NHWC(32F, 8, 4, 4, 10), "x");
18
1
  ccv_nnc_tensor_symbol_t sum = ccv_nnc_tensor_symbol_new(symbolic_graph, CPU_TENSOR_NHWC(32F, 8, 4, 4, 1), "sum");
19
1
  ccv_nnc_tensor_symbol_t mean = ccv_nnc_tensor_symbol_new(symbolic_graph, CPU_TENSOR_NHWC(32F, 8, 4, 4, 1), "mean");
20
1
  ccv_nnc_tensor_symbol_t whitening = ccv_nnc_tensor_symbol_new(symbolic_graph, CPU_TENSOR_NHWC(32F, 8, 4, 4, 10), "whitening");
21
1
  ccv_nnc_tensor_symbol_t sqr = ccv_nnc_tensor_symbol_new(symbolic_graph, CPU_TENSOR_NHWC(32F, 8, 4, 4, 10), "sqr");
22
1
  ccv_nnc_tensor_symbol_t varsum = ccv_nnc_tensor_symbol_new(symbolic_graph, CPU_TENSOR_NHWC(32F, 8, 4, 4, 1), "varsum");
23
1
  ccv_nnc_tensor_symbol_t var = ccv_nnc_tensor_symbol_new(symbolic_graph, CPU_TENSOR_NHWC(32F, 8, 4, 4, 1), "var");
24
1
  ccv_nnc_tensor_symbol_t logvar = ccv_nnc_tensor_symbol_new(symbolic_graph, CPU_TENSOR_NHWC(32F, 8, 4, 4, 1), "logvar");
25
1
  ccv_nnc_tensor_symbol_t logvar_2 = ccv_nnc_tensor_symbol_new(symbolic_graph, CPU_TENSOR_NHWC(32F, 8, 4, 4, 1), "logvar");
26
1
  ccv_nnc_tensor_symbol_t std = ccv_nnc_tensor_symbol_new(symbolic_graph, CPU_TENSOR_NHWC(32F, 8, 4, 4, 1), "std");
27
1
  ccv_nnc_tensor_symbol_t inv_std = ccv_nnc_tensor_symbol_new(symbolic_graph, CPU_TENSOR_NHWC(32F, 8, 4, 4, 1), "inv_std");
28
1
  ccv_nnc_tensor_symbol_t y = ccv_nnc_tensor_symbol_new(symbolic_graph, CPU_TENSOR_NHWC(32F, 8, 4, 4, 10), "y");
29
1
  ccv_nnc_graph_exec_symbol_new(symbolic_graph, CMD_REDUCE_SUM_FORWARD(3), TENSOR_SYMBOL_LIST(x), TENSOR_SYMBOL_LIST(sum), "sum");
30
1
  ccv_nnc_graph_exec_symbol_new(symbolic_graph, CMD_SCALAR_MUL_FORWARD(1.0 / 10), TENSOR_SYMBOL_LIST(sum), TENSOR_SYMBOL_LIST(mean), "mean");
31
1
  ccv_nnc_graph_exec_symbol_new(symbolic_graph, CMD_ADD_FORWARD(1, -1), TENSOR_SYMBOL_LIST(x, mean), TENSOR_SYMBOL_LIST(whitening), "whitening");
32
1
  ccv_nnc_graph_exec_symbol_new(symbolic_graph, CMD_EWPROD_FORWARD(), TENSOR_SYMBOL_LIST(whitening, whitening), TENSOR_SYMBOL_LIST(sqr), "sqr");
33
1
  ccv_nnc_graph_exec_symbol_new(symbolic_graph, CMD_REDUCE_SUM_FORWARD(3), TENSOR_SYMBOL_LIST(sqr), TENSOR_SYMBOL_LIST(varsum), "varsum");
34
1
  ccv_nnc_graph_exec_symbol_new(symbolic_graph, CMD_SCALAR_MUL_FORWARD(1.0 / 10), TENSOR_SYMBOL_LIST(varsum), TENSOR_SYMBOL_LIST(var), "var");
35
1
  ccv_nnc_graph_exec_symbol_new(symbolic_graph, CMD_EWLOG_FORWARD(), TENSOR_SYMBOL_LIST(var), TENSOR_SYMBOL_LIST(logvar), "log(var)");
36
1
  ccv_nnc_graph_exec_symbol_new(symbolic_graph, CMD_SCALAR_MUL_FORWARD(0.5), TENSOR_SYMBOL_LIST(logvar), TENSOR_SYMBOL_LIST(logvar_2), "log(var)/2");
37
1
  ccv_nnc_graph_exec_symbol_new(symbolic_graph, CMD_EWEXP_FORWARD(), TENSOR_SYMBOL_LIST(logvar_2), TENSOR_SYMBOL_LIST(std), "std");
38
1
  ccv_nnc_graph_exec_symbol_new(symbolic_graph, CMD_EWDIV_FORWARD(), TENSOR_SYMBOL_LIST(NO_TENSOR_SYMBOL, std), TENSOR_SYMBOL_LIST(inv_std), "1/std");
39
1
  ccv_nnc_graph_exec_symbol_new(symbolic_graph, CMD_MUL_FORWARD(1), TENSOR_SYMBOL_LIST(whitening, inv_std), TENSOR_SYMBOL_LIST(y), "y");
40
1
  ccv_nnc_graph_exec_symbol_autogen(symbolic_graph, 0, 0, CCV_NNC_AUTOGEN_ALL_EXECS | CCV_NNC_AUTOGEN_SOURCES_AND_DESTINATIONS);
41
1
  SYMBOLIC_GRAPH_GEN(symbolic_graph, CCV_NNC_LONG_DOT_GRAPH);
42
1
  ccv_nnc_graph_t* graph = 0;
43
1
  ccv_nnc_tensor_arena_t* tensor_arena = 0;
44
1
  ccv_nnc_graph_exec_arena_t* graph_exec_arena = 0;
45
1
  ccv_nnc_symbolic_graph_compile(symbolic_graph, ccv_nnc_default_compile_params, 0, 0, 0, 0, SYMBOLIC_GRAPH_SOURCES(symbolic_graph), SYMBOLIC_GRAPH_DESTINATIONS(symbolic_graph), &graph, &tensor_arena, &graph_exec_arena);
46
1
  GRAPH_GEN(graph, CCV_NNC_LONG_DOT_GRAPH);
47
1
  ccv_nnc_tensor_t* const x_tensor = ccv_nnc_tensor_from_symbol(tensor_arena, x);
48
1
  dsfmt_t dsfmt;
49
1
  int i;
50
1
  dsfmt_init_gen_rand(&dsfmt, 1);
51
1.28k
  for (i = 0; i < 8 * 4 * 4 * 10; 
i++1.28k
)
52
1.28k
    x_tensor->data.f32[i] = dsfmt_genrand_open_close(&dsfmt);
53
1
  ccv_nnc_symbolic_graph_t* const group_norm_symbolic_graph = ccv_nnc_symbolic_graph_new();
54
1
  ccv_nnc_tensor_symbol_t bx = ccv_nnc_tensor_symbol_new(group_norm_symbolic_graph, CPU_TENSOR_NHWC(32F, 8, 4, 4, 10), "x");
55
1
  ccv_nnc_tensor_symbol_t by = ccv_nnc_tensor_symbol_new(group_norm_symbolic_graph, CPU_TENSOR_NHWC(32F, 8, 4, 4, 10), "y");
56
1
  ccv_nnc_tensor_symbol_t scale = ccv_nnc_tensor_symbol_new(group_norm_symbolic_graph, CPU_TENSOR_NHWC(32F, 1, 4, 4, 10), "scale");
57
1
  ccv_nnc_tensor_symbol_t bias = ccv_nnc_tensor_symbol_new(group_norm_symbolic_graph, CPU_TENSOR_NHWC(32F, 1, 4, 4, 10), "bias");
58
1
  ccv_nnc_tensor_symbol_t saved_mean = ccv_nnc_tensor_symbol_new(group_norm_symbolic_graph, CPU_TENSOR_NHWC(32F, 8, 4, 4, 1), "saved_mean");
59
1
  ccv_nnc_tensor_symbol_t saved_inv_std = ccv_nnc_tensor_symbol_new(group_norm_symbolic_graph, CPU_TENSOR_NHWC(32F, 8, 4, 4, 1), "saved_inv_std");
60
1
  ccv_nnc_graph_exec_symbol_new(group_norm_symbolic_graph, CMD_SET_FORWARD(1), 0, 0, TENSOR_SYMBOL_LIST(scale), "set_scale");
61
1
  ccv_nnc_graph_exec_symbol_new(group_norm_symbolic_graph, CMD_SET_FORWARD(0), 0, 0, TENSOR_SYMBOL_LIST(bias), "set_bias");
62
1
  ccv_nnc_graph_exec_symbol_new(group_norm_symbolic_graph, CMD_GROUP_NORM_FORWARD(3, 1, 0, 1), TENSOR_SYMBOL_LIST(bx, scale, bias), TENSOR_SYMBOL_LIST(by, saved_mean, saved_inv_std), "group_norm");
63
1
  ccv_nnc_graph_exec_symbol_autogen(group_norm_symbolic_graph, 0, 0, CCV_NNC_AUTOGEN_ALL_EXECS | CCV_NNC_AUTOGEN_SOURCES_AND_DESTINATIONS);
64
1
  ccv_nnc_graph_t* group_norm_graph = 0;
65
1
  ccv_nnc_tensor_arena_t* group_norm_tensor_arena = 0;
66
1
  ccv_nnc_graph_exec_arena_t* group_norm_graph_exec_arena = 0;
67
1
  ccv_nnc_symbolic_graph_compile(group_norm_symbolic_graph, ccv_nnc_default_compile_params, 0, 0, 0, 0, SYMBOLIC_GRAPH_SOURCES(group_norm_symbolic_graph), SYMBOLIC_GRAPH_DESTINATIONS(group_norm_symbolic_graph), &group_norm_graph, &group_norm_tensor_arena, &group_norm_graph_exec_arena);
68
1
  ccv_nnc_tensor_t* const bx_tensor = ccv_nnc_tensor_from_symbol(group_norm_tensor_arena, bx);
69
1
  memcpy(bx_tensor->data.f32, x_tensor->data.f32, sizeof(float) * 8 * 4 * 4 * 10);
70
1
  ccv_nnc_graph_run(graph, 0, TRAVERSE_FULL, 0, 0);
71
1
  ccv_nnc_graph_run(group_norm_graph, 0, TRAVERSE_FULL, 0, 0);
72
1
  ccv_nnc_tensor_t* const y_tensor = ccv_nnc_tensor_from_symbol(tensor_arena, y);
73
1
  ccv_nnc_tensor_t* const by_tensor = ccv_nnc_tensor_from_symbol(group_norm_tensor_arena, by);
74
1
  REQUIRE_TENSOR_EQ(y_tensor, by_tensor, "graph computed result should match batch norm op result");
75
1
  ccv_nnc_symbolic_graph_free(symbolic_graph);
76
1
  ccv_nnc_tensor_arena_free(tensor_arena);
77
1
  ccv_nnc_graph_exec_arena_free(graph_exec_arena);
78
1
  ccv_nnc_graph_free(graph);
79
1
  ccv_nnc_symbolic_graph_free(group_norm_symbolic_graph);
80
1
  ccv_nnc_tensor_arena_free(group_norm_tensor_arena);
81
1
  ccv_nnc_graph_exec_arena_free(group_norm_graph_exec_arena);
82
1
  ccv_nnc_graph_free(group_norm_graph);
83
1
}
84
85
TEST_CASE("compare simple group norm (like layer norm) gradient with other symbolic graph")
86
1
{
87
1
  ccv_nnc_symbolic_graph_t* const symbolic_graph = ccv_nnc_symbolic_graph_new();
88
1
  ccv_nnc_tensor_symbol_t x = ccv_nnc_tensor_symbol_new(symbolic_graph, CPU_TENSOR_NHWC(32F, 8, 4, 4, 10), "x");
89
1
  ccv_nnc_tensor_symbol_t sum = ccv_nnc_tensor_symbol_new(symbolic_graph, CPU_TENSOR_NHWC(32F, 8, 4, 4, 1), "sum");
90
1
  ccv_nnc_tensor_symbol_t mean = ccv_nnc_tensor_symbol_new(symbolic_graph, CPU_TENSOR_NHWC(32F, 8, 4, 4, 1), "mean");
91
1
  ccv_nnc_tensor_symbol_t whitening = ccv_nnc_tensor_symbol_new(symbolic_graph, CPU_TENSOR_NHWC(32F, 8, 4, 4, 10), "whitening");
92
1
  ccv_nnc_tensor_symbol_t sqr = ccv_nnc_tensor_symbol_new(symbolic_graph, CPU_TENSOR_NHWC(32F, 8, 4, 4, 10), "sqr");
93
1
  ccv_nnc_tensor_symbol_t varsum = ccv_nnc_tensor_symbol_new(symbolic_graph, CPU_TENSOR_NHWC(32F, 8, 4, 4, 1), "varsum");
94
1
  ccv_nnc_tensor_symbol_t var = ccv_nnc_tensor_symbol_new(symbolic_graph, CPU_TENSOR_NHWC(32F, 8, 4, 4, 1), "var");
95
1
  ccv_nnc_tensor_symbol_t logvar = ccv_nnc_tensor_symbol_new(symbolic_graph, CPU_TENSOR_NHWC(32F, 8, 4, 4, 1), "logvar");
96
1
  ccv_nnc_tensor_symbol_t logvar_2 = ccv_nnc_tensor_symbol_new(symbolic_graph, CPU_TENSOR_NHWC(32F, 8, 4, 4, 1), "logvar");
97
1
  ccv_nnc_tensor_symbol_t std = ccv_nnc_tensor_symbol_new(symbolic_graph, CPU_TENSOR_NHWC(32F, 8, 4, 4, 1), "std");
98
1
  ccv_nnc_tensor_symbol_t inv_std = ccv_nnc_tensor_symbol_new(symbolic_graph, CPU_TENSOR_NHWC(32F, 8, 4, 4, 1), "inv_std");
99
1
  ccv_nnc_tensor_symbol_t y = ccv_nnc_tensor_symbol_new(symbolic_graph, CPU_TENSOR_NHWC(32F, 8, 4, 4, 10), "y");
100
1
  ccv_nnc_graph_exec_symbol_new(symbolic_graph, CMD_REDUCE_SUM_FORWARD(3), TENSOR_SYMBOL_LIST(x), TENSOR_SYMBOL_LIST(sum), "sum");
101
1
  ccv_nnc_graph_exec_symbol_new(symbolic_graph, CMD_SCALAR_MUL_FORWARD(1.0 / 10), TENSOR_SYMBOL_LIST(sum), TENSOR_SYMBOL_LIST(mean), "mean");
102
1
  ccv_nnc_graph_exec_symbol_new(symbolic_graph, CMD_ADD_FORWARD(1, -1), TENSOR_SYMBOL_LIST(x, mean), TENSOR_SYMBOL_LIST(whitening), "whitening");
103
1
  ccv_nnc_graph_exec_symbol_new(symbolic_graph, CMD_EWPROD_FORWARD(), TENSOR_SYMBOL_LIST(whitening, whitening), TENSOR_SYMBOL_LIST(sqr), "sqr");
104
1
  ccv_nnc_graph_exec_symbol_new(symbolic_graph, CMD_REDUCE_SUM_FORWARD(3), TENSOR_SYMBOL_LIST(sqr), TENSOR_SYMBOL_LIST(varsum), "varsum");
105
1
  ccv_nnc_graph_exec_symbol_new(symbolic_graph, CMD_SCALAR_MUL_FORWARD(1.0 / 10), TENSOR_SYMBOL_LIST(varsum), TENSOR_SYMBOL_LIST(var), "var");
106
1
  ccv_nnc_graph_exec_symbol_new(symbolic_graph, CMD_EWLOG_FORWARD(), TENSOR_SYMBOL_LIST(var), TENSOR_SYMBOL_LIST(logvar), "log(var)");
107
1
  ccv_nnc_graph_exec_symbol_new(symbolic_graph, CMD_SCALAR_MUL_FORWARD(0.5), TENSOR_SYMBOL_LIST(logvar), TENSOR_SYMBOL_LIST(logvar_2), "log(var)/2");
108
1
  ccv_nnc_graph_exec_symbol_new(symbolic_graph, CMD_EWEXP_FORWARD(), TENSOR_SYMBOL_LIST(logvar_2), TENSOR_SYMBOL_LIST(std), "std");
109
1
  ccv_nnc_graph_exec_symbol_new(symbolic_graph, CMD_EWDIV_FORWARD(), TENSOR_SYMBOL_LIST(NO_TENSOR_SYMBOL, std), TENSOR_SYMBOL_LIST(inv_std), "1/std");
110
1
  ccv_nnc_graph_exec_symbol_new(symbolic_graph, CMD_MUL_FORWARD(1), TENSOR_SYMBOL_LIST(whitening, inv_std), TENSOR_SYMBOL_LIST(y), "y");
111
1
  ccv_nnc_graph_exec_symbol_autogen(symbolic_graph, 0, 0, CCV_NNC_AUTOGEN_ALL_EXECS | CCV_NNC_AUTOGEN_SOURCES_AND_DESTINATIONS);
112
1
  ccv_nnc_symbolic_graph_backward(symbolic_graph, TENSOR_SYMBOL_LIST(y), TENSOR_SYMBOL_LIST(x), SYMBOLIC_GRAPH_SOURCES(symbolic_graph), SYMBOLIC_GRAPH_DESTINATIONS(symbolic_graph));
113
1
  ccv_nnc_graph_exec_symbol_autogen(symbolic_graph, 0, 0, CCV_NNC_AUTOGEN_ALL_EXECS | CCV_NNC_AUTOGEN_SOURCES_AND_DESTINATIONS);
114
1
  ccv_nnc_tensor_symbol_t dy = ccv_nnc_tensor_symbol_for_backward(symbolic_graph, y);
115
1
  ccv_nnc_tensor_symbol_t dx = ccv_nnc_tensor_symbol_for_backward(symbolic_graph, x);
116
1
  SYMBOLIC_GRAPH_GEN(symbolic_graph, CCV_NNC_LONG_DOT_GRAPH);
117
1
  ccv_nnc_graph_t* graph = 0;
118
1
  ccv_nnc_tensor_arena_t* tensor_arena = 0;
119
1
  ccv_nnc_graph_exec_arena_t* graph_exec_arena = 0;
120
1
  ccv_nnc_symbolic_graph_compile(symbolic_graph, ccv_nnc_default_compile_params, 0, 0, 0, 0, SYMBOLIC_GRAPH_SOURCES(symbolic_graph), SYMBOLIC_GRAPH_DESTINATIONS(symbolic_graph), &graph, &tensor_arena, &graph_exec_arena);
121
1
  GRAPH_GEN(graph, CCV_NNC_LONG_DOT_GRAPH);
122
1
  ccv_nnc_tensor_t* const x_tensor = ccv_nnc_tensor_from_symbol(tensor_arena, x);
123
1
  dsfmt_t dsfmt;
124
1
  int i;
125
1
  dsfmt_init_gen_rand(&dsfmt, 1);
126
1.28k
  for (i = 0; i < 8 * 4 * 4 * 10; 
i++1.28k
)
127
1.28k
    x_tensor->data.f32[i] = dsfmt_genrand_open_close(&dsfmt);
128
1
  ccv_nnc_symbolic_graph_t* const group_norm_symbolic_graph = ccv_nnc_symbolic_graph_new();
129
1
  ccv_nnc_tensor_symbol_t bx = ccv_nnc_tensor_symbol_new(group_norm_symbolic_graph, CPU_TENSOR_NHWC(32F, 8, 4, 4, 10), "x");
130
1
  ccv_nnc_tensor_symbol_t by = ccv_nnc_tensor_symbol_new(group_norm_symbolic_graph, CPU_TENSOR_NHWC(32F, 8, 4, 4, 10), "y");
131
1
  ccv_nnc_tensor_symbol_t scale = ccv_nnc_tensor_symbol_new(group_norm_symbolic_graph, CPU_TENSOR_NHWC(32F, 1, 4, 4, 10), "scale");
132
1
  ccv_nnc_tensor_symbol_t bias = ccv_nnc_tensor_symbol_new(group_norm_symbolic_graph, CPU_TENSOR_NHWC(32F, 1, 4, 4, 10), "bias");
133
1
  ccv_nnc_tensor_symbol_t saved_mean = ccv_nnc_tensor_symbol_new(group_norm_symbolic_graph, CPU_TENSOR_NHWC(32F, 8, 4, 4, 1), "saved_mean");
134
1
  ccv_nnc_tensor_symbol_t saved_inv_std = ccv_nnc_tensor_symbol_new(group_norm_symbolic_graph, CPU_TENSOR_NHWC(32F, 8, 4, 4, 1), "saved_inv_std");
135
1
  ccv_nnc_graph_exec_symbol_new(group_norm_symbolic_graph, CMD_SET_FORWARD(1), 0, 0, TENSOR_SYMBOL_LIST(scale), "set_scale");
136
1
  ccv_nnc_graph_exec_symbol_new(group_norm_symbolic_graph, CMD_SET_FORWARD(0), 0, 0, TENSOR_SYMBOL_LIST(bias), "set_bias");
137
1
  ccv_nnc_graph_exec_symbol_new(group_norm_symbolic_graph, CMD_GROUP_NORM_FORWARD(3, 1, 0, 1), TENSOR_SYMBOL_LIST(bx, scale, bias), TENSOR_SYMBOL_LIST(by, saved_mean, saved_inv_std), "group_norm");
138
1
  ccv_nnc_graph_exec_symbol_autogen(group_norm_symbolic_graph, 0, 0, CCV_NNC_AUTOGEN_ALL_EXECS | CCV_NNC_AUTOGEN_SOURCES_AND_DESTINATIONS);
139
1
  ccv_nnc_symbolic_graph_backward(group_norm_symbolic_graph, TENSOR_SYMBOL_LIST(by), TENSOR_SYMBOL_LIST(bx, scale, bias), SYMBOLIC_GRAPH_SOURCES(group_norm_symbolic_graph), SYMBOLIC_GRAPH_DESTINATIONS(group_norm_symbolic_graph));
140
1
  ccv_nnc_graph_exec_symbol_autogen(group_norm_symbolic_graph, 0, 0, CCV_NNC_AUTOGEN_ALL_EXECS | CCV_NNC_AUTOGEN_SOURCES_AND_DESTINATIONS);
141
1
  ccv_nnc_tensor_symbol_t dby = ccv_nnc_tensor_symbol_for_backward(group_norm_symbolic_graph, by);
142
1
  ccv_nnc_tensor_symbol_t dbx = ccv_nnc_tensor_symbol_for_backward(group_norm_symbolic_graph, bx);
143
1
  ccv_nnc_graph_t* group_norm_graph = 0;
144
1
  ccv_nnc_tensor_arena_t* group_norm_tensor_arena = 0;
145
1
  ccv_nnc_graph_exec_arena_t* group_norm_graph_exec_arena = 0;
146
1
  ccv_nnc_symbolic_graph_compile(group_norm_symbolic_graph, ccv_nnc_default_compile_params, 0, 0, 0, 0, SYMBOLIC_GRAPH_SOURCES(group_norm_symbolic_graph), SYMBOLIC_GRAPH_DESTINATIONS(group_norm_symbolic_graph), &group_norm_graph, &group_norm_tensor_arena, &group_norm_graph_exec_arena);
147
1
  ccv_nnc_tensor_t* const bx_tensor = ccv_nnc_tensor_from_symbol(group_norm_tensor_arena, bx);
148
1
  ccv_nnc_tensor_t* const dy_tensor = ccv_nnc_tensor_from_symbol(tensor_arena, dy);
149
1
  ccv_nnc_tensor_t* const dby_tensor = ccv_nnc_tensor_from_symbol(group_norm_tensor_arena, dby);
150
1.28k
  for (i = 0; i < 8 * 4 * 4 * 10; 
i++1.28k
)
151
1.28k
    dby_tensor->data.f32[i] = dy_tensor->data.f32[i] = dsfmt_genrand_open_close(&dsfmt) * 2 - 1;
152
1
  memcpy(bx_tensor->data.f32, x_tensor->data.f32, sizeof(float) * 8 * 4 * 4 * 10);
153
1
  ccv_nnc_graph_run(graph, 0, TRAVERSE_FULL, 0, 0);
154
1
  ccv_nnc_graph_run(group_norm_graph, 0, TRAVERSE_FULL, 0, 0);
155
1
  ccv_nnc_tensor_t* const dx_tensor = ccv_nnc_tensor_from_symbol(tensor_arena, dx);
156
1
  ccv_nnc_tensor_t* const dbx_tensor = ccv_nnc_tensor_from_symbol(group_norm_tensor_arena, dbx);
157
1
  REQUIRE_ARRAY_EQ_WITH_TOLERANCE(float, dx_tensor->data.f32, dbx_tensor->data.f32, 8 * 4 * 4 * 10, 1e-5, "graph computed result should match layer norm op result");
158
1
  ccv_nnc_symbolic_graph_free(symbolic_graph);
159
1
  ccv_nnc_tensor_arena_free(tensor_arena);
160
1
  ccv_nnc_graph_exec_arena_free(graph_exec_arena);
161
1
  ccv_nnc_graph_free(graph);
162
1
  ccv_nnc_symbolic_graph_free(group_norm_symbolic_graph);
163
1
  ccv_nnc_tensor_arena_free(group_norm_tensor_arena);
164
1
  ccv_nnc_graph_exec_arena_free(group_norm_graph_exec_arena);
165
1
  ccv_nnc_graph_free(group_norm_graph);
166
1
}
167
168
TEST_CASE("implement group norm with other symbolic graph")
169
1
{
170
1
  ccv_nnc_symbolic_graph_t* const symbolic_graph = ccv_nnc_symbolic_graph_new();
171
1
  ccv_nnc_tensor_symbol_t x = ccv_nnc_tensor_symbol_new(symbolic_graph, CPU_TENSOR_NHWC(32F, 8, 4, 4, 5), "x");
172
1
  ccv_nnc_tensor_symbol_t sum = ccv_nnc_tensor_symbol_new(symbolic_graph, CPU_TENSOR_NHWC(32F, 8, 4, 1, 5), "sum");
173
1
  ccv_nnc_tensor_symbol_t mean = ccv_nnc_tensor_symbol_new(symbolic_graph, CPU_TENSOR_NHWC(32F, 8, 4, 1, 5), "mean");
174
1
  ccv_nnc_tensor_symbol_t whitening = ccv_nnc_tensor_symbol_new(symbolic_graph, CPU_TENSOR_NHWC(32F, 8, 4, 4, 5), "whitening");
175
1
  ccv_nnc_tensor_symbol_t sqr = ccv_nnc_tensor_symbol_new(symbolic_graph, CPU_TENSOR_NHWC(32F, 8, 4, 4, 5), "sqr");
176
1
  ccv_nnc_tensor_symbol_t varsum = ccv_nnc_tensor_symbol_new(symbolic_graph, CPU_TENSOR_NHWC(32F, 8, 4, 1, 5), "varsum");
177
1
  ccv_nnc_tensor_symbol_t var = ccv_nnc_tensor_symbol_new(symbolic_graph, CPU_TENSOR_NHWC(32F, 8, 4, 1, 5), "var");
178
1
  ccv_nnc_tensor_symbol_t logvar = ccv_nnc_tensor_symbol_new(symbolic_graph, CPU_TENSOR_NHWC(32F, 8, 4, 1, 5), "logvar");
179
1
  ccv_nnc_tensor_symbol_t logvar_2 = ccv_nnc_tensor_symbol_new(symbolic_graph, CPU_TENSOR_NHWC(32F, 8, 4, 1, 5), "logvar");
180
1
  ccv_nnc_tensor_symbol_t std = ccv_nnc_tensor_symbol_new(symbolic_graph, CPU_TENSOR_NHWC(32F, 8, 4, 1, 5), "std");
181
1
  ccv_nnc_tensor_symbol_t inv_std = ccv_nnc_tensor_symbol_new(symbolic_graph, CPU_TENSOR_NHWC(32F, 8, 4, 1, 5), "inv_std");
182
1
  ccv_nnc_tensor_symbol_t y = ccv_nnc_tensor_symbol_new(symbolic_graph, CPU_TENSOR_NHWC(32F, 8, 4, 4, 5), "y");
183
1
  ccv_nnc_graph_exec_symbol_new(symbolic_graph, CMD_REDUCE_SUM_FORWARD(1), TENSOR_SYMBOL_LIST(x), TENSOR_SYMBOL_LIST(sum), "sum");
184
1
  ccv_nnc_graph_exec_symbol_new(symbolic_graph, CMD_SCALAR_MUL_FORWARD(1.0 / 4), TENSOR_SYMBOL_LIST(sum), TENSOR_SYMBOL_LIST(mean), "mean");
185
1
  ccv_nnc_graph_exec_symbol_new(symbolic_graph, CMD_ADD_FORWARD(1, -1), TENSOR_SYMBOL_LIST(x, mean), TENSOR_SYMBOL_LIST(whitening), "whitening");
186
1
  ccv_nnc_graph_exec_symbol_new(symbolic_graph, CMD_EWPROD_FORWARD(), TENSOR_SYMBOL_LIST(whitening, whitening), TENSOR_SYMBOL_LIST(sqr), "sqr");
187
1
  ccv_nnc_graph_exec_symbol_new(symbolic_graph, CMD_REDUCE_SUM_FORWARD(1), TENSOR_SYMBOL_LIST(sqr), TENSOR_SYMBOL_LIST(varsum), "varsum");
188
1
  ccv_nnc_graph_exec_symbol_new(symbolic_graph, CMD_SCALAR_MUL_FORWARD(1.0 / 4), TENSOR_SYMBOL_LIST(varsum), TENSOR_SYMBOL_LIST(var), "var");
189
1
  ccv_nnc_graph_exec_symbol_new(symbolic_graph, CMD_EWLOG_FORWARD(), TENSOR_SYMBOL_LIST(var), TENSOR_SYMBOL_LIST(logvar), "log(var)");
190
1
  ccv_nnc_graph_exec_symbol_new(symbolic_graph, CMD_SCALAR_MUL_FORWARD(0.5), TENSOR_SYMBOL_LIST(logvar), TENSOR_SYMBOL_LIST(logvar_2), "log(var)/2");
191
1
  ccv_nnc_graph_exec_symbol_new(symbolic_graph, CMD_EWEXP_FORWARD(), TENSOR_SYMBOL_LIST(logvar_2), TENSOR_SYMBOL_LIST(std), "std");
192
1
  ccv_nnc_graph_exec_symbol_new(symbolic_graph, CMD_EWDIV_FORWARD(), TENSOR_SYMBOL_LIST(NO_TENSOR_SYMBOL, std), TENSOR_SYMBOL_LIST(inv_std), "1/std");
193
1
  ccv_nnc_graph_exec_symbol_new(symbolic_graph, CMD_MUL_FORWARD(1), TENSOR_SYMBOL_LIST(whitening, inv_std), TENSOR_SYMBOL_LIST(y), "y");
194
1
  ccv_nnc_graph_exec_symbol_autogen(symbolic_graph, 0, 0, CCV_NNC_AUTOGEN_ALL_EXECS | CCV_NNC_AUTOGEN_SOURCES_AND_DESTINATIONS);
195
1
  SYMBOLIC_GRAPH_GEN(symbolic_graph, CCV_NNC_LONG_DOT_GRAPH);
196
1
  ccv_nnc_graph_t* graph = 0;
197
1
  ccv_nnc_tensor_arena_t* tensor_arena = 0;
198
1
  ccv_nnc_graph_exec_arena_t* graph_exec_arena = 0;
199
1
  ccv_nnc_symbolic_graph_compile(symbolic_graph, ccv_nnc_default_compile_params, 0, 0, 0, 0, SYMBOLIC_GRAPH_SOURCES(symbolic_graph), SYMBOLIC_GRAPH_DESTINATIONS(symbolic_graph), &graph, &tensor_arena, &graph_exec_arena);
200
1
  GRAPH_GEN(graph, CCV_NNC_LONG_DOT_GRAPH);
201
1
  ccv_nnc_tensor_t* const x_tensor = ccv_nnc_tensor_from_symbol(tensor_arena, x);
202
1
  dsfmt_t dsfmt;
203
1
  int i;
204
1
  dsfmt_init_gen_rand(&dsfmt, 1);
205
641
  for (i = 0; i < 8 * 4 * 4 * 5; 
i++640
)
206
640
    x_tensor->data.f32[i] = dsfmt_genrand_open_close(&dsfmt);
207
1
  ccv_nnc_symbolic_graph_t* const group_norm_symbolic_graph = ccv_nnc_symbolic_graph_new();
208
1
  ccv_nnc_tensor_symbol_t bx = ccv_nnc_tensor_symbol_new(group_norm_symbolic_graph, CPU_TENSOR_NHWC(32F, 8, 16, 5), "x");
209
1
  ccv_nnc_tensor_symbol_t by = ccv_nnc_tensor_symbol_new(group_norm_symbolic_graph, CPU_TENSOR_NHWC(32F, 8, 16, 5), "y");
210
1
  ccv_nnc_tensor_symbol_t scale = ccv_nnc_tensor_symbol_new(group_norm_symbolic_graph, CPU_TENSOR_NHWC(32F, 1, 16, 5), "scale");
211
1
  ccv_nnc_tensor_symbol_t bias = ccv_nnc_tensor_symbol_new(group_norm_symbolic_graph, CPU_TENSOR_NHWC(32F, 1, 16, 5), "bias");
212
1
  ccv_nnc_tensor_symbol_t saved_mean = ccv_nnc_tensor_symbol_new(group_norm_symbolic_graph, CPU_TENSOR_NHWC(32F, 8, 4, 5), "saved_mean");
213
1
  ccv_nnc_tensor_symbol_t saved_inv_std = ccv_nnc_tensor_symbol_new(group_norm_symbolic_graph, CPU_TENSOR_NHWC(32F, 8, 4, 5), "saved_inv_std");
214
1
  ccv_nnc_graph_exec_symbol_new(group_norm_symbolic_graph, CMD_SET_FORWARD(1), 0, 0, TENSOR_SYMBOL_LIST(scale), "set_scale");
215
1
  ccv_nnc_graph_exec_symbol_new(group_norm_symbolic_graph, CMD_SET_FORWARD(0), 0, 0, TENSOR_SYMBOL_LIST(bias), "set_bias");
216
1
  ccv_nnc_graph_exec_symbol_new(group_norm_symbolic_graph, CMD_GROUP_NORM_FORWARD(1, 4, 0, 1), TENSOR_SYMBOL_LIST(bx, scale, bias), TENSOR_SYMBOL_LIST(by, saved_mean, saved_inv_std), "group_norm");
217
1
  ccv_nnc_graph_exec_symbol_autogen(group_norm_symbolic_graph, 0, 0, CCV_NNC_AUTOGEN_ALL_EXECS | CCV_NNC_AUTOGEN_SOURCES_AND_DESTINATIONS);
218
1
  ccv_nnc_graph_t* group_norm_graph = 0;
219
1
  ccv_nnc_tensor_arena_t* group_norm_tensor_arena = 0;
220
1
  ccv_nnc_graph_exec_arena_t* group_norm_graph_exec_arena = 0;
221
1
  ccv_nnc_symbolic_graph_compile(group_norm_symbolic_graph, ccv_nnc_default_compile_params, 0, 0, 0, 0, SYMBOLIC_GRAPH_SOURCES(group_norm_symbolic_graph), SYMBOLIC_GRAPH_DESTINATIONS(group_norm_symbolic_graph), &group_norm_graph, &group_norm_tensor_arena, &group_norm_graph_exec_arena);
222
1
  ccv_nnc_tensor_t* const bx_tensor = ccv_nnc_tensor_from_symbol(group_norm_tensor_arena, bx);
223
1
  memcpy(bx_tensor->data.f32, x_tensor->data.f32, sizeof(float) * 8 * 16 * 5);
224
1
  ccv_nnc_graph_run(graph, 0, TRAVERSE_FULL, 0, 0);
225
1
  ccv_nnc_graph_run(group_norm_graph, 0, TRAVERSE_FULL, 0, 0);
226
1
  ccv_nnc_tensor_t* const y_tensor = ccv_nnc_tensor_from_symbol(tensor_arena, y);
227
1
  ccv_nnc_tensor_t* const by_tensor = ccv_nnc_tensor_from_symbol(group_norm_tensor_arena, by);
228
1
  REQUIRE_ARRAY_EQ_WITH_TOLERANCE(float, y_tensor->data.f32, by_tensor->data.f32, 8 * 16 * 5, 1e-5, "graph computed result should match batch norm op result");
229
1
  ccv_nnc_symbolic_graph_free(symbolic_graph);
230
1
  ccv_nnc_tensor_arena_free(tensor_arena);
231
1
  ccv_nnc_graph_exec_arena_free(graph_exec_arena);
232
1
  ccv_nnc_graph_free(graph);
233
1
  ccv_nnc_symbolic_graph_free(group_norm_symbolic_graph);
234
1
  ccv_nnc_tensor_arena_free(group_norm_tensor_arena);
235
1
  ccv_nnc_graph_exec_arena_free(group_norm_graph_exec_arena);
236
1
  ccv_nnc_graph_free(group_norm_graph);
237
1
}
238
239
TEST_CASE("compare group norm gradient with other symbolic graph")
240
1
{
241
1
  ccv_nnc_symbolic_graph_t* const symbolic_graph = ccv_nnc_symbolic_graph_new();
242
1
  ccv_nnc_tensor_symbol_t x = ccv_nnc_tensor_symbol_new(symbolic_graph, CPU_TENSOR_NHWC(32F, 8, 4, 4, 5), "x");
243
1
  ccv_nnc_tensor_symbol_t sum = ccv_nnc_tensor_symbol_new(symbolic_graph, CPU_TENSOR_NHWC(32F, 8, 4, 1, 5), "sum");
244
1
  ccv_nnc_tensor_symbol_t mean = ccv_nnc_tensor_symbol_new(symbolic_graph, CPU_TENSOR_NHWC(32F, 8, 4, 1, 5), "mean");
245
1
  ccv_nnc_tensor_symbol_t whitening = ccv_nnc_tensor_symbol_new(symbolic_graph, CPU_TENSOR_NHWC(32F, 8, 4, 4, 5), "whitening");
246
1
  ccv_nnc_tensor_symbol_t sqr = ccv_nnc_tensor_symbol_new(symbolic_graph, CPU_TENSOR_NHWC(32F, 8, 4, 4, 5), "sqr");
247
1
  ccv_nnc_tensor_symbol_t varsum = ccv_nnc_tensor_symbol_new(symbolic_graph, CPU_TENSOR_NHWC(32F, 8, 4, 1, 5), "varsum");
248
1
  ccv_nnc_tensor_symbol_t var = ccv_nnc_tensor_symbol_new(symbolic_graph, CPU_TENSOR_NHWC(32F, 8, 4, 1, 5), "var");
249
1
  ccv_nnc_tensor_symbol_t logvar = ccv_nnc_tensor_symbol_new(symbolic_graph, CPU_TENSOR_NHWC(32F, 8, 4, 1, 5), "logvar");
250
1
  ccv_nnc_tensor_symbol_t logvar_2 = ccv_nnc_tensor_symbol_new(symbolic_graph, CPU_TENSOR_NHWC(32F, 8, 4, 1, 5), "logvar");
251
1
  ccv_nnc_tensor_symbol_t std = ccv_nnc_tensor_symbol_new(symbolic_graph, CPU_TENSOR_NHWC(32F, 8, 4, 1, 5), "std");
252
1
  ccv_nnc_tensor_symbol_t inv_std = ccv_nnc_tensor_symbol_new(symbolic_graph, CPU_TENSOR_NHWC(32F, 8, 4, 1, 5), "inv_std");
253
1
  ccv_nnc_tensor_symbol_t y = ccv_nnc_tensor_symbol_new(symbolic_graph, CPU_TENSOR_NHWC(32F, 8, 4, 4, 5), "y");
254
1
  ccv_nnc_graph_exec_symbol_new(symbolic_graph, CMD_REDUCE_SUM_FORWARD(1), TENSOR_SYMBOL_LIST(x), TENSOR_SYMBOL_LIST(sum), "sum");
255
1
  ccv_nnc_graph_exec_symbol_new(symbolic_graph, CMD_SCALAR_MUL_FORWARD(1.0 / 4), TENSOR_SYMBOL_LIST(sum), TENSOR_SYMBOL_LIST(mean), "mean");
256
1
  ccv_nnc_graph_exec_symbol_new(symbolic_graph, CMD_ADD_FORWARD(1, -1), TENSOR_SYMBOL_LIST(x, mean), TENSOR_SYMBOL_LIST(whitening), "whitening");
257
1
  ccv_nnc_graph_exec_symbol_new(symbolic_graph, CMD_EWPROD_FORWARD(), TENSOR_SYMBOL_LIST(whitening, whitening), TENSOR_SYMBOL_LIST(sqr), "sqr");
258
1
  ccv_nnc_graph_exec_symbol_new(symbolic_graph, CMD_REDUCE_SUM_FORWARD(1), TENSOR_SYMBOL_LIST(sqr), TENSOR_SYMBOL_LIST(varsum), "varsum");
259
1
  ccv_nnc_graph_exec_symbol_new(symbolic_graph, CMD_SCALAR_MUL_FORWARD(1.0 / 4), TENSOR_SYMBOL_LIST(varsum), TENSOR_SYMBOL_LIST(var), "var");
260
1
  ccv_nnc_graph_exec_symbol_new(symbolic_graph, CMD_EWLOG_FORWARD(), TENSOR_SYMBOL_LIST(var), TENSOR_SYMBOL_LIST(logvar), "log(var)");
261
1
  ccv_nnc_graph_exec_symbol_new(symbolic_graph, CMD_SCALAR_MUL_FORWARD(0.5), TENSOR_SYMBOL_LIST(logvar), TENSOR_SYMBOL_LIST(logvar_2), "log(var)/2");
262
1
  ccv_nnc_graph_exec_symbol_new(symbolic_graph, CMD_EWEXP_FORWARD(), TENSOR_SYMBOL_LIST(logvar_2), TENSOR_SYMBOL_LIST(std), "std");
263
1
  ccv_nnc_graph_exec_symbol_new(symbolic_graph, CMD_EWDIV_FORWARD(), TENSOR_SYMBOL_LIST(NO_TENSOR_SYMBOL, std), TENSOR_SYMBOL_LIST(inv_std), "1/std");
264
1
  ccv_nnc_graph_exec_symbol_new(symbolic_graph, CMD_MUL_FORWARD(1), TENSOR_SYMBOL_LIST(whitening, inv_std), TENSOR_SYMBOL_LIST(y), "y");
265
1
  ccv_nnc_graph_exec_symbol_autogen(symbolic_graph, 0, 0, CCV_NNC_AUTOGEN_ALL_EXECS | CCV_NNC_AUTOGEN_SOURCES_AND_DESTINATIONS);
266
1
  ccv_nnc_symbolic_graph_backward(symbolic_graph, TENSOR_SYMBOL_LIST(y), TENSOR_SYMBOL_LIST(x), SYMBOLIC_GRAPH_SOURCES(symbolic_graph), SYMBOLIC_GRAPH_DESTINATIONS(symbolic_graph));
267
1
  ccv_nnc_graph_exec_symbol_autogen(symbolic_graph, 0, 0, CCV_NNC_AUTOGEN_ALL_EXECS | CCV_NNC_AUTOGEN_SOURCES_AND_DESTINATIONS);
268
1
  ccv_nnc_tensor_symbol_t dy = ccv_nnc_tensor_symbol_for_backward(symbolic_graph, y);
269
1
  ccv_nnc_tensor_symbol_t dx = ccv_nnc_tensor_symbol_for_backward(symbolic_graph, x);
270
1
  SYMBOLIC_GRAPH_GEN(symbolic_graph, CCV_NNC_LONG_DOT_GRAPH);
271
1
  ccv_nnc_graph_t* graph = 0;
272
1
  ccv_nnc_tensor_arena_t* tensor_arena = 0;
273
1
  ccv_nnc_graph_exec_arena_t* graph_exec_arena = 0;
274
1
  ccv_nnc_symbolic_graph_compile(symbolic_graph, ccv_nnc_default_compile_params, 0, 0, 0, 0, SYMBOLIC_GRAPH_SOURCES(symbolic_graph), SYMBOLIC_GRAPH_DESTINATIONS(symbolic_graph), &graph, &tensor_arena, &graph_exec_arena);
275
1
  GRAPH_GEN(graph, CCV_NNC_LONG_DOT_GRAPH);
276
1
  ccv_nnc_tensor_t* const x_tensor = ccv_nnc_tensor_from_symbol(tensor_arena, x);
277
1
  dsfmt_t dsfmt;
278
1
  int i;
279
1
  dsfmt_init_gen_rand(&dsfmt, 1);
280
641
  for (i = 0; i < 8 * 4 * 4 * 5; 
i++640
)
281
640
    x_tensor->data.f32[i] = dsfmt_genrand_open_close(&dsfmt);
282
1
  ccv_nnc_symbolic_graph_t* const group_norm_symbolic_graph = ccv_nnc_symbolic_graph_new();
283
1
  ccv_nnc_tensor_symbol_t bx = ccv_nnc_tensor_symbol_new(group_norm_symbolic_graph, CPU_TENSOR_NHWC(32F, 8, 16, 5), "x");
284
1
  ccv_nnc_tensor_symbol_t by = ccv_nnc_tensor_symbol_new(group_norm_symbolic_graph, CPU_TENSOR_NHWC(32F, 8, 16, 5), "y");
285
1
  ccv_nnc_tensor_symbol_t scale = ccv_nnc_tensor_symbol_new(group_norm_symbolic_graph, CPU_TENSOR_NHWC(32F, 1, 16, 5), "scale");
286
1
  ccv_nnc_tensor_symbol_t bias = ccv_nnc_tensor_symbol_new(group_norm_symbolic_graph, CPU_TENSOR_NHWC(32F, 1, 16, 5), "bias");
287
1
  ccv_nnc_tensor_symbol_t saved_mean = ccv_nnc_tensor_symbol_new(group_norm_symbolic_graph, CPU_TENSOR_NHWC(32F, 8, 4, 5), "saved_mean");
288
1
  ccv_nnc_tensor_symbol_t saved_inv_std = ccv_nnc_tensor_symbol_new(group_norm_symbolic_graph, CPU_TENSOR_NHWC(32F, 8, 4, 5), "saved_inv_std");
289
1
  ccv_nnc_graph_exec_symbol_new(group_norm_symbolic_graph, CMD_SET_FORWARD(1), 0, 0, TENSOR_SYMBOL_LIST(scale), "set_scale");
290
1
  ccv_nnc_graph_exec_symbol_new(group_norm_symbolic_graph, CMD_SET_FORWARD(0), 0, 0, TENSOR_SYMBOL_LIST(bias), "set_bias");
291
1
  ccv_nnc_graph_exec_symbol_new(group_norm_symbolic_graph, CMD_GROUP_NORM_FORWARD(1, 4, 0, 1), TENSOR_SYMBOL_LIST(bx, scale, bias), TENSOR_SYMBOL_LIST(by, saved_mean, saved_inv_std), "group_norm");
292
1
  ccv_nnc_graph_exec_symbol_autogen(group_norm_symbolic_graph, 0, 0, CCV_NNC_AUTOGEN_ALL_EXECS | CCV_NNC_AUTOGEN_SOURCES_AND_DESTINATIONS);
293
1
  ccv_nnc_symbolic_graph_backward(group_norm_symbolic_graph, TENSOR_SYMBOL_LIST(by), TENSOR_SYMBOL_LIST(bx, scale, bias), SYMBOLIC_GRAPH_SOURCES(group_norm_symbolic_graph), SYMBOLIC_GRAPH_DESTINATIONS(group_norm_symbolic_graph));
294
1
  ccv_nnc_graph_exec_symbol_autogen(group_norm_symbolic_graph, 0, 0, CCV_NNC_AUTOGEN_ALL_EXECS | CCV_NNC_AUTOGEN_SOURCES_AND_DESTINATIONS);
295
1
  ccv_nnc_tensor_symbol_t dby = ccv_nnc_tensor_symbol_for_backward(group_norm_symbolic_graph, by);
296
1
  ccv_nnc_tensor_symbol_t dbx = ccv_nnc_tensor_symbol_for_backward(group_norm_symbolic_graph, bx);
297
1
  ccv_nnc_graph_t* group_norm_graph = 0;
298
1
  ccv_nnc_tensor_arena_t* group_norm_tensor_arena = 0;
299
1
  ccv_nnc_graph_exec_arena_t* group_norm_graph_exec_arena = 0;
300
1
  ccv_nnc_symbolic_graph_compile(group_norm_symbolic_graph, ccv_nnc_default_compile_params, 0, 0, 0, 0, SYMBOLIC_GRAPH_SOURCES(group_norm_symbolic_graph), SYMBOLIC_GRAPH_DESTINATIONS(group_norm_symbolic_graph), &group_norm_graph, &group_norm_tensor_arena, &group_norm_graph_exec_arena);
301
1
  ccv_nnc_tensor_t* const bx_tensor = ccv_nnc_tensor_from_symbol(group_norm_tensor_arena, bx);
302
1
  ccv_nnc_tensor_t* const dy_tensor = ccv_nnc_tensor_from_symbol(tensor_arena, dy);
303
1
  ccv_nnc_tensor_t* const dby_tensor = ccv_nnc_tensor_from_symbol(group_norm_tensor_arena, dby);
304
641
  for (i = 0; i < 8 * 16 * 5; 
i++640
)
305
640
    dby_tensor->data.f32[i] = dy_tensor->data.f32[i] = dsfmt_genrand_open_close(&dsfmt) * 2 - 1;
306
1
  memcpy(bx_tensor->data.f32, x_tensor->data.f32, sizeof(float) * 8 * 16 * 5);
307
1
  ccv_nnc_graph_run(graph, 0, TRAVERSE_FULL, 0, 0);
308
1
  ccv_nnc_graph_run(group_norm_graph, 0, TRAVERSE_FULL, 0, 0);
309
1
  ccv_nnc_tensor_t* const dx_tensor = ccv_nnc_tensor_from_symbol(tensor_arena, dx);
310
1
  ccv_nnc_tensor_t* const dbx_tensor = ccv_nnc_tensor_from_symbol(group_norm_tensor_arena, dbx);
311
1
  REQUIRE_ARRAY_EQ_WITH_TOLERANCE(float, dx_tensor->data.f32, dbx_tensor->data.f32, 8 * 16 * 5, 1e-5, "graph computed result should match layer norm op result");
312
1
  ccv_nnc_symbolic_graph_free(symbolic_graph);
313
1
  ccv_nnc_tensor_arena_free(tensor_arena);
314
1
  ccv_nnc_graph_exec_arena_free(graph_exec_arena);
315
1
  ccv_nnc_graph_free(graph);
316
1
  ccv_nnc_symbolic_graph_free(group_norm_symbolic_graph);
317
1
  ccv_nnc_tensor_arena_free(group_norm_tensor_arena);
318
1
  ccv_nnc_graph_exec_arena_free(group_norm_graph_exec_arena);
319
1
  ccv_nnc_graph_free(group_norm_graph);
320
1
}
321
322
TEST_CASE("implement group norm with other symbolic graph without scale / bias")
323
1
{
324
1
  ccv_nnc_symbolic_graph_t* const symbolic_graph = ccv_nnc_symbolic_graph_new();
325
1
  ccv_nnc_tensor_symbol_t x = ccv_nnc_tensor_symbol_new(symbolic_graph, CPU_TENSOR_NHWC(32F, 8, 4, 4, 5), "x");
326
1
  ccv_nnc_tensor_symbol_t sum = ccv_nnc_tensor_symbol_new(symbolic_graph, CPU_TENSOR_NHWC(32F, 8, 4, 1, 5), "sum");
327
1
  ccv_nnc_tensor_symbol_t mean = ccv_nnc_tensor_symbol_new(symbolic_graph, CPU_TENSOR_NHWC(32F, 8, 4, 1, 5), "mean");
328
1
  ccv_nnc_tensor_symbol_t whitening = ccv_nnc_tensor_symbol_new(symbolic_graph, CPU_TENSOR_NHWC(32F, 8, 4, 4, 5), "whitening");
329
1
  ccv_nnc_tensor_symbol_t sqr = ccv_nnc_tensor_symbol_new(symbolic_graph, CPU_TENSOR_NHWC(32F, 8, 4, 4, 5), "sqr");
330
1
  ccv_nnc_tensor_symbol_t varsum = ccv_nnc_tensor_symbol_new(symbolic_graph, CPU_TENSOR_NHWC(32F, 8, 4, 1, 5), "varsum");
331
1
  ccv_nnc_tensor_symbol_t var = ccv_nnc_tensor_symbol_new(symbolic_graph, CPU_TENSOR_NHWC(32F, 8, 4, 1, 5), "var");
332
1
  ccv_nnc_tensor_symbol_t logvar = ccv_nnc_tensor_symbol_new(symbolic_graph, CPU_TENSOR_NHWC(32F, 8, 4, 1, 5), "logvar");
333
1
  ccv_nnc_tensor_symbol_t logvar_2 = ccv_nnc_tensor_symbol_new(symbolic_graph, CPU_TENSOR_NHWC(32F, 8, 4, 1, 5), "logvar");
334
1
  ccv_nnc_tensor_symbol_t std = ccv_nnc_tensor_symbol_new(symbolic_graph, CPU_TENSOR_NHWC(32F, 8, 4, 1, 5), "std");
335
1
  ccv_nnc_tensor_symbol_t inv_std = ccv_nnc_tensor_symbol_new(symbolic_graph, CPU_TENSOR_NHWC(32F, 8, 4, 1, 5), "inv_std");
336
1
  ccv_nnc_tensor_symbol_t y = ccv_nnc_tensor_symbol_new(symbolic_graph, CPU_TENSOR_NHWC(32F, 8, 4, 4, 5), "y");
337
1
  ccv_nnc_graph_exec_symbol_new(symbolic_graph, CMD_REDUCE_SUM_FORWARD(1), TENSOR_SYMBOL_LIST(x), TENSOR_SYMBOL_LIST(sum), "sum");
338
1
  ccv_nnc_graph_exec_symbol_new(symbolic_graph, CMD_SCALAR_MUL_FORWARD(1.0 / 4), TENSOR_SYMBOL_LIST(sum), TENSOR_SYMBOL_LIST(mean), "mean");
339
1
  ccv_nnc_graph_exec_symbol_new(symbolic_graph, CMD_ADD_FORWARD(1, -1), TENSOR_SYMBOL_LIST(x, mean), TENSOR_SYMBOL_LIST(whitening), "whitening");
340
1
  ccv_nnc_graph_exec_symbol_new(symbolic_graph, CMD_EWPROD_FORWARD(), TENSOR_SYMBOL_LIST(whitening, whitening), TENSOR_SYMBOL_LIST(sqr), "sqr");
341
1
  ccv_nnc_graph_exec_symbol_new(symbolic_graph, CMD_REDUCE_SUM_FORWARD(1), TENSOR_SYMBOL_LIST(sqr), TENSOR_SYMBOL_LIST(varsum), "varsum");
342
1
  ccv_nnc_graph_exec_symbol_new(symbolic_graph, CMD_SCALAR_MUL_FORWARD(1.0 / 4), TENSOR_SYMBOL_LIST(varsum), TENSOR_SYMBOL_LIST(var), "var");
343
1
  ccv_nnc_graph_exec_symbol_new(symbolic_graph, CMD_EWLOG_FORWARD(), TENSOR_SYMBOL_LIST(var), TENSOR_SYMBOL_LIST(logvar), "log(var)");
344
1
  ccv_nnc_graph_exec_symbol_new(symbolic_graph, CMD_SCALAR_MUL_FORWARD(0.5), TENSOR_SYMBOL_LIST(logvar), TENSOR_SYMBOL_LIST(logvar_2), "log(var)/2");
345
1
  ccv_nnc_graph_exec_symbol_new(symbolic_graph, CMD_EWEXP_FORWARD(), TENSOR_SYMBOL_LIST(logvar_2), TENSOR_SYMBOL_LIST(std), "std");
346
1
  ccv_nnc_graph_exec_symbol_new(symbolic_graph, CMD_EWDIV_FORWARD(), TENSOR_SYMBOL_LIST(NO_TENSOR_SYMBOL, std), TENSOR_SYMBOL_LIST(inv_std), "1/std");
347
1
  ccv_nnc_graph_exec_symbol_new(symbolic_graph, CMD_MUL_FORWARD(1), TENSOR_SYMBOL_LIST(whitening, inv_std), TENSOR_SYMBOL_LIST(y), "y");
348
1
  ccv_nnc_graph_exec_symbol_autogen(symbolic_graph, 0, 0, CCV_NNC_AUTOGEN_ALL_EXECS | CCV_NNC_AUTOGEN_SOURCES_AND_DESTINATIONS);
349
1
  SYMBOLIC_GRAPH_GEN(symbolic_graph, CCV_NNC_LONG_DOT_GRAPH);
350
1
  ccv_nnc_graph_t* graph = 0;
351
1
  ccv_nnc_tensor_arena_t* tensor_arena = 0;
352
1
  ccv_nnc_graph_exec_arena_t* graph_exec_arena = 0;
353
1
  ccv_nnc_symbolic_graph_compile(symbolic_graph, ccv_nnc_default_compile_params, 0, 0, 0, 0, SYMBOLIC_GRAPH_SOURCES(symbolic_graph), SYMBOLIC_GRAPH_DESTINATIONS(symbolic_graph), &graph, &tensor_arena, &graph_exec_arena);
354
1
  GRAPH_GEN(graph, CCV_NNC_LONG_DOT_GRAPH);
355
1
  ccv_nnc_tensor_t* const x_tensor = ccv_nnc_tensor_from_symbol(tensor_arena, x);
356
1
  dsfmt_t dsfmt;
357
1
  int i;
358
1
  dsfmt_init_gen_rand(&dsfmt, 1);
359
641
  for (i = 0; i < 8 * 4 * 4 * 5; 
i++640
)
360
640
    x_tensor->data.f32[i] = dsfmt_genrand_open_close(&dsfmt);
361
1
  ccv_nnc_symbolic_graph_t* const group_norm_symbolic_graph = ccv_nnc_symbolic_graph_new();
362
1
  ccv_nnc_tensor_symbol_t bx = ccv_nnc_tensor_symbol_new(group_norm_symbolic_graph, CPU_TENSOR_NHWC(32F, 8, 16, 5), "x");
363
1
  ccv_nnc_tensor_symbol_t by = ccv_nnc_tensor_symbol_new(group_norm_symbolic_graph, CPU_TENSOR_NHWC(32F, 8, 16, 5), "y");
364
1
  ccv_nnc_tensor_symbol_t saved_mean = ccv_nnc_tensor_symbol_new(group_norm_symbolic_graph, CPU_TENSOR_NHWC(32F, 8, 4, 5), "saved_mean");
365
1
  ccv_nnc_tensor_symbol_t saved_inv_std = ccv_nnc_tensor_symbol_new(group_norm_symbolic_graph, CPU_TENSOR_NHWC(32F, 8, 4, 5), "saved_inv_std");
366
1
  ccv_nnc_graph_exec_symbol_new(group_norm_symbolic_graph, CMD_GROUP_NORM_FORWARD(1, 4, 0, 0), TENSOR_SYMBOL_LIST(bx), TENSOR_SYMBOL_LIST(by, saved_mean, saved_inv_std), "group_norm");
367
1
  ccv_nnc_graph_exec_symbol_autogen(group_norm_symbolic_graph, 0, 0, CCV_NNC_AUTOGEN_ALL_EXECS | CCV_NNC_AUTOGEN_SOURCES_AND_DESTINATIONS);
368
1
  ccv_nnc_graph_t* group_norm_graph = 0;
369
1
  ccv_nnc_tensor_arena_t* group_norm_tensor_arena = 0;
370
1
  ccv_nnc_graph_exec_arena_t* group_norm_graph_exec_arena = 0;
371
1
  ccv_nnc_symbolic_graph_compile(group_norm_symbolic_graph, ccv_nnc_default_compile_params, 0, 0, 0, 0, SYMBOLIC_GRAPH_SOURCES(group_norm_symbolic_graph), SYMBOLIC_GRAPH_DESTINATIONS(group_norm_symbolic_graph), &group_norm_graph, &group_norm_tensor_arena, &group_norm_graph_exec_arena);
372
1
  ccv_nnc_tensor_t* const bx_tensor = ccv_nnc_tensor_from_symbol(group_norm_tensor_arena, bx);
373
1
  memcpy(bx_tensor->data.f32, x_tensor->data.f32, sizeof(float) * 8 * 16 * 5);
374
1
  ccv_nnc_graph_run(graph, 0, TRAVERSE_FULL, 0, 0);
375
1
  ccv_nnc_graph_run(group_norm_graph, 0, TRAVERSE_FULL, 0, 0);
376
1
  ccv_nnc_tensor_t* const y_tensor = ccv_nnc_tensor_from_symbol(tensor_arena, y);
377
1
  ccv_nnc_tensor_t* const by_tensor = ccv_nnc_tensor_from_symbol(group_norm_tensor_arena, by);
378
1
  REQUIRE_ARRAY_EQ_WITH_TOLERANCE(float, y_tensor->data.f32, by_tensor->data.f32, 8 * 16 * 5, 1e-5, "graph computed result should match batch norm op result");
379
1
  ccv_nnc_symbolic_graph_free(symbolic_graph);
380
1
  ccv_nnc_tensor_arena_free(tensor_arena);
381
1
  ccv_nnc_graph_exec_arena_free(graph_exec_arena);
382
1
  ccv_nnc_graph_free(graph);
383
1
  ccv_nnc_symbolic_graph_free(group_norm_symbolic_graph);
384
1
  ccv_nnc_tensor_arena_free(group_norm_tensor_arena);
385
1
  ccv_nnc_graph_exec_arena_free(group_norm_graph_exec_arena);
386
1
  ccv_nnc_graph_free(group_norm_graph);
387
1
}
388
389
TEST_CASE("compare group norm gradient with other symbolic graph without scale / bias")
390
1
{
391
1
  ccv_nnc_symbolic_graph_t* const symbolic_graph = ccv_nnc_symbolic_graph_new();
392
1
  ccv_nnc_tensor_symbol_t x = ccv_nnc_tensor_symbol_new(symbolic_graph, CPU_TENSOR_NHWC(32F, 8, 4, 4, 5), "x");
393
1
  ccv_nnc_tensor_symbol_t sum = ccv_nnc_tensor_symbol_new(symbolic_graph, CPU_TENSOR_NHWC(32F, 8, 4, 1, 5), "sum");
394
1
  ccv_nnc_tensor_symbol_t mean = ccv_nnc_tensor_symbol_new(symbolic_graph, CPU_TENSOR_NHWC(32F, 8, 4, 1, 5), "mean");
395
1
  ccv_nnc_tensor_symbol_t whitening = ccv_nnc_tensor_symbol_new(symbolic_graph, CPU_TENSOR_NHWC(32F, 8, 4, 4, 5), "whitening");
396
1
  ccv_nnc_tensor_symbol_t sqr = ccv_nnc_tensor_symbol_new(symbolic_graph, CPU_TENSOR_NHWC(32F, 8, 4, 4, 5), "sqr");
397
1
  ccv_nnc_tensor_symbol_t varsum = ccv_nnc_tensor_symbol_new(symbolic_graph, CPU_TENSOR_NHWC(32F, 8, 4, 1, 5), "varsum");
398
1
  ccv_nnc_tensor_symbol_t var = ccv_nnc_tensor_symbol_new(symbolic_graph, CPU_TENSOR_NHWC(32F, 8, 4, 1, 5), "var");
399
1
  ccv_nnc_tensor_symbol_t logvar = ccv_nnc_tensor_symbol_new(symbolic_graph, CPU_TENSOR_NHWC(32F, 8, 4, 1, 5), "logvar");
400
1
  ccv_nnc_tensor_symbol_t logvar_2 = ccv_nnc_tensor_symbol_new(symbolic_graph, CPU_TENSOR_NHWC(32F, 8, 4, 1, 5), "logvar");
401
1
  ccv_nnc_tensor_symbol_t std = ccv_nnc_tensor_symbol_new(symbolic_graph, CPU_TENSOR_NHWC(32F, 8, 4, 1, 5), "std");
402
1
  ccv_nnc_tensor_symbol_t inv_std = ccv_nnc_tensor_symbol_new(symbolic_graph, CPU_TENSOR_NHWC(32F, 8, 4, 1, 5), "inv_std");
403
1
  ccv_nnc_tensor_symbol_t y = ccv_nnc_tensor_symbol_new(symbolic_graph, CPU_TENSOR_NHWC(32F, 8, 4, 4, 5), "y");
404
1
  ccv_nnc_graph_exec_symbol_new(symbolic_graph, CMD_REDUCE_SUM_FORWARD(1), TENSOR_SYMBOL_LIST(x), TENSOR_SYMBOL_LIST(sum), "sum");
405
1
  ccv_nnc_graph_exec_symbol_new(symbolic_graph, CMD_SCALAR_MUL_FORWARD(1.0 / 4), TENSOR_SYMBOL_LIST(sum), TENSOR_SYMBOL_LIST(mean), "mean");
406
1
  ccv_nnc_graph_exec_symbol_new(symbolic_graph, CMD_ADD_FORWARD(1, -1), TENSOR_SYMBOL_LIST(x, mean), TENSOR_SYMBOL_LIST(whitening), "whitening");
407
1
  ccv_nnc_graph_exec_symbol_new(symbolic_graph, CMD_EWPROD_FORWARD(), TENSOR_SYMBOL_LIST(whitening, whitening), TENSOR_SYMBOL_LIST(sqr), "sqr");
408
1
  ccv_nnc_graph_exec_symbol_new(symbolic_graph, CMD_REDUCE_SUM_FORWARD(1), TENSOR_SYMBOL_LIST(sqr), TENSOR_SYMBOL_LIST(varsum), "varsum");
409
1
  ccv_nnc_graph_exec_symbol_new(symbolic_graph, CMD_SCALAR_MUL_FORWARD(1.0 / 4), TENSOR_SYMBOL_LIST(varsum), TENSOR_SYMBOL_LIST(var), "var");
410
1
  ccv_nnc_graph_exec_symbol_new(symbolic_graph, CMD_EWLOG_FORWARD(), TENSOR_SYMBOL_LIST(var), TENSOR_SYMBOL_LIST(logvar), "log(var)");
411
1
  ccv_nnc_graph_exec_symbol_new(symbolic_graph, CMD_SCALAR_MUL_FORWARD(0.5), TENSOR_SYMBOL_LIST(logvar), TENSOR_SYMBOL_LIST(logvar_2), "log(var)/2");
412
1
  ccv_nnc_graph_exec_symbol_new(symbolic_graph, CMD_EWEXP_FORWARD(), TENSOR_SYMBOL_LIST(logvar_2), TENSOR_SYMBOL_LIST(std), "std");
413
1
  ccv_nnc_graph_exec_symbol_new(symbolic_graph, CMD_EWDIV_FORWARD(), TENSOR_SYMBOL_LIST(NO_TENSOR_SYMBOL, std), TENSOR_SYMBOL_LIST(inv_std), "1/std");
414
1
  ccv_nnc_graph_exec_symbol_new(symbolic_graph, CMD_MUL_FORWARD(1), TENSOR_SYMBOL_LIST(whitening, inv_std), TENSOR_SYMBOL_LIST(y), "y");
415
1
  ccv_nnc_graph_exec_symbol_autogen(symbolic_graph, 0, 0, CCV_NNC_AUTOGEN_ALL_EXECS | CCV_NNC_AUTOGEN_SOURCES_AND_DESTINATIONS);
416
1
  ccv_nnc_symbolic_graph_backward(symbolic_graph, TENSOR_SYMBOL_LIST(y), TENSOR_SYMBOL_LIST(x), SYMBOLIC_GRAPH_SOURCES(symbolic_graph), SYMBOLIC_GRAPH_DESTINATIONS(symbolic_graph));
417
1
  ccv_nnc_graph_exec_symbol_autogen(symbolic_graph, 0, 0, CCV_NNC_AUTOGEN_ALL_EXECS | CCV_NNC_AUTOGEN_SOURCES_AND_DESTINATIONS);
418
1
  ccv_nnc_tensor_symbol_t dy = ccv_nnc_tensor_symbol_for_backward(symbolic_graph, y);
419
1
  ccv_nnc_tensor_symbol_t dx = ccv_nnc_tensor_symbol_for_backward(symbolic_graph, x);
420
1
  SYMBOLIC_GRAPH_GEN(symbolic_graph, CCV_NNC_LONG_DOT_GRAPH);
421
1
  ccv_nnc_graph_t* graph = 0;
422
1
  ccv_nnc_tensor_arena_t* tensor_arena = 0;
423
1
  ccv_nnc_graph_exec_arena_t* graph_exec_arena = 0;
424
1
  ccv_nnc_symbolic_graph_compile(symbolic_graph, ccv_nnc_default_compile_params, 0, 0, 0, 0, SYMBOLIC_GRAPH_SOURCES(symbolic_graph), SYMBOLIC_GRAPH_DESTINATIONS(symbolic_graph), &graph, &tensor_arena, &graph_exec_arena);
425
1
  GRAPH_GEN(graph, CCV_NNC_LONG_DOT_GRAPH);
426
1
  ccv_nnc_tensor_t* const x_tensor = ccv_nnc_tensor_from_symbol(tensor_arena, x);
427
1
  dsfmt_t dsfmt;
428
1
  int i;
429
1
  dsfmt_init_gen_rand(&dsfmt, 1);
430
641
  for (i = 0; i < 8 * 4 * 4 * 5; 
i++640
)
431
640
    x_tensor->data.f32[i] = dsfmt_genrand_open_close(&dsfmt);
432
1
  ccv_nnc_symbolic_graph_t* const group_norm_symbolic_graph = ccv_nnc_symbolic_graph_new();
433
1
  ccv_nnc_tensor_symbol_t bx = ccv_nnc_tensor_symbol_new(group_norm_symbolic_graph, CPU_TENSOR_NHWC(32F, 8, 16, 5), "x");
434
1
  ccv_nnc_tensor_symbol_t by = ccv_nnc_tensor_symbol_new(group_norm_symbolic_graph, CPU_TENSOR_NHWC(32F, 8, 16, 5), "y");
435
1
  ccv_nnc_tensor_symbol_t saved_mean = ccv_nnc_tensor_symbol_new(group_norm_symbolic_graph, CPU_TENSOR_NHWC(32F, 8, 4, 5), "saved_mean");
436
1
  ccv_nnc_tensor_symbol_t saved_inv_std = ccv_nnc_tensor_symbol_new(group_norm_symbolic_graph, CPU_TENSOR_NHWC(32F, 8, 4, 5), "saved_inv_std");
437
1
  ccv_nnc_graph_exec_symbol_new(group_norm_symbolic_graph, CMD_GROUP_NORM_FORWARD(1, 4, 0, 0), TENSOR_SYMBOL_LIST(bx), TENSOR_SYMBOL_LIST(by, saved_mean, saved_inv_std), "group_norm");
438
1
  ccv_nnc_graph_exec_symbol_autogen(group_norm_symbolic_graph, 0, 0, CCV_NNC_AUTOGEN_ALL_EXECS | CCV_NNC_AUTOGEN_SOURCES_AND_DESTINATIONS);
439
1
  ccv_nnc_symbolic_graph_backward(group_norm_symbolic_graph, TENSOR_SYMBOL_LIST(by), TENSOR_SYMBOL_LIST(bx), SYMBOLIC_GRAPH_SOURCES(group_norm_symbolic_graph), SYMBOLIC_GRAPH_DESTINATIONS(group_norm_symbolic_graph));
440
1
  ccv_nnc_graph_exec_symbol_autogen(group_norm_symbolic_graph, 0, 0, CCV_NNC_AUTOGEN_ALL_EXECS | CCV_NNC_AUTOGEN_SOURCES_AND_DESTINATIONS);
441
1
  ccv_nnc_tensor_symbol_t dby = ccv_nnc_tensor_symbol_for_backward(group_norm_symbolic_graph, by);
442
1
  ccv_nnc_tensor_symbol_t dbx = ccv_nnc_tensor_symbol_for_backward(group_norm_symbolic_graph, bx);
443
1
  ccv_nnc_graph_t* group_norm_graph = 0;
444
1
  ccv_nnc_tensor_arena_t* group_norm_tensor_arena = 0;
445
1
  ccv_nnc_graph_exec_arena_t* group_norm_graph_exec_arena = 0;
446
1
  ccv_nnc_symbolic_graph_compile(group_norm_symbolic_graph, ccv_nnc_default_compile_params, 0, 0, 0, 0, SYMBOLIC_GRAPH_SOURCES(group_norm_symbolic_graph), SYMBOLIC_GRAPH_DESTINATIONS(group_norm_symbolic_graph), &group_norm_graph, &group_norm_tensor_arena, &group_norm_graph_exec_arena);
447
1
  ccv_nnc_tensor_t* const bx_tensor = ccv_nnc_tensor_from_symbol(group_norm_tensor_arena, bx);
448
1
  ccv_nnc_tensor_t* const dy_tensor = ccv_nnc_tensor_from_symbol(tensor_arena, dy);
449
1
  ccv_nnc_tensor_t* const dby_tensor = ccv_nnc_tensor_from_symbol(group_norm_tensor_arena, dby);
450
641
  for (i = 0; i < 8 * 16 * 5; 
i++640
)
451
640
    dby_tensor->data.f32[i] = dy_tensor->data.f32[i] = dsfmt_genrand_open_close(&dsfmt) * 2 - 1;
452
1
  memcpy(bx_tensor->data.f32, x_tensor->data.f32, sizeof(float) * 8 * 16 * 5);
453
1
  ccv_nnc_graph_run(graph, 0, TRAVERSE_FULL, 0, 0);
454
1
  ccv_nnc_graph_run(group_norm_graph, 0, TRAVERSE_FULL, 0, 0);
455
1
  ccv_nnc_tensor_t* const dx_tensor = ccv_nnc_tensor_from_symbol(tensor_arena, dx);
456
1
  ccv_nnc_tensor_t* const dbx_tensor = ccv_nnc_tensor_from_symbol(group_norm_tensor_arena, dbx);
457
1
  REQUIRE_ARRAY_EQ_WITH_TOLERANCE(float, dx_tensor->data.f32, dbx_tensor->data.f32, 8 * 16 * 5, 1e-5, "graph computed result should match layer norm op result");
458
1
  ccv_nnc_symbolic_graph_free(symbolic_graph);
459
1
  ccv_nnc_tensor_arena_free(tensor_arena);
460
1
  ccv_nnc_graph_exec_arena_free(graph_exec_arena);
461
1
  ccv_nnc_graph_free(graph);
462
1
  ccv_nnc_symbolic_graph_free(group_norm_symbolic_graph);
463
1
  ccv_nnc_tensor_arena_free(group_norm_tensor_arena);
464
1
  ccv_nnc_graph_exec_arena_free(group_norm_graph_exec_arena);
465
1
  ccv_nnc_graph_free(group_norm_graph);
466
1
}
467
468
#include "case_main.h"