/home/liu/actions-runner/_work/ccv/ccv/test/unit/nnc/group.norm.tests.c
Line | Count | Source |
1 | | #include "case.h" |
2 | | #include "ccv_case.h" |
3 | | #include "ccv_nnc_case.h" |
4 | | #include <ccv.h> |
5 | | #include <nnc/ccv_nnc.h> |
6 | | #include <nnc/ccv_nnc_easy.h> |
7 | | #include "3rdparty/dsfmt/dSFMT.h" |
8 | | |
9 | | TEST_SETUP() |
10 | | { |
11 | | ccv_nnc_init(); |
12 | | } |
13 | | |
14 | | TEST_CASE("implement simple group norm (like layer norm) with other symbolic graph") |
15 | 1 | { |
16 | 1 | ccv_nnc_symbolic_graph_t* const symbolic_graph = ccv_nnc_symbolic_graph_new(); |
17 | 1 | ccv_nnc_tensor_symbol_t x = ccv_nnc_tensor_symbol_new(symbolic_graph, CPU_TENSOR_NHWC(32F, 8, 4, 4, 10), "x"); |
18 | 1 | ccv_nnc_tensor_symbol_t sum = ccv_nnc_tensor_symbol_new(symbolic_graph, CPU_TENSOR_NHWC(32F, 8, 4, 4, 1), "sum"); |
19 | 1 | ccv_nnc_tensor_symbol_t mean = ccv_nnc_tensor_symbol_new(symbolic_graph, CPU_TENSOR_NHWC(32F, 8, 4, 4, 1), "mean"); |
20 | 1 | ccv_nnc_tensor_symbol_t whitening = ccv_nnc_tensor_symbol_new(symbolic_graph, CPU_TENSOR_NHWC(32F, 8, 4, 4, 10), "whitening"); |
21 | 1 | ccv_nnc_tensor_symbol_t sqr = ccv_nnc_tensor_symbol_new(symbolic_graph, CPU_TENSOR_NHWC(32F, 8, 4, 4, 10), "sqr"); |
22 | 1 | ccv_nnc_tensor_symbol_t varsum = ccv_nnc_tensor_symbol_new(symbolic_graph, CPU_TENSOR_NHWC(32F, 8, 4, 4, 1), "varsum"); |
23 | 1 | ccv_nnc_tensor_symbol_t var = ccv_nnc_tensor_symbol_new(symbolic_graph, CPU_TENSOR_NHWC(32F, 8, 4, 4, 1), "var"); |
24 | 1 | ccv_nnc_tensor_symbol_t logvar = ccv_nnc_tensor_symbol_new(symbolic_graph, CPU_TENSOR_NHWC(32F, 8, 4, 4, 1), "logvar"); |
25 | 1 | ccv_nnc_tensor_symbol_t logvar_2 = ccv_nnc_tensor_symbol_new(symbolic_graph, CPU_TENSOR_NHWC(32F, 8, 4, 4, 1), "logvar"); |
26 | 1 | ccv_nnc_tensor_symbol_t std = ccv_nnc_tensor_symbol_new(symbolic_graph, CPU_TENSOR_NHWC(32F, 8, 4, 4, 1), "std"); |
27 | 1 | ccv_nnc_tensor_symbol_t inv_std = ccv_nnc_tensor_symbol_new(symbolic_graph, CPU_TENSOR_NHWC(32F, 8, 4, 4, 1), "inv_std"); |
28 | 1 | ccv_nnc_tensor_symbol_t y = ccv_nnc_tensor_symbol_new(symbolic_graph, CPU_TENSOR_NHWC(32F, 8, 4, 4, 10), "y"); |
29 | 1 | ccv_nnc_graph_exec_symbol_new(symbolic_graph, CMD_REDUCE_SUM_FORWARD(3), TENSOR_SYMBOL_LIST(x), TENSOR_SYMBOL_LIST(sum), "sum"); |
30 | 1 | ccv_nnc_graph_exec_symbol_new(symbolic_graph, CMD_SCALAR_MUL_FORWARD(1.0 / 10), TENSOR_SYMBOL_LIST(sum), TENSOR_SYMBOL_LIST(mean), "mean"); |
31 | 1 | ccv_nnc_graph_exec_symbol_new(symbolic_graph, CMD_ADD_FORWARD(1, -1), TENSOR_SYMBOL_LIST(x, mean), TENSOR_SYMBOL_LIST(whitening), "whitening"); |
32 | 1 | ccv_nnc_graph_exec_symbol_new(symbolic_graph, CMD_EWPROD_FORWARD(), TENSOR_SYMBOL_LIST(whitening, whitening), TENSOR_SYMBOL_LIST(sqr), "sqr"); |
33 | 1 | ccv_nnc_graph_exec_symbol_new(symbolic_graph, CMD_REDUCE_SUM_FORWARD(3), TENSOR_SYMBOL_LIST(sqr), TENSOR_SYMBOL_LIST(varsum), "varsum"); |
34 | 1 | ccv_nnc_graph_exec_symbol_new(symbolic_graph, CMD_SCALAR_MUL_FORWARD(1.0 / 10), TENSOR_SYMBOL_LIST(varsum), TENSOR_SYMBOL_LIST(var), "var"); |
35 | 1 | ccv_nnc_graph_exec_symbol_new(symbolic_graph, CMD_EWLOG_FORWARD(), TENSOR_SYMBOL_LIST(var), TENSOR_SYMBOL_LIST(logvar), "log(var)"); |
36 | 1 | ccv_nnc_graph_exec_symbol_new(symbolic_graph, CMD_SCALAR_MUL_FORWARD(0.5), TENSOR_SYMBOL_LIST(logvar), TENSOR_SYMBOL_LIST(logvar_2), "log(var)/2"); |
37 | 1 | ccv_nnc_graph_exec_symbol_new(symbolic_graph, CMD_EWEXP_FORWARD(), TENSOR_SYMBOL_LIST(logvar_2), TENSOR_SYMBOL_LIST(std), "std"); |
38 | 1 | ccv_nnc_graph_exec_symbol_new(symbolic_graph, CMD_EWDIV_FORWARD(), TENSOR_SYMBOL_LIST(NO_TENSOR_SYMBOL, std), TENSOR_SYMBOL_LIST(inv_std), "1/std"); |
39 | 1 | ccv_nnc_graph_exec_symbol_new(symbolic_graph, CMD_MUL_FORWARD(1), TENSOR_SYMBOL_LIST(whitening, inv_std), TENSOR_SYMBOL_LIST(y), "y"); |
40 | 1 | ccv_nnc_graph_exec_symbol_autogen(symbolic_graph, 0, 0, CCV_NNC_AUTOGEN_ALL_EXECS | CCV_NNC_AUTOGEN_SOURCES_AND_DESTINATIONS); |
41 | 1 | SYMBOLIC_GRAPH_GEN(symbolic_graph, CCV_NNC_LONG_DOT_GRAPH); |
42 | 1 | ccv_nnc_graph_t* graph = 0; |
43 | 1 | ccv_nnc_tensor_arena_t* tensor_arena = 0; |
44 | 1 | ccv_nnc_graph_exec_arena_t* graph_exec_arena = 0; |
45 | 1 | ccv_nnc_symbolic_graph_compile(symbolic_graph, ccv_nnc_default_compile_params, 0, 0, 0, 0, SYMBOLIC_GRAPH_SOURCES(symbolic_graph), SYMBOLIC_GRAPH_DESTINATIONS(symbolic_graph), &graph, &tensor_arena, &graph_exec_arena); |
46 | 1 | GRAPH_GEN(graph, CCV_NNC_LONG_DOT_GRAPH); |
47 | 1 | ccv_nnc_tensor_t* const x_tensor = ccv_nnc_tensor_from_symbol(tensor_arena, x); |
48 | 1 | dsfmt_t dsfmt; |
49 | 1 | int i; |
50 | 1 | dsfmt_init_gen_rand(&dsfmt, 1); |
51 | 1.28k | for (i = 0; i < 8 * 4 * 4 * 10; i++1.28k ) |
52 | 1.28k | x_tensor->data.f32[i] = dsfmt_genrand_open_close(&dsfmt); |
53 | 1 | ccv_nnc_symbolic_graph_t* const group_norm_symbolic_graph = ccv_nnc_symbolic_graph_new(); |
54 | 1 | ccv_nnc_tensor_symbol_t bx = ccv_nnc_tensor_symbol_new(group_norm_symbolic_graph, CPU_TENSOR_NHWC(32F, 8, 4, 4, 10), "x"); |
55 | 1 | ccv_nnc_tensor_symbol_t by = ccv_nnc_tensor_symbol_new(group_norm_symbolic_graph, CPU_TENSOR_NHWC(32F, 8, 4, 4, 10), "y"); |
56 | 1 | ccv_nnc_tensor_symbol_t scale = ccv_nnc_tensor_symbol_new(group_norm_symbolic_graph, CPU_TENSOR_NHWC(32F, 1, 4, 4, 10), "scale"); |
57 | 1 | ccv_nnc_tensor_symbol_t bias = ccv_nnc_tensor_symbol_new(group_norm_symbolic_graph, CPU_TENSOR_NHWC(32F, 1, 4, 4, 10), "bias"); |
58 | 1 | ccv_nnc_tensor_symbol_t saved_mean = ccv_nnc_tensor_symbol_new(group_norm_symbolic_graph, CPU_TENSOR_NHWC(32F, 8, 4, 4, 1), "saved_mean"); |
59 | 1 | ccv_nnc_tensor_symbol_t saved_inv_std = ccv_nnc_tensor_symbol_new(group_norm_symbolic_graph, CPU_TENSOR_NHWC(32F, 8, 4, 4, 1), "saved_inv_std"); |
60 | 1 | ccv_nnc_graph_exec_symbol_new(group_norm_symbolic_graph, CMD_SET_FORWARD(1), 0, 0, TENSOR_SYMBOL_LIST(scale), "set_scale"); |
61 | 1 | ccv_nnc_graph_exec_symbol_new(group_norm_symbolic_graph, CMD_SET_FORWARD(0), 0, 0, TENSOR_SYMBOL_LIST(bias), "set_bias"); |
62 | 1 | ccv_nnc_graph_exec_symbol_new(group_norm_symbolic_graph, CMD_GROUP_NORM_FORWARD(3, 1, 0, 1), TENSOR_SYMBOL_LIST(bx, scale, bias), TENSOR_SYMBOL_LIST(by, saved_mean, saved_inv_std), "group_norm"); |
63 | 1 | ccv_nnc_graph_exec_symbol_autogen(group_norm_symbolic_graph, 0, 0, CCV_NNC_AUTOGEN_ALL_EXECS | CCV_NNC_AUTOGEN_SOURCES_AND_DESTINATIONS); |
64 | 1 | ccv_nnc_graph_t* group_norm_graph = 0; |
65 | 1 | ccv_nnc_tensor_arena_t* group_norm_tensor_arena = 0; |
66 | 1 | ccv_nnc_graph_exec_arena_t* group_norm_graph_exec_arena = 0; |
67 | 1 | ccv_nnc_symbolic_graph_compile(group_norm_symbolic_graph, ccv_nnc_default_compile_params, 0, 0, 0, 0, SYMBOLIC_GRAPH_SOURCES(group_norm_symbolic_graph), SYMBOLIC_GRAPH_DESTINATIONS(group_norm_symbolic_graph), &group_norm_graph, &group_norm_tensor_arena, &group_norm_graph_exec_arena); |
68 | 1 | ccv_nnc_tensor_t* const bx_tensor = ccv_nnc_tensor_from_symbol(group_norm_tensor_arena, bx); |
69 | 1 | memcpy(bx_tensor->data.f32, x_tensor->data.f32, sizeof(float) * 8 * 4 * 4 * 10); |
70 | 1 | ccv_nnc_graph_run(graph, 0, TRAVERSE_FULL, 0, 0); |
71 | 1 | ccv_nnc_graph_run(group_norm_graph, 0, TRAVERSE_FULL, 0, 0); |
72 | 1 | ccv_nnc_tensor_t* const y_tensor = ccv_nnc_tensor_from_symbol(tensor_arena, y); |
73 | 1 | ccv_nnc_tensor_t* const by_tensor = ccv_nnc_tensor_from_symbol(group_norm_tensor_arena, by); |
74 | 1 | REQUIRE_TENSOR_EQ(y_tensor, by_tensor, "graph computed result should match batch norm op result"); |
75 | 1 | ccv_nnc_symbolic_graph_free(symbolic_graph); |
76 | 1 | ccv_nnc_tensor_arena_free(tensor_arena); |
77 | 1 | ccv_nnc_graph_exec_arena_free(graph_exec_arena); |
78 | 1 | ccv_nnc_graph_free(graph); |
79 | 1 | ccv_nnc_symbolic_graph_free(group_norm_symbolic_graph); |
80 | 1 | ccv_nnc_tensor_arena_free(group_norm_tensor_arena); |
81 | 1 | ccv_nnc_graph_exec_arena_free(group_norm_graph_exec_arena); |
82 | 1 | ccv_nnc_graph_free(group_norm_graph); |
83 | 1 | } |
84 | | |
85 | | TEST_CASE("compare simple group norm (like layer norm) gradient with other symbolic graph") |
86 | 1 | { |
87 | 1 | ccv_nnc_symbolic_graph_t* const symbolic_graph = ccv_nnc_symbolic_graph_new(); |
88 | 1 | ccv_nnc_tensor_symbol_t x = ccv_nnc_tensor_symbol_new(symbolic_graph, CPU_TENSOR_NHWC(32F, 8, 4, 4, 10), "x"); |
89 | 1 | ccv_nnc_tensor_symbol_t sum = ccv_nnc_tensor_symbol_new(symbolic_graph, CPU_TENSOR_NHWC(32F, 8, 4, 4, 1), "sum"); |
90 | 1 | ccv_nnc_tensor_symbol_t mean = ccv_nnc_tensor_symbol_new(symbolic_graph, CPU_TENSOR_NHWC(32F, 8, 4, 4, 1), "mean"); |
91 | 1 | ccv_nnc_tensor_symbol_t whitening = ccv_nnc_tensor_symbol_new(symbolic_graph, CPU_TENSOR_NHWC(32F, 8, 4, 4, 10), "whitening"); |
92 | 1 | ccv_nnc_tensor_symbol_t sqr = ccv_nnc_tensor_symbol_new(symbolic_graph, CPU_TENSOR_NHWC(32F, 8, 4, 4, 10), "sqr"); |
93 | 1 | ccv_nnc_tensor_symbol_t varsum = ccv_nnc_tensor_symbol_new(symbolic_graph, CPU_TENSOR_NHWC(32F, 8, 4, 4, 1), "varsum"); |
94 | 1 | ccv_nnc_tensor_symbol_t var = ccv_nnc_tensor_symbol_new(symbolic_graph, CPU_TENSOR_NHWC(32F, 8, 4, 4, 1), "var"); |
95 | 1 | ccv_nnc_tensor_symbol_t logvar = ccv_nnc_tensor_symbol_new(symbolic_graph, CPU_TENSOR_NHWC(32F, 8, 4, 4, 1), "logvar"); |
96 | 1 | ccv_nnc_tensor_symbol_t logvar_2 = ccv_nnc_tensor_symbol_new(symbolic_graph, CPU_TENSOR_NHWC(32F, 8, 4, 4, 1), "logvar"); |
97 | 1 | ccv_nnc_tensor_symbol_t std = ccv_nnc_tensor_symbol_new(symbolic_graph, CPU_TENSOR_NHWC(32F, 8, 4, 4, 1), "std"); |
98 | 1 | ccv_nnc_tensor_symbol_t inv_std = ccv_nnc_tensor_symbol_new(symbolic_graph, CPU_TENSOR_NHWC(32F, 8, 4, 4, 1), "inv_std"); |
99 | 1 | ccv_nnc_tensor_symbol_t y = ccv_nnc_tensor_symbol_new(symbolic_graph, CPU_TENSOR_NHWC(32F, 8, 4, 4, 10), "y"); |
100 | 1 | ccv_nnc_graph_exec_symbol_new(symbolic_graph, CMD_REDUCE_SUM_FORWARD(3), TENSOR_SYMBOL_LIST(x), TENSOR_SYMBOL_LIST(sum), "sum"); |
101 | 1 | ccv_nnc_graph_exec_symbol_new(symbolic_graph, CMD_SCALAR_MUL_FORWARD(1.0 / 10), TENSOR_SYMBOL_LIST(sum), TENSOR_SYMBOL_LIST(mean), "mean"); |
102 | 1 | ccv_nnc_graph_exec_symbol_new(symbolic_graph, CMD_ADD_FORWARD(1, -1), TENSOR_SYMBOL_LIST(x, mean), TENSOR_SYMBOL_LIST(whitening), "whitening"); |
103 | 1 | ccv_nnc_graph_exec_symbol_new(symbolic_graph, CMD_EWPROD_FORWARD(), TENSOR_SYMBOL_LIST(whitening, whitening), TENSOR_SYMBOL_LIST(sqr), "sqr"); |
104 | 1 | ccv_nnc_graph_exec_symbol_new(symbolic_graph, CMD_REDUCE_SUM_FORWARD(3), TENSOR_SYMBOL_LIST(sqr), TENSOR_SYMBOL_LIST(varsum), "varsum"); |
105 | 1 | ccv_nnc_graph_exec_symbol_new(symbolic_graph, CMD_SCALAR_MUL_FORWARD(1.0 / 10), TENSOR_SYMBOL_LIST(varsum), TENSOR_SYMBOL_LIST(var), "var"); |
106 | 1 | ccv_nnc_graph_exec_symbol_new(symbolic_graph, CMD_EWLOG_FORWARD(), TENSOR_SYMBOL_LIST(var), TENSOR_SYMBOL_LIST(logvar), "log(var)"); |
107 | 1 | ccv_nnc_graph_exec_symbol_new(symbolic_graph, CMD_SCALAR_MUL_FORWARD(0.5), TENSOR_SYMBOL_LIST(logvar), TENSOR_SYMBOL_LIST(logvar_2), "log(var)/2"); |
108 | 1 | ccv_nnc_graph_exec_symbol_new(symbolic_graph, CMD_EWEXP_FORWARD(), TENSOR_SYMBOL_LIST(logvar_2), TENSOR_SYMBOL_LIST(std), "std"); |
109 | 1 | ccv_nnc_graph_exec_symbol_new(symbolic_graph, CMD_EWDIV_FORWARD(), TENSOR_SYMBOL_LIST(NO_TENSOR_SYMBOL, std), TENSOR_SYMBOL_LIST(inv_std), "1/std"); |
110 | 1 | ccv_nnc_graph_exec_symbol_new(symbolic_graph, CMD_MUL_FORWARD(1), TENSOR_SYMBOL_LIST(whitening, inv_std), TENSOR_SYMBOL_LIST(y), "y"); |
111 | 1 | ccv_nnc_graph_exec_symbol_autogen(symbolic_graph, 0, 0, CCV_NNC_AUTOGEN_ALL_EXECS | CCV_NNC_AUTOGEN_SOURCES_AND_DESTINATIONS); |
112 | 1 | ccv_nnc_symbolic_graph_backward(symbolic_graph, TENSOR_SYMBOL_LIST(y), TENSOR_SYMBOL_LIST(x), SYMBOLIC_GRAPH_SOURCES(symbolic_graph), SYMBOLIC_GRAPH_DESTINATIONS(symbolic_graph)); |
113 | 1 | ccv_nnc_graph_exec_symbol_autogen(symbolic_graph, 0, 0, CCV_NNC_AUTOGEN_ALL_EXECS | CCV_NNC_AUTOGEN_SOURCES_AND_DESTINATIONS); |
114 | 1 | ccv_nnc_tensor_symbol_t dy = ccv_nnc_tensor_symbol_for_backward(symbolic_graph, y); |
115 | 1 | ccv_nnc_tensor_symbol_t dx = ccv_nnc_tensor_symbol_for_backward(symbolic_graph, x); |
116 | 1 | SYMBOLIC_GRAPH_GEN(symbolic_graph, CCV_NNC_LONG_DOT_GRAPH); |
117 | 1 | ccv_nnc_graph_t* graph = 0; |
118 | 1 | ccv_nnc_tensor_arena_t* tensor_arena = 0; |
119 | 1 | ccv_nnc_graph_exec_arena_t* graph_exec_arena = 0; |
120 | 1 | ccv_nnc_symbolic_graph_compile(symbolic_graph, ccv_nnc_default_compile_params, 0, 0, 0, 0, SYMBOLIC_GRAPH_SOURCES(symbolic_graph), SYMBOLIC_GRAPH_DESTINATIONS(symbolic_graph), &graph, &tensor_arena, &graph_exec_arena); |
121 | 1 | GRAPH_GEN(graph, CCV_NNC_LONG_DOT_GRAPH); |
122 | 1 | ccv_nnc_tensor_t* const x_tensor = ccv_nnc_tensor_from_symbol(tensor_arena, x); |
123 | 1 | dsfmt_t dsfmt; |
124 | 1 | int i; |
125 | 1 | dsfmt_init_gen_rand(&dsfmt, 1); |
126 | 1.28k | for (i = 0; i < 8 * 4 * 4 * 10; i++1.28k ) |
127 | 1.28k | x_tensor->data.f32[i] = dsfmt_genrand_open_close(&dsfmt); |
128 | 1 | ccv_nnc_symbolic_graph_t* const group_norm_symbolic_graph = ccv_nnc_symbolic_graph_new(); |
129 | 1 | ccv_nnc_tensor_symbol_t bx = ccv_nnc_tensor_symbol_new(group_norm_symbolic_graph, CPU_TENSOR_NHWC(32F, 8, 4, 4, 10), "x"); |
130 | 1 | ccv_nnc_tensor_symbol_t by = ccv_nnc_tensor_symbol_new(group_norm_symbolic_graph, CPU_TENSOR_NHWC(32F, 8, 4, 4, 10), "y"); |
131 | 1 | ccv_nnc_tensor_symbol_t scale = ccv_nnc_tensor_symbol_new(group_norm_symbolic_graph, CPU_TENSOR_NHWC(32F, 1, 4, 4, 10), "scale"); |
132 | 1 | ccv_nnc_tensor_symbol_t bias = ccv_nnc_tensor_symbol_new(group_norm_symbolic_graph, CPU_TENSOR_NHWC(32F, 1, 4, 4, 10), "bias"); |
133 | 1 | ccv_nnc_tensor_symbol_t saved_mean = ccv_nnc_tensor_symbol_new(group_norm_symbolic_graph, CPU_TENSOR_NHWC(32F, 8, 4, 4, 1), "saved_mean"); |
134 | 1 | ccv_nnc_tensor_symbol_t saved_inv_std = ccv_nnc_tensor_symbol_new(group_norm_symbolic_graph, CPU_TENSOR_NHWC(32F, 8, 4, 4, 1), "saved_inv_std"); |
135 | 1 | ccv_nnc_graph_exec_symbol_new(group_norm_symbolic_graph, CMD_SET_FORWARD(1), 0, 0, TENSOR_SYMBOL_LIST(scale), "set_scale"); |
136 | 1 | ccv_nnc_graph_exec_symbol_new(group_norm_symbolic_graph, CMD_SET_FORWARD(0), 0, 0, TENSOR_SYMBOL_LIST(bias), "set_bias"); |
137 | 1 | ccv_nnc_graph_exec_symbol_new(group_norm_symbolic_graph, CMD_GROUP_NORM_FORWARD(3, 1, 0, 1), TENSOR_SYMBOL_LIST(bx, scale, bias), TENSOR_SYMBOL_LIST(by, saved_mean, saved_inv_std), "group_norm"); |
138 | 1 | ccv_nnc_graph_exec_symbol_autogen(group_norm_symbolic_graph, 0, 0, CCV_NNC_AUTOGEN_ALL_EXECS | CCV_NNC_AUTOGEN_SOURCES_AND_DESTINATIONS); |
139 | 1 | ccv_nnc_symbolic_graph_backward(group_norm_symbolic_graph, TENSOR_SYMBOL_LIST(by), TENSOR_SYMBOL_LIST(bx, scale, bias), SYMBOLIC_GRAPH_SOURCES(group_norm_symbolic_graph), SYMBOLIC_GRAPH_DESTINATIONS(group_norm_symbolic_graph)); |
140 | 1 | ccv_nnc_graph_exec_symbol_autogen(group_norm_symbolic_graph, 0, 0, CCV_NNC_AUTOGEN_ALL_EXECS | CCV_NNC_AUTOGEN_SOURCES_AND_DESTINATIONS); |
141 | 1 | ccv_nnc_tensor_symbol_t dby = ccv_nnc_tensor_symbol_for_backward(group_norm_symbolic_graph, by); |
142 | 1 | ccv_nnc_tensor_symbol_t dbx = ccv_nnc_tensor_symbol_for_backward(group_norm_symbolic_graph, bx); |
143 | 1 | ccv_nnc_graph_t* group_norm_graph = 0; |
144 | 1 | ccv_nnc_tensor_arena_t* group_norm_tensor_arena = 0; |
145 | 1 | ccv_nnc_graph_exec_arena_t* group_norm_graph_exec_arena = 0; |
146 | 1 | ccv_nnc_symbolic_graph_compile(group_norm_symbolic_graph, ccv_nnc_default_compile_params, 0, 0, 0, 0, SYMBOLIC_GRAPH_SOURCES(group_norm_symbolic_graph), SYMBOLIC_GRAPH_DESTINATIONS(group_norm_symbolic_graph), &group_norm_graph, &group_norm_tensor_arena, &group_norm_graph_exec_arena); |
147 | 1 | ccv_nnc_tensor_t* const bx_tensor = ccv_nnc_tensor_from_symbol(group_norm_tensor_arena, bx); |
148 | 1 | ccv_nnc_tensor_t* const dy_tensor = ccv_nnc_tensor_from_symbol(tensor_arena, dy); |
149 | 1 | ccv_nnc_tensor_t* const dby_tensor = ccv_nnc_tensor_from_symbol(group_norm_tensor_arena, dby); |
150 | 1.28k | for (i = 0; i < 8 * 4 * 4 * 10; i++1.28k ) |
151 | 1.28k | dby_tensor->data.f32[i] = dy_tensor->data.f32[i] = dsfmt_genrand_open_close(&dsfmt) * 2 - 1; |
152 | 1 | memcpy(bx_tensor->data.f32, x_tensor->data.f32, sizeof(float) * 8 * 4 * 4 * 10); |
153 | 1 | ccv_nnc_graph_run(graph, 0, TRAVERSE_FULL, 0, 0); |
154 | 1 | ccv_nnc_graph_run(group_norm_graph, 0, TRAVERSE_FULL, 0, 0); |
155 | 1 | ccv_nnc_tensor_t* const dx_tensor = ccv_nnc_tensor_from_symbol(tensor_arena, dx); |
156 | 1 | ccv_nnc_tensor_t* const dbx_tensor = ccv_nnc_tensor_from_symbol(group_norm_tensor_arena, dbx); |
157 | 1 | REQUIRE_ARRAY_EQ_WITH_TOLERANCE(float, dx_tensor->data.f32, dbx_tensor->data.f32, 8 * 4 * 4 * 10, 1e-5, "graph computed result should match layer norm op result"); |
158 | 1 | ccv_nnc_symbolic_graph_free(symbolic_graph); |
159 | 1 | ccv_nnc_tensor_arena_free(tensor_arena); |
160 | 1 | ccv_nnc_graph_exec_arena_free(graph_exec_arena); |
161 | 1 | ccv_nnc_graph_free(graph); |
162 | 1 | ccv_nnc_symbolic_graph_free(group_norm_symbolic_graph); |
163 | 1 | ccv_nnc_tensor_arena_free(group_norm_tensor_arena); |
164 | 1 | ccv_nnc_graph_exec_arena_free(group_norm_graph_exec_arena); |
165 | 1 | ccv_nnc_graph_free(group_norm_graph); |
166 | 1 | } |
167 | | |
168 | | TEST_CASE("implement group norm with other symbolic graph") |
169 | 1 | { |
170 | 1 | ccv_nnc_symbolic_graph_t* const symbolic_graph = ccv_nnc_symbolic_graph_new(); |
171 | 1 | ccv_nnc_tensor_symbol_t x = ccv_nnc_tensor_symbol_new(symbolic_graph, CPU_TENSOR_NHWC(32F, 8, 4, 4, 5), "x"); |
172 | 1 | ccv_nnc_tensor_symbol_t sum = ccv_nnc_tensor_symbol_new(symbolic_graph, CPU_TENSOR_NHWC(32F, 8, 4, 1, 5), "sum"); |
173 | 1 | ccv_nnc_tensor_symbol_t mean = ccv_nnc_tensor_symbol_new(symbolic_graph, CPU_TENSOR_NHWC(32F, 8, 4, 1, 5), "mean"); |
174 | 1 | ccv_nnc_tensor_symbol_t whitening = ccv_nnc_tensor_symbol_new(symbolic_graph, CPU_TENSOR_NHWC(32F, 8, 4, 4, 5), "whitening"); |
175 | 1 | ccv_nnc_tensor_symbol_t sqr = ccv_nnc_tensor_symbol_new(symbolic_graph, CPU_TENSOR_NHWC(32F, 8, 4, 4, 5), "sqr"); |
176 | 1 | ccv_nnc_tensor_symbol_t varsum = ccv_nnc_tensor_symbol_new(symbolic_graph, CPU_TENSOR_NHWC(32F, 8, 4, 1, 5), "varsum"); |
177 | 1 | ccv_nnc_tensor_symbol_t var = ccv_nnc_tensor_symbol_new(symbolic_graph, CPU_TENSOR_NHWC(32F, 8, 4, 1, 5), "var"); |
178 | 1 | ccv_nnc_tensor_symbol_t logvar = ccv_nnc_tensor_symbol_new(symbolic_graph, CPU_TENSOR_NHWC(32F, 8, 4, 1, 5), "logvar"); |
179 | 1 | ccv_nnc_tensor_symbol_t logvar_2 = ccv_nnc_tensor_symbol_new(symbolic_graph, CPU_TENSOR_NHWC(32F, 8, 4, 1, 5), "logvar"); |
180 | 1 | ccv_nnc_tensor_symbol_t std = ccv_nnc_tensor_symbol_new(symbolic_graph, CPU_TENSOR_NHWC(32F, 8, 4, 1, 5), "std"); |
181 | 1 | ccv_nnc_tensor_symbol_t inv_std = ccv_nnc_tensor_symbol_new(symbolic_graph, CPU_TENSOR_NHWC(32F, 8, 4, 1, 5), "inv_std"); |
182 | 1 | ccv_nnc_tensor_symbol_t y = ccv_nnc_tensor_symbol_new(symbolic_graph, CPU_TENSOR_NHWC(32F, 8, 4, 4, 5), "y"); |
183 | 1 | ccv_nnc_graph_exec_symbol_new(symbolic_graph, CMD_REDUCE_SUM_FORWARD(1), TENSOR_SYMBOL_LIST(x), TENSOR_SYMBOL_LIST(sum), "sum"); |
184 | 1 | ccv_nnc_graph_exec_symbol_new(symbolic_graph, CMD_SCALAR_MUL_FORWARD(1.0 / 4), TENSOR_SYMBOL_LIST(sum), TENSOR_SYMBOL_LIST(mean), "mean"); |
185 | 1 | ccv_nnc_graph_exec_symbol_new(symbolic_graph, CMD_ADD_FORWARD(1, -1), TENSOR_SYMBOL_LIST(x, mean), TENSOR_SYMBOL_LIST(whitening), "whitening"); |
186 | 1 | ccv_nnc_graph_exec_symbol_new(symbolic_graph, CMD_EWPROD_FORWARD(), TENSOR_SYMBOL_LIST(whitening, whitening), TENSOR_SYMBOL_LIST(sqr), "sqr"); |
187 | 1 | ccv_nnc_graph_exec_symbol_new(symbolic_graph, CMD_REDUCE_SUM_FORWARD(1), TENSOR_SYMBOL_LIST(sqr), TENSOR_SYMBOL_LIST(varsum), "varsum"); |
188 | 1 | ccv_nnc_graph_exec_symbol_new(symbolic_graph, CMD_SCALAR_MUL_FORWARD(1.0 / 4), TENSOR_SYMBOL_LIST(varsum), TENSOR_SYMBOL_LIST(var), "var"); |
189 | 1 | ccv_nnc_graph_exec_symbol_new(symbolic_graph, CMD_EWLOG_FORWARD(), TENSOR_SYMBOL_LIST(var), TENSOR_SYMBOL_LIST(logvar), "log(var)"); |
190 | 1 | ccv_nnc_graph_exec_symbol_new(symbolic_graph, CMD_SCALAR_MUL_FORWARD(0.5), TENSOR_SYMBOL_LIST(logvar), TENSOR_SYMBOL_LIST(logvar_2), "log(var)/2"); |
191 | 1 | ccv_nnc_graph_exec_symbol_new(symbolic_graph, CMD_EWEXP_FORWARD(), TENSOR_SYMBOL_LIST(logvar_2), TENSOR_SYMBOL_LIST(std), "std"); |
192 | 1 | ccv_nnc_graph_exec_symbol_new(symbolic_graph, CMD_EWDIV_FORWARD(), TENSOR_SYMBOL_LIST(NO_TENSOR_SYMBOL, std), TENSOR_SYMBOL_LIST(inv_std), "1/std"); |
193 | 1 | ccv_nnc_graph_exec_symbol_new(symbolic_graph, CMD_MUL_FORWARD(1), TENSOR_SYMBOL_LIST(whitening, inv_std), TENSOR_SYMBOL_LIST(y), "y"); |
194 | 1 | ccv_nnc_graph_exec_symbol_autogen(symbolic_graph, 0, 0, CCV_NNC_AUTOGEN_ALL_EXECS | CCV_NNC_AUTOGEN_SOURCES_AND_DESTINATIONS); |
195 | 1 | SYMBOLIC_GRAPH_GEN(symbolic_graph, CCV_NNC_LONG_DOT_GRAPH); |
196 | 1 | ccv_nnc_graph_t* graph = 0; |
197 | 1 | ccv_nnc_tensor_arena_t* tensor_arena = 0; |
198 | 1 | ccv_nnc_graph_exec_arena_t* graph_exec_arena = 0; |
199 | 1 | ccv_nnc_symbolic_graph_compile(symbolic_graph, ccv_nnc_default_compile_params, 0, 0, 0, 0, SYMBOLIC_GRAPH_SOURCES(symbolic_graph), SYMBOLIC_GRAPH_DESTINATIONS(symbolic_graph), &graph, &tensor_arena, &graph_exec_arena); |
200 | 1 | GRAPH_GEN(graph, CCV_NNC_LONG_DOT_GRAPH); |
201 | 1 | ccv_nnc_tensor_t* const x_tensor = ccv_nnc_tensor_from_symbol(tensor_arena, x); |
202 | 1 | dsfmt_t dsfmt; |
203 | 1 | int i; |
204 | 1 | dsfmt_init_gen_rand(&dsfmt, 1); |
205 | 641 | for (i = 0; i < 8 * 4 * 4 * 5; i++640 ) |
206 | 640 | x_tensor->data.f32[i] = dsfmt_genrand_open_close(&dsfmt); |
207 | 1 | ccv_nnc_symbolic_graph_t* const group_norm_symbolic_graph = ccv_nnc_symbolic_graph_new(); |
208 | 1 | ccv_nnc_tensor_symbol_t bx = ccv_nnc_tensor_symbol_new(group_norm_symbolic_graph, CPU_TENSOR_NHWC(32F, 8, 16, 5), "x"); |
209 | 1 | ccv_nnc_tensor_symbol_t by = ccv_nnc_tensor_symbol_new(group_norm_symbolic_graph, CPU_TENSOR_NHWC(32F, 8, 16, 5), "y"); |
210 | 1 | ccv_nnc_tensor_symbol_t scale = ccv_nnc_tensor_symbol_new(group_norm_symbolic_graph, CPU_TENSOR_NHWC(32F, 1, 16, 5), "scale"); |
211 | 1 | ccv_nnc_tensor_symbol_t bias = ccv_nnc_tensor_symbol_new(group_norm_symbolic_graph, CPU_TENSOR_NHWC(32F, 1, 16, 5), "bias"); |
212 | 1 | ccv_nnc_tensor_symbol_t saved_mean = ccv_nnc_tensor_symbol_new(group_norm_symbolic_graph, CPU_TENSOR_NHWC(32F, 8, 4, 5), "saved_mean"); |
213 | 1 | ccv_nnc_tensor_symbol_t saved_inv_std = ccv_nnc_tensor_symbol_new(group_norm_symbolic_graph, CPU_TENSOR_NHWC(32F, 8, 4, 5), "saved_inv_std"); |
214 | 1 | ccv_nnc_graph_exec_symbol_new(group_norm_symbolic_graph, CMD_SET_FORWARD(1), 0, 0, TENSOR_SYMBOL_LIST(scale), "set_scale"); |
215 | 1 | ccv_nnc_graph_exec_symbol_new(group_norm_symbolic_graph, CMD_SET_FORWARD(0), 0, 0, TENSOR_SYMBOL_LIST(bias), "set_bias"); |
216 | 1 | ccv_nnc_graph_exec_symbol_new(group_norm_symbolic_graph, CMD_GROUP_NORM_FORWARD(1, 4, 0, 1), TENSOR_SYMBOL_LIST(bx, scale, bias), TENSOR_SYMBOL_LIST(by, saved_mean, saved_inv_std), "group_norm"); |
217 | 1 | ccv_nnc_graph_exec_symbol_autogen(group_norm_symbolic_graph, 0, 0, CCV_NNC_AUTOGEN_ALL_EXECS | CCV_NNC_AUTOGEN_SOURCES_AND_DESTINATIONS); |
218 | 1 | ccv_nnc_graph_t* group_norm_graph = 0; |
219 | 1 | ccv_nnc_tensor_arena_t* group_norm_tensor_arena = 0; |
220 | 1 | ccv_nnc_graph_exec_arena_t* group_norm_graph_exec_arena = 0; |
221 | 1 | ccv_nnc_symbolic_graph_compile(group_norm_symbolic_graph, ccv_nnc_default_compile_params, 0, 0, 0, 0, SYMBOLIC_GRAPH_SOURCES(group_norm_symbolic_graph), SYMBOLIC_GRAPH_DESTINATIONS(group_norm_symbolic_graph), &group_norm_graph, &group_norm_tensor_arena, &group_norm_graph_exec_arena); |
222 | 1 | ccv_nnc_tensor_t* const bx_tensor = ccv_nnc_tensor_from_symbol(group_norm_tensor_arena, bx); |
223 | 1 | memcpy(bx_tensor->data.f32, x_tensor->data.f32, sizeof(float) * 8 * 16 * 5); |
224 | 1 | ccv_nnc_graph_run(graph, 0, TRAVERSE_FULL, 0, 0); |
225 | 1 | ccv_nnc_graph_run(group_norm_graph, 0, TRAVERSE_FULL, 0, 0); |
226 | 1 | ccv_nnc_tensor_t* const y_tensor = ccv_nnc_tensor_from_symbol(tensor_arena, y); |
227 | 1 | ccv_nnc_tensor_t* const by_tensor = ccv_nnc_tensor_from_symbol(group_norm_tensor_arena, by); |
228 | 1 | REQUIRE_ARRAY_EQ_WITH_TOLERANCE(float, y_tensor->data.f32, by_tensor->data.f32, 8 * 16 * 5, 1e-5, "graph computed result should match batch norm op result"); |
229 | 1 | ccv_nnc_symbolic_graph_free(symbolic_graph); |
230 | 1 | ccv_nnc_tensor_arena_free(tensor_arena); |
231 | 1 | ccv_nnc_graph_exec_arena_free(graph_exec_arena); |
232 | 1 | ccv_nnc_graph_free(graph); |
233 | 1 | ccv_nnc_symbolic_graph_free(group_norm_symbolic_graph); |
234 | 1 | ccv_nnc_tensor_arena_free(group_norm_tensor_arena); |
235 | 1 | ccv_nnc_graph_exec_arena_free(group_norm_graph_exec_arena); |
236 | 1 | ccv_nnc_graph_free(group_norm_graph); |
237 | 1 | } |
238 | | |
239 | | TEST_CASE("compare group norm gradient with other symbolic graph") |
240 | 1 | { |
241 | 1 | ccv_nnc_symbolic_graph_t* const symbolic_graph = ccv_nnc_symbolic_graph_new(); |
242 | 1 | ccv_nnc_tensor_symbol_t x = ccv_nnc_tensor_symbol_new(symbolic_graph, CPU_TENSOR_NHWC(32F, 8, 4, 4, 5), "x"); |
243 | 1 | ccv_nnc_tensor_symbol_t sum = ccv_nnc_tensor_symbol_new(symbolic_graph, CPU_TENSOR_NHWC(32F, 8, 4, 1, 5), "sum"); |
244 | 1 | ccv_nnc_tensor_symbol_t mean = ccv_nnc_tensor_symbol_new(symbolic_graph, CPU_TENSOR_NHWC(32F, 8, 4, 1, 5), "mean"); |
245 | 1 | ccv_nnc_tensor_symbol_t whitening = ccv_nnc_tensor_symbol_new(symbolic_graph, CPU_TENSOR_NHWC(32F, 8, 4, 4, 5), "whitening"); |
246 | 1 | ccv_nnc_tensor_symbol_t sqr = ccv_nnc_tensor_symbol_new(symbolic_graph, CPU_TENSOR_NHWC(32F, 8, 4, 4, 5), "sqr"); |
247 | 1 | ccv_nnc_tensor_symbol_t varsum = ccv_nnc_tensor_symbol_new(symbolic_graph, CPU_TENSOR_NHWC(32F, 8, 4, 1, 5), "varsum"); |
248 | 1 | ccv_nnc_tensor_symbol_t var = ccv_nnc_tensor_symbol_new(symbolic_graph, CPU_TENSOR_NHWC(32F, 8, 4, 1, 5), "var"); |
249 | 1 | ccv_nnc_tensor_symbol_t logvar = ccv_nnc_tensor_symbol_new(symbolic_graph, CPU_TENSOR_NHWC(32F, 8, 4, 1, 5), "logvar"); |
250 | 1 | ccv_nnc_tensor_symbol_t logvar_2 = ccv_nnc_tensor_symbol_new(symbolic_graph, CPU_TENSOR_NHWC(32F, 8, 4, 1, 5), "logvar"); |
251 | 1 | ccv_nnc_tensor_symbol_t std = ccv_nnc_tensor_symbol_new(symbolic_graph, CPU_TENSOR_NHWC(32F, 8, 4, 1, 5), "std"); |
252 | 1 | ccv_nnc_tensor_symbol_t inv_std = ccv_nnc_tensor_symbol_new(symbolic_graph, CPU_TENSOR_NHWC(32F, 8, 4, 1, 5), "inv_std"); |
253 | 1 | ccv_nnc_tensor_symbol_t y = ccv_nnc_tensor_symbol_new(symbolic_graph, CPU_TENSOR_NHWC(32F, 8, 4, 4, 5), "y"); |
254 | 1 | ccv_nnc_graph_exec_symbol_new(symbolic_graph, CMD_REDUCE_SUM_FORWARD(1), TENSOR_SYMBOL_LIST(x), TENSOR_SYMBOL_LIST(sum), "sum"); |
255 | 1 | ccv_nnc_graph_exec_symbol_new(symbolic_graph, CMD_SCALAR_MUL_FORWARD(1.0 / 4), TENSOR_SYMBOL_LIST(sum), TENSOR_SYMBOL_LIST(mean), "mean"); |
256 | 1 | ccv_nnc_graph_exec_symbol_new(symbolic_graph, CMD_ADD_FORWARD(1, -1), TENSOR_SYMBOL_LIST(x, mean), TENSOR_SYMBOL_LIST(whitening), "whitening"); |
257 | 1 | ccv_nnc_graph_exec_symbol_new(symbolic_graph, CMD_EWPROD_FORWARD(), TENSOR_SYMBOL_LIST(whitening, whitening), TENSOR_SYMBOL_LIST(sqr), "sqr"); |
258 | 1 | ccv_nnc_graph_exec_symbol_new(symbolic_graph, CMD_REDUCE_SUM_FORWARD(1), TENSOR_SYMBOL_LIST(sqr), TENSOR_SYMBOL_LIST(varsum), "varsum"); |
259 | 1 | ccv_nnc_graph_exec_symbol_new(symbolic_graph, CMD_SCALAR_MUL_FORWARD(1.0 / 4), TENSOR_SYMBOL_LIST(varsum), TENSOR_SYMBOL_LIST(var), "var"); |
260 | 1 | ccv_nnc_graph_exec_symbol_new(symbolic_graph, CMD_EWLOG_FORWARD(), TENSOR_SYMBOL_LIST(var), TENSOR_SYMBOL_LIST(logvar), "log(var)"); |
261 | 1 | ccv_nnc_graph_exec_symbol_new(symbolic_graph, CMD_SCALAR_MUL_FORWARD(0.5), TENSOR_SYMBOL_LIST(logvar), TENSOR_SYMBOL_LIST(logvar_2), "log(var)/2"); |
262 | 1 | ccv_nnc_graph_exec_symbol_new(symbolic_graph, CMD_EWEXP_FORWARD(), TENSOR_SYMBOL_LIST(logvar_2), TENSOR_SYMBOL_LIST(std), "std"); |
263 | 1 | ccv_nnc_graph_exec_symbol_new(symbolic_graph, CMD_EWDIV_FORWARD(), TENSOR_SYMBOL_LIST(NO_TENSOR_SYMBOL, std), TENSOR_SYMBOL_LIST(inv_std), "1/std"); |
264 | 1 | ccv_nnc_graph_exec_symbol_new(symbolic_graph, CMD_MUL_FORWARD(1), TENSOR_SYMBOL_LIST(whitening, inv_std), TENSOR_SYMBOL_LIST(y), "y"); |
265 | 1 | ccv_nnc_graph_exec_symbol_autogen(symbolic_graph, 0, 0, CCV_NNC_AUTOGEN_ALL_EXECS | CCV_NNC_AUTOGEN_SOURCES_AND_DESTINATIONS); |
266 | 1 | ccv_nnc_symbolic_graph_backward(symbolic_graph, TENSOR_SYMBOL_LIST(y), TENSOR_SYMBOL_LIST(x), SYMBOLIC_GRAPH_SOURCES(symbolic_graph), SYMBOLIC_GRAPH_DESTINATIONS(symbolic_graph)); |
267 | 1 | ccv_nnc_graph_exec_symbol_autogen(symbolic_graph, 0, 0, CCV_NNC_AUTOGEN_ALL_EXECS | CCV_NNC_AUTOGEN_SOURCES_AND_DESTINATIONS); |
268 | 1 | ccv_nnc_tensor_symbol_t dy = ccv_nnc_tensor_symbol_for_backward(symbolic_graph, y); |
269 | 1 | ccv_nnc_tensor_symbol_t dx = ccv_nnc_tensor_symbol_for_backward(symbolic_graph, x); |
270 | 1 | SYMBOLIC_GRAPH_GEN(symbolic_graph, CCV_NNC_LONG_DOT_GRAPH); |
271 | 1 | ccv_nnc_graph_t* graph = 0; |
272 | 1 | ccv_nnc_tensor_arena_t* tensor_arena = 0; |
273 | 1 | ccv_nnc_graph_exec_arena_t* graph_exec_arena = 0; |
274 | 1 | ccv_nnc_symbolic_graph_compile(symbolic_graph, ccv_nnc_default_compile_params, 0, 0, 0, 0, SYMBOLIC_GRAPH_SOURCES(symbolic_graph), SYMBOLIC_GRAPH_DESTINATIONS(symbolic_graph), &graph, &tensor_arena, &graph_exec_arena); |
275 | 1 | GRAPH_GEN(graph, CCV_NNC_LONG_DOT_GRAPH); |
276 | 1 | ccv_nnc_tensor_t* const x_tensor = ccv_nnc_tensor_from_symbol(tensor_arena, x); |
277 | 1 | dsfmt_t dsfmt; |
278 | 1 | int i; |
279 | 1 | dsfmt_init_gen_rand(&dsfmt, 1); |
280 | 641 | for (i = 0; i < 8 * 4 * 4 * 5; i++640 ) |
281 | 640 | x_tensor->data.f32[i] = dsfmt_genrand_open_close(&dsfmt); |
282 | 1 | ccv_nnc_symbolic_graph_t* const group_norm_symbolic_graph = ccv_nnc_symbolic_graph_new(); |
283 | 1 | ccv_nnc_tensor_symbol_t bx = ccv_nnc_tensor_symbol_new(group_norm_symbolic_graph, CPU_TENSOR_NHWC(32F, 8, 16, 5), "x"); |
284 | 1 | ccv_nnc_tensor_symbol_t by = ccv_nnc_tensor_symbol_new(group_norm_symbolic_graph, CPU_TENSOR_NHWC(32F, 8, 16, 5), "y"); |
285 | 1 | ccv_nnc_tensor_symbol_t scale = ccv_nnc_tensor_symbol_new(group_norm_symbolic_graph, CPU_TENSOR_NHWC(32F, 1, 16, 5), "scale"); |
286 | 1 | ccv_nnc_tensor_symbol_t bias = ccv_nnc_tensor_symbol_new(group_norm_symbolic_graph, CPU_TENSOR_NHWC(32F, 1, 16, 5), "bias"); |
287 | 1 | ccv_nnc_tensor_symbol_t saved_mean = ccv_nnc_tensor_symbol_new(group_norm_symbolic_graph, CPU_TENSOR_NHWC(32F, 8, 4, 5), "saved_mean"); |
288 | 1 | ccv_nnc_tensor_symbol_t saved_inv_std = ccv_nnc_tensor_symbol_new(group_norm_symbolic_graph, CPU_TENSOR_NHWC(32F, 8, 4, 5), "saved_inv_std"); |
289 | 1 | ccv_nnc_graph_exec_symbol_new(group_norm_symbolic_graph, CMD_SET_FORWARD(1), 0, 0, TENSOR_SYMBOL_LIST(scale), "set_scale"); |
290 | 1 | ccv_nnc_graph_exec_symbol_new(group_norm_symbolic_graph, CMD_SET_FORWARD(0), 0, 0, TENSOR_SYMBOL_LIST(bias), "set_bias"); |
291 | 1 | ccv_nnc_graph_exec_symbol_new(group_norm_symbolic_graph, CMD_GROUP_NORM_FORWARD(1, 4, 0, 1), TENSOR_SYMBOL_LIST(bx, scale, bias), TENSOR_SYMBOL_LIST(by, saved_mean, saved_inv_std), "group_norm"); |
292 | 1 | ccv_nnc_graph_exec_symbol_autogen(group_norm_symbolic_graph, 0, 0, CCV_NNC_AUTOGEN_ALL_EXECS | CCV_NNC_AUTOGEN_SOURCES_AND_DESTINATIONS); |
293 | 1 | ccv_nnc_symbolic_graph_backward(group_norm_symbolic_graph, TENSOR_SYMBOL_LIST(by), TENSOR_SYMBOL_LIST(bx, scale, bias), SYMBOLIC_GRAPH_SOURCES(group_norm_symbolic_graph), SYMBOLIC_GRAPH_DESTINATIONS(group_norm_symbolic_graph)); |
294 | 1 | ccv_nnc_graph_exec_symbol_autogen(group_norm_symbolic_graph, 0, 0, CCV_NNC_AUTOGEN_ALL_EXECS | CCV_NNC_AUTOGEN_SOURCES_AND_DESTINATIONS); |
295 | 1 | ccv_nnc_tensor_symbol_t dby = ccv_nnc_tensor_symbol_for_backward(group_norm_symbolic_graph, by); |
296 | 1 | ccv_nnc_tensor_symbol_t dbx = ccv_nnc_tensor_symbol_for_backward(group_norm_symbolic_graph, bx); |
297 | 1 | ccv_nnc_graph_t* group_norm_graph = 0; |
298 | 1 | ccv_nnc_tensor_arena_t* group_norm_tensor_arena = 0; |
299 | 1 | ccv_nnc_graph_exec_arena_t* group_norm_graph_exec_arena = 0; |
300 | 1 | ccv_nnc_symbolic_graph_compile(group_norm_symbolic_graph, ccv_nnc_default_compile_params, 0, 0, 0, 0, SYMBOLIC_GRAPH_SOURCES(group_norm_symbolic_graph), SYMBOLIC_GRAPH_DESTINATIONS(group_norm_symbolic_graph), &group_norm_graph, &group_norm_tensor_arena, &group_norm_graph_exec_arena); |
301 | 1 | ccv_nnc_tensor_t* const bx_tensor = ccv_nnc_tensor_from_symbol(group_norm_tensor_arena, bx); |
302 | 1 | ccv_nnc_tensor_t* const dy_tensor = ccv_nnc_tensor_from_symbol(tensor_arena, dy); |
303 | 1 | ccv_nnc_tensor_t* const dby_tensor = ccv_nnc_tensor_from_symbol(group_norm_tensor_arena, dby); |
304 | 641 | for (i = 0; i < 8 * 16 * 5; i++640 ) |
305 | 640 | dby_tensor->data.f32[i] = dy_tensor->data.f32[i] = dsfmt_genrand_open_close(&dsfmt) * 2 - 1; |
306 | 1 | memcpy(bx_tensor->data.f32, x_tensor->data.f32, sizeof(float) * 8 * 16 * 5); |
307 | 1 | ccv_nnc_graph_run(graph, 0, TRAVERSE_FULL, 0, 0); |
308 | 1 | ccv_nnc_graph_run(group_norm_graph, 0, TRAVERSE_FULL, 0, 0); |
309 | 1 | ccv_nnc_tensor_t* const dx_tensor = ccv_nnc_tensor_from_symbol(tensor_arena, dx); |
310 | 1 | ccv_nnc_tensor_t* const dbx_tensor = ccv_nnc_tensor_from_symbol(group_norm_tensor_arena, dbx); |
311 | 1 | REQUIRE_ARRAY_EQ_WITH_TOLERANCE(float, dx_tensor->data.f32, dbx_tensor->data.f32, 8 * 16 * 5, 1e-5, "graph computed result should match layer norm op result"); |
312 | 1 | ccv_nnc_symbolic_graph_free(symbolic_graph); |
313 | 1 | ccv_nnc_tensor_arena_free(tensor_arena); |
314 | 1 | ccv_nnc_graph_exec_arena_free(graph_exec_arena); |
315 | 1 | ccv_nnc_graph_free(graph); |
316 | 1 | ccv_nnc_symbolic_graph_free(group_norm_symbolic_graph); |
317 | 1 | ccv_nnc_tensor_arena_free(group_norm_tensor_arena); |
318 | 1 | ccv_nnc_graph_exec_arena_free(group_norm_graph_exec_arena); |
319 | 1 | ccv_nnc_graph_free(group_norm_graph); |
320 | 1 | } |
321 | | |
322 | | TEST_CASE("implement group norm with other symbolic graph without scale / bias") |
323 | 1 | { |
324 | 1 | ccv_nnc_symbolic_graph_t* const symbolic_graph = ccv_nnc_symbolic_graph_new(); |
325 | 1 | ccv_nnc_tensor_symbol_t x = ccv_nnc_tensor_symbol_new(symbolic_graph, CPU_TENSOR_NHWC(32F, 8, 4, 4, 5), "x"); |
326 | 1 | ccv_nnc_tensor_symbol_t sum = ccv_nnc_tensor_symbol_new(symbolic_graph, CPU_TENSOR_NHWC(32F, 8, 4, 1, 5), "sum"); |
327 | 1 | ccv_nnc_tensor_symbol_t mean = ccv_nnc_tensor_symbol_new(symbolic_graph, CPU_TENSOR_NHWC(32F, 8, 4, 1, 5), "mean"); |
328 | 1 | ccv_nnc_tensor_symbol_t whitening = ccv_nnc_tensor_symbol_new(symbolic_graph, CPU_TENSOR_NHWC(32F, 8, 4, 4, 5), "whitening"); |
329 | 1 | ccv_nnc_tensor_symbol_t sqr = ccv_nnc_tensor_symbol_new(symbolic_graph, CPU_TENSOR_NHWC(32F, 8, 4, 4, 5), "sqr"); |
330 | 1 | ccv_nnc_tensor_symbol_t varsum = ccv_nnc_tensor_symbol_new(symbolic_graph, CPU_TENSOR_NHWC(32F, 8, 4, 1, 5), "varsum"); |
331 | 1 | ccv_nnc_tensor_symbol_t var = ccv_nnc_tensor_symbol_new(symbolic_graph, CPU_TENSOR_NHWC(32F, 8, 4, 1, 5), "var"); |
332 | 1 | ccv_nnc_tensor_symbol_t logvar = ccv_nnc_tensor_symbol_new(symbolic_graph, CPU_TENSOR_NHWC(32F, 8, 4, 1, 5), "logvar"); |
333 | 1 | ccv_nnc_tensor_symbol_t logvar_2 = ccv_nnc_tensor_symbol_new(symbolic_graph, CPU_TENSOR_NHWC(32F, 8, 4, 1, 5), "logvar"); |
334 | 1 | ccv_nnc_tensor_symbol_t std = ccv_nnc_tensor_symbol_new(symbolic_graph, CPU_TENSOR_NHWC(32F, 8, 4, 1, 5), "std"); |
335 | 1 | ccv_nnc_tensor_symbol_t inv_std = ccv_nnc_tensor_symbol_new(symbolic_graph, CPU_TENSOR_NHWC(32F, 8, 4, 1, 5), "inv_std"); |
336 | 1 | ccv_nnc_tensor_symbol_t y = ccv_nnc_tensor_symbol_new(symbolic_graph, CPU_TENSOR_NHWC(32F, 8, 4, 4, 5), "y"); |
337 | 1 | ccv_nnc_graph_exec_symbol_new(symbolic_graph, CMD_REDUCE_SUM_FORWARD(1), TENSOR_SYMBOL_LIST(x), TENSOR_SYMBOL_LIST(sum), "sum"); |
338 | 1 | ccv_nnc_graph_exec_symbol_new(symbolic_graph, CMD_SCALAR_MUL_FORWARD(1.0 / 4), TENSOR_SYMBOL_LIST(sum), TENSOR_SYMBOL_LIST(mean), "mean"); |
339 | 1 | ccv_nnc_graph_exec_symbol_new(symbolic_graph, CMD_ADD_FORWARD(1, -1), TENSOR_SYMBOL_LIST(x, mean), TENSOR_SYMBOL_LIST(whitening), "whitening"); |
340 | 1 | ccv_nnc_graph_exec_symbol_new(symbolic_graph, CMD_EWPROD_FORWARD(), TENSOR_SYMBOL_LIST(whitening, whitening), TENSOR_SYMBOL_LIST(sqr), "sqr"); |
341 | 1 | ccv_nnc_graph_exec_symbol_new(symbolic_graph, CMD_REDUCE_SUM_FORWARD(1), TENSOR_SYMBOL_LIST(sqr), TENSOR_SYMBOL_LIST(varsum), "varsum"); |
342 | 1 | ccv_nnc_graph_exec_symbol_new(symbolic_graph, CMD_SCALAR_MUL_FORWARD(1.0 / 4), TENSOR_SYMBOL_LIST(varsum), TENSOR_SYMBOL_LIST(var), "var"); |
343 | 1 | ccv_nnc_graph_exec_symbol_new(symbolic_graph, CMD_EWLOG_FORWARD(), TENSOR_SYMBOL_LIST(var), TENSOR_SYMBOL_LIST(logvar), "log(var)"); |
344 | 1 | ccv_nnc_graph_exec_symbol_new(symbolic_graph, CMD_SCALAR_MUL_FORWARD(0.5), TENSOR_SYMBOL_LIST(logvar), TENSOR_SYMBOL_LIST(logvar_2), "log(var)/2"); |
345 | 1 | ccv_nnc_graph_exec_symbol_new(symbolic_graph, CMD_EWEXP_FORWARD(), TENSOR_SYMBOL_LIST(logvar_2), TENSOR_SYMBOL_LIST(std), "std"); |
346 | 1 | ccv_nnc_graph_exec_symbol_new(symbolic_graph, CMD_EWDIV_FORWARD(), TENSOR_SYMBOL_LIST(NO_TENSOR_SYMBOL, std), TENSOR_SYMBOL_LIST(inv_std), "1/std"); |
347 | 1 | ccv_nnc_graph_exec_symbol_new(symbolic_graph, CMD_MUL_FORWARD(1), TENSOR_SYMBOL_LIST(whitening, inv_std), TENSOR_SYMBOL_LIST(y), "y"); |
348 | 1 | ccv_nnc_graph_exec_symbol_autogen(symbolic_graph, 0, 0, CCV_NNC_AUTOGEN_ALL_EXECS | CCV_NNC_AUTOGEN_SOURCES_AND_DESTINATIONS); |
349 | 1 | SYMBOLIC_GRAPH_GEN(symbolic_graph, CCV_NNC_LONG_DOT_GRAPH); |
350 | 1 | ccv_nnc_graph_t* graph = 0; |
351 | 1 | ccv_nnc_tensor_arena_t* tensor_arena = 0; |
352 | 1 | ccv_nnc_graph_exec_arena_t* graph_exec_arena = 0; |
353 | 1 | ccv_nnc_symbolic_graph_compile(symbolic_graph, ccv_nnc_default_compile_params, 0, 0, 0, 0, SYMBOLIC_GRAPH_SOURCES(symbolic_graph), SYMBOLIC_GRAPH_DESTINATIONS(symbolic_graph), &graph, &tensor_arena, &graph_exec_arena); |
354 | 1 | GRAPH_GEN(graph, CCV_NNC_LONG_DOT_GRAPH); |
355 | 1 | ccv_nnc_tensor_t* const x_tensor = ccv_nnc_tensor_from_symbol(tensor_arena, x); |
356 | 1 | dsfmt_t dsfmt; |
357 | 1 | int i; |
358 | 1 | dsfmt_init_gen_rand(&dsfmt, 1); |
359 | 641 | for (i = 0; i < 8 * 4 * 4 * 5; i++640 ) |
360 | 640 | x_tensor->data.f32[i] = dsfmt_genrand_open_close(&dsfmt); |
361 | 1 | ccv_nnc_symbolic_graph_t* const group_norm_symbolic_graph = ccv_nnc_symbolic_graph_new(); |
362 | 1 | ccv_nnc_tensor_symbol_t bx = ccv_nnc_tensor_symbol_new(group_norm_symbolic_graph, CPU_TENSOR_NHWC(32F, 8, 16, 5), "x"); |
363 | 1 | ccv_nnc_tensor_symbol_t by = ccv_nnc_tensor_symbol_new(group_norm_symbolic_graph, CPU_TENSOR_NHWC(32F, 8, 16, 5), "y"); |
364 | 1 | ccv_nnc_tensor_symbol_t saved_mean = ccv_nnc_tensor_symbol_new(group_norm_symbolic_graph, CPU_TENSOR_NHWC(32F, 8, 4, 5), "saved_mean"); |
365 | 1 | ccv_nnc_tensor_symbol_t saved_inv_std = ccv_nnc_tensor_symbol_new(group_norm_symbolic_graph, CPU_TENSOR_NHWC(32F, 8, 4, 5), "saved_inv_std"); |
366 | 1 | ccv_nnc_graph_exec_symbol_new(group_norm_symbolic_graph, CMD_GROUP_NORM_FORWARD(1, 4, 0, 0), TENSOR_SYMBOL_LIST(bx), TENSOR_SYMBOL_LIST(by, saved_mean, saved_inv_std), "group_norm"); |
367 | 1 | ccv_nnc_graph_exec_symbol_autogen(group_norm_symbolic_graph, 0, 0, CCV_NNC_AUTOGEN_ALL_EXECS | CCV_NNC_AUTOGEN_SOURCES_AND_DESTINATIONS); |
368 | 1 | ccv_nnc_graph_t* group_norm_graph = 0; |
369 | 1 | ccv_nnc_tensor_arena_t* group_norm_tensor_arena = 0; |
370 | 1 | ccv_nnc_graph_exec_arena_t* group_norm_graph_exec_arena = 0; |
371 | 1 | ccv_nnc_symbolic_graph_compile(group_norm_symbolic_graph, ccv_nnc_default_compile_params, 0, 0, 0, 0, SYMBOLIC_GRAPH_SOURCES(group_norm_symbolic_graph), SYMBOLIC_GRAPH_DESTINATIONS(group_norm_symbolic_graph), &group_norm_graph, &group_norm_tensor_arena, &group_norm_graph_exec_arena); |
372 | 1 | ccv_nnc_tensor_t* const bx_tensor = ccv_nnc_tensor_from_symbol(group_norm_tensor_arena, bx); |
373 | 1 | memcpy(bx_tensor->data.f32, x_tensor->data.f32, sizeof(float) * 8 * 16 * 5); |
374 | 1 | ccv_nnc_graph_run(graph, 0, TRAVERSE_FULL, 0, 0); |
375 | 1 | ccv_nnc_graph_run(group_norm_graph, 0, TRAVERSE_FULL, 0, 0); |
376 | 1 | ccv_nnc_tensor_t* const y_tensor = ccv_nnc_tensor_from_symbol(tensor_arena, y); |
377 | 1 | ccv_nnc_tensor_t* const by_tensor = ccv_nnc_tensor_from_symbol(group_norm_tensor_arena, by); |
378 | 1 | REQUIRE_ARRAY_EQ_WITH_TOLERANCE(float, y_tensor->data.f32, by_tensor->data.f32, 8 * 16 * 5, 1e-5, "graph computed result should match batch norm op result"); |
379 | 1 | ccv_nnc_symbolic_graph_free(symbolic_graph); |
380 | 1 | ccv_nnc_tensor_arena_free(tensor_arena); |
381 | 1 | ccv_nnc_graph_exec_arena_free(graph_exec_arena); |
382 | 1 | ccv_nnc_graph_free(graph); |
383 | 1 | ccv_nnc_symbolic_graph_free(group_norm_symbolic_graph); |
384 | 1 | ccv_nnc_tensor_arena_free(group_norm_tensor_arena); |
385 | 1 | ccv_nnc_graph_exec_arena_free(group_norm_graph_exec_arena); |
386 | 1 | ccv_nnc_graph_free(group_norm_graph); |
387 | 1 | } |
388 | | |
389 | | TEST_CASE("compare group norm gradient with other symbolic graph without scale / bias") |
390 | 1 | { |
391 | 1 | ccv_nnc_symbolic_graph_t* const symbolic_graph = ccv_nnc_symbolic_graph_new(); |
392 | 1 | ccv_nnc_tensor_symbol_t x = ccv_nnc_tensor_symbol_new(symbolic_graph, CPU_TENSOR_NHWC(32F, 8, 4, 4, 5), "x"); |
393 | 1 | ccv_nnc_tensor_symbol_t sum = ccv_nnc_tensor_symbol_new(symbolic_graph, CPU_TENSOR_NHWC(32F, 8, 4, 1, 5), "sum"); |
394 | 1 | ccv_nnc_tensor_symbol_t mean = ccv_nnc_tensor_symbol_new(symbolic_graph, CPU_TENSOR_NHWC(32F, 8, 4, 1, 5), "mean"); |
395 | 1 | ccv_nnc_tensor_symbol_t whitening = ccv_nnc_tensor_symbol_new(symbolic_graph, CPU_TENSOR_NHWC(32F, 8, 4, 4, 5), "whitening"); |
396 | 1 | ccv_nnc_tensor_symbol_t sqr = ccv_nnc_tensor_symbol_new(symbolic_graph, CPU_TENSOR_NHWC(32F, 8, 4, 4, 5), "sqr"); |
397 | 1 | ccv_nnc_tensor_symbol_t varsum = ccv_nnc_tensor_symbol_new(symbolic_graph, CPU_TENSOR_NHWC(32F, 8, 4, 1, 5), "varsum"); |
398 | 1 | ccv_nnc_tensor_symbol_t var = ccv_nnc_tensor_symbol_new(symbolic_graph, CPU_TENSOR_NHWC(32F, 8, 4, 1, 5), "var"); |
399 | 1 | ccv_nnc_tensor_symbol_t logvar = ccv_nnc_tensor_symbol_new(symbolic_graph, CPU_TENSOR_NHWC(32F, 8, 4, 1, 5), "logvar"); |
400 | 1 | ccv_nnc_tensor_symbol_t logvar_2 = ccv_nnc_tensor_symbol_new(symbolic_graph, CPU_TENSOR_NHWC(32F, 8, 4, 1, 5), "logvar"); |
401 | 1 | ccv_nnc_tensor_symbol_t std = ccv_nnc_tensor_symbol_new(symbolic_graph, CPU_TENSOR_NHWC(32F, 8, 4, 1, 5), "std"); |
402 | 1 | ccv_nnc_tensor_symbol_t inv_std = ccv_nnc_tensor_symbol_new(symbolic_graph, CPU_TENSOR_NHWC(32F, 8, 4, 1, 5), "inv_std"); |
403 | 1 | ccv_nnc_tensor_symbol_t y = ccv_nnc_tensor_symbol_new(symbolic_graph, CPU_TENSOR_NHWC(32F, 8, 4, 4, 5), "y"); |
404 | 1 | ccv_nnc_graph_exec_symbol_new(symbolic_graph, CMD_REDUCE_SUM_FORWARD(1), TENSOR_SYMBOL_LIST(x), TENSOR_SYMBOL_LIST(sum), "sum"); |
405 | 1 | ccv_nnc_graph_exec_symbol_new(symbolic_graph, CMD_SCALAR_MUL_FORWARD(1.0 / 4), TENSOR_SYMBOL_LIST(sum), TENSOR_SYMBOL_LIST(mean), "mean"); |
406 | 1 | ccv_nnc_graph_exec_symbol_new(symbolic_graph, CMD_ADD_FORWARD(1, -1), TENSOR_SYMBOL_LIST(x, mean), TENSOR_SYMBOL_LIST(whitening), "whitening"); |
407 | 1 | ccv_nnc_graph_exec_symbol_new(symbolic_graph, CMD_EWPROD_FORWARD(), TENSOR_SYMBOL_LIST(whitening, whitening), TENSOR_SYMBOL_LIST(sqr), "sqr"); |
408 | 1 | ccv_nnc_graph_exec_symbol_new(symbolic_graph, CMD_REDUCE_SUM_FORWARD(1), TENSOR_SYMBOL_LIST(sqr), TENSOR_SYMBOL_LIST(varsum), "varsum"); |
409 | 1 | ccv_nnc_graph_exec_symbol_new(symbolic_graph, CMD_SCALAR_MUL_FORWARD(1.0 / 4), TENSOR_SYMBOL_LIST(varsum), TENSOR_SYMBOL_LIST(var), "var"); |
410 | 1 | ccv_nnc_graph_exec_symbol_new(symbolic_graph, CMD_EWLOG_FORWARD(), TENSOR_SYMBOL_LIST(var), TENSOR_SYMBOL_LIST(logvar), "log(var)"); |
411 | 1 | ccv_nnc_graph_exec_symbol_new(symbolic_graph, CMD_SCALAR_MUL_FORWARD(0.5), TENSOR_SYMBOL_LIST(logvar), TENSOR_SYMBOL_LIST(logvar_2), "log(var)/2"); |
412 | 1 | ccv_nnc_graph_exec_symbol_new(symbolic_graph, CMD_EWEXP_FORWARD(), TENSOR_SYMBOL_LIST(logvar_2), TENSOR_SYMBOL_LIST(std), "std"); |
413 | 1 | ccv_nnc_graph_exec_symbol_new(symbolic_graph, CMD_EWDIV_FORWARD(), TENSOR_SYMBOL_LIST(NO_TENSOR_SYMBOL, std), TENSOR_SYMBOL_LIST(inv_std), "1/std"); |
414 | 1 | ccv_nnc_graph_exec_symbol_new(symbolic_graph, CMD_MUL_FORWARD(1), TENSOR_SYMBOL_LIST(whitening, inv_std), TENSOR_SYMBOL_LIST(y), "y"); |
415 | 1 | ccv_nnc_graph_exec_symbol_autogen(symbolic_graph, 0, 0, CCV_NNC_AUTOGEN_ALL_EXECS | CCV_NNC_AUTOGEN_SOURCES_AND_DESTINATIONS); |
416 | 1 | ccv_nnc_symbolic_graph_backward(symbolic_graph, TENSOR_SYMBOL_LIST(y), TENSOR_SYMBOL_LIST(x), SYMBOLIC_GRAPH_SOURCES(symbolic_graph), SYMBOLIC_GRAPH_DESTINATIONS(symbolic_graph)); |
417 | 1 | ccv_nnc_graph_exec_symbol_autogen(symbolic_graph, 0, 0, CCV_NNC_AUTOGEN_ALL_EXECS | CCV_NNC_AUTOGEN_SOURCES_AND_DESTINATIONS); |
418 | 1 | ccv_nnc_tensor_symbol_t dy = ccv_nnc_tensor_symbol_for_backward(symbolic_graph, y); |
419 | 1 | ccv_nnc_tensor_symbol_t dx = ccv_nnc_tensor_symbol_for_backward(symbolic_graph, x); |
420 | 1 | SYMBOLIC_GRAPH_GEN(symbolic_graph, CCV_NNC_LONG_DOT_GRAPH); |
421 | 1 | ccv_nnc_graph_t* graph = 0; |
422 | 1 | ccv_nnc_tensor_arena_t* tensor_arena = 0; |
423 | 1 | ccv_nnc_graph_exec_arena_t* graph_exec_arena = 0; |
424 | 1 | ccv_nnc_symbolic_graph_compile(symbolic_graph, ccv_nnc_default_compile_params, 0, 0, 0, 0, SYMBOLIC_GRAPH_SOURCES(symbolic_graph), SYMBOLIC_GRAPH_DESTINATIONS(symbolic_graph), &graph, &tensor_arena, &graph_exec_arena); |
425 | 1 | GRAPH_GEN(graph, CCV_NNC_LONG_DOT_GRAPH); |
426 | 1 | ccv_nnc_tensor_t* const x_tensor = ccv_nnc_tensor_from_symbol(tensor_arena, x); |
427 | 1 | dsfmt_t dsfmt; |
428 | 1 | int i; |
429 | 1 | dsfmt_init_gen_rand(&dsfmt, 1); |
430 | 641 | for (i = 0; i < 8 * 4 * 4 * 5; i++640 ) |
431 | 640 | x_tensor->data.f32[i] = dsfmt_genrand_open_close(&dsfmt); |
432 | 1 | ccv_nnc_symbolic_graph_t* const group_norm_symbolic_graph = ccv_nnc_symbolic_graph_new(); |
433 | 1 | ccv_nnc_tensor_symbol_t bx = ccv_nnc_tensor_symbol_new(group_norm_symbolic_graph, CPU_TENSOR_NHWC(32F, 8, 16, 5), "x"); |
434 | 1 | ccv_nnc_tensor_symbol_t by = ccv_nnc_tensor_symbol_new(group_norm_symbolic_graph, CPU_TENSOR_NHWC(32F, 8, 16, 5), "y"); |
435 | 1 | ccv_nnc_tensor_symbol_t saved_mean = ccv_nnc_tensor_symbol_new(group_norm_symbolic_graph, CPU_TENSOR_NHWC(32F, 8, 4, 5), "saved_mean"); |
436 | 1 | ccv_nnc_tensor_symbol_t saved_inv_std = ccv_nnc_tensor_symbol_new(group_norm_symbolic_graph, CPU_TENSOR_NHWC(32F, 8, 4, 5), "saved_inv_std"); |
437 | 1 | ccv_nnc_graph_exec_symbol_new(group_norm_symbolic_graph, CMD_GROUP_NORM_FORWARD(1, 4, 0, 0), TENSOR_SYMBOL_LIST(bx), TENSOR_SYMBOL_LIST(by, saved_mean, saved_inv_std), "group_norm"); |
438 | 1 | ccv_nnc_graph_exec_symbol_autogen(group_norm_symbolic_graph, 0, 0, CCV_NNC_AUTOGEN_ALL_EXECS | CCV_NNC_AUTOGEN_SOURCES_AND_DESTINATIONS); |
439 | 1 | ccv_nnc_symbolic_graph_backward(group_norm_symbolic_graph, TENSOR_SYMBOL_LIST(by), TENSOR_SYMBOL_LIST(bx), SYMBOLIC_GRAPH_SOURCES(group_norm_symbolic_graph), SYMBOLIC_GRAPH_DESTINATIONS(group_norm_symbolic_graph)); |
440 | 1 | ccv_nnc_graph_exec_symbol_autogen(group_norm_symbolic_graph, 0, 0, CCV_NNC_AUTOGEN_ALL_EXECS | CCV_NNC_AUTOGEN_SOURCES_AND_DESTINATIONS); |
441 | 1 | ccv_nnc_tensor_symbol_t dby = ccv_nnc_tensor_symbol_for_backward(group_norm_symbolic_graph, by); |
442 | 1 | ccv_nnc_tensor_symbol_t dbx = ccv_nnc_tensor_symbol_for_backward(group_norm_symbolic_graph, bx); |
443 | 1 | ccv_nnc_graph_t* group_norm_graph = 0; |
444 | 1 | ccv_nnc_tensor_arena_t* group_norm_tensor_arena = 0; |
445 | 1 | ccv_nnc_graph_exec_arena_t* group_norm_graph_exec_arena = 0; |
446 | 1 | ccv_nnc_symbolic_graph_compile(group_norm_symbolic_graph, ccv_nnc_default_compile_params, 0, 0, 0, 0, SYMBOLIC_GRAPH_SOURCES(group_norm_symbolic_graph), SYMBOLIC_GRAPH_DESTINATIONS(group_norm_symbolic_graph), &group_norm_graph, &group_norm_tensor_arena, &group_norm_graph_exec_arena); |
447 | 1 | ccv_nnc_tensor_t* const bx_tensor = ccv_nnc_tensor_from_symbol(group_norm_tensor_arena, bx); |
448 | 1 | ccv_nnc_tensor_t* const dy_tensor = ccv_nnc_tensor_from_symbol(tensor_arena, dy); |
449 | 1 | ccv_nnc_tensor_t* const dby_tensor = ccv_nnc_tensor_from_symbol(group_norm_tensor_arena, dby); |
450 | 641 | for (i = 0; i < 8 * 16 * 5; i++640 ) |
451 | 640 | dby_tensor->data.f32[i] = dy_tensor->data.f32[i] = dsfmt_genrand_open_close(&dsfmt) * 2 - 1; |
452 | 1 | memcpy(bx_tensor->data.f32, x_tensor->data.f32, sizeof(float) * 8 * 16 * 5); |
453 | 1 | ccv_nnc_graph_run(graph, 0, TRAVERSE_FULL, 0, 0); |
454 | 1 | ccv_nnc_graph_run(group_norm_graph, 0, TRAVERSE_FULL, 0, 0); |
455 | 1 | ccv_nnc_tensor_t* const dx_tensor = ccv_nnc_tensor_from_symbol(tensor_arena, dx); |
456 | 1 | ccv_nnc_tensor_t* const dbx_tensor = ccv_nnc_tensor_from_symbol(group_norm_tensor_arena, dbx); |
457 | 1 | REQUIRE_ARRAY_EQ_WITH_TOLERANCE(float, dx_tensor->data.f32, dbx_tensor->data.f32, 8 * 16 * 5, 1e-5, "graph computed result should match layer norm op result"); |
458 | 1 | ccv_nnc_symbolic_graph_free(symbolic_graph); |
459 | 1 | ccv_nnc_tensor_arena_free(tensor_arena); |
460 | 1 | ccv_nnc_graph_exec_arena_free(graph_exec_arena); |
461 | 1 | ccv_nnc_graph_free(graph); |
462 | 1 | ccv_nnc_symbolic_graph_free(group_norm_symbolic_graph); |
463 | 1 | ccv_nnc_tensor_arena_free(group_norm_tensor_arena); |
464 | 1 | ccv_nnc_graph_exec_arena_free(group_norm_graph_exec_arena); |
465 | 1 | ccv_nnc_graph_free(group_norm_graph); |
466 | 1 | } |
467 | | |
468 | | #include "case_main.h" |