Coverage Report

Created: 2019-07-03 22:50

/home/liu/buildslave/linux-x64-runtests/build/test/unit/nnc/batch.norm.tests.c
Line
Count
Source
1
#include "case.h"
2
#include "ccv_case.h"
3
#include "ccv_nnc_case.h"
4
#include <ccv.h>
5
#include <nnc/ccv_nnc.h>
6
#include <nnc/ccv_nnc_easy.h>
7
#include "3rdparty/dsfmt/dSFMT.h"
8
9
TEST_SETUP()
10
{
11
  ccv_nnc_init();
12
}
13
14
TEST_CASE("implement batch norm with fine-grained symbolic graph")
15
1
{
16
1
  ccv_nnc_symbolic_graph_t* const symbolic_graph = ccv_nnc_symbolic_graph_new();
17
1
  ccv_nnc_tensor_symbol_t x = ccv_nnc_tensor_symbol_new(symbolic_graph, CPU_TENSOR_NHWC(32F, 8, 4, 4, 10), "x");
18
1
  ccv_nnc_tensor_symbol_t sum = ccv_nnc_tensor_symbol_new(symbolic_graph, CPU_TENSOR_NHWC(32F, 10), "sum");
19
1
  ccv_nnc_tensor_symbol_t mean = ccv_nnc_tensor_symbol_new(symbolic_graph, CPU_TENSOR_NHWC(32F, 10), "mean");
20
1
  ccv_nnc_tensor_symbol_t whitening = ccv_nnc_tensor_symbol_new(symbolic_graph, CPU_TENSOR_NHWC(32F, 8, 4, 4, 10), "whitening");
21
1
  ccv_nnc_tensor_symbol_t sqr = ccv_nnc_tensor_symbol_new(symbolic_graph, CPU_TENSOR_NHWC(32F, 8, 4, 4, 10), "sqr");
22
1
  ccv_nnc_tensor_symbol_t varsum = ccv_nnc_tensor_symbol_new(symbolic_graph, CPU_TENSOR_NHWC(32F, 10), "varsum");
23
1
  ccv_nnc_tensor_symbol_t var = ccv_nnc_tensor_symbol_new(symbolic_graph, CPU_TENSOR_NHWC(32F, 10), "var");
24
1
  ccv_nnc_tensor_symbol_t logvar = ccv_nnc_tensor_symbol_new(symbolic_graph, CPU_TENSOR_NHWC(32F, 10), "logvar");
25
1
  ccv_nnc_tensor_symbol_t logvar_2 = ccv_nnc_tensor_symbol_new(symbolic_graph, CPU_TENSOR_NHWC(32F, 10), "logvar");
26
1
  ccv_nnc_tensor_symbol_t std = ccv_nnc_tensor_symbol_new(symbolic_graph, CPU_TENSOR_NHWC(32F, 10), "std");
27
1
  ccv_nnc_tensor_symbol_t inv_std = ccv_nnc_tensor_symbol_new(symbolic_graph, CPU_TENSOR_NHWC(32F, 10), "inv_std");
28
1
  ccv_nnc_tensor_symbol_t y = ccv_nnc_tensor_symbol_new(symbolic_graph, CPU_TENSOR_NHWC(32F, 8, 4, 4, 10), "y");
29
1
  ccv_nnc_graph_exec_symbol_new(symbolic_graph, CMD_REDUCE_SUM_FORWARD(0, 1, 2), TENSOR_SYMBOL_LIST(x), TENSOR_SYMBOL_LIST(sum), "sum");
30
1
  ccv_nnc_graph_exec_symbol_new(symbolic_graph, CMD_SCALAR_MUL_FORWARD(1.0 / (8 * 4 * 4)), TENSOR_SYMBOL_LIST(sum), TENSOR_SYMBOL_LIST(mean), "mean");
31
1
  ccv_nnc_graph_exec_symbol_new(symbolic_graph, CMD_ADD_FORWARD(1, -1), TENSOR_SYMBOL_LIST(x, mean), TENSOR_SYMBOL_LIST(whitening), "whitening");
32
1
  ccv_nnc_graph_exec_symbol_new(symbolic_graph, CMD_EWPROD_FORWARD(), TENSOR_SYMBOL_LIST(whitening, whitening), TENSOR_SYMBOL_LIST(sqr), "sqr");
33
1
  ccv_nnc_graph_exec_symbol_new(symbolic_graph, CMD_REDUCE_SUM_FORWARD(0, 1, 2), TENSOR_SYMBOL_LIST(sqr), TENSOR_SYMBOL_LIST(varsum), "varsum");
34
1
  ccv_nnc_graph_exec_symbol_new(symbolic_graph, CMD_SCALAR_MUL_FORWARD(1.0 / (8 * 4 * 4)), TENSOR_SYMBOL_LIST(varsum), TENSOR_SYMBOL_LIST(var), "var");
35
1
  ccv_nnc_graph_exec_symbol_new(symbolic_graph, CMD_EWLOG_FORWARD(), TENSOR_SYMBOL_LIST(var), TENSOR_SYMBOL_LIST(logvar), "log(var)");
36
1
  ccv_nnc_graph_exec_symbol_new(symbolic_graph, CMD_SCALAR_MUL_FORWARD(0.5), TENSOR_SYMBOL_LIST(logvar), TENSOR_SYMBOL_LIST(logvar_2), "log(var)/2");
37
1
  ccv_nnc_graph_exec_symbol_new(symbolic_graph, CMD_EWEXP_FORWARD(), TENSOR_SYMBOL_LIST(logvar_2), TENSOR_SYMBOL_LIST(std), "std");
38
1
  ccv_nnc_graph_exec_symbol_new(symbolic_graph, CMD_EWDIV_FORWARD(), TENSOR_SYMBOL_LIST(NO_TENSOR_SYMBOL, std), TENSOR_SYMBOL_LIST(inv_std), "1/std");
39
1
  ccv_nnc_graph_exec_symbol_new(symbolic_graph, CMD_MUL_FORWARD(1, 1), TENSOR_SYMBOL_LIST(whitening, inv_std), TENSOR_SYMBOL_LIST(y), "y");
40
1
  ccv_nnc_graph_exec_symbol_autogen(symbolic_graph, 0, 0, CCV_NNC_AUTOGEN_ALL_EXECS | CCV_NNC_AUTOGEN_SOURCES_AND_DESTINATIONS);
41
1
  SYMBOLIC_GRAPH_GEN(symbolic_graph, CCV_NNC_LONG_DOT_GRAPH);
42
1
  ccv_nnc_graph_t* graph = 0;
43
1
  ccv_nnc_tensor_arena_t* tensor_arena = 0;
44
1
  ccv_nnc_graph_exec_arena_t* graph_exec_arena = 0;
45
1
  ccv_nnc_symbolic_graph_compile(symbolic_graph, 0, 0, 0, 0, SYMBOLIC_GRAPH_SOURCES(symbolic_graph), SYMBOLIC_GRAPH_DESTINATIONS(symbolic_graph), &graph, &tensor_arena, &graph_exec_arena);
46
1
  GRAPH_GEN(graph, CCV_NNC_LONG_DOT_GRAPH);
47
1
  ccv_nnc_tensor_t* const x_tensor = ccv_nnc_tensor_from_symbol(tensor_arena, x);
48
1
  dsfmt_t dsfmt;
49
1
  int i;
50
1
  dsfmt_init_gen_rand(&dsfmt, 1);
51
1.28k
  for (i = 0; i < 8 * 4 * 4 * 10; 
i++1.28k
)
52
1.28k
    x_tensor->data.f32[i] = dsfmt_genrand_open_close(&dsfmt);
53
1
  ccv_nnc_symbolic_graph_t* const batch_norm_symbolic_graph = ccv_nnc_symbolic_graph_new();
54
1
  ccv_nnc_tensor_symbol_t bx = ccv_nnc_tensor_symbol_new(batch_norm_symbolic_graph, CPU_TENSOR_NHWC(32F, 8, 4, 4, 10), "x");
55
1
  ccv_nnc_tensor_symbol_t by = ccv_nnc_tensor_symbol_new(batch_norm_symbolic_graph, CPU_TENSOR_NHWC(32F, 8, 4, 4, 10), "y");
56
1
  ccv_nnc_tensor_symbol_t scale = ccv_nnc_tensor_symbol_new(batch_norm_symbolic_graph, CPU_TENSOR_NHWC(32F, 10), "scale");
57
1
  ccv_nnc_tensor_symbol_t bias = ccv_nnc_tensor_symbol_new(batch_norm_symbolic_graph, CPU_TENSOR_NHWC(32F, 10), "bias");
58
1
  ccv_nnc_tensor_symbol_t bmean = ccv_nnc_tensor_symbol_new(batch_norm_symbolic_graph, CPU_TENSOR_NHWC(32F, 10), "mean");
59
1
  ccv_nnc_tensor_symbol_t bvar = ccv_nnc_tensor_symbol_new(batch_norm_symbolic_graph, CPU_TENSOR_NHWC(32F, 10), "var");
60
1
  ccv_nnc_tensor_symbol_t bmean_out = ccv_nnc_tensor_symbol_new(batch_norm_symbolic_graph, CPU_TENSOR_NHWC(32F, 10), "mean");
61
1
  ccv_nnc_tensor_symbol_t bvar_out = ccv_nnc_tensor_symbol_new(batch_norm_symbolic_graph, CPU_TENSOR_NHWC(32F, 10), "var");
62
1
  ccv_nnc_tensor_symbol_t saved_mean = ccv_nnc_tensor_symbol_new(batch_norm_symbolic_graph, CPU_TENSOR_NHWC(32F, 10), "saved_mean");
63
1
  ccv_nnc_tensor_symbol_t saved_inv_std = ccv_nnc_tensor_symbol_new(batch_norm_symbolic_graph, CPU_TENSOR_NHWC(32F, 10), "saved_inv_std");
64
1
  ccv_nnc_graph_exec_symbol_new(batch_norm_symbolic_graph, CMD_SET_FORWARD(1), 0, 0, TENSOR_SYMBOL_LIST(scale), "set_scale");
65
1
  ccv_nnc_graph_exec_symbol_new(batch_norm_symbolic_graph, CMD_SET_FORWARD(0), 0, 0, TENSOR_SYMBOL_LIST(bias), "set_bias");
66
1
  ccv_nnc_graph_exec_symbol_new(batch_norm_symbolic_graph, CMD_BATCH_NORM_FORWARD(0, 0, 0.9, 0, 1, 2), TENSOR_SYMBOL_LIST(bx, scale, bias, bmean, bvar), TENSOR_SYMBOL_LIST(by, bmean_out, bvar_out, saved_mean, saved_inv_std), "batch_norm");
67
1
  ccv_nnc_graph_exec_symbol_autogen(batch_norm_symbolic_graph, 0, 0, CCV_NNC_AUTOGEN_ALL_EXECS | CCV_NNC_AUTOGEN_SOURCES_AND_DESTINATIONS);
68
1
  ccv_nnc_graph_t* batch_norm_graph = 0;
69
1
  ccv_nnc_tensor_arena_t* batch_norm_tensor_arena = 0;
70
1
  ccv_nnc_graph_exec_arena_t* batch_norm_graph_exec_arena = 0;
71
1
  ccv_nnc_symbolic_graph_compile(batch_norm_symbolic_graph, 0, 0, 0, 0, SYMBOLIC_GRAPH_SOURCES(batch_norm_symbolic_graph), SYMBOLIC_GRAPH_DESTINATIONS(batch_norm_symbolic_graph), &batch_norm_graph, &batch_norm_tensor_arena, &batch_norm_graph_exec_arena);
72
1
  ccv_nnc_tensor_t* const bx_tensor = ccv_nnc_tensor_from_symbol(batch_norm_tensor_arena, bx);
73
1
  memcpy(bx_tensor->data.f32, x_tensor->data.f32, sizeof(float) * 8 * 4 * 4 * 10);
74
1
  ccv_nnc_graph_run(graph, 0, 0, 0, TRAVERSE_FULL);
75
1
  ccv_nnc_graph_run(batch_norm_graph, 0, 0, 0, TRAVERSE_FULL);
76
1
  ccv_nnc_tensor_t* const y_tensor = ccv_nnc_tensor_from_symbol(tensor_arena, y);
77
1
  ccv_nnc_tensor_t* const by_tensor = ccv_nnc_tensor_from_symbol(batch_norm_tensor_arena, by);
78
1
  REQUIRE_TENSOR_EQ(y_tensor, by_tensor, "graph computed result should match batch norm op result");
79
1
  ccv_nnc_symbolic_graph_free(symbolic_graph);
80
1
  ccv_nnc_tensor_arena_free(tensor_arena);
81
1
  ccv_nnc_graph_exec_arena_free(graph_exec_arena);
82
1
  ccv_nnc_graph_free(graph);
83
1
  ccv_nnc_symbolic_graph_free(batch_norm_symbolic_graph);
84
1
  ccv_nnc_tensor_arena_free(batch_norm_tensor_arena);
85
1
  ccv_nnc_graph_exec_arena_free(batch_norm_graph_exec_arena);
86
1
  ccv_nnc_graph_free(batch_norm_graph);
87
1
}
88
89
TEST_CASE("compare batch norm gradient with fine-grained symbolic graph")
90
1
{
91
1
  ccv_nnc_symbolic_graph_t* const symbolic_graph = ccv_nnc_symbolic_graph_new();
92
1
  ccv_nnc_tensor_symbol_t x = ccv_nnc_tensor_symbol_new(symbolic_graph, CPU_TENSOR_NHWC(32F, 2, 2, 2, 10), "x");
93
1
  ccv_nnc_tensor_symbol_t sum = ccv_nnc_tensor_symbol_new(symbolic_graph, CPU_TENSOR_NHWC(32F, 10), "sum");
94
1
  ccv_nnc_tensor_symbol_t mean = ccv_nnc_tensor_symbol_new(symbolic_graph, CPU_TENSOR_NHWC(32F, 10), "mean");
95
1
  ccv_nnc_tensor_symbol_t whitening = ccv_nnc_tensor_symbol_new(symbolic_graph, CPU_TENSOR_NHWC(32F, 2, 2, 2, 10), "whitening");
96
1
  ccv_nnc_tensor_symbol_t sqr = ccv_nnc_tensor_symbol_new(symbolic_graph, CPU_TENSOR_NHWC(32F, 2, 2, 2, 10), "sqr");
97
1
  ccv_nnc_tensor_symbol_t varsum = ccv_nnc_tensor_symbol_new(symbolic_graph, CPU_TENSOR_NHWC(32F, 10), "varsum");
98
1
  ccv_nnc_tensor_symbol_t var = ccv_nnc_tensor_symbol_new(symbolic_graph, CPU_TENSOR_NHWC(32F, 10), "var");
99
1
  ccv_nnc_tensor_symbol_t std = ccv_nnc_tensor_symbol_new(symbolic_graph, CPU_TENSOR_NHWC(32F, 10), "std");
100
1
  ccv_nnc_tensor_symbol_t inv_std = ccv_nnc_tensor_symbol_new(symbolic_graph, CPU_TENSOR_NHWC(32F, 10), "inv_std");
101
1
  ccv_nnc_tensor_symbol_t y = ccv_nnc_tensor_symbol_new(symbolic_graph, CPU_TENSOR_NHWC(32F, 2, 2, 2, 10), "y");
102
1
  ccv_nnc_graph_exec_symbol_new(symbolic_graph, CMD_REDUCE_SUM_FORWARD(0, 1, 2), TENSOR_SYMBOL_LIST(x), TENSOR_SYMBOL_LIST(sum), "sum");
103
1
  ccv_nnc_graph_exec_symbol_new(symbolic_graph, CMD_SCALAR_MUL_FORWARD(1.0 / (2 * 2 * 2)), TENSOR_SYMBOL_LIST(sum), TENSOR_SYMBOL_LIST(mean), "mean");
104
1
  ccv_nnc_graph_exec_symbol_new(symbolic_graph, CMD_ADD_FORWARD(1, -1), TENSOR_SYMBOL_LIST(x, mean), TENSOR_SYMBOL_LIST(whitening), "whitening");
105
1
  ccv_nnc_graph_exec_symbol_new(symbolic_graph, CMD_EWPROD_FORWARD(), TENSOR_SYMBOL_LIST(whitening, whitening), TENSOR_SYMBOL_LIST(sqr), "sqr");
106
1
  ccv_nnc_graph_exec_symbol_new(symbolic_graph, CMD_REDUCE_SUM_FORWARD(0, 1, 2), TENSOR_SYMBOL_LIST(sqr), TENSOR_SYMBOL_LIST(varsum), "varsum");
107
1
  ccv_nnc_graph_exec_symbol_new(symbolic_graph, CMD_SCALAR_MUL_FORWARD(1.0 / (2 * 2 * 2)), TENSOR_SYMBOL_LIST(varsum), TENSOR_SYMBOL_LIST(var), "var");
108
1
  ccv_nnc_graph_exec_symbol_new(symbolic_graph, CMD_EWSQRT_FORWARD(), TENSOR_SYMBOL_LIST(var), TENSOR_SYMBOL_LIST(std), "sqrt(var)");
109
1
  ccv_nnc_graph_exec_symbol_new(symbolic_graph, CMD_EWDIV_FORWARD(), TENSOR_SYMBOL_LIST(NO_TENSOR_SYMBOL, std), TENSOR_SYMBOL_LIST(inv_std), "1/std");
110
1
  ccv_nnc_graph_exec_symbol_new(symbolic_graph, CMD_MUL_FORWARD(1, 1), TENSOR_SYMBOL_LIST(whitening, inv_std), TENSOR_SYMBOL_LIST(y), "y");
111
1
  ccv_nnc_graph_exec_symbol_autogen(symbolic_graph, 0, 0, CCV_NNC_AUTOGEN_ALL_EXECS | CCV_NNC_AUTOGEN_SOURCES_AND_DESTINATIONS);
112
1
  ccv_nnc_symbolic_graph_backward(symbolic_graph, TENSOR_SYMBOL_LIST(y), TENSOR_SYMBOL_LIST(x), SYMBOLIC_GRAPH_SOURCES(symbolic_graph), SYMBOLIC_GRAPH_DESTINATIONS(symbolic_graph));
113
1
  ccv_nnc_graph_exec_symbol_autogen(symbolic_graph, 0, 0, CCV_NNC_AUTOGEN_ALL_EXECS | CCV_NNC_AUTOGEN_SOURCES_AND_DESTINATIONS);
114
1
  SYMBOLIC_GRAPH_GEN(symbolic_graph, CCV_NNC_LONG_DOT_GRAPH);
115
1
  ccv_nnc_tensor_symbol_t dy = ccv_nnc_tensor_symbol_for_backward(symbolic_graph, y);
116
1
  ccv_nnc_tensor_symbol_t dx = ccv_nnc_tensor_symbol_for_backward(symbolic_graph, x);
117
1
  ccv_nnc_graph_t* graph = 0;
118
1
  ccv_nnc_tensor_arena_t* tensor_arena = 0;
119
1
  ccv_nnc_graph_exec_arena_t* graph_exec_arena = 0;
120
1
  ccv_nnc_symbolic_graph_compile(symbolic_graph, 0, 0, 0, 0, SYMBOLIC_GRAPH_SOURCES(symbolic_graph), SYMBOLIC_GRAPH_DESTINATIONS(symbolic_graph), &graph, &tensor_arena, &graph_exec_arena);
121
1
  GRAPH_GEN(graph, CCV_NNC_LONG_DOT_GRAPH);
122
1
  ccv_nnc_tensor_t* const x_tensor = ccv_nnc_tensor_from_symbol(tensor_arena, x);
123
1
  dsfmt_t dsfmt;
124
1
  int i;
125
1
  dsfmt_init_gen_rand(&dsfmt, 1);
126
81
  for (i = 0; i < 2 * 2 * 2 * 10; 
i++80
)
127
80
    x_tensor->data.f32[i] = dsfmt_genrand_open_close(&dsfmt);
128
1
  ccv_nnc_symbolic_graph_t* const batch_norm_symbolic_graph = ccv_nnc_symbolic_graph_new();
129
1
  ccv_nnc_tensor_symbol_t bx = ccv_nnc_tensor_symbol_new(batch_norm_symbolic_graph, CPU_TENSOR_NHWC(32F, 2, 2, 2, 10), "x");
130
1
  ccv_nnc_tensor_symbol_t by = ccv_nnc_tensor_symbol_new(batch_norm_symbolic_graph, CPU_TENSOR_NHWC(32F, 2, 2, 2, 10), "y");
131
1
  ccv_nnc_tensor_symbol_t scale = ccv_nnc_tensor_symbol_new(batch_norm_symbolic_graph, CPU_TENSOR_NHWC(32F, 10), "scale");
132
1
  ccv_nnc_tensor_symbol_t bias = ccv_nnc_tensor_symbol_new(batch_norm_symbolic_graph, CPU_TENSOR_NHWC(32F, 10), "bias");
133
1
  ccv_nnc_tensor_symbol_t bmean = ccv_nnc_tensor_symbol_new(batch_norm_symbolic_graph, CPU_TENSOR_NHWC(32F, 10), "mean");
134
1
  ccv_nnc_tensor_symbol_t bvar = ccv_nnc_tensor_symbol_new(batch_norm_symbolic_graph, CPU_TENSOR_NHWC(32F, 10), "var");
135
1
  ccv_nnc_tensor_symbol_set_flags(batch_norm_symbolic_graph, bmean, CCV_NNC_TENSOR_SYMBOL_INIT_ZEROS);
136
1
  ccv_nnc_tensor_symbol_set_flags(batch_norm_symbolic_graph, bvar, CCV_NNC_TENSOR_SYMBOL_INIT_ZEROS);
137
1
  ccv_nnc_tensor_symbol_t bmean_out = ccv_nnc_tensor_symbol_new(batch_norm_symbolic_graph, CPU_TENSOR_NHWC(32F, 10), "mean");
138
1
  ccv_nnc_tensor_symbol_t bvar_out = ccv_nnc_tensor_symbol_new(batch_norm_symbolic_graph, CPU_TENSOR_NHWC(32F, 10), "var");
139
1
  ccv_nnc_tensor_symbol_t saved_mean = ccv_nnc_tensor_symbol_new(batch_norm_symbolic_graph, CPU_TENSOR_NHWC(32F, 10), "saved_mean");
140
1
  ccv_nnc_tensor_symbol_t saved_inv_std = ccv_nnc_tensor_symbol_new(batch_norm_symbolic_graph, CPU_TENSOR_NHWC(32F, 10), "saved_inv_std");
141
1
  ccv_nnc_graph_exec_symbol_new(batch_norm_symbolic_graph, CMD_SET_FORWARD(1), 0, 0, TENSOR_SYMBOL_LIST(scale), "set_scale");
142
1
  ccv_nnc_graph_exec_symbol_new(batch_norm_symbolic_graph, CMD_SET_FORWARD(0), 0, 0, TENSOR_SYMBOL_LIST(bias), "set_bias");
143
1
  ccv_nnc_graph_exec_symbol_new(batch_norm_symbolic_graph, CMD_BATCH_NORM_FORWARD(0, 0, 0.9, 0, 1, 2), TENSOR_SYMBOL_LIST(bx, scale, bias, bmean, bvar), TENSOR_SYMBOL_LIST(by, bmean_out, bvar_out, saved_mean, saved_inv_std), "batch_norm");
144
1
  ccv_nnc_graph_exec_symbol_autogen(batch_norm_symbolic_graph, 0, 0, CCV_NNC_AUTOGEN_ALL_EXECS | CCV_NNC_AUTOGEN_SOURCES_AND_DESTINATIONS);
145
1
  ccv_nnc_symbolic_graph_backward(batch_norm_symbolic_graph, TENSOR_SYMBOL_LIST(by), TENSOR_SYMBOL_LIST(bx, scale, bias), SYMBOLIC_GRAPH_SOURCES(batch_norm_symbolic_graph), SYMBOLIC_GRAPH_DESTINATIONS(batch_norm_symbolic_graph));
146
1
  ccv_nnc_graph_exec_symbol_autogen(batch_norm_symbolic_graph, 0, 0, CCV_NNC_AUTOGEN_ALL_EXECS | CCV_NNC_AUTOGEN_SOURCES_AND_DESTINATIONS);
147
1
  ccv_nnc_tensor_symbol_t dby = ccv_nnc_tensor_symbol_for_backward(batch_norm_symbolic_graph, by);
148
1
  ccv_nnc_tensor_symbol_t dbx = ccv_nnc_tensor_symbol_for_backward(batch_norm_symbolic_graph, bx);
149
1
  ccv_nnc_graph_t* batch_norm_graph = 0;
150
1
  ccv_nnc_tensor_arena_t* batch_norm_tensor_arena = 0;
151
1
  ccv_nnc_graph_exec_arena_t* batch_norm_graph_exec_arena = 0;
152
1
  ccv_nnc_symbolic_graph_compile(batch_norm_symbolic_graph, 0, 0, 0, 0, SYMBOLIC_GRAPH_SOURCES(batch_norm_symbolic_graph), SYMBOLIC_GRAPH_DESTINATIONS(batch_norm_symbolic_graph), &batch_norm_graph, &batch_norm_tensor_arena, &batch_norm_graph_exec_arena);
153
1
  ccv_nnc_tensor_t* const bx_tensor = ccv_nnc_tensor_from_symbol(batch_norm_tensor_arena, bx);
154
1
  memcpy(bx_tensor->data.f32, x_tensor->data.f32, sizeof(float) * 2 * 2 * 2 * 10);
155
1
  ccv_nnc_tensor_t* const dy_tensor = ccv_nnc_tensor_from_symbol(tensor_arena, dy);
156
1
  ccv_nnc_tensor_t* const dby_tensor = ccv_nnc_tensor_from_symbol(batch_norm_tensor_arena, dby);
157
81
  for (i = 0; i < 2 * 2 * 2 * 10; 
i++80
)
158
80
    dby_tensor->data.f32[i] = dy_tensor->data.f32[i] = dsfmt_genrand_open_close(&dsfmt) * 2 - 1;
159
1
  ccv_nnc_graph_run(graph, 0, 0, 0, TRAVERSE_FULL);
160
1
  ccv_nnc_graph_run(batch_norm_graph, 0, 0, 0, TRAVERSE_FULL);
161
1
  ccv_nnc_tensor_t* const dx_tensor = ccv_nnc_tensor_from_symbol(tensor_arena, dx);
162
1
  ccv_nnc_tensor_t* const dbx_tensor = ccv_nnc_tensor_from_symbol(batch_norm_tensor_arena, dbx);
163
1
  REQUIRE_TENSOR_EQ(dx_tensor, dbx_tensor, "graph computed result should match batch norm op result");
164
1
  ccv_nnc_symbolic_graph_free(symbolic_graph);
165
1
  ccv_nnc_tensor_arena_free(tensor_arena);
166
1
  ccv_nnc_graph_exec_arena_free(graph_exec_arena);
167
1
  ccv_nnc_graph_free(graph);
168
1
  ccv_nnc_symbolic_graph_free(batch_norm_symbolic_graph);
169
1
  ccv_nnc_tensor_arena_free(batch_norm_tensor_arena);
170
1
  ccv_nnc_graph_exec_arena_free(batch_norm_graph_exec_arena);
171
1
  ccv_nnc_graph_free(batch_norm_graph);
172
1
}
173
174
TEST_CASE("compare aggregated mean / var from batch norm with binded tensors on outputs")
175
1
{
176
1
  ccv_nnc_symbolic_graph_t* const symbolic_graph = ccv_nnc_symbolic_graph_new();
177
1
  ccv_nnc_tensor_symbol_t bx = ccv_nnc_tensor_symbol_new(symbolic_graph, CPU_TENSOR_NHWC(32F, 8, 4, 4, 10), "x");
178
1
  ccv_nnc_tensor_symbol_t by = ccv_nnc_tensor_symbol_new(symbolic_graph, CPU_TENSOR_NHWC(32F, 8, 4, 4, 10), "y");
179
1
  ccv_nnc_tensor_symbol_t scale = ccv_nnc_tensor_symbol_new(symbolic_graph, CPU_TENSOR_NHWC(32F, 10), "scale");
180
1
  ccv_nnc_tensor_symbol_t bias = ccv_nnc_tensor_symbol_new(symbolic_graph, CPU_TENSOR_NHWC(32F, 10), "bias");
181
1
  ccv_nnc_tensor_symbol_t bmean = ccv_nnc_tensor_symbol_new(symbolic_graph, CPU_TENSOR_NHWC(32F, 10), "mean");
182
1
  ccv_nnc_tensor_symbol_t bvar = ccv_nnc_tensor_symbol_new(symbolic_graph, CPU_TENSOR_NHWC(32F, 10), "var");
183
1
  ccv_nnc_tensor_symbol_t bmean_out = ccv_nnc_tensor_symbol_new(symbolic_graph, CPU_TENSOR_NHWC(32F, 10), "mean");
184
1
  ccv_nnc_tensor_symbol_t bvar_out = ccv_nnc_tensor_symbol_new(symbolic_graph, CPU_TENSOR_NHWC(32F, 10), "var");
185
1
  ccv_nnc_tensor_symbol_t saved_mean = ccv_nnc_tensor_symbol_new(symbolic_graph, CPU_TENSOR_NHWC(32F, 10), "saved_mean");
186
1
  ccv_nnc_tensor_symbol_t saved_inv_std = ccv_nnc_tensor_symbol_new(symbolic_graph, CPU_TENSOR_NHWC(32F, 10), "saved_inv_std");
187
1
  ccv_nnc_graph_exec_symbol_new(symbolic_graph, CMD_SET_FORWARD(1), 0, 0, TENSOR_SYMBOL_LIST(scale), "set_scale");
188
1
  ccv_nnc_graph_exec_symbol_new(symbolic_graph, CMD_SET_FORWARD(0), 0, 0, TENSOR_SYMBOL_LIST(bias), "set_bias");
189
1
  ccv_nnc_graph_exec_symbol_new(symbolic_graph, CMD_BATCH_NORM_FORWARD(0, 0, 0.9, 0, 1, 2), TENSOR_SYMBOL_LIST(bx, scale, bias, bmean, bvar), TENSOR_SYMBOL_LIST(by, bmean_out, bvar_out, saved_mean, saved_inv_std), "batch_norm");
190
1
  ccv_nnc_graph_exec_symbol_autogen(symbolic_graph, 0, 0, CCV_NNC_AUTOGEN_ALL_EXECS | CCV_NNC_AUTOGEN_SOURCES_AND_DESTINATIONS);
191
1
  ccv_nnc_graph_t* graph0 = 0;
192
1
  ccv_nnc_tensor_arena_t* tensor_arena0 = 0;
193
1
  ccv_nnc_graph_exec_arena_t* graph_exec_arena0 = 0;
194
1
  ccv_nnc_symbolic_graph_compile(symbolic_graph, 0, 0, 0, 0, SYMBOLIC_GRAPH_SOURCES(symbolic_graph), SYMBOLIC_GRAPH_DESTINATIONS(symbolic_graph), &graph0, &tensor_arena0, &graph_exec_arena0);
195
1
  ccv_nnc_tensor_t* const x1_tensor = ccv_nnc_tensor_new(0, CPU_TENSOR_NHWC(32F, 8, 4, 4, 10), 0);
196
1
  ccv_nnc_tensor_t* const x2_tensor = ccv_nnc_tensor_new(0, CPU_TENSOR_NHWC(32F, 8, 4, 4, 10), 0);
197
1
  dsfmt_t dsfmt;
198
1
  int i;
199
1
  dsfmt_init_gen_rand(&dsfmt, 1);
200
1.28k
  for (i = 0; i < 8 * 4 * 4 * 10; 
i++1.28k
)
201
1.28k
  {
202
1.28k
    x1_tensor->data.f32[i] = dsfmt_genrand_open_close(&dsfmt);
203
1.28k
    x2_tensor->data.f32[i] = dsfmt_genrand_open_close(&dsfmt);
204
1.28k
  }
205
1
  ccv_nnc_tensor_zero(ccv_nnc_tensor_from_symbol(tensor_arena0, bmean));
206
1
  ccv_nnc_tensor_zero(ccv_nnc_tensor_from_symbol(tensor_arena0, bvar));
207
1
  REQUIRE(ccv_nnc_tensor_from_symbol(tensor_arena0, bmean)->data.u8 == ccv_nnc_tensor_from_symbol(tensor_arena0, bmean_out)->data.u8, "enforced in-place symbol for mean");
208
1
  REQUIRE(ccv_nnc_tensor_from_symbol(tensor_arena0, bvar)->data.u8 == ccv_nnc_tensor_from_symbol(tensor_arena0, bvar_out)->data.u8, "enforced in-place symbol for var");
209
1
  ccv_nnc_tensor_t* const bx_tensor = ccv_nnc_tensor_from_symbol(tensor_arena0, bx);
210
1
  memcpy(bx_tensor->data.f32, x1_tensor->data.f32, sizeof(float) * 8 * 4 * 4 * 10);
211
1
  ccv_nnc_graph_run(graph0, 0, 0, 0, TRAVERSE_FULL);
212
1
  memcpy(bx_tensor->data.f32, x2_tensor->data.f32, sizeof(float) * 8 * 4 * 4 * 10);
213
1
  ccv_nnc_graph_run(graph0, 0, 0, 0, TRAVERSE_FULL);
214
1
  ccv_nnc_graph_t* graph1 = 0;
215
1
  ccv_nnc_tensor_arena_t* tensor_arena1 = 0;
216
1
  ccv_nnc_graph_exec_arena_t* graph_exec_arena1 = 0;
217
1
  ccv_nnc_tensor_t* const bmean_tensor = ccv_nnc_tensor_new(0, CPU_TENSOR_NHWC(32F, 10), 0);
218
1
  ccv_nnc_tensor_zero(bmean_tensor);
219
1
  ccv_nnc_tensor_t* const bvar_tensor = ccv_nnc_tensor_new(0, CPU_TENSOR_NHWC(32F, 10), 0);
220
1
  ccv_nnc_tensor_zero(bvar_tensor);
221
1
  ccv_nnc_symbolic_graph_compile(symbolic_graph,
222
1
    TENSOR_BIND_MAP(KV(bx, x1_tensor), KV(bmean_out, bmean_tensor), KV(bvar_out, bvar_tensor)),
223
1
    0, 0, SYMBOLIC_GRAPH_SOURCES(symbolic_graph), SYMBOLIC_GRAPH_DESTINATIONS(symbolic_graph), &graph1, &tensor_arena1, &graph_exec_arena1);
224
1
  ccv_nnc_graph_run(graph1, 0, 0, 0, TRAVERSE_FULL);
225
1
  ccv_nnc_tensor_bind_symbol(tensor_arena1, bx, x2_tensor);
226
1
  ccv_nnc_graph_run(graph1, 0, 0, 0, TRAVERSE_FULL);
227
1
  ccv_nnc_tensor_t* const bmean_out_tensor = ccv_nnc_tensor_from_symbol(tensor_arena0, bmean_out);
228
1
  ccv_nnc_tensor_t* const bvar_out_tensor = ccv_nnc_tensor_from_symbol(tensor_arena0, bvar_out);
229
1
  REQUIRE_TENSOR_EQ(bmean_tensor, bmean_out_tensor, "updated mean should be the same");
230
1
  REQUIRE_TENSOR_EQ(bvar_tensor, bvar_out_tensor, "updated var should be the same");
231
1
  ccv_nnc_symbolic_graph_free(symbolic_graph);
232
1
  ccv_nnc_graph_free(graph0);
233
1
  ccv_nnc_tensor_arena_free(tensor_arena0);
234
1
  ccv_nnc_graph_exec_arena_free(graph_exec_arena0);
235
1
  ccv_nnc_graph_free(graph1);
236
1
  ccv_nnc_tensor_arena_free(tensor_arena1);
237
1
  ccv_nnc_graph_exec_arena_free(graph_exec_arena1);
238
1
  ccv_nnc_tensor_free(x1_tensor);
239
1
  ccv_nnc_tensor_free(x2_tensor);
240
1
  ccv_nnc_tensor_free(bmean_tensor);
241
1
  ccv_nnc_tensor_free(bvar_tensor);
242
1
}
243
244
TEST_CASE("compare aggregated mean / var from batch norm with binded tensors on inputs")
245
1
{
246
1
  ccv_nnc_symbolic_graph_t* const symbolic_graph = ccv_nnc_symbolic_graph_new();
247
1
  ccv_nnc_tensor_symbol_t bx = ccv_nnc_tensor_symbol_new(symbolic_graph, CPU_TENSOR_NHWC(32F, 8, 4, 4, 10), "x");
248
1
  ccv_nnc_tensor_symbol_t by = ccv_nnc_tensor_symbol_new(symbolic_graph, CPU_TENSOR_NHWC(32F, 8, 4, 4, 10), "y");
249
1
  ccv_nnc_tensor_symbol_t scale = ccv_nnc_tensor_symbol_new(symbolic_graph, CPU_TENSOR_NHWC(32F, 10), "scale");
250
1
  ccv_nnc_tensor_symbol_t bias = ccv_nnc_tensor_symbol_new(symbolic_graph, CPU_TENSOR_NHWC(32F, 10), "bias");
251
1
  ccv_nnc_tensor_symbol_t bmean = ccv_nnc_tensor_symbol_new(symbolic_graph, CPU_TENSOR_NHWC(32F, 10), "mean");
252
1
  ccv_nnc_tensor_symbol_t bvar = ccv_nnc_tensor_symbol_new(symbolic_graph, CPU_TENSOR_NHWC(32F, 10), "var");
253
1
  ccv_nnc_tensor_symbol_t bmean_out = ccv_nnc_tensor_symbol_new(symbolic_graph, CPU_TENSOR_NHWC(32F, 10), "mean");
254
1
  ccv_nnc_tensor_symbol_t bvar_out = ccv_nnc_tensor_symbol_new(symbolic_graph, CPU_TENSOR_NHWC(32F, 10), "var");
255
1
  ccv_nnc_tensor_symbol_t saved_mean = ccv_nnc_tensor_symbol_new(symbolic_graph, CPU_TENSOR_NHWC(32F, 10), "saved_mean");
256
1
  ccv_nnc_tensor_symbol_t saved_inv_std = ccv_nnc_tensor_symbol_new(symbolic_graph, CPU_TENSOR_NHWC(32F, 10), "saved_inv_std");
257
1
  ccv_nnc_graph_exec_symbol_new(symbolic_graph, CMD_SET_FORWARD(1), 0, 0, TENSOR_SYMBOL_LIST(scale), "set_scale");
258
1
  ccv_nnc_graph_exec_symbol_new(symbolic_graph, CMD_SET_FORWARD(0), 0, 0, TENSOR_SYMBOL_LIST(bias), "set_bias");
259
1
  ccv_nnc_graph_exec_symbol_new(symbolic_graph, CMD_BATCH_NORM_FORWARD(0, 0, 0.9, 0, 1, 2), TENSOR_SYMBOL_LIST(bx, scale, bias, bmean, bvar), TENSOR_SYMBOL_LIST(by, bmean_out, bvar_out, saved_mean, saved_inv_std), "batch_norm");
260
1
  ccv_nnc_graph_exec_symbol_autogen(symbolic_graph, 0, 0, CCV_NNC_AUTOGEN_ALL_EXECS | CCV_NNC_AUTOGEN_SOURCES_AND_DESTINATIONS);
261
1
  ccv_nnc_graph_t* graph0 = 0;
262
1
  ccv_nnc_tensor_arena_t* tensor_arena0 = 0;
263
1
  ccv_nnc_graph_exec_arena_t* graph_exec_arena0 = 0;
264
1
  ccv_nnc_symbolic_graph_compile(symbolic_graph, 0, 0, 0, 0, SYMBOLIC_GRAPH_SOURCES(symbolic_graph), SYMBOLIC_GRAPH_DESTINATIONS(symbolic_graph), &graph0, &tensor_arena0, &graph_exec_arena0);
265
1
  ccv_nnc_tensor_t* const x1_tensor = ccv_nnc_tensor_new(0, CPU_TENSOR_NHWC(32F, 8, 4, 4, 10), 0);
266
1
  ccv_nnc_tensor_t* const x2_tensor = ccv_nnc_tensor_new(0, CPU_TENSOR_NHWC(32F, 8, 4, 4, 10), 0);
267
1
  dsfmt_t dsfmt;
268
1
  int i;
269
1
  dsfmt_init_gen_rand(&dsfmt, 1);
270
1.28k
  for (i = 0; i < 8 * 4 * 4 * 10; 
i++1.28k
)
271
1.28k
  {
272
1.28k
    x1_tensor->data.f32[i] = dsfmt_genrand_open_close(&dsfmt);
273
1.28k
    x2_tensor->data.f32[i] = dsfmt_genrand_open_close(&dsfmt);
274
1.28k
  }
275
1
  ccv_nnc_tensor_zero(ccv_nnc_tensor_from_symbol(tensor_arena0, bmean));
276
1
  ccv_nnc_tensor_zero(ccv_nnc_tensor_from_symbol(tensor_arena0, bvar));
277
1
  REQUIRE(ccv_nnc_tensor_from_symbol(tensor_arena0, bmean)->data.u8 == ccv_nnc_tensor_from_symbol(tensor_arena0, bmean_out)->data.u8, "enforced in-place symbol for mean");
278
1
  REQUIRE(ccv_nnc_tensor_from_symbol(tensor_arena0, bvar)->data.u8 == ccv_nnc_tensor_from_symbol(tensor_arena0, bvar_out)->data.u8, "enforced in-place symbol for var");
279
1
  ccv_nnc_tensor_t* const bx_tensor = ccv_nnc_tensor_from_symbol(tensor_arena0, bx);
280
1
  memcpy(bx_tensor->data.f32, x1_tensor->data.f32, sizeof(float) * 8 * 4 * 4 * 10);
281
1
  ccv_nnc_graph_run(graph0, 0, 0, 0, TRAVERSE_FULL);
282
1
  memcpy(bx_tensor->data.f32, x2_tensor->data.f32, sizeof(float) * 8 * 4 * 4 * 10);
283
1
  ccv_nnc_graph_run(graph0, 0, 0, 0, TRAVERSE_FULL);
284
1
  ccv_nnc_graph_t* graph1 = 0;
285
1
  ccv_nnc_tensor_arena_t* tensor_arena1 = 0;
286
1
  ccv_nnc_graph_exec_arena_t* graph_exec_arena1 = 0;
287
1
  ccv_nnc_tensor_t* const bmean_tensor = ccv_nnc_tensor_new(0, CPU_TENSOR_NHWC(32F, 10), 0);
288
1
  ccv_nnc_tensor_zero(bmean_tensor);
289
1
  ccv_nnc_tensor_t* const bvar_tensor = ccv_nnc_tensor_new(0, CPU_TENSOR_NHWC(32F, 10), 0);
290
1
  ccv_nnc_tensor_zero(bvar_tensor);
291
1
  ccv_nnc_symbolic_graph_compile(symbolic_graph,
292
1
    TENSOR_BIND_MAP(KV(bx, x1_tensor), KV(bmean, bmean_tensor), KV(bvar, bvar_tensor)),
293
1
    0, 0, SYMBOLIC_GRAPH_SOURCES(symbolic_graph), SYMBOLIC_GRAPH_DESTINATIONS(symbolic_graph), &graph1, &tensor_arena1, &graph_exec_arena1);
294
1
  ccv_nnc_graph_run(graph1, 0, 0, 0, TRAVERSE_FULL);
295
1
  ccv_nnc_tensor_bind_symbol(tensor_arena1, bx, x2_tensor);
296
1
  ccv_nnc_graph_run(graph1, 0, 0, 0, TRAVERSE_FULL);
297
1
  ccv_nnc_tensor_t* const bmean_out_tensor = ccv_nnc_tensor_from_symbol(tensor_arena0, bmean_out);
298
1
  ccv_nnc_tensor_t* const bvar_out_tensor = ccv_nnc_tensor_from_symbol(tensor_arena0, bvar_out);
299
1
  REQUIRE_TENSOR_EQ(bmean_tensor, bmean_out_tensor, "updated mean should be the same");
300
1
  REQUIRE_TENSOR_EQ(bvar_tensor, bvar_out_tensor, "updated var should be the same");
301
1
  ccv_nnc_symbolic_graph_free(symbolic_graph);
302
1
  ccv_nnc_graph_free(graph0);
303
1
  ccv_nnc_tensor_arena_free(tensor_arena0);
304
1
  ccv_nnc_graph_exec_arena_free(graph_exec_arena0);
305
1
  ccv_nnc_graph_free(graph1);
306
1
  ccv_nnc_tensor_arena_free(tensor_arena1);
307
1
  ccv_nnc_graph_exec_arena_free(graph_exec_arena1);
308
1
  ccv_nnc_tensor_free(x1_tensor);
309
1
  ccv_nnc_tensor_free(x2_tensor);
310
1
  ccv_nnc_tensor_free(bmean_tensor);
311
1
  ccv_nnc_tensor_free(bvar_tensor);
312
1
}
313
314
TEST_CASE("compare aggregated mean / var from batch norm with late binded tensors on outputs")
315
1
{
316
1
  ccv_nnc_symbolic_graph_t* const symbolic_graph = ccv_nnc_symbolic_graph_new();
317
1
  ccv_nnc_tensor_symbol_t bx = ccv_nnc_tensor_symbol_new(symbolic_graph, CPU_TENSOR_NHWC(32F, 8, 4, 4, 10), "x");
318
1
  ccv_nnc_tensor_symbol_t by = ccv_nnc_tensor_symbol_new(symbolic_graph, CPU_TENSOR_NHWC(32F, 8, 4, 4, 10), "y");
319
1
  ccv_nnc_tensor_symbol_t scale = ccv_nnc_tensor_symbol_new(symbolic_graph, CPU_TENSOR_NHWC(32F, 10), "scale");
320
1
  ccv_nnc_tensor_symbol_t bias = ccv_nnc_tensor_symbol_new(symbolic_graph, CPU_TENSOR_NHWC(32F, 10), "bias");
321
1
  ccv_nnc_tensor_symbol_t bmean = ccv_nnc_tensor_symbol_new(symbolic_graph, CPU_TENSOR_NHWC(32F, 10), "mean");
322
1
  ccv_nnc_tensor_symbol_t bvar = ccv_nnc_tensor_symbol_new(symbolic_graph, CPU_TENSOR_NHWC(32F, 10), "var");
323
1
  ccv_nnc_tensor_symbol_t bmean_out = ccv_nnc_tensor_symbol_new(symbolic_graph, CPU_TENSOR_NHWC(32F, 10), "mean");
324
1
  ccv_nnc_tensor_symbol_t bvar_out = ccv_nnc_tensor_symbol_new(symbolic_graph, CPU_TENSOR_NHWC(32F, 10), "var");
325
1
  ccv_nnc_tensor_symbol_t saved_mean = ccv_nnc_tensor_symbol_new(symbolic_graph, CPU_TENSOR_NHWC(32F, 10), "saved_mean");
326
1
  ccv_nnc_tensor_symbol_t saved_inv_std = ccv_nnc_tensor_symbol_new(symbolic_graph, CPU_TENSOR_NHWC(32F, 10), "saved_inv_std");
327
1
  ccv_nnc_graph_exec_symbol_new(symbolic_graph, CMD_SET_FORWARD(1), 0, 0, TENSOR_SYMBOL_LIST(scale), "set_scale");
328
1
  ccv_nnc_graph_exec_symbol_new(symbolic_graph, CMD_SET_FORWARD(0), 0, 0, TENSOR_SYMBOL_LIST(bias), "set_bias");
329
1
  ccv_nnc_graph_exec_symbol_new(symbolic_graph, CMD_BATCH_NORM_FORWARD(0, 0, 0.9, 0, 1, 2), TENSOR_SYMBOL_LIST(bx, scale, bias, bmean, bvar), TENSOR_SYMBOL_LIST(by, bmean_out, bvar_out, saved_mean, saved_inv_std), "batch_norm");
330
1
  ccv_nnc_graph_exec_symbol_autogen(symbolic_graph, 0, 0, CCV_NNC_AUTOGEN_ALL_EXECS | CCV_NNC_AUTOGEN_SOURCES_AND_DESTINATIONS);
331
1
  ccv_nnc_graph_t* graph0 = 0;
332
1
  ccv_nnc_tensor_arena_t* tensor_arena0 = 0;
333
1
  ccv_nnc_graph_exec_arena_t* graph_exec_arena0 = 0;
334
1
  ccv_nnc_symbolic_graph_compile(symbolic_graph, 0, 0, 0, 0, SYMBOLIC_GRAPH_SOURCES(symbolic_graph), SYMBOLIC_GRAPH_DESTINATIONS(symbolic_graph), &graph0, &tensor_arena0, &graph_exec_arena0);
335
1
  ccv_nnc_tensor_t* const x1_tensor = ccv_nnc_tensor_new(0, CPU_TENSOR_NHWC(32F, 8, 4, 4, 10), 0);
336
1
  ccv_nnc_tensor_t* const x2_tensor = ccv_nnc_tensor_new(0, CPU_TENSOR_NHWC(32F, 8, 4, 4, 10), 0);
337
1
  dsfmt_t dsfmt;
338
1
  int i;
339
1
  dsfmt_init_gen_rand(&dsfmt, 1);
340
1.28k
  for (i = 0; i < 8 * 4 * 4 * 10; 
i++1.28k
)
341
1.28k
  {
342
1.28k
    x1_tensor->data.f32[i] = dsfmt_genrand_open_close(&dsfmt);
343
1.28k
    x2_tensor->data.f32[i] = dsfmt_genrand_open_close(&dsfmt);
344
1.28k
  }
345
1
  ccv_nnc_tensor_zero(ccv_nnc_tensor_from_symbol(tensor_arena0, bmean));
346
1
  REQUIRE(ccv_nnc_tensor_from_symbol(tensor_arena0, bmean)->data.u8 == ccv_nnc_tensor_from_symbol(tensor_arena0, bmean_out)->data.u8, "enforced in-place symbol for mean");
347
1
  REQUIRE(ccv_nnc_tensor_from_symbol(tensor_arena0, bvar)->data.u8 == ccv_nnc_tensor_from_symbol(tensor_arena0, bvar_out)->data.u8, "enforced in-place symbol for var");
348
1
  ccv_nnc_tensor_zero(ccv_nnc_tensor_from_symbol(tensor_arena0, bvar));
349
1
  ccv_nnc_tensor_t* const bx_tensor = ccv_nnc_tensor_from_symbol(tensor_arena0, bx);
350
1
  memcpy(bx_tensor->data.f32, x1_tensor->data.f32, sizeof(float) * 8 * 4 * 4 * 10);
351
1
  ccv_nnc_graph_run(graph0, 0, 0, 0, TRAVERSE_FULL);
352
1
  memcpy(bx_tensor->data.f32, x2_tensor->data.f32, sizeof(float) * 8 * 4 * 4 * 10);
353
1
  ccv_nnc_graph_run(graph0, 0, 0, 0, TRAVERSE_FULL);
354
1
  ccv_nnc_graph_t* graph1 = 0;
355
1
  ccv_nnc_tensor_arena_t* tensor_arena1 = 0;
356
1
  ccv_nnc_graph_exec_arena_t* graph_exec_arena1 = 0;
357
1
  ccv_nnc_tensor_t* const bmean_tensor = ccv_nnc_tensor_new(0, CPU_TENSOR_NHWC(32F, 10), 0);
358
1
  ccv_nnc_tensor_zero(bmean_tensor);
359
1
  ccv_nnc_tensor_t* const bvar_tensor = ccv_nnc_tensor_new(0, CPU_TENSOR_NHWC(32F, 10), 0);
360
1
  ccv_nnc_tensor_zero(bvar_tensor);
361
1
  ccv_nnc_symbolic_graph_compile(symbolic_graph,
362
1
    TENSOR_BIND_MAP(KV(bx, x1_tensor)),
363
1
    0, 0, SYMBOLIC_GRAPH_SOURCES(symbolic_graph), SYMBOLIC_GRAPH_DESTINATIONS(symbolic_graph), &graph1, &tensor_arena1, &graph_exec_arena1);
364
1
  ccv_nnc_tensor_bind_symbol(tensor_arena1, bmean_out, bmean_tensor);
365
1
  ccv_nnc_tensor_bind_symbol(tensor_arena1, bvar_out, bvar_tensor);
366
1
  ccv_nnc_graph_run(graph1, 0, 0, 0, TRAVERSE_FULL);
367
1
  ccv_nnc_tensor_bind_symbol(tensor_arena1, bx, x2_tensor);
368
1
  ccv_nnc_graph_run(graph1, 0, 0, 0, TRAVERSE_FULL);
369
1
  ccv_nnc_tensor_t* const bmean_out_tensor = ccv_nnc_tensor_from_symbol(tensor_arena0, bmean_out);
370
1
  ccv_nnc_tensor_t* const bvar_out_tensor = ccv_nnc_tensor_from_symbol(tensor_arena0, bvar_out);
371
1
  REQUIRE_TENSOR_EQ(bmean_tensor, bmean_out_tensor, "updated mean should be the same");
372
1
  REQUIRE_TENSOR_EQ(bvar_tensor, bvar_out_tensor, "updated var should be the same");
373
1
  ccv_nnc_symbolic_graph_free(symbolic_graph);
374
1
  ccv_nnc_graph_free(graph0);
375
1
  ccv_nnc_tensor_arena_free(tensor_arena0);
376
1
  ccv_nnc_graph_exec_arena_free(graph_exec_arena0);
377
1
  ccv_nnc_graph_free(graph1);
378
1
  ccv_nnc_tensor_arena_free(tensor_arena1);
379
1
  ccv_nnc_graph_exec_arena_free(graph_exec_arena1);
380
1
  ccv_nnc_tensor_free(x1_tensor);
381
1
  ccv_nnc_tensor_free(x2_tensor);
382
1
  ccv_nnc_tensor_free(bmean_tensor);
383
1
  ccv_nnc_tensor_free(bvar_tensor);
384
1
}
385
386
TEST_CASE("compare aggregated mean / var from batch norm with late binded tensors on inputs")
387
1
{
388
1
  ccv_nnc_symbolic_graph_t* const symbolic_graph = ccv_nnc_symbolic_graph_new();
389
1
  ccv_nnc_tensor_symbol_t bx = ccv_nnc_tensor_symbol_new(symbolic_graph, CPU_TENSOR_NHWC(32F, 8, 4, 4, 10), "x");
390
1
  ccv_nnc_tensor_symbol_t by = ccv_nnc_tensor_symbol_new(symbolic_graph, CPU_TENSOR_NHWC(32F, 8, 4, 4, 10), "y");
391
1
  ccv_nnc_tensor_symbol_t scale = ccv_nnc_tensor_symbol_new(symbolic_graph, CPU_TENSOR_NHWC(32F, 10), "scale");
392
1
  ccv_nnc_tensor_symbol_t bias = ccv_nnc_tensor_symbol_new(symbolic_graph, CPU_TENSOR_NHWC(32F, 10), "bias");
393
1
  ccv_nnc_tensor_symbol_t bmean = ccv_nnc_tensor_symbol_new(symbolic_graph, CPU_TENSOR_NHWC(32F, 10), "mean");
394
1
  ccv_nnc_tensor_symbol_t bvar = ccv_nnc_tensor_symbol_new(symbolic_graph, CPU_TENSOR_NHWC(32F, 10), "var");
395
1
  ccv_nnc_tensor_symbol_t bmean_out = ccv_nnc_tensor_symbol_new(symbolic_graph, CPU_TENSOR_NHWC(32F, 10), "mean");
396
1
  ccv_nnc_tensor_symbol_t bvar_out = ccv_nnc_tensor_symbol_new(symbolic_graph, CPU_TENSOR_NHWC(32F, 10), "var");
397
1
  ccv_nnc_tensor_symbol_t saved_mean = ccv_nnc_tensor_symbol_new(symbolic_graph, CPU_TENSOR_NHWC(32F, 10), "saved_mean");
398
1
  ccv_nnc_tensor_symbol_t saved_inv_std = ccv_nnc_tensor_symbol_new(symbolic_graph, CPU_TENSOR_NHWC(32F, 10), "saved_inv_std");
399
1
  ccv_nnc_graph_exec_symbol_new(symbolic_graph, CMD_SET_FORWARD(1), 0, 0, TENSOR_SYMBOL_LIST(scale), "set_scale");
400
1
  ccv_nnc_graph_exec_symbol_new(symbolic_graph, CMD_SET_FORWARD(0), 0, 0, TENSOR_SYMBOL_LIST(bias), "set_bias");
401
1
  ccv_nnc_graph_exec_symbol_new(symbolic_graph, CMD_BATCH_NORM_FORWARD(0, 0, 0.9, 0, 1, 2), TENSOR_SYMBOL_LIST(bx, scale, bias, bmean, bvar), TENSOR_SYMBOL_LIST(by, bmean_out, bvar_out, saved_mean, saved_inv_std), "batch_norm");
402
1
  ccv_nnc_graph_exec_symbol_autogen(symbolic_graph, 0, 0, CCV_NNC_AUTOGEN_ALL_EXECS | CCV_NNC_AUTOGEN_SOURCES_AND_DESTINATIONS);
403
1
  ccv_nnc_graph_t* graph0 = 0;
404
1
  ccv_nnc_tensor_arena_t* tensor_arena0 = 0;
405
1
  ccv_nnc_graph_exec_arena_t* graph_exec_arena0 = 0;
406
1
  ccv_nnc_symbolic_graph_compile(symbolic_graph, 0, 0, 0, 0, SYMBOLIC_GRAPH_SOURCES(symbolic_graph), SYMBOLIC_GRAPH_DESTINATIONS(symbolic_graph), &graph0, &tensor_arena0, &graph_exec_arena0);
407
1
  ccv_nnc_tensor_t* const x1_tensor = ccv_nnc_tensor_new(0, CPU_TENSOR_NHWC(32F, 8, 4, 4, 10), 0);
408
1
  ccv_nnc_tensor_t* const x2_tensor = ccv_nnc_tensor_new(0, CPU_TENSOR_NHWC(32F, 8, 4, 4, 10), 0);
409
1
  dsfmt_t dsfmt;
410
1
  int i;
411
1
  dsfmt_init_gen_rand(&dsfmt, 1);
412
1.28k
  for (i = 0; i < 8 * 4 * 4 * 10; 
i++1.28k
)
413
1.28k
  {
414
1.28k
    x1_tensor->data.f32[i] = dsfmt_genrand_open_close(&dsfmt);
415
1.28k
    x2_tensor->data.f32[i] = dsfmt_genrand_open_close(&dsfmt);
416
1.28k
  }
417
1
  ccv_nnc_tensor_zero(ccv_nnc_tensor_from_symbol(tensor_arena0, bmean));
418
1
  ccv_nnc_tensor_zero(ccv_nnc_tensor_from_symbol(tensor_arena0, bvar));
419
1
  REQUIRE(ccv_nnc_tensor_from_symbol(tensor_arena0, bmean)->data.u8 == ccv_nnc_tensor_from_symbol(tensor_arena0, bmean_out)->data.u8, "enforced in-place symbol for mean");
420
1
  REQUIRE(ccv_nnc_tensor_from_symbol(tensor_arena0, bvar)->data.u8 == ccv_nnc_tensor_from_symbol(tensor_arena0, bvar_out)->data.u8, "enforced in-place symbol for var");
421
1
  ccv_nnc_tensor_t* const bx_tensor = ccv_nnc_tensor_from_symbol(tensor_arena0, bx);
422
1
  memcpy(bx_tensor->data.f32, x1_tensor->data.f32, sizeof(float) * 8 * 4 * 4 * 10);
423
1
  ccv_nnc_graph_run(graph0, 0, 0, 0, TRAVERSE_FULL);
424
1
  memcpy(bx_tensor->data.f32, x2_tensor->data.f32, sizeof(float) * 8 * 4 * 4 * 10);
425
1
  ccv_nnc_graph_run(graph0, 0, 0, 0, TRAVERSE_FULL);
426
1
  ccv_nnc_graph_t* graph1 = 0;
427
1
  ccv_nnc_tensor_arena_t* tensor_arena1 = 0;
428
1
  ccv_nnc_graph_exec_arena_t* graph_exec_arena1 = 0;
429
1
  ccv_nnc_tensor_t* const bmean_tensor = ccv_nnc_tensor_new(0, CPU_TENSOR_NHWC(32F, 10), 0);
430
1
  ccv_nnc_tensor_zero(bmean_tensor);
431
1
  ccv_nnc_tensor_t* const bvar_tensor = ccv_nnc_tensor_new(0, CPU_TENSOR_NHWC(32F, 10), 0);
432
1
  ccv_nnc_tensor_zero(bvar_tensor);
433
1
  ccv_nnc_symbolic_graph_compile(symbolic_graph,
434
1
    TENSOR_BIND_MAP(KV(bx, x1_tensor)),
435
1
    0, 0, SYMBOLIC_GRAPH_SOURCES(symbolic_graph), SYMBOLIC_GRAPH_DESTINATIONS(symbolic_graph), &graph1, &tensor_arena1, &graph_exec_arena1);
436
1
  ccv_nnc_tensor_bind_symbol(tensor_arena1, bmean, bmean_tensor);
437
1
  ccv_nnc_tensor_bind_symbol(tensor_arena1, bvar, bvar_tensor);
438
1
  ccv_nnc_graph_run(graph1, 0, 0, 0, TRAVERSE_FULL);
439
1
  ccv_nnc_tensor_bind_symbol(tensor_arena1, bx, x2_tensor);
440
1
  ccv_nnc_graph_run(graph1, 0, 0, 0, TRAVERSE_FULL);
441
1
  ccv_nnc_tensor_t* const bmean_out_tensor = ccv_nnc_tensor_from_symbol(tensor_arena0, bmean_out);
442
1
  ccv_nnc_tensor_t* const bvar_out_tensor = ccv_nnc_tensor_from_symbol(tensor_arena0, bvar_out);
443
1
  REQUIRE_TENSOR_EQ(bmean_tensor, bmean_out_tensor, "updated mean should be the same");
444
1
  REQUIRE_TENSOR_EQ(bvar_tensor, bvar_out_tensor, "updated var should be the same");
445
1
  ccv_nnc_symbolic_graph_free(symbolic_graph);
446
1
  ccv_nnc_graph_free(graph0);
447
1
  ccv_nnc_tensor_arena_free(tensor_arena0);
448
1
  ccv_nnc_graph_exec_arena_free(graph_exec_arena0);
449
1
  ccv_nnc_graph_free(graph1);
450
1
  ccv_nnc_tensor_arena_free(tensor_arena1);
451
1
  ccv_nnc_graph_exec_arena_free(graph_exec_arena1);
452
1
  ccv_nnc_tensor_free(x1_tensor);
453
1
  ccv_nnc_tensor_free(x2_tensor);
454
1
  ccv_nnc_tensor_free(bmean_tensor);
455
1
  ccv_nnc_tensor_free(bvar_tensor);
456
1
}
457
458
#include "case_main.h"