Coverage Report

Created: 2022-08-03 23:52

/home/liu/buildslave/linux-x64-runtests/build/test/unit/nnc/dynamic.graph.tests.c
Line
Count
Source (jump to first uncovered line)
1
#include "case.h"
2
#include "ccv_case.h"
3
#include "ccv_nnc_case.h"
4
#include <ccv.h>
5
#include <nnc/ccv_nnc.h>
6
#include <nnc/ccv_nnc_easy.h>
7
#include "3rdparty/dsfmt/dSFMT.h"
8
9
TEST_SETUP()
10
{
11
  ccv_nnc_init();
12
}
13
14
TEST_CASE("dynamic graph to compute log(19)")
15
1
{
16
1
  ccv_nnc_dynamic_graph_t* const graph = ccv_nnc_dynamic_graph_new();
17
1
  ccv_nnc_tensor_variable_t a = ccv_nnc_tensor_variable_new(graph, CPU_TENSOR_NHWC(32F, 1));
18
1
  ccv_nnc_tensor_from_variable(graph, a)->data.f32[0] = 19;
19
1
  ccv_nnc_tensor_variable_t b = ccv_nnc_tensor_variable_new(graph);
20
1
  ccv_nnc_dynamic_graph_exec(graph, CMD_EWLOG_FORWARD(), ccv_nnc_no_hint, 0, TENSOR_VARIABLE_LIST(a), TENSOR_VARIABLE_LIST(b), 0, 0);
21
1
  REQUIRE_EQ_WITH_TOLERANCE(ccv_nnc_tensor_from_variable(graph, b)->data.f32[0], logf(19), 1e-5, "log(19) result should be equal.");
22
1
  DYNAMIC_GRAPH_GEN(graph, CCV_NNC_LONG_DOT_GRAPH);
23
1
  ccv_nnc_dynamic_graph_free(graph);
24
1
}
25
26
TEST_CASE("dynamic graph to compute reciprocal")
27
1
{
28
1
  ccv_nnc_dynamic_graph_t* const graph = ccv_nnc_dynamic_graph_new();
29
1
  ccv_nnc_tensor_variable_t a = ccv_nnc_tensor_variable_new(graph, CPU_TENSOR_NHWC(32F, 1));
30
1
  ccv_nnc_tensor_from_variable(graph, a)->data.f32[0] = 19;
31
1
  ccv_nnc_tensor_variable_t b = ccv_nnc_tensor_variable_new(graph);
32
1
  ccv_nnc_dynamic_graph_exec(graph, CMD_EWDIV_FORWARD(), ccv_nnc_no_hint, 0, TENSOR_VARIABLE_LIST(0, a), TENSOR_VARIABLE_LIST(b), 0, 0);
33
1
  REQUIRE_EQ_WITH_TOLERANCE(ccv_nnc_tensor_from_variable(graph, b)->data.f32[0], 1.0 / 19, 1e-5, "1.0 / 19 result should be equal.");
34
1
  DYNAMIC_GRAPH_GEN(graph, CCV_NNC_LONG_DOT_GRAPH);
35
1
  ccv_nnc_dynamic_graph_free(graph);
36
1
}
37
38
TEST_CASE("dynamic graph with alias")
39
1
{
40
1
  ccv_nnc_dynamic_graph_t* const graph = ccv_nnc_dynamic_graph_new();
41
1
  ccv_nnc_tensor_variable_t const a = ccv_nnc_tensor_variable_new(graph, CPU_TENSOR_NHWC(32F, 2));
42
1
  ccv_nnc_tensor_variable_t const a0 = ccv_nnc_tensor_variable_alias_new(graph, a, DIM_ALLOC(1), DIM_ALLOC(), CPU_TENSOR_NHWC(32F, 1));
43
1
  ccv_nnc_tensor_variable_t const a1 = ccv_nnc_tensor_variable_alias_new(graph, a, ccv_nnc_no_ofs, DIM_ALLOC(), CPU_TENSOR_NHWC(32F, 1));
44
1
  ccv_nnc_tensor_t* const b0 = ccv_nnc_tensor_new(0, CPU_TENSOR_NHWC(32F, 2), 0);
45
1
  ccv_nnc_tensor_variable_set(graph, a, b0);
46
1
  b0->data.f32[0] = 10;
47
1
  b0->data.f32[1] = 11;
48
1
  ccv_nnc_tensor_t* const b1 = ccv_nnc_tensor_new(0, CPU_TENSOR_NHWC(32F, 2), 0);
49
1
  b1->data.f32[0] = 20;
50
1
  b1->data.f32[1] = 21;
51
1
  REQUIRE(CCV_IS_TENSOR_VIEW(ccv_nnc_tensor_from_variable(graph, a0)), "Complex vector is a tensor view");
52
1
  REQUIRE(!CCV_IS_TENSOR_VIEW(ccv_nnc_tensor_from_variable(graph, a1)), "Simple vector is not a tensor view");
53
1
  REQUIRE_EQ(ccv_nnc_tensor_from_variable(graph, a0)->data.f32[0], 11, "should be b0[1]");
54
1
  REQUIRE_EQ(ccv_nnc_tensor_from_variable(graph, a1)->data.f32[0], 10, "should be b0[0]");
55
1
  REQUIRE(!CCV_IS_TENSOR_VIEW(ccv_nnc_tensor_from_variable(graph, a1)), "no need to be a tensor view");
56
1
  ccv_nnc_tensor_variable_set(graph, a, b1);
57
1
  REQUIRE_EQ(ccv_nnc_tensor_from_variable(graph, a0)->data.f32[0], 21, "should be b1[1]");
58
1
  REQUIRE_EQ(ccv_nnc_tensor_from_variable(graph, a1)->data.f32[0], 20, "should be b1[0]");
59
1
  REQUIRE(!CCV_IS_TENSOR_VIEW(ccv_nnc_tensor_from_variable(graph, a1)), "no need to be a tensor view");
60
1
  ccv_nnc_dynamic_graph_free(graph);
61
1
  ccv_nnc_tensor_free(b0);
62
1
  ccv_nnc_tensor_free(b1);
63
1
}
64
65
TEST_CASE("dynamic graph alias an alias")
66
1
{
67
1
  ccv_nnc_dynamic_graph_t* const graph = ccv_nnc_dynamic_graph_new();
68
1
  ccv_nnc_tensor_variable_t const a = ccv_nnc_tensor_variable_new(graph, CPU_TENSOR_NHWC(32F, 2, 3));
69
1
  ccv_nnc_tensor_variable_t const a0 = ccv_nnc_tensor_variable_alias_new(graph, a, DIM_ALLOC(1, 0), DIM_ALLOC(2, 3), CPU_TENSOR_NHWC(32F, 1, 3));
70
1
  ccv_nnc_tensor_variable_t const a1 = ccv_nnc_tensor_variable_alias_new(graph, a0, DIM_ALLOC(1), DIM_ALLOC(3), CPU_TENSOR_NHWC(32F, 2));
71
1
  ccv_nnc_tensor_t* const b0 = ccv_nnc_tensor_new(0, CPU_TENSOR_NHWC(32F, 2, 3), 0);
72
1
  ccv_nnc_tensor_variable_set(graph, a, b0);
73
1
  b0->data.f32[0] = 10;
74
1
  b0->data.f32[1] = 11;
75
1
  b0->data.f32[2] = 12;
76
1
  b0->data.f32[3] = 13;
77
1
  b0->data.f32[4] = 14;
78
1
  b0->data.f32[5] = 15;
79
1
  ccv_nnc_tensor_t* const b1 = ccv_nnc_tensor_new(0, CPU_TENSOR_NHWC(32F, 2, 3), 0);
80
1
  b1->data.f32[0] = 20;
81
1
  b1->data.f32[1] = 21;
82
1
  b1->data.f32[2] = 22;
83
1
  b1->data.f32[3] = 23;
84
1
  b1->data.f32[4] = 24;
85
1
  b1->data.f32[5] = 25;
86
1
  REQUIRE(CCV_IS_TENSOR_VIEW(ccv_nnc_tensor_from_variable(graph, a0)), "Complex vector is a tensor view");
87
1
  REQUIRE(CCV_IS_TENSOR_VIEW(ccv_nnc_tensor_from_variable(graph, a1)), "Complex vector is a tensor view");
88
1
  REQUIRE_EQ(ccv_nnc_tensor_from_variable(graph, a0)->data.f32[0], 13, "should be b0[1, 0]");
89
1
  REQUIRE_EQ(ccv_nnc_tensor_from_variable(graph, a0)->data.f32[1], 14, "should be b0[1, 1]");
90
1
  REQUIRE_EQ(ccv_nnc_tensor_from_variable(graph, a0)->data.f32[2], 15, "should be b0[1, 2]");
91
1
  REQUIRE_EQ(ccv_nnc_tensor_from_variable(graph, a1)->data.f32[0], 14, "should be b0[1, 1]");
92
1
  REQUIRE_EQ(ccv_nnc_tensor_from_variable(graph, a1)->data.f32[1], 15, "should be b0[1, 2]");
93
1
  ccv_nnc_tensor_variable_set(graph, a, b1);
94
1
  REQUIRE_EQ(ccv_nnc_tensor_from_variable(graph, a0)->data.f32[0], 23, "should be b1[1, 0]");
95
1
  REQUIRE_EQ(ccv_nnc_tensor_from_variable(graph, a0)->data.f32[1], 24, "should be b1[1, 1]");
96
1
  REQUIRE_EQ(ccv_nnc_tensor_from_variable(graph, a0)->data.f32[2], 25, "should be b1[1, 2]");
97
1
  REQUIRE_EQ(ccv_nnc_tensor_from_variable(graph, a1)->data.f32[0], 24, "should be b1[1, 1]");
98
1
  REQUIRE_EQ(ccv_nnc_tensor_from_variable(graph, a1)->data.f32[1], 25, "should be b1[1, 2]");
99
1
  ccv_nnc_dynamic_graph_free(graph);
100
1
  ccv_nnc_tensor_free(b0);
101
1
  ccv_nnc_tensor_free(b1);
102
1
}
103
104
TEST_CASE("dynamic graph to compute f(x) = x * log(x) + 1.2 * x, f'(x) where x = 19")
105
1
{
106
1
  ccv_nnc_dynamic_graph_t* const graph = ccv_nnc_dynamic_graph_new();
107
1
  ccv_nnc_tensor_variable_t x = ccv_nnc_tensor_variable_new(graph, CPU_TENSOR_NHWC(32F, 1));
108
1
  ccv_nnc_tensor_from_variable(graph, x)->data.f32[0] = 19;
109
1
  ccv_nnc_tensor_variable_t f = ccv_nnc_tensor_variable_new(graph);
110
1
  ccv_nnc_dynamic_graph_exec(graph, CMD_EWLOG_FORWARD(), ccv_nnc_no_hint, 0, TENSOR_VARIABLE_LIST(x), TENSOR_VARIABLE_LIST(f), 0, 0);
111
1
  ccv_nnc_dynamic_graph_exec(graph, CMD_EWPROD_FORWARD(), ccv_nnc_no_hint, 0, TENSOR_VARIABLE_LIST(x, f), TENSOR_VARIABLE_LIST(f), 0, 0);
112
1
  ccv_nnc_tensor_variable_t y = ccv_nnc_tensor_variable_new(graph, CPU_TENSOR_NHWC(32F, 1));
113
1
  ccv_nnc_tensor_from_variable(graph, y)->data.f32[0] = 1.2;
114
1
  ccv_nnc_tensor_variable_t z = ccv_nnc_tensor_variable_new(graph);
115
1
  ccv_nnc_dynamic_graph_exec(graph, CMD_EWPROD_FORWARD(), ccv_nnc_no_hint, 0, TENSOR_VARIABLE_LIST(x, y), TENSOR_VARIABLE_LIST(z), 0, 0);
116
1
  ccv_nnc_dynamic_graph_exec(graph, CMD_EWSUM_FORWARD(), ccv_nnc_no_hint, 0, TENSOR_VARIABLE_LIST(f, z), TENSOR_VARIABLE_LIST(f), 0, 0);
117
1
  REQUIRE_EQ_WITH_TOLERANCE(ccv_nnc_tensor_from_variable(graph, f)->data.f32[0], 19 * logf(19) + 1.2 * 19, 1e-5, "f(x) = 1.2 * 19 + 19 * log(19)");
118
  // Do gradient computation multiple times.
119
1
  ccv_nnc_tensor_variable_t dx = ccv_nnc_tensor_variable_new(graph);
120
1
  ccv_nnc_dynamic_graph_backward(graph, TENSOR_VARIABLE_LIST(f), 0, TENSOR_VARIABLE_LIST(x), TENSOR_VARIABLE_LIST(dx), 0);
121
1
  REQUIRE_EQ_WITH_TOLERANCE(ccv_nnc_tensor_from_variable(graph, dx)->data.f32[0], logf(19) + 1 + 1.2, 1e-5, "f'(x) = 1.2 + log(19) + 19 * 1 / 19");
122
1
  ccv_nnc_tensor_variable_t dy = ccv_nnc_tensor_variable_new(graph);
123
1
  ccv_nnc_dynamic_graph_backward(graph, TENSOR_VARIABLE_LIST(f), 0, TENSOR_VARIABLE_LIST(y), TENSOR_VARIABLE_LIST(dy), 0);
124
1
  REQUIRE_EQ_WITH_TOLERANCE(ccv_nnc_tensor_from_variable(graph, dy)->data.f32[0], 19, 1e-5, "f'(y) = 19");
125
1
  ccv_nnc_tensor_variable_t dz = ccv_nnc_tensor_variable_new(graph);
126
1
  ccv_nnc_dynamic_graph_backward(graph, TENSOR_VARIABLE_LIST(f), 0, TENSOR_VARIABLE_LIST(z), TENSOR_VARIABLE_LIST(dz), 0);
127
1
  REQUIRE_EQ_WITH_TOLERANCE(ccv_nnc_tensor_from_variable(graph, dz)->data.f32[0], 1, 1e-5, "f'(z) = 1");
128
1
  ccv_nnc_tensor_variable_free(graph, dy);
129
1
  dy = ccv_nnc_tensor_variable_new(graph);
130
1
  ccv_nnc_dynamic_graph_exec(graph, CMD_SET_FORWARD(0), ccv_nnc_no_hint, 0, TENSOR_VARIABLE_LIST(), TENSOR_VARIABLE_LIST(dx), 0, 0);
131
1
  ccv_nnc_dynamic_graph_backward(graph, TENSOR_VARIABLE_LIST(f), 0, TENSOR_VARIABLE_LIST(y, x), TENSOR_VARIABLE_LIST(dy, dx), 0);
132
1
  REQUIRE_EQ_WITH_TOLERANCE(ccv_nnc_tensor_from_variable(graph, dx)->data.f32[0], logf(19) + 1 + 1.2, 1e-5, "f'(x) = 1.2 + log(19) + 19 * 1 / 19");
133
1
  REQUIRE_EQ_WITH_TOLERANCE(ccv_nnc_tensor_from_variable(graph, dy)->data.f32[0], 19, 1e-5, "f'(y) = 19");
134
1
  DYNAMIC_GRAPH_GEN(graph, CCV_NNC_LONG_DOT_GRAPH);
135
1
  ccv_nnc_dynamic_graph_free(graph);
136
1
}
137
138
TEST_CASE("dynamic graph with dense net (extensive use of alias)")
139
1
{
140
1
  ccv_nnc_dynamic_graph_t* const graph = ccv_nnc_dynamic_graph_new();
141
1
  ccv_nnc_tensor_variable_t x = ccv_nnc_tensor_variable_new(graph, CPU_TENSOR_NHWC(32F, 1, 4));
142
1
  ccv_nnc_tensor_from_variable(graph, x)->data.f32[0] = 0.472;
143
1
  ccv_nnc_tensor_variable_t x1 = ccv_nnc_tensor_variable_alias_new(graph, x, ccv_nnc_no_ofs, DIM_ALLOC(1, 4), CPU_TENSOR_NHWC(32F, 1, 1));
144
1
  ccv_nnc_tensor_variable_t w1 = ccv_nnc_tensor_variable_new(graph, CPU_TENSOR_NHWC(32F, 1, 1));
145
1
  ccv_nnc_tensor_from_variable(graph, w1)->data.f32[0] = 0.234;
146
1
  ccv_nnc_tensor_variable_t b1 = ccv_nnc_tensor_variable_new(graph, CPU_TENSOR_NHWC(32F, 1));
147
1
  ccv_nnc_tensor_from_variable(graph, b1)->data.f32[0] = 0.1;
148
1
  ccv_nnc_tensor_variable_t x11 = ccv_nnc_tensor_variable_alias_new(graph, x, DIM_ALLOC(0, 1), DIM_ALLOC(1, 4), CPU_TENSOR_NHWC(32F, 1, 1));
149
1
  ccv_nnc_dynamic_graph_exec(graph, CMD_GEMM_FORWARD(NO_TRANSPOSE, TRANSPOSE(0, 1)), ccv_nnc_no_hint, 0, TENSOR_VARIABLE_LIST(x1, w1, b1), TENSOR_VARIABLE_LIST(x11), 0, 0);
150
1
  ccv_nnc_tensor_variable_t x2 = ccv_nnc_tensor_variable_alias_new(graph, x, ccv_nnc_no_ofs, DIM_ALLOC(1, 4), CPU_TENSOR_NHWC(32F, 1, 2));
151
1
  ccv_nnc_tensor_variable_t w2 = ccv_nnc_tensor_variable_new(graph, CPU_TENSOR_NHWC(32F, 1, 2));
152
1
  ccv_nnc_tensor_from_variable(graph, w2)->data.f32[0] = 0.374;
153
1
  ccv_nnc_tensor_from_variable(graph, w2)->data.f32[1] = 0.886;
154
1
  ccv_nnc_tensor_variable_t b2 = ccv_nnc_tensor_variable_new(graph, CPU_TENSOR_NHWC(32F, 1));
155
1
  ccv_nnc_tensor_from_variable(graph, b2)->data.f32[0] = 0.2;
156
1
  ccv_nnc_tensor_variable_t x21 = ccv_nnc_tensor_variable_alias_new(graph, x, DIM_ALLOC(0, 2), DIM_ALLOC(1, 4), CPU_TENSOR_NHWC(32F, 1, 1));
157
1
  ccv_nnc_dynamic_graph_exec(graph, CMD_GEMM_FORWARD(NO_TRANSPOSE, TRANSPOSE(0, 1)), ccv_nnc_no_hint, 0, TENSOR_VARIABLE_LIST(x2, w2, b2), TENSOR_VARIABLE_LIST(x21), 0, 0);
158
1
  ccv_nnc_tensor_variable_t x3 = ccv_nnc_tensor_variable_alias_new(graph, x, ccv_nnc_no_ofs, DIM_ALLOC(1, 4), CPU_TENSOR_NHWC(32F, 1, 3));
159
1
  ccv_nnc_tensor_variable_t w3 = ccv_nnc_tensor_variable_new(graph, CPU_TENSOR_NHWC(32F, 1, 3));
160
1
  ccv_nnc_tensor_from_variable(graph, w3)->data.f32[0] = 0.484;
161
1
  ccv_nnc_tensor_from_variable(graph, w3)->data.f32[1] = 0.912;
162
1
  ccv_nnc_tensor_from_variable(graph, w3)->data.f32[2] = 0.235;
163
1
  ccv_nnc_tensor_variable_t b3 = ccv_nnc_tensor_variable_new(graph, CPU_TENSOR_NHWC(32F, 1));
164
1
  ccv_nnc_tensor_from_variable(graph, b3)->data.f32[0] = 0.3;
165
1
  ccv_nnc_tensor_variable_t x31 = ccv_nnc_tensor_variable_alias_new(graph, x, DIM_ALLOC(0, 3), DIM_ALLOC(1, 4), CPU_TENSOR_NHWC(32F, 1, 1));
166
1
  ccv_nnc_dynamic_graph_exec(graph, CMD_GEMM_FORWARD(NO_TRANSPOSE, TRANSPOSE(0, 1)), ccv_nnc_no_hint, 0, TENSOR_VARIABLE_LIST(x3, w3, b3), TENSOR_VARIABLE_LIST(x31), 0, 0);
167
1
  DYNAMIC_GRAPH_GEN(graph, CCV_NNC_LONG_DOT_GRAPH);
168
1
  ccv_nnc_tensor_t* xt = ccv_nnc_tensor_new(0, CPU_TENSOR_NHWC(32F, 1, 4), 0);
169
1
  xt->data.f32[0] = 0.472;
170
1
  xt->data.f32[1] = xt->data.f32[0] * 0.234 + 0.1;
171
1
  xt->data.f32[2] = xt->data.f32[0] * 0.374 + xt->data.f32[1] * 0.886 + 0.2;
172
1
  xt->data.f32[3] = xt->data.f32[0] * 0.484 + xt->data.f32[1] * 0.912 + xt->data.f32[2] * 0.235 + 0.3;
173
1
  REQUIRE_MATRIX_EQ(ccv_nnc_tensor_from_variable(graph, x), xt, "1x4 matrix should be exactly the same");
174
1
  ccv_nnc_tensor_free(xt);
175
1
  ccv_nnc_tensor_variable_t dw1 = ccv_nnc_tensor_variable_new(graph);
176
1
  ccv_nnc_dynamic_graph_backward(graph, TENSOR_VARIABLE_LIST(x), 0, TENSOR_VARIABLE_LIST(w1), TENSOR_VARIABLE_LIST(dw1), 0);
177
1
  REQUIRE_EQ_WITH_TOLERANCE((0.235 * 0.886 + 0.912) * 0.472, ccv_nnc_tensor_from_variable(graph, dw1)->data.f32[0], 1e-5, "the gradient should be equal to a complicated result");
178
1
  ccv_nnc_dynamic_graph_free(graph);
179
1
}
180
181
TEST_CASE("batch norm in dynamic graph (enforce inplace)")
182
1
{
183
1
  ccv_nnc_dynamic_graph_t* const graph = ccv_nnc_dynamic_graph_new();
184
1
  ccv_nnc_tensor_variable_t x = ccv_nnc_tensor_variable_new(graph, CPU_TENSOR_NHWC(32F, 2, 2, 2, 10));
185
1
  ccv_nnc_tensor_variable_t y = ccv_nnc_tensor_variable_new(graph);
186
1
  ccv_nnc_tensor_variable_t scale = ccv_nnc_tensor_variable_new(graph, CPU_TENSOR_NHWC(32F, 10));
187
1
  ccv_nnc_tensor_variable_t bias = ccv_nnc_tensor_variable_new(graph, CPU_TENSOR_NHWC(32F, 10));
188
1
  ccv_nnc_tensor_variable_t mean = ccv_nnc_tensor_variable_new(graph, CPU_TENSOR_NHWC(32F, 10));
189
1
  ccv_nnc_tensor_variable_t var = ccv_nnc_tensor_variable_new(graph, CPU_TENSOR_NHWC(32F, 10));
190
1
  ccv_nnc_tensor_variable_t saved_mean = ccv_nnc_tensor_variable_new(graph);
191
1
  ccv_nnc_tensor_variable_t saved_inv_std = ccv_nnc_tensor_variable_new(graph);
192
1
  dsfmt_t dsfmt;
193
1
  int i;
194
1
  dsfmt_init_gen_rand(&dsfmt, 1);
195
81
  for (i = 0; i < 2 * 2 * 2 * 10; 
i++80
)
196
80
    ccv_nnc_tensor_from_variable(graph, x)->data.f32[i] = dsfmt_genrand_open_close(&dsfmt);
197
11
  for (i = 0; i < 10; 
i++10
)
198
10
    ccv_nnc_tensor_from_variable(graph, scale)->data.f32[i] = 1;
199
11
  for (i = 0; i < 10; 
i++10
)
200
10
    ccv_nnc_tensor_from_variable(graph, bias)->data.f32[i] = 0;
201
11
  for (i = 0; i < 10; 
i++10
)
202
10
    ccv_nnc_tensor_from_variable(graph, mean)->data.f32[i] = 0;
203
1
  ccv_nnc_tensor_t* mean_tensor_ptr = ccv_nnc_tensor_from_variable(graph, mean);
204
11
  for (i = 0; i < 10; 
i++10
)
205
10
    ccv_nnc_tensor_from_variable(graph, var)->data.f32[i] = 0;
206
1
  ccv_nnc_tensor_t* var_tensor_ptr = ccv_nnc_tensor_from_variable(graph, var);
207
1
  ccv_nnc_dynamic_graph_exec(graph, CMD_BATCH_NORM_FORWARD(0, 0, 0.9, 0, 1, 2), ccv_nnc_no_hint, 0, TENSOR_VARIABLE_LIST(x, scale, bias, mean, var), TENSOR_VARIABLE_LIST(y, mean, var, saved_mean, saved_inv_std), 0, 0);
208
1
  DYNAMIC_GRAPH_GEN(graph, CCV_NNC_LONG_DOT_GRAPH);
209
1
  REQUIRE(mean_tensor_ptr == ccv_nnc_tensor_from_variable(graph, mean), "enforced inplace, tensor view pointer unchanged");
210
1
  REQUIRE(var_tensor_ptr == ccv_nnc_tensor_from_variable(graph, var), "enforced inplace, tensor view pointer unchanged");
211
1
  ccv_nnc_tensor_t* x_tensor = ccv_nnc_tensor_new(0, CPU_TENSOR_NHWC(32F, 2, 2, 2, 10), 0);
212
1
  ccv_nnc_tensor_t* y_tensor = ccv_nnc_tensor_new(0, CPU_TENSOR_NHWC(32F, 2, 2, 2, 10), 0);
213
1
  ccv_nnc_tensor_t* scale_tensor = ccv_nnc_tensor_new(0, CPU_TENSOR_NHWC(32F, 10), 0);
214
1
  ccv_nnc_tensor_t* bias_tensor = ccv_nnc_tensor_new(0, CPU_TENSOR_NHWC(32F, 10), 0);
215
1
  ccv_nnc_tensor_t* mean_tensor = ccv_nnc_tensor_new(0, CPU_TENSOR_NHWC(32F, 10), 0);
216
1
  ccv_nnc_tensor_t* var_tensor = ccv_nnc_tensor_new(0, CPU_TENSOR_NHWC(32F, 10), 0);
217
1
  ccv_nnc_tensor_t* saved_mean_tensor = ccv_nnc_tensor_new(0, CPU_TENSOR_NHWC(32F, 1, 1, 1, 10), 0);
218
1
  ccv_nnc_tensor_t* saved_inv_std_tensor = ccv_nnc_tensor_new(0, CPU_TENSOR_NHWC(32F, 1, 1, 1, 10), 0);
219
1
  memcpy(x_tensor->data.f32, ccv_nnc_tensor_from_variable(graph, x)->data.f32, sizeof(float) * 2 * 2 * 2 * 10);
220
11
  for (i = 0; i < 10; 
i++10
)
221
10
    scale_tensor->data.f32[i] = 1;
222
1
  memset(bias_tensor->data.f32, 0, sizeof(float) * 10);
223
1
  memset(mean_tensor->data.f32, 0, sizeof(float) * 10);
224
1
  memset(var_tensor->data.f32, 0, sizeof(float) * 10);
225
1
  ccv_nnc_cmd_exec(CMD_BATCH_NORM_FORWARD(0, 0, 0.9, 0, 1, 2), ccv_nnc_no_hint, 0, TENSOR_LIST(x_tensor, scale_tensor, bias_tensor, mean_tensor, var_tensor), TENSOR_LIST(y_tensor, mean_tensor, var_tensor, saved_mean_tensor, saved_inv_std_tensor), 0);
226
1
  REQUIRE_TENSOR_EQ(y_tensor, ccv_nnc_tensor_from_variable(graph, y), "y should be equal");
227
1
  REQUIRE_TENSOR_EQ(mean_tensor, ccv_nnc_tensor_from_variable(graph, mean), "mean should be equal");
228
1
  REQUIRE_TENSOR_EQ(var_tensor, ccv_nnc_tensor_from_variable(graph, var), "var should be equal");
229
1
  REQUIRE_TENSOR_EQ(saved_mean_tensor, ccv_nnc_tensor_from_variable(graph, saved_mean), "saved_mean should be equal");
230
1
  REQUIRE_TENSOR_EQ(saved_inv_std_tensor, ccv_nnc_tensor_from_variable(graph, saved_inv_std), "saved_inv_std should be equal");
231
1
  ccv_nnc_dynamic_graph_free(graph);
232
1
  ccv_nnc_tensor_free(x_tensor);
233
1
  ccv_nnc_tensor_free(y_tensor);
234
1
  ccv_nnc_tensor_free(scale_tensor);
235
1
  ccv_nnc_tensor_free(bias_tensor);
236
1
  ccv_nnc_tensor_free(mean_tensor);
237
1
  ccv_nnc_tensor_free(var_tensor);
238
1
  ccv_nnc_tensor_free(saved_mean_tensor);
239
1
  ccv_nnc_tensor_free(saved_inv_std_tensor);
240
1
}
241
242
TEST_CASE("empty inputs / outputs for dynamic graph")
243
1
{
244
1
  ccv_nnc_dynamic_graph_t* const graph = ccv_nnc_dynamic_graph_new();
245
1
  ccv_nnc_tensor_variable_t df = ccv_nnc_tensor_variable_new(graph, CPU_TENSOR_NHWC(32F, 1));
246
1
  ccv_nnc_tensor_variable_t x = ccv_nnc_tensor_variable_new(graph, CPU_TENSOR_NHWC(32F, 1));
247
1
  ccv_nnc_tensor_variable_t y = ccv_nnc_tensor_variable_new(graph);
248
1
  ccv_nnc_tensor_from_variable(graph, df)->data.f32[0] = 1;
249
1
  ccv_nnc_tensor_from_variable(graph, x)->data.f32[0] = 10;
250
1
  ccv_nnc_dynamic_graph_exec(graph, CMD_EWDIV_BACKWARD(), ccv_nnc_no_hint, 0, TENSOR_VARIABLE_LIST(df, 0, x), TENSOR_VARIABLE_LIST(y, 0), 0, 0);
251
1
  DYNAMIC_GRAPH_GEN(graph, CCV_NNC_LONG_DOT_GRAPH);
252
1
  REQUIRE_EQ_WITH_TOLERANCE(ccv_nnc_tensor_from_variable(graph, y)->data.f32[0], 1. / 10, 1e-5, "div backward should equal to 1 / 10");
253
1
  ccv_nnc_dynamic_graph_free(graph);
254
1
}
255
256
TEST_CASE("long dynamic graph with unused variables freed")
257
1
{
258
1
  ccv_nnc_dynamic_graph_t* const graph = ccv_nnc_dynamic_graph_new();
259
1
  int i;
260
1
  ccv_nnc_tensor_variable_t x = ccv_nnc_tensor_variable_new(graph, CPU_TENSOR_NHWC(32F, 1));
261
1
  ccv_nnc_tensor_variable_t y = ccv_nnc_tensor_variable_new(graph, CPU_TENSOR_NHWC(32F, 1));
262
1
  ccv_nnc_tensor_from_variable(graph, x)->data.f32[0] = 32;
263
1
  ccv_nnc_tensor_from_variable(graph, y)->data.f32[0] = 0.5;
264
11
  for (i = 0; i < 10; 
i++10
)
265
10
  {
266
10
    ccv_nnc_tensor_variable_t z = ccv_nnc_tensor_variable_new(graph);
267
10
    if (i < 7)
268
7
      ccv_nnc_dynamic_graph_exec(graph, CMD_EWPROD_FORWARD(), ccv_nnc_no_hint, 0, TENSOR_VARIABLE_LIST(x, y), TENSOR_VARIABLE_LIST(z), 0, 0);
269
3
    else {
270
3
      if (i == 7)
271
1
        ccv_nnc_tensor_variable_free(graph, y); // No longer need y.
272
3
      ccv_nnc_dynamic_graph_exec(graph, CMD_EWPROD_FORWARD(), ccv_nnc_no_hint, 0, TENSOR_VARIABLE_LIST(x, x), TENSOR_VARIABLE_LIST(z), 0, 0);
273
3
    }
274
10
    if (i < 9)
275
9
      ccv_nnc_tensor_variable_free(graph, x);
276
10
    x = z;
277
10
  }
278
1
  DYNAMIC_GRAPH_GEN(graph, CCV_NNC_LONG_DOT_GRAPH);
279
1
  float g = 32;
280
11
  for (i = 0; i < 10; 
i++10
)
281
10
  {
282
10
    if (i < 7)
283
7
      g = g * 0.5;
284
3
    else
285
3
      g = g * g;
286
10
  }
287
1
  REQUIRE_EQ_WITH_TOLERANCE(ccv_nnc_tensor_from_variable(graph, x)->data.f32[0], g, 1e-5, "x should equal to the computed result");
288
1
  ccv_nnc_dynamic_graph_free(graph);
289
1
}
290
291
TEST_CASE("repeat multiple x * y with y as a constant")
292
1
{
293
1
  ccv_nnc_dynamic_graph_t* const graph = ccv_nnc_dynamic_graph_new();
294
1
  int i;
295
1
  ccv_nnc_tensor_variable_t x = ccv_nnc_tensor_variable_new(graph, CPU_TENSOR_NHWC(32F, 1));
296
1
  ccv_nnc_tensor_variable_t y = ccv_nnc_tensor_constant_new(graph, CPU_TENSOR_NHWC(32F, 1));
297
1
  ccv_nnc_tensor_from_variable(graph, x)->data.f32[0] = 32;
298
1
  ccv_nnc_tensor_from_variable(graph, y)->data.f32[0] = 0.5;
299
11
  for (i = 0; i < 10; 
i++10
)
300
10
  {
301
10
    ccv_nnc_tensor_variable_t z = ccv_nnc_tensor_variable_new(graph);
302
10
    ccv_nnc_dynamic_graph_exec(graph, CMD_EWPROD_FORWARD(), ccv_nnc_no_hint, 0, TENSOR_VARIABLE_LIST(x, y), TENSOR_VARIABLE_LIST(z), 0, 0);
303
10
    ccv_nnc_tensor_variable_free(graph, x);
304
10
    x = z;
305
10
  }
306
1
  DYNAMIC_GRAPH_GEN(graph, CCV_NNC_LONG_DOT_GRAPH);
307
1
  float g = 32;
308
11
  for (i = 0; i < 10; 
i++10
)
309
10
    g = g * 0.5;
310
1
  REQUIRE_EQ_WITH_TOLERANCE(ccv_nnc_tensor_from_variable(graph, x)->data.f32[0], g, 1e-5, "x should equal to the computed result");
311
1
  ccv_nnc_dynamic_graph_free(graph);
312
1
}
313
314
static int _ccv_tensor_variable_freed = 0;
315
316
static void _ccv_tensor_variable_hook(ccv_nnc_dynamic_graph_t* const graph, const ccv_nnc_tensor_t* const tensor, void* const context)
317
7
{
318
7
  ++_ccv_tensor_variable_freed;
319
7
}
320
321
TEST_CASE("repeat multiple x * y with y as a constant, compute d(x)")
322
1
{
323
1
  ccv_nnc_dynamic_graph_t* const graph = ccv_nnc_dynamic_graph_new();
324
1
  int i;
325
1
  _ccv_tensor_variable_freed = 0;
326
1
  ccv_nnc_tensor_variable_t x = ccv_nnc_tensor_variable_new(graph, CPU_TENSOR_NHWC(32F, 1));
327
1
  ccv_nnc_tensor_variable_t y = ccv_nnc_tensor_constant_new(graph, CPU_TENSOR_NHWC(32F, 1));
328
1
  ccv_nnc_tensor_variable_t ox = x;
329
1
  ccv_nnc_tensor_variable_destructor_hook(graph, x, _ccv_tensor_variable_hook, 0);
330
1
  ccv_nnc_tensor_variable_destructor_hook(graph, y, _ccv_tensor_variable_hook, 0);
331
1
  ccv_nnc_tensor_from_variable(graph, x)->data.f32[0] = 32;
332
1
  ccv_nnc_tensor_from_variable(graph, y)->data.f32[0] = 0.9;
333
5
  for (i = 0; i < 4; 
i++4
)
334
4
  {
335
4
    ccv_nnc_tensor_variable_t z = ccv_nnc_tensor_variable_new(graph);
336
4
    ccv_nnc_tensor_variable_destructor_hook(graph, z, _ccv_tensor_variable_hook, 0);
337
4
    ccv_nnc_dynamic_graph_exec(graph, CMD_EWPROD_FORWARD(), ccv_nnc_no_hint, 0, TENSOR_VARIABLE_LIST(x, y), TENSOR_VARIABLE_LIST(z), 0, 0);
338
4
    if (i > 0)
339
3
      ccv_nnc_tensor_variable_free(graph, x);
340
4
    x = z;
341
4
  }
342
1
  REQUIRE_EQ(0, _ccv_tensor_variable_freed, "none of these are freed");
343
1
  ccv_nnc_tensor_variable_t dx = ccv_nnc_tensor_variable_new(graph);
344
1
  ccv_nnc_tensor_variable_destructor_hook(graph, dx, _ccv_tensor_variable_hook, 0);
345
1
  ccv_nnc_dynamic_graph_backward(graph, TENSOR_VARIABLE_LIST(x), 0, TENSOR_VARIABLE_LIST(ox), TENSOR_VARIABLE_LIST(dx), 0);
346
1
  ccv_nnc_tensor_variable_free(graph, ox);
347
  // We freed all 4 x (ox, when i = 1, 2, 3).
348
1
  REQUIRE_EQ(4, _ccv_tensor_variable_freed, "all are freed except dx");
349
1
  _ccv_tensor_variable_freed = 0;
350
1
  DYNAMIC_GRAPH_GEN(graph, CCV_NNC_LONG_DOT_GRAPH);
351
1
  float g = 1;
352
5
  for (i = 0; i < 4; 
i++4
)
353
4
    g = g * 0.9;
354
1
  REQUIRE_EQ_WITH_TOLERANCE(ccv_nnc_tensor_from_variable(graph, dx)->data.f32[0], g, 1e-5, "x should equal to the computed result");
355
1
  ccv_nnc_dynamic_graph_free(graph);
356
  // y, dx, and z.
357
1
  REQUIRE_EQ(3, _ccv_tensor_variable_freed, "dx freed");
358
1
}
359
360
TEST_CASE("compute f(x) = x * log(x) + x, f'(x) when x = 10 (and intermediate results all freed)")
361
1
{
362
1
  ccv_nnc_dynamic_graph_t* const graph = ccv_nnc_dynamic_graph_new();
363
1
  ccv_nnc_tensor_variable_t x = ccv_nnc_tensor_variable_new(graph, CPU_TENSOR_NHWC(32F, 1));
364
1
  ccv_nnc_tensor_variable_t y = ccv_nnc_tensor_variable_new(graph);
365
1
  ccv_nnc_tensor_variable_t f = ccv_nnc_tensor_variable_new(graph);
366
1
  ccv_nnc_tensor_from_variable(graph, x)->data.f32[0] = 10;
367
1
  ccv_nnc_dynamic_graph_exec(graph, CMD_EWLOG_FORWARD(), ccv_nnc_no_hint, 0, TENSOR_VARIABLE_LIST(x), TENSOR_VARIABLE_LIST(y), 0, 0);
368
1
  ccv_nnc_dynamic_graph_exec(graph, CMD_EWPROD_FORWARD(), ccv_nnc_no_hint, 0, TENSOR_VARIABLE_LIST(x, y), TENSOR_VARIABLE_LIST(y), 0, 0);
369
1
  ccv_nnc_dynamic_graph_exec(graph, CMD_EWSUM_FORWARD(), ccv_nnc_no_hint, 0, TENSOR_VARIABLE_LIST(y, x), TENSOR_VARIABLE_LIST(f), 0, 0);
370
1
  ccv_nnc_tensor_variable_t df = ccv_nnc_tensor_variable_new(graph, CPU_TENSOR_NHWC(32F, 1));
371
1
  ccv_nnc_dynamic_graph_exec(graph, CMD_SET_FORWARD(1), ccv_nnc_no_hint, 0, 0, 0, TENSOR_VARIABLE_LIST(df), 0, 0);
372
  // x will be accumulated on to itself.
373
1
  ccv_nnc_dynamic_graph_backward(graph, TENSOR_VARIABLE_LIST(f), &df, TENSOR_VARIABLE_LIST(x), TENSOR_VARIABLE_LIST(x), 0);
374
1
  DYNAMIC_GRAPH_GEN(graph, CCV_NNC_LONG_DOT_GRAPH);
375
1
  REQUIRE_EQ_WITH_TOLERANCE(ccv_nnc_tensor_from_variable(graph, x)->data.f32[0], 10 + log(10) + 1 + 1, 1e-5, "dx should equal to the computed result");
376
1
  ccv_nnc_dynamic_graph_free(graph);
377
1
}
378
379
TEST_CASE("dynamic graph with binded value")
380
1
{
381
1
  ccv_nnc_dynamic_graph_t* const graph = ccv_nnc_dynamic_graph_new();
382
1
  ccv_nnc_tensor_variable_t x = ccv_nnc_tensor_variable_new(graph, CPU_TENSOR_NHWC(32F, 1));
383
1
  ccv_nnc_tensor_t* const x_tensor = ccv_nnc_tensor_new(0, CPU_TENSOR_NHWC(32F, 1), 0);
384
1
  x_tensor->data.f32[0] = 10;
385
1
  ccv_nnc_tensor_variable_set(graph, x, x_tensor);
386
1
  ccv_nnc_tensor_variable_t y = ccv_nnc_tensor_variable_new(graph);
387
1
  ccv_nnc_dynamic_graph_exec(graph, CMD_EWLOG_FORWARD(), ccv_nnc_no_hint, 0, TENSOR_VARIABLE_LIST(x), TENSOR_VARIABLE_LIST(y), 0, 0);
388
1
  DYNAMIC_GRAPH_GEN(graph, CCV_NNC_LONG_DOT_GRAPH);
389
1
  REQUIRE_EQ_WITH_TOLERANCE(ccv_nnc_tensor_from_variable(graph, y)->data.f32[0], log(10), 1e-5, "dx should equal to the computed result");
390
1
  ccv_nnc_dynamic_graph_free(graph);
391
1
  ccv_nnc_tensor_free(x_tensor);
392
1
}
393
394
TEST_CASE("dynamic graph to evaluate cnnp model")
395
1
{
396
1
  ccv_nnc_dynamic_graph_t* const graph = ccv_nnc_dynamic_graph_new();
397
1
  ccv_cnnp_model_t* const linear = ccv_cnnp_dense(1, 0, 0);
398
1
  ccv_nnc_tensor_variable_t z = ccv_nnc_tensor_constant_new(graph, CPU_TENSOR_NHWC(32F, 1));
399
1
  ccv_nnc_tensor_from_variable(graph, z)->data.f32[0] = 5;
400
1
  int i;
401
101
  for (i = 0; i < 100; 
i++100
)
402
100
  {
403
100
    ccv_nnc_tensor_variable_t x;
404
100
    if (i % 2 == 1)
405
50
    {
406
50
      x = ccv_nnc_tensor_variable_new(graph, CPU_TENSOR_NHWC(32F, 2, 1));
407
50
      ccv_nnc_tensor_from_variable(graph, x)->data.f32[0] = 10;
408
50
      ccv_nnc_tensor_from_variable(graph, x)->data.f32[1] = 10;
409
50
    } else {
410
50
      x = ccv_nnc_tensor_variable_new(graph, CPU_TENSOR_NHWC(32F, 1));
411
50
      ccv_nnc_tensor_from_variable(graph, x)->data.f32[0] = 10;
412
50
    }
413
100
    ccv_nnc_tensor_variable_t y = ccv_nnc_tensor_variable_new(graph);
414
100
    ccv_nnc_dynamic_graph_exec(graph, CMD_EWLOG_FORWARD(), ccv_nnc_no_hint, 0, TENSOR_VARIABLE_LIST(x), TENSOR_VARIABLE_LIST(y), 0, 0);
415
100
    ccv_nnc_dynamic_graph_evaluate(graph, linear, 0, TENSOR_VARIABLE_LIST(y), TENSOR_VARIABLE_LIST(y), 0, 0);
416
100
    ccv_nnc_dynamic_graph_exec(graph, CMD_ADD_FORWARD(1, -1), ccv_nnc_no_hint, 0, TENSOR_VARIABLE_LIST(y, z), TENSOR_VARIABLE_LIST(y), 0, 0);
417
100
    ccv_nnc_tensor_variable_t f = ccv_nnc_tensor_variable_new(graph);
418
100
    ccv_nnc_dynamic_graph_exec(graph, CMD_EWPROD_FORWARD(), ccv_nnc_no_hint, 0, TENSOR_VARIABLE_LIST(y, y), TENSOR_VARIABLE_LIST(f), 0, 0);
419
100
    ccv_nnc_tensor_variable_t dx = ccv_nnc_tensor_variable_new(graph);
420
100
    ccv_nnc_dynamic_graph_backward(graph, TENSOR_VARIABLE_LIST(f), 0, TENSOR_VARIABLE_LIST(x), TENSOR_VARIABLE_LIST(dx), 0);
421
100
    ccv_cnnp_model_set_minimizer(linear, CMD_SGD_FORWARD(0, 0.01, 1, 0.01, 0, 0), 0, 0, 0);
422
100
    ccv_nnc_dynamic_graph_apply_gradients(graph, CMD_SGD_FORWARD(0, 0.01, 1, 0.01, 0, 0), TENSOR_VARIABLE_LIST(), TENSOR_VARIABLE_LIST(), 0, 0, 0);
423
100
    ccv_nnc_tensor_variable_free(graph, x);
424
100
    ccv_nnc_tensor_variable_free(graph, y);
425
100
    ccv_nnc_tensor_variable_free(graph, f);
426
100
    ccv_nnc_tensor_variable_free(graph, dx);
427
100
  }
428
1
  ccv_nnc_tensor_variable_t x = ccv_nnc_tensor_variable_new(graph, CPU_TENSOR_NHWC(32F, 1));
429
1
  ccv_nnc_tensor_from_variable(graph, x)->data.f32[0] = 10;
430
1
  ccv_nnc_tensor_variable_t y = ccv_nnc_tensor_variable_new(graph);
431
1
  ccv_nnc_dynamic_graph_exec(graph, CMD_EWLOG_FORWARD(), ccv_nnc_no_hint, 0, TENSOR_VARIABLE_LIST(x), TENSOR_VARIABLE_LIST(y), 0, 0);
432
1
  ccv_nnc_dynamic_graph_evaluate(graph, linear, 1, TENSOR_VARIABLE_LIST(y), TENSOR_VARIABLE_LIST(y), 0, 0);
433
1
  REQUIRE_EQ_WITH_TOLERANCE(ccv_nnc_tensor_from_variable(graph, y)->data.f32[0], 5, 1e-2, "linear model should be trained to generate the same value as z");
434
1
  ccv_nnc_tensor_variable_t iy = ccv_nnc_tensor_constant_new(graph, CPU_TENSOR_NHWC(32F, 1));
435
1
  ccv_nnc_tensor_from_variable(graph, iy)->data.f32[0] = log(10);
436
1
  ccv_nnc_tensor_variable_t iz = ccv_nnc_tensor_variable_new(graph, CPU_TENSOR_NHWC(32F, 1));
437
1
  ccv_nnc_dynamic_graph_evaluate(graph, linear, 1, TENSOR_VARIABLE_LIST(iy), TENSOR_VARIABLE_LIST(iz), 0, 0);
438
1
  REQUIRE_EQ_WITH_TOLERANCE(ccv_nnc_tensor_from_variable(graph, iz)->data.f32[0], 5, 1e-2, "linear model should be trained to generate the same value as z");
439
1
  DYNAMIC_GRAPH_GEN(graph, CCV_NNC_LONG_DOT_GRAPH);
440
1
  ccv_nnc_dynamic_graph_free(graph);
441
1
  ccv_cnnp_model_free(linear);
442
1
}
443
444
TEST_CASE("dynamic graph to evaluate cnnp model without any parameters")
445
1
{
446
1
  ccv_nnc_dynamic_graph_t* const graph = ccv_nnc_dynamic_graph_new();
447
1
  ccv_nnc_tensor_variable_t a = ccv_nnc_tensor_variable_new(graph, CPU_TENSOR_NHWC(32F, 1));
448
1
  ccv_nnc_tensor_from_variable(graph, a)->data.f32[0] = 1.23;
449
1
  ccv_nnc_tensor_variable_t b = ccv_nnc_tensor_variable_new(graph, CPU_TENSOR_NHWC(32F, 1));
450
1
  ccv_nnc_tensor_from_variable(graph, b)->data.f32[0] = 2;
451
1
  ccv_nnc_tensor_variable_t c = ccv_nnc_tensor_variable_new(graph, CPU_TENSOR_NHWC(32F, 1));
452
1
  ccv_cnnp_model_t* const mul = ccv_cnnp_mul(1, "mul");
453
1
  ccv_nnc_dynamic_graph_evaluate(graph, mul, 1, TENSOR_VARIABLE_LIST(a, b), TENSOR_VARIABLE_LIST(c), 0, 0);
454
1
  ccv_nnc_tensor_variable_t da = ccv_nnc_tensor_variable_new(graph);
455
1
  ccv_nnc_dynamic_graph_backward(graph, TENSOR_VARIABLE_LIST(c), 0, TENSOR_VARIABLE_LIST(a), TENSOR_VARIABLE_LIST(da), 0);
456
1
  ccv_cnnp_model_set_minimizer(mul, CMD_SGD_FORWARD(0, 0.01, 1, 0.01, 0, 0), 0, 0, 0);
457
1
  ccv_nnc_dynamic_graph_apply_gradients(graph, CMD_SGD_FORWARD(0, 0.01, 1, 0.01, 0, 0), TENSOR_VARIABLE_LIST(), TENSOR_VARIABLE_LIST(), 0, 0, 0);
458
1
  ccv_cnnp_model_free(mul);
459
1
  REQUIRE_EQ_WITH_TOLERANCE(ccv_nnc_tensor_from_variable(graph, c)->data.f32[0], 2.46, 1e-5, "should be equal");
460
1
  REQUIRE_EQ_WITH_TOLERANCE(ccv_nnc_tensor_from_variable(graph, da)->data.f32[0], 2, 1e-5, "should be equal");
461
1
  ccv_nnc_dynamic_graph_free(graph);
462
1
}
463
464
TEST_CASE("dynamic graph to evaluate cnnp model and simply accumulate gradients")
465
1
{
466
1
  ccv_nnc_dynamic_graph_t* const graph = ccv_nnc_dynamic_graph_new();
467
1
  ccv_cnnp_model_t* const linear = ccv_cnnp_dense(1, 0, 0);
468
1
  ccv_nnc_tensor_variable_t z = ccv_nnc_tensor_constant_new(graph, CPU_TENSOR_NHWC(32F, 1));
469
1
  ccv_nnc_tensor_from_variable(graph, z)->data.f32[0] = 5;
470
1
  int i;
471
101
  for (i = 0; i < 100; 
i++100
)
472
100
  {
473
100
    ccv_nnc_tensor_variable_t x;
474
100
    if (i % 2 == 1)
475
50
    {
476
50
      x = ccv_nnc_tensor_variable_new(graph, CPU_TENSOR_NHWC(32F, 2, 1));
477
50
      ccv_nnc_tensor_from_variable(graph, x)->data.f32[0] = 10;
478
50
      ccv_nnc_tensor_from_variable(graph, x)->data.f32[1] = 10;
479
50
    } else {
480
50
      x = ccv_nnc_tensor_variable_new(graph, CPU_TENSOR_NHWC(32F, 1));
481
50
      ccv_nnc_tensor_from_variable(graph, x)->data.f32[0] = 10;
482
50
    }
483
100
    ccv_nnc_tensor_variable_t y = ccv_nnc_tensor_variable_new(graph);
484
100
    ccv_nnc_dynamic_graph_exec(graph, CMD_EWLOG_FORWARD(), ccv_nnc_no_hint, 0, TENSOR_VARIABLE_LIST(x), TENSOR_VARIABLE_LIST(y), 0, 0);
485
100
    ccv_nnc_dynamic_graph_evaluate(graph, linear, 0, TENSOR_VARIABLE_LIST(y), TENSOR_VARIABLE_LIST(y), 0, 0);
486
100
    ccv_nnc_dynamic_graph_exec(graph, CMD_ADD_FORWARD(1, -1), ccv_nnc_no_hint, 0, TENSOR_VARIABLE_LIST(y, z), TENSOR_VARIABLE_LIST(y), 0, 0);
487
100
    ccv_nnc_tensor_variable_t f = ccv_nnc_tensor_variable_new(graph);
488
100
    ccv_nnc_dynamic_graph_exec(graph, CMD_EWPROD_FORWARD(), ccv_nnc_no_hint, 0, TENSOR_VARIABLE_LIST(y, y), TENSOR_VARIABLE_LIST(f), 0, 0);
489
100
    ccv_nnc_tensor_variable_t dx = ccv_nnc_tensor_variable_new(graph);
490
100
    ccv_nnc_dynamic_graph_backward(graph, TENSOR_VARIABLE_LIST(f), 0, TENSOR_VARIABLE_LIST(x), TENSOR_VARIABLE_LIST(dx), 0);
491
100
    ccv_nnc_tensor_variable_free(graph, x);
492
100
    ccv_nnc_tensor_variable_free(graph, y);
493
100
    ccv_nnc_tensor_variable_free(graph, f);
494
100
    ccv_nnc_tensor_variable_free(graph, dx);
495
100
    if ((i % 2) == 1)
496
50
    {
497
50
      ccv_cnnp_model_set_minimizer(linear, CMD_SGD_FORWARD(0, 0.01, 1, 0.01, 0, 0), 0, 0, 0);
498
50
      ccv_nnc_dynamic_graph_apply_gradients(graph, CMD_SGD_FORWARD(0, 0.01, 1, 0.01, 0, 0), TENSOR_VARIABLE_LIST(), TENSOR_VARIABLE_LIST(), 0, 0, 0);
499
50
    }
500
100
  }
501
1
  ccv_nnc_tensor_variable_t x = ccv_nnc_tensor_variable_new(graph, CPU_TENSOR_NHWC(32F, 1));
502
1
  ccv_nnc_tensor_from_variable(graph, x)->data.f32[0] = 10;
503
1
  ccv_nnc_tensor_variable_t y = ccv_nnc_tensor_variable_new(graph);
504
1
  ccv_nnc_dynamic_graph_exec(graph, CMD_EWLOG_FORWARD(), ccv_nnc_no_hint, 0, TENSOR_VARIABLE_LIST(x), TENSOR_VARIABLE_LIST(y), 0, 0);
505
1
  ccv_nnc_dynamic_graph_evaluate(graph, linear, 1, TENSOR_VARIABLE_LIST(y), TENSOR_VARIABLE_LIST(y), 0, 0);
506
1
  REQUIRE_EQ_WITH_TOLERANCE(ccv_nnc_tensor_from_variable(graph, y)->data.f32[0], 5, 1e-2, "linear model should be trained to generate the same value as z");
507
1
  DYNAMIC_GRAPH_GEN(graph, CCV_NNC_LONG_DOT_GRAPH);
508
1
  ccv_nnc_dynamic_graph_free(graph);
509
1
  ccv_cnnp_model_free(linear);
510
1
}
511
512
TEST_CASE("dynamic graph to accumulate gradients cross cnnp models")
513
1
{
514
1
  ccv_nnc_dynamic_graph_t* const graph = ccv_nnc_dynamic_graph_new();
515
1
  ccv_cnnp_model_t* const linear = ccv_cnnp_dense(1, 1, 0);
516
1
  ccv_nnc_tensor_variable_t a = ccv_nnc_tensor_variable_new(graph, CPU_TENSOR_NHWC(32F, 1));
517
1
  ccv_nnc_dynamic_graph_exec(graph, CMD_SET_FORWARD(0.2485), ccv_nnc_no_hint, 0, TENSOR_VARIABLE_LIST(), TENSOR_VARIABLE_LIST(a), 0, 0);
518
1
  const ccv_nnc_tensor_param_t input = CPU_TENSOR_NHWC(32F, 1);
519
1
  ccv_cnnp_model_compile(linear, &input, 1, CMD_NOOP(), CMD_NOOP());
520
1
  ccv_cnnp_model_set_parameter(linear, ccv_cnnp_model_parameters(linear, CCV_CNNP_PARAMETER_SELECT_WEIGHT, 0), ccv_nnc_tensor_from_variable(graph, a));
521
1
  ccv_nnc_tensor_variable_t a_grad = ccv_nnc_tensor_variable_new(graph);
522
1
  ccv_nnc_tensor_variable_t saved_aux = ccv_nnc_tensor_variable_new(graph, CPU_TENSOR_NHWC(32F, 1));
523
1
  int i;
524
1.00k
  for (i = 0; i < 1000; 
i++1.00k
)
525
1.00k
  {
526
1.00k
    ccv_nnc_tensor_variable_t x = ccv_nnc_tensor_variable_new(graph, CPU_TENSOR_NHWC(32F, 1));
527
1.00k
    ccv_nnc_tensor_from_variable(graph, x)->data.f32[0] = i;
528
1.00k
    ccv_nnc_tensor_variable_t t = ccv_nnc_tensor_variable_new(graph, CPU_TENSOR_NHWC(32F, 1));
529
1.00k
    ccv_nnc_tensor_from_variable(graph, t)->data.f32[0] = -i;
530
1.00k
    ccv_nnc_tensor_variable_t y = ccv_nnc_tensor_variable_new(graph);
531
1.00k
    ccv_nnc_dynamic_graph_exec(graph, CMD_MUL_FORWARD(1), ccv_nnc_no_hint, 0, TENSOR_VARIABLE_LIST(x, a), TENSOR_VARIABLE_LIST(y), 0, 0);
532
1.00k
    ccv_nnc_tensor_variable_t z = ccv_nnc_tensor_variable_new(graph);
533
1.00k
    ccv_nnc_dynamic_graph_evaluate(graph, linear, 0, TENSOR_VARIABLE_LIST(y), TENSOR_VARIABLE_LIST(z), 0, 0);
534
1.00k
    ccv_nnc_tensor_variable_t n = ccv_nnc_tensor_variable_new(graph);
535
1.00k
    ccv_nnc_dynamic_graph_exec(graph, CMD_ADD_FORWARD(1, -1), ccv_nnc_no_hint, 0, TENSOR_VARIABLE_LIST(z, t), TENSOR_VARIABLE_LIST(n), 0, 0);
536
1.00k
    ccv_nnc_tensor_variable_t f = ccv_nnc_tensor_variable_new(graph);
537
1.00k
    ccv_nnc_dynamic_graph_exec(graph, CMD_MUL_FORWARD(1), ccv_nnc_no_hint, 0, TENSOR_VARIABLE_LIST(n, n), TENSOR_VARIABLE_LIST(f), 0, 0);
538
1.00k
    ccv_nnc_dynamic_graph_backward(graph, TENSOR_VARIABLE_LIST(f), 0, TENSOR_VARIABLE_LIST(a), TENSOR_VARIABLE_LIST(a_grad), 0);
539
1.00k
    ccv_nnc_tensor_variable_free(graph, n);
540
1.00k
    ccv_nnc_tensor_variable_free(graph, z);
541
1.00k
    ccv_nnc_tensor_variable_free(graph, y);
542
1.00k
    ccv_nnc_tensor_variable_free(graph, x);
543
1.00k
    ccv_nnc_tensor_variable_free(graph, t);
544
1.00k
    ccv_nnc_tensor_variable_free(graph, f);
545
1.00k
    if (((i + 1) % 5) == 0)
546
200
    {
547
200
      DYNAMIC_GRAPH_GEN(graph, CCV_NNC_LONG_DOT_GRAPH);
548
200
      REQUIRE_EQ(ccv_nnc_dynamic_graph_bookkeeping_count(graph, CCV_NNC_SYMBOL_GRAPH_EXEC), 0, "everything should be freed");
549
200
      float lr = 0.001;
550
200
      if (i >= 200)
551
160
        lr = 0.0001;
552
40
      else if (i >= 600)
553
0
        lr = 0.00001;
554
200
      ccv_cnnp_model_set_minimizer(linear, CMD_SGD_FORWARD(0, lr, 0.001, 0, 0, 0), 0, 0, 0);
555
200
      ccv_nnc_dynamic_graph_apply_gradients(graph, CMD_SGD_FORWARD(0, lr, 0.001, 0, 0, 0), TENSOR_VARIABLE_LIST(a_grad), TENSOR_VARIABLE_LIST(a), &saved_aux, 0, 0);
556
200
    }
557
1.00k
  }
558
1
  ccv_nnc_tensor_variable_t x = ccv_nnc_tensor_variable_new(graph, CPU_TENSOR_NHWC(32F, 1));
559
1
  ccv_nnc_tensor_from_variable(graph, x)->data.f32[0] = 5;
560
1
  ccv_nnc_tensor_variable_t y = ccv_nnc_tensor_variable_new(graph);
561
1
  ccv_nnc_dynamic_graph_exec(graph, CMD_MUL_FORWARD(1), ccv_nnc_no_hint, 0, TENSOR_VARIABLE_LIST(x, a), TENSOR_VARIABLE_LIST(y), 0, 0);
562
1
  ccv_nnc_tensor_variable_t z = ccv_nnc_tensor_variable_new(graph);
563
1
  ccv_nnc_dynamic_graph_evaluate(graph, linear, 0, TENSOR_VARIABLE_LIST(y), TENSOR_VARIABLE_LIST(z), 0, 0);
564
1
  REQUIRE_EQ_WITH_TOLERANCE(ccv_nnc_tensor_from_variable(graph, z)->data.f32[0], -5, 1e-2, "linear model should be trained to generate the same value as z");
565
1
  ccv_nnc_dynamic_graph_free(graph);
566
1
  ccv_cnnp_model_free(linear);
567
1
}
568
569
TEST_CASE("dynamic graph to accumulate gradients cross cnnp models with aliases")
570
1
{
571
1
  ccv_nnc_dynamic_graph_t* const graph = ccv_nnc_dynamic_graph_new();
572
1
  ccv_cnnp_model_t* const linear = ccv_cnnp_sequential_new(MODEL_LIST(
573
1
    ccv_cnnp_dense(1, 1, 0),
574
1
    ccv_cnnp_reshape(DIM_ALLOC(1), DIM_ALLOC(), DIM_ALLOC(), 0),
575
1
  ), 0);
576
1
  ccv_nnc_tensor_variable_t a = ccv_nnc_tensor_variable_new(graph, CPU_TENSOR_NHWC(32F, 1));
577
1
  ccv_nnc_dynamic_graph_exec(graph, CMD_SET_FORWARD(0.2485), ccv_nnc_no_hint, 0, TENSOR_VARIABLE_LIST(), TENSOR_VARIABLE_LIST(a), 0, 0);
578
1
  const ccv_nnc_tensor_param_t input = CPU_TENSOR_NHWC(32F, 1);
579
1
  ccv_cnnp_model_compile(linear, &input, 1, CMD_NOOP(), CMD_NOOP());
580
1
  ccv_cnnp_model_set_parameter(linear, ccv_cnnp_model_parameters(linear, CCV_CNNP_PARAMETER_SELECT_WEIGHT, 0), ccv_nnc_tensor_from_variable(graph, a));
581
1
  ccv_nnc_tensor_variable_t a_grad = ccv_nnc_tensor_variable_new(graph);
582
1
  ccv_nnc_tensor_variable_t saved_aux = ccv_nnc_tensor_variable_new(graph, CPU_TENSOR_NHWC(32F, 1));
583
1
  int i;
584
1.00k
  for (i = 0; i < 1000; 
i++1.00k
)
585
1.00k
  {
586
1.00k
    ccv_nnc_tensor_variable_t x = ccv_nnc_tensor_variable_new(graph, CPU_TENSOR_NHWC(32F, 1));
587
1.00k
    ccv_nnc_tensor_from_variable(graph, x)->data.f32[0] = i;
588
1.00k
    ccv_nnc_tensor_variable_t t = ccv_nnc_tensor_variable_new(graph, CPU_TENSOR_NHWC(32F, 1));
589
1.00k
    ccv_nnc_tensor_from_variable(graph, t)->data.f32[0] = -i;
590
1.00k
    ccv_nnc_tensor_variable_t y = ccv_nnc_tensor_variable_new(graph);
591
1.00k
    ccv_nnc_dynamic_graph_exec(graph, CMD_MUL_FORWARD(1), ccv_nnc_no_hint, 0, TENSOR_VARIABLE_LIST(x, a), TENSOR_VARIABLE_LIST(y), 0, 0);
592
1.00k
    ccv_nnc_tensor_variable_t y_alias = ccv_nnc_tensor_variable_alias_new(graph, y, DIM_ALLOC(), DIM_ALLOC(1), CPU_TENSOR_NHWC(32F, 1));
593
1.00k
    ccv_nnc_tensor_variable_t z = ccv_nnc_tensor_variable_new(graph);
594
1.00k
    ccv_nnc_dynamic_graph_evaluate(graph, linear, 0, TENSOR_VARIABLE_LIST(y_alias), TENSOR_VARIABLE_LIST(z), 0, 0);
595
1.00k
    ccv_nnc_tensor_variable_t n = ccv_nnc_tensor_variable_new(graph);
596
1.00k
    ccv_nnc_dynamic_graph_exec(graph, CMD_ADD_FORWARD(1, -1), ccv_nnc_no_hint, 0, TENSOR_VARIABLE_LIST(z, t), TENSOR_VARIABLE_LIST(n), 0, 0);
597
1.00k
    ccv_nnc_tensor_variable_t f = ccv_nnc_tensor_variable_new(graph);
598
1.00k
    ccv_nnc_dynamic_graph_exec(graph, CMD_MUL_FORWARD(1), ccv_nnc_no_hint, 0, TENSOR_VARIABLE_LIST(n, n), TENSOR_VARIABLE_LIST(f), 0, 0);
599
1.00k
    ccv_nnc_dynamic_graph_backward(graph, TENSOR_VARIABLE_LIST(f), 0, TENSOR_VARIABLE_LIST(a), TENSOR_VARIABLE_LIST(a_grad), 0);
600
1.00k
    ccv_nnc_tensor_variable_free(graph, n);
601
1.00k
    ccv_nnc_tensor_variable_free(graph, z);
602
1.00k
    ccv_nnc_tensor_variable_free(graph, y);
603
1.00k
    ccv_nnc_tensor_variable_free(graph, y_alias);
604
1.00k
    ccv_nnc_tensor_variable_free(graph, x);
605
1.00k
    ccv_nnc_tensor_variable_free(graph, t);
606
1.00k
    ccv_nnc_tensor_variable_free(graph, f);
607
1.00k
    if (((i + 1) % 5) == 0)
608
200
    {
609
200
      REQUIRE_EQ(ccv_nnc_dynamic_graph_bookkeeping_count(graph, CCV_NNC_SYMBOL_GRAPH_EXEC), 0, "everything should be freed");
610
200
      DYNAMIC_GRAPH_GEN(graph, CCV_NNC_LONG_DOT_GRAPH);
611
200
      float lr = 0.001;
612
200
      if (i >= 200)
613
160
        lr = 0.0001;
614
40
      else if (i >= 600)
615
0
        lr = 0.00001;
616
200
      ccv_cnnp_model_set_minimizer(linear, CMD_SGD_FORWARD(0, lr, 0.001, 0, 0, 0), 0, 0, 0);
617
200
      ccv_nnc_dynamic_graph_apply_gradients(graph, CMD_SGD_FORWARD(0, lr, 0.001, 0, 0, 0), TENSOR_VARIABLE_LIST(a_grad), TENSOR_VARIABLE_LIST(a), &saved_aux, 0, 0);
618
200
    }
619
1.00k
  }
620
1
  ccv_nnc_tensor_variable_t x = ccv_nnc_tensor_variable_new(graph, CPU_TENSOR_NHWC(32F, 1));
621
1
  ccv_nnc_tensor_from_variable(graph, x)->data.f32[0] = 5;
622
1
  ccv_nnc_tensor_variable_t y = ccv_nnc_tensor_variable_new(graph);
623
1
  ccv_nnc_dynamic_graph_exec(graph, CMD_MUL_FORWARD(1), ccv_nnc_no_hint, 0, TENSOR_VARIABLE_LIST(x, a), TENSOR_VARIABLE_LIST(y), 0, 0);
624
1
  ccv_nnc_tensor_variable_t z = ccv_nnc_tensor_variable_new(graph);
625
1
  ccv_nnc_dynamic_graph_evaluate(graph, linear, 0, TENSOR_VARIABLE_LIST(y), TENSOR_VARIABLE_LIST(z), 0, 0);
626
1
  REQUIRE_EQ_WITH_TOLERANCE(ccv_nnc_tensor_from_variable(graph, z)->data.f32[0], -5, 1e-2, "linear model should be trained to generate the same value as z");
627
1
  ccv_nnc_dynamic_graph_free(graph);
628
1
  ccv_cnnp_model_free(linear);
629
1
}
630
631
TEST_CASE("dynamic graph to compute f(x) = x * log(x) + 1.2 * x, f'(x) and sum on x = 19, 10")
632
1
{
633
1
  ccv_nnc_dynamic_graph_t* const graph = ccv_nnc_dynamic_graph_new();
634
1
  ccv_nnc_tensor_variable_t x = ccv_nnc_tensor_variable_new(graph, CPU_TENSOR_NHWC(32F, 1));
635
1
  ccv_nnc_tensor_from_variable(graph, x)->data.f32[0] = 19;
636
1
  ccv_nnc_tensor_variable_t f = ccv_nnc_tensor_variable_new(graph);
637
1
  ccv_nnc_dynamic_graph_exec(graph, CMD_EWLOG_FORWARD(), ccv_nnc_no_hint, 0, TENSOR_VARIABLE_LIST(x), TENSOR_VARIABLE_LIST(f), 0, 0);
638
1
  ccv_nnc_dynamic_graph_exec(graph, CMD_EWPROD_FORWARD(), ccv_nnc_no_hint, 0, TENSOR_VARIABLE_LIST(x, f), TENSOR_VARIABLE_LIST(f), 0, 0);
639
1
  ccv_nnc_tensor_variable_t y = ccv_nnc_tensor_variable_new(graph, CPU_TENSOR_NHWC(32F, 1));
640
1
  ccv_nnc_tensor_from_variable(graph, y)->data.f32[0] = 1.2;
641
1
  ccv_nnc_tensor_variable_t z = ccv_nnc_tensor_variable_new(graph);
642
1
  ccv_nnc_dynamic_graph_exec(graph, CMD_EWPROD_FORWARD(), ccv_nnc_no_hint, 0, TENSOR_VARIABLE_LIST(x, y), TENSOR_VARIABLE_LIST(z), 0, 0);
643
1
  ccv_nnc_dynamic_graph_exec(graph, CMD_EWSUM_FORWARD(), ccv_nnc_no_hint, 0, TENSOR_VARIABLE_LIST(f, z), TENSOR_VARIABLE_LIST(f), 0, 0);
644
1
  REQUIRE_EQ_WITH_TOLERANCE(ccv_nnc_tensor_from_variable(graph, f)->data.f32[0], 19 * logf(19) + 1.2 * 19, 1e-5, "f(x) = 1.2 * 19 + 19 * log(19)");
645
  // Do gradient computation multiple times.
646
1
  ccv_nnc_tensor_variable_t dx = ccv_nnc_tensor_variable_new(graph);
647
1
  ccv_nnc_dynamic_graph_backward(graph, TENSOR_VARIABLE_LIST(f), 0, TENSOR_VARIABLE_LIST(x), TENSOR_VARIABLE_LIST(dx), 0);
648
1
  REQUIRE_EQ_WITH_TOLERANCE(ccv_nnc_tensor_from_variable(graph, dx)->data.f32[0], logf(19) + 1 + 1.2, 1e-5, "f'(x) = 1.2 + log(19) + 19 * 1 / 19");
649
1
  ccv_nnc_tensor_variable_free(graph, x);
650
1
  x = ccv_nnc_tensor_variable_new(graph, CPU_TENSOR_NHWC(32F, 1));
651
1
  ccv_nnc_tensor_from_variable(graph, x)->data.f32[0] = 10;
652
1
  ccv_nnc_dynamic_graph_exec(graph, CMD_EWLOG_FORWARD(), ccv_nnc_no_hint, 0, TENSOR_VARIABLE_LIST(x), TENSOR_VARIABLE_LIST(f), 0, 0);
653
1
  ccv_nnc_dynamic_graph_exec(graph, CMD_EWPROD_FORWARD(), ccv_nnc_no_hint, 0, TENSOR_VARIABLE_LIST(x, f), TENSOR_VARIABLE_LIST(f), 0, 0);
654
1
  ccv_nnc_tensor_from_variable(graph, y)->data.f32[0] = 1.2;
655
1
  ccv_nnc_dynamic_graph_exec(graph, CMD_EWPROD_FORWARD(), ccv_nnc_no_hint, 0, TENSOR_VARIABLE_LIST(x, y), TENSOR_VARIABLE_LIST(z), 0, 0);
656
1
  ccv_nnc_dynamic_graph_exec(graph, CMD_EWSUM_FORWARD(), ccv_nnc_no_hint, 0, TENSOR_VARIABLE_LIST(f, z), TENSOR_VARIABLE_LIST(f), 0, 0);
657
1
  REQUIRE_EQ_WITH_TOLERANCE(ccv_nnc_tensor_from_variable(graph, f)->data.f32[0], 10 * logf(10) + 1.2 * 10, 1e-5, "f(x) = 1.2 * 10 + 10 * log(10)");
658
1
  ccv_nnc_dynamic_graph_backward(graph, TENSOR_VARIABLE_LIST(f), 0, TENSOR_VARIABLE_LIST(x), TENSOR_VARIABLE_LIST(dx), 0);
659
1
  REQUIRE_EQ_WITH_TOLERANCE(ccv_nnc_tensor_from_variable(graph, dx)->data.f32[0], logf(19) + 1 + 1.2 + logf(10) + 1 + 1.2, 1e-5, "f'(x) = 1.2 + log(19) + 19 * 1 / 19 + 1.2 + log(19) + 19 * 1 / 19");
660
1
  DYNAMIC_GRAPH_GEN(graph, CCV_NNC_LONG_DOT_GRAPH);
661
1
  ccv_nnc_dynamic_graph_free(graph);
662
1
}
663
664
TEST_CASE("dynamic graph to compute f(x) = x * log(x) + y' where y = 1.2 * x and y' is an alias to y")
665
1
{
666
1
  ccv_nnc_dynamic_graph_t* const graph = ccv_nnc_dynamic_graph_new();
667
1
  ccv_nnc_tensor_variable_t x = ccv_nnc_tensor_variable_new(graph, CPU_TENSOR_NHWC(32F, 1));
668
1
  ccv_nnc_tensor_from_variable(graph, x)->data.f32[0] = 19;
669
1
  ccv_nnc_tensor_variable_t f = ccv_nnc_tensor_variable_new(graph);
670
1
  ccv_nnc_dynamic_graph_exec(graph, CMD_EWLOG_FORWARD(), ccv_nnc_no_hint, 0, TENSOR_VARIABLE_LIST(x), TENSOR_VARIABLE_LIST(f), 0, 0);
671
1
  ccv_nnc_dynamic_graph_exec(graph, CMD_EWPROD_FORWARD(), ccv_nnc_no_hint, 0, TENSOR_VARIABLE_LIST(x, f), TENSOR_VARIABLE_LIST(f), 0, 0);
672
1
  ccv_nnc_tensor_variable_t z = ccv_nnc_tensor_variable_new(graph, CPU_TENSOR_NHWC(32F, 1));
673
1
  ccv_nnc_tensor_from_variable(graph, z)->data.f32[0] = 1.2;
674
1
  ccv_nnc_tensor_variable_t y = ccv_nnc_tensor_variable_new(graph, CPU_TENSOR_NHWC(32F, 1));
675
1
  ccv_nnc_tensor_variable_t y_hat = ccv_nnc_tensor_variable_alias_new(graph, y, ccv_nnc_no_ofs, DIM_ALLOC(), CPU_TENSOR_NHWC(32F, 1));
676
1
  ccv_nnc_dynamic_graph_exec(graph, CMD_EWPROD_FORWARD(), ccv_nnc_no_hint, 0, TENSOR_VARIABLE_LIST(x, z), TENSOR_VARIABLE_LIST(y), 0, 0);
677
1
  ccv_nnc_dynamic_graph_exec(graph, CMD_EWSUM_FORWARD(), ccv_nnc_no_hint, 0, TENSOR_VARIABLE_LIST(f, y_hat), TENSOR_VARIABLE_LIST(f), 0, 0);
678
1
  REQUIRE_EQ_WITH_TOLERANCE(ccv_nnc_tensor_from_variable(graph, f)->data.f32[0], 19 * logf(19) + 1.2 * 19, 1e-5, "f(x) = 1.2 * 19 + 19 * log(19)");
679
1
  ccv_nnc_tensor_variable_free(graph, y_hat);
680
1
  ccv_nnc_tensor_variable_free(graph, f);
681
1
  ccv_nnc_tensor_variable_free(graph, x);
682
1
  ccv_nnc_tensor_variable_free(graph, z);
683
1
  ccv_nnc_tensor_variable_free(graph, y);
684
1
  REQUIRE_EQ(ccv_nnc_dynamic_graph_bookkeeping_count(graph, CCV_NNC_SYMBOL_TENSOR), 0, "should have no tensor left");
685
1
  ccv_nnc_dynamic_graph_free(graph);
686
1
}
687
688
TEST_CASE("dynamic graph to compute f(x) = x * log(x) + y' where y'' = 1.2 * x and both y'' and y' are aliases to y")
689
1
{
690
1
  ccv_nnc_dynamic_graph_t* const graph = ccv_nnc_dynamic_graph_new();
691
1
  ccv_nnc_tensor_variable_t x = ccv_nnc_tensor_variable_new(graph, CPU_TENSOR_NHWC(32F, 1));
692
1
  ccv_nnc_tensor_from_variable(graph, x)->data.f32[0] = 19;
693
1
  ccv_nnc_tensor_variable_t f = ccv_nnc_tensor_variable_new(graph);
694
1
  ccv_nnc_dynamic_graph_exec(graph, CMD_EWLOG_FORWARD(), ccv_nnc_no_hint, 0, TENSOR_VARIABLE_LIST(x), TENSOR_VARIABLE_LIST(f), 0, 0);
695
1
  ccv_nnc_dynamic_graph_exec(graph, CMD_EWPROD_FORWARD(), ccv_nnc_no_hint, 0, TENSOR_VARIABLE_LIST(x, f), TENSOR_VARIABLE_LIST(f), 0, 0);
696
1
  ccv_nnc_tensor_variable_t z = ccv_nnc_tensor_variable_new(graph, CPU_TENSOR_NHWC(32F, 1));
697
1
  ccv_nnc_tensor_from_variable(graph, z)->data.f32[0] = 1.2;
698
1
  ccv_nnc_tensor_variable_t y = ccv_nnc_tensor_variable_new(graph, CPU_TENSOR_NHWC(32F, 1));
699
1
  ccv_nnc_tensor_variable_t y_hat = ccv_nnc_tensor_variable_alias_new(graph, y, ccv_nnc_no_ofs, DIM_ALLOC(), CPU_TENSOR_NHWC(32F, 1));
700
1
  ccv_nnc_tensor_variable_t y_double_hat = ccv_nnc_tensor_variable_alias_new(graph, y, ccv_nnc_no_ofs, DIM_ALLOC(), CPU_TENSOR_NHWC(32F, 1));
701
1
  ccv_nnc_dynamic_graph_exec(graph, CMD_EWPROD_FORWARD(), ccv_nnc_no_hint, 0, TENSOR_VARIABLE_LIST(x, z), TENSOR_VARIABLE_LIST(y_double_hat), 0, 0);
702
1
  ccv_nnc_dynamic_graph_exec(graph, CMD_EWSUM_FORWARD(), ccv_nnc_no_hint, 0, TENSOR_VARIABLE_LIST(f, y_hat), TENSOR_VARIABLE_LIST(f), 0, 0);
703
1
  REQUIRE_EQ_WITH_TOLERANCE(ccv_nnc_tensor_from_variable(graph, f)->data.f32[0], 19 * logf(19) + 1.2 * 19, 1e-5, "f(x) = 1.2 * 19 + 19 * log(19)");
704
1
  ccv_nnc_tensor_variable_free(graph, y_hat);
705
1
  ccv_nnc_tensor_variable_free(graph, y_double_hat);
706
1
  ccv_nnc_tensor_variable_free(graph, f);
707
1
  ccv_nnc_tensor_variable_free(graph, x);
708
1
  ccv_nnc_tensor_variable_free(graph, z);
709
1
  ccv_nnc_tensor_variable_free(graph, y);
710
1
  REQUIRE_EQ(ccv_nnc_dynamic_graph_bookkeeping_count(graph, CCV_NNC_SYMBOL_TENSOR), 0, "should have no tensor left");
711
1
  ccv_nnc_dynamic_graph_free(graph);
712
1
}
713
714
TEST_CASE("dynamic graph to compute f(x) = x * y, where y[0] = 10 * z, y[1] = 2 * z, z = [2], x = [10], should free all variables")
715
1
{
716
1
  ccv_nnc_dynamic_graph_t* const graph = ccv_nnc_dynamic_graph_new();
717
1
  ccv_nnc_tensor_variable_t x = ccv_nnc_tensor_variable_new(graph, CPU_TENSOR_NHWC(32F, 1));
718
1
  ccv_nnc_tensor_variable_t y = ccv_nnc_tensor_variable_new(graph, CPU_TENSOR_NHWC(32F, 2));
719
1
  ccv_nnc_tensor_variable_t z = ccv_nnc_tensor_variable_new(graph, CPU_TENSOR_NHWC(32F, 1));
720
1
  ccv_nnc_dynamic_graph_exec(graph, CMD_SET_FORWARD(2), ccv_nnc_no_hint, 0, TENSOR_VARIABLE_LIST(), TENSOR_VARIABLE_LIST(z), 0, 0);
721
1
  ccv_nnc_dynamic_graph_exec(graph, CMD_SET_FORWARD(10), ccv_nnc_no_hint, 0, TENSOR_VARIABLE_LIST(), TENSOR_VARIABLE_LIST(x), 0, 0);
722
1
  ccv_nnc_tensor_variable_t y_1 = ccv_nnc_tensor_variable_alias_new(graph, y, ccv_nnc_no_ofs, DIM_ALLOC(2), CPU_TENSOR_NHWC(32F, 1));
723
1
  ccv_nnc_tensor_variable_t y_2 = ccv_nnc_tensor_variable_alias_new(graph, y, DIM_ALLOC(1), DIM_ALLOC(2), CPU_TENSOR_NHWC(32F, 1));
724
1
  ccv_nnc_dynamic_graph_exec(graph, CMD_SCALAR_MUL_FORWARD(10), ccv_nnc_no_hint, 0, TENSOR_VARIABLE_LIST(z), TENSOR_VARIABLE_LIST(y_1), 0, 0);
725
1
  ccv_nnc_dynamic_graph_exec(graph, CMD_SCALAR_MUL_FORWARD(2), ccv_nnc_no_hint, 0, TENSOR_VARIABLE_LIST(z), TENSOR_VARIABLE_LIST(y_2), 0, 0);
726
1
  ccv_nnc_tensor_variable_t f = ccv_nnc_tensor_variable_new(graph);
727
1
  ccv_nnc_dynamic_graph_exec(graph, CMD_MUL_FORWARD(1), ccv_nnc_no_hint, 0, TENSOR_VARIABLE_LIST(x, y), TENSOR_VARIABLE_LIST(f), 0, 0);
728
1
  float gt[] = {10 * 2 * 10, 2 * 2 * 10};
729
1
  REQUIRE_ARRAY_EQ_WITH_TOLERANCE(float, ccv_nnc_tensor_from_variable(graph, f)->data.f32, gt, 2, 1e-5, "should be equal");
730
1
  ccv_nnc_tensor_variable_free(graph, z);
731
1
  ccv_nnc_tensor_variable_free(graph, y_1);
732
1
  ccv_nnc_tensor_variable_free(graph, y_2);
733
1
  ccv_nnc_tensor_variable_free(graph, x);
734
1
  ccv_nnc_tensor_variable_free(graph, y);
735
1
  ccv_nnc_tensor_variable_free(graph, f);
736
1
  REQUIRE_EQ(ccv_nnc_dynamic_graph_bookkeeping_count(graph, CCV_NNC_SYMBOL_GRAPH_EXEC), 0, "everything should be freed");
737
1
  REQUIRE_EQ(ccv_nnc_dynamic_graph_bookkeeping_count(graph, CCV_NNC_SYMBOL_TENSOR), 0, "everything should be freed");
738
1
  ccv_nnc_dynamic_graph_free(graph);
739
1
}
740
741
TEST_CASE("dynamic graph to compute f(x) = x * y, where y[0] = 10 * z, y[1] = 2 * z, z = [2], x = [10], freed y[0], y[1] before use")
742
1
{
743
1
  ccv_nnc_dynamic_graph_t* const graph = ccv_nnc_dynamic_graph_new();
744
1
  ccv_nnc_tensor_variable_t x = ccv_nnc_tensor_variable_new(graph, CPU_TENSOR_NHWC(32F, 1));
745
1
  ccv_nnc_tensor_variable_t y = ccv_nnc_tensor_variable_new(graph, CPU_TENSOR_NHWC(32F, 2));
746
1
  ccv_nnc_tensor_variable_t z = ccv_nnc_tensor_variable_new(graph, CPU_TENSOR_NHWC(32F, 1));
747
1
  ccv_nnc_dynamic_graph_exec(graph, CMD_SET_FORWARD(2), ccv_nnc_no_hint, 0, TENSOR_VARIABLE_LIST(), TENSOR_VARIABLE_LIST(z), 0, 0);
748
1
  ccv_nnc_dynamic_graph_exec(graph, CMD_SET_FORWARD(10), ccv_nnc_no_hint, 0, TENSOR_VARIABLE_LIST(), TENSOR_VARIABLE_LIST(x), 0, 0);
749
1
  ccv_nnc_tensor_variable_t y_1 = ccv_nnc_tensor_variable_alias_new(graph, y, ccv_nnc_no_ofs, DIM_ALLOC(2), CPU_TENSOR_NHWC(32F, 1));
750
1
  ccv_nnc_tensor_variable_t y_2 = ccv_nnc_tensor_variable_alias_new(graph, y, DIM_ALLOC(1), DIM_ALLOC(2), CPU_TENSOR_NHWC(32F, 1));
751
1
  ccv_nnc_dynamic_graph_exec(graph, CMD_SCALAR_MUL_FORWARD(10), ccv_nnc_no_hint, 0, TENSOR_VARIABLE_LIST(z), TENSOR_VARIABLE_LIST(y_1), 0, 0);
752
1
  ccv_nnc_dynamic_graph_exec(graph, CMD_SCALAR_MUL_FORWARD(2), ccv_nnc_no_hint, 0, TENSOR_VARIABLE_LIST(z), TENSOR_VARIABLE_LIST(y_2), 0, 0);
753
1
  ccv_nnc_tensor_variable_free(graph, y_1);
754
1
  ccv_nnc_tensor_variable_free(graph, y_2);
755
1
  ccv_nnc_tensor_variable_t f = ccv_nnc_tensor_variable_new(graph);
756
1
  ccv_nnc_dynamic_graph_exec(graph, CMD_MUL_FORWARD(1), ccv_nnc_no_hint, 0, TENSOR_VARIABLE_LIST(x, y), TENSOR_VARIABLE_LIST(f), 0, 0);
757
1
  REQUIRE_EQ(ccv_nnc_dynamic_graph_bookkeeping_count(graph, CCV_NNC_SYMBOL_GRAPH_EXEC), 3, "should keep 3 ops");
758
1
  float gt[] = {10 * 2 * 10, 2 * 2 * 10};
759
1
  REQUIRE_ARRAY_EQ_WITH_TOLERANCE(float, ccv_nnc_tensor_from_variable(graph, f)->data.f32, gt, 2, 1e-5, "should be equal");
760
1
  ccv_nnc_tensor_variable_free(graph, z);
761
1
  ccv_nnc_tensor_variable_free(graph, x);
762
1
  ccv_nnc_tensor_variable_free(graph, y);
763
1
  ccv_nnc_tensor_variable_free(graph, f);
764
1
  REQUIRE_EQ(ccv_nnc_dynamic_graph_bookkeeping_count(graph, CCV_NNC_SYMBOL_GRAPH_EXEC), 0, "everything should be freed");
765
1
  REQUIRE_EQ(ccv_nnc_dynamic_graph_bookkeeping_count(graph, CCV_NNC_SYMBOL_TENSOR), 0, "everything should be freed");
766
1
  ccv_nnc_dynamic_graph_free(graph);
767
1
}
768
769
TEST_CASE("dynamic graph to compute f(x) = x * y', where y[0, 0] = 2, y[0, 1] = 4, x = [10], y' = y^T, gradient against y")
770
1
{
771
1
  ccv_nnc_dynamic_graph_t* const graph = ccv_nnc_dynamic_graph_new();
772
1
  ccv_nnc_tensor_variable_t x = ccv_nnc_tensor_variable_new(graph, CPU_TENSOR_NHWC(32F, 1));
773
1
  ccv_nnc_tensor_variable_t y = ccv_nnc_tensor_variable_new(graph, CPU_TENSOR_NHWC(32F, 2));
774
1
  ccv_nnc_tensor_from_variable(graph, y)->data.f32[0] = 2;
775
1
  ccv_nnc_tensor_from_variable(graph, y)->data.f32[0] = 4;
776
1
  ccv_nnc_dynamic_graph_exec(graph, CMD_SET_FORWARD(10), ccv_nnc_no_hint, 0, TENSOR_VARIABLE_LIST(), TENSOR_VARIABLE_LIST(x), 0, 0);
777
1
  ccv_nnc_tensor_variable_t y_t = ccv_nnc_tensor_variable_alias_new(graph, y, ccv_nnc_no_ofs, DIM_ALLOC(), CPU_TENSOR_NHWC(32F, 2, 1));
778
1
  ccv_nnc_tensor_variable_t f = ccv_nnc_tensor_variable_new(graph);
779
1
  ccv_nnc_dynamic_graph_exec(graph, CMD_MUL_FORWARD(1), ccv_nnc_no_hint, 0, TENSOR_VARIABLE_LIST(x, y_t), TENSOR_VARIABLE_LIST(f), 0, 0);
780
1
  ccv_nnc_tensor_variable_t dy_t = ccv_nnc_tensor_variable_new(graph);
781
1
  ccv_nnc_tensor_variable_free(graph, x); // Cannot free y_t, it has to be freed in lock step with y.
782
1
  ccv_nnc_tensor_variable_free(graph, y);
783
1
  ccv_nnc_dynamic_graph_backward(graph, TENSOR_VARIABLE_LIST(f), 0, TENSOR_VARIABLE_LIST(y_t), TENSOR_VARIABLE_LIST(dy_t), 0);
784
1
  float gt[] = {10, 10};
785
1
  REQUIRE_ARRAY_EQ_WITH_TOLERANCE(float, ccv_nnc_tensor_from_variable(graph, dy_t)->data.f32, gt, 2, 1e-5, "should be equal");
786
1
  ccv_nnc_tensor_variable_free(graph, y_t);
787
1
  ccv_nnc_tensor_variable_free(graph, f);
788
1
  REQUIRE_EQ(ccv_nnc_dynamic_graph_bookkeeping_count(graph, CCV_NNC_SYMBOL_GRAPH_EXEC), 0, "everything should be freed");
789
1
  REQUIRE_EQ(ccv_nnc_dynamic_graph_bookkeeping_count(graph, CCV_NNC_SYMBOL_TENSOR), 0, "everything should be freed");
790
1
  ccv_nnc_dynamic_graph_free(graph);
791
1
}
792
793
TEST_CASE("long chain to autograd, simulate random free of unused tensors due to garbage collection")
794
1
{
795
1
  ccv_nnc_dynamic_graph_t* const graph = ccv_nnc_dynamic_graph_new();
796
1
  ccv_nnc_tensor_variable_t x[10];
797
1
  ccv_nnc_tensor_variable_t f[10];
798
1
  ccv_nnc_tensor_variable_t y = ccv_nnc_tensor_variable_new(graph, CPU_TENSOR_NHWC(32F, 1));
799
1
  ccv_nnc_tensor_variable_t dy[3];
800
1
  ccv_nnc_tensor_from_variable(graph, y)->data.f32[0] = 1.3;
801
1
  int i, j;
802
11
  for (i = 0; i < 10; 
i++10
)
803
10
  {
804
10
    x[i] = ccv_nnc_tensor_variable_new(graph, CPU_TENSOR_NHWC(32F, 1));
805
10
    ccv_nnc_tensor_from_variable(graph, x[i])->data.f32[0] = i + 1;
806
10
    f[i] = ccv_nnc_tensor_variable_new(graph);
807
10
    ccv_nnc_dynamic_graph_exec(graph, CMD_MUL_FORWARD(1), ccv_nnc_no_hint, 0, TENSOR_VARIABLE_LIST(x[i], y), TENSOR_VARIABLE_LIST(f[i]), 0, 0);
808
10
    if (i >= 5)
809
5
    {
810
5
      if (i - 5 < 3)
811
3
        dy[i - 5] = ccv_nnc_tensor_variable_new(graph);
812
5
      ccv_nnc_dynamic_graph_backward(graph, TENSOR_VARIABLE_LIST(f[i]), 0, TENSOR_VARIABLE_LIST(y), TENSOR_VARIABLE_LIST(dy[ccv_min(2, i - 5)]), 0);
813
5
    }
814
10
    if (i == 7)
815
6
      
for (j = 0; 1
j < 5;
j++5
)
816
5
        ccv_nnc_tensor_variable_free(graph, f[j]);
817
10
  }
818
1
  REQUIRE_EQ_WITH_TOLERANCE(ccv_nnc_tensor_from_variable(graph, f[9])->data.f32[0], 1.3 * 10, 1e-5, "should equal to 1.3 * 10");
819
1
  REQUIRE_EQ_WITH_TOLERANCE(ccv_nnc_tensor_from_variable(graph, dy[2])->data.f32[0], 8 + 9 + 10, 1e-5, "should equal to sum of the last 3");
820
11
  for (i = 0; i < 10; 
i++10
)
821
10
    ccv_nnc_tensor_variable_free(graph, x[i]);
822
1
  ccv_nnc_dynamic_graph_free(graph);
823
1
}
824
825
TEST_CASE("parallel exec with a stream, focus on uma tensors")
826
1
{
827
1
  ccv_nnc_dynamic_graph_t* const graph = ccv_nnc_dynamic_graph_new();
828
1
  ccv_nnc_stream_context_t* const stream = ccv_nnc_stream_context_new(CCV_STREAM_CONTEXT_CPU);
829
1
  ccv_nnc_tensor_variable_t x[2];
830
1
  x[0] = ccv_nnc_tensor_variable_new(graph, CPU_TENSOR_NHWC(32F, 1));
831
1
  x[1] = ccv_nnc_tensor_variable_new(graph, CPU_TENSOR_NHWC(32F, 1));
832
1
  ccv_nnc_tensor_from_variable(graph, x[0])->data.f32[0] = 10;
833
1
  ccv_nnc_tensor_from_variable(graph, x[1])->data.f32[0] = 9;
834
1
  ccv_nnc_tensor_variable_t y[2];
835
1
  y[0] = ccv_nnc_tensor_variable_new(graph);
836
1
  y[1] = ccv_nnc_tensor_variable_new(graph);
837
1
  ccv_nnc_dynamic_graph_exec(graph, CMD_SCALAR_MUL_FORWARD(0.4), ccv_nnc_no_hint, 0, TENSOR_VARIABLE_LIST(x[0], x[1]), TENSOR_VARIABLE_LIST(y[0], y[1]), 2, stream);
838
1
  ccv_nnc_stream_context_wait(stream);
839
1
  ccv_nnc_stream_context_free(stream);
840
1
  REQUIRE_EQ_WITH_TOLERANCE(ccv_nnc_tensor_from_variable(graph, y[0])->data.f32[0], 10 * 0.4, 1e-5, "should match the result");
841
1
  REQUIRE_EQ_WITH_TOLERANCE(ccv_nnc_tensor_from_variable(graph, y[1])->data.f32[0], 9 * 0.4, 1e-5, "should match the result");
842
1
  ccv_nnc_dynamic_graph_free(graph);
843
1
}
844
845
TEST_CASE("parallel exec with a stream, focus on potential memory issues")
846
1
{
847
1
  ccv_nnc_dynamic_graph_t* const graph = ccv_nnc_dynamic_graph_new();
848
1
  ccv_nnc_stream_context_t* const stream = ccv_nnc_stream_context_new(CCV_STREAM_CONTEXT_CPU);
849
1
  ccv_nnc_tensor_variable_t x[2];
850
1
  x[0] = ccv_nnc_tensor_variable_new(graph, CPU_NUMA_TENSOR_NHWC(000, 32F, 1));
851
1
  x[1] = ccv_nnc_tensor_variable_new(graph, CPU_NUMA_TENSOR_NHWC(000, 32F, 1));
852
1
  ccv_nnc_tensor_from_variable(graph, x[0])->data.f32[0] = 10;
853
1
  ccv_nnc_tensor_from_variable(graph, x[1])->data.f32[0] = 9;
854
1
  ccv_nnc_tensor_variable_t y[2];
855
1
  y[0] = ccv_nnc_tensor_variable_new(graph);
856
1
  y[1] = ccv_nnc_tensor_variable_new(graph);
857
1
  ccv_nnc_dynamic_graph_exec(graph, CMD_SCALAR_MUL_FORWARD(0.4), ccv_nnc_no_hint, 0, TENSOR_VARIABLE_LIST(x[0], x[1]), TENSOR_VARIABLE_LIST(y[0], y[1]), 2, stream);
858
1
  ccv_nnc_stream_context_wait(stream);
859
1
  ccv_nnc_stream_context_free(stream);
860
1
  REQUIRE_EQ_WITH_TOLERANCE(ccv_nnc_tensor_from_variable(graph, y[0])->data.f32[0], 10 * 0.4, 1e-5, "should match the result");
861
1
  REQUIRE_EQ_WITH_TOLERANCE(ccv_nnc_tensor_from_variable(graph, y[1])->data.f32[0], 9 * 0.4, 1e-5, "should match the result");
862
1
  ccv_nnc_dynamic_graph_free(graph);
863
1
}
864
865
TEST_CASE("query whether a tensor variable depends on another")
866
1
{
867
1
  ccv_nnc_dynamic_graph_t* const graph = ccv_nnc_dynamic_graph_new();
868
1
  ccv_nnc_tensor_variable_t a = ccv_nnc_tensor_variable_new(graph, CPU_TENSOR_NHWC(32F, 1));
869
1
  ccv_nnc_tensor_from_variable(graph, a)->data.f32[0] = 10;
870
1
  ccv_nnc_tensor_variable_t b = ccv_nnc_tensor_variable_new(graph);
871
1
  ccv_nnc_dynamic_graph_exec(graph, CMD_SCALAR_MUL_FORWARD(0.5), ccv_nnc_no_hint, 0, TENSOR_VARIABLE_LIST(a), TENSOR_VARIABLE_LIST(b), 0, 0);
872
1
  ccv_nnc_tensor_variable_t c = ccv_nnc_tensor_variable_new(graph, CPU_TENSOR_NHWC(32F, 1));
873
1
  ccv_nnc_tensor_from_variable(graph, c)->data.f32[0] = 9;
874
1
  ccv_nnc_tensor_variable_t d = ccv_nnc_tensor_variable_new(graph);
875
1
  ccv_nnc_dynamic_graph_exec(graph, CMD_SCALAR_MUL_FORWARD(0.4), ccv_nnc_no_hint, 0, TENSOR_VARIABLE_LIST(c), TENSOR_VARIABLE_LIST(d), 0, 0);
876
1
  ccv_nnc_tensor_variable_t e = ccv_nnc_tensor_variable_new(graph);
877
1
  ccv_nnc_dynamic_graph_exec(graph, CMD_SCALAR_MUL_FORWARD(2), ccv_nnc_no_hint, 0, TENSOR_VARIABLE_LIST(d), TENSOR_VARIABLE_LIST(e), 0, 0);
878
1
  ccv_nnc_tensor_variable_t f = ccv_nnc_tensor_variable_new(graph);
879
1
  ccv_nnc_dynamic_graph_exec(graph, CMD_EWSUM_FORWARD(), ccv_nnc_no_hint, 0, TENSOR_VARIABLE_LIST(a, c), TENSOR_VARIABLE_LIST(f), 0, 0);
880
1
  ccv_nnc_tensor_variable_t g = ccv_nnc_tensor_variable_new(graph);
881
1
  ccv_nnc_dynamic_graph_exec(graph, CMD_EWSUM_FORWARD(), ccv_nnc_no_hint, 0, TENSOR_VARIABLE_LIST(e, b), TENSOR_VARIABLE_LIST(g), 0, 0);
882
1
  REQUIRE_EQ_WITH_TOLERANCE(ccv_nnc_tensor_from_variable(graph, f)->data.f32[0], 10 + 9, 1e-5, "should match the result");
883
1
  REQUIRE_EQ_WITH_TOLERANCE(ccv_nnc_tensor_from_variable(graph, g)->data.f32[0], 10 * 0.5 + 9 * 0.4 * 2, 1e-5, "should match the result");
884
1
  uint64_t bitmask = 1;
885
1
  ccv_nnc_dynamic_graph_has_effect_to_tensor_variables(graph, TENSOR_VARIABLE_LIST(a, b, f), TENSOR_VARIABLE_LIST(g), &bitmask);
886
1
  REQUIRE(bitmask == 3, "a and b should be ancestors to g");
887
1
  bitmask = 8;
888
1
  ccv_nnc_dynamic_graph_has_effect_to_tensor_variables(graph, TENSOR_VARIABLE_LIST(a, e, c, f), TENSOR_VARIABLE_LIST(g), &bitmask);
889
1
  REQUIRE(bitmask == 7, "a, e and c should be ancestors to g");
890
1
  bitmask = 2;
891
1
  ccv_nnc_dynamic_graph_has_effect_to_tensor_variables(graph, TENSOR_VARIABLE_LIST(a, e, c), TENSOR_VARIABLE_LIST(f), &bitmask);
892
1
  REQUIRE(bitmask == 5, "a and c should be ancestors to f");
893
1
  ccv_nnc_dynamic_graph_free(graph);
894
1
}
895
896
TEST_CASE("compute f(x) = 0.5 * log(x), detach log(x) means 0.5 * y should be freed")
897
1
{
898
1
  ccv_nnc_dynamic_graph_t* const graph = ccv_nnc_dynamic_graph_new();
899
1
  ccv_nnc_tensor_variable_t x = ccv_nnc_tensor_variable_new(graph, CPU_TENSOR_NHWC(32F, 1));
900
1
  ccv_nnc_tensor_from_variable(graph, x)->data.f32[0] = 10;
901
1
  ccv_nnc_tensor_variable_t y = ccv_nnc_tensor_variable_new(graph);
902
1
  ccv_nnc_dynamic_graph_exec(graph, CMD_EWLOG_FORWARD(), ccv_nnc_no_hint, 0, TENSOR_VARIABLE_LIST(x), TENSOR_VARIABLE_LIST(y), 0, 0);
903
1
  REQUIRE_EQ_WITH_TOLERANCE(ccv_nnc_tensor_from_variable(graph, y)->data.f32[0], log(10), 1e-5, "should match the result");
904
1
  ccv_nnc_tensor_variable_t z = ccv_nnc_tensor_variable_new(graph);
905
1
  ccv_nnc_dynamic_graph_exec(graph, CMD_SCALAR_MUL_FORWARD(0.5), ccv_nnc_no_hint, 0, TENSOR_VARIABLE_LIST(y), TENSOR_VARIABLE_LIST(z), 0, 0);
906
1
  REQUIRE_EQ_WITH_TOLERANCE(ccv_nnc_tensor_from_variable(graph, z)->data.f32[0], 0.5 * log(10), 1e-5, "should match the result");
907
1
  REQUIRE_EQ(ccv_nnc_dynamic_graph_bookkeeping_count(graph, CCV_NNC_SYMBOL_GRAPH_EXEC), 2, "nothing should be freed");
908
1
  REQUIRE_EQ(ccv_nnc_dynamic_graph_bookkeeping_count(graph, CCV_NNC_SYMBOL_TENSOR), 3, "nothing should be freed");
909
1
  uint64_t bitmask = 0;
910
1
  ccv_nnc_dynamic_graph_has_effect_to_tensor_variables(graph, TENSOR_VARIABLE_LIST(x), TENSOR_VARIABLE_LIST(z), &bitmask);
911
1
  REQUIRE(bitmask == 1, "before detach, it should have effect");
912
1
  ccv_nnc_tensor_variable_detach(graph, y);
913
1
  REQUIRE_EQ(ccv_nnc_dynamic_graph_bookkeeping_count(graph, CCV_NNC_SYMBOL_GRAPH_EXEC), 1, "0.5 * y should be freed");
914
1
  REQUIRE_EQ(ccv_nnc_dynamic_graph_bookkeeping_count(graph, CCV_NNC_SYMBOL_TENSOR), 3, "nothing should be freed");
915
1
  bitmask = 0;
916
1
  ccv_nnc_dynamic_graph_has_effect_to_tensor_variables(graph, TENSOR_VARIABLE_LIST(x), TENSOR_VARIABLE_LIST(z), &bitmask);
917
1
  REQUIRE(bitmask == 0, "after detach, it should have no effect");
918
1
  ccv_nnc_tensor_variable_free(graph, x);
919
1
  REQUIRE_EQ(ccv_nnc_dynamic_graph_bookkeeping_count(graph, CCV_NNC_SYMBOL_GRAPH_EXEC), 0, "everything should be freed");
920
1
  REQUIRE_EQ(ccv_nnc_dynamic_graph_bookkeeping_count(graph, CCV_NNC_SYMBOL_TENSOR), 1, "x and y should be freed");
921
1
  ccv_nnc_tensor_variable_free(graph, y);
922
1
  ccv_nnc_tensor_variable_free(graph, z);
923
1
  REQUIRE_EQ(ccv_nnc_dynamic_graph_bookkeeping_count(graph, CCV_NNC_SYMBOL_TENSOR), 0, "everything should be freed");
924
1
  ccv_nnc_dynamic_graph_free(graph);
925
1
}
926
927
TEST_CASE("compute f(x) = 0.5 * log(x), detach log(x) after freeing x means all computations on the graph should be freed")
928
1
{
929
1
  ccv_nnc_dynamic_graph_t* const graph = ccv_nnc_dynamic_graph_new();
930
1
  ccv_nnc_tensor_variable_t x = ccv_nnc_tensor_variable_new(graph, CPU_TENSOR_NHWC(32F, 1));
931
1
  ccv_nnc_tensor_from_variable(graph, x)->data.f32[0] = 10;
932
1
  ccv_nnc_tensor_variable_t y = ccv_nnc_tensor_variable_new(graph);
933
1
  ccv_nnc_dynamic_graph_exec(graph, CMD_EWLOG_FORWARD(), ccv_nnc_no_hint, 0, TENSOR_VARIABLE_LIST(x), TENSOR_VARIABLE_LIST(y), 0, 0);
934
1
  REQUIRE_EQ_WITH_TOLERANCE(ccv_nnc_tensor_from_variable(graph, y)->data.f32[0], log(10), 1e-5, "should match the result");
935
1
  ccv_nnc_tensor_variable_t z = ccv_nnc_tensor_variable_new(graph);
936
1
  ccv_nnc_dynamic_graph_exec(graph, CMD_SCALAR_MUL_FORWARD(0.5), ccv_nnc_no_hint, 0, TENSOR_VARIABLE_LIST(y), TENSOR_VARIABLE_LIST(z), 0, 0);
937
1
  REQUIRE_EQ_WITH_TOLERANCE(ccv_nnc_tensor_from_variable(graph, z)->data.f32[0], 0.5 * log(10), 1e-5, "should match the result");
938
1
  REQUIRE_EQ(ccv_nnc_dynamic_graph_bookkeeping_count(graph, CCV_NNC_SYMBOL_GRAPH_EXEC), 2, "nothing should be freed");
939
1
  REQUIRE_EQ(ccv_nnc_dynamic_graph_bookkeeping_count(graph, CCV_NNC_SYMBOL_TENSOR), 3, "nothing should be freed");
940
  // free x first before detaching y.
941
1
  ccv_nnc_tensor_variable_free(graph, x);
942
1
  REQUIRE_EQ(ccv_nnc_dynamic_graph_bookkeeping_count(graph, CCV_NNC_SYMBOL_GRAPH_EXEC), 1, "log(x) should be freed");
943
1
  REQUIRE_EQ(ccv_nnc_dynamic_graph_bookkeeping_count(graph, CCV_NNC_SYMBOL_TENSOR), 2, "only x should be freed");
944
1
  ccv_nnc_tensor_variable_detach(graph, y);
945
1
  REQUIRE_EQ(ccv_nnc_dynamic_graph_bookkeeping_count(graph, CCV_NNC_SYMBOL_GRAPH_EXEC), 0, "everything should be freed");
946
1
  REQUIRE_EQ(ccv_nnc_dynamic_graph_bookkeeping_count(graph, CCV_NNC_SYMBOL_TENSOR), 1, "x and y should be freed");
947
1
  ccv_nnc_tensor_variable_free(graph, y);
948
1
  ccv_nnc_tensor_variable_free(graph, z);
949
1
  REQUIRE_EQ(ccv_nnc_dynamic_graph_bookkeeping_count(graph, CCV_NNC_SYMBOL_TENSOR), 0, "everything should be freed");
950
1
  ccv_nnc_dynamic_graph_free(graph);
951
1
}
952
953
TEST_CASE("compute f(x) = a * log(x), detach log(x) means nothing changed")
954
1
{
955
1
  ccv_nnc_dynamic_graph_t* const graph = ccv_nnc_dynamic_graph_new();
956
1
  ccv_nnc_tensor_variable_t x = ccv_nnc_tensor_variable_new(graph, CPU_TENSOR_NHWC(32F, 1));
957
1
  ccv_nnc_tensor_from_variable(graph, x)->data.f32[0] = 10;
958
1
  ccv_nnc_tensor_variable_t a = ccv_nnc_tensor_variable_new(graph, CPU_TENSOR_NHWC(32F, 1));
959
1
  ccv_nnc_tensor_from_variable(graph, a)->data.f32[0] = 5;
960
1
  ccv_nnc_tensor_variable_t y = ccv_nnc_tensor_variable_new(graph);
961
1
  ccv_nnc_dynamic_graph_exec(graph, CMD_EWLOG_FORWARD(), ccv_nnc_no_hint, 0, TENSOR_VARIABLE_LIST(x), TENSOR_VARIABLE_LIST(y), 0, 0);
962
1
  REQUIRE_EQ_WITH_TOLERANCE(ccv_nnc_tensor_from_variable(graph, y)->data.f32[0], log(10), 1e-5, "should match the result");
963
1
  ccv_nnc_tensor_variable_t z = ccv_nnc_tensor_variable_new(graph);
964
1
  ccv_nnc_dynamic_graph_exec(graph, CMD_MUL_FORWARD(1), ccv_nnc_no_hint, 0, TENSOR_VARIABLE_LIST(a, y), TENSOR_VARIABLE_LIST(z), 0, 0);
965
1
  REQUIRE_EQ_WITH_TOLERANCE(ccv_nnc_tensor_from_variable(graph, z)->data.f32[0], 5 * log(10), 1e-5, "should match the result");
966
1
  REQUIRE_EQ(ccv_nnc_dynamic_graph_bookkeeping_count(graph, CCV_NNC_SYMBOL_GRAPH_EXEC), 2, "nothing should be freed");
967
1
  REQUIRE_EQ(ccv_nnc_dynamic_graph_bookkeeping_count(graph, CCV_NNC_SYMBOL_TENSOR), 4, "nothing should be freed");
968
1
  uint64_t bitmask = 0;
969
1
  ccv_nnc_dynamic_graph_has_effect_to_tensor_variables(graph, TENSOR_VARIABLE_LIST(x), TENSOR_VARIABLE_LIST(z), &bitmask);
970
1
  REQUIRE(bitmask == 1, "before detach, it should have effect");
971
1
  ccv_nnc_tensor_variable_detach(graph, y);
972
1
  REQUIRE_EQ(ccv_nnc_dynamic_graph_bookkeeping_count(graph, CCV_NNC_SYMBOL_GRAPH_EXEC), 2, "nothing should be freed");
973
1
  REQUIRE_EQ(ccv_nnc_dynamic_graph_bookkeeping_count(graph, CCV_NNC_SYMBOL_TENSOR), 4, "nothing should be freed");
974
1
  bitmask = 0;
975
1
  ccv_nnc_dynamic_graph_has_effect_to_tensor_variables(graph, TENSOR_VARIABLE_LIST(x), TENSOR_VARIABLE_LIST(z), &bitmask);
976
1
  REQUIRE(bitmask == 0, "after detach, it should have no effect");
977
1
  ccv_nnc_tensor_variable_free(graph, a);
978
1
  REQUIRE_EQ(ccv_nnc_dynamic_graph_bookkeeping_count(graph, CCV_NNC_SYMBOL_GRAPH_EXEC), 1, "a * y should be freed");
979
1
  REQUIRE_EQ(ccv_nnc_dynamic_graph_bookkeeping_count(graph, CCV_NNC_SYMBOL_TENSOR), 3, "a should be freed");
980
1
  ccv_nnc_tensor_variable_free(graph, x);
981
1
  REQUIRE_EQ(ccv_nnc_dynamic_graph_bookkeeping_count(graph, CCV_NNC_SYMBOL_GRAPH_EXEC), 0, "everything should be freed");
982
1
  REQUIRE_EQ(ccv_nnc_dynamic_graph_bookkeeping_count(graph, CCV_NNC_SYMBOL_TENSOR), 1, "x and y should be freed");
983
1
  ccv_nnc_tensor_variable_free(graph, y);
984
1
  ccv_nnc_tensor_variable_free(graph, z);
985
1
  REQUIRE_EQ(ccv_nnc_dynamic_graph_bookkeeping_count(graph, CCV_NNC_SYMBOL_TENSOR), 0, "everything should be freed");
986
1
  ccv_nnc_dynamic_graph_free(graph);
987
1
}
988
989
TEST_CASE("compute f(x) = exp(x) * log(x), detach log(x) means nothing changed")
990
1
{
991
1
  ccv_nnc_dynamic_graph_t* const graph = ccv_nnc_dynamic_graph_new();
992
1
  ccv_nnc_tensor_variable_t x = ccv_nnc_tensor_variable_new(graph, CPU_TENSOR_NHWC(32F, 1));
993
1
  ccv_nnc_tensor_from_variable(graph, x)->data.f32[0] = 10;
994
1
  ccv_nnc_tensor_variable_t a = ccv_nnc_tensor_variable_new(graph);
995
1
  ccv_nnc_dynamic_graph_exec(graph, CMD_EWEXP_FORWARD(), ccv_nnc_no_hint, 0, TENSOR_VARIABLE_LIST(x), TENSOR_VARIABLE_LIST(a), 0, 0);
996
1
  REQUIRE_EQ_WITH_TOLERANCE(ccv_nnc_tensor_from_variable(graph, a)->data.f32[0], exp(10), 1e-2, "should match the result");
997
1
  ccv_nnc_tensor_variable_t y = ccv_nnc_tensor_variable_new(graph);
998
1
  ccv_nnc_dynamic_graph_exec(graph, CMD_EWLOG_FORWARD(), ccv_nnc_no_hint, 0, TENSOR_VARIABLE_LIST(x), TENSOR_VARIABLE_LIST(y), 0, 0);
999
1
  REQUIRE_EQ_WITH_TOLERANCE(ccv_nnc_tensor_from_variable(graph, y)->data.f32[0], log(10), 1e-5, "should match the result");
1000
1
  ccv_nnc_tensor_variable_t z = ccv_nnc_tensor_variable_new(graph);
1001
1
  ccv_nnc_dynamic_graph_exec(graph, CMD_MUL_FORWARD(1), ccv_nnc_no_hint, 0, TENSOR_VARIABLE_LIST(a, y), TENSOR_VARIABLE_LIST(z), 0, 0);
1002
1
  REQUIRE_EQ_WITH_TOLERANCE(ccv_nnc_tensor_from_variable(graph, z)->data.f32[0], exp(10) * log(10), 1e-2, "should match the result");
1003
1
  REQUIRE_EQ(ccv_nnc_dynamic_graph_bookkeeping_count(graph, CCV_NNC_SYMBOL_GRAPH_EXEC), 3, "nothing should be freed");
1004
1
  REQUIRE_EQ(ccv_nnc_dynamic_graph_bookkeeping_count(graph, CCV_NNC_SYMBOL_TENSOR), 4, "nothing should be freed");
1005
1
  uint64_t bitmask = 0;
1006
1
  ccv_nnc_dynamic_graph_has_effect_to_tensor_variables(graph, TENSOR_VARIABLE_LIST(x), TENSOR_VARIABLE_LIST(z), &bitmask);
1007
1
  REQUIRE(bitmask == 1, "before detach, it should have effect");
1008
1
  ccv_nnc_tensor_variable_detach(graph, y);
1009
1
  REQUIRE_EQ(ccv_nnc_dynamic_graph_bookkeeping_count(graph, CCV_NNC_SYMBOL_GRAPH_EXEC), 3, "nothing should be freed");
1010
1
  REQUIRE_EQ(ccv_nnc_dynamic_graph_bookkeeping_count(graph, CCV_NNC_SYMBOL_TENSOR), 4, "nothing should be freed");
1011
1
  bitmask = 0;
1012
1
  ccv_nnc_dynamic_graph_has_effect_to_tensor_variables(graph, TENSOR_VARIABLE_LIST(x), TENSOR_VARIABLE_LIST(z), &bitmask);
1013
1
  REQUIRE(bitmask == 1, "after detach, it should have still effect (through exp(x))");
1014
1
  ccv_nnc_tensor_variable_detach(graph, a);
1015
1
  REQUIRE_EQ(ccv_nnc_dynamic_graph_bookkeeping_count(graph, CCV_NNC_SYMBOL_GRAPH_EXEC), 2, "a * y should be freed");
1016
1
  REQUIRE_EQ(ccv_nnc_dynamic_graph_bookkeeping_count(graph, CCV_NNC_SYMBOL_TENSOR), 4, "nothing should be freed");
1017
1
  bitmask = 0;
1018
1
  ccv_nnc_dynamic_graph_has_effect_to_tensor_variables(graph, TENSOR_VARIABLE_LIST(x), TENSOR_VARIABLE_LIST(z), &bitmask);
1019
1
  REQUIRE(bitmask == 0, "after a freed, it should have no effect");
1020
1
  ccv_nnc_tensor_variable_free(graph, a);
1021
1
  REQUIRE_EQ(ccv_nnc_dynamic_graph_bookkeeping_count(graph, CCV_NNC_SYMBOL_TENSOR), 3, "a should be freed");
1022
1
  ccv_nnc_tensor_variable_free(graph, x);
1023
1
  REQUIRE_EQ(ccv_nnc_dynamic_graph_bookkeeping_count(graph, CCV_NNC_SYMBOL_GRAPH_EXEC), 0, "everything should be freed");
1024
1
  REQUIRE_EQ(ccv_nnc_dynamic_graph_bookkeeping_count(graph, CCV_NNC_SYMBOL_TENSOR), 1, "x and y should be freed");
1025
1
  ccv_nnc_tensor_variable_free(graph, y);
1026
1
  ccv_nnc_tensor_variable_free(graph, z);
1027
1
  REQUIRE_EQ(ccv_nnc_dynamic_graph_bookkeeping_count(graph, CCV_NNC_SYMBOL_TENSOR), 0, "everything should be freed");
1028
1
  ccv_nnc_dynamic_graph_free(graph);
1029
1
}
1030
1031
TEST_CASE("compute f(x) = exp(0.5 * log(x)), detach log(x) won't free exp(_)")
1032
1
{
1033
1
  ccv_nnc_dynamic_graph_t* const graph = ccv_nnc_dynamic_graph_new();
1034
1
  ccv_nnc_tensor_variable_t x = ccv_nnc_tensor_variable_new(graph, CPU_TENSOR_NHWC(32F, 1));
1035
1
  ccv_nnc_tensor_from_variable(graph, x)->data.f32[0] = 10;
1036
1
  ccv_nnc_tensor_variable_t y = ccv_nnc_tensor_variable_new(graph);
1037
1
  ccv_nnc_dynamic_graph_exec(graph, CMD_EWLOG_FORWARD(), ccv_nnc_no_hint, 0, TENSOR_VARIABLE_LIST(x), TENSOR_VARIABLE_LIST(y), 0, 0);
1038
1
  REQUIRE_EQ_WITH_TOLERANCE(ccv_nnc_tensor_from_variable(graph, y)->data.f32[0], log(10), 1e-5, "should match the result");
1039
1
  ccv_nnc_tensor_variable_t z = ccv_nnc_tensor_variable_new(graph);
1040
1
  ccv_nnc_dynamic_graph_exec(graph, CMD_SCALAR_MUL_FORWARD(0.5), ccv_nnc_no_hint, 0, TENSOR_VARIABLE_LIST(y), TENSOR_VARIABLE_LIST(z), 0, 0);
1041
1
  REQUIRE_EQ_WITH_TOLERANCE(ccv_nnc_tensor_from_variable(graph, z)->data.f32[0], 0.5 * log(10), 1e-5, "should match the result");
1042
1
  ccv_nnc_tensor_variable_t f = ccv_nnc_tensor_variable_new(graph);
1043
1
  ccv_nnc_dynamic_graph_exec(graph, CMD_EWEXP_FORWARD(), ccv_nnc_no_hint, 0, TENSOR_VARIABLE_LIST(z), TENSOR_VARIABLE_LIST(f), 0, 0);
1044
1
  REQUIRE_EQ_WITH_TOLERANCE(ccv_nnc_tensor_from_variable(graph, f)->data.f32[0], exp(0.5 * log(10)), 1e-5, "should match the result");
1045
1
  uint64_t bitmask = 0;
1046
1
  ccv_nnc_dynamic_graph_has_effect_to_tensor_variables(graph, TENSOR_VARIABLE_LIST(x), TENSOR_VARIABLE_LIST(f), &bitmask);
1047
1
  REQUIRE(bitmask == 1, "before detach, it should have effect");
1048
1
  ccv_nnc_tensor_variable_detach(graph, y);
1049
1
  REQUIRE_EQ(ccv_nnc_dynamic_graph_bookkeeping_count(graph, CCV_NNC_SYMBOL_GRAPH_EXEC), 2, "0.5 * y should be freed");
1050
1
  REQUIRE_EQ(ccv_nnc_dynamic_graph_bookkeeping_count(graph, CCV_NNC_SYMBOL_TENSOR), 4, "nothing should be freed");
1051
1
  bitmask = 0;
1052
1
  ccv_nnc_dynamic_graph_has_effect_to_tensor_variables(graph, TENSOR_VARIABLE_LIST(x), TENSOR_VARIABLE_LIST(f), &bitmask);
1053
1
  REQUIRE(bitmask == 0, "after detach, it should have no effect");
1054
1
  ccv_nnc_tensor_variable_free(graph, f);
1055
1
  REQUIRE_EQ(ccv_nnc_dynamic_graph_bookkeeping_count(graph, CCV_NNC_SYMBOL_GRAPH_EXEC), 1, "exp(_) should be freed");
1056
1
  REQUIRE_EQ(ccv_nnc_dynamic_graph_bookkeeping_count(graph, CCV_NNC_SYMBOL_TENSOR), 3, "f should be freed");
1057
1
  ccv_nnc_tensor_variable_free(graph, z);
1058
1
  REQUIRE_EQ(ccv_nnc_dynamic_graph_bookkeeping_count(graph, CCV_NNC_SYMBOL_GRAPH_EXEC), 1, "nothing should change");
1059
1
  REQUIRE_EQ(ccv_nnc_dynamic_graph_bookkeeping_count(graph, CCV_NNC_SYMBOL_TENSOR), 2, "z should be freed");
1060
1
  ccv_nnc_tensor_variable_free(graph, y);
1061
1
  REQUIRE_EQ(ccv_nnc_dynamic_graph_bookkeeping_count(graph, CCV_NNC_SYMBOL_GRAPH_EXEC), 0, "log(x) should be freed");
1062
1
  REQUIRE_EQ(ccv_nnc_dynamic_graph_bookkeeping_count(graph, CCV_NNC_SYMBOL_TENSOR), 1, "y should be freed");
1063
1
  ccv_nnc_tensor_variable_free(graph, x);
1064
1
  REQUIRE_EQ(ccv_nnc_dynamic_graph_bookkeeping_count(graph, CCV_NNC_SYMBOL_TENSOR), 0, "x should be freed");
1065
1
  ccv_nnc_dynamic_graph_free(graph);
1066
1
}
1067
1068
#include "case_main.h"