Coverage Report

Created: 2025-02-24 17:43

next uncovered line (L), next uncovered region (R), next uncovered branch (B)
/home/liu/actions-runner/_work/ccv/ccv/test/unit/nnc/dynamic.graph.tests.c
Line
Count
Source
1
#include "case.h"
2
#include "ccv_case.h"
3
#include "ccv_nnc_case.h"
4
#include <ccv.h>
5
#include <nnc/ccv_nnc.h>
6
#include <nnc/ccv_nnc_easy.h>
7
#include "3rdparty/dsfmt/dSFMT.h"
8
9
TEST_SETUP()
10
{
11
  ccv_nnc_init();
12
}
13
14
TEST_CASE("dynamic graph to compute log(19)")
15
1
{
16
1
  ccv_nnc_dynamic_graph_t* const graph = ccv_nnc_dynamic_graph_new();
17
1
  ccv_nnc_tensor_variable_t a = ccv_nnc_tensor_variable_new(graph, CPU_TENSOR_NHWC(32F, 1));
18
1
  ccv_nnc_tensor_from_variable(graph, a)->data.f32[0] = 19;
19
1
  ccv_nnc_tensor_variable_t b = ccv_nnc_tensor_variable_new(graph);
20
1
  ccv_nnc_dynamic_graph_exec(graph, CMD_EWLOG_FORWARD(), ccv_nnc_no_hint, 0, TENSOR_VARIABLE_LIST(a), TENSOR_VARIABLE_LIST(b), 0, 0);
21
1
  REQUIRE_EQ_WITH_TOLERANCE(ccv_nnc_tensor_from_variable(graph, b)->data.f32[0], logf(19), 1e-5, "log(19) result should be equal.");
22
1
  DYNAMIC_GRAPH_GEN(graph, CCV_NNC_LONG_DOT_GRAPH);
23
1
  ccv_nnc_dynamic_graph_free(graph);
24
1
}
25
26
TEST_CASE("dynamic graph to compute reciprocal")
27
1
{
28
1
  ccv_nnc_dynamic_graph_t* const graph = ccv_nnc_dynamic_graph_new();
29
1
  ccv_nnc_tensor_variable_t a = ccv_nnc_tensor_variable_new(graph, CPU_TENSOR_NHWC(32F, 1));
30
1
  ccv_nnc_tensor_from_variable(graph, a)->data.f32[0] = 19;
31
1
  ccv_nnc_tensor_variable_t b = ccv_nnc_tensor_variable_new(graph);
32
1
  ccv_nnc_dynamic_graph_exec(graph, CMD_EWDIV_FORWARD(), ccv_nnc_no_hint, 0, TENSOR_VARIABLE_LIST(0, a), TENSOR_VARIABLE_LIST(b), 0, 0);
33
1
  REQUIRE_EQ_WITH_TOLERANCE(ccv_nnc_tensor_from_variable(graph, b)->data.f32[0], 1.0 / 19, 1e-5, "1.0 / 19 result should be equal.");
34
1
  DYNAMIC_GRAPH_GEN(graph, CCV_NNC_LONG_DOT_GRAPH);
35
1
  ccv_nnc_dynamic_graph_free(graph);
36
1
}
37
38
TEST_CASE("dynamic graph with alias")
39
1
{
40
1
  ccv_nnc_dynamic_graph_t* const graph = ccv_nnc_dynamic_graph_new();
41
1
  ccv_nnc_tensor_variable_t const a = ccv_nnc_tensor_variable_new(graph, CPU_TENSOR_NHWC(32F, 2));
42
1
  ccv_nnc_tensor_variable_t const a0 = ccv_nnc_tensor_variable_alias_new(graph, a, DIM_ALLOC(1), DIM_ALLOC(), CPU_TENSOR_NHWC(32F, 1));
43
1
  ccv_nnc_tensor_variable_t const a1 = ccv_nnc_tensor_variable_alias_new(graph, a, ccv_nnc_no_ofs, DIM_ALLOC(), CPU_TENSOR_NHWC(32F, 1));
44
1
  ccv_nnc_tensor_t* const b0 = ccv_nnc_tensor_new(0, CPU_TENSOR_NHWC(32F, 2), 0);
45
1
  ccv_nnc_tensor_variable_set(graph, a, b0);
46
1
  b0->data.f32[0] = 10;
47
1
  b0->data.f32[1] = 11;
48
1
  ccv_nnc_tensor_t* const b1 = ccv_nnc_tensor_new(0, CPU_TENSOR_NHWC(32F, 2), 0);
49
1
  b1->data.f32[0] = 20;
50
1
  b1->data.f32[1] = 21;
51
1
  REQUIRE(CCV_IS_TENSOR_VIEW(ccv_nnc_tensor_from_variable(graph, a0)), "Complex vector is a tensor view");
52
1
  REQUIRE(!CCV_IS_TENSOR_VIEW(ccv_nnc_tensor_from_variable(graph, a1)), "Simple vector is not a tensor view");
53
1
  REQUIRE_EQ(ccv_nnc_tensor_from_variable(graph, a0)->data.f32[0], 11, "should be b0[1]");
54
1
  REQUIRE_EQ(ccv_nnc_tensor_from_variable(graph, a1)->data.f32[0], 10, "should be b0[0]");
55
1
  REQUIRE(!CCV_IS_TENSOR_VIEW(ccv_nnc_tensor_from_variable(graph, a1)), "no need to be a tensor view");
56
1
  ccv_nnc_tensor_variable_set(graph, a, b1);
57
1
  REQUIRE_EQ(ccv_nnc_tensor_from_variable(graph, a0)->data.f32[0], 21, "should be b1[1]");
58
1
  REQUIRE_EQ(ccv_nnc_tensor_from_variable(graph, a1)->data.f32[0], 20, "should be b1[0]");
59
1
  REQUIRE(!CCV_IS_TENSOR_VIEW(ccv_nnc_tensor_from_variable(graph, a1)), "no need to be a tensor view");
60
1
  ccv_nnc_dynamic_graph_free(graph);
61
1
  ccv_nnc_tensor_free(b0);
62
1
  ccv_nnc_tensor_free(b1);
63
1
}
64
65
TEST_CASE("dynamic graph alias an alias")
66
1
{
67
1
  ccv_nnc_dynamic_graph_t* const graph = ccv_nnc_dynamic_graph_new();
68
1
  ccv_nnc_tensor_variable_t const a = ccv_nnc_tensor_variable_new(graph, CPU_TENSOR_NHWC(32F, 2, 3));
69
1
  ccv_nnc_tensor_variable_t const a0 = ccv_nnc_tensor_variable_alias_new(graph, a, DIM_ALLOC(1, 0), DIM_ALLOC(3, 1), CPU_TENSOR_NHWC(32F, 1, 3));
70
1
  ccv_nnc_tensor_variable_t const a1 = ccv_nnc_tensor_variable_alias_new(graph, a0, DIM_ALLOC(1), DIM_ALLOC(1), CPU_TENSOR_NHWC(32F, 2));
71
1
  ccv_nnc_tensor_t* const b0 = ccv_nnc_tensor_new(0, CPU_TENSOR_NHWC(32F, 2, 3), 0);
72
1
  ccv_nnc_tensor_variable_set(graph, a, b0);
73
1
  b0->data.f32[0] = 10;
74
1
  b0->data.f32[1] = 11;
75
1
  b0->data.f32[2] = 12;
76
1
  b0->data.f32[3] = 13;
77
1
  b0->data.f32[4] = 14;
78
1
  b0->data.f32[5] = 15;
79
1
  ccv_nnc_tensor_t* const b1 = ccv_nnc_tensor_new(0, CPU_TENSOR_NHWC(32F, 2, 3), 0);
80
1
  b1->data.f32[0] = 20;
81
1
  b1->data.f32[1] = 21;
82
1
  b1->data.f32[2] = 22;
83
1
  b1->data.f32[3] = 23;
84
1
  b1->data.f32[4] = 24;
85
1
  b1->data.f32[5] = 25;
86
1
  REQUIRE(CCV_IS_TENSOR_VIEW(ccv_nnc_tensor_from_variable(graph, a0)), "Complex vector is a tensor view");
87
1
  REQUIRE(CCV_IS_TENSOR_VIEW(ccv_nnc_tensor_from_variable(graph, a1)), "Complex vector is a tensor view");
88
1
  REQUIRE_EQ(ccv_nnc_tensor_from_variable(graph, a0)->data.f32[0], 13, "should be b0[1, 0]");
89
1
  REQUIRE_EQ(ccv_nnc_tensor_from_variable(graph, a0)->data.f32[1], 14, "should be b0[1, 1]");
90
1
  REQUIRE_EQ(ccv_nnc_tensor_from_variable(graph, a0)->data.f32[2], 15, "should be b0[1, 2]");
91
1
  REQUIRE_EQ(ccv_nnc_tensor_from_variable(graph, a1)->data.f32[0], 14, "should be b0[1, 1]");
92
1
  REQUIRE_EQ(ccv_nnc_tensor_from_variable(graph, a1)->data.f32[1], 15, "should be b0[1, 2]");
93
1
  ccv_nnc_tensor_variable_set(graph, a, b1);
94
1
  REQUIRE_EQ(ccv_nnc_tensor_from_variable(graph, a0)->data.f32[0], 23, "should be b1[1, 0]");
95
1
  REQUIRE_EQ(ccv_nnc_tensor_from_variable(graph, a0)->data.f32[1], 24, "should be b1[1, 1]");
96
1
  REQUIRE_EQ(ccv_nnc_tensor_from_variable(graph, a0)->data.f32[2], 25, "should be b1[1, 2]");
97
1
  REQUIRE_EQ(ccv_nnc_tensor_from_variable(graph, a1)->data.f32[0], 24, "should be b1[1, 1]");
98
1
  REQUIRE_EQ(ccv_nnc_tensor_from_variable(graph, a1)->data.f32[1], 25, "should be b1[1, 2]");
99
1
  ccv_nnc_dynamic_graph_free(graph);
100
1
  ccv_nnc_tensor_free(b0);
101
1
  ccv_nnc_tensor_free(b1);
102
1
}
103
104
TEST_CASE("dynamic graph to compute f(x) = x * log(x) + 1.2 * x, f'(x) where x = 19")
105
1
{
106
1
  ccv_nnc_dynamic_graph_t* const graph = ccv_nnc_dynamic_graph_new();
107
1
  ccv_nnc_tensor_variable_t x = ccv_nnc_tensor_variable_new(graph, CPU_TENSOR_NHWC(32F, 1));
108
1
  ccv_nnc_tensor_from_variable(graph, x)->data.f32[0] = 19;
109
1
  ccv_nnc_tensor_variable_t f = ccv_nnc_tensor_variable_new(graph);
110
1
  ccv_nnc_dynamic_graph_exec(graph, CMD_EWLOG_FORWARD(), ccv_nnc_no_hint, 0, TENSOR_VARIABLE_LIST(x), TENSOR_VARIABLE_LIST(f), 0, 0);
111
1
  ccv_nnc_dynamic_graph_exec(graph, CMD_EWPROD_FORWARD(), ccv_nnc_no_hint, 0, TENSOR_VARIABLE_LIST(x, f), TENSOR_VARIABLE_LIST(f), 0, 0);
112
1
  ccv_nnc_tensor_variable_t y = ccv_nnc_tensor_variable_new(graph, CPU_TENSOR_NHWC(32F, 1));
113
1
  ccv_nnc_tensor_from_variable(graph, y)->data.f32[0] = 1.2;
114
1
  ccv_nnc_tensor_variable_t z = ccv_nnc_tensor_variable_new(graph);
115
1
  ccv_nnc_dynamic_graph_exec(graph, CMD_EWPROD_FORWARD(), ccv_nnc_no_hint, 0, TENSOR_VARIABLE_LIST(x, y), TENSOR_VARIABLE_LIST(z), 0, 0);
116
1
  ccv_nnc_dynamic_graph_exec(graph, CMD_EWSUM_FORWARD(), ccv_nnc_no_hint, 0, TENSOR_VARIABLE_LIST(f, z), TENSOR_VARIABLE_LIST(f), 0, 0);
117
1
  REQUIRE_EQ_WITH_TOLERANCE(ccv_nnc_tensor_from_variable(graph, f)->data.f32[0], 19 * logf(19) + 1.2 * 19, 1e-5, "f(x) = 1.2 * 19 + 19 * log(19)");
118
  // Do gradient computation multiple times.
119
1
  ccv_nnc_tensor_variable_t dx = ccv_nnc_tensor_variable_new(graph);
120
1
  ccv_nnc_dynamic_graph_backward(graph, TENSOR_VARIABLE_LIST(f), 0, TENSOR_VARIABLE_LIST(x), TENSOR_VARIABLE_LIST(dx), 0);
121
1
  REQUIRE_EQ_WITH_TOLERANCE(ccv_nnc_tensor_from_variable(graph, dx)->data.f32[0], logf(19) + 1 + 1.2, 1e-5, "f'(x) = 1.2 + log(19) + 19 * 1 / 19");
122
1
  ccv_nnc_tensor_variable_t dy = ccv_nnc_tensor_variable_new(graph);
123
1
  ccv_nnc_dynamic_graph_backward(graph, TENSOR_VARIABLE_LIST(f), 0, TENSOR_VARIABLE_LIST(y), TENSOR_VARIABLE_LIST(dy), 0);
124
1
  REQUIRE_EQ_WITH_TOLERANCE(ccv_nnc_tensor_from_variable(graph, dy)->data.f32[0], 19, 1e-5, "f'(y) = 19");
125
1
  ccv_nnc_tensor_variable_t dz = ccv_nnc_tensor_variable_new(graph);
126
1
  ccv_nnc_dynamic_graph_backward(graph, TENSOR_VARIABLE_LIST(f), 0, TENSOR_VARIABLE_LIST(z), TENSOR_VARIABLE_LIST(dz), 0);
127
1
  REQUIRE_EQ_WITH_TOLERANCE(ccv_nnc_tensor_from_variable(graph, dz)->data.f32[0], 1, 1e-5, "f'(z) = 1");
128
1
  ccv_nnc_tensor_variable_free(graph, dy);
129
1
  dy = ccv_nnc_tensor_variable_new(graph);
130
1
  ccv_nnc_dynamic_graph_exec(graph, CMD_SET_FORWARD(0), ccv_nnc_no_hint, 0, TENSOR_VARIABLE_LIST(), TENSOR_VARIABLE_LIST(dx), 0, 0);
131
1
  ccv_nnc_dynamic_graph_backward(graph, TENSOR_VARIABLE_LIST(f), 0, TENSOR_VARIABLE_LIST(y, x), TENSOR_VARIABLE_LIST(dy, dx), 0);
132
1
  REQUIRE_EQ_WITH_TOLERANCE(ccv_nnc_tensor_from_variable(graph, dx)->data.f32[0], logf(19) + 1 + 1.2, 1e-5, "f'(x) = 1.2 + log(19) + 19 * 1 / 19");
133
1
  REQUIRE_EQ_WITH_TOLERANCE(ccv_nnc_tensor_from_variable(graph, dy)->data.f32[0], 19, 1e-5, "f'(y) = 19");
134
1
  DYNAMIC_GRAPH_GEN(graph, CCV_NNC_LONG_DOT_GRAPH);
135
1
  ccv_nnc_dynamic_graph_free(graph);
136
1
}
137
138
TEST_CASE("dynamic graph with dense net (extensive use of alias)")
139
1
{
140
1
  ccv_nnc_dynamic_graph_t* const graph = ccv_nnc_dynamic_graph_new();
141
1
  ccv_nnc_tensor_variable_t x = ccv_nnc_tensor_variable_new(graph, CPU_TENSOR_NHWC(32F, 1, 4));
142
1
  ccv_nnc_tensor_from_variable(graph, x)->data.f32[0] = 0.472;
143
1
  ccv_nnc_tensor_variable_t x1 = ccv_nnc_tensor_variable_alias_new(graph, x, ccv_nnc_no_ofs, DIM_ALLOC(4, 1), CPU_TENSOR_NHWC(32F, 1, 1));
144
1
  ccv_nnc_tensor_variable_t w1 = ccv_nnc_tensor_variable_new(graph, CPU_TENSOR_NHWC(32F, 1, 1));
145
1
  ccv_nnc_tensor_from_variable(graph, w1)->data.f32[0] = 0.234;
146
1
  ccv_nnc_tensor_variable_t b1 = ccv_nnc_tensor_variable_new(graph, CPU_TENSOR_NHWC(32F, 1));
147
1
  ccv_nnc_tensor_from_variable(graph, b1)->data.f32[0] = 0.1;
148
1
  ccv_nnc_tensor_variable_t x11 = ccv_nnc_tensor_variable_alias_new(graph, x, DIM_ALLOC(0, 1), DIM_ALLOC(4, 1), CPU_TENSOR_NHWC(32F, 1, 1));
149
1
  ccv_nnc_dynamic_graph_exec(graph, CMD_GEMM_FORWARD(NO_TRANSPOSE, TRANSPOSE(0, 1)), ccv_nnc_no_hint, 0, TENSOR_VARIABLE_LIST(x1, w1, b1), TENSOR_VARIABLE_LIST(x11), 0, 0);
150
1
  ccv_nnc_tensor_variable_t x2 = ccv_nnc_tensor_variable_alias_new(graph, x, ccv_nnc_no_ofs, DIM_ALLOC(4, 1), CPU_TENSOR_NHWC(32F, 1, 2));
151
1
  ccv_nnc_tensor_variable_t w2 = ccv_nnc_tensor_variable_new(graph, CPU_TENSOR_NHWC(32F, 1, 2));
152
1
  ccv_nnc_tensor_from_variable(graph, w2)->data.f32[0] = 0.374;
153
1
  ccv_nnc_tensor_from_variable(graph, w2)->data.f32[1] = 0.886;
154
1
  ccv_nnc_tensor_variable_t b2 = ccv_nnc_tensor_variable_new(graph, CPU_TENSOR_NHWC(32F, 1));
155
1
  ccv_nnc_tensor_from_variable(graph, b2)->data.f32[0] = 0.2;
156
1
  ccv_nnc_tensor_variable_t x21 = ccv_nnc_tensor_variable_alias_new(graph, x, DIM_ALLOC(0, 2), DIM_ALLOC(4, 1), CPU_TENSOR_NHWC(32F, 1, 1));
157
1
  ccv_nnc_dynamic_graph_exec(graph, CMD_GEMM_FORWARD(NO_TRANSPOSE, TRANSPOSE(0, 1)), ccv_nnc_no_hint, 0, TENSOR_VARIABLE_LIST(x2, w2, b2), TENSOR_VARIABLE_LIST(x21), 0, 0);
158
1
  ccv_nnc_tensor_variable_t x3 = ccv_nnc_tensor_variable_alias_new(graph, x, ccv_nnc_no_ofs, DIM_ALLOC(4, 1), CPU_TENSOR_NHWC(32F, 1, 3));
159
1
  ccv_nnc_tensor_variable_t w3 = ccv_nnc_tensor_variable_new(graph, CPU_TENSOR_NHWC(32F, 1, 3));
160
1
  ccv_nnc_tensor_from_variable(graph, w3)->data.f32[0] = 0.484;
161
1
  ccv_nnc_tensor_from_variable(graph, w3)->data.f32[1] = 0.912;
162
1
  ccv_nnc_tensor_from_variable(graph, w3)->data.f32[2] = 0.235;
163
1
  ccv_nnc_tensor_variable_t b3 = ccv_nnc_tensor_variable_new(graph, CPU_TENSOR_NHWC(32F, 1));
164
1
  ccv_nnc_tensor_from_variable(graph, b3)->data.f32[0] = 0.3;
165
1
  ccv_nnc_tensor_variable_t x31 = ccv_nnc_tensor_variable_alias_new(graph, x, DIM_ALLOC(0, 3), DIM_ALLOC(4, 1), CPU_TENSOR_NHWC(32F, 1, 1));
166
1
  ccv_nnc_dynamic_graph_exec(graph, CMD_GEMM_FORWARD(NO_TRANSPOSE, TRANSPOSE(0, 1)), ccv_nnc_no_hint, 0, TENSOR_VARIABLE_LIST(x3, w3, b3), TENSOR_VARIABLE_LIST(x31), 0, 0);
167
1
  DYNAMIC_GRAPH_GEN(graph, CCV_NNC_LONG_DOT_GRAPH);
168
1
  ccv_nnc_tensor_t* xt = ccv_nnc_tensor_new(0, CPU_TENSOR_NHWC(32F, 1, 4), 0);
169
1
  xt->data.f32[0] = 0.472;
170
1
  xt->data.f32[1] = xt->data.f32[0] * 0.234 + 0.1;
171
1
  xt->data.f32[2] = xt->data.f32[0] * 0.374 + xt->data.f32[1] * 0.886 + 0.2;
172
1
  xt->data.f32[3] = xt->data.f32[0] * 0.484 + xt->data.f32[1] * 0.912 + xt->data.f32[2] * 0.235 + 0.3;
173
1
  REQUIRE_MATRIX_EQ(ccv_nnc_tensor_from_variable(graph, x), xt, "1x4 matrix should be exactly the same");
174
1
  ccv_nnc_tensor_free(xt);
175
1
  ccv_nnc_tensor_variable_t dw1 = ccv_nnc_tensor_variable_new(graph);
176
1
  ccv_nnc_dynamic_graph_backward(graph, TENSOR_VARIABLE_LIST(x), 0, TENSOR_VARIABLE_LIST(w1), TENSOR_VARIABLE_LIST(dw1), 0);
177
1
  REQUIRE_EQ_WITH_TOLERANCE((0.235 * 0.886 + 0.912) * 0.472, ccv_nnc_tensor_from_variable(graph, dw1)->data.f32[0], 1e-5, "the gradient should be equal to a complicated result");
178
1
  ccv_nnc_dynamic_graph_free(graph);
179
1
}
180
181
TEST_CASE("batch norm in dynamic graph (enforce inplace)")
182
1
{
183
1
  ccv_nnc_dynamic_graph_t* const graph = ccv_nnc_dynamic_graph_new();
184
1
  ccv_nnc_tensor_variable_t x = ccv_nnc_tensor_variable_new(graph, CPU_TENSOR_NHWC(32F, 2, 2, 2, 10));
185
1
  ccv_nnc_tensor_variable_t y = ccv_nnc_tensor_variable_new(graph);
186
1
  ccv_nnc_tensor_variable_t scale = ccv_nnc_tensor_variable_new(graph, CPU_TENSOR_NHWC(32F, 10));
187
1
  ccv_nnc_tensor_variable_t bias = ccv_nnc_tensor_variable_new(graph, CPU_TENSOR_NHWC(32F, 10));
188
1
  ccv_nnc_tensor_variable_t mean = ccv_nnc_tensor_variable_new(graph, CPU_TENSOR_NHWC(32F, 10));
189
1
  ccv_nnc_tensor_variable_t var = ccv_nnc_tensor_variable_new(graph, CPU_TENSOR_NHWC(32F, 10));
190
1
  ccv_nnc_tensor_variable_t saved_mean = ccv_nnc_tensor_variable_new(graph);
191
1
  ccv_nnc_tensor_variable_t saved_inv_std = ccv_nnc_tensor_variable_new(graph);
192
1
  dsfmt_t dsfmt;
193
1
  int i;
194
1
  dsfmt_init_gen_rand(&dsfmt, 1);
195
81
  for (i = 0; i < 2 * 2 * 2 * 10; 
i++80
)
196
80
    ccv_nnc_tensor_from_variable(graph, x)->data.f32[i] = dsfmt_genrand_open_close(&dsfmt);
197
11
  for (i = 0; i < 10; 
i++10
)
198
10
    ccv_nnc_tensor_from_variable(graph, scale)->data.f32[i] = 1;
199
11
  for (i = 0; i < 10; 
i++10
)
200
10
    ccv_nnc_tensor_from_variable(graph, bias)->data.f32[i] = 0;
201
11
  for (i = 0; i < 10; 
i++10
)
202
10
    ccv_nnc_tensor_from_variable(graph, mean)->data.f32[i] = 0;
203
1
  ccv_nnc_tensor_t* mean_tensor_ptr = ccv_nnc_tensor_from_variable(graph, mean);
204
11
  for (i = 0; i < 10; 
i++10
)
205
10
    ccv_nnc_tensor_from_variable(graph, var)->data.f32[i] = 0;
206
1
  ccv_nnc_tensor_t* var_tensor_ptr = ccv_nnc_tensor_from_variable(graph, var);
207
1
  ccv_nnc_dynamic_graph_exec(graph, CMD_BATCH_NORM_FORWARD(0, 0, 0.9, 0, 1, 2), ccv_nnc_no_hint, 0, TENSOR_VARIABLE_LIST(x, scale, bias, mean, var), TENSOR_VARIABLE_LIST(y, mean, var, saved_mean, saved_inv_std), 0, 0);
208
1
  DYNAMIC_GRAPH_GEN(graph, CCV_NNC_LONG_DOT_GRAPH);
209
1
  REQUIRE(mean_tensor_ptr == ccv_nnc_tensor_from_variable(graph, mean), "enforced inplace, tensor view pointer unchanged");
210
1
  REQUIRE(var_tensor_ptr == ccv_nnc_tensor_from_variable(graph, var), "enforced inplace, tensor view pointer unchanged");
211
1
  ccv_nnc_tensor_t* x_tensor = ccv_nnc_tensor_new(0, CPU_TENSOR_NHWC(32F, 2, 2, 2, 10), 0);
212
1
  ccv_nnc_tensor_t* y_tensor = ccv_nnc_tensor_new(0, CPU_TENSOR_NHWC(32F, 2, 2, 2, 10), 0);
213
1
  ccv_nnc_tensor_t* scale_tensor = ccv_nnc_tensor_new(0, CPU_TENSOR_NHWC(32F, 10), 0);
214
1
  ccv_nnc_tensor_t* bias_tensor = ccv_nnc_tensor_new(0, CPU_TENSOR_NHWC(32F, 10), 0);
215
1
  ccv_nnc_tensor_t* mean_tensor = ccv_nnc_tensor_new(0, CPU_TENSOR_NHWC(32F, 10), 0);
216
1
  ccv_nnc_tensor_t* var_tensor = ccv_nnc_tensor_new(0, CPU_TENSOR_NHWC(32F, 10), 0);
217
1
  ccv_nnc_tensor_t* saved_mean_tensor = ccv_nnc_tensor_new(0, CPU_TENSOR_NHWC(32F, 1, 1, 1, 10), 0);
218
1
  ccv_nnc_tensor_t* saved_inv_std_tensor = ccv_nnc_tensor_new(0, CPU_TENSOR_NHWC(32F, 1, 1, 1, 10), 0);
219
1
  memcpy(x_tensor->data.f32, ccv_nnc_tensor_from_variable(graph, x)->data.f32, sizeof(float) * 2 * 2 * 2 * 10);
220
11
  for (i = 0; i < 10; 
i++10
)
221
10
    scale_tensor->data.f32[i] = 1;
222
1
  memset(bias_tensor->data.f32, 0, sizeof(float) * 10);
223
1
  memset(mean_tensor->data.f32, 0, sizeof(float) * 10);
224
1
  memset(var_tensor->data.f32, 0, sizeof(float) * 10);
225
1
  ccv_nnc_cmd_exec(CMD_BATCH_NORM_FORWARD(0, 0, 0.9, 0, 1, 2), ccv_nnc_no_hint, 0, TENSOR_LIST(x_tensor, scale_tensor, bias_tensor, mean_tensor, var_tensor), TENSOR_LIST(y_tensor, mean_tensor, var_tensor, saved_mean_tensor, saved_inv_std_tensor), 0);
226
1
  REQUIRE_TENSOR_EQ(y_tensor, ccv_nnc_tensor_from_variable(graph, y), "y should be equal");
227
1
  REQUIRE_TENSOR_EQ(mean_tensor, ccv_nnc_tensor_from_variable(graph, mean), "mean should be equal");
228
1
  REQUIRE_TENSOR_EQ(var_tensor, ccv_nnc_tensor_from_variable(graph, var), "var should be equal");
229
1
  REQUIRE_TENSOR_EQ(saved_mean_tensor, ccv_nnc_tensor_from_variable(graph, saved_mean), "saved_mean should be equal");
230
1
  REQUIRE_TENSOR_EQ(saved_inv_std_tensor, ccv_nnc_tensor_from_variable(graph, saved_inv_std), "saved_inv_std should be equal");
231
1
  ccv_nnc_dynamic_graph_free(graph);
232
1
  ccv_nnc_tensor_free(x_tensor);
233
1
  ccv_nnc_tensor_free(y_tensor);
234
1
  ccv_nnc_tensor_free(scale_tensor);
235
1
  ccv_nnc_tensor_free(bias_tensor);
236
1
  ccv_nnc_tensor_free(mean_tensor);
237
1
  ccv_nnc_tensor_free(var_tensor);
238
1
  ccv_nnc_tensor_free(saved_mean_tensor);
239
1
  ccv_nnc_tensor_free(saved_inv_std_tensor);
240
1
}
241
242
TEST_CASE("empty inputs / outputs for dynamic graph")
243
1
{
244
1
  ccv_nnc_dynamic_graph_t* const graph = ccv_nnc_dynamic_graph_new();
245
1
  ccv_nnc_tensor_variable_t df = ccv_nnc_tensor_variable_new(graph, CPU_TENSOR_NHWC(32F, 1));
246
1
  ccv_nnc_tensor_variable_t x = ccv_nnc_tensor_variable_new(graph, CPU_TENSOR_NHWC(32F, 1));
247
1
  ccv_nnc_tensor_variable_t y = ccv_nnc_tensor_variable_new(graph);
248
1
  ccv_nnc_tensor_from_variable(graph, df)->data.f32[0] = 1;
249
1
  ccv_nnc_tensor_from_variable(graph, x)->data.f32[0] = 10;
250
1
  ccv_nnc_dynamic_graph_exec(graph, CMD_EWDIV_BACKWARD(), ccv_nnc_no_hint, 0, TENSOR_VARIABLE_LIST(df, 0, x), TENSOR_VARIABLE_LIST(y, 0), 0, 0);
251
1
  DYNAMIC_GRAPH_GEN(graph, CCV_NNC_LONG_DOT_GRAPH);
252
1
  REQUIRE_EQ_WITH_TOLERANCE(ccv_nnc_tensor_from_variable(graph, y)->data.f32[0], 1. / 10, 1e-5, "div backward should equal to 1 / 10");
253
1
  ccv_nnc_dynamic_graph_free(graph);
254
1
}
255
256
TEST_CASE("long dynamic graph with unused variables freed")
257
1
{
258
1
  ccv_nnc_dynamic_graph_t* const graph = ccv_nnc_dynamic_graph_new();
259
1
  int i;
260
1
  ccv_nnc_tensor_variable_t x = ccv_nnc_tensor_variable_new(graph, CPU_TENSOR_NHWC(32F, 1));
261
1
  ccv_nnc_tensor_variable_t y = ccv_nnc_tensor_variable_new(graph, CPU_TENSOR_NHWC(32F, 1));
262
1
  ccv_nnc_tensor_from_variable(graph, x)->data.f32[0] = 32;
263
1
  ccv_nnc_tensor_from_variable(graph, y)->data.f32[0] = 0.5;
264
11
  for (i = 0; i < 10; 
i++10
)
265
10
  {
266
10
    ccv_nnc_tensor_variable_t z = ccv_nnc_tensor_variable_new(graph);
267
10
    if (i < 7)
268
7
      ccv_nnc_dynamic_graph_exec(graph, CMD_EWPROD_FORWARD(), ccv_nnc_no_hint, 0, TENSOR_VARIABLE_LIST(x, y), TENSOR_VARIABLE_LIST(z), 0, 0);
269
3
    else {
270
3
      if (i == 7)
271
1
        ccv_nnc_tensor_variable_free(graph, y); // No longer need y.
272
3
      ccv_nnc_dynamic_graph_exec(graph, CMD_EWPROD_FORWARD(), ccv_nnc_no_hint, 0, TENSOR_VARIABLE_LIST(x, x), TENSOR_VARIABLE_LIST(z), 0, 0);
273
3
    }
274
10
    if (i < 9)
275
9
      ccv_nnc_tensor_variable_free(graph, x);
276
10
    x = z;
277
10
  }
278
1
  DYNAMIC_GRAPH_GEN(graph, CCV_NNC_LONG_DOT_GRAPH);
279
1
  float g = 32;
280
11
  for (i = 0; i < 10; 
i++10
)
281
10
  {
282
10
    if (i < 7)
283
7
      g = g * 0.5;
284
3
    else
285
3
      g = g * g;
286
10
  }
287
1
  REQUIRE_EQ_WITH_TOLERANCE(ccv_nnc_tensor_from_variable(graph, x)->data.f32[0], g, 1e-5, "x should equal to the computed result");
288
1
  ccv_nnc_dynamic_graph_free(graph);
289
1
}
290
291
TEST_CASE("repeat multiple x * y with y as a constant")
292
1
{
293
1
  ccv_nnc_dynamic_graph_t* const graph = ccv_nnc_dynamic_graph_new();
294
1
  int i;
295
1
  ccv_nnc_tensor_variable_t x = ccv_nnc_tensor_variable_new(graph, CPU_TENSOR_NHWC(32F, 1));
296
1
  ccv_nnc_tensor_variable_t y = ccv_nnc_tensor_constant_new(graph, CPU_TENSOR_NHWC(32F, 1));
297
1
  ccv_nnc_tensor_from_variable(graph, x)->data.f32[0] = 32;
298
1
  ccv_nnc_tensor_from_variable(graph, y)->data.f32[0] = 0.5;
299
11
  for (i = 0; i < 10; 
i++10
)
300
10
  {
301
10
    ccv_nnc_tensor_variable_t z = ccv_nnc_tensor_variable_new(graph);
302
10
    ccv_nnc_dynamic_graph_exec(graph, CMD_EWPROD_FORWARD(), ccv_nnc_no_hint, 0, TENSOR_VARIABLE_LIST(x, y), TENSOR_VARIABLE_LIST(z), 0, 0);
303
10
    ccv_nnc_tensor_variable_free(graph, x);
304
10
    x = z;
305
10
  }
306
1
  DYNAMIC_GRAPH_GEN(graph, CCV_NNC_LONG_DOT_GRAPH);
307
1
  float g = 32;
308
11
  for (i = 0; i < 10; 
i++10
)
309
10
    g = g * 0.5;
310
1
  REQUIRE_EQ_WITH_TOLERANCE(ccv_nnc_tensor_from_variable(graph, x)->data.f32[0], g, 1e-5, "x should equal to the computed result");
311
1
  ccv_nnc_dynamic_graph_free(graph);
312
1
}
313
314
static int _ccv_tensor_variable_freed = 0;
315
316
static void _ccv_tensor_variable_hook(ccv_nnc_dynamic_graph_t* const graph, const ccv_nnc_tensor_t* const tensor, void* const context)
317
7
{
318
7
  ++_ccv_tensor_variable_freed;
319
7
}
320
321
TEST_CASE("repeat multiple x * y with y as a constant, compute d(x)")
322
1
{
323
1
  ccv_nnc_dynamic_graph_t* const graph = ccv_nnc_dynamic_graph_new();
324
1
  int i;
325
1
  _ccv_tensor_variable_freed = 0;
326
1
  ccv_nnc_tensor_variable_t x = ccv_nnc_tensor_variable_new(graph, CPU_TENSOR_NHWC(32F, 1));
327
1
  ccv_nnc_tensor_variable_t y = ccv_nnc_tensor_constant_new(graph, CPU_TENSOR_NHWC(32F, 1));
328
1
  ccv_nnc_tensor_variable_t ox = x;
329
1
  ccv_nnc_tensor_variable_destructor_hook(graph, x, _ccv_tensor_variable_hook, 0);
330
1
  ccv_nnc_tensor_variable_destructor_hook(graph, y, _ccv_tensor_variable_hook, 0);
331
1
  ccv_nnc_tensor_from_variable(graph, x)->data.f32[0] = 32;
332
1
  ccv_nnc_tensor_from_variable(graph, y)->data.f32[0] = 0.9;
333
5
  for (i = 0; i < 4; 
i++4
)
334
4
  {
335
4
    ccv_nnc_tensor_variable_t z = ccv_nnc_tensor_variable_new(graph);
336
4
    ccv_nnc_tensor_variable_destructor_hook(graph, z, _ccv_tensor_variable_hook, 0);
337
4
    ccv_nnc_dynamic_graph_exec(graph, CMD_EWPROD_FORWARD(), ccv_nnc_no_hint, 0, TENSOR_VARIABLE_LIST(x, y), TENSOR_VARIABLE_LIST(z), 0, 0);
338
4
    if (i > 0)
339
3
      ccv_nnc_tensor_variable_free(graph, x);
340
4
    x = z;
341
4
  }
342
1
  REQUIRE_EQ(0, _ccv_tensor_variable_freed, "none of these are freed");
343
1
  ccv_nnc_tensor_variable_t dx = ccv_nnc_tensor_variable_new(graph);
344
1
  ccv_nnc_tensor_variable_destructor_hook(graph, dx, _ccv_tensor_variable_hook, 0);
345
1
  ccv_nnc_dynamic_graph_backward(graph, TENSOR_VARIABLE_LIST(x), 0, TENSOR_VARIABLE_LIST(ox), TENSOR_VARIABLE_LIST(dx), 0);
346
1
  ccv_nnc_tensor_variable_free(graph, ox);
347
  // We freed all 4 x (ox, when i = 1, 2, 3).
348
1
  REQUIRE_EQ(4, _ccv_tensor_variable_freed, "all are freed except dx");
349
1
  _ccv_tensor_variable_freed = 0;
350
1
  DYNAMIC_GRAPH_GEN(graph, CCV_NNC_LONG_DOT_GRAPH);
351
1
  float g = 1;
352
5
  for (i = 0; i < 4; 
i++4
)
353
4
    g = g * 0.9;
354
1
  REQUIRE_EQ_WITH_TOLERANCE(ccv_nnc_tensor_from_variable(graph, dx)->data.f32[0], g, 1e-5, "x should equal to the computed result");
355
1
  ccv_nnc_dynamic_graph_free(graph);
356
  // y, dx, and z.
357
1
  REQUIRE_EQ(3, _ccv_tensor_variable_freed, "dx freed");
358
1
}
359
360
TEST_CASE("compute f(x) = x * log(x) + x, f'(x) when x = 10 (and intermediate results all freed)")
361
1
{
362
1
  ccv_nnc_dynamic_graph_t* const graph = ccv_nnc_dynamic_graph_new();
363
1
  ccv_nnc_tensor_variable_t x = ccv_nnc_tensor_variable_new(graph, CPU_TENSOR_NHWC(32F, 1));
364
1
  ccv_nnc_tensor_variable_t y = ccv_nnc_tensor_variable_new(graph);
365
1
  ccv_nnc_tensor_variable_t f = ccv_nnc_tensor_variable_new(graph);
366
1
  ccv_nnc_tensor_from_variable(graph, x)->data.f32[0] = 10;
367
1
  ccv_nnc_dynamic_graph_exec(graph, CMD_EWLOG_FORWARD(), ccv_nnc_no_hint, 0, TENSOR_VARIABLE_LIST(x), TENSOR_VARIABLE_LIST(y), 0, 0);
368
1
  ccv_nnc_dynamic_graph_exec(graph, CMD_EWPROD_FORWARD(), ccv_nnc_no_hint, 0, TENSOR_VARIABLE_LIST(x, y), TENSOR_VARIABLE_LIST(y), 0, 0);
369
1
  ccv_nnc_dynamic_graph_exec(graph, CMD_EWSUM_FORWARD(), ccv_nnc_no_hint, 0, TENSOR_VARIABLE_LIST(y, x), TENSOR_VARIABLE_LIST(f), 0, 0);
370
1
  ccv_nnc_tensor_variable_t df = ccv_nnc_tensor_variable_new(graph, CPU_TENSOR_NHWC(32F, 1));
371
1
  ccv_nnc_dynamic_graph_exec(graph, CMD_SET_FORWARD(1), ccv_nnc_no_hint, 0, 0, 0, TENSOR_VARIABLE_LIST(df), 0, 0);
372
  // x will be accumulated on to itself.
373
1
  ccv_nnc_dynamic_graph_backward(graph, TENSOR_VARIABLE_LIST(f), &df, TENSOR_VARIABLE_LIST(x), TENSOR_VARIABLE_LIST(x), 0);
374
1
  DYNAMIC_GRAPH_GEN(graph, CCV_NNC_LONG_DOT_GRAPH);
375
1
  REQUIRE_EQ_WITH_TOLERANCE(ccv_nnc_tensor_from_variable(graph, x)->data.f32[0], 10 + log(10) + 1 + 1, 1e-5, "dx should equal to the computed result");
376
1
  ccv_nnc_dynamic_graph_free(graph);
377
1
}
378
379
TEST_CASE("dynamic graph with binded value")
380
1
{
381
1
  ccv_nnc_dynamic_graph_t* const graph = ccv_nnc_dynamic_graph_new();
382
1
  ccv_nnc_tensor_variable_t x = ccv_nnc_tensor_variable_new(graph, CPU_TENSOR_NHWC(32F, 1));
383
1
  ccv_nnc_tensor_t* const x_tensor = ccv_nnc_tensor_new(0, CPU_TENSOR_NHWC(32F, 1), 0);
384
1
  x_tensor->data.f32[0] = 10;
385
1
  ccv_nnc_tensor_variable_set(graph, x, x_tensor);
386
1
  ccv_nnc_tensor_variable_t y = ccv_nnc_tensor_variable_new(graph);
387
1
  ccv_nnc_dynamic_graph_exec(graph, CMD_EWLOG_FORWARD(), ccv_nnc_no_hint, 0, TENSOR_VARIABLE_LIST(x), TENSOR_VARIABLE_LIST(y), 0, 0);
388
1
  DYNAMIC_GRAPH_GEN(graph, CCV_NNC_LONG_DOT_GRAPH);
389
1
  REQUIRE_EQ_WITH_TOLERANCE(ccv_nnc_tensor_from_variable(graph, y)->data.f32[0], log(10), 1e-5, "dx should equal to the computed result");
390
1
  ccv_nnc_dynamic_graph_free(graph);
391
1
  ccv_nnc_tensor_free(x_tensor);
392
1
}
393
394
TEST_CASE("dynamic graph to evaluate cnnp model")
395
1
{
396
1
  ccv_nnc_dynamic_graph_t* const graph = ccv_nnc_dynamic_graph_new();
397
1
  ccv_cnnp_model_t* const linear = ccv_cnnp_dense(1, 0, 0, 1, 0);
398
1
  ccv_nnc_tensor_variable_t z = ccv_nnc_tensor_constant_new(graph, CPU_TENSOR_NHWC(32F, 1));
399
1
  ccv_nnc_tensor_from_variable(graph, z)->data.f32[0] = 5;
400
1
  int i;
401
101
  for (i = 0; i < 100; 
i++100
)
402
100
  {
403
100
    ccv_nnc_tensor_variable_t x;
404
100
    if (i % 2 == 1)
405
50
    {
406
50
      x = ccv_nnc_tensor_variable_new(graph, CPU_TENSOR_NHWC(32F, 2, 1));
407
50
      ccv_nnc_tensor_from_variable(graph, x)->data.f32[0] = 10;
408
50
      ccv_nnc_tensor_from_variable(graph, x)->data.f32[1] = 10;
409
50
    } else {
410
50
      x = ccv_nnc_tensor_variable_new(graph, CPU_TENSOR_NHWC(32F, 1));
411
50
      ccv_nnc_tensor_from_variable(graph, x)->data.f32[0] = 10;
412
50
    }
413
100
    ccv_nnc_tensor_variable_t y = ccv_nnc_tensor_variable_new(graph);
414
100
    ccv_nnc_dynamic_graph_exec(graph, CMD_EWLOG_FORWARD(), ccv_nnc_no_hint, 0, TENSOR_VARIABLE_LIST(x), TENSOR_VARIABLE_LIST(y), 0, 0);
415
100
    ccv_nnc_dynamic_graph_evaluate(graph, linear, 0, TENSOR_VARIABLE_LIST(y), TENSOR_VARIABLE_LIST(y), 0, 0);
416
100
    ccv_nnc_dynamic_graph_exec(graph, CMD_ADD_FORWARD(1, -1), ccv_nnc_no_hint, 0, TENSOR_VARIABLE_LIST(y, z), TENSOR_VARIABLE_LIST(y), 0, 0);
417
100
    ccv_nnc_tensor_variable_t f = ccv_nnc_tensor_variable_new(graph);
418
100
    ccv_nnc_dynamic_graph_exec(graph, CMD_EWPROD_FORWARD(), ccv_nnc_no_hint, 0, TENSOR_VARIABLE_LIST(y, y), TENSOR_VARIABLE_LIST(f), 0, 0);
419
100
    ccv_nnc_tensor_variable_t dx = ccv_nnc_tensor_variable_new(graph);
420
100
    ccv_nnc_dynamic_graph_backward(graph, TENSOR_VARIABLE_LIST(f), 0, TENSOR_VARIABLE_LIST(x), TENSOR_VARIABLE_LIST(dx), 0);
421
100
    ccv_cnnp_model_set_minimizer(linear, CMD_SGD_FORWARD(0, 0.01, 1, 0.01, 0, 0), 0, 0, 0);
422
100
    ccv_nnc_dynamic_graph_apply_gradients(graph, CMD_SGD_FORWARD(0, 0.01, 1, 0.01, 0, 0), TENSOR_VARIABLE_LIST(), TENSOR_VARIABLE_LIST(), 0, 0, 0);
423
100
    ccv_nnc_tensor_variable_free(graph, x);
424
100
    ccv_nnc_tensor_variable_free(graph, y);
425
100
    ccv_nnc_tensor_variable_free(graph, f);
426
100
    ccv_nnc_tensor_variable_free(graph, dx);
427
100
  }
428
1
  ccv_nnc_tensor_variable_t x = ccv_nnc_tensor_variable_new(graph, CPU_TENSOR_NHWC(32F, 1));
429
1
  ccv_nnc_tensor_from_variable(graph, x)->data.f32[0] = 10;
430
1
  ccv_nnc_tensor_variable_t y = ccv_nnc_tensor_variable_new(graph);
431
1
  ccv_nnc_dynamic_graph_exec(graph, CMD_EWLOG_FORWARD(), ccv_nnc_no_hint, 0, TENSOR_VARIABLE_LIST(x), TENSOR_VARIABLE_LIST(y), 0, 0);
432
1
  ccv_nnc_dynamic_graph_evaluate(graph, linear, 1, TENSOR_VARIABLE_LIST(y), TENSOR_VARIABLE_LIST(y), 0, 0);
433
1
  REQUIRE_EQ_WITH_TOLERANCE(ccv_nnc_tensor_from_variable(graph, y)->data.f32[0], 5, 1e-2, "linear model should be trained to generate the same value as z");
434
1
  ccv_nnc_tensor_variable_t iy = ccv_nnc_tensor_constant_new(graph, CPU_TENSOR_NHWC(32F, 1));
435
1
  ccv_nnc_tensor_from_variable(graph, iy)->data.f32[0] = log(10);
436
1
  ccv_nnc_tensor_variable_t iz = ccv_nnc_tensor_variable_new(graph, CPU_TENSOR_NHWC(32F, 1));
437
1
  ccv_nnc_dynamic_graph_evaluate(graph, linear, 1, TENSOR_VARIABLE_LIST(iy), TENSOR_VARIABLE_LIST(iz), 0, 0);
438
1
  REQUIRE_EQ_WITH_TOLERANCE(ccv_nnc_tensor_from_variable(graph, iz)->data.f32[0], 5, 1e-2, "linear model should be trained to generate the same value as z");
439
1
  DYNAMIC_GRAPH_GEN(graph, CCV_NNC_LONG_DOT_GRAPH);
440
1
  ccv_nnc_dynamic_graph_free(graph);
441
1
  ccv_cnnp_model_free(linear);
442
1
}
443
444
TEST_CASE("dynamic graph to evaluate cnnp model without any parameters")
445
1
{
446
1
  ccv_nnc_dynamic_graph_t* const graph = ccv_nnc_dynamic_graph_new();
447
1
  ccv_nnc_tensor_variable_t a = ccv_nnc_tensor_variable_new(graph, CPU_TENSOR_NHWC(32F, 1));
448
1
  ccv_nnc_tensor_from_variable(graph, a)->data.f32[0] = 1.23;
449
1
  ccv_nnc_tensor_variable_t b = ccv_nnc_tensor_variable_new(graph, CPU_TENSOR_NHWC(32F, 1));
450
1
  ccv_nnc_tensor_from_variable(graph, b)->data.f32[0] = 2;
451
1
  ccv_nnc_tensor_variable_t c = ccv_nnc_tensor_variable_new(graph, CPU_TENSOR_NHWC(32F, 1));
452
1
  ccv_cnnp_model_t* const mul = ccv_cnnp_mul(1, "mul");
453
1
  ccv_nnc_dynamic_graph_evaluate(graph, mul, 1, TENSOR_VARIABLE_LIST(a, b), TENSOR_VARIABLE_LIST(c), 0, 0);
454
1
  ccv_nnc_tensor_variable_t da = ccv_nnc_tensor_variable_new(graph);
455
1
  ccv_nnc_dynamic_graph_backward(graph, TENSOR_VARIABLE_LIST(c), 0, TENSOR_VARIABLE_LIST(a), TENSOR_VARIABLE_LIST(da), 0);
456
1
  ccv_cnnp_model_set_minimizer(mul, CMD_SGD_FORWARD(0, 0.01, 1, 0.01, 0, 0), 0, 0, 0);
457
1
  ccv_nnc_dynamic_graph_apply_gradients(graph, CMD_SGD_FORWARD(0, 0.01, 1, 0.01, 0, 0), TENSOR_VARIABLE_LIST(), TENSOR_VARIABLE_LIST(), 0, 0, 0);
458
1
  ccv_cnnp_model_free(mul);
459
1
  REQUIRE_EQ_WITH_TOLERANCE(ccv_nnc_tensor_from_variable(graph, c)->data.f32[0], 2.46, 1e-5, "should be equal");
460
1
  REQUIRE_EQ_WITH_TOLERANCE(ccv_nnc_tensor_from_variable(graph, da)->data.f32[0], 2, 1e-5, "should be equal");
461
1
  ccv_nnc_dynamic_graph_free(graph);
462
1
}
463
464
TEST_CASE("dynamic graph to evaluate cnnp model without any parameters with div")
465
1
{
466
1
  ccv_nnc_dynamic_graph_t* const graph = ccv_nnc_dynamic_graph_new();
467
1
  ccv_nnc_tensor_variable_t a = ccv_nnc_tensor_variable_new(graph, CPU_TENSOR_NHWC(32F, 1));
468
1
  ccv_nnc_tensor_from_variable(graph, a)->data.f32[0] = 1.23;
469
1
  ccv_nnc_tensor_variable_t b = ccv_nnc_tensor_variable_new(graph, CPU_TENSOR_NHWC(32F, 1));
470
1
  ccv_nnc_tensor_from_variable(graph, b)->data.f32[0] = 2;
471
1
  ccv_nnc_tensor_variable_t c = ccv_nnc_tensor_variable_new(graph, CPU_TENSOR_NHWC(32F, 1));
472
1
  ccv_cnnp_model_t* const div = ccv_cnnp_div(0, "div");
473
1
  ccv_nnc_dynamic_graph_evaluate(graph, div, 1, TENSOR_VARIABLE_LIST(a, b), TENSOR_VARIABLE_LIST(c), 0, 0);
474
1
  ccv_nnc_tensor_variable_t da = ccv_nnc_tensor_variable_new(graph);
475
1
  ccv_nnc_dynamic_graph_backward(graph, TENSOR_VARIABLE_LIST(c), 0, TENSOR_VARIABLE_LIST(a), TENSOR_VARIABLE_LIST(da), 0);
476
1
  ccv_cnnp_model_set_minimizer(div, CMD_SGD_FORWARD(0, 0.01, 1, 0.01, 0, 0), 0, 0, 0);
477
1
  ccv_nnc_dynamic_graph_apply_gradients(graph, CMD_SGD_FORWARD(0, 0.01, 1, 0.01, 0, 0), TENSOR_VARIABLE_LIST(), TENSOR_VARIABLE_LIST(), 0, 0, 0);
478
1
  ccv_cnnp_model_free(div);
479
1
  REQUIRE_EQ_WITH_TOLERANCE(ccv_nnc_tensor_from_variable(graph, c)->data.f32[0], 1.23 / 2, 1e-5, "should be equal");
480
1
  REQUIRE_EQ_WITH_TOLERANCE(ccv_nnc_tensor_from_variable(graph, da)->data.f32[0], 1.0 / 2, 1e-5, "should be equal");
481
1
  ccv_nnc_dynamic_graph_free(graph);
482
1
}
483
484
TEST_CASE("dynamic graph to evaluate cnnp model without any parameters with reciprocal")
485
1
{
486
1
  ccv_nnc_dynamic_graph_t* const graph = ccv_nnc_dynamic_graph_new();
487
1
  ccv_nnc_tensor_variable_t a = ccv_nnc_tensor_variable_new(graph, CPU_TENSOR_NHWC(32F, 1));
488
1
  ccv_nnc_tensor_from_variable(graph, a)->data.f32[0] = 1.23;
489
1
  ccv_nnc_tensor_variable_t c = ccv_nnc_tensor_variable_new(graph, CPU_TENSOR_NHWC(32F, 1));
490
1
  ccv_cnnp_model_t* const div = ccv_cnnp_div(1, "div");
491
1
  ccv_nnc_dynamic_graph_evaluate(graph, div, 1, TENSOR_VARIABLE_LIST(a), TENSOR_VARIABLE_LIST(c), 0, 0);
492
1
  ccv_nnc_tensor_variable_t da = ccv_nnc_tensor_variable_new(graph);
493
1
  ccv_nnc_dynamic_graph_backward(graph, TENSOR_VARIABLE_LIST(c), 0, TENSOR_VARIABLE_LIST(a), TENSOR_VARIABLE_LIST(da), 0);
494
1
  ccv_cnnp_model_set_minimizer(div, CMD_SGD_FORWARD(0, 0.01, 1, 0.01, 0, 0), 0, 0, 0);
495
1
  ccv_nnc_dynamic_graph_apply_gradients(graph, CMD_SGD_FORWARD(0, 0.01, 1, 0.01, 0, 0), TENSOR_VARIABLE_LIST(), TENSOR_VARIABLE_LIST(), 0, 0, 0);
496
1
  ccv_cnnp_model_free(div);
497
1
  REQUIRE_EQ_WITH_TOLERANCE(ccv_nnc_tensor_from_variable(graph, c)->data.f32[0], 1 / 1.23, 1e-5, "should be equal");
498
1
  REQUIRE_EQ_WITH_TOLERANCE(ccv_nnc_tensor_from_variable(graph, da)->data.f32[0], -1 / (1.23 * 1.23), 1e-5, "should be equal");
499
1
  ccv_nnc_dynamic_graph_free(graph);
500
1
}
501
502
TEST_CASE("dynamic graph to evaluate cnnp model and simply accumulate gradients")
503
1
{
504
1
  ccv_nnc_dynamic_graph_t* const graph = ccv_nnc_dynamic_graph_new();
505
1
  ccv_cnnp_model_t* const linear = ccv_cnnp_dense(1, 0, 0, 1, 0);
506
1
  ccv_nnc_tensor_variable_t z = ccv_nnc_tensor_constant_new(graph, CPU_TENSOR_NHWC(32F, 1));
507
1
  ccv_nnc_tensor_from_variable(graph, z)->data.f32[0] = 5;
508
1
  int i;
509
101
  for (i = 0; i < 100; 
i++100
)
510
100
  {
511
100
    ccv_nnc_tensor_variable_t x;
512
100
    if (i % 2 == 1)
513
50
    {
514
50
      x = ccv_nnc_tensor_variable_new(graph, CPU_TENSOR_NHWC(32F, 2, 1));
515
50
      ccv_nnc_tensor_from_variable(graph, x)->data.f32[0] = 10;
516
50
      ccv_nnc_tensor_from_variable(graph, x)->data.f32[1] = 10;
517
50
    } else {
518
50
      x = ccv_nnc_tensor_variable_new(graph, CPU_TENSOR_NHWC(32F, 1));
519
50
      ccv_nnc_tensor_from_variable(graph, x)->data.f32[0] = 10;
520
50
    }
521
100
    ccv_nnc_tensor_variable_t y = ccv_nnc_tensor_variable_new(graph);
522
100
    ccv_nnc_dynamic_graph_exec(graph, CMD_EWLOG_FORWARD(), ccv_nnc_no_hint, 0, TENSOR_VARIABLE_LIST(x), TENSOR_VARIABLE_LIST(y), 0, 0);
523
100
    ccv_nnc_dynamic_graph_evaluate(graph, linear, 0, TENSOR_VARIABLE_LIST(y), TENSOR_VARIABLE_LIST(y), 0, 0);
524
100
    ccv_nnc_dynamic_graph_exec(graph, CMD_ADD_FORWARD(1, -1), ccv_nnc_no_hint, 0, TENSOR_VARIABLE_LIST(y, z), TENSOR_VARIABLE_LIST(y), 0, 0);
525
100
    ccv_nnc_tensor_variable_t f = ccv_nnc_tensor_variable_new(graph);
526
100
    ccv_nnc_dynamic_graph_exec(graph, CMD_EWPROD_FORWARD(), ccv_nnc_no_hint, 0, TENSOR_VARIABLE_LIST(y, y), TENSOR_VARIABLE_LIST(f), 0, 0);
527
100
    ccv_nnc_tensor_variable_t dx = ccv_nnc_tensor_variable_new(graph);
528
100
    ccv_nnc_dynamic_graph_backward(graph, TENSOR_VARIABLE_LIST(f), 0, TENSOR_VARIABLE_LIST(x), TENSOR_VARIABLE_LIST(dx), 0);
529
100
    ccv_nnc_tensor_variable_free(graph, x);
530
100
    ccv_nnc_tensor_variable_free(graph, y);
531
100
    ccv_nnc_tensor_variable_free(graph, f);
532
100
    ccv_nnc_tensor_variable_free(graph, dx);
533
100
    if ((i % 2) == 1)
534
50
    {
535
50
      ccv_cnnp_model_set_minimizer(linear, CMD_SGD_FORWARD(0, 0.01, 1, 0.01, 0, 0), 0, 0, 0);
536
50
      ccv_nnc_dynamic_graph_apply_gradients(graph, CMD_SGD_FORWARD(0, 0.01, 1, 0.01, 0, 0), TENSOR_VARIABLE_LIST(), TENSOR_VARIABLE_LIST(), 0, 0, 0);
537
50
    }
538
100
  }
539
1
  ccv_nnc_tensor_variable_t x = ccv_nnc_tensor_variable_new(graph, CPU_TENSOR_NHWC(32F, 1));
540
1
  ccv_nnc_tensor_from_variable(graph, x)->data.f32[0] = 10;
541
1
  ccv_nnc_tensor_variable_t y = ccv_nnc_tensor_variable_new(graph);
542
1
  ccv_nnc_dynamic_graph_exec(graph, CMD_EWLOG_FORWARD(), ccv_nnc_no_hint, 0, TENSOR_VARIABLE_LIST(x), TENSOR_VARIABLE_LIST(y), 0, 0);
543
1
  ccv_nnc_dynamic_graph_evaluate(graph, linear, 1, TENSOR_VARIABLE_LIST(y), TENSOR_VARIABLE_LIST(y), 0, 0);
544
1
  REQUIRE_EQ_WITH_TOLERANCE(ccv_nnc_tensor_from_variable(graph, y)->data.f32[0], 5, 1e-2, "linear model should be trained to generate the same value as z");
545
1
  DYNAMIC_GRAPH_GEN(graph, CCV_NNC_LONG_DOT_GRAPH);
546
1
  ccv_nnc_dynamic_graph_free(graph);
547
1
  ccv_cnnp_model_free(linear);
548
1
}
549
550
TEST_CASE("dynamic graph to accumulate gradients cross cnnp models")
551
1
{
552
1
  ccv_nnc_dynamic_graph_t* const graph = ccv_nnc_dynamic_graph_new();
553
1
  ccv_cnnp_model_t* const linear = ccv_cnnp_dense(1, 1, 0, 1, 0);
554
1
  ccv_nnc_tensor_variable_t a = ccv_nnc_tensor_variable_new(graph, CPU_TENSOR_NHWC(32F, 1));
555
1
  ccv_nnc_dynamic_graph_exec(graph, CMD_SET_FORWARD(0.2485), ccv_nnc_no_hint, 0, TENSOR_VARIABLE_LIST(), TENSOR_VARIABLE_LIST(a), 0, 0);
556
1
  const ccv_nnc_tensor_param_t input = CPU_TENSOR_NHWC(32F, 1);
557
1
  ccv_cnnp_model_compile(linear, &input, 1, CMD_NOOP(), CMD_NOOP());
558
1
  ccv_cnnp_model_set_parameter(linear, ccv_cnnp_model_parameters(linear, CCV_CNNP_PARAMETER_SELECT_WEIGHT, 0), ccv_nnc_tensor_from_variable(graph, a));
559
1
  ccv_nnc_tensor_variable_t a_grad = ccv_nnc_tensor_variable_new(graph);
560
1
  ccv_nnc_tensor_variable_t saved_aux = ccv_nnc_tensor_variable_new(graph, CPU_TENSOR_NHWC(32F, 1));
561
1
  int i;
562
1.00k
  for (i = 0; i < 1000; 
i++1.00k
)
563
1.00k
  {
564
1.00k
    ccv_nnc_tensor_variable_t x = ccv_nnc_tensor_variable_new(graph, CPU_TENSOR_NHWC(32F, 1));
565
1.00k
    ccv_nnc_tensor_from_variable(graph, x)->data.f32[0] = i;
566
1.00k
    ccv_nnc_tensor_variable_t t = ccv_nnc_tensor_variable_new(graph, CPU_TENSOR_NHWC(32F, 1));
567
1.00k
    ccv_nnc_tensor_from_variable(graph, t)->data.f32[0] = -i;
568
1.00k
    ccv_nnc_tensor_variable_t y = ccv_nnc_tensor_variable_new(graph);
569
1.00k
    ccv_nnc_dynamic_graph_exec(graph, CMD_MUL_FORWARD(1), ccv_nnc_no_hint, 0, TENSOR_VARIABLE_LIST(x, a), TENSOR_VARIABLE_LIST(y), 0, 0);
570
1.00k
    ccv_nnc_tensor_variable_t z = ccv_nnc_tensor_variable_new(graph);
571
1.00k
    ccv_nnc_dynamic_graph_evaluate(graph, linear, 0, TENSOR_VARIABLE_LIST(y), TENSOR_VARIABLE_LIST(z), 0, 0);
572
1.00k
    ccv_nnc_tensor_variable_t n = ccv_nnc_tensor_variable_new(graph);
573
1.00k
    ccv_nnc_dynamic_graph_exec(graph, CMD_ADD_FORWARD(1, -1), ccv_nnc_no_hint, 0, TENSOR_VARIABLE_LIST(z, t), TENSOR_VARIABLE_LIST(n), 0, 0);
574
1.00k
    ccv_nnc_tensor_variable_t f = ccv_nnc_tensor_variable_new(graph);
575
1.00k
    ccv_nnc_dynamic_graph_exec(graph, CMD_MUL_FORWARD(1), ccv_nnc_no_hint, 0, TENSOR_VARIABLE_LIST(n, n), TENSOR_VARIABLE_LIST(f), 0, 0);
576
1.00k
    ccv_nnc_dynamic_graph_backward(graph, TENSOR_VARIABLE_LIST(f), 0, TENSOR_VARIABLE_LIST(a), TENSOR_VARIABLE_LIST(a_grad), 0);
577
1.00k
    ccv_nnc_tensor_variable_free(graph, n);
578
1.00k
    ccv_nnc_tensor_variable_free(graph, z);
579
1.00k
    ccv_nnc_tensor_variable_free(graph, y);
580
1.00k
    ccv_nnc_tensor_variable_free(graph, x);
581
1.00k
    ccv_nnc_tensor_variable_free(graph, t);
582
1.00k
    ccv_nnc_tensor_variable_free(graph, f);
583
1.00k
    if (((i + 1) % 5) == 0)
584
200
    {
585
200
      DYNAMIC_GRAPH_GEN(graph, CCV_NNC_LONG_DOT_GRAPH);
586
200
      REQUIRE_EQ(ccv_nnc_dynamic_graph_bookkeeping_count(graph, CCV_NNC_SYMBOL_GRAPH_EXEC), 0, "everything should be freed");
587
200
      float lr = 0.001;
588
200
      if (i >= 200)
589
160
        lr = 0.0001;
590
40
      else if (i >= 600)
591
0
        lr = 0.00001;
592
200
      ccv_cnnp_model_set_minimizer(linear, CMD_SGD_FORWARD(0, lr, 0.001, 0, 0, 0), 0, 0, 0);
593
200
      ccv_nnc_dynamic_graph_apply_gradients(graph, CMD_SGD_FORWARD(0, lr, 0.001, 0, 0, 0), TENSOR_VARIABLE_LIST(a_grad), TENSOR_VARIABLE_LIST(a), &saved_aux, 0, 0);
594
200
    }
595
1.00k
  }
596
1
  ccv_nnc_tensor_variable_t x = ccv_nnc_tensor_variable_new(graph, CPU_TENSOR_NHWC(32F, 1));
597
1
  ccv_nnc_tensor_from_variable(graph, x)->data.f32[0] = 5;
598
1
  ccv_nnc_tensor_variable_t y = ccv_nnc_tensor_variable_new(graph);
599
1
  ccv_nnc_dynamic_graph_exec(graph, CMD_MUL_FORWARD(1), ccv_nnc_no_hint, 0, TENSOR_VARIABLE_LIST(x, a), TENSOR_VARIABLE_LIST(y), 0, 0);
600
1
  ccv_nnc_tensor_variable_t z = ccv_nnc_tensor_variable_new(graph);
601
1
  ccv_nnc_dynamic_graph_evaluate(graph, linear, 0, TENSOR_VARIABLE_LIST(y), TENSOR_VARIABLE_LIST(z), 0, 0);
602
1
  REQUIRE_EQ_WITH_TOLERANCE(ccv_nnc_tensor_from_variable(graph, z)->data.f32[0], -5, 1e-2, "linear model should be trained to generate the same value as z");
603
1
  ccv_nnc_dynamic_graph_free(graph);
604
1
  ccv_cnnp_model_free(linear);
605
1
}
606
607
TEST_CASE("dynamic graph to accumulate gradients cross cnnp models with aliases")
608
1
{
609
1
  ccv_nnc_dynamic_graph_t* const graph = ccv_nnc_dynamic_graph_new();
610
1
  ccv_cnnp_model_t* const linear = ccv_cnnp_sequential_new(MODEL_LIST(
611
1
    ccv_cnnp_dense(1, 1, 0, 1, 0),
612
1
    ccv_cnnp_reshape(0, DIM_ALLOC(1), DIM_ALLOC(), DIM_ALLOC(), 0),
613
1
  ), 1, 0);
614
1
  ccv_nnc_tensor_variable_t a = ccv_nnc_tensor_variable_new(graph, CPU_TENSOR_NHWC(32F, 1));
615
1
  ccv_nnc_dynamic_graph_exec(graph, CMD_SET_FORWARD(0.2485), ccv_nnc_no_hint, 0, TENSOR_VARIABLE_LIST(), TENSOR_VARIABLE_LIST(a), 0, 0);
616
1
  const ccv_nnc_tensor_param_t input = CPU_TENSOR_NHWC(32F, 1);
617
1
  ccv_cnnp_model_compile(linear, &input, 1, CMD_NOOP(), CMD_NOOP());
618
1
  ccv_cnnp_model_set_parameter(linear, ccv_cnnp_model_parameters(linear, CCV_CNNP_PARAMETER_SELECT_WEIGHT, 0), ccv_nnc_tensor_from_variable(graph, a));
619
1
  ccv_nnc_tensor_variable_t a_grad = ccv_nnc_tensor_variable_new(graph);
620
1
  ccv_nnc_tensor_variable_t saved_aux = ccv_nnc_tensor_variable_new(graph, CPU_TENSOR_NHWC(32F, 1));
621
1
  int i;
622
1.00k
  for (i = 0; i < 1000; 
i++1.00k
)
623
1.00k
  {
624
1.00k
    ccv_nnc_tensor_variable_t x = ccv_nnc_tensor_variable_new(graph, CPU_TENSOR_NHWC(32F, 1));
625
1.00k
    ccv_nnc_tensor_from_variable(graph, x)->data.f32[0] = i;
626
1.00k
    ccv_nnc_tensor_variable_t t = ccv_nnc_tensor_variable_new(graph, CPU_TENSOR_NHWC(32F, 1));
627
1.00k
    ccv_nnc_tensor_from_variable(graph, t)->data.f32[0] = -i;
628
1.00k
    ccv_nnc_tensor_variable_t y = ccv_nnc_tensor_variable_new(graph);
629
1.00k
    ccv_nnc_dynamic_graph_exec(graph, CMD_MUL_FORWARD(1), ccv_nnc_no_hint, 0, TENSOR_VARIABLE_LIST(x, a), TENSOR_VARIABLE_LIST(y), 0, 0);
630
1.00k
    ccv_nnc_tensor_variable_t y_alias = ccv_nnc_tensor_variable_alias_new(graph, y, DIM_ALLOC(), DIM_ALLOC(1), CPU_TENSOR_NHWC(32F, 1));
631
1.00k
    ccv_nnc_tensor_variable_t z = ccv_nnc_tensor_variable_new(graph);
632
1.00k
    ccv_nnc_dynamic_graph_evaluate(graph, linear, 0, TENSOR_VARIABLE_LIST(y_alias), TENSOR_VARIABLE_LIST(z), 0, 0);
633
1.00k
    ccv_nnc_tensor_variable_t n = ccv_nnc_tensor_variable_new(graph);
634
1.00k
    ccv_nnc_dynamic_graph_exec(graph, CMD_ADD_FORWARD(1, -1), ccv_nnc_no_hint, 0, TENSOR_VARIABLE_LIST(z, t), TENSOR_VARIABLE_LIST(n), 0, 0);
635
1.00k
    ccv_nnc_tensor_variable_t f = ccv_nnc_tensor_variable_new(graph);
636
1.00k
    ccv_nnc_dynamic_graph_exec(graph, CMD_MUL_FORWARD(1), ccv_nnc_no_hint, 0, TENSOR_VARIABLE_LIST(n, n), TENSOR_VARIABLE_LIST(f), 0, 0);
637
1.00k
    ccv_nnc_dynamic_graph_backward(graph, TENSOR_VARIABLE_LIST(f), 0, TENSOR_VARIABLE_LIST(a), TENSOR_VARIABLE_LIST(a_grad), 0);
638
1.00k
    ccv_nnc_tensor_variable_free(graph, n);
639
1.00k
    ccv_nnc_tensor_variable_free(graph, z);
640
1.00k
    ccv_nnc_tensor_variable_free(graph, y);
641
1.00k
    ccv_nnc_tensor_variable_free(graph, y_alias);
642
1.00k
    ccv_nnc_tensor_variable_free(graph, x);
643
1.00k
    ccv_nnc_tensor_variable_free(graph, t);
644
1.00k
    ccv_nnc_tensor_variable_free(graph, f);
645
1.00k
    if (((i + 1) % 5) == 0)
646
200
    {
647
200
      REQUIRE_EQ(ccv_nnc_dynamic_graph_bookkeeping_count(graph, CCV_NNC_SYMBOL_GRAPH_EXEC), 0, "everything should be freed");
648
200
      DYNAMIC_GRAPH_GEN(graph, CCV_NNC_LONG_DOT_GRAPH);
649
200
      float lr = 0.001;
650
200
      if (i >= 200)
651
160
        lr = 0.0001;
652
40
      else if (i >= 600)
653
0
        lr = 0.00001;
654
200
      ccv_cnnp_model_set_minimizer(linear, CMD_SGD_FORWARD(0, lr, 0.001, 0, 0, 0), 0, 0, 0);
655
200
      ccv_nnc_dynamic_graph_apply_gradients(graph, CMD_SGD_FORWARD(0, lr, 0.001, 0, 0, 0), TENSOR_VARIABLE_LIST(a_grad), TENSOR_VARIABLE_LIST(a), &saved_aux, 0, 0);
656
200
    }
657
1.00k
  }
658
1
  ccv_nnc_tensor_variable_t x = ccv_nnc_tensor_variable_new(graph, CPU_TENSOR_NHWC(32F, 1));
659
1
  ccv_nnc_tensor_from_variable(graph, x)->data.f32[0] = 5;
660
1
  ccv_nnc_tensor_variable_t y = ccv_nnc_tensor_variable_new(graph);
661
1
  ccv_nnc_dynamic_graph_exec(graph, CMD_MUL_FORWARD(1), ccv_nnc_no_hint, 0, TENSOR_VARIABLE_LIST(x, a), TENSOR_VARIABLE_LIST(y), 0, 0);
662
1
  ccv_nnc_tensor_variable_t z = ccv_nnc_tensor_variable_new(graph);
663
1
  ccv_nnc_dynamic_graph_evaluate(graph, linear, 0, TENSOR_VARIABLE_LIST(y), TENSOR_VARIABLE_LIST(z), 0, 0);
664
1
  REQUIRE_EQ_WITH_TOLERANCE(ccv_nnc_tensor_from_variable(graph, z)->data.f32[0], -5, 1e-2, "linear model should be trained to generate the same value as z");
665
1
  ccv_nnc_dynamic_graph_free(graph);
666
1
  ccv_cnnp_model_free(linear);
667
1
}
668
669
TEST_CASE("dynamic graph to use cnnp model for permute and reshape")
670
1
{
671
1
  ccv_cnnp_model_t* const sequential = ccv_cnnp_sequential_new(MODEL_LIST(
672
1
    ccv_cnnp_reshape(0, DIM_ALLOC(4, 3, 2), DIM_ALLOC(), DIM_ALLOC(), 0),
673
1
    ccv_cnnp_permute(DIM_ALLOC(2, 0, 1), 0),
674
1
    ccv_cnnp_reshape(0, DIM_ALLOC(2 * 4, 3), DIM_ALLOC(), DIM_ALLOC(), 0),
675
1
  ), 1, 0);
676
1
  ccv_nnc_dynamic_graph_t* const graph = ccv_nnc_dynamic_graph_new();
677
1
  ccv_nnc_tensor_variable_t x = ccv_nnc_tensor_variable_new(graph, CPU_TENSOR_NHWC(32F, 4, 3, 2));
678
1
  int i;
679
25
  for (i = 0; i < 4 * 3 * 2; 
i++24
)
680
24
    ccv_nnc_tensor_from_variable(graph, x)->data.f32[i] = i;
681
1
  ccv_nnc_tensor_variable_t y = ccv_nnc_tensor_variable_new(graph);
682
1
  ccv_nnc_dynamic_graph_evaluate(graph, sequential, 0, TENSOR_VARIABLE_LIST(x), TENSOR_VARIABLE_LIST(y), 0, 0);
683
  // 0, 1,
684
  // 2, 3,
685
  // 4, 5
686
  //
687
  // 6, 7,
688
  // 8, 9,
689
  // 10, 11
690
  //
691
  // 12, 13,
692
  // 14, 15,
693
  // 16, 17
694
  //
695
  // 18, 19,
696
  // 20, 21,
697
  // 22, 23
698
  //
699
  // After permute:
700
  // 0, 2, 4,
701
  // 6, 8, 10,
702
  // 12, 14, 16,
703
  // 18, 20, 22
704
  //
705
  // 1, 3, 5,
706
  // 7, 9, 11,
707
  // 13, 15, 17,
708
  // 19, 21, 23
709
1
  float btp[] = {
710
1
    0, 2, 4,
711
1
    6, 8, 10,
712
1
    12, 14, 16,
713
1
    18, 20, 22,
714
1
    1, 3, 5,
715
1
    7, 9, 11,
716
1
    13, 15, 17,
717
1
    19, 21, 23
718
1
  };
719
1
  ccv_nnc_tensor_t bt = ccv_nnc_tensor(btp, CPU_TENSOR_NHWC(32F, 2 * 4, 3), 0);
720
1
  REQUIRE_TENSOR_EQ(ccv_nnc_tensor_from_variable(graph, y), &bt, "should materialize permute before reshape");
721
1
  ccv_nnc_dynamic_graph_free(graph);
722
1
  ccv_cnnp_model_free(sequential);
723
1
}
724
725
TEST_CASE("dynamic graph to compute f(x) = x * log(x) + 1.2 * x, f'(x) and sum on x = 19, 10")
726
1
{
727
1
  ccv_nnc_dynamic_graph_t* const graph = ccv_nnc_dynamic_graph_new();
728
1
  ccv_nnc_tensor_variable_t x = ccv_nnc_tensor_variable_new(graph, CPU_TENSOR_NHWC(32F, 1));
729
1
  ccv_nnc_tensor_from_variable(graph, x)->data.f32[0] = 19;
730
1
  ccv_nnc_tensor_variable_t f = ccv_nnc_tensor_variable_new(graph);
731
1
  ccv_nnc_dynamic_graph_exec(graph, CMD_EWLOG_FORWARD(), ccv_nnc_no_hint, 0, TENSOR_VARIABLE_LIST(x), TENSOR_VARIABLE_LIST(f), 0, 0);
732
1
  ccv_nnc_dynamic_graph_exec(graph, CMD_EWPROD_FORWARD(), ccv_nnc_no_hint, 0, TENSOR_VARIABLE_LIST(x, f), TENSOR_VARIABLE_LIST(f), 0, 0);
733
1
  ccv_nnc_tensor_variable_t y = ccv_nnc_tensor_variable_new(graph, CPU_TENSOR_NHWC(32F, 1));
734
1
  ccv_nnc_tensor_from_variable(graph, y)->data.f32[0] = 1.2;
735
1
  ccv_nnc_tensor_variable_t z = ccv_nnc_tensor_variable_new(graph);
736
1
  ccv_nnc_dynamic_graph_exec(graph, CMD_EWPROD_FORWARD(), ccv_nnc_no_hint, 0, TENSOR_VARIABLE_LIST(x, y), TENSOR_VARIABLE_LIST(z), 0, 0);
737
1
  ccv_nnc_dynamic_graph_exec(graph, CMD_EWSUM_FORWARD(), ccv_nnc_no_hint, 0, TENSOR_VARIABLE_LIST(f, z), TENSOR_VARIABLE_LIST(f), 0, 0);
738
1
  REQUIRE_EQ_WITH_TOLERANCE(ccv_nnc_tensor_from_variable(graph, f)->data.f32[0], 19 * logf(19) + 1.2 * 19, 1e-5, "f(x) = 1.2 * 19 + 19 * log(19)");
739
  // Do gradient computation multiple times.
740
1
  ccv_nnc_tensor_variable_t dx = ccv_nnc_tensor_variable_new(graph);
741
1
  ccv_nnc_dynamic_graph_backward(graph, TENSOR_VARIABLE_LIST(f), 0, TENSOR_VARIABLE_LIST(x), TENSOR_VARIABLE_LIST(dx), 0);
742
1
  REQUIRE_EQ_WITH_TOLERANCE(ccv_nnc_tensor_from_variable(graph, dx)->data.f32[0], logf(19) + 1 + 1.2, 1e-5, "f'(x) = 1.2 + log(19) + 19 * 1 / 19");
743
1
  ccv_nnc_tensor_variable_free(graph, x);
744
1
  x = ccv_nnc_tensor_variable_new(graph, CPU_TENSOR_NHWC(32F, 1));
745
1
  ccv_nnc_tensor_from_variable(graph, x)->data.f32[0] = 10;
746
1
  ccv_nnc_dynamic_graph_exec(graph, CMD_EWLOG_FORWARD(), ccv_nnc_no_hint, 0, TENSOR_VARIABLE_LIST(x), TENSOR_VARIABLE_LIST(f), 0, 0);
747
1
  ccv_nnc_dynamic_graph_exec(graph, CMD_EWPROD_FORWARD(), ccv_nnc_no_hint, 0, TENSOR_VARIABLE_LIST(x, f), TENSOR_VARIABLE_LIST(f), 0, 0);
748
1
  ccv_nnc_tensor_from_variable(graph, y)->data.f32[0] = 1.2;
749
1
  ccv_nnc_dynamic_graph_exec(graph, CMD_EWPROD_FORWARD(), ccv_nnc_no_hint, 0, TENSOR_VARIABLE_LIST(x, y), TENSOR_VARIABLE_LIST(z), 0, 0);
750
1
  ccv_nnc_dynamic_graph_exec(graph, CMD_EWSUM_FORWARD(), ccv_nnc_no_hint, 0, TENSOR_VARIABLE_LIST(f, z), TENSOR_VARIABLE_LIST(f), 0, 0);
751
1
  REQUIRE_EQ_WITH_TOLERANCE(ccv_nnc_tensor_from_variable(graph, f)->data.f32[0], 10 * logf(10) + 1.2 * 10, 1e-5, "f(x) = 1.2 * 10 + 10 * log(10)");
752
1
  ccv_nnc_dynamic_graph_backward(graph, TENSOR_VARIABLE_LIST(f), 0, TENSOR_VARIABLE_LIST(x), TENSOR_VARIABLE_LIST(dx), 0);
753
1
  REQUIRE_EQ_WITH_TOLERANCE(ccv_nnc_tensor_from_variable(graph, dx)->data.f32[0], logf(19) + 1 + 1.2 + logf(10) + 1 + 1.2, 1e-5, "f'(x) = 1.2 + log(19) + 19 * 1 / 19 + 1.2 + log(19) + 19 * 1 / 19");
754
1
  DYNAMIC_GRAPH_GEN(graph, CCV_NNC_LONG_DOT_GRAPH);
755
1
  ccv_nnc_dynamic_graph_free(graph);
756
1
}
757
758
TEST_CASE("dynamic graph to compute f(x) = x * log(x) + y' where y = 1.2 * x and y' is an alias to y")
759
1
{
760
1
  ccv_nnc_dynamic_graph_t* const graph = ccv_nnc_dynamic_graph_new();
761
1
  ccv_nnc_tensor_variable_t x = ccv_nnc_tensor_variable_new(graph, CPU_TENSOR_NHWC(32F, 1));
762
1
  ccv_nnc_tensor_from_variable(graph, x)->data.f32[0] = 19;
763
1
  ccv_nnc_tensor_variable_t f = ccv_nnc_tensor_variable_new(graph);
764
1
  ccv_nnc_dynamic_graph_exec(graph, CMD_EWLOG_FORWARD(), ccv_nnc_no_hint, 0, TENSOR_VARIABLE_LIST(x), TENSOR_VARIABLE_LIST(f), 0, 0);
765
1
  ccv_nnc_dynamic_graph_exec(graph, CMD_EWPROD_FORWARD(), ccv_nnc_no_hint, 0, TENSOR_VARIABLE_LIST(x, f), TENSOR_VARIABLE_LIST(f), 0, 0);
766
1
  ccv_nnc_tensor_variable_t z = ccv_nnc_tensor_variable_new(graph, CPU_TENSOR_NHWC(32F, 1));
767
1
  ccv_nnc_tensor_from_variable(graph, z)->data.f32[0] = 1.2;
768
1
  ccv_nnc_tensor_variable_t y = ccv_nnc_tensor_variable_new(graph, CPU_TENSOR_NHWC(32F, 1));
769
1
  ccv_nnc_tensor_variable_t y_hat = ccv_nnc_tensor_variable_alias_new(graph, y, ccv_nnc_no_ofs, DIM_ALLOC(), CPU_TENSOR_NHWC(32F, 1));
770
1
  ccv_nnc_dynamic_graph_exec(graph, CMD_EWPROD_FORWARD(), ccv_nnc_no_hint, 0, TENSOR_VARIABLE_LIST(x, z), TENSOR_VARIABLE_LIST(y), 0, 0);
771
1
  ccv_nnc_dynamic_graph_exec(graph, CMD_EWSUM_FORWARD(), ccv_nnc_no_hint, 0, TENSOR_VARIABLE_LIST(f, y_hat), TENSOR_VARIABLE_LIST(f), 0, 0);
772
1
  REQUIRE_EQ_WITH_TOLERANCE(ccv_nnc_tensor_from_variable(graph, f)->data.f32[0], 19 * logf(19) + 1.2 * 19, 1e-5, "f(x) = 1.2 * 19 + 19 * log(19)");
773
1
  ccv_nnc_tensor_variable_free(graph, y_hat);
774
1
  ccv_nnc_tensor_variable_free(graph, f);
775
1
  ccv_nnc_tensor_variable_free(graph, x);
776
1
  ccv_nnc_tensor_variable_free(graph, z);
777
1
  ccv_nnc_tensor_variable_free(graph, y);
778
1
  REQUIRE_EQ(ccv_nnc_dynamic_graph_bookkeeping_count(graph, CCV_NNC_SYMBOL_TENSOR), 0, "should have no tensor left");
779
1
  ccv_nnc_dynamic_graph_free(graph);
780
1
}
781
782
TEST_CASE("dynamic graph to compute f(x) = x * log(x) + y' where y'' = 1.2 * x and both y'' and y' are aliases to y")
783
1
{
784
1
  ccv_nnc_dynamic_graph_t* const graph = ccv_nnc_dynamic_graph_new();
785
1
  ccv_nnc_tensor_variable_t x = ccv_nnc_tensor_variable_new(graph, CPU_TENSOR_NHWC(32F, 1));
786
1
  ccv_nnc_tensor_from_variable(graph, x)->data.f32[0] = 19;
787
1
  ccv_nnc_tensor_variable_t f = ccv_nnc_tensor_variable_new(graph);
788
1
  ccv_nnc_dynamic_graph_exec(graph, CMD_EWLOG_FORWARD(), ccv_nnc_no_hint, 0, TENSOR_VARIABLE_LIST(x), TENSOR_VARIABLE_LIST(f), 0, 0);
789
1
  ccv_nnc_dynamic_graph_exec(graph, CMD_EWPROD_FORWARD(), ccv_nnc_no_hint, 0, TENSOR_VARIABLE_LIST(x, f), TENSOR_VARIABLE_LIST(f), 0, 0);
790
1
  ccv_nnc_tensor_variable_t z = ccv_nnc_tensor_variable_new(graph, CPU_TENSOR_NHWC(32F, 1));
791
1
  ccv_nnc_tensor_from_variable(graph, z)->data.f32[0] = 1.2;
792
1
  ccv_nnc_tensor_variable_t y = ccv_nnc_tensor_variable_new(graph, CPU_TENSOR_NHWC(32F, 1));
793
1
  ccv_nnc_tensor_variable_t y_hat = ccv_nnc_tensor_variable_alias_new(graph, y, ccv_nnc_no_ofs, DIM_ALLOC(), CPU_TENSOR_NHWC(32F, 1));
794
1
  ccv_nnc_tensor_variable_t y_double_hat = ccv_nnc_tensor_variable_alias_new(graph, y, ccv_nnc_no_ofs, DIM_ALLOC(), CPU_TENSOR_NHWC(32F, 1));
795
1
  ccv_nnc_dynamic_graph_exec(graph, CMD_EWPROD_FORWARD(), ccv_nnc_no_hint, 0, TENSOR_VARIABLE_LIST(x, z), TENSOR_VARIABLE_LIST(y_double_hat), 0, 0);
796
1
  ccv_nnc_dynamic_graph_exec(graph, CMD_EWSUM_FORWARD(), ccv_nnc_no_hint, 0, TENSOR_VARIABLE_LIST(f, y_hat), TENSOR_VARIABLE_LIST(f), 0, 0);
797
1
  REQUIRE_EQ_WITH_TOLERANCE(ccv_nnc_tensor_from_variable(graph, f)->data.f32[0], 19 * logf(19) + 1.2 * 19, 1e-5, "f(x) = 1.2 * 19 + 19 * log(19)");
798
1
  ccv_nnc_tensor_variable_free(graph, y_hat);
799
1
  ccv_nnc_tensor_variable_free(graph, y_double_hat);
800
1
  ccv_nnc_tensor_variable_free(graph, f);
801
1
  ccv_nnc_tensor_variable_free(graph, x);
802
1
  ccv_nnc_tensor_variable_free(graph, z);
803
1
  ccv_nnc_tensor_variable_free(graph, y);
804
1
  REQUIRE_EQ(ccv_nnc_dynamic_graph_bookkeeping_count(graph, CCV_NNC_SYMBOL_TENSOR), 0, "should have no tensor left");
805
1
  ccv_nnc_dynamic_graph_free(graph);
806
1
}
807
808
TEST_CASE("dynamic graph to compute f(x) = x * y, where y[0] = 10 * z, y[1] = 2 * z, z = [2], x = [10], should free all variables")
809
1
{
810
1
  ccv_nnc_dynamic_graph_t* const graph = ccv_nnc_dynamic_graph_new();
811
1
  ccv_nnc_tensor_variable_t x = ccv_nnc_tensor_variable_new(graph, CPU_TENSOR_NHWC(32F, 1));
812
1
  ccv_nnc_tensor_variable_t y = ccv_nnc_tensor_variable_new(graph, CPU_TENSOR_NHWC(32F, 2));
813
1
  ccv_nnc_tensor_variable_t z = ccv_nnc_tensor_variable_new(graph, CPU_TENSOR_NHWC(32F, 1));
814
1
  ccv_nnc_dynamic_graph_exec(graph, CMD_SET_FORWARD(2), ccv_nnc_no_hint, 0, TENSOR_VARIABLE_LIST(), TENSOR_VARIABLE_LIST(z), 0, 0);
815
1
  ccv_nnc_dynamic_graph_exec(graph, CMD_SET_FORWARD(10), ccv_nnc_no_hint, 0, TENSOR_VARIABLE_LIST(), TENSOR_VARIABLE_LIST(x), 0, 0);
816
1
  ccv_nnc_tensor_variable_t y_1 = ccv_nnc_tensor_variable_alias_new(graph, y, ccv_nnc_no_ofs, DIM_ALLOC(1), CPU_TENSOR_NHWC(32F, 1));
817
1
  ccv_nnc_tensor_variable_t y_2 = ccv_nnc_tensor_variable_alias_new(graph, y, DIM_ALLOC(1), DIM_ALLOC(1), CPU_TENSOR_NHWC(32F, 1));
818
1
  ccv_nnc_dynamic_graph_exec(graph, CMD_SCALAR_MUL_FORWARD(10), ccv_nnc_no_hint, 0, TENSOR_VARIABLE_LIST(z), TENSOR_VARIABLE_LIST(y_1), 0, 0);
819
1
  ccv_nnc_dynamic_graph_exec(graph, CMD_SCALAR_MUL_FORWARD(2), ccv_nnc_no_hint, 0, TENSOR_VARIABLE_LIST(z), TENSOR_VARIABLE_LIST(y_2), 0, 0);
820
1
  ccv_nnc_tensor_variable_t f = ccv_nnc_tensor_variable_new(graph);
821
1
  ccv_nnc_dynamic_graph_exec(graph, CMD_MUL_FORWARD(1), ccv_nnc_no_hint, 0, TENSOR_VARIABLE_LIST(x, y), TENSOR_VARIABLE_LIST(f), 0, 0);
822
1
  float gt[] = {10 * 2 * 10, 2 * 2 * 10};
823
1
  REQUIRE_ARRAY_EQ_WITH_TOLERANCE(float, ccv_nnc_tensor_from_variable(graph, f)->data.f32, gt, 2, 1e-5, "should be equal");
824
1
  ccv_nnc_tensor_variable_free(graph, z);
825
1
  ccv_nnc_tensor_variable_free(graph, y_1);
826
1
  ccv_nnc_tensor_variable_free(graph, y_2);
827
1
  ccv_nnc_tensor_variable_free(graph, x);
828
1
  ccv_nnc_tensor_variable_free(graph, y);
829
1
  ccv_nnc_tensor_variable_free(graph, f);
830
1
  REQUIRE_EQ(ccv_nnc_dynamic_graph_bookkeeping_count(graph, CCV_NNC_SYMBOL_GRAPH_EXEC), 0, "everything should be freed");
831
1
  REQUIRE_EQ(ccv_nnc_dynamic_graph_bookkeeping_count(graph, CCV_NNC_SYMBOL_TENSOR), 0, "everything should be freed");
832
1
  ccv_nnc_dynamic_graph_free(graph);
833
1
}
834
835
TEST_CASE("dynamic graph to compute f(x) = x * y, where y[0] = 10 * z, y[1] = 2 * z, z = [2], x = [10], freed y[0], y[1] before use")
836
1
{
837
1
  ccv_nnc_dynamic_graph_t* const graph = ccv_nnc_dynamic_graph_new();
838
1
  ccv_nnc_tensor_variable_t x = ccv_nnc_tensor_variable_new(graph, CPU_TENSOR_NHWC(32F, 1));
839
1
  ccv_nnc_tensor_variable_t y = ccv_nnc_tensor_variable_new(graph, CPU_TENSOR_NHWC(32F, 2));
840
1
  ccv_nnc_tensor_variable_t z = ccv_nnc_tensor_variable_new(graph, CPU_TENSOR_NHWC(32F, 1));
841
1
  ccv_nnc_dynamic_graph_exec(graph, CMD_SET_FORWARD(2), ccv_nnc_no_hint, 0, TENSOR_VARIABLE_LIST(), TENSOR_VARIABLE_LIST(z), 0, 0);
842
1
  ccv_nnc_dynamic_graph_exec(graph, CMD_SET_FORWARD(10), ccv_nnc_no_hint, 0, TENSOR_VARIABLE_LIST(), TENSOR_VARIABLE_LIST(x), 0, 0);
843
1
  ccv_nnc_tensor_variable_t y_1 = ccv_nnc_tensor_variable_alias_new(graph, y, ccv_nnc_no_ofs, DIM_ALLOC(1), CPU_TENSOR_NHWC(32F, 1));
844
1
  ccv_nnc_tensor_variable_t y_2 = ccv_nnc_tensor_variable_alias_new(graph, y, DIM_ALLOC(1), DIM_ALLOC(1), CPU_TENSOR_NHWC(32F, 1));
845
1
  ccv_nnc_dynamic_graph_exec(graph, CMD_SCALAR_MUL_FORWARD(10), ccv_nnc_no_hint, 0, TENSOR_VARIABLE_LIST(z), TENSOR_VARIABLE_LIST(y_1), 0, 0);
846
1
  ccv_nnc_dynamic_graph_exec(graph, CMD_SCALAR_MUL_FORWARD(2), ccv_nnc_no_hint, 0, TENSOR_VARIABLE_LIST(z), TENSOR_VARIABLE_LIST(y_2), 0, 0);
847
1
  ccv_nnc_tensor_variable_free(graph, y_1);
848
1
  ccv_nnc_tensor_variable_free(graph, y_2);
849
1
  ccv_nnc_tensor_variable_t f = ccv_nnc_tensor_variable_new(graph);
850
1
  ccv_nnc_dynamic_graph_exec(graph, CMD_MUL_FORWARD(1), ccv_nnc_no_hint, 0, TENSOR_VARIABLE_LIST(x, y), TENSOR_VARIABLE_LIST(f), 0, 0);
851
1
  REQUIRE_EQ(ccv_nnc_dynamic_graph_bookkeeping_count(graph, CCV_NNC_SYMBOL_GRAPH_EXEC), 3, "should keep 3 ops");
852
1
  float gt[] = {10 * 2 * 10, 2 * 2 * 10};
853
1
  REQUIRE_ARRAY_EQ_WITH_TOLERANCE(float, ccv_nnc_tensor_from_variable(graph, f)->data.f32, gt, 2, 1e-5, "should be equal");
854
1
  ccv_nnc_tensor_variable_free(graph, z);
855
1
  ccv_nnc_tensor_variable_free(graph, x);
856
1
  ccv_nnc_tensor_variable_free(graph, y);
857
1
  ccv_nnc_tensor_variable_free(graph, f);
858
1
  REQUIRE_EQ(ccv_nnc_dynamic_graph_bookkeeping_count(graph, CCV_NNC_SYMBOL_GRAPH_EXEC), 0, "everything should be freed");
859
1
  REQUIRE_EQ(ccv_nnc_dynamic_graph_bookkeeping_count(graph, CCV_NNC_SYMBOL_TENSOR), 0, "everything should be freed");
860
1
  ccv_nnc_dynamic_graph_free(graph);
861
1
}
862
863
TEST_CASE("dynamic graph to compute f(x) = x * y', where y[0, 0] = 2, y[0, 1] = 4, x = [10], y' = y^T, gradient against y")
864
1
{
865
1
  ccv_nnc_dynamic_graph_t* const graph = ccv_nnc_dynamic_graph_new();
866
1
  ccv_nnc_tensor_variable_t x = ccv_nnc_tensor_variable_new(graph, CPU_TENSOR_NHWC(32F, 1));
867
1
  ccv_nnc_tensor_variable_t y = ccv_nnc_tensor_variable_new(graph, CPU_TENSOR_NHWC(32F, 2));
868
1
  ccv_nnc_tensor_from_variable(graph, y)->data.f32[0] = 2;
869
1
  ccv_nnc_tensor_from_variable(graph, y)->data.f32[0] = 4;
870
1
  ccv_nnc_dynamic_graph_exec(graph, CMD_SET_FORWARD(10), ccv_nnc_no_hint, 0, TENSOR_VARIABLE_LIST(), TENSOR_VARIABLE_LIST(x), 0, 0);
871
1
  ccv_nnc_tensor_variable_t y_t = ccv_nnc_tensor_variable_alias_new(graph, y, ccv_nnc_no_ofs, DIM_ALLOC(), CPU_TENSOR_NHWC(32F, 2, 1));
872
1
  ccv_nnc_tensor_variable_t f = ccv_nnc_tensor_variable_new(graph);
873
1
  ccv_nnc_dynamic_graph_exec(graph, CMD_MUL_FORWARD(1), ccv_nnc_no_hint, 0, TENSOR_VARIABLE_LIST(x, y_t), TENSOR_VARIABLE_LIST(f), 0, 0);
874
1
  ccv_nnc_tensor_variable_t dy_t = ccv_nnc_tensor_variable_new(graph);
875
1
  ccv_nnc_tensor_variable_free(graph, x); // Cannot free y_t, it has to be freed in lock step with y.
876
1
  ccv_nnc_tensor_variable_free(graph, y);
877
1
  ccv_nnc_dynamic_graph_backward(graph, TENSOR_VARIABLE_LIST(f), 0, TENSOR_VARIABLE_LIST(y_t), TENSOR_VARIABLE_LIST(dy_t), 0);
878
1
  float gt[] = {10, 10};
879
1
  REQUIRE_ARRAY_EQ_WITH_TOLERANCE(float, ccv_nnc_tensor_from_variable(graph, dy_t)->data.f32, gt, 2, 1e-5, "should be equal");
880
1
  ccv_nnc_tensor_variable_free(graph, y_t);
881
1
  ccv_nnc_tensor_variable_free(graph, f);
882
1
  REQUIRE_EQ(ccv_nnc_dynamic_graph_bookkeeping_count(graph, CCV_NNC_SYMBOL_GRAPH_EXEC), 0, "everything should be freed");
883
1
  REQUIRE_EQ(ccv_nnc_dynamic_graph_bookkeeping_count(graph, CCV_NNC_SYMBOL_TENSOR), 0, "everything should be freed");
884
1
  ccv_nnc_dynamic_graph_free(graph);
885
1
}
886
887
TEST_CASE("long chain to autograd, simulate random free of unused tensors due to garbage collection")
888
1
{
889
1
  ccv_nnc_dynamic_graph_t* const graph = ccv_nnc_dynamic_graph_new();
890
1
  ccv_nnc_tensor_variable_t x[10];
891
1
  ccv_nnc_tensor_variable_t f[10];
892
1
  ccv_nnc_tensor_variable_t y = ccv_nnc_tensor_variable_new(graph, CPU_TENSOR_NHWC(32F, 1));
893
1
  ccv_nnc_tensor_variable_t dy[3];
894
1
  ccv_nnc_tensor_from_variable(graph, y)->data.f32[0] = 1.3;
895
1
  int i, j;
896
11
  for (i = 0; i < 10; 
i++10
)
897
10
  {
898
10
    x[i] = ccv_nnc_tensor_variable_new(graph, CPU_TENSOR_NHWC(32F, 1));
899
10
    ccv_nnc_tensor_from_variable(graph, x[i])->data.f32[0] = i + 1;
900
10
    f[i] = ccv_nnc_tensor_variable_new(graph);
901
10
    ccv_nnc_dynamic_graph_exec(graph, CMD_MUL_FORWARD(1), ccv_nnc_no_hint, 0, TENSOR_VARIABLE_LIST(x[i], y), TENSOR_VARIABLE_LIST(f[i]), 0, 0);
902
10
    if (i >= 5)
903
5
    {
904
5
      if (i - 5 < 3)
905
3
        dy[i - 5] = ccv_nnc_tensor_variable_new(graph);
906
5
      ccv_nnc_dynamic_graph_backward(graph, TENSOR_VARIABLE_LIST(f[i]), 0, TENSOR_VARIABLE_LIST(y), TENSOR_VARIABLE_LIST(dy[ccv_min(2, i - 5)]), 0);
907
5
    }
908
10
    if (i == 7)
909
6
      
for (j = 0; 1
j < 5;
j++5
)
910
5
        ccv_nnc_tensor_variable_free(graph, f[j]);
911
10
  }
912
1
  REQUIRE_EQ_WITH_TOLERANCE(ccv_nnc_tensor_from_variable(graph, f[9])->data.f32[0], 1.3 * 10, 1e-5, "should equal to 1.3 * 10");
913
1
  REQUIRE_EQ_WITH_TOLERANCE(ccv_nnc_tensor_from_variable(graph, dy[2])->data.f32[0], 8 + 9 + 10, 1e-5, "should equal to sum of the last 3");
914
11
  for (i = 0; i < 10; 
i++10
)
915
10
    ccv_nnc_tensor_variable_free(graph, x[i]);
916
1
  ccv_nnc_dynamic_graph_free(graph);
917
1
}
918
919
TEST_CASE("parallel exec with a stream, focus on uma tensors")
920
1
{
921
1
  ccv_nnc_dynamic_graph_t* const graph = ccv_nnc_dynamic_graph_new();
922
1
  ccv_nnc_stream_context_t* const stream = ccv_nnc_stream_context_new(CCV_STREAM_CONTEXT_CPU);
923
1
  ccv_nnc_tensor_variable_t x[2];
924
1
  x[0] = ccv_nnc_tensor_variable_new(graph, CPU_TENSOR_NHWC(32F, 1));
925
1
  x[1] = ccv_nnc_tensor_variable_new(graph, CPU_TENSOR_NHWC(32F, 1));
926
1
  ccv_nnc_tensor_from_variable(graph, x[0])->data.f32[0] = 10;
927
1
  ccv_nnc_tensor_from_variable(graph, x[1])->data.f32[0] = 9;
928
1
  ccv_nnc_tensor_variable_t y[2];
929
1
  y[0] = ccv_nnc_tensor_variable_new(graph);
930
1
  y[1] = ccv_nnc_tensor_variable_new(graph);
931
1
  ccv_nnc_dynamic_graph_exec(graph, CMD_SCALAR_MUL_FORWARD(0.4), ccv_nnc_no_hint, 0, TENSOR_VARIABLE_LIST(x[0], x[1]), TENSOR_VARIABLE_LIST(y[0], y[1]), 2, stream);
932
1
  ccv_nnc_stream_context_wait(stream);
933
1
  ccv_nnc_stream_context_free(stream);
934
1
  REQUIRE_EQ_WITH_TOLERANCE(ccv_nnc_tensor_from_variable(graph, y[0])->data.f32[0], 10 * 0.4, 1e-5, "should match the result");
935
1
  REQUIRE_EQ_WITH_TOLERANCE(ccv_nnc_tensor_from_variable(graph, y[1])->data.f32[0], 9 * 0.4, 1e-5, "should match the result");
936
1
  ccv_nnc_dynamic_graph_free(graph);
937
1
}
938
939
TEST_CASE("parallel exec with a stream, focus on potential memory issues")
940
1
{
941
1
  ccv_nnc_dynamic_graph_t* const graph = ccv_nnc_dynamic_graph_new();
942
1
  ccv_nnc_stream_context_t* const stream = ccv_nnc_stream_context_new(CCV_STREAM_CONTEXT_CPU);
943
1
  ccv_nnc_tensor_variable_t x[2];
944
1
  x[0] = ccv_nnc_tensor_variable_new(graph, CPU_NUMA_TENSOR_NHWC(000, 32F, 1));
945
1
  x[1] = ccv_nnc_tensor_variable_new(graph, CPU_NUMA_TENSOR_NHWC(000, 32F, 1));
946
1
  ccv_nnc_tensor_from_variable(graph, x[0])->data.f32[0] = 10;
947
1
  ccv_nnc_tensor_from_variable(graph, x[1])->data.f32[0] = 9;
948
1
  ccv_nnc_tensor_variable_t y[2];
949
1
  y[0] = ccv_nnc_tensor_variable_new(graph);
950
1
  y[1] = ccv_nnc_tensor_variable_new(graph);
951
1
  ccv_nnc_dynamic_graph_exec(graph, CMD_SCALAR_MUL_FORWARD(0.4), ccv_nnc_no_hint, 0, TENSOR_VARIABLE_LIST(x[0], x[1]), TENSOR_VARIABLE_LIST(y[0], y[1]), 2, stream);
952
1
  ccv_nnc_stream_context_wait(stream);
953
1
  ccv_nnc_stream_context_free(stream);
954
1
  REQUIRE_EQ_WITH_TOLERANCE(ccv_nnc_tensor_from_variable(graph, y[0])->data.f32[0], 10 * 0.4, 1e-5, "should match the result");
955
1
  REQUIRE_EQ_WITH_TOLERANCE(ccv_nnc_tensor_from_variable(graph, y[1])->data.f32[0], 9 * 0.4, 1e-5, "should match the result");
956
1
  ccv_nnc_dynamic_graph_free(graph);
957
1
}
958
959
TEST_CASE("query whether a tensor variable depends on another")
960
1
{
961
1
  ccv_nnc_dynamic_graph_t* const graph = ccv_nnc_dynamic_graph_new();
962
1
  ccv_nnc_tensor_variable_t a = ccv_nnc_tensor_variable_new(graph, CPU_TENSOR_NHWC(32F, 1));
963
1
  ccv_nnc_tensor_from_variable(graph, a)->data.f32[0] = 10;
964
1
  ccv_nnc_tensor_variable_t b = ccv_nnc_tensor_variable_new(graph);
965
1
  ccv_nnc_dynamic_graph_exec(graph, CMD_SCALAR_MUL_FORWARD(0.5), ccv_nnc_no_hint, 0, TENSOR_VARIABLE_LIST(a), TENSOR_VARIABLE_LIST(b), 0, 0);
966
1
  ccv_nnc_tensor_variable_t c = ccv_nnc_tensor_variable_new(graph, CPU_TENSOR_NHWC(32F, 1));
967
1
  ccv_nnc_tensor_from_variable(graph, c)->data.f32[0] = 9;
968
1
  ccv_nnc_tensor_variable_t d = ccv_nnc_tensor_variable_new(graph);
969
1
  ccv_nnc_dynamic_graph_exec(graph, CMD_SCALAR_MUL_FORWARD(0.4), ccv_nnc_no_hint, 0, TENSOR_VARIABLE_LIST(c), TENSOR_VARIABLE_LIST(d), 0, 0);
970
1
  ccv_nnc_tensor_variable_t e = ccv_nnc_tensor_variable_new(graph);
971
1
  ccv_nnc_dynamic_graph_exec(graph, CMD_SCALAR_MUL_FORWARD(2), ccv_nnc_no_hint, 0, TENSOR_VARIABLE_LIST(d), TENSOR_VARIABLE_LIST(e), 0, 0);
972
1
  ccv_nnc_tensor_variable_t f = ccv_nnc_tensor_variable_new(graph);
973
1
  ccv_nnc_dynamic_graph_exec(graph, CMD_EWSUM_FORWARD(), ccv_nnc_no_hint, 0, TENSOR_VARIABLE_LIST(a, c), TENSOR_VARIABLE_LIST(f), 0, 0);
974
1
  ccv_nnc_tensor_variable_t g = ccv_nnc_tensor_variable_new(graph);
975
1
  ccv_nnc_dynamic_graph_exec(graph, CMD_EWSUM_FORWARD(), ccv_nnc_no_hint, 0, TENSOR_VARIABLE_LIST(e, b), TENSOR_VARIABLE_LIST(g), 0, 0);
976
1
  REQUIRE_EQ_WITH_TOLERANCE(ccv_nnc_tensor_from_variable(graph, f)->data.f32[0], 10 + 9, 1e-5, "should match the result");
977
1
  REQUIRE_EQ_WITH_TOLERANCE(ccv_nnc_tensor_from_variable(graph, g)->data.f32[0], 10 * 0.5 + 9 * 0.4 * 2, 1e-5, "should match the result");
978
1
  uint64_t bitmask = 1;
979
1
  ccv_nnc_dynamic_graph_has_effect_to_tensor_variables(graph, TENSOR_VARIABLE_LIST(a, b, f), TENSOR_VARIABLE_LIST(g), &bitmask);
980
1
  REQUIRE(bitmask == 3, "a and b should be ancestors to g");
981
1
  bitmask = 8;
982
1
  ccv_nnc_dynamic_graph_has_effect_to_tensor_variables(graph, TENSOR_VARIABLE_LIST(a, e, c, f), TENSOR_VARIABLE_LIST(g), &bitmask);
983
1
  REQUIRE(bitmask == 7, "a, e and c should be ancestors to g");
984
1
  bitmask = 2;
985
1
  ccv_nnc_dynamic_graph_has_effect_to_tensor_variables(graph, TENSOR_VARIABLE_LIST(a, e, c), TENSOR_VARIABLE_LIST(f), &bitmask);
986
1
  REQUIRE(bitmask == 5, "a and c should be ancestors to f");
987
1
  ccv_nnc_dynamic_graph_free(graph);
988
1
}
989
990
TEST_CASE("compute f(x) = 0.5 * log(x), detach log(x) means 0.5 * y should be freed")
991
1
{
992
1
  ccv_nnc_dynamic_graph_t* const graph = ccv_nnc_dynamic_graph_new();
993
1
  ccv_nnc_tensor_variable_t x = ccv_nnc_tensor_variable_new(graph, CPU_TENSOR_NHWC(32F, 1));
994
1
  ccv_nnc_tensor_from_variable(graph, x)->data.f32[0] = 10;
995
1
  ccv_nnc_tensor_variable_t y = ccv_nnc_tensor_variable_new(graph);
996
1
  ccv_nnc_dynamic_graph_exec(graph, CMD_EWLOG_FORWARD(), ccv_nnc_no_hint, 0, TENSOR_VARIABLE_LIST(x), TENSOR_VARIABLE_LIST(y), 0, 0);
997
1
  REQUIRE_EQ_WITH_TOLERANCE(ccv_nnc_tensor_from_variable(graph, y)->data.f32[0], log(10), 1e-5, "should match the result");
998
1
  ccv_nnc_tensor_variable_t z = ccv_nnc_tensor_variable_new(graph);
999
1
  ccv_nnc_dynamic_graph_exec(graph, CMD_SCALAR_MUL_FORWARD(0.5), ccv_nnc_no_hint, 0, TENSOR_VARIABLE_LIST(y), TENSOR_VARIABLE_LIST(z), 0, 0);
1000
1
  REQUIRE_EQ_WITH_TOLERANCE(ccv_nnc_tensor_from_variable(graph, z)->data.f32[0], 0.5 * log(10), 1e-5, "should match the result");
1001
1
  REQUIRE_EQ(ccv_nnc_dynamic_graph_bookkeeping_count(graph, CCV_NNC_SYMBOL_GRAPH_EXEC), 2, "nothing should be freed");
1002
1
  REQUIRE_EQ(ccv_nnc_dynamic_graph_bookkeeping_count(graph, CCV_NNC_SYMBOL_TENSOR), 3, "nothing should be freed");
1003
1
  uint64_t bitmask = 0;
1004
1
  ccv_nnc_dynamic_graph_has_effect_to_tensor_variables(graph, TENSOR_VARIABLE_LIST(x), TENSOR_VARIABLE_LIST(z), &bitmask);
1005
1
  REQUIRE(bitmask == 1, "before detach, it should have effect");
1006
1
  ccv_nnc_tensor_variable_detach(graph, y);
1007
1
  REQUIRE_EQ(ccv_nnc_dynamic_graph_bookkeeping_count(graph, CCV_NNC_SYMBOL_GRAPH_EXEC), 1, "0.5 * y should be freed");
1008
1
  REQUIRE_EQ(ccv_nnc_dynamic_graph_bookkeeping_count(graph, CCV_NNC_SYMBOL_TENSOR), 3, "nothing should be freed");
1009
1
  bitmask = 0;
1010
1
  ccv_nnc_dynamic_graph_has_effect_to_tensor_variables(graph, TENSOR_VARIABLE_LIST(x), TENSOR_VARIABLE_LIST(z), &bitmask);
1011
1
  REQUIRE(bitmask == 0, "after detach, it should have no effect");
1012
1
  ccv_nnc_tensor_variable_free(graph, x);
1013
1
  REQUIRE_EQ(ccv_nnc_dynamic_graph_bookkeeping_count(graph, CCV_NNC_SYMBOL_GRAPH_EXEC), 0, "everything should be freed");
1014
1
  REQUIRE_EQ(ccv_nnc_dynamic_graph_bookkeeping_count(graph, CCV_NNC_SYMBOL_TENSOR), 1, "x and y should be freed");
1015
1
  ccv_nnc_tensor_variable_free(graph, y);
1016
1
  ccv_nnc_tensor_variable_free(graph, z);
1017
1
  REQUIRE_EQ(ccv_nnc_dynamic_graph_bookkeeping_count(graph, CCV_NNC_SYMBOL_TENSOR), 0, "everything should be freed");
1018
1
  ccv_nnc_dynamic_graph_free(graph);
1019
1
}
1020
1021
TEST_CASE("compute f(x) = 0.5 * log(x), detach log(x) after freeing x means all computations on the graph should be freed")
1022
1
{
1023
1
  ccv_nnc_dynamic_graph_t* const graph = ccv_nnc_dynamic_graph_new();
1024
1
  ccv_nnc_tensor_variable_t x = ccv_nnc_tensor_variable_new(graph, CPU_TENSOR_NHWC(32F, 1));
1025
1
  ccv_nnc_tensor_from_variable(graph, x)->data.f32[0] = 10;
1026
1
  ccv_nnc_tensor_variable_t y = ccv_nnc_tensor_variable_new(graph);
1027
1
  ccv_nnc_dynamic_graph_exec(graph, CMD_EWLOG_FORWARD(), ccv_nnc_no_hint, 0, TENSOR_VARIABLE_LIST(x), TENSOR_VARIABLE_LIST(y), 0, 0);
1028
1
  REQUIRE_EQ_WITH_TOLERANCE(ccv_nnc_tensor_from_variable(graph, y)->data.f32[0], log(10), 1e-5, "should match the result");
1029
1
  ccv_nnc_tensor_variable_t z = ccv_nnc_tensor_variable_new(graph);
1030
1
  ccv_nnc_dynamic_graph_exec(graph, CMD_SCALAR_MUL_FORWARD(0.5), ccv_nnc_no_hint, 0, TENSOR_VARIABLE_LIST(y), TENSOR_VARIABLE_LIST(z), 0, 0);
1031
1
  REQUIRE_EQ_WITH_TOLERANCE(ccv_nnc_tensor_from_variable(graph, z)->data.f32[0], 0.5 * log(10), 1e-5, "should match the result");
1032
1
  REQUIRE_EQ(ccv_nnc_dynamic_graph_bookkeeping_count(graph, CCV_NNC_SYMBOL_GRAPH_EXEC), 2, "nothing should be freed");
1033
1
  REQUIRE_EQ(ccv_nnc_dynamic_graph_bookkeeping_count(graph, CCV_NNC_SYMBOL_TENSOR), 3, "nothing should be freed");
1034
  // free x first before detaching y.
1035
1
  ccv_nnc_tensor_variable_free(graph, x);
1036
1
  REQUIRE_EQ(ccv_nnc_dynamic_graph_bookkeeping_count(graph, CCV_NNC_SYMBOL_GRAPH_EXEC), 1, "log(x) should be freed");
1037
1
  REQUIRE_EQ(ccv_nnc_dynamic_graph_bookkeeping_count(graph, CCV_NNC_SYMBOL_TENSOR), 2, "only x should be freed");
1038
1
  ccv_nnc_tensor_variable_detach(graph, y);
1039
1
  REQUIRE_EQ(ccv_nnc_dynamic_graph_bookkeeping_count(graph, CCV_NNC_SYMBOL_GRAPH_EXEC), 0, "everything should be freed");
1040
1
  REQUIRE_EQ(ccv_nnc_dynamic_graph_bookkeeping_count(graph, CCV_NNC_SYMBOL_TENSOR), 1, "x and y should be freed");
1041
1
  ccv_nnc_tensor_variable_free(graph, y);
1042
1
  ccv_nnc_tensor_variable_free(graph, z);
1043
1
  REQUIRE_EQ(ccv_nnc_dynamic_graph_bookkeeping_count(graph, CCV_NNC_SYMBOL_TENSOR), 0, "everything should be freed");
1044
1
  ccv_nnc_dynamic_graph_free(graph);
1045
1
}
1046
1047
TEST_CASE("compute f(x) = a * log(x), detach log(x) means nothing changed")
1048
1
{
1049
1
  ccv_nnc_dynamic_graph_t* const graph = ccv_nnc_dynamic_graph_new();
1050
1
  ccv_nnc_tensor_variable_t x = ccv_nnc_tensor_variable_new(graph, CPU_TENSOR_NHWC(32F, 1));
1051
1
  ccv_nnc_tensor_from_variable(graph, x)->data.f32[0] = 10;
1052
1
  ccv_nnc_tensor_variable_t a = ccv_nnc_tensor_variable_new(graph, CPU_TENSOR_NHWC(32F, 1));
1053
1
  ccv_nnc_tensor_from_variable(graph, a)->data.f32[0] = 5;
1054
1
  ccv_nnc_tensor_variable_t y = ccv_nnc_tensor_variable_new(graph);
1055
1
  ccv_nnc_dynamic_graph_exec(graph, CMD_EWLOG_FORWARD(), ccv_nnc_no_hint, 0, TENSOR_VARIABLE_LIST(x), TENSOR_VARIABLE_LIST(y), 0, 0);
1056
1
  REQUIRE_EQ_WITH_TOLERANCE(ccv_nnc_tensor_from_variable(graph, y)->data.f32[0], log(10), 1e-5, "should match the result");
1057
1
  ccv_nnc_tensor_variable_t z = ccv_nnc_tensor_variable_new(graph);
1058
1
  ccv_nnc_dynamic_graph_exec(graph, CMD_MUL_FORWARD(1), ccv_nnc_no_hint, 0, TENSOR_VARIABLE_LIST(a, y), TENSOR_VARIABLE_LIST(z), 0, 0);
1059
1
  REQUIRE_EQ_WITH_TOLERANCE(ccv_nnc_tensor_from_variable(graph, z)->data.f32[0], 5 * log(10), 1e-5, "should match the result");
1060
1
  REQUIRE_EQ(ccv_nnc_dynamic_graph_bookkeeping_count(graph, CCV_NNC_SYMBOL_GRAPH_EXEC), 2, "nothing should be freed");
1061
1
  REQUIRE_EQ(ccv_nnc_dynamic_graph_bookkeeping_count(graph, CCV_NNC_SYMBOL_TENSOR), 4, "nothing should be freed");
1062
1
  uint64_t bitmask = 0;
1063
1
  ccv_nnc_dynamic_graph_has_effect_to_tensor_variables(graph, TENSOR_VARIABLE_LIST(x), TENSOR_VARIABLE_LIST(z), &bitmask);
1064
1
  REQUIRE(bitmask == 1, "before detach, it should have effect");
1065
1
  ccv_nnc_tensor_variable_detach(graph, y);
1066
1
  REQUIRE_EQ(ccv_nnc_dynamic_graph_bookkeeping_count(graph, CCV_NNC_SYMBOL_GRAPH_EXEC), 2, "nothing should be freed");
1067
1
  REQUIRE_EQ(ccv_nnc_dynamic_graph_bookkeeping_count(graph, CCV_NNC_SYMBOL_TENSOR), 4, "nothing should be freed");
1068
1
  bitmask = 0;
1069
1
  ccv_nnc_dynamic_graph_has_effect_to_tensor_variables(graph, TENSOR_VARIABLE_LIST(x), TENSOR_VARIABLE_LIST(z), &bitmask);
1070
1
  REQUIRE(bitmask == 0, "after detach, it should have no effect");
1071
1
  ccv_nnc_tensor_variable_free(graph, a);
1072
1
  REQUIRE_EQ(ccv_nnc_dynamic_graph_bookkeeping_count(graph, CCV_NNC_SYMBOL_GRAPH_EXEC), 1, "a * y should be freed");
1073
1
  REQUIRE_EQ(ccv_nnc_dynamic_graph_bookkeeping_count(graph, CCV_NNC_SYMBOL_TENSOR), 3, "a should be freed");
1074
1
  ccv_nnc_tensor_variable_free(graph, x);
1075
1
  REQUIRE_EQ(ccv_nnc_dynamic_graph_bookkeeping_count(graph, CCV_NNC_SYMBOL_GRAPH_EXEC), 0, "everything should be freed");
1076
1
  REQUIRE_EQ(ccv_nnc_dynamic_graph_bookkeeping_count(graph, CCV_NNC_SYMBOL_TENSOR), 1, "x and y should be freed");
1077
1
  ccv_nnc_tensor_variable_free(graph, y);
1078
1
  ccv_nnc_tensor_variable_free(graph, z);
1079
1
  REQUIRE_EQ(ccv_nnc_dynamic_graph_bookkeeping_count(graph, CCV_NNC_SYMBOL_TENSOR), 0, "everything should be freed");
1080
1
  ccv_nnc_dynamic_graph_free(graph);
1081
1
}
1082
1083
TEST_CASE("compute f(x) = exp(x) * log(x), detach log(x) means nothing changed")
1084
1
{
1085
1
  ccv_nnc_dynamic_graph_t* const graph = ccv_nnc_dynamic_graph_new();
1086
1
  ccv_nnc_tensor_variable_t x = ccv_nnc_tensor_variable_new(graph, CPU_TENSOR_NHWC(32F, 1));
1087
1
  ccv_nnc_tensor_from_variable(graph, x)->data.f32[0] = 10;
1088
1
  ccv_nnc_tensor_variable_t a = ccv_nnc_tensor_variable_new(graph);
1089
1
  ccv_nnc_dynamic_graph_exec(graph, CMD_EWEXP_FORWARD(), ccv_nnc_no_hint, 0, TENSOR_VARIABLE_LIST(x), TENSOR_VARIABLE_LIST(a), 0, 0);
1090
1
  REQUIRE_EQ_WITH_TOLERANCE(ccv_nnc_tensor_from_variable(graph, a)->data.f32[0], exp(10), 1e-2, "should match the result");
1091
1
  ccv_nnc_tensor_variable_t y = ccv_nnc_tensor_variable_new(graph);
1092
1
  ccv_nnc_dynamic_graph_exec(graph, CMD_EWLOG_FORWARD(), ccv_nnc_no_hint, 0, TENSOR_VARIABLE_LIST(x), TENSOR_VARIABLE_LIST(y), 0, 0);
1093
1
  REQUIRE_EQ_WITH_TOLERANCE(ccv_nnc_tensor_from_variable(graph, y)->data.f32[0], log(10), 1e-5, "should match the result");
1094
1
  ccv_nnc_tensor_variable_t z = ccv_nnc_tensor_variable_new(graph);
1095
1
  ccv_nnc_dynamic_graph_exec(graph, CMD_MUL_FORWARD(1), ccv_nnc_no_hint, 0, TENSOR_VARIABLE_LIST(a, y), TENSOR_VARIABLE_LIST(z), 0, 0);
1096
1
  REQUIRE_EQ_WITH_TOLERANCE(ccv_nnc_tensor_from_variable(graph, z)->data.f32[0], exp(10) * log(10), 1e-2, "should match the result");
1097
1
  REQUIRE_EQ(ccv_nnc_dynamic_graph_bookkeeping_count(graph, CCV_NNC_SYMBOL_GRAPH_EXEC), 3, "nothing should be freed");
1098
1
  REQUIRE_EQ(ccv_nnc_dynamic_graph_bookkeeping_count(graph, CCV_NNC_SYMBOL_TENSOR), 4, "nothing should be freed");
1099
1
  uint64_t bitmask = 0;
1100
1
  ccv_nnc_dynamic_graph_has_effect_to_tensor_variables(graph, TENSOR_VARIABLE_LIST(x), TENSOR_VARIABLE_LIST(z), &bitmask);
1101
1
  REQUIRE(bitmask == 1, "before detach, it should have effect");
1102
1
  ccv_nnc_tensor_variable_detach(graph, y);
1103
1
  REQUIRE_EQ(ccv_nnc_dynamic_graph_bookkeeping_count(graph, CCV_NNC_SYMBOL_GRAPH_EXEC), 3, "nothing should be freed");
1104
1
  REQUIRE_EQ(ccv_nnc_dynamic_graph_bookkeeping_count(graph, CCV_NNC_SYMBOL_TENSOR), 4, "nothing should be freed");
1105
1
  bitmask = 0;
1106
1
  ccv_nnc_dynamic_graph_has_effect_to_tensor_variables(graph, TENSOR_VARIABLE_LIST(x), TENSOR_VARIABLE_LIST(z), &bitmask);
1107
1
  REQUIRE(bitmask == 1, "after detach, it should have still effect (through exp(x))");
1108
1
  ccv_nnc_tensor_variable_detach(graph, a);
1109
1
  REQUIRE_EQ(ccv_nnc_dynamic_graph_bookkeeping_count(graph, CCV_NNC_SYMBOL_GRAPH_EXEC), 2, "a * y should be freed");
1110
1
  REQUIRE_EQ(ccv_nnc_dynamic_graph_bookkeeping_count(graph, CCV_NNC_SYMBOL_TENSOR), 4, "nothing should be freed");
1111
1
  bitmask = 0;
1112
1
  ccv_nnc_dynamic_graph_has_effect_to_tensor_variables(graph, TENSOR_VARIABLE_LIST(x), TENSOR_VARIABLE_LIST(z), &bitmask);
1113
1
  REQUIRE(bitmask == 0, "after a freed, it should have no effect");
1114
1
  ccv_nnc_tensor_variable_free(graph, a);
1115
1
  REQUIRE_EQ(ccv_nnc_dynamic_graph_bookkeeping_count(graph, CCV_NNC_SYMBOL_TENSOR), 3, "a should be freed");
1116
1
  ccv_nnc_tensor_variable_free(graph, x);
1117
1
  REQUIRE_EQ(ccv_nnc_dynamic_graph_bookkeeping_count(graph, CCV_NNC_SYMBOL_GRAPH_EXEC), 0, "everything should be freed");
1118
1
  REQUIRE_EQ(ccv_nnc_dynamic_graph_bookkeeping_count(graph, CCV_NNC_SYMBOL_TENSOR), 1, "x and y should be freed");
1119
1
  ccv_nnc_tensor_variable_free(graph, y);
1120
1
  ccv_nnc_tensor_variable_free(graph, z);
1121
1
  REQUIRE_EQ(ccv_nnc_dynamic_graph_bookkeeping_count(graph, CCV_NNC_SYMBOL_TENSOR), 0, "everything should be freed");
1122
1
  ccv_nnc_dynamic_graph_free(graph);
1123
1
}
1124
1125
TEST_CASE("compute f(x) = exp(0.5 * log(x)), detach log(x) won't free exp(_)")
1126
1
{
1127
1
  ccv_nnc_dynamic_graph_t* const graph = ccv_nnc_dynamic_graph_new();
1128
1
  ccv_nnc_tensor_variable_t x = ccv_nnc_tensor_variable_new(graph, CPU_TENSOR_NHWC(32F, 1));
1129
1
  ccv_nnc_tensor_from_variable(graph, x)->data.f32[0] = 10;
1130
1
  ccv_nnc_tensor_variable_t y = ccv_nnc_tensor_variable_new(graph);
1131
1
  ccv_nnc_dynamic_graph_exec(graph, CMD_EWLOG_FORWARD(), ccv_nnc_no_hint, 0, TENSOR_VARIABLE_LIST(x), TENSOR_VARIABLE_LIST(y), 0, 0);
1132
1
  REQUIRE_EQ_WITH_TOLERANCE(ccv_nnc_tensor_from_variable(graph, y)->data.f32[0], log(10), 1e-5, "should match the result");
1133
1
  ccv_nnc_tensor_variable_t z = ccv_nnc_tensor_variable_new(graph);
1134
1
  ccv_nnc_dynamic_graph_exec(graph, CMD_SCALAR_MUL_FORWARD(0.5), ccv_nnc_no_hint, 0, TENSOR_VARIABLE_LIST(y), TENSOR_VARIABLE_LIST(z), 0, 0);
1135
1
  REQUIRE_EQ_WITH_TOLERANCE(ccv_nnc_tensor_from_variable(graph, z)->data.f32[0], 0.5 * log(10), 1e-5, "should match the result");
1136
1
  ccv_nnc_tensor_variable_t f = ccv_nnc_tensor_variable_new(graph);
1137
1
  ccv_nnc_dynamic_graph_exec(graph, CMD_EWEXP_FORWARD(), ccv_nnc_no_hint, 0, TENSOR_VARIABLE_LIST(z), TENSOR_VARIABLE_LIST(f), 0, 0);
1138
1
  REQUIRE_EQ_WITH_TOLERANCE(ccv_nnc_tensor_from_variable(graph, f)->data.f32[0], exp(0.5 * log(10)), 1e-5, "should match the result");
1139
1
  uint64_t bitmask = 0;
1140
1
  ccv_nnc_dynamic_graph_has_effect_to_tensor_variables(graph, TENSOR_VARIABLE_LIST(x), TENSOR_VARIABLE_LIST(f), &bitmask);
1141
1
  REQUIRE(bitmask == 1, "before detach, it should have effect");
1142
1
  ccv_nnc_tensor_variable_detach(graph, y);
1143
1
  REQUIRE_EQ(ccv_nnc_dynamic_graph_bookkeeping_count(graph, CCV_NNC_SYMBOL_GRAPH_EXEC), 2, "0.5 * y should be freed");
1144
1
  REQUIRE_EQ(ccv_nnc_dynamic_graph_bookkeeping_count(graph, CCV_NNC_SYMBOL_TENSOR), 4, "nothing should be freed");
1145
1
  bitmask = 0;
1146
1
  ccv_nnc_dynamic_graph_has_effect_to_tensor_variables(graph, TENSOR_VARIABLE_LIST(x), TENSOR_VARIABLE_LIST(f), &bitmask);
1147
1
  REQUIRE(bitmask == 0, "after detach, it should have no effect");
1148
1
  ccv_nnc_tensor_variable_free(graph, f);
1149
1
  REQUIRE_EQ(ccv_nnc_dynamic_graph_bookkeeping_count(graph, CCV_NNC_SYMBOL_GRAPH_EXEC), 1, "exp(_) should be freed");
1150
1
  REQUIRE_EQ(ccv_nnc_dynamic_graph_bookkeeping_count(graph, CCV_NNC_SYMBOL_TENSOR), 3, "f should be freed");
1151
1
  ccv_nnc_tensor_variable_free(graph, z);
1152
1
  REQUIRE_EQ(ccv_nnc_dynamic_graph_bookkeeping_count(graph, CCV_NNC_SYMBOL_GRAPH_EXEC), 1, "nothing should change");
1153
1
  REQUIRE_EQ(ccv_nnc_dynamic_graph_bookkeeping_count(graph, CCV_NNC_SYMBOL_TENSOR), 2, "z should be freed");
1154
1
  ccv_nnc_tensor_variable_free(graph, y);
1155
1
  REQUIRE_EQ(ccv_nnc_dynamic_graph_bookkeeping_count(graph, CCV_NNC_SYMBOL_GRAPH_EXEC), 0, "log(x) should be freed");
1156
1
  REQUIRE_EQ(ccv_nnc_dynamic_graph_bookkeeping_count(graph, CCV_NNC_SYMBOL_TENSOR), 1, "y should be freed");
1157
1
  ccv_nnc_tensor_variable_free(graph, x);
1158
1
  REQUIRE_EQ(ccv_nnc_dynamic_graph_bookkeeping_count(graph, CCV_NNC_SYMBOL_TENSOR), 0, "x should be freed");
1159
1
  ccv_nnc_dynamic_graph_free(graph);
1160
1
}
1161
1162
#include "case_main.h"