Coverage Report

Created: 2021-09-21 22:26

/home/liu/buildslave/linux-x64-runtests/build/test/unit/nnc/dynamic.graph.tests.c
Line
Count
Source (jump to first uncovered line)
1
#include "case.h"
2
#include "ccv_case.h"
3
#include "ccv_nnc_case.h"
4
#include <ccv.h>
5
#include <nnc/ccv_nnc.h>
6
#include <nnc/ccv_nnc_easy.h>
7
#include "3rdparty/dsfmt/dSFMT.h"
8
9
TEST_SETUP()
10
{
11
  ccv_nnc_init();
12
}
13
14
TEST_CASE("dynamic graph to compute log(19)")
15
1
{
16
1
  ccv_nnc_dynamic_graph_t* const graph = ccv_nnc_dynamic_graph_new();
17
1
  ccv_nnc_tensor_variable_t a = ccv_nnc_tensor_variable_new(graph, CPU_TENSOR_NHWC(32F, 1));
18
1
  ccv_nnc_tensor_from_variable(graph, a)->data.f32[0] = 19;
19
1
  ccv_nnc_tensor_variable_t b = ccv_nnc_tensor_variable_new(graph);
20
1
  ccv_nnc_dynamic_graph_exec(graph, CMD_EWLOG_FORWARD(), ccv_nnc_no_hint, 0, TENSOR_VARIABLE_LIST(a), TENSOR_VARIABLE_LIST(b), 0, 0);
21
1
  REQUIRE_EQ_WITH_TOLERANCE(ccv_nnc_tensor_from_variable(graph, b)->data.f32[0], logf(19), 1e-5, "log(19) result should be equal.");
22
1
  DYNAMIC_GRAPH_GEN(graph, CCV_NNC_LONG_DOT_GRAPH);
23
1
  ccv_nnc_dynamic_graph_free(graph);
24
1
}
25
26
TEST_CASE("dynamic graph with alias")
27
1
{
28
1
  ccv_nnc_dynamic_graph_t* const graph = ccv_nnc_dynamic_graph_new();
29
1
  ccv_nnc_tensor_variable_t const a = ccv_nnc_tensor_variable_new(graph, CPU_TENSOR_NHWC(32F, 2));
30
1
  ccv_nnc_tensor_variable_t const a0 = ccv_nnc_tensor_variable_alias_new(graph, a, DIM_ALLOC(1), DIM_ALLOC(), CPU_TENSOR_NHWC(32F, 1));
31
1
  ccv_nnc_tensor_variable_t const a1 = ccv_nnc_tensor_variable_alias_new(graph, a, ccv_nnc_no_ofs, DIM_ALLOC(), CPU_TENSOR_NHWC(32F, 1));
32
1
  ccv_nnc_tensor_t* const b0 = ccv_nnc_tensor_new(0, CPU_TENSOR_NHWC(32F, 2), 0);
33
1
  ccv_nnc_tensor_variable_set(graph, a, b0);
34
1
  b0->data.f32[0] = 10;
35
1
  b0->data.f32[1] = 11;
36
1
  ccv_nnc_tensor_t* const b1 = ccv_nnc_tensor_new(0, CPU_TENSOR_NHWC(32F, 2), 0);
37
1
  b1->data.f32[0] = 20;
38
1
  b1->data.f32[1] = 21;
39
1
  REQUIRE_EQ(ccv_nnc_tensor_from_variable(graph, a0)->data.f32[0], 11, "should be b0[1]");
40
1
  REQUIRE_EQ(ccv_nnc_tensor_from_variable(graph, a1)->data.f32[0], 10, "should be b0[0]");
41
1
  REQUIRE(!CCV_IS_TENSOR_VIEW(ccv_nnc_tensor_from_variable(graph, a1)), "no need to be a tensor view");
42
1
  ccv_nnc_tensor_variable_set(graph, a, b1);
43
1
  REQUIRE_EQ(ccv_nnc_tensor_from_variable(graph, a0)->data.f32[0], 21, "should be b1[1]");
44
1
  REQUIRE_EQ(ccv_nnc_tensor_from_variable(graph, a1)->data.f32[0], 20, "should be b1[0]");
45
1
  REQUIRE(!CCV_IS_TENSOR_VIEW(ccv_nnc_tensor_from_variable(graph, a1)), "no need to be a tensor view");
46
1
  ccv_nnc_dynamic_graph_free(graph);
47
1
  ccv_nnc_tensor_free(b0);
48
1
  ccv_nnc_tensor_free(b1);
49
1
}
50
51
TEST_CASE("dynamic graph to compute f(x) = x * log(x) + 1.2 * x, f'(x) where x = 19")
52
1
{
53
1
  ccv_nnc_dynamic_graph_t* const graph = ccv_nnc_dynamic_graph_new();
54
1
  ccv_nnc_tensor_variable_t x = ccv_nnc_tensor_variable_new(graph, CPU_TENSOR_NHWC(32F, 1));
55
1
  ccv_nnc_tensor_from_variable(graph, x)->data.f32[0] = 19;
56
1
  ccv_nnc_tensor_variable_t f = ccv_nnc_tensor_variable_new(graph);
57
1
  ccv_nnc_dynamic_graph_exec(graph, CMD_EWLOG_FORWARD(), ccv_nnc_no_hint, 0, TENSOR_VARIABLE_LIST(x), TENSOR_VARIABLE_LIST(f), 0, 0);
58
1
  ccv_nnc_dynamic_graph_exec(graph, CMD_EWPROD_FORWARD(), ccv_nnc_no_hint, 0, TENSOR_VARIABLE_LIST(x, f), TENSOR_VARIABLE_LIST(f), 0, 0);
59
1
  ccv_nnc_tensor_variable_t y = ccv_nnc_tensor_variable_new(graph, CPU_TENSOR_NHWC(32F, 1));
60
1
  ccv_nnc_tensor_from_variable(graph, y)->data.f32[0] = 1.2;
61
1
  ccv_nnc_tensor_variable_t z = ccv_nnc_tensor_variable_new(graph);
62
1
  ccv_nnc_dynamic_graph_exec(graph, CMD_EWPROD_FORWARD(), ccv_nnc_no_hint, 0, TENSOR_VARIABLE_LIST(x, y), TENSOR_VARIABLE_LIST(z), 0, 0);
63
1
  ccv_nnc_dynamic_graph_exec(graph, CMD_EWSUM_FORWARD(), ccv_nnc_no_hint, 0, TENSOR_VARIABLE_LIST(f, z), TENSOR_VARIABLE_LIST(f), 0, 0);
64
1
  REQUIRE_EQ_WITH_TOLERANCE(ccv_nnc_tensor_from_variable(graph, f)->data.f32[0], 19 * logf(19) + 1.2 * 19, 1e-5, "f(x) = 1.2 * 19 + 19 * log(19)");
65
1
  // Do gradient computation multiple times.
66
1
  ccv_nnc_tensor_variable_t dx = ccv_nnc_tensor_variable_new(graph);
67
1
  ccv_nnc_dynamic_graph_backward(graph, TENSOR_VARIABLE_LIST(f), 0, TENSOR_VARIABLE_LIST(x), TENSOR_VARIABLE_LIST(dx), 0);
68
1
  REQUIRE_EQ_WITH_TOLERANCE(ccv_nnc_tensor_from_variable(graph, dx)->data.f32[0], logf(19) + 1 + 1.2, 1e-5, "f'(x) = 1.2 + log(19) + 19 * 1 / 19");
69
1
  ccv_nnc_tensor_variable_t dy = ccv_nnc_tensor_variable_new(graph);
70
1
  ccv_nnc_dynamic_graph_backward(graph, TENSOR_VARIABLE_LIST(f), 0, TENSOR_VARIABLE_LIST(y), TENSOR_VARIABLE_LIST(dy), 0);
71
1
  REQUIRE_EQ_WITH_TOLERANCE(ccv_nnc_tensor_from_variable(graph, dy)->data.f32[0], 19, 1e-5, "f'(y) = 19");
72
1
  ccv_nnc_tensor_variable_t dz = ccv_nnc_tensor_variable_new(graph);
73
1
  ccv_nnc_dynamic_graph_backward(graph, TENSOR_VARIABLE_LIST(f), 0, TENSOR_VARIABLE_LIST(z), TENSOR_VARIABLE_LIST(dz), 0);
74
1
  REQUIRE_EQ_WITH_TOLERANCE(ccv_nnc_tensor_from_variable(graph, dz)->data.f32[0], 1, 1e-5, "f'(z) = 1");
75
1
  ccv_nnc_tensor_variable_free(graph, dy);
76
1
  dy = ccv_nnc_tensor_variable_new(graph);
77
1
  ccv_nnc_dynamic_graph_exec(graph, CMD_SET_FORWARD(0), ccv_nnc_no_hint, 0, TENSOR_VARIABLE_LIST(), TENSOR_VARIABLE_LIST(dx), 0, 0);
78
1
  ccv_nnc_dynamic_graph_backward(graph, TENSOR_VARIABLE_LIST(f), 0, TENSOR_VARIABLE_LIST(y, x), TENSOR_VARIABLE_LIST(dy, dx), 0);
79
1
  REQUIRE_EQ_WITH_TOLERANCE(ccv_nnc_tensor_from_variable(graph, dx)->data.f32[0], logf(19) + 1 + 1.2, 1e-5, "f'(x) = 1.2 + log(19) + 19 * 1 / 19");
80
1
  REQUIRE_EQ_WITH_TOLERANCE(ccv_nnc_tensor_from_variable(graph, dy)->data.f32[0], 19, 1e-5, "f'(y) = 19");
81
1
  DYNAMIC_GRAPH_GEN(graph, CCV_NNC_LONG_DOT_GRAPH);
82
1
  ccv_nnc_dynamic_graph_free(graph);
83
1
}
84
85
TEST_CASE("dynamic graph with dense net (extensive use of alias)")
86
1
{
87
1
  ccv_nnc_dynamic_graph_t* const graph = ccv_nnc_dynamic_graph_new();
88
1
  ccv_nnc_tensor_variable_t x = ccv_nnc_tensor_variable_new(graph, CPU_TENSOR_NHWC(32F, 1, 4));
89
1
  ccv_nnc_tensor_from_variable(graph, x)->data.f32[0] = 0.472;
90
1
  ccv_nnc_tensor_variable_t x1 = ccv_nnc_tensor_variable_alias_new(graph, x, ccv_nnc_no_ofs, DIM_ALLOC(1, 4), CPU_TENSOR_NHWC(32F, 1, 1));
91
1
  ccv_nnc_tensor_variable_t w1 = ccv_nnc_tensor_variable_new(graph, CPU_TENSOR_NHWC(32F, 1, 1));
92
1
  ccv_nnc_tensor_from_variable(graph, w1)->data.f32[0] = 0.234;
93
1
  ccv_nnc_tensor_variable_t b1 = ccv_nnc_tensor_variable_new(graph, CPU_TENSOR_NHWC(32F, 1));
94
1
  ccv_nnc_tensor_from_variable(graph, b1)->data.f32[0] = 0.1;
95
1
  ccv_nnc_tensor_variable_t x11 = ccv_nnc_tensor_variable_alias_new(graph, x, DIM_ALLOC(0, 1), DIM_ALLOC(1, 4), CPU_TENSOR_NHWC(32F, 1, 1));
96
1
  ccv_nnc_dynamic_graph_exec(graph, CMD_GEMM_FORWARD(NO_TRANSPOSE, TRANSPOSE(0, 1)), ccv_nnc_no_hint, 0, TENSOR_VARIABLE_LIST(x1, w1, b1), TENSOR_VARIABLE_LIST(x11), 0, 0);
97
1
  ccv_nnc_tensor_variable_t x2 = ccv_nnc_tensor_variable_alias_new(graph, x, ccv_nnc_no_ofs, DIM_ALLOC(1, 4), CPU_TENSOR_NHWC(32F, 1, 2));
98
1
  ccv_nnc_tensor_variable_t w2 = ccv_nnc_tensor_variable_new(graph, CPU_TENSOR_NHWC(32F, 1, 2));
99
1
  ccv_nnc_tensor_from_variable(graph, w2)->data.f32[0] = 0.374;
100
1
  ccv_nnc_tensor_from_variable(graph, w2)->data.f32[1] = 0.886;
101
1
  ccv_nnc_tensor_variable_t b2 = ccv_nnc_tensor_variable_new(graph, CPU_TENSOR_NHWC(32F, 1));
102
1
  ccv_nnc_tensor_from_variable(graph, b2)->data.f32[0] = 0.2;
103
1
  ccv_nnc_tensor_variable_t x21 = ccv_nnc_tensor_variable_alias_new(graph, x, DIM_ALLOC(0, 2), DIM_ALLOC(1, 4), CPU_TENSOR_NHWC(32F, 1, 1));
104
1
  ccv_nnc_dynamic_graph_exec(graph, CMD_GEMM_FORWARD(NO_TRANSPOSE, TRANSPOSE(0, 1)), ccv_nnc_no_hint, 0, TENSOR_VARIABLE_LIST(x2, w2, b2), TENSOR_VARIABLE_LIST(x21), 0, 0);
105
1
  ccv_nnc_tensor_variable_t x3 = ccv_nnc_tensor_variable_alias_new(graph, x, ccv_nnc_no_ofs, DIM_ALLOC(1, 4), CPU_TENSOR_NHWC(32F, 1, 3));
106
1
  ccv_nnc_tensor_variable_t w3 = ccv_nnc_tensor_variable_new(graph, CPU_TENSOR_NHWC(32F, 1, 3));
107
1
  ccv_nnc_tensor_from_variable(graph, w3)->data.f32[0] = 0.484;
108
1
  ccv_nnc_tensor_from_variable(graph, w3)->data.f32[1] = 0.912;
109
1
  ccv_nnc_tensor_from_variable(graph, w3)->data.f32[2] = 0.235;
110
1
  ccv_nnc_tensor_variable_t b3 = ccv_nnc_tensor_variable_new(graph, CPU_TENSOR_NHWC(32F, 1));
111
1
  ccv_nnc_tensor_from_variable(graph, b3)->data.f32[0] = 0.3;
112
1
  ccv_nnc_tensor_variable_t x31 = ccv_nnc_tensor_variable_alias_new(graph, x, DIM_ALLOC(0, 3), DIM_ALLOC(1, 4), CPU_TENSOR_NHWC(32F, 1, 1));
113
1
  ccv_nnc_dynamic_graph_exec(graph, CMD_GEMM_FORWARD(NO_TRANSPOSE, TRANSPOSE(0, 1)), ccv_nnc_no_hint, 0, TENSOR_VARIABLE_LIST(x3, w3, b3), TENSOR_VARIABLE_LIST(x31), 0, 0);
114
1
  DYNAMIC_GRAPH_GEN(graph, CCV_NNC_LONG_DOT_GRAPH);
115
1
  ccv_nnc_tensor_t* xt = ccv_nnc_tensor_new(0, CPU_TENSOR_NHWC(32F, 1, 4), 0);
116
1
  xt->data.f32[0] = 0.472;
117
1
  xt->data.f32[1] = xt->data.f32[0] * 0.234 + 0.1;
118
1
  xt->data.f32[2] = xt->data.f32[0] * 0.374 + xt->data.f32[1] * 0.886 + 0.2;
119
1
  xt->data.f32[3] = xt->data.f32[0] * 0.484 + xt->data.f32[1] * 0.912 + xt->data.f32[2] * 0.235 + 0.3;
120
1
  REQUIRE_MATRIX_EQ(ccv_nnc_tensor_from_variable(graph, x), xt, "1x4 matrix should be exactly the same");
121
1
  ccv_nnc_tensor_free(xt);
122
1
  ccv_nnc_tensor_variable_t dw1 = ccv_nnc_tensor_variable_new(graph);
123
1
  ccv_nnc_dynamic_graph_backward(graph, TENSOR_VARIABLE_LIST(x), 0, TENSOR_VARIABLE_LIST(w1), TENSOR_VARIABLE_LIST(dw1), 0);
124
1
  REQUIRE_EQ_WITH_TOLERANCE((0.235 * 0.886 + 0.912) * 0.472, ccv_nnc_tensor_from_variable(graph, dw1)->data.f32[0], 1e-5, "the gradient should be equal to a complicated result");
125
1
  ccv_nnc_dynamic_graph_free(graph);
126
1
}
127
128
TEST_CASE("batch norm in dynamic graph (enforce inplace)")
129
1
{
130
1
  ccv_nnc_dynamic_graph_t* const graph = ccv_nnc_dynamic_graph_new();
131
1
  ccv_nnc_tensor_variable_t x = ccv_nnc_tensor_variable_new(graph, CPU_TENSOR_NHWC(32F, 2, 2, 2, 10));
132
1
  ccv_nnc_tensor_variable_t y = ccv_nnc_tensor_variable_new(graph);
133
1
  ccv_nnc_tensor_variable_t scale = ccv_nnc_tensor_variable_new(graph, CPU_TENSOR_NHWC(32F, 10));
134
1
  ccv_nnc_tensor_variable_t bias = ccv_nnc_tensor_variable_new(graph, CPU_TENSOR_NHWC(32F, 10));
135
1
  ccv_nnc_tensor_variable_t mean = ccv_nnc_tensor_variable_new(graph, CPU_TENSOR_NHWC(32F, 10));
136
1
  ccv_nnc_tensor_variable_t var = ccv_nnc_tensor_variable_new(graph, CPU_TENSOR_NHWC(32F, 10));
137
1
  ccv_nnc_tensor_variable_t saved_mean = ccv_nnc_tensor_variable_new(graph);
138
1
  ccv_nnc_tensor_variable_t saved_inv_std = ccv_nnc_tensor_variable_new(graph);
139
1
  dsfmt_t dsfmt;
140
1
  int i;
141
1
  dsfmt_init_gen_rand(&dsfmt, 1);
142
81
  for (i = 0; i < 2 * 2 * 2 * 10; 
i++80
)
143
80
    ccv_nnc_tensor_from_variable(graph, x)->data.f32[i] = dsfmt_genrand_open_close(&dsfmt);
144
11
  for (i = 0; i < 10; 
i++10
)
145
10
    ccv_nnc_tensor_from_variable(graph, scale)->data.f32[i] = 1;
146
11
  for (i = 0; i < 10; 
i++10
)
147
10
    ccv_nnc_tensor_from_variable(graph, bias)->data.f32[i] = 0;
148
11
  for (i = 0; i < 10; 
i++10
)
149
10
    ccv_nnc_tensor_from_variable(graph, mean)->data.f32[i] = 0;
150
1
  ccv_nnc_tensor_t* mean_tensor_ptr = ccv_nnc_tensor_from_variable(graph, mean);
151
11
  for (i = 0; i < 10; 
i++10
)
152
10
    ccv_nnc_tensor_from_variable(graph, var)->data.f32[i] = 0;
153
1
  ccv_nnc_tensor_t* var_tensor_ptr = ccv_nnc_tensor_from_variable(graph, var);
154
1
  ccv_nnc_dynamic_graph_exec(graph, CMD_BATCH_NORM_FORWARD(0, 0, 0.9, 0, 1, 2), ccv_nnc_no_hint, 0, TENSOR_VARIABLE_LIST(x, scale, bias, mean, var), TENSOR_VARIABLE_LIST(y, mean, var, saved_mean, saved_inv_std), 0, 0);
155
1
  DYNAMIC_GRAPH_GEN(graph, CCV_NNC_LONG_DOT_GRAPH);
156
1
  REQUIRE(mean_tensor_ptr == ccv_nnc_tensor_from_variable(graph, mean), "enforced inplace, tensor view pointer unchanged");
157
1
  REQUIRE(var_tensor_ptr == ccv_nnc_tensor_from_variable(graph, var), "enforced inplace, tensor view pointer unchanged");
158
1
  ccv_nnc_tensor_t* x_tensor = ccv_nnc_tensor_new(0, CPU_TENSOR_NHWC(32F, 2, 2, 2, 10), 0);
159
1
  ccv_nnc_tensor_t* y_tensor = ccv_nnc_tensor_new(0, CPU_TENSOR_NHWC(32F, 2, 2, 2, 10), 0);
160
1
  ccv_nnc_tensor_t* scale_tensor = ccv_nnc_tensor_new(0, CPU_TENSOR_NHWC(32F, 10), 0);
161
1
  ccv_nnc_tensor_t* bias_tensor = ccv_nnc_tensor_new(0, CPU_TENSOR_NHWC(32F, 10), 0);
162
1
  ccv_nnc_tensor_t* mean_tensor = ccv_nnc_tensor_new(0, CPU_TENSOR_NHWC(32F, 10), 0);
163
1
  ccv_nnc_tensor_t* var_tensor = ccv_nnc_tensor_new(0, CPU_TENSOR_NHWC(32F, 10), 0);
164
1
  ccv_nnc_tensor_t* saved_mean_tensor = ccv_nnc_tensor_new(0, CPU_TENSOR_NHWC(32F, 1, 1, 1, 10), 0);
165
1
  ccv_nnc_tensor_t* saved_inv_std_tensor = ccv_nnc_tensor_new(0, CPU_TENSOR_NHWC(32F, 1, 1, 1, 10), 0);
166
1
  memcpy(x_tensor->data.f32, ccv_nnc_tensor_from_variable(graph, x)->data.f32, sizeof(float) * 2 * 2 * 2 * 10);
167
11
  for (i = 0; i < 10; 
i++10
)
168
10
    scale_tensor->data.f32[i] = 1;
169
1
  memset(bias_tensor->data.f32, 0, sizeof(float) * 10);
170
1
  memset(mean_tensor->data.f32, 0, sizeof(float) * 10);
171
1
  memset(var_tensor->data.f32, 0, sizeof(float) * 10);
172
1
  ccv_nnc_cmd_exec(CMD_BATCH_NORM_FORWARD(0, 0, 0.9, 0, 1, 2), ccv_nnc_no_hint, 0, TENSOR_LIST(x_tensor, scale_tensor, bias_tensor, mean_tensor, var_tensor), TENSOR_LIST(y_tensor, mean_tensor, var_tensor, saved_mean_tensor, saved_inv_std_tensor), 0);
173
1
  REQUIRE_TENSOR_EQ(y_tensor, ccv_nnc_tensor_from_variable(graph, y), "y should be equal");
174
1
  REQUIRE_TENSOR_EQ(mean_tensor, ccv_nnc_tensor_from_variable(graph, mean), "mean should be equal");
175
1
  REQUIRE_TENSOR_EQ(var_tensor, ccv_nnc_tensor_from_variable(graph, var), "var should be equal");
176
1
  REQUIRE_TENSOR_EQ(saved_mean_tensor, ccv_nnc_tensor_from_variable(graph, saved_mean), "saved_mean should be equal");
177
1
  REQUIRE_TENSOR_EQ(saved_inv_std_tensor, ccv_nnc_tensor_from_variable(graph, saved_inv_std), "saved_inv_std should be equal");
178
1
  ccv_nnc_dynamic_graph_free(graph);
179
1
  ccv_nnc_tensor_free(x_tensor);
180
1
  ccv_nnc_tensor_free(y_tensor);
181
1
  ccv_nnc_tensor_free(scale_tensor);
182
1
  ccv_nnc_tensor_free(bias_tensor);
183
1
  ccv_nnc_tensor_free(mean_tensor);
184
1
  ccv_nnc_tensor_free(var_tensor);
185
1
  ccv_nnc_tensor_free(saved_mean_tensor);
186
1
  ccv_nnc_tensor_free(saved_inv_std_tensor);
187
1
}
188
189
TEST_CASE("empty inputs / outputs for dynamic graph")
190
1
{
191
1
  ccv_nnc_dynamic_graph_t* const graph = ccv_nnc_dynamic_graph_new();
192
1
  ccv_nnc_tensor_variable_t df = ccv_nnc_tensor_variable_new(graph, CPU_TENSOR_NHWC(32F, 1));
193
1
  ccv_nnc_tensor_variable_t x = ccv_nnc_tensor_variable_new(graph, CPU_TENSOR_NHWC(32F, 1));
194
1
  ccv_nnc_tensor_variable_t y = ccv_nnc_tensor_variable_new(graph);
195
1
  ccv_nnc_tensor_from_variable(graph, df)->data.f32[0] = 1;
196
1
  ccv_nnc_tensor_from_variable(graph, x)->data.f32[0] = 10;
197
1
  ccv_nnc_dynamic_graph_exec(graph, CMD_EWDIV_BACKWARD(), ccv_nnc_no_hint, 0, TENSOR_VARIABLE_LIST(df, 0, x), TENSOR_VARIABLE_LIST(y, 0), 0, 0);
198
1
  DYNAMIC_GRAPH_GEN(graph, CCV_NNC_LONG_DOT_GRAPH);
199
1
  REQUIRE_EQ_WITH_TOLERANCE(ccv_nnc_tensor_from_variable(graph, y)->data.f32[0], 1. / 10, 1e-5, "div backward should equal to 1 / 10");
200
1
  ccv_nnc_dynamic_graph_free(graph);
201
1
}
202
203
TEST_CASE("long dynamic graph with unused variables freed")
204
1
{
205
1
  ccv_nnc_dynamic_graph_t* const graph = ccv_nnc_dynamic_graph_new();
206
1
  int i;
207
1
  ccv_nnc_tensor_variable_t x = ccv_nnc_tensor_variable_new(graph, CPU_TENSOR_NHWC(32F, 1));
208
1
  ccv_nnc_tensor_variable_t y = ccv_nnc_tensor_variable_new(graph, CPU_TENSOR_NHWC(32F, 1));
209
1
  ccv_nnc_tensor_from_variable(graph, x)->data.f32[0] = 32;
210
1
  ccv_nnc_tensor_from_variable(graph, y)->data.f32[0] = 0.5;
211
11
  for (i = 0; i < 10; 
i++10
)
212
10
  {
213
10
    ccv_nnc_tensor_variable_t z = ccv_nnc_tensor_variable_new(graph);
214
10
    if (i < 7)
215
7
      ccv_nnc_dynamic_graph_exec(graph, CMD_EWPROD_FORWARD(), ccv_nnc_no_hint, 0, TENSOR_VARIABLE_LIST(x, y), TENSOR_VARIABLE_LIST(z), 0, 0);
216
3
    else {
217
3
      if (i == 7)
218
1
        ccv_nnc_tensor_variable_free(graph, y); // No longer need y.
219
3
      ccv_nnc_dynamic_graph_exec(graph, CMD_EWPROD_FORWARD(), ccv_nnc_no_hint, 0, TENSOR_VARIABLE_LIST(x, x), TENSOR_VARIABLE_LIST(z), 0, 0);
220
3
    }
221
10
    if (i < 9)
222
9
      ccv_nnc_tensor_variable_free(graph, x);
223
10
    x = z;
224
10
  }
225
1
  DYNAMIC_GRAPH_GEN(graph, CCV_NNC_LONG_DOT_GRAPH);
226
1
  float g = 32;
227
11
  for (i = 0; i < 10; 
i++10
)
228
10
  {
229
10
    if (i < 7)
230
7
      g = g * 0.5;
231
3
    else
232
3
      g = g * g;
233
10
  }
234
1
  REQUIRE_EQ_WITH_TOLERANCE(ccv_nnc_tensor_from_variable(graph, x)->data.f32[0], g, 1e-5, "x should equal to the computed result");
235
1
  ccv_nnc_dynamic_graph_free(graph);
236
1
}
237
238
TEST_CASE("repeat multiple x * y with y as a constant")
239
1
{
240
1
  ccv_nnc_dynamic_graph_t* const graph = ccv_nnc_dynamic_graph_new();
241
1
  int i;
242
1
  ccv_nnc_tensor_variable_t x = ccv_nnc_tensor_variable_new(graph, CPU_TENSOR_NHWC(32F, 1));
243
1
  ccv_nnc_tensor_variable_t y = ccv_nnc_tensor_constant_new(graph, CPU_TENSOR_NHWC(32F, 1));
244
1
  ccv_nnc_tensor_from_variable(graph, x)->data.f32[0] = 32;
245
1
  ccv_nnc_tensor_from_variable(graph, y)->data.f32[0] = 0.5;
246
11
  for (i = 0; i < 10; 
i++10
)
247
10
  {
248
10
    ccv_nnc_tensor_variable_t z = ccv_nnc_tensor_variable_new(graph);
249
10
    ccv_nnc_dynamic_graph_exec(graph, CMD_EWPROD_FORWARD(), ccv_nnc_no_hint, 0, TENSOR_VARIABLE_LIST(x, y), TENSOR_VARIABLE_LIST(z), 0, 0);
250
10
    ccv_nnc_tensor_variable_free(graph, x);
251
10
    x = z;
252
10
  }
253
1
  DYNAMIC_GRAPH_GEN(graph, CCV_NNC_LONG_DOT_GRAPH);
254
1
  float g = 32;
255
11
  for (i = 0; i < 10; 
i++10
)
256
10
    g = g * 0.5;
257
1
  REQUIRE_EQ_WITH_TOLERANCE(ccv_nnc_tensor_from_variable(graph, x)->data.f32[0], g, 1e-5, "x should equal to the computed result");
258
1
  ccv_nnc_dynamic_graph_free(graph);
259
1
}
260
261
static int _ccv_tensor_variable_freed = 0;
262
263
static void _ccv_tensor_variable_hook(ccv_nnc_dynamic_graph_t* const graph, const ccv_nnc_tensor_t* const tensor, void* const context)
264
7
{
265
7
  ++_ccv_tensor_variable_freed;
266
7
}
267
268
TEST_CASE("repeat multiple x * y with y as a constant, compute d(x)")
269
1
{
270
1
  ccv_nnc_dynamic_graph_t* const graph = ccv_nnc_dynamic_graph_new();
271
1
  int i;
272
1
  _ccv_tensor_variable_freed = 0;
273
1
  ccv_nnc_tensor_variable_t x = ccv_nnc_tensor_variable_new(graph, CPU_TENSOR_NHWC(32F, 1));
274
1
  ccv_nnc_tensor_variable_t y = ccv_nnc_tensor_constant_new(graph, CPU_TENSOR_NHWC(32F, 1));
275
1
  ccv_nnc_tensor_variable_t ox = x;
276
1
  ccv_nnc_tensor_variable_destructor_hook(graph, x, _ccv_tensor_variable_hook, 0);
277
1
  ccv_nnc_tensor_variable_destructor_hook(graph, y, _ccv_tensor_variable_hook, 0);
278
1
  ccv_nnc_tensor_from_variable(graph, x)->data.f32[0] = 32;
279
1
  ccv_nnc_tensor_from_variable(graph, y)->data.f32[0] = 0.9;
280
5
  for (i = 0; i < 4; 
i++4
)
281
4
  {
282
4
    ccv_nnc_tensor_variable_t z = ccv_nnc_tensor_variable_new(graph);
283
4
    ccv_nnc_tensor_variable_destructor_hook(graph, z, _ccv_tensor_variable_hook, 0);
284
4
    ccv_nnc_dynamic_graph_exec(graph, CMD_EWPROD_FORWARD(), ccv_nnc_no_hint, 0, TENSOR_VARIABLE_LIST(x, y), TENSOR_VARIABLE_LIST(z), 0, 0);
285
4
    if (i > 0)
286
3
      ccv_nnc_tensor_variable_free(graph, x);
287
4
    x = z;
288
4
  }
289
1
  REQUIRE_EQ(0, _ccv_tensor_variable_freed, "none of these are freed");
290
1
  ccv_nnc_tensor_variable_t dx = ccv_nnc_tensor_variable_new(graph);
291
1
  ccv_nnc_tensor_variable_destructor_hook(graph, dx, _ccv_tensor_variable_hook, 0);
292
1
  ccv_nnc_dynamic_graph_backward(graph, TENSOR_VARIABLE_LIST(x), 0, TENSOR_VARIABLE_LIST(ox), TENSOR_VARIABLE_LIST(dx), 0);
293
1
  ccv_nnc_tensor_variable_free(graph, ox);
294
1
  // We freed all 4 x (ox, when i = 1, 2, 3).
295
1
  REQUIRE_EQ(4, _ccv_tensor_variable_freed, "all are freed except dx");
296
1
  _ccv_tensor_variable_freed = 0;
297
1
  DYNAMIC_GRAPH_GEN(graph, CCV_NNC_LONG_DOT_GRAPH);
298
1
  float g = 1;
299
5
  for (i = 0; i < 4; 
i++4
)
300
4
    g = g * 0.9;
301
1
  REQUIRE_EQ_WITH_TOLERANCE(ccv_nnc_tensor_from_variable(graph, dx)->data.f32[0], g, 1e-5, "x should equal to the computed result");
302
1
  ccv_nnc_dynamic_graph_free(graph);
303
1
  // y, dx, and z.
304
1
  REQUIRE_EQ(3, _ccv_tensor_variable_freed, "dx freed");
305
1
}
306
307
TEST_CASE("compute f(x) = x * log(x) + x, f'(x) when x = 10 (and intermediate results all freed)")
308
1
{
309
1
  ccv_nnc_dynamic_graph_t* const graph = ccv_nnc_dynamic_graph_new();
310
1
  ccv_nnc_tensor_variable_t x = ccv_nnc_tensor_variable_new(graph, CPU_TENSOR_NHWC(32F, 1));
311
1
  ccv_nnc_tensor_variable_t y = ccv_nnc_tensor_variable_new(graph);
312
1
  ccv_nnc_tensor_variable_t f = ccv_nnc_tensor_variable_new(graph);
313
1
  ccv_nnc_tensor_from_variable(graph, x)->data.f32[0] = 10;
314
1
  ccv_nnc_dynamic_graph_exec(graph, CMD_EWLOG_FORWARD(), ccv_nnc_no_hint, 0, TENSOR_VARIABLE_LIST(x), TENSOR_VARIABLE_LIST(y), 0, 0);
315
1
  ccv_nnc_dynamic_graph_exec(graph, CMD_EWPROD_FORWARD(), ccv_nnc_no_hint, 0, TENSOR_VARIABLE_LIST(x, y), TENSOR_VARIABLE_LIST(y), 0, 0);
316
1
  ccv_nnc_dynamic_graph_exec(graph, CMD_EWSUM_FORWARD(), ccv_nnc_no_hint, 0, TENSOR_VARIABLE_LIST(y, x), TENSOR_VARIABLE_LIST(f), 0, 0);
317
1
  ccv_nnc_tensor_variable_t df = ccv_nnc_tensor_variable_new(graph, CPU_TENSOR_NHWC(32F, 1));
318
1
  ccv_nnc_dynamic_graph_exec(graph, CMD_SET_FORWARD(1), ccv_nnc_no_hint, 0, 0, 0, TENSOR_VARIABLE_LIST(df), 0, 0);
319
1
  // x will be accumulated on to itself.
320
1
  ccv_nnc_dynamic_graph_backward(graph, TENSOR_VARIABLE_LIST(f), &df, TENSOR_VARIABLE_LIST(x), TENSOR_VARIABLE_LIST(x), 0);
321
1
  DYNAMIC_GRAPH_GEN(graph, CCV_NNC_LONG_DOT_GRAPH);
322
1
  REQUIRE_EQ_WITH_TOLERANCE(ccv_nnc_tensor_from_variable(graph, x)->data.f32[0], 10 + log(10) + 1 + 1, 1e-5, "dx should equal to the computed result");
323
1
  ccv_nnc_dynamic_graph_free(graph);
324
1
}
325
326
TEST_CASE("dynamic graph with binded value")
327
1
{
328
1
  ccv_nnc_dynamic_graph_t* const graph = ccv_nnc_dynamic_graph_new();
329
1
  ccv_nnc_tensor_variable_t x = ccv_nnc_tensor_variable_new(graph, CPU_TENSOR_NHWC(32F, 1));
330
1
  ccv_nnc_tensor_t* const x_tensor = ccv_nnc_tensor_new(0, CPU_TENSOR_NHWC(32F, 1), 0);
331
1
  x_tensor->data.f32[0] = 10;
332
1
  ccv_nnc_tensor_variable_set(graph, x, x_tensor);
333
1
  ccv_nnc_tensor_variable_t y = ccv_nnc_tensor_variable_new(graph);
334
1
  ccv_nnc_dynamic_graph_exec(graph, CMD_EWLOG_FORWARD(), ccv_nnc_no_hint, 0, TENSOR_VARIABLE_LIST(x), TENSOR_VARIABLE_LIST(y), 0, 0);
335
1
  DYNAMIC_GRAPH_GEN(graph, CCV_NNC_LONG_DOT_GRAPH);
336
1
  REQUIRE_EQ_WITH_TOLERANCE(ccv_nnc_tensor_from_variable(graph, y)->data.f32[0], log(10), 1e-5, "dx should equal to the computed result");
337
1
  ccv_nnc_dynamic_graph_free(graph);
338
1
  ccv_nnc_tensor_free(x_tensor);
339
1
}
340
341
TEST_CASE("dynamic graph to evaluate cnnp model")
342
1
{
343
1
  ccv_nnc_dynamic_graph_t* const graph = ccv_nnc_dynamic_graph_new();
344
1
  ccv_cnnp_model_t* const linear = ccv_cnnp_dense(1, 0, 0);
345
1
  ccv_nnc_tensor_variable_t z = ccv_nnc_tensor_constant_new(graph, CPU_TENSOR_NHWC(32F, 1));
346
1
  ccv_nnc_tensor_from_variable(graph, z)->data.f32[0] = 5;
347
1
  int i;
348
101
  for (i = 0; i < 100; 
i++100
)
349
100
  {
350
100
    ccv_nnc_tensor_variable_t x;
351
100
    if (i % 2 == 1)
352
50
    {
353
50
      x = ccv_nnc_tensor_variable_new(graph, CPU_TENSOR_NHWC(32F, 2, 1));
354
50
      ccv_nnc_tensor_from_variable(graph, x)->data.f32[0] = 10;
355
50
      ccv_nnc_tensor_from_variable(graph, x)->data.f32[1] = 10;
356
50
    } else {
357
50
      x = ccv_nnc_tensor_variable_new(graph, CPU_TENSOR_NHWC(32F, 1));
358
50
      ccv_nnc_tensor_from_variable(graph, x)->data.f32[0] = 10;
359
50
    }
360
100
    ccv_nnc_tensor_variable_t y = ccv_nnc_tensor_variable_new(graph);
361
100
    ccv_nnc_dynamic_graph_exec(graph, CMD_EWLOG_FORWARD(), ccv_nnc_no_hint, 0, TENSOR_VARIABLE_LIST(x), TENSOR_VARIABLE_LIST(y), 0, 0);
362
100
    ccv_nnc_dynamic_graph_evaluate(graph, linear, 0, TENSOR_VARIABLE_LIST(y), TENSOR_VARIABLE_LIST(y), 0, 0);
363
100
    ccv_nnc_dynamic_graph_exec(graph, CMD_ADD_FORWARD(1, -1), ccv_nnc_no_hint, 0, TENSOR_VARIABLE_LIST(y, z), TENSOR_VARIABLE_LIST(y), 0, 0);
364
100
    ccv_nnc_tensor_variable_t f = ccv_nnc_tensor_variable_new(graph);
365
100
    ccv_nnc_dynamic_graph_exec(graph, CMD_EWPROD_FORWARD(), ccv_nnc_no_hint, 0, TENSOR_VARIABLE_LIST(y, y), TENSOR_VARIABLE_LIST(f), 0, 0);
366
100
    ccv_nnc_tensor_variable_t dx = ccv_nnc_tensor_variable_new(graph);
367
100
    ccv_nnc_dynamic_graph_backward(graph, TENSOR_VARIABLE_LIST(f), 0, TENSOR_VARIABLE_LIST(x), TENSOR_VARIABLE_LIST(dx), 0);
368
100
    ccv_cnnp_model_set_minimizer(linear, CMD_SGD_FORWARD(0, 0.01, 1, 0.01, 0, 0), 0, 0, 0);
369
100
    ccv_nnc_dynamic_graph_apply_gradients(graph, CMD_SGD_FORWARD(0, 0.01, 1, 0.01, 0, 0), TENSOR_VARIABLE_LIST(), TENSOR_VARIABLE_LIST(), 0, 0, 0);
370
100
    ccv_nnc_tensor_variable_free(graph, x);
371
100
    ccv_nnc_tensor_variable_free(graph, y);
372
100
    ccv_nnc_tensor_variable_free(graph, f);
373
100
    ccv_nnc_tensor_variable_free(graph, dx);
374
100
  }
375
1
  ccv_nnc_tensor_variable_t x = ccv_nnc_tensor_variable_new(graph, CPU_TENSOR_NHWC(32F, 1));
376
1
  ccv_nnc_tensor_from_variable(graph, x)->data.f32[0] = 10;
377
1
  ccv_nnc_tensor_variable_t y = ccv_nnc_tensor_variable_new(graph);
378
1
  ccv_nnc_dynamic_graph_exec(graph, CMD_EWLOG_FORWARD(), ccv_nnc_no_hint, 0, TENSOR_VARIABLE_LIST(x), TENSOR_VARIABLE_LIST(y), 0, 0);
379
1
  ccv_nnc_dynamic_graph_evaluate(graph, linear, 1, TENSOR_VARIABLE_LIST(y), TENSOR_VARIABLE_LIST(y), 0, 0);
380
1
  REQUIRE_EQ_WITH_TOLERANCE(ccv_nnc_tensor_from_variable(graph, y)->data.f32[0], 5, 1e-2, "linear model should be trained to generate the same value as z");
381
1
  ccv_nnc_tensor_variable_t iy = ccv_nnc_tensor_constant_new(graph, CPU_TENSOR_NHWC(32F, 1));
382
1
  ccv_nnc_tensor_from_variable(graph, iy)->data.f32[0] = log(10);
383
1
  ccv_nnc_tensor_variable_t iz = ccv_nnc_tensor_variable_new(graph, CPU_TENSOR_NHWC(32F, 1));
384
1
  ccv_nnc_dynamic_graph_evaluate(graph, linear, 1, TENSOR_VARIABLE_LIST(iy), TENSOR_VARIABLE_LIST(iz), 0, 0);
385
1
  REQUIRE_EQ_WITH_TOLERANCE(ccv_nnc_tensor_from_variable(graph, iz)->data.f32[0], 5, 1e-2, "linear model should be trained to generate the same value as z");
386
1
  DYNAMIC_GRAPH_GEN(graph, CCV_NNC_LONG_DOT_GRAPH);
387
1
  ccv_nnc_dynamic_graph_free(graph);
388
1
  ccv_cnnp_model_free(linear);
389
1
}
390
391
TEST_CASE("dynamic graph to evaluate cnnp model without any parameters")
392
1
{
393
1
  ccv_nnc_dynamic_graph_t* const graph = ccv_nnc_dynamic_graph_new();
394
1
  ccv_nnc_tensor_variable_t a = ccv_nnc_tensor_variable_new(graph, CPU_TENSOR_NHWC(32F, 1));
395
1
  ccv_nnc_tensor_from_variable(graph, a)->data.f32[0] = 1.23;
396
1
  ccv_nnc_tensor_variable_t b = ccv_nnc_tensor_variable_new(graph, CPU_TENSOR_NHWC(32F, 1));
397
1
  ccv_nnc_tensor_from_variable(graph, b)->data.f32[0] = 2;
398
1
  ccv_nnc_tensor_variable_t c = ccv_nnc_tensor_variable_new(graph, CPU_TENSOR_NHWC(32F, 1));
399
1
  ccv_cnnp_model_t* const mul = ccv_cnnp_mul(1, "mul");
400
1
  ccv_nnc_dynamic_graph_evaluate(graph, mul, 1, TENSOR_VARIABLE_LIST(a, b), TENSOR_VARIABLE_LIST(c), 0, 0);
401
1
  ccv_nnc_tensor_variable_t da = ccv_nnc_tensor_variable_new(graph);
402
1
  ccv_nnc_dynamic_graph_backward(graph, TENSOR_VARIABLE_LIST(c), 0, TENSOR_VARIABLE_LIST(a), TENSOR_VARIABLE_LIST(da), 0);
403
1
  ccv_cnnp_model_set_minimizer(mul, CMD_SGD_FORWARD(0, 0.01, 1, 0.01, 0, 0), 0, 0, 0);
404
1
  ccv_nnc_dynamic_graph_apply_gradients(graph, CMD_SGD_FORWARD(0, 0.01, 1, 0.01, 0, 0), TENSOR_VARIABLE_LIST(), TENSOR_VARIABLE_LIST(), 0, 0, 0);
405
1
  ccv_cnnp_model_free(mul);
406
1
  REQUIRE_EQ_WITH_TOLERANCE(ccv_nnc_tensor_from_variable(graph, c)->data.f32[0], 2.46, 1e-5, "should be equal");
407
1
  REQUIRE_EQ_WITH_TOLERANCE(ccv_nnc_tensor_from_variable(graph, da)->data.f32[0], 2, 1e-5, "should be equal");
408
1
  ccv_nnc_dynamic_graph_free(graph);
409
1
}
410
411
TEST_CASE("dynamic graph to evaluate cnnp model and simply accumulate gradients")
412
1
{
413
1
  ccv_nnc_dynamic_graph_t* const graph = ccv_nnc_dynamic_graph_new();
414
1
  ccv_cnnp_model_t* const linear = ccv_cnnp_dense(1, 0, 0);
415
1
  ccv_nnc_tensor_variable_t z = ccv_nnc_tensor_constant_new(graph, CPU_TENSOR_NHWC(32F, 1));
416
1
  ccv_nnc_tensor_from_variable(graph, z)->data.f32[0] = 5;
417
1
  int i;
418
101
  for (i = 0; i < 100; 
i++100
)
419
100
  {
420
100
    ccv_nnc_tensor_variable_t x;
421
100
    if (i % 2 == 1)
422
50
    {
423
50
      x = ccv_nnc_tensor_variable_new(graph, CPU_TENSOR_NHWC(32F, 2, 1));
424
50
      ccv_nnc_tensor_from_variable(graph, x)->data.f32[0] = 10;
425
50
      ccv_nnc_tensor_from_variable(graph, x)->data.f32[1] = 10;
426
50
    } else {
427
50
      x = ccv_nnc_tensor_variable_new(graph, CPU_TENSOR_NHWC(32F, 1));
428
50
      ccv_nnc_tensor_from_variable(graph, x)->data.f32[0] = 10;
429
50
    }
430
100
    ccv_nnc_tensor_variable_t y = ccv_nnc_tensor_variable_new(graph);
431
100
    ccv_nnc_dynamic_graph_exec(graph, CMD_EWLOG_FORWARD(), ccv_nnc_no_hint, 0, TENSOR_VARIABLE_LIST(x), TENSOR_VARIABLE_LIST(y), 0, 0);
432
100
    ccv_nnc_dynamic_graph_evaluate(graph, linear, 0, TENSOR_VARIABLE_LIST(y), TENSOR_VARIABLE_LIST(y), 0, 0);
433
100
    ccv_nnc_dynamic_graph_exec(graph, CMD_ADD_FORWARD(1, -1), ccv_nnc_no_hint, 0, TENSOR_VARIABLE_LIST(y, z), TENSOR_VARIABLE_LIST(y), 0, 0);
434
100
    ccv_nnc_tensor_variable_t f = ccv_nnc_tensor_variable_new(graph);
435
100
    ccv_nnc_dynamic_graph_exec(graph, CMD_EWPROD_FORWARD(), ccv_nnc_no_hint, 0, TENSOR_VARIABLE_LIST(y, y), TENSOR_VARIABLE_LIST(f), 0, 0);
436
100
    ccv_nnc_tensor_variable_t dx = ccv_nnc_tensor_variable_new(graph);
437
100
    ccv_nnc_dynamic_graph_backward(graph, TENSOR_VARIABLE_LIST(f), 0, TENSOR_VARIABLE_LIST(x), TENSOR_VARIABLE_LIST(dx), 0);
438
100
    ccv_nnc_tensor_variable_free(graph, x);
439
100
    ccv_nnc_tensor_variable_free(graph, y);
440
100
    ccv_nnc_tensor_variable_free(graph, f);
441
100
    ccv_nnc_tensor_variable_free(graph, dx);
442
100
    if ((i % 2) == 1)
443
50
    {
444
50
      ccv_cnnp_model_set_minimizer(linear, CMD_SGD_FORWARD(0, 0.01, 1, 0.01, 0, 0), 0, 0, 0);
445
50
      ccv_nnc_dynamic_graph_apply_gradients(graph, CMD_SGD_FORWARD(0, 0.01, 1, 0.01, 0, 0), TENSOR_VARIABLE_LIST(), TENSOR_VARIABLE_LIST(), 0, 0, 0);
446
50
    }
447
100
  }
448
1
  ccv_nnc_tensor_variable_t x = ccv_nnc_tensor_variable_new(graph, CPU_TENSOR_NHWC(32F, 1));
449
1
  ccv_nnc_tensor_from_variable(graph, x)->data.f32[0] = 10;
450
1
  ccv_nnc_tensor_variable_t y = ccv_nnc_tensor_variable_new(graph);
451
1
  ccv_nnc_dynamic_graph_exec(graph, CMD_EWLOG_FORWARD(), ccv_nnc_no_hint, 0, TENSOR_VARIABLE_LIST(x), TENSOR_VARIABLE_LIST(y), 0, 0);
452
1
  ccv_nnc_dynamic_graph_evaluate(graph, linear, 1, TENSOR_VARIABLE_LIST(y), TENSOR_VARIABLE_LIST(y), 0, 0);
453
1
  REQUIRE_EQ_WITH_TOLERANCE(ccv_nnc_tensor_from_variable(graph, y)->data.f32[0], 5, 1e-2, "linear model should be trained to generate the same value as z");
454
1
  DYNAMIC_GRAPH_GEN(graph, CCV_NNC_LONG_DOT_GRAPH);
455
1
  ccv_nnc_dynamic_graph_free(graph);
456
1
  ccv_cnnp_model_free(linear);
457
1
}
458
459
TEST_CASE("dynamic graph to accumulate gradients cross cnnp models")
460
1
{
461
1
  ccv_nnc_dynamic_graph_t* const graph = ccv_nnc_dynamic_graph_new();
462
1
  ccv_cnnp_model_t* const linear = ccv_cnnp_dense(1, 1, 0);
463
1
  ccv_nnc_tensor_variable_t a = ccv_nnc_tensor_variable_new(graph, CPU_TENSOR_NHWC(32F, 1));
464
1
  ccv_nnc_dynamic_graph_exec(graph, CMD_SET_FORWARD(0.2485), ccv_nnc_no_hint, 0, TENSOR_VARIABLE_LIST(), TENSOR_VARIABLE_LIST(a), 0, 0);
465
1
  const ccv_nnc_tensor_param_t input = CPU_TENSOR_NHWC(32F, 1);
466
1
  ccv_cnnp_model_compile(linear, &input, 1, CMD_NOOP(), CMD_NOOP());
467
1
  ccv_cnnp_model_set_parameter(linear, ccv_cnnp_model_parameters(linear, CCV_CNNP_PARAMETER_SELECT_WEIGHT, 0), ccv_nnc_tensor_from_variable(graph, a));
468
1
  ccv_nnc_tensor_variable_t a_grad = ccv_nnc_tensor_variable_new(graph);
469
1
  ccv_nnc_tensor_variable_t saved_aux = ccv_nnc_tensor_variable_new(graph, CPU_TENSOR_NHWC(32F, 1));
470
1
  int i;
471
1.00k
  for (i = 0; i < 1000; 
i++1.00k
)
472
1.00k
  {
473
1.00k
    ccv_nnc_tensor_variable_t x = ccv_nnc_tensor_variable_new(graph, CPU_TENSOR_NHWC(32F, 1));
474
1.00k
    ccv_nnc_tensor_from_variable(graph, x)->data.f32[0] = i;
475
1.00k
    ccv_nnc_tensor_variable_t t = ccv_nnc_tensor_variable_new(graph, CPU_TENSOR_NHWC(32F, 1));
476
1.00k
    ccv_nnc_tensor_from_variable(graph, t)->data.f32[0] = -i;
477
1.00k
    ccv_nnc_tensor_variable_t y = ccv_nnc_tensor_variable_new(graph);
478
1.00k
    ccv_nnc_dynamic_graph_exec(graph, CMD_MUL_FORWARD(1), ccv_nnc_no_hint, 0, TENSOR_VARIABLE_LIST(x, a), TENSOR_VARIABLE_LIST(y), 0, 0);
479
1.00k
    ccv_nnc_tensor_variable_t z = ccv_nnc_tensor_variable_new(graph);
480
1.00k
    ccv_nnc_dynamic_graph_evaluate(graph, linear, 0, TENSOR_VARIABLE_LIST(y), TENSOR_VARIABLE_LIST(z), 0, 0);
481
1.00k
    ccv_nnc_tensor_variable_t n = ccv_nnc_tensor_variable_new(graph);
482
1.00k
    ccv_nnc_dynamic_graph_exec(graph, CMD_ADD_FORWARD(1, -1), ccv_nnc_no_hint, 0, TENSOR_VARIABLE_LIST(z, t), TENSOR_VARIABLE_LIST(n), 0, 0);
483
1.00k
    ccv_nnc_tensor_variable_t f = ccv_nnc_tensor_variable_new(graph);
484
1.00k
    ccv_nnc_dynamic_graph_exec(graph, CMD_MUL_FORWARD(1), ccv_nnc_no_hint, 0, TENSOR_VARIABLE_LIST(n, n), TENSOR_VARIABLE_LIST(f), 0, 0);
485
1.00k
    ccv_nnc_dynamic_graph_backward(graph, TENSOR_VARIABLE_LIST(f), 0, TENSOR_VARIABLE_LIST(a), TENSOR_VARIABLE_LIST(a_grad), 0);
486
1.00k
    ccv_nnc_tensor_variable_free(graph, n);
487
1.00k
    ccv_nnc_tensor_variable_free(graph, z);
488
1.00k
    ccv_nnc_tensor_variable_free(graph, y);
489
1.00k
    ccv_nnc_tensor_variable_free(graph, x);
490
1.00k
    ccv_nnc_tensor_variable_free(graph, t);
491
1.00k
    ccv_nnc_tensor_variable_free(graph, f);
492
1.00k
    if (((i + 1) % 5) == 0)
493
200
    {
494
200
      DYNAMIC_GRAPH_GEN(graph, CCV_NNC_LONG_DOT_GRAPH);
495
200
      REQUIRE_EQ(ccv_nnc_dynamic_graph_bookkeeping_count(graph, CCV_NNC_SYMBOL_GRAPH_EXEC), 0, "everything should be freed");
496
200
      float lr = 0.001;
497
200
      if (i >= 200)
498
160
        lr = 0.0001;
499
40
      else if (i >= 600)
500
0
        lr = 0.00001;
501
200
      ccv_cnnp_model_set_minimizer(linear, CMD_SGD_FORWARD(0, lr, 0.001, 0, 0, 0), 0, 0, 0);
502
200
      ccv_nnc_dynamic_graph_apply_gradients(graph, CMD_SGD_FORWARD(0, lr, 0.001, 0, 0, 0), TENSOR_VARIABLE_LIST(a_grad), TENSOR_VARIABLE_LIST(a), &saved_aux, 0, 0);
503
200
    }
504
1.00k
  }
505
1
  ccv_nnc_tensor_variable_t x = ccv_nnc_tensor_variable_new(graph, CPU_TENSOR_NHWC(32F, 1));
506
1
  ccv_nnc_tensor_from_variable(graph, x)->data.f32[0] = 5;
507
1
  ccv_nnc_tensor_variable_t y = ccv_nnc_tensor_variable_new(graph);
508
1
  ccv_nnc_dynamic_graph_exec(graph, CMD_MUL_FORWARD(1), ccv_nnc_no_hint, 0, TENSOR_VARIABLE_LIST(x, a), TENSOR_VARIABLE_LIST(y), 0, 0);
509
1
  ccv_nnc_tensor_variable_t z = ccv_nnc_tensor_variable_new(graph);
510
1
  ccv_nnc_dynamic_graph_evaluate(graph, linear, 0, TENSOR_VARIABLE_LIST(y), TENSOR_VARIABLE_LIST(z), 0, 0);
511
1
  REQUIRE_EQ_WITH_TOLERANCE(ccv_nnc_tensor_from_variable(graph, z)->data.f32[0], -5, 1e-2, "linear model should be trained to generate the same value as z");
512
1
  ccv_nnc_dynamic_graph_free(graph);
513
1
  ccv_cnnp_model_free(linear);
514
1
}
515
516
TEST_CASE("dynamic graph to accumulate gradients cross cnnp models with aliases")
517
1
{
518
1
  ccv_nnc_dynamic_graph_t* const graph = ccv_nnc_dynamic_graph_new();
519
1
  ccv_cnnp_model_t* const linear = ccv_cnnp_sequential_new(MODEL_LIST(
520
1
    ccv_cnnp_dense(1, 1, 0),
521
1
    ccv_cnnp_reshape(DIM_ALLOC(1), DIM_ALLOC(), DIM_ALLOC(), 0),
522
1
  ), 0);
523
1
  ccv_nnc_tensor_variable_t a = ccv_nnc_tensor_variable_new(graph, CPU_TENSOR_NHWC(32F, 1));
524
1
  ccv_nnc_dynamic_graph_exec(graph, CMD_SET_FORWARD(0.2485), ccv_nnc_no_hint, 0, TENSOR_VARIABLE_LIST(), TENSOR_VARIABLE_LIST(a), 0, 0);
525
1
  const ccv_nnc_tensor_param_t input = CPU_TENSOR_NHWC(32F, 1);
526
1
  ccv_cnnp_model_compile(linear, &input, 1, CMD_NOOP(), CMD_NOOP());
527
1
  ccv_cnnp_model_set_parameter(linear, ccv_cnnp_model_parameters(linear, CCV_CNNP_PARAMETER_SELECT_WEIGHT, 0), ccv_nnc_tensor_from_variable(graph, a));
528
1
  ccv_nnc_tensor_variable_t a_grad = ccv_nnc_tensor_variable_new(graph);
529
1
  ccv_nnc_tensor_variable_t saved_aux = ccv_nnc_tensor_variable_new(graph, CPU_TENSOR_NHWC(32F, 1));
530
1
  int i;
531
1.00k
  for (i = 0; i < 1000; 
i++1.00k
)
532
1.00k
  {
533
1.00k
    ccv_nnc_tensor_variable_t x = ccv_nnc_tensor_variable_new(graph, CPU_TENSOR_NHWC(32F, 1));
534
1.00k
    ccv_nnc_tensor_from_variable(graph, x)->data.f32[0] = i;
535
1.00k
    ccv_nnc_tensor_variable_t t = ccv_nnc_tensor_variable_new(graph, CPU_TENSOR_NHWC(32F, 1));
536
1.00k
    ccv_nnc_tensor_from_variable(graph, t)->data.f32[0] = -i;
537
1.00k
    ccv_nnc_tensor_variable_t y = ccv_nnc_tensor_variable_new(graph);
538
1.00k
    ccv_nnc_dynamic_graph_exec(graph, CMD_MUL_FORWARD(1), ccv_nnc_no_hint, 0, TENSOR_VARIABLE_LIST(x, a), TENSOR_VARIABLE_LIST(y), 0, 0);
539
1.00k
    ccv_nnc_tensor_variable_t y_alias = ccv_nnc_tensor_variable_alias_new(graph, y, DIM_ALLOC(), DIM_ALLOC(1), CPU_TENSOR_NHWC(32F, 1));
540
1.00k
    ccv_nnc_tensor_variable_t z = ccv_nnc_tensor_variable_new(graph);
541
1.00k
    ccv_nnc_dynamic_graph_evaluate(graph, linear, 0, TENSOR_VARIABLE_LIST(y_alias), TENSOR_VARIABLE_LIST(z), 0, 0);
542
1.00k
    ccv_nnc_tensor_variable_t n = ccv_nnc_tensor_variable_new(graph);
543
1.00k
    ccv_nnc_dynamic_graph_exec(graph, CMD_ADD_FORWARD(1, -1), ccv_nnc_no_hint, 0, TENSOR_VARIABLE_LIST(z, t), TENSOR_VARIABLE_LIST(n), 0, 0);
544
1.00k
    ccv_nnc_tensor_variable_t f = ccv_nnc_tensor_variable_new(graph);
545
1.00k
    ccv_nnc_dynamic_graph_exec(graph, CMD_MUL_FORWARD(1), ccv_nnc_no_hint, 0, TENSOR_VARIABLE_LIST(n, n), TENSOR_VARIABLE_LIST(f), 0, 0);
546
1.00k
    ccv_nnc_dynamic_graph_backward(graph, TENSOR_VARIABLE_LIST(f), 0, TENSOR_VARIABLE_LIST(a), TENSOR_VARIABLE_LIST(a_grad), 0);
547
1.00k
    ccv_nnc_tensor_variable_free(graph, n);
548
1.00k
    ccv_nnc_tensor_variable_free(graph, z);
549
1.00k
    ccv_nnc_tensor_variable_free(graph, y);
550
1.00k
    ccv_nnc_tensor_variable_free(graph, y_alias);
551
1.00k
    ccv_nnc_tensor_variable_free(graph, x);
552
1.00k
    ccv_nnc_tensor_variable_free(graph, t);
553
1.00k
    ccv_nnc_tensor_variable_free(graph, f);
554
1.00k
    if (((i + 1) % 5) == 0)
555
200
    {
556
200
      REQUIRE_EQ(ccv_nnc_dynamic_graph_bookkeeping_count(graph, CCV_NNC_SYMBOL_GRAPH_EXEC), 0, "everything should be freed");
557
200
      DYNAMIC_GRAPH_GEN(graph, CCV_NNC_LONG_DOT_GRAPH);
558
200
      float lr = 0.001;
559
200
      if (i >= 200)
560
160
        lr = 0.0001;
561
40
      else if (i >= 600)
562
0
        lr = 0.00001;
563
200
      ccv_cnnp_model_set_minimizer(linear, CMD_SGD_FORWARD(0, lr, 0.001, 0, 0, 0), 0, 0, 0);
564
200
      ccv_nnc_dynamic_graph_apply_gradients(graph, CMD_SGD_FORWARD(0, lr, 0.001, 0, 0, 0), TENSOR_VARIABLE_LIST(a_grad), TENSOR_VARIABLE_LIST(a), &saved_aux, 0, 0);
565
200
    }
566
1.00k
  }
567
1
  ccv_nnc_tensor_variable_t x = ccv_nnc_tensor_variable_new(graph, CPU_TENSOR_NHWC(32F, 1));
568
1
  ccv_nnc_tensor_from_variable(graph, x)->data.f32[0] = 5;
569
1
  ccv_nnc_tensor_variable_t y = ccv_nnc_tensor_variable_new(graph);
570
1
  ccv_nnc_dynamic_graph_exec(graph, CMD_MUL_FORWARD(1), ccv_nnc_no_hint, 0, TENSOR_VARIABLE_LIST(x, a), TENSOR_VARIABLE_LIST(y), 0, 0);
571
1
  ccv_nnc_tensor_variable_t z = ccv_nnc_tensor_variable_new(graph);
572
1
  ccv_nnc_dynamic_graph_evaluate(graph, linear, 0, TENSOR_VARIABLE_LIST(y), TENSOR_VARIABLE_LIST(z), 0, 0);
573
1
  REQUIRE_EQ_WITH_TOLERANCE(ccv_nnc_tensor_from_variable(graph, z)->data.f32[0], -5, 1e-2, "linear model should be trained to generate the same value as z");
574
1
  ccv_nnc_dynamic_graph_free(graph);
575
1
  ccv_cnnp_model_free(linear);
576
1
}
577
578
TEST_CASE("dynamic graph to compute f(x) = x * log(x) + 1.2 * x, f'(x) and sum on x = 19, 10")
579
1
{
580
1
  ccv_nnc_dynamic_graph_t* const graph = ccv_nnc_dynamic_graph_new();
581
1
  ccv_nnc_tensor_variable_t x = ccv_nnc_tensor_variable_new(graph, CPU_TENSOR_NHWC(32F, 1));
582
1
  ccv_nnc_tensor_from_variable(graph, x)->data.f32[0] = 19;
583
1
  ccv_nnc_tensor_variable_t f = ccv_nnc_tensor_variable_new(graph);
584
1
  ccv_nnc_dynamic_graph_exec(graph, CMD_EWLOG_FORWARD(), ccv_nnc_no_hint, 0, TENSOR_VARIABLE_LIST(x), TENSOR_VARIABLE_LIST(f), 0, 0);
585
1
  ccv_nnc_dynamic_graph_exec(graph, CMD_EWPROD_FORWARD(), ccv_nnc_no_hint, 0, TENSOR_VARIABLE_LIST(x, f), TENSOR_VARIABLE_LIST(f), 0, 0);
586
1
  ccv_nnc_tensor_variable_t y = ccv_nnc_tensor_variable_new(graph, CPU_TENSOR_NHWC(32F, 1));
587
1
  ccv_nnc_tensor_from_variable(graph, y)->data.f32[0] = 1.2;
588
1
  ccv_nnc_tensor_variable_t z = ccv_nnc_tensor_variable_new(graph);
589
1
  ccv_nnc_dynamic_graph_exec(graph, CMD_EWPROD_FORWARD(), ccv_nnc_no_hint, 0, TENSOR_VARIABLE_LIST(x, y), TENSOR_VARIABLE_LIST(z), 0, 0);
590
1
  ccv_nnc_dynamic_graph_exec(graph, CMD_EWSUM_FORWARD(), ccv_nnc_no_hint, 0, TENSOR_VARIABLE_LIST(f, z), TENSOR_VARIABLE_LIST(f), 0, 0);
591
1
  REQUIRE_EQ_WITH_TOLERANCE(ccv_nnc_tensor_from_variable(graph, f)->data.f32[0], 19 * logf(19) + 1.2 * 19, 1e-5, "f(x) = 1.2 * 19 + 19 * log(19)");
592
1
  // Do gradient computation multiple times.
593
1
  ccv_nnc_tensor_variable_t dx = ccv_nnc_tensor_variable_new(graph);
594
1
  ccv_nnc_dynamic_graph_backward(graph, TENSOR_VARIABLE_LIST(f), 0, TENSOR_VARIABLE_LIST(x), TENSOR_VARIABLE_LIST(dx), 0);
595
1
  REQUIRE_EQ_WITH_TOLERANCE(ccv_nnc_tensor_from_variable(graph, dx)->data.f32[0], logf(19) + 1 + 1.2, 1e-5, "f'(x) = 1.2 + log(19) + 19 * 1 / 19");
596
1
  ccv_nnc_tensor_variable_free(graph, x);
597
1
  x = ccv_nnc_tensor_variable_new(graph, CPU_TENSOR_NHWC(32F, 1));
598
1
  ccv_nnc_tensor_from_variable(graph, x)->data.f32[0] = 10;
599
1
  ccv_nnc_dynamic_graph_exec(graph, CMD_EWLOG_FORWARD(), ccv_nnc_no_hint, 0, TENSOR_VARIABLE_LIST(x), TENSOR_VARIABLE_LIST(f), 0, 0);
600
1
  ccv_nnc_dynamic_graph_exec(graph, CMD_EWPROD_FORWARD(), ccv_nnc_no_hint, 0, TENSOR_VARIABLE_LIST(x, f), TENSOR_VARIABLE_LIST(f), 0, 0);
601
1
  ccv_nnc_tensor_from_variable(graph, y)->data.f32[0] = 1.2;
602
1
  ccv_nnc_dynamic_graph_exec(graph, CMD_EWPROD_FORWARD(), ccv_nnc_no_hint, 0, TENSOR_VARIABLE_LIST(x, y), TENSOR_VARIABLE_LIST(z), 0, 0);
603
1
  ccv_nnc_dynamic_graph_exec(graph, CMD_EWSUM_FORWARD(), ccv_nnc_no_hint, 0, TENSOR_VARIABLE_LIST(f, z), TENSOR_VARIABLE_LIST(f), 0, 0);
604
1
  REQUIRE_EQ_WITH_TOLERANCE(ccv_nnc_tensor_from_variable(graph, f)->data.f32[0], 10 * logf(10) + 1.2 * 10, 1e-5, "f(x) = 1.2 * 10 + 10 * log(10)");
605
1
  ccv_nnc_dynamic_graph_backward(graph, TENSOR_VARIABLE_LIST(f), 0, TENSOR_VARIABLE_LIST(x), TENSOR_VARIABLE_LIST(dx), 0);
606
1
  REQUIRE_EQ_WITH_TOLERANCE(ccv_nnc_tensor_from_variable(graph, dx)->data.f32[0], logf(19) + 1 + 1.2 + logf(10) + 1 + 1.2, 1e-5, "f'(x) = 1.2 + log(19) + 19 * 1 / 19 + 1.2 + log(19) + 19 * 1 / 19");
607
1
  DYNAMIC_GRAPH_GEN(graph, CCV_NNC_LONG_DOT_GRAPH);
608
1
  ccv_nnc_dynamic_graph_free(graph);
609
1
}
610
611
TEST_CASE("dynamic graph to compute f(x) = x * log(x) + y' where y = 1.2 * x and y' is an alias to y")
612
1
{
613
1
  ccv_nnc_dynamic_graph_t* const graph = ccv_nnc_dynamic_graph_new();
614
1
  ccv_nnc_tensor_variable_t x = ccv_nnc_tensor_variable_new(graph, CPU_TENSOR_NHWC(32F, 1));
615
1
  ccv_nnc_tensor_from_variable(graph, x)->data.f32[0] = 19;
616
1
  ccv_nnc_tensor_variable_t f = ccv_nnc_tensor_variable_new(graph);
617
1
  ccv_nnc_dynamic_graph_exec(graph, CMD_EWLOG_FORWARD(), ccv_nnc_no_hint, 0, TENSOR_VARIABLE_LIST(x), TENSOR_VARIABLE_LIST(f), 0, 0);
618
1
  ccv_nnc_dynamic_graph_exec(graph, CMD_EWPROD_FORWARD(), ccv_nnc_no_hint, 0, TENSOR_VARIABLE_LIST(x, f), TENSOR_VARIABLE_LIST(f), 0, 0);
619
1
  ccv_nnc_tensor_variable_t z = ccv_nnc_tensor_variable_new(graph, CPU_TENSOR_NHWC(32F, 1));
620
1
  ccv_nnc_tensor_from_variable(graph, z)->data.f32[0] = 1.2;
621
1
  ccv_nnc_tensor_variable_t y = ccv_nnc_tensor_variable_new(graph, CPU_TENSOR_NHWC(32F, 1));
622
1
  ccv_nnc_tensor_variable_t y_hat = ccv_nnc_tensor_variable_alias_new(graph, y, ccv_nnc_no_ofs, DIM_ALLOC(), CPU_TENSOR_NHWC(32F, 1));
623
1
  ccv_nnc_dynamic_graph_exec(graph, CMD_EWPROD_FORWARD(), ccv_nnc_no_hint, 0, TENSOR_VARIABLE_LIST(x, z), TENSOR_VARIABLE_LIST(y), 0, 0);
624
1
  ccv_nnc_dynamic_graph_exec(graph, CMD_EWSUM_FORWARD(), ccv_nnc_no_hint, 0, TENSOR_VARIABLE_LIST(f, y_hat), TENSOR_VARIABLE_LIST(f), 0, 0);
625
1
  REQUIRE_EQ_WITH_TOLERANCE(ccv_nnc_tensor_from_variable(graph, f)->data.f32[0], 19 * logf(19) + 1.2 * 19, 1e-5, "f(x) = 1.2 * 19 + 19 * log(19)");
626
1
  ccv_nnc_tensor_variable_free(graph, y_hat);
627
1
  ccv_nnc_tensor_variable_free(graph, f);
628
1
  ccv_nnc_tensor_variable_free(graph, x);
629
1
  ccv_nnc_tensor_variable_free(graph, z);
630
1
  ccv_nnc_tensor_variable_free(graph, y);
631
1
  REQUIRE_EQ(ccv_nnc_dynamic_graph_bookkeeping_count(graph, CCV_NNC_SYMBOL_TENSOR), 0, "should have no tensor left");
632
1
  ccv_nnc_dynamic_graph_free(graph);
633
1
}
634
635
TEST_CASE("dynamic graph to compute f(x) = x * log(x) + y' where y'' = 1.2 * x and both y'' and y' are aliases to y")
636
1
{
637
1
  ccv_nnc_dynamic_graph_t* const graph = ccv_nnc_dynamic_graph_new();
638
1
  ccv_nnc_tensor_variable_t x = ccv_nnc_tensor_variable_new(graph, CPU_TENSOR_NHWC(32F, 1));
639
1
  ccv_nnc_tensor_from_variable(graph, x)->data.f32[0] = 19;
640
1
  ccv_nnc_tensor_variable_t f = ccv_nnc_tensor_variable_new(graph);
641
1
  ccv_nnc_dynamic_graph_exec(graph, CMD_EWLOG_FORWARD(), ccv_nnc_no_hint, 0, TENSOR_VARIABLE_LIST(x), TENSOR_VARIABLE_LIST(f), 0, 0);
642
1
  ccv_nnc_dynamic_graph_exec(graph, CMD_EWPROD_FORWARD(), ccv_nnc_no_hint, 0, TENSOR_VARIABLE_LIST(x, f), TENSOR_VARIABLE_LIST(f), 0, 0);
643
1
  ccv_nnc_tensor_variable_t z = ccv_nnc_tensor_variable_new(graph, CPU_TENSOR_NHWC(32F, 1));
644
1
  ccv_nnc_tensor_from_variable(graph, z)->data.f32[0] = 1.2;
645
1
  ccv_nnc_tensor_variable_t y = ccv_nnc_tensor_variable_new(graph, CPU_TENSOR_NHWC(32F, 1));
646
1
  ccv_nnc_tensor_variable_t y_hat = ccv_nnc_tensor_variable_alias_new(graph, y, ccv_nnc_no_ofs, DIM_ALLOC(), CPU_TENSOR_NHWC(32F, 1));
647
1
  ccv_nnc_tensor_variable_t y_double_hat = ccv_nnc_tensor_variable_alias_new(graph, y, ccv_nnc_no_ofs, DIM_ALLOC(), CPU_TENSOR_NHWC(32F, 1));
648
1
  ccv_nnc_dynamic_graph_exec(graph, CMD_EWPROD_FORWARD(), ccv_nnc_no_hint, 0, TENSOR_VARIABLE_LIST(x, z), TENSOR_VARIABLE_LIST(y_double_hat), 0, 0);
649
1
  ccv_nnc_dynamic_graph_exec(graph, CMD_EWSUM_FORWARD(), ccv_nnc_no_hint, 0, TENSOR_VARIABLE_LIST(f, y_hat), TENSOR_VARIABLE_LIST(f), 0, 0);
650
1
  REQUIRE_EQ_WITH_TOLERANCE(ccv_nnc_tensor_from_variable(graph, f)->data.f32[0], 19 * logf(19) + 1.2 * 19, 1e-5, "f(x) = 1.2 * 19 + 19 * log(19)");
651
1
  ccv_nnc_tensor_variable_free(graph, y_hat);
652
1
  ccv_nnc_tensor_variable_free(graph, y_double_hat);
653
1
  ccv_nnc_tensor_variable_free(graph, f);
654
1
  ccv_nnc_tensor_variable_free(graph, x);
655
1
  ccv_nnc_tensor_variable_free(graph, z);
656
1
  ccv_nnc_tensor_variable_free(graph, y);
657
1
  REQUIRE_EQ(ccv_nnc_dynamic_graph_bookkeeping_count(graph, CCV_NNC_SYMBOL_TENSOR), 0, "should have no tensor left");
658
1
  ccv_nnc_dynamic_graph_free(graph);
659
1
}
660
661
TEST_CASE("dynamic graph to compute f(x) = x * y, where y[0] = 10 * z, y[1] = 2 * z, z = [2], x = [10], should free all variables")
662
1
{
663
1
  ccv_nnc_dynamic_graph_t* const graph = ccv_nnc_dynamic_graph_new();
664
1
  ccv_nnc_tensor_variable_t x = ccv_nnc_tensor_variable_new(graph, CPU_TENSOR_NHWC(32F, 1));
665
1
  ccv_nnc_tensor_variable_t y = ccv_nnc_tensor_variable_new(graph, CPU_TENSOR_NHWC(32F, 2));
666
1
  ccv_nnc_tensor_variable_t z = ccv_nnc_tensor_variable_new(graph, CPU_TENSOR_NHWC(32F, 1));
667
1
  ccv_nnc_dynamic_graph_exec(graph, CMD_SET_FORWARD(2), ccv_nnc_no_hint, 0, TENSOR_VARIABLE_LIST(), TENSOR_VARIABLE_LIST(z), 0, 0);
668
1
  ccv_nnc_dynamic_graph_exec(graph, CMD_SET_FORWARD(10), ccv_nnc_no_hint, 0, TENSOR_VARIABLE_LIST(), TENSOR_VARIABLE_LIST(x), 0, 0);
669
1
  ccv_nnc_tensor_variable_t y_1 = ccv_nnc_tensor_variable_alias_new(graph, y, ccv_nnc_no_ofs, DIM_ALLOC(2), CPU_TENSOR_NHWC(32F, 1));
670
1
  ccv_nnc_tensor_variable_t y_2 = ccv_nnc_tensor_variable_alias_new(graph, y, DIM_ALLOC(1), DIM_ALLOC(2), CPU_TENSOR_NHWC(32F, 1));
671
1
  ccv_nnc_dynamic_graph_exec(graph, CMD_SCALAR_MUL_FORWARD(10), ccv_nnc_no_hint, 0, TENSOR_VARIABLE_LIST(z), TENSOR_VARIABLE_LIST(y_1), 0, 0);
672
1
  ccv_nnc_dynamic_graph_exec(graph, CMD_SCALAR_MUL_FORWARD(2), ccv_nnc_no_hint, 0, TENSOR_VARIABLE_LIST(z), TENSOR_VARIABLE_LIST(y_2), 0, 0);
673
1
  ccv_nnc_tensor_variable_t f = ccv_nnc_tensor_variable_new(graph);
674
1
  ccv_nnc_dynamic_graph_exec(graph, CMD_MUL_FORWARD(1), ccv_nnc_no_hint, 0, TENSOR_VARIABLE_LIST(x, y), TENSOR_VARIABLE_LIST(f), 0, 0);
675
1
  float gt[] = {10 * 2 * 10, 2 * 2 * 10};
676
1
  REQUIRE_ARRAY_EQ_WITH_TOLERANCE(float, ccv_nnc_tensor_from_variable(graph, f)->data.f32, gt, 2, 1e-5, "should be equal");
677
1
  ccv_nnc_tensor_variable_free(graph, z);
678
1
  ccv_nnc_tensor_variable_free(graph, y_1);
679
1
  ccv_nnc_tensor_variable_free(graph, y_2);
680
1
  ccv_nnc_tensor_variable_free(graph, x);
681
1
  ccv_nnc_tensor_variable_free(graph, y);
682
1
  ccv_nnc_tensor_variable_free(graph, f);
683
1
  REQUIRE_EQ(ccv_nnc_dynamic_graph_bookkeeping_count(graph, CCV_NNC_SYMBOL_GRAPH_EXEC), 0, "everything should be freed");
684
1
  REQUIRE_EQ(ccv_nnc_dynamic_graph_bookkeeping_count(graph, CCV_NNC_SYMBOL_TENSOR), 0, "everything should be freed");
685
1
  ccv_nnc_dynamic_graph_free(graph);
686
1
}
687
688
TEST_CASE("dynamic graph to compute f(x) = x * y, where y[0] = 10 * z, y[1] = 2 * z, z = [2], x = [10], freed y[0], y[1] before use")
689
1
{
690
1
  ccv_nnc_dynamic_graph_t* const graph = ccv_nnc_dynamic_graph_new();
691
1
  ccv_nnc_tensor_variable_t x = ccv_nnc_tensor_variable_new(graph, CPU_TENSOR_NHWC(32F, 1));
692
1
  ccv_nnc_tensor_variable_t y = ccv_nnc_tensor_variable_new(graph, CPU_TENSOR_NHWC(32F, 2));
693
1
  ccv_nnc_tensor_variable_t z = ccv_nnc_tensor_variable_new(graph, CPU_TENSOR_NHWC(32F, 1));
694
1
  ccv_nnc_dynamic_graph_exec(graph, CMD_SET_FORWARD(2), ccv_nnc_no_hint, 0, TENSOR_VARIABLE_LIST(), TENSOR_VARIABLE_LIST(z), 0, 0);
695
1
  ccv_nnc_dynamic_graph_exec(graph, CMD_SET_FORWARD(10), ccv_nnc_no_hint, 0, TENSOR_VARIABLE_LIST(), TENSOR_VARIABLE_LIST(x), 0, 0);
696
1
  ccv_nnc_tensor_variable_t y_1 = ccv_nnc_tensor_variable_alias_new(graph, y, ccv_nnc_no_ofs, DIM_ALLOC(2), CPU_TENSOR_NHWC(32F, 1));
697
1
  ccv_nnc_tensor_variable_t y_2 = ccv_nnc_tensor_variable_alias_new(graph, y, DIM_ALLOC(1), DIM_ALLOC(2), CPU_TENSOR_NHWC(32F, 1));
698
1
  ccv_nnc_dynamic_graph_exec(graph, CMD_SCALAR_MUL_FORWARD(10), ccv_nnc_no_hint, 0, TENSOR_VARIABLE_LIST(z), TENSOR_VARIABLE_LIST(y_1), 0, 0);
699
1
  ccv_nnc_dynamic_graph_exec(graph, CMD_SCALAR_MUL_FORWARD(2), ccv_nnc_no_hint, 0, TENSOR_VARIABLE_LIST(z), TENSOR_VARIABLE_LIST(y_2), 0, 0);
700
1
  ccv_nnc_tensor_variable_free(graph, y_1);
701
1
  ccv_nnc_tensor_variable_free(graph, y_2);
702
1
  ccv_nnc_tensor_variable_t f = ccv_nnc_tensor_variable_new(graph);
703
1
  ccv_nnc_dynamic_graph_exec(graph, CMD_MUL_FORWARD(1), ccv_nnc_no_hint, 0, TENSOR_VARIABLE_LIST(x, y), TENSOR_VARIABLE_LIST(f), 0, 0);
704
1
  REQUIRE_EQ(ccv_nnc_dynamic_graph_bookkeeping_count(graph, CCV_NNC_SYMBOL_GRAPH_EXEC), 3, "should keep 3 ops");
705
1
  float gt[] = {10 * 2 * 10, 2 * 2 * 10};
706
1
  REQUIRE_ARRAY_EQ_WITH_TOLERANCE(float, ccv_nnc_tensor_from_variable(graph, f)->data.f32, gt, 2, 1e-5, "should be equal");
707
1
  ccv_nnc_tensor_variable_free(graph, z);
708
1
  ccv_nnc_tensor_variable_free(graph, x);
709
1
  ccv_nnc_tensor_variable_free(graph, y);
710
1
  ccv_nnc_tensor_variable_free(graph, f);
711
1
  REQUIRE_EQ(ccv_nnc_dynamic_graph_bookkeeping_count(graph, CCV_NNC_SYMBOL_GRAPH_EXEC), 0, "everything should be freed");
712
1
  REQUIRE_EQ(ccv_nnc_dynamic_graph_bookkeeping_count(graph, CCV_NNC_SYMBOL_TENSOR), 0, "everything should be freed");
713
1
  ccv_nnc_dynamic_graph_free(graph);
714
1
}
715
716
TEST_CASE("long chain to autograd, simulate random free of unused tensors due to garbage collection")
717
1
{
718
1
  ccv_nnc_dynamic_graph_t* const graph = ccv_nnc_dynamic_graph_new();
719
1
  ccv_nnc_tensor_variable_t x[10];
720
1
  ccv_nnc_tensor_variable_t f[10];
721
1
  ccv_nnc_tensor_variable_t y = ccv_nnc_tensor_variable_new(graph, CPU_TENSOR_NHWC(32F, 1));
722
1
  ccv_nnc_tensor_variable_t dy[3];
723
1
  ccv_nnc_tensor_from_variable(graph, y)->data.f32[0] = 1.3;
724
1
  int i, j;
725
11
  for (i = 0; i < 10; 
i++10
)
726
10
  {
727
10
    x[i] = ccv_nnc_tensor_variable_new(graph, CPU_TENSOR_NHWC(32F, 1));
728
10
    ccv_nnc_tensor_from_variable(graph, x[i])->data.f32[0] = i + 1;
729
10
    f[i] = ccv_nnc_tensor_variable_new(graph);
730
10
    ccv_nnc_dynamic_graph_exec(graph, CMD_MUL_FORWARD(1), ccv_nnc_no_hint, 0, TENSOR_VARIABLE_LIST(x[i], y), TENSOR_VARIABLE_LIST(f[i]), 0, 0);
731
10
    if (i >= 5)
732
5
    {
733
5
      if (i - 5 < 3)
734
3
        dy[i - 5] = ccv_nnc_tensor_variable_new(graph);
735
5
      ccv_nnc_dynamic_graph_backward(graph, TENSOR_VARIABLE_LIST(f[i]), 0, TENSOR_VARIABLE_LIST(y), TENSOR_VARIABLE_LIST(dy[ccv_min(2, i - 5)]), 0);
736
5
    }
737
10
    if (i == 7)
738
6
      
for (j = 0; 1
j < 5;
j++5
)
739
5
        ccv_nnc_tensor_variable_free(graph, f[j]);
740
10
  }
741
1
  REQUIRE_EQ_WITH_TOLERANCE(ccv_nnc_tensor_from_variable(graph, f[9])->data.f32[0], 1.3 * 10, 1e-5, "should equal to 1.3 * 10");
742
1
  REQUIRE_EQ_WITH_TOLERANCE(ccv_nnc_tensor_from_variable(graph, dy[2])->data.f32[0], 8 + 9 + 10, 1e-5, "should equal to sum of the last 3");
743
11
  for (i = 0; i < 10; 
i++10
)
744
10
    ccv_nnc_tensor_variable_free(graph, x[i]);
745
1
  ccv_nnc_dynamic_graph_free(graph);
746
1
}
747
748
TEST_CASE("parallel exec with a stream, focus on uma tensors")
749
1
{
750
1
  ccv_nnc_dynamic_graph_t* const graph = ccv_nnc_dynamic_graph_new();
751
1
  ccv_nnc_stream_context_t* const stream = ccv_nnc_stream_context_new(CCV_STREAM_CONTEXT_CPU);
752
1
  ccv_nnc_tensor_variable_t x[2];
753
1
  x[0] = ccv_nnc_tensor_variable_new(graph, CPU_TENSOR_NHWC(32F, 1));
754
1
  x[1] = ccv_nnc_tensor_variable_new(graph, CPU_TENSOR_NHWC(32F, 1));
755
1
  ccv_nnc_tensor_from_variable(graph, x[0])->data.f32[0] = 10;
756
1
  ccv_nnc_tensor_from_variable(graph, x[1])->data.f32[0] = 9;
757
1
  ccv_nnc_tensor_variable_t y[2];
758
1
  y[0] = ccv_nnc_tensor_variable_new(graph);
759
1
  y[1] = ccv_nnc_tensor_variable_new(graph);
760
1
  ccv_nnc_dynamic_graph_exec(graph, CMD_SCALAR_MUL_FORWARD(0.4), ccv_nnc_no_hint, 0, TENSOR_VARIABLE_LIST(x[0], x[1]), TENSOR_VARIABLE_LIST(y[0], y[1]), 2, stream);
761
1
  ccv_nnc_stream_context_wait(stream);
762
1
  ccv_nnc_stream_context_free(stream);
763
1
  REQUIRE_EQ_WITH_TOLERANCE(ccv_nnc_tensor_from_variable(graph, y[0])->data.f32[0], 10 * 0.4, 1e-5, "should match the result");
764
1
  REQUIRE_EQ_WITH_TOLERANCE(ccv_nnc_tensor_from_variable(graph, y[1])->data.f32[0], 9 * 0.4, 1e-5, "should match the result");
765
1
  ccv_nnc_dynamic_graph_free(graph);
766
1
}
767
768
TEST_CASE("parallel exec with a stream, focus on potential memory issues")
769
1
{
770
1
  ccv_nnc_dynamic_graph_t* const graph = ccv_nnc_dynamic_graph_new();
771
1
  ccv_nnc_stream_context_t* const stream = ccv_nnc_stream_context_new(CCV_STREAM_CONTEXT_CPU);
772
1
  ccv_nnc_tensor_variable_t x[2];
773
1
  x[0] = ccv_nnc_tensor_variable_new(graph, CPU_NUMA_TENSOR_NHWC(000, 32F, 1));
774
1
  x[1] = ccv_nnc_tensor_variable_new(graph, CPU_NUMA_TENSOR_NHWC(000, 32F, 1));
775
1
  ccv_nnc_tensor_from_variable(graph, x[0])->data.f32[0] = 10;
776
1
  ccv_nnc_tensor_from_variable(graph, x[1])->data.f32[0] = 9;
777
1
  ccv_nnc_tensor_variable_t y[2];
778
1
  y[0] = ccv_nnc_tensor_variable_new(graph);
779
1
  y[1] = ccv_nnc_tensor_variable_new(graph);
780
1
  ccv_nnc_dynamic_graph_exec(graph, CMD_SCALAR_MUL_FORWARD(0.4), ccv_nnc_no_hint, 0, TENSOR_VARIABLE_LIST(x[0], x[1]), TENSOR_VARIABLE_LIST(y[0], y[1]), 2, stream);
781
1
  ccv_nnc_stream_context_wait(stream);
782
1
  ccv_nnc_stream_context_free(stream);
783
1
  REQUIRE_EQ_WITH_TOLERANCE(ccv_nnc_tensor_from_variable(graph, y[0])->data.f32[0], 10 * 0.4, 1e-5, "should match the result");
784
1
  REQUIRE_EQ_WITH_TOLERANCE(ccv_nnc_tensor_from_variable(graph, y[1])->data.f32[0], 9 * 0.4, 1e-5, "should match the result");
785
1
  ccv_nnc_dynamic_graph_free(graph);
786
1
}
787
788
TEST_CASE("query whether a tensor variable depends on another")
789
1
{
790
1
  ccv_nnc_dynamic_graph_t* const graph = ccv_nnc_dynamic_graph_new();
791
1
  ccv_nnc_tensor_variable_t a = ccv_nnc_tensor_variable_new(graph, CPU_TENSOR_NHWC(32F, 1));
792
1
  ccv_nnc_tensor_from_variable(graph, a)->data.f32[0] = 10;
793
1
  ccv_nnc_tensor_variable_t b = ccv_nnc_tensor_variable_new(graph);
794
1
  ccv_nnc_dynamic_graph_exec(graph, CMD_SCALAR_MUL_FORWARD(0.5), ccv_nnc_no_hint, 0, TENSOR_VARIABLE_LIST(a), TENSOR_VARIABLE_LIST(b), 0, 0);
795
1
  ccv_nnc_tensor_variable_t c = ccv_nnc_tensor_variable_new(graph, CPU_TENSOR_NHWC(32F, 1));
796
1
  ccv_nnc_tensor_from_variable(graph, c)->data.f32[0] = 9;
797
1
  ccv_nnc_tensor_variable_t d = ccv_nnc_tensor_variable_new(graph);
798
1
  ccv_nnc_dynamic_graph_exec(graph, CMD_SCALAR_MUL_FORWARD(0.4), ccv_nnc_no_hint, 0, TENSOR_VARIABLE_LIST(c), TENSOR_VARIABLE_LIST(d), 0, 0);
799
1
  ccv_nnc_tensor_variable_t e = ccv_nnc_tensor_variable_new(graph);
800
1
  ccv_nnc_dynamic_graph_exec(graph, CMD_SCALAR_MUL_FORWARD(2), ccv_nnc_no_hint, 0, TENSOR_VARIABLE_LIST(d), TENSOR_VARIABLE_LIST(e), 0, 0);
801
1
  ccv_nnc_tensor_variable_t f = ccv_nnc_tensor_variable_new(graph);
802
1
  ccv_nnc_dynamic_graph_exec(graph, CMD_EWSUM_FORWARD(), ccv_nnc_no_hint, 0, TENSOR_VARIABLE_LIST(a, c), TENSOR_VARIABLE_LIST(f), 0, 0);
803
1
  ccv_nnc_tensor_variable_t g = ccv_nnc_tensor_variable_new(graph);
804
1
  ccv_nnc_dynamic_graph_exec(graph, CMD_EWSUM_FORWARD(), ccv_nnc_no_hint, 0, TENSOR_VARIABLE_LIST(e, b), TENSOR_VARIABLE_LIST(g), 0, 0);
805
1
  REQUIRE_EQ_WITH_TOLERANCE(ccv_nnc_tensor_from_variable(graph, f)->data.f32[0], 10 + 9, 1e-5, "should match the result");
806
1
  REQUIRE_EQ_WITH_TOLERANCE(ccv_nnc_tensor_from_variable(graph, g)->data.f32[0], 10 * 0.5 + 9 * 0.4 * 2, 1e-5, "should match the result");
807
1
  uint64_t bitmask = 1;
808
1
  ccv_nnc_dynamic_graph_has_effect_to_tensor_variables(graph, TENSOR_VARIABLE_LIST(a, b, f), TENSOR_VARIABLE_LIST(g), &bitmask);
809
1
  REQUIRE(bitmask == 3, "a and b should be ancestors to g");
810
1
  bitmask = 8;
811
1
  ccv_nnc_dynamic_graph_has_effect_to_tensor_variables(graph, TENSOR_VARIABLE_LIST(a, e, c, f), TENSOR_VARIABLE_LIST(g), &bitmask);
812
1
  REQUIRE(bitmask == 7, "a, e and c should be ancestors to g");
813
1
  bitmask = 2;
814
1
  ccv_nnc_dynamic_graph_has_effect_to_tensor_variables(graph, TENSOR_VARIABLE_LIST(a, e, c), TENSOR_VARIABLE_LIST(f), &bitmask);
815
1
  REQUIRE(bitmask == 5, "a and c should be ancestors to f");
816
1
  ccv_nnc_dynamic_graph_free(graph);
817
1
}
818
819
#include "case_main.h"