/home/liu/actions-runner/_work/ccv/ccv/test/unit/nnc/dynamic.graph.tests.c
Line | Count | Source |
1 | | #include "case.h" |
2 | | #include "ccv_case.h" |
3 | | #include "ccv_nnc_case.h" |
4 | | #include <ccv.h> |
5 | | #include <nnc/ccv_nnc.h> |
6 | | #include <nnc/ccv_nnc_easy.h> |
7 | | #include "3rdparty/dsfmt/dSFMT.h" |
8 | | |
9 | | TEST_SETUP() |
10 | | { |
11 | | ccv_nnc_init(); |
12 | | } |
13 | | |
14 | | TEST_CASE("dynamic graph to compute log(19)") |
15 | 1 | { |
16 | 1 | ccv_nnc_dynamic_graph_t* const graph = ccv_nnc_dynamic_graph_new(); |
17 | 1 | ccv_nnc_tensor_variable_t a = ccv_nnc_tensor_variable_new(graph, CPU_TENSOR_NHWC(32F, 1)); |
18 | 1 | ccv_nnc_tensor_from_variable(graph, a)->data.f32[0] = 19; |
19 | 1 | ccv_nnc_tensor_variable_t b = ccv_nnc_tensor_variable_new(graph); |
20 | 1 | ccv_nnc_dynamic_graph_exec(graph, CMD_EWLOG_FORWARD(), ccv_nnc_no_hint, 0, TENSOR_VARIABLE_LIST(a), TENSOR_VARIABLE_LIST(b), 0, 0); |
21 | 1 | REQUIRE_EQ_WITH_TOLERANCE(ccv_nnc_tensor_from_variable(graph, b)->data.f32[0], logf(19), 1e-5, "log(19) result should be equal."); |
22 | 1 | DYNAMIC_GRAPH_GEN(graph, CCV_NNC_LONG_DOT_GRAPH); |
23 | 1 | ccv_nnc_dynamic_graph_free(graph); |
24 | 1 | } |
25 | | |
26 | | TEST_CASE("dynamic graph to compute reciprocal") |
27 | 1 | { |
28 | 1 | ccv_nnc_dynamic_graph_t* const graph = ccv_nnc_dynamic_graph_new(); |
29 | 1 | ccv_nnc_tensor_variable_t a = ccv_nnc_tensor_variable_new(graph, CPU_TENSOR_NHWC(32F, 1)); |
30 | 1 | ccv_nnc_tensor_from_variable(graph, a)->data.f32[0] = 19; |
31 | 1 | ccv_nnc_tensor_variable_t b = ccv_nnc_tensor_variable_new(graph); |
32 | 1 | ccv_nnc_dynamic_graph_exec(graph, CMD_EWDIV_FORWARD(), ccv_nnc_no_hint, 0, TENSOR_VARIABLE_LIST(0, a), TENSOR_VARIABLE_LIST(b), 0, 0); |
33 | 1 | REQUIRE_EQ_WITH_TOLERANCE(ccv_nnc_tensor_from_variable(graph, b)->data.f32[0], 1.0 / 19, 1e-5, "1.0 / 19 result should be equal."); |
34 | 1 | DYNAMIC_GRAPH_GEN(graph, CCV_NNC_LONG_DOT_GRAPH); |
35 | 1 | ccv_nnc_dynamic_graph_free(graph); |
36 | 1 | } |
37 | | |
38 | | TEST_CASE("dynamic graph with alias") |
39 | 1 | { |
40 | 1 | ccv_nnc_dynamic_graph_t* const graph = ccv_nnc_dynamic_graph_new(); |
41 | 1 | ccv_nnc_tensor_variable_t const a = ccv_nnc_tensor_variable_new(graph, CPU_TENSOR_NHWC(32F, 2)); |
42 | 1 | ccv_nnc_tensor_variable_t const a0 = ccv_nnc_tensor_variable_alias_new(graph, a, DIM_ALLOC(1), DIM_ALLOC(), CPU_TENSOR_NHWC(32F, 1)); |
43 | 1 | ccv_nnc_tensor_variable_t const a1 = ccv_nnc_tensor_variable_alias_new(graph, a, ccv_nnc_no_ofs, DIM_ALLOC(), CPU_TENSOR_NHWC(32F, 1)); |
44 | 1 | ccv_nnc_tensor_t* const b0 = ccv_nnc_tensor_new(0, CPU_TENSOR_NHWC(32F, 2), 0); |
45 | 1 | ccv_nnc_tensor_variable_set(graph, a, b0); |
46 | 1 | b0->data.f32[0] = 10; |
47 | 1 | b0->data.f32[1] = 11; |
48 | 1 | ccv_nnc_tensor_t* const b1 = ccv_nnc_tensor_new(0, CPU_TENSOR_NHWC(32F, 2), 0); |
49 | 1 | b1->data.f32[0] = 20; |
50 | 1 | b1->data.f32[1] = 21; |
51 | 1 | REQUIRE(CCV_IS_TENSOR_VIEW(ccv_nnc_tensor_from_variable(graph, a0)), "Complex vector is a tensor view"); |
52 | 1 | REQUIRE(!CCV_IS_TENSOR_VIEW(ccv_nnc_tensor_from_variable(graph, a1)), "Simple vector is not a tensor view"); |
53 | 1 | REQUIRE_EQ(ccv_nnc_tensor_from_variable(graph, a0)->data.f32[0], 11, "should be b0[1]"); |
54 | 1 | REQUIRE_EQ(ccv_nnc_tensor_from_variable(graph, a1)->data.f32[0], 10, "should be b0[0]"); |
55 | 1 | REQUIRE(!CCV_IS_TENSOR_VIEW(ccv_nnc_tensor_from_variable(graph, a1)), "no need to be a tensor view"); |
56 | 1 | ccv_nnc_tensor_variable_set(graph, a, b1); |
57 | 1 | REQUIRE_EQ(ccv_nnc_tensor_from_variable(graph, a0)->data.f32[0], 21, "should be b1[1]"); |
58 | 1 | REQUIRE_EQ(ccv_nnc_tensor_from_variable(graph, a1)->data.f32[0], 20, "should be b1[0]"); |
59 | 1 | REQUIRE(!CCV_IS_TENSOR_VIEW(ccv_nnc_tensor_from_variable(graph, a1)), "no need to be a tensor view"); |
60 | 1 | ccv_nnc_dynamic_graph_free(graph); |
61 | 1 | ccv_nnc_tensor_free(b0); |
62 | 1 | ccv_nnc_tensor_free(b1); |
63 | 1 | } |
64 | | |
65 | | TEST_CASE("dynamic graph alias an alias") |
66 | 1 | { |
67 | 1 | ccv_nnc_dynamic_graph_t* const graph = ccv_nnc_dynamic_graph_new(); |
68 | 1 | ccv_nnc_tensor_variable_t const a = ccv_nnc_tensor_variable_new(graph, CPU_TENSOR_NHWC(32F, 2, 3)); |
69 | 1 | ccv_nnc_tensor_variable_t const a0 = ccv_nnc_tensor_variable_alias_new(graph, a, DIM_ALLOC(1, 0), DIM_ALLOC(3, 1), CPU_TENSOR_NHWC(32F, 1, 3)); |
70 | 1 | ccv_nnc_tensor_variable_t const a1 = ccv_nnc_tensor_variable_alias_new(graph, a0, DIM_ALLOC(1), DIM_ALLOC(1), CPU_TENSOR_NHWC(32F, 2)); |
71 | 1 | ccv_nnc_tensor_t* const b0 = ccv_nnc_tensor_new(0, CPU_TENSOR_NHWC(32F, 2, 3), 0); |
72 | 1 | ccv_nnc_tensor_variable_set(graph, a, b0); |
73 | 1 | b0->data.f32[0] = 10; |
74 | 1 | b0->data.f32[1] = 11; |
75 | 1 | b0->data.f32[2] = 12; |
76 | 1 | b0->data.f32[3] = 13; |
77 | 1 | b0->data.f32[4] = 14; |
78 | 1 | b0->data.f32[5] = 15; |
79 | 1 | ccv_nnc_tensor_t* const b1 = ccv_nnc_tensor_new(0, CPU_TENSOR_NHWC(32F, 2, 3), 0); |
80 | 1 | b1->data.f32[0] = 20; |
81 | 1 | b1->data.f32[1] = 21; |
82 | 1 | b1->data.f32[2] = 22; |
83 | 1 | b1->data.f32[3] = 23; |
84 | 1 | b1->data.f32[4] = 24; |
85 | 1 | b1->data.f32[5] = 25; |
86 | 1 | REQUIRE(CCV_IS_TENSOR_VIEW(ccv_nnc_tensor_from_variable(graph, a0)), "Complex vector is a tensor view"); |
87 | 1 | REQUIRE(CCV_IS_TENSOR_VIEW(ccv_nnc_tensor_from_variable(graph, a1)), "Complex vector is a tensor view"); |
88 | 1 | REQUIRE_EQ(ccv_nnc_tensor_from_variable(graph, a0)->data.f32[0], 13, "should be b0[1, 0]"); |
89 | 1 | REQUIRE_EQ(ccv_nnc_tensor_from_variable(graph, a0)->data.f32[1], 14, "should be b0[1, 1]"); |
90 | 1 | REQUIRE_EQ(ccv_nnc_tensor_from_variable(graph, a0)->data.f32[2], 15, "should be b0[1, 2]"); |
91 | 1 | REQUIRE_EQ(ccv_nnc_tensor_from_variable(graph, a1)->data.f32[0], 14, "should be b0[1, 1]"); |
92 | 1 | REQUIRE_EQ(ccv_nnc_tensor_from_variable(graph, a1)->data.f32[1], 15, "should be b0[1, 2]"); |
93 | 1 | ccv_nnc_tensor_variable_set(graph, a, b1); |
94 | 1 | REQUIRE_EQ(ccv_nnc_tensor_from_variable(graph, a0)->data.f32[0], 23, "should be b1[1, 0]"); |
95 | 1 | REQUIRE_EQ(ccv_nnc_tensor_from_variable(graph, a0)->data.f32[1], 24, "should be b1[1, 1]"); |
96 | 1 | REQUIRE_EQ(ccv_nnc_tensor_from_variable(graph, a0)->data.f32[2], 25, "should be b1[1, 2]"); |
97 | 1 | REQUIRE_EQ(ccv_nnc_tensor_from_variable(graph, a1)->data.f32[0], 24, "should be b1[1, 1]"); |
98 | 1 | REQUIRE_EQ(ccv_nnc_tensor_from_variable(graph, a1)->data.f32[1], 25, "should be b1[1, 2]"); |
99 | 1 | ccv_nnc_dynamic_graph_free(graph); |
100 | 1 | ccv_nnc_tensor_free(b0); |
101 | 1 | ccv_nnc_tensor_free(b1); |
102 | 1 | } |
103 | | |
104 | | TEST_CASE("dynamic graph to compute f(x) = x * log(x) + 1.2 * x, f'(x) where x = 19") |
105 | 1 | { |
106 | 1 | ccv_nnc_dynamic_graph_t* const graph = ccv_nnc_dynamic_graph_new(); |
107 | 1 | ccv_nnc_tensor_variable_t x = ccv_nnc_tensor_variable_new(graph, CPU_TENSOR_NHWC(32F, 1)); |
108 | 1 | ccv_nnc_tensor_from_variable(graph, x)->data.f32[0] = 19; |
109 | 1 | ccv_nnc_tensor_variable_t f = ccv_nnc_tensor_variable_new(graph); |
110 | 1 | ccv_nnc_dynamic_graph_exec(graph, CMD_EWLOG_FORWARD(), ccv_nnc_no_hint, 0, TENSOR_VARIABLE_LIST(x), TENSOR_VARIABLE_LIST(f), 0, 0); |
111 | 1 | ccv_nnc_dynamic_graph_exec(graph, CMD_EWPROD_FORWARD(), ccv_nnc_no_hint, 0, TENSOR_VARIABLE_LIST(x, f), TENSOR_VARIABLE_LIST(f), 0, 0); |
112 | 1 | ccv_nnc_tensor_variable_t y = ccv_nnc_tensor_variable_new(graph, CPU_TENSOR_NHWC(32F, 1)); |
113 | 1 | ccv_nnc_tensor_from_variable(graph, y)->data.f32[0] = 1.2; |
114 | 1 | ccv_nnc_tensor_variable_t z = ccv_nnc_tensor_variable_new(graph); |
115 | 1 | ccv_nnc_dynamic_graph_exec(graph, CMD_EWPROD_FORWARD(), ccv_nnc_no_hint, 0, TENSOR_VARIABLE_LIST(x, y), TENSOR_VARIABLE_LIST(z), 0, 0); |
116 | 1 | ccv_nnc_dynamic_graph_exec(graph, CMD_EWSUM_FORWARD(), ccv_nnc_no_hint, 0, TENSOR_VARIABLE_LIST(f, z), TENSOR_VARIABLE_LIST(f), 0, 0); |
117 | 1 | REQUIRE_EQ_WITH_TOLERANCE(ccv_nnc_tensor_from_variable(graph, f)->data.f32[0], 19 * logf(19) + 1.2 * 19, 1e-5, "f(x) = 1.2 * 19 + 19 * log(19)"); |
118 | | // Do gradient computation multiple times. |
119 | 1 | ccv_nnc_tensor_variable_t dx = ccv_nnc_tensor_variable_new(graph); |
120 | 1 | ccv_nnc_dynamic_graph_backward(graph, TENSOR_VARIABLE_LIST(f), 0, TENSOR_VARIABLE_LIST(x), TENSOR_VARIABLE_LIST(dx), 0); |
121 | 1 | REQUIRE_EQ_WITH_TOLERANCE(ccv_nnc_tensor_from_variable(graph, dx)->data.f32[0], logf(19) + 1 + 1.2, 1e-5, "f'(x) = 1.2 + log(19) + 19 * 1 / 19"); |
122 | 1 | ccv_nnc_tensor_variable_t dy = ccv_nnc_tensor_variable_new(graph); |
123 | 1 | ccv_nnc_dynamic_graph_backward(graph, TENSOR_VARIABLE_LIST(f), 0, TENSOR_VARIABLE_LIST(y), TENSOR_VARIABLE_LIST(dy), 0); |
124 | 1 | REQUIRE_EQ_WITH_TOLERANCE(ccv_nnc_tensor_from_variable(graph, dy)->data.f32[0], 19, 1e-5, "f'(y) = 19"); |
125 | 1 | ccv_nnc_tensor_variable_t dz = ccv_nnc_tensor_variable_new(graph); |
126 | 1 | ccv_nnc_dynamic_graph_backward(graph, TENSOR_VARIABLE_LIST(f), 0, TENSOR_VARIABLE_LIST(z), TENSOR_VARIABLE_LIST(dz), 0); |
127 | 1 | REQUIRE_EQ_WITH_TOLERANCE(ccv_nnc_tensor_from_variable(graph, dz)->data.f32[0], 1, 1e-5, "f'(z) = 1"); |
128 | 1 | ccv_nnc_tensor_variable_free(graph, dy); |
129 | 1 | dy = ccv_nnc_tensor_variable_new(graph); |
130 | 1 | ccv_nnc_dynamic_graph_exec(graph, CMD_SET_FORWARD(0), ccv_nnc_no_hint, 0, TENSOR_VARIABLE_LIST(), TENSOR_VARIABLE_LIST(dx), 0, 0); |
131 | 1 | ccv_nnc_dynamic_graph_backward(graph, TENSOR_VARIABLE_LIST(f), 0, TENSOR_VARIABLE_LIST(y, x), TENSOR_VARIABLE_LIST(dy, dx), 0); |
132 | 1 | REQUIRE_EQ_WITH_TOLERANCE(ccv_nnc_tensor_from_variable(graph, dx)->data.f32[0], logf(19) + 1 + 1.2, 1e-5, "f'(x) = 1.2 + log(19) + 19 * 1 / 19"); |
133 | 1 | REQUIRE_EQ_WITH_TOLERANCE(ccv_nnc_tensor_from_variable(graph, dy)->data.f32[0], 19, 1e-5, "f'(y) = 19"); |
134 | 1 | DYNAMIC_GRAPH_GEN(graph, CCV_NNC_LONG_DOT_GRAPH); |
135 | 1 | ccv_nnc_dynamic_graph_free(graph); |
136 | 1 | } |
137 | | |
138 | | TEST_CASE("dynamic graph with dense net (extensive use of alias)") |
139 | 1 | { |
140 | 1 | ccv_nnc_dynamic_graph_t* const graph = ccv_nnc_dynamic_graph_new(); |
141 | 1 | ccv_nnc_tensor_variable_t x = ccv_nnc_tensor_variable_new(graph, CPU_TENSOR_NHWC(32F, 1, 4)); |
142 | 1 | ccv_nnc_tensor_from_variable(graph, x)->data.f32[0] = 0.472; |
143 | 1 | ccv_nnc_tensor_variable_t x1 = ccv_nnc_tensor_variable_alias_new(graph, x, ccv_nnc_no_ofs, DIM_ALLOC(4, 1), CPU_TENSOR_NHWC(32F, 1, 1)); |
144 | 1 | ccv_nnc_tensor_variable_t w1 = ccv_nnc_tensor_variable_new(graph, CPU_TENSOR_NHWC(32F, 1, 1)); |
145 | 1 | ccv_nnc_tensor_from_variable(graph, w1)->data.f32[0] = 0.234; |
146 | 1 | ccv_nnc_tensor_variable_t b1 = ccv_nnc_tensor_variable_new(graph, CPU_TENSOR_NHWC(32F, 1)); |
147 | 1 | ccv_nnc_tensor_from_variable(graph, b1)->data.f32[0] = 0.1; |
148 | 1 | ccv_nnc_tensor_variable_t x11 = ccv_nnc_tensor_variable_alias_new(graph, x, DIM_ALLOC(0, 1), DIM_ALLOC(4, 1), CPU_TENSOR_NHWC(32F, 1, 1)); |
149 | 1 | ccv_nnc_dynamic_graph_exec(graph, CMD_GEMM_FORWARD(NO_TRANSPOSE, TRANSPOSE(0, 1)), ccv_nnc_no_hint, 0, TENSOR_VARIABLE_LIST(x1, w1, b1), TENSOR_VARIABLE_LIST(x11), 0, 0); |
150 | 1 | ccv_nnc_tensor_variable_t x2 = ccv_nnc_tensor_variable_alias_new(graph, x, ccv_nnc_no_ofs, DIM_ALLOC(4, 1), CPU_TENSOR_NHWC(32F, 1, 2)); |
151 | 1 | ccv_nnc_tensor_variable_t w2 = ccv_nnc_tensor_variable_new(graph, CPU_TENSOR_NHWC(32F, 1, 2)); |
152 | 1 | ccv_nnc_tensor_from_variable(graph, w2)->data.f32[0] = 0.374; |
153 | 1 | ccv_nnc_tensor_from_variable(graph, w2)->data.f32[1] = 0.886; |
154 | 1 | ccv_nnc_tensor_variable_t b2 = ccv_nnc_tensor_variable_new(graph, CPU_TENSOR_NHWC(32F, 1)); |
155 | 1 | ccv_nnc_tensor_from_variable(graph, b2)->data.f32[0] = 0.2; |
156 | 1 | ccv_nnc_tensor_variable_t x21 = ccv_nnc_tensor_variable_alias_new(graph, x, DIM_ALLOC(0, 2), DIM_ALLOC(4, 1), CPU_TENSOR_NHWC(32F, 1, 1)); |
157 | 1 | ccv_nnc_dynamic_graph_exec(graph, CMD_GEMM_FORWARD(NO_TRANSPOSE, TRANSPOSE(0, 1)), ccv_nnc_no_hint, 0, TENSOR_VARIABLE_LIST(x2, w2, b2), TENSOR_VARIABLE_LIST(x21), 0, 0); |
158 | 1 | ccv_nnc_tensor_variable_t x3 = ccv_nnc_tensor_variable_alias_new(graph, x, ccv_nnc_no_ofs, DIM_ALLOC(4, 1), CPU_TENSOR_NHWC(32F, 1, 3)); |
159 | 1 | ccv_nnc_tensor_variable_t w3 = ccv_nnc_tensor_variable_new(graph, CPU_TENSOR_NHWC(32F, 1, 3)); |
160 | 1 | ccv_nnc_tensor_from_variable(graph, w3)->data.f32[0] = 0.484; |
161 | 1 | ccv_nnc_tensor_from_variable(graph, w3)->data.f32[1] = 0.912; |
162 | 1 | ccv_nnc_tensor_from_variable(graph, w3)->data.f32[2] = 0.235; |
163 | 1 | ccv_nnc_tensor_variable_t b3 = ccv_nnc_tensor_variable_new(graph, CPU_TENSOR_NHWC(32F, 1)); |
164 | 1 | ccv_nnc_tensor_from_variable(graph, b3)->data.f32[0] = 0.3; |
165 | 1 | ccv_nnc_tensor_variable_t x31 = ccv_nnc_tensor_variable_alias_new(graph, x, DIM_ALLOC(0, 3), DIM_ALLOC(4, 1), CPU_TENSOR_NHWC(32F, 1, 1)); |
166 | 1 | ccv_nnc_dynamic_graph_exec(graph, CMD_GEMM_FORWARD(NO_TRANSPOSE, TRANSPOSE(0, 1)), ccv_nnc_no_hint, 0, TENSOR_VARIABLE_LIST(x3, w3, b3), TENSOR_VARIABLE_LIST(x31), 0, 0); |
167 | 1 | DYNAMIC_GRAPH_GEN(graph, CCV_NNC_LONG_DOT_GRAPH); |
168 | 1 | ccv_nnc_tensor_t* xt = ccv_nnc_tensor_new(0, CPU_TENSOR_NHWC(32F, 1, 4), 0); |
169 | 1 | xt->data.f32[0] = 0.472; |
170 | 1 | xt->data.f32[1] = xt->data.f32[0] * 0.234 + 0.1; |
171 | 1 | xt->data.f32[2] = xt->data.f32[0] * 0.374 + xt->data.f32[1] * 0.886 + 0.2; |
172 | 1 | xt->data.f32[3] = xt->data.f32[0] * 0.484 + xt->data.f32[1] * 0.912 + xt->data.f32[2] * 0.235 + 0.3; |
173 | 1 | REQUIRE_MATRIX_EQ(ccv_nnc_tensor_from_variable(graph, x), xt, "1x4 matrix should be exactly the same"); |
174 | 1 | ccv_nnc_tensor_free(xt); |
175 | 1 | ccv_nnc_tensor_variable_t dw1 = ccv_nnc_tensor_variable_new(graph); |
176 | 1 | ccv_nnc_dynamic_graph_backward(graph, TENSOR_VARIABLE_LIST(x), 0, TENSOR_VARIABLE_LIST(w1), TENSOR_VARIABLE_LIST(dw1), 0); |
177 | 1 | REQUIRE_EQ_WITH_TOLERANCE((0.235 * 0.886 + 0.912) * 0.472, ccv_nnc_tensor_from_variable(graph, dw1)->data.f32[0], 1e-5, "the gradient should be equal to a complicated result"); |
178 | 1 | ccv_nnc_dynamic_graph_free(graph); |
179 | 1 | } |
180 | | |
181 | | TEST_CASE("batch norm in dynamic graph (enforce inplace)") |
182 | 1 | { |
183 | 1 | ccv_nnc_dynamic_graph_t* const graph = ccv_nnc_dynamic_graph_new(); |
184 | 1 | ccv_nnc_tensor_variable_t x = ccv_nnc_tensor_variable_new(graph, CPU_TENSOR_NHWC(32F, 2, 2, 2, 10)); |
185 | 1 | ccv_nnc_tensor_variable_t y = ccv_nnc_tensor_variable_new(graph); |
186 | 1 | ccv_nnc_tensor_variable_t scale = ccv_nnc_tensor_variable_new(graph, CPU_TENSOR_NHWC(32F, 10)); |
187 | 1 | ccv_nnc_tensor_variable_t bias = ccv_nnc_tensor_variable_new(graph, CPU_TENSOR_NHWC(32F, 10)); |
188 | 1 | ccv_nnc_tensor_variable_t mean = ccv_nnc_tensor_variable_new(graph, CPU_TENSOR_NHWC(32F, 10)); |
189 | 1 | ccv_nnc_tensor_variable_t var = ccv_nnc_tensor_variable_new(graph, CPU_TENSOR_NHWC(32F, 10)); |
190 | 1 | ccv_nnc_tensor_variable_t saved_mean = ccv_nnc_tensor_variable_new(graph); |
191 | 1 | ccv_nnc_tensor_variable_t saved_inv_std = ccv_nnc_tensor_variable_new(graph); |
192 | 1 | dsfmt_t dsfmt; |
193 | 1 | int i; |
194 | 1 | dsfmt_init_gen_rand(&dsfmt, 1); |
195 | 81 | for (i = 0; i < 2 * 2 * 2 * 10; i++80 ) |
196 | 80 | ccv_nnc_tensor_from_variable(graph, x)->data.f32[i] = dsfmt_genrand_open_close(&dsfmt); |
197 | 11 | for (i = 0; i < 10; i++10 ) |
198 | 10 | ccv_nnc_tensor_from_variable(graph, scale)->data.f32[i] = 1; |
199 | 11 | for (i = 0; i < 10; i++10 ) |
200 | 10 | ccv_nnc_tensor_from_variable(graph, bias)->data.f32[i] = 0; |
201 | 11 | for (i = 0; i < 10; i++10 ) |
202 | 10 | ccv_nnc_tensor_from_variable(graph, mean)->data.f32[i] = 0; |
203 | 1 | ccv_nnc_tensor_t* mean_tensor_ptr = ccv_nnc_tensor_from_variable(graph, mean); |
204 | 11 | for (i = 0; i < 10; i++10 ) |
205 | 10 | ccv_nnc_tensor_from_variable(graph, var)->data.f32[i] = 0; |
206 | 1 | ccv_nnc_tensor_t* var_tensor_ptr = ccv_nnc_tensor_from_variable(graph, var); |
207 | 1 | ccv_nnc_dynamic_graph_exec(graph, CMD_BATCH_NORM_FORWARD(0, 0, 0.9, 0, 1, 2), ccv_nnc_no_hint, 0, TENSOR_VARIABLE_LIST(x, scale, bias, mean, var), TENSOR_VARIABLE_LIST(y, mean, var, saved_mean, saved_inv_std), 0, 0); |
208 | 1 | DYNAMIC_GRAPH_GEN(graph, CCV_NNC_LONG_DOT_GRAPH); |
209 | 1 | REQUIRE(mean_tensor_ptr == ccv_nnc_tensor_from_variable(graph, mean), "enforced inplace, tensor view pointer unchanged"); |
210 | 1 | REQUIRE(var_tensor_ptr == ccv_nnc_tensor_from_variable(graph, var), "enforced inplace, tensor view pointer unchanged"); |
211 | 1 | ccv_nnc_tensor_t* x_tensor = ccv_nnc_tensor_new(0, CPU_TENSOR_NHWC(32F, 2, 2, 2, 10), 0); |
212 | 1 | ccv_nnc_tensor_t* y_tensor = ccv_nnc_tensor_new(0, CPU_TENSOR_NHWC(32F, 2, 2, 2, 10), 0); |
213 | 1 | ccv_nnc_tensor_t* scale_tensor = ccv_nnc_tensor_new(0, CPU_TENSOR_NHWC(32F, 10), 0); |
214 | 1 | ccv_nnc_tensor_t* bias_tensor = ccv_nnc_tensor_new(0, CPU_TENSOR_NHWC(32F, 10), 0); |
215 | 1 | ccv_nnc_tensor_t* mean_tensor = ccv_nnc_tensor_new(0, CPU_TENSOR_NHWC(32F, 10), 0); |
216 | 1 | ccv_nnc_tensor_t* var_tensor = ccv_nnc_tensor_new(0, CPU_TENSOR_NHWC(32F, 10), 0); |
217 | 1 | ccv_nnc_tensor_t* saved_mean_tensor = ccv_nnc_tensor_new(0, CPU_TENSOR_NHWC(32F, 1, 1, 1, 10), 0); |
218 | 1 | ccv_nnc_tensor_t* saved_inv_std_tensor = ccv_nnc_tensor_new(0, CPU_TENSOR_NHWC(32F, 1, 1, 1, 10), 0); |
219 | 1 | memcpy(x_tensor->data.f32, ccv_nnc_tensor_from_variable(graph, x)->data.f32, sizeof(float) * 2 * 2 * 2 * 10); |
220 | 11 | for (i = 0; i < 10; i++10 ) |
221 | 10 | scale_tensor->data.f32[i] = 1; |
222 | 1 | memset(bias_tensor->data.f32, 0, sizeof(float) * 10); |
223 | 1 | memset(mean_tensor->data.f32, 0, sizeof(float) * 10); |
224 | 1 | memset(var_tensor->data.f32, 0, sizeof(float) * 10); |
225 | 1 | ccv_nnc_cmd_exec(CMD_BATCH_NORM_FORWARD(0, 0, 0.9, 0, 1, 2), ccv_nnc_no_hint, 0, TENSOR_LIST(x_tensor, scale_tensor, bias_tensor, mean_tensor, var_tensor), TENSOR_LIST(y_tensor, mean_tensor, var_tensor, saved_mean_tensor, saved_inv_std_tensor), 0); |
226 | 1 | REQUIRE_TENSOR_EQ(y_tensor, ccv_nnc_tensor_from_variable(graph, y), "y should be equal"); |
227 | 1 | REQUIRE_TENSOR_EQ(mean_tensor, ccv_nnc_tensor_from_variable(graph, mean), "mean should be equal"); |
228 | 1 | REQUIRE_TENSOR_EQ(var_tensor, ccv_nnc_tensor_from_variable(graph, var), "var should be equal"); |
229 | 1 | REQUIRE_TENSOR_EQ(saved_mean_tensor, ccv_nnc_tensor_from_variable(graph, saved_mean), "saved_mean should be equal"); |
230 | 1 | REQUIRE_TENSOR_EQ(saved_inv_std_tensor, ccv_nnc_tensor_from_variable(graph, saved_inv_std), "saved_inv_std should be equal"); |
231 | 1 | ccv_nnc_dynamic_graph_free(graph); |
232 | 1 | ccv_nnc_tensor_free(x_tensor); |
233 | 1 | ccv_nnc_tensor_free(y_tensor); |
234 | 1 | ccv_nnc_tensor_free(scale_tensor); |
235 | 1 | ccv_nnc_tensor_free(bias_tensor); |
236 | 1 | ccv_nnc_tensor_free(mean_tensor); |
237 | 1 | ccv_nnc_tensor_free(var_tensor); |
238 | 1 | ccv_nnc_tensor_free(saved_mean_tensor); |
239 | 1 | ccv_nnc_tensor_free(saved_inv_std_tensor); |
240 | 1 | } |
241 | | |
242 | | TEST_CASE("empty inputs / outputs for dynamic graph") |
243 | 1 | { |
244 | 1 | ccv_nnc_dynamic_graph_t* const graph = ccv_nnc_dynamic_graph_new(); |
245 | 1 | ccv_nnc_tensor_variable_t df = ccv_nnc_tensor_variable_new(graph, CPU_TENSOR_NHWC(32F, 1)); |
246 | 1 | ccv_nnc_tensor_variable_t x = ccv_nnc_tensor_variable_new(graph, CPU_TENSOR_NHWC(32F, 1)); |
247 | 1 | ccv_nnc_tensor_variable_t y = ccv_nnc_tensor_variable_new(graph); |
248 | 1 | ccv_nnc_tensor_from_variable(graph, df)->data.f32[0] = 1; |
249 | 1 | ccv_nnc_tensor_from_variable(graph, x)->data.f32[0] = 10; |
250 | 1 | ccv_nnc_dynamic_graph_exec(graph, CMD_EWDIV_BACKWARD(), ccv_nnc_no_hint, 0, TENSOR_VARIABLE_LIST(df, 0, x), TENSOR_VARIABLE_LIST(y, 0), 0, 0); |
251 | 1 | DYNAMIC_GRAPH_GEN(graph, CCV_NNC_LONG_DOT_GRAPH); |
252 | 1 | REQUIRE_EQ_WITH_TOLERANCE(ccv_nnc_tensor_from_variable(graph, y)->data.f32[0], 1. / 10, 1e-5, "div backward should equal to 1 / 10"); |
253 | 1 | ccv_nnc_dynamic_graph_free(graph); |
254 | 1 | } |
255 | | |
256 | | TEST_CASE("long dynamic graph with unused variables freed") |
257 | 1 | { |
258 | 1 | ccv_nnc_dynamic_graph_t* const graph = ccv_nnc_dynamic_graph_new(); |
259 | 1 | int i; |
260 | 1 | ccv_nnc_tensor_variable_t x = ccv_nnc_tensor_variable_new(graph, CPU_TENSOR_NHWC(32F, 1)); |
261 | 1 | ccv_nnc_tensor_variable_t y = ccv_nnc_tensor_variable_new(graph, CPU_TENSOR_NHWC(32F, 1)); |
262 | 1 | ccv_nnc_tensor_from_variable(graph, x)->data.f32[0] = 32; |
263 | 1 | ccv_nnc_tensor_from_variable(graph, y)->data.f32[0] = 0.5; |
264 | 11 | for (i = 0; i < 10; i++10 ) |
265 | 10 | { |
266 | 10 | ccv_nnc_tensor_variable_t z = ccv_nnc_tensor_variable_new(graph); |
267 | 10 | if (i < 7) |
268 | 7 | ccv_nnc_dynamic_graph_exec(graph, CMD_EWPROD_FORWARD(), ccv_nnc_no_hint, 0, TENSOR_VARIABLE_LIST(x, y), TENSOR_VARIABLE_LIST(z), 0, 0); |
269 | 3 | else { |
270 | 3 | if (i == 7) |
271 | 1 | ccv_nnc_tensor_variable_free(graph, y); // No longer need y. |
272 | 3 | ccv_nnc_dynamic_graph_exec(graph, CMD_EWPROD_FORWARD(), ccv_nnc_no_hint, 0, TENSOR_VARIABLE_LIST(x, x), TENSOR_VARIABLE_LIST(z), 0, 0); |
273 | 3 | } |
274 | 10 | if (i < 9) |
275 | 9 | ccv_nnc_tensor_variable_free(graph, x); |
276 | 10 | x = z; |
277 | 10 | } |
278 | 1 | DYNAMIC_GRAPH_GEN(graph, CCV_NNC_LONG_DOT_GRAPH); |
279 | 1 | float g = 32; |
280 | 11 | for (i = 0; i < 10; i++10 ) |
281 | 10 | { |
282 | 10 | if (i < 7) |
283 | 7 | g = g * 0.5; |
284 | 3 | else |
285 | 3 | g = g * g; |
286 | 10 | } |
287 | 1 | REQUIRE_EQ_WITH_TOLERANCE(ccv_nnc_tensor_from_variable(graph, x)->data.f32[0], g, 1e-5, "x should equal to the computed result"); |
288 | 1 | ccv_nnc_dynamic_graph_free(graph); |
289 | 1 | } |
290 | | |
291 | | TEST_CASE("repeat multiple x * y with y as a constant") |
292 | 1 | { |
293 | 1 | ccv_nnc_dynamic_graph_t* const graph = ccv_nnc_dynamic_graph_new(); |
294 | 1 | int i; |
295 | 1 | ccv_nnc_tensor_variable_t x = ccv_nnc_tensor_variable_new(graph, CPU_TENSOR_NHWC(32F, 1)); |
296 | 1 | ccv_nnc_tensor_variable_t y = ccv_nnc_tensor_constant_new(graph, CPU_TENSOR_NHWC(32F, 1)); |
297 | 1 | ccv_nnc_tensor_from_variable(graph, x)->data.f32[0] = 32; |
298 | 1 | ccv_nnc_tensor_from_variable(graph, y)->data.f32[0] = 0.5; |
299 | 11 | for (i = 0; i < 10; i++10 ) |
300 | 10 | { |
301 | 10 | ccv_nnc_tensor_variable_t z = ccv_nnc_tensor_variable_new(graph); |
302 | 10 | ccv_nnc_dynamic_graph_exec(graph, CMD_EWPROD_FORWARD(), ccv_nnc_no_hint, 0, TENSOR_VARIABLE_LIST(x, y), TENSOR_VARIABLE_LIST(z), 0, 0); |
303 | 10 | ccv_nnc_tensor_variable_free(graph, x); |
304 | 10 | x = z; |
305 | 10 | } |
306 | 1 | DYNAMIC_GRAPH_GEN(graph, CCV_NNC_LONG_DOT_GRAPH); |
307 | 1 | float g = 32; |
308 | 11 | for (i = 0; i < 10; i++10 ) |
309 | 10 | g = g * 0.5; |
310 | 1 | REQUIRE_EQ_WITH_TOLERANCE(ccv_nnc_tensor_from_variable(graph, x)->data.f32[0], g, 1e-5, "x should equal to the computed result"); |
311 | 1 | ccv_nnc_dynamic_graph_free(graph); |
312 | 1 | } |
313 | | |
314 | | static int _ccv_tensor_variable_freed = 0; |
315 | | |
316 | | static void _ccv_tensor_variable_hook(ccv_nnc_dynamic_graph_t* const graph, const ccv_nnc_tensor_t* const tensor, void* const context) |
317 | 7 | { |
318 | 7 | ++_ccv_tensor_variable_freed; |
319 | 7 | } |
320 | | |
321 | | TEST_CASE("repeat multiple x * y with y as a constant, compute d(x)") |
322 | 1 | { |
323 | 1 | ccv_nnc_dynamic_graph_t* const graph = ccv_nnc_dynamic_graph_new(); |
324 | 1 | int i; |
325 | 1 | _ccv_tensor_variable_freed = 0; |
326 | 1 | ccv_nnc_tensor_variable_t x = ccv_nnc_tensor_variable_new(graph, CPU_TENSOR_NHWC(32F, 1)); |
327 | 1 | ccv_nnc_tensor_variable_t y = ccv_nnc_tensor_constant_new(graph, CPU_TENSOR_NHWC(32F, 1)); |
328 | 1 | ccv_nnc_tensor_variable_t ox = x; |
329 | 1 | ccv_nnc_tensor_variable_destructor_hook(graph, x, _ccv_tensor_variable_hook, 0); |
330 | 1 | ccv_nnc_tensor_variable_destructor_hook(graph, y, _ccv_tensor_variable_hook, 0); |
331 | 1 | ccv_nnc_tensor_from_variable(graph, x)->data.f32[0] = 32; |
332 | 1 | ccv_nnc_tensor_from_variable(graph, y)->data.f32[0] = 0.9; |
333 | 5 | for (i = 0; i < 4; i++4 ) |
334 | 4 | { |
335 | 4 | ccv_nnc_tensor_variable_t z = ccv_nnc_tensor_variable_new(graph); |
336 | 4 | ccv_nnc_tensor_variable_destructor_hook(graph, z, _ccv_tensor_variable_hook, 0); |
337 | 4 | ccv_nnc_dynamic_graph_exec(graph, CMD_EWPROD_FORWARD(), ccv_nnc_no_hint, 0, TENSOR_VARIABLE_LIST(x, y), TENSOR_VARIABLE_LIST(z), 0, 0); |
338 | 4 | if (i > 0) |
339 | 3 | ccv_nnc_tensor_variable_free(graph, x); |
340 | 4 | x = z; |
341 | 4 | } |
342 | 1 | REQUIRE_EQ(0, _ccv_tensor_variable_freed, "none of these are freed"); |
343 | 1 | ccv_nnc_tensor_variable_t dx = ccv_nnc_tensor_variable_new(graph); |
344 | 1 | ccv_nnc_tensor_variable_destructor_hook(graph, dx, _ccv_tensor_variable_hook, 0); |
345 | 1 | ccv_nnc_dynamic_graph_backward(graph, TENSOR_VARIABLE_LIST(x), 0, TENSOR_VARIABLE_LIST(ox), TENSOR_VARIABLE_LIST(dx), 0); |
346 | 1 | ccv_nnc_tensor_variable_free(graph, ox); |
347 | | // We freed all 4 x (ox, when i = 1, 2, 3). |
348 | 1 | REQUIRE_EQ(4, _ccv_tensor_variable_freed, "all are freed except dx"); |
349 | 1 | _ccv_tensor_variable_freed = 0; |
350 | 1 | DYNAMIC_GRAPH_GEN(graph, CCV_NNC_LONG_DOT_GRAPH); |
351 | 1 | float g = 1; |
352 | 5 | for (i = 0; i < 4; i++4 ) |
353 | 4 | g = g * 0.9; |
354 | 1 | REQUIRE_EQ_WITH_TOLERANCE(ccv_nnc_tensor_from_variable(graph, dx)->data.f32[0], g, 1e-5, "x should equal to the computed result"); |
355 | 1 | ccv_nnc_dynamic_graph_free(graph); |
356 | | // y, dx, and z. |
357 | 1 | REQUIRE_EQ(3, _ccv_tensor_variable_freed, "dx freed"); |
358 | 1 | } |
359 | | |
360 | | TEST_CASE("compute f(x) = x * log(x) + x, f'(x) when x = 10 (and intermediate results all freed)") |
361 | 1 | { |
362 | 1 | ccv_nnc_dynamic_graph_t* const graph = ccv_nnc_dynamic_graph_new(); |
363 | 1 | ccv_nnc_tensor_variable_t x = ccv_nnc_tensor_variable_new(graph, CPU_TENSOR_NHWC(32F, 1)); |
364 | 1 | ccv_nnc_tensor_variable_t y = ccv_nnc_tensor_variable_new(graph); |
365 | 1 | ccv_nnc_tensor_variable_t f = ccv_nnc_tensor_variable_new(graph); |
366 | 1 | ccv_nnc_tensor_from_variable(graph, x)->data.f32[0] = 10; |
367 | 1 | ccv_nnc_dynamic_graph_exec(graph, CMD_EWLOG_FORWARD(), ccv_nnc_no_hint, 0, TENSOR_VARIABLE_LIST(x), TENSOR_VARIABLE_LIST(y), 0, 0); |
368 | 1 | ccv_nnc_dynamic_graph_exec(graph, CMD_EWPROD_FORWARD(), ccv_nnc_no_hint, 0, TENSOR_VARIABLE_LIST(x, y), TENSOR_VARIABLE_LIST(y), 0, 0); |
369 | 1 | ccv_nnc_dynamic_graph_exec(graph, CMD_EWSUM_FORWARD(), ccv_nnc_no_hint, 0, TENSOR_VARIABLE_LIST(y, x), TENSOR_VARIABLE_LIST(f), 0, 0); |
370 | 1 | ccv_nnc_tensor_variable_t df = ccv_nnc_tensor_variable_new(graph, CPU_TENSOR_NHWC(32F, 1)); |
371 | 1 | ccv_nnc_dynamic_graph_exec(graph, CMD_SET_FORWARD(1), ccv_nnc_no_hint, 0, 0, 0, TENSOR_VARIABLE_LIST(df), 0, 0); |
372 | | // x will be accumulated on to itself. |
373 | 1 | ccv_nnc_dynamic_graph_backward(graph, TENSOR_VARIABLE_LIST(f), &df, TENSOR_VARIABLE_LIST(x), TENSOR_VARIABLE_LIST(x), 0); |
374 | 1 | DYNAMIC_GRAPH_GEN(graph, CCV_NNC_LONG_DOT_GRAPH); |
375 | 1 | REQUIRE_EQ_WITH_TOLERANCE(ccv_nnc_tensor_from_variable(graph, x)->data.f32[0], 10 + log(10) + 1 + 1, 1e-5, "dx should equal to the computed result"); |
376 | 1 | ccv_nnc_dynamic_graph_free(graph); |
377 | 1 | } |
378 | | |
379 | | TEST_CASE("dynamic graph with binded value") |
380 | 1 | { |
381 | 1 | ccv_nnc_dynamic_graph_t* const graph = ccv_nnc_dynamic_graph_new(); |
382 | 1 | ccv_nnc_tensor_variable_t x = ccv_nnc_tensor_variable_new(graph, CPU_TENSOR_NHWC(32F, 1)); |
383 | 1 | ccv_nnc_tensor_t* const x_tensor = ccv_nnc_tensor_new(0, CPU_TENSOR_NHWC(32F, 1), 0); |
384 | 1 | x_tensor->data.f32[0] = 10; |
385 | 1 | ccv_nnc_tensor_variable_set(graph, x, x_tensor); |
386 | 1 | ccv_nnc_tensor_variable_t y = ccv_nnc_tensor_variable_new(graph); |
387 | 1 | ccv_nnc_dynamic_graph_exec(graph, CMD_EWLOG_FORWARD(), ccv_nnc_no_hint, 0, TENSOR_VARIABLE_LIST(x), TENSOR_VARIABLE_LIST(y), 0, 0); |
388 | 1 | DYNAMIC_GRAPH_GEN(graph, CCV_NNC_LONG_DOT_GRAPH); |
389 | 1 | REQUIRE_EQ_WITH_TOLERANCE(ccv_nnc_tensor_from_variable(graph, y)->data.f32[0], log(10), 1e-5, "dx should equal to the computed result"); |
390 | 1 | ccv_nnc_dynamic_graph_free(graph); |
391 | 1 | ccv_nnc_tensor_free(x_tensor); |
392 | 1 | } |
393 | | |
394 | | TEST_CASE("dynamic graph to evaluate cnnp model") |
395 | 1 | { |
396 | 1 | ccv_nnc_dynamic_graph_t* const graph = ccv_nnc_dynamic_graph_new(); |
397 | 1 | ccv_cnnp_model_t* const linear = ccv_cnnp_dense(1, 0, 0, 1, 0); |
398 | 1 | ccv_nnc_tensor_variable_t z = ccv_nnc_tensor_constant_new(graph, CPU_TENSOR_NHWC(32F, 1)); |
399 | 1 | ccv_nnc_tensor_from_variable(graph, z)->data.f32[0] = 5; |
400 | 1 | int i; |
401 | 101 | for (i = 0; i < 100; i++100 ) |
402 | 100 | { |
403 | 100 | ccv_nnc_tensor_variable_t x; |
404 | 100 | if (i % 2 == 1) |
405 | 50 | { |
406 | 50 | x = ccv_nnc_tensor_variable_new(graph, CPU_TENSOR_NHWC(32F, 2, 1)); |
407 | 50 | ccv_nnc_tensor_from_variable(graph, x)->data.f32[0] = 10; |
408 | 50 | ccv_nnc_tensor_from_variable(graph, x)->data.f32[1] = 10; |
409 | 50 | } else { |
410 | 50 | x = ccv_nnc_tensor_variable_new(graph, CPU_TENSOR_NHWC(32F, 1)); |
411 | 50 | ccv_nnc_tensor_from_variable(graph, x)->data.f32[0] = 10; |
412 | 50 | } |
413 | 100 | ccv_nnc_tensor_variable_t y = ccv_nnc_tensor_variable_new(graph); |
414 | 100 | ccv_nnc_dynamic_graph_exec(graph, CMD_EWLOG_FORWARD(), ccv_nnc_no_hint, 0, TENSOR_VARIABLE_LIST(x), TENSOR_VARIABLE_LIST(y), 0, 0); |
415 | 100 | ccv_nnc_dynamic_graph_evaluate(graph, linear, 0, TENSOR_VARIABLE_LIST(y), TENSOR_VARIABLE_LIST(y), 0, 0); |
416 | 100 | ccv_nnc_dynamic_graph_exec(graph, CMD_ADD_FORWARD(1, -1), ccv_nnc_no_hint, 0, TENSOR_VARIABLE_LIST(y, z), TENSOR_VARIABLE_LIST(y), 0, 0); |
417 | 100 | ccv_nnc_tensor_variable_t f = ccv_nnc_tensor_variable_new(graph); |
418 | 100 | ccv_nnc_dynamic_graph_exec(graph, CMD_EWPROD_FORWARD(), ccv_nnc_no_hint, 0, TENSOR_VARIABLE_LIST(y, y), TENSOR_VARIABLE_LIST(f), 0, 0); |
419 | 100 | ccv_nnc_tensor_variable_t dx = ccv_nnc_tensor_variable_new(graph); |
420 | 100 | ccv_nnc_dynamic_graph_backward(graph, TENSOR_VARIABLE_LIST(f), 0, TENSOR_VARIABLE_LIST(x), TENSOR_VARIABLE_LIST(dx), 0); |
421 | 100 | ccv_cnnp_model_set_minimizer(linear, CMD_SGD_FORWARD(0, 0.01, 1, 0.01, 0, 0), 0, 0, 0); |
422 | 100 | ccv_nnc_dynamic_graph_apply_gradients(graph, CMD_SGD_FORWARD(0, 0.01, 1, 0.01, 0, 0), TENSOR_VARIABLE_LIST(), TENSOR_VARIABLE_LIST(), 0, 0, 0); |
423 | 100 | ccv_nnc_tensor_variable_free(graph, x); |
424 | 100 | ccv_nnc_tensor_variable_free(graph, y); |
425 | 100 | ccv_nnc_tensor_variable_free(graph, f); |
426 | 100 | ccv_nnc_tensor_variable_free(graph, dx); |
427 | 100 | } |
428 | 1 | ccv_nnc_tensor_variable_t x = ccv_nnc_tensor_variable_new(graph, CPU_TENSOR_NHWC(32F, 1)); |
429 | 1 | ccv_nnc_tensor_from_variable(graph, x)->data.f32[0] = 10; |
430 | 1 | ccv_nnc_tensor_variable_t y = ccv_nnc_tensor_variable_new(graph); |
431 | 1 | ccv_nnc_dynamic_graph_exec(graph, CMD_EWLOG_FORWARD(), ccv_nnc_no_hint, 0, TENSOR_VARIABLE_LIST(x), TENSOR_VARIABLE_LIST(y), 0, 0); |
432 | 1 | ccv_nnc_dynamic_graph_evaluate(graph, linear, 1, TENSOR_VARIABLE_LIST(y), TENSOR_VARIABLE_LIST(y), 0, 0); |
433 | 1 | REQUIRE_EQ_WITH_TOLERANCE(ccv_nnc_tensor_from_variable(graph, y)->data.f32[0], 5, 1e-2, "linear model should be trained to generate the same value as z"); |
434 | 1 | ccv_nnc_tensor_variable_t iy = ccv_nnc_tensor_constant_new(graph, CPU_TENSOR_NHWC(32F, 1)); |
435 | 1 | ccv_nnc_tensor_from_variable(graph, iy)->data.f32[0] = log(10); |
436 | 1 | ccv_nnc_tensor_variable_t iz = ccv_nnc_tensor_variable_new(graph, CPU_TENSOR_NHWC(32F, 1)); |
437 | 1 | ccv_nnc_dynamic_graph_evaluate(graph, linear, 1, TENSOR_VARIABLE_LIST(iy), TENSOR_VARIABLE_LIST(iz), 0, 0); |
438 | 1 | REQUIRE_EQ_WITH_TOLERANCE(ccv_nnc_tensor_from_variable(graph, iz)->data.f32[0], 5, 1e-2, "linear model should be trained to generate the same value as z"); |
439 | 1 | DYNAMIC_GRAPH_GEN(graph, CCV_NNC_LONG_DOT_GRAPH); |
440 | 1 | ccv_nnc_dynamic_graph_free(graph); |
441 | 1 | ccv_cnnp_model_free(linear); |
442 | 1 | } |
443 | | |
444 | | TEST_CASE("dynamic graph to evaluate cnnp model without any parameters") |
445 | 1 | { |
446 | 1 | ccv_nnc_dynamic_graph_t* const graph = ccv_nnc_dynamic_graph_new(); |
447 | 1 | ccv_nnc_tensor_variable_t a = ccv_nnc_tensor_variable_new(graph, CPU_TENSOR_NHWC(32F, 1)); |
448 | 1 | ccv_nnc_tensor_from_variable(graph, a)->data.f32[0] = 1.23; |
449 | 1 | ccv_nnc_tensor_variable_t b = ccv_nnc_tensor_variable_new(graph, CPU_TENSOR_NHWC(32F, 1)); |
450 | 1 | ccv_nnc_tensor_from_variable(graph, b)->data.f32[0] = 2; |
451 | 1 | ccv_nnc_tensor_variable_t c = ccv_nnc_tensor_variable_new(graph, CPU_TENSOR_NHWC(32F, 1)); |
452 | 1 | ccv_cnnp_model_t* const mul = ccv_cnnp_mul(1, "mul"); |
453 | 1 | ccv_nnc_dynamic_graph_evaluate(graph, mul, 1, TENSOR_VARIABLE_LIST(a, b), TENSOR_VARIABLE_LIST(c), 0, 0); |
454 | 1 | ccv_nnc_tensor_variable_t da = ccv_nnc_tensor_variable_new(graph); |
455 | 1 | ccv_nnc_dynamic_graph_backward(graph, TENSOR_VARIABLE_LIST(c), 0, TENSOR_VARIABLE_LIST(a), TENSOR_VARIABLE_LIST(da), 0); |
456 | 1 | ccv_cnnp_model_set_minimizer(mul, CMD_SGD_FORWARD(0, 0.01, 1, 0.01, 0, 0), 0, 0, 0); |
457 | 1 | ccv_nnc_dynamic_graph_apply_gradients(graph, CMD_SGD_FORWARD(0, 0.01, 1, 0.01, 0, 0), TENSOR_VARIABLE_LIST(), TENSOR_VARIABLE_LIST(), 0, 0, 0); |
458 | 1 | ccv_cnnp_model_free(mul); |
459 | 1 | REQUIRE_EQ_WITH_TOLERANCE(ccv_nnc_tensor_from_variable(graph, c)->data.f32[0], 2.46, 1e-5, "should be equal"); |
460 | 1 | REQUIRE_EQ_WITH_TOLERANCE(ccv_nnc_tensor_from_variable(graph, da)->data.f32[0], 2, 1e-5, "should be equal"); |
461 | 1 | ccv_nnc_dynamic_graph_free(graph); |
462 | 1 | } |
463 | | |
464 | | TEST_CASE("dynamic graph to evaluate cnnp model without any parameters with div") |
465 | 1 | { |
466 | 1 | ccv_nnc_dynamic_graph_t* const graph = ccv_nnc_dynamic_graph_new(); |
467 | 1 | ccv_nnc_tensor_variable_t a = ccv_nnc_tensor_variable_new(graph, CPU_TENSOR_NHWC(32F, 1)); |
468 | 1 | ccv_nnc_tensor_from_variable(graph, a)->data.f32[0] = 1.23; |
469 | 1 | ccv_nnc_tensor_variable_t b = ccv_nnc_tensor_variable_new(graph, CPU_TENSOR_NHWC(32F, 1)); |
470 | 1 | ccv_nnc_tensor_from_variable(graph, b)->data.f32[0] = 2; |
471 | 1 | ccv_nnc_tensor_variable_t c = ccv_nnc_tensor_variable_new(graph, CPU_TENSOR_NHWC(32F, 1)); |
472 | 1 | ccv_cnnp_model_t* const div = ccv_cnnp_div(0, "div"); |
473 | 1 | ccv_nnc_dynamic_graph_evaluate(graph, div, 1, TENSOR_VARIABLE_LIST(a, b), TENSOR_VARIABLE_LIST(c), 0, 0); |
474 | 1 | ccv_nnc_tensor_variable_t da = ccv_nnc_tensor_variable_new(graph); |
475 | 1 | ccv_nnc_dynamic_graph_backward(graph, TENSOR_VARIABLE_LIST(c), 0, TENSOR_VARIABLE_LIST(a), TENSOR_VARIABLE_LIST(da), 0); |
476 | 1 | ccv_cnnp_model_set_minimizer(div, CMD_SGD_FORWARD(0, 0.01, 1, 0.01, 0, 0), 0, 0, 0); |
477 | 1 | ccv_nnc_dynamic_graph_apply_gradients(graph, CMD_SGD_FORWARD(0, 0.01, 1, 0.01, 0, 0), TENSOR_VARIABLE_LIST(), TENSOR_VARIABLE_LIST(), 0, 0, 0); |
478 | 1 | ccv_cnnp_model_free(div); |
479 | 1 | REQUIRE_EQ_WITH_TOLERANCE(ccv_nnc_tensor_from_variable(graph, c)->data.f32[0], 1.23 / 2, 1e-5, "should be equal"); |
480 | 1 | REQUIRE_EQ_WITH_TOLERANCE(ccv_nnc_tensor_from_variable(graph, da)->data.f32[0], 1.0 / 2, 1e-5, "should be equal"); |
481 | 1 | ccv_nnc_dynamic_graph_free(graph); |
482 | 1 | } |
483 | | |
484 | | TEST_CASE("dynamic graph to evaluate cnnp model without any parameters with reciprocal") |
485 | 1 | { |
486 | 1 | ccv_nnc_dynamic_graph_t* const graph = ccv_nnc_dynamic_graph_new(); |
487 | 1 | ccv_nnc_tensor_variable_t a = ccv_nnc_tensor_variable_new(graph, CPU_TENSOR_NHWC(32F, 1)); |
488 | 1 | ccv_nnc_tensor_from_variable(graph, a)->data.f32[0] = 1.23; |
489 | 1 | ccv_nnc_tensor_variable_t c = ccv_nnc_tensor_variable_new(graph, CPU_TENSOR_NHWC(32F, 1)); |
490 | 1 | ccv_cnnp_model_t* const div = ccv_cnnp_div(1, "div"); |
491 | 1 | ccv_nnc_dynamic_graph_evaluate(graph, div, 1, TENSOR_VARIABLE_LIST(a), TENSOR_VARIABLE_LIST(c), 0, 0); |
492 | 1 | ccv_nnc_tensor_variable_t da = ccv_nnc_tensor_variable_new(graph); |
493 | 1 | ccv_nnc_dynamic_graph_backward(graph, TENSOR_VARIABLE_LIST(c), 0, TENSOR_VARIABLE_LIST(a), TENSOR_VARIABLE_LIST(da), 0); |
494 | 1 | ccv_cnnp_model_set_minimizer(div, CMD_SGD_FORWARD(0, 0.01, 1, 0.01, 0, 0), 0, 0, 0); |
495 | 1 | ccv_nnc_dynamic_graph_apply_gradients(graph, CMD_SGD_FORWARD(0, 0.01, 1, 0.01, 0, 0), TENSOR_VARIABLE_LIST(), TENSOR_VARIABLE_LIST(), 0, 0, 0); |
496 | 1 | ccv_cnnp_model_free(div); |
497 | 1 | REQUIRE_EQ_WITH_TOLERANCE(ccv_nnc_tensor_from_variable(graph, c)->data.f32[0], 1 / 1.23, 1e-5, "should be equal"); |
498 | 1 | REQUIRE_EQ_WITH_TOLERANCE(ccv_nnc_tensor_from_variable(graph, da)->data.f32[0], -1 / (1.23 * 1.23), 1e-5, "should be equal"); |
499 | 1 | ccv_nnc_dynamic_graph_free(graph); |
500 | 1 | } |
501 | | |
502 | | TEST_CASE("dynamic graph to evaluate cnnp model and simply accumulate gradients") |
503 | 1 | { |
504 | 1 | ccv_nnc_dynamic_graph_t* const graph = ccv_nnc_dynamic_graph_new(); |
505 | 1 | ccv_cnnp_model_t* const linear = ccv_cnnp_dense(1, 0, 0, 1, 0); |
506 | 1 | ccv_nnc_tensor_variable_t z = ccv_nnc_tensor_constant_new(graph, CPU_TENSOR_NHWC(32F, 1)); |
507 | 1 | ccv_nnc_tensor_from_variable(graph, z)->data.f32[0] = 5; |
508 | 1 | int i; |
509 | 101 | for (i = 0; i < 100; i++100 ) |
510 | 100 | { |
511 | 100 | ccv_nnc_tensor_variable_t x; |
512 | 100 | if (i % 2 == 1) |
513 | 50 | { |
514 | 50 | x = ccv_nnc_tensor_variable_new(graph, CPU_TENSOR_NHWC(32F, 2, 1)); |
515 | 50 | ccv_nnc_tensor_from_variable(graph, x)->data.f32[0] = 10; |
516 | 50 | ccv_nnc_tensor_from_variable(graph, x)->data.f32[1] = 10; |
517 | 50 | } else { |
518 | 50 | x = ccv_nnc_tensor_variable_new(graph, CPU_TENSOR_NHWC(32F, 1)); |
519 | 50 | ccv_nnc_tensor_from_variable(graph, x)->data.f32[0] = 10; |
520 | 50 | } |
521 | 100 | ccv_nnc_tensor_variable_t y = ccv_nnc_tensor_variable_new(graph); |
522 | 100 | ccv_nnc_dynamic_graph_exec(graph, CMD_EWLOG_FORWARD(), ccv_nnc_no_hint, 0, TENSOR_VARIABLE_LIST(x), TENSOR_VARIABLE_LIST(y), 0, 0); |
523 | 100 | ccv_nnc_dynamic_graph_evaluate(graph, linear, 0, TENSOR_VARIABLE_LIST(y), TENSOR_VARIABLE_LIST(y), 0, 0); |
524 | 100 | ccv_nnc_dynamic_graph_exec(graph, CMD_ADD_FORWARD(1, -1), ccv_nnc_no_hint, 0, TENSOR_VARIABLE_LIST(y, z), TENSOR_VARIABLE_LIST(y), 0, 0); |
525 | 100 | ccv_nnc_tensor_variable_t f = ccv_nnc_tensor_variable_new(graph); |
526 | 100 | ccv_nnc_dynamic_graph_exec(graph, CMD_EWPROD_FORWARD(), ccv_nnc_no_hint, 0, TENSOR_VARIABLE_LIST(y, y), TENSOR_VARIABLE_LIST(f), 0, 0); |
527 | 100 | ccv_nnc_tensor_variable_t dx = ccv_nnc_tensor_variable_new(graph); |
528 | 100 | ccv_nnc_dynamic_graph_backward(graph, TENSOR_VARIABLE_LIST(f), 0, TENSOR_VARIABLE_LIST(x), TENSOR_VARIABLE_LIST(dx), 0); |
529 | 100 | ccv_nnc_tensor_variable_free(graph, x); |
530 | 100 | ccv_nnc_tensor_variable_free(graph, y); |
531 | 100 | ccv_nnc_tensor_variable_free(graph, f); |
532 | 100 | ccv_nnc_tensor_variable_free(graph, dx); |
533 | 100 | if ((i % 2) == 1) |
534 | 50 | { |
535 | 50 | ccv_cnnp_model_set_minimizer(linear, CMD_SGD_FORWARD(0, 0.01, 1, 0.01, 0, 0), 0, 0, 0); |
536 | 50 | ccv_nnc_dynamic_graph_apply_gradients(graph, CMD_SGD_FORWARD(0, 0.01, 1, 0.01, 0, 0), TENSOR_VARIABLE_LIST(), TENSOR_VARIABLE_LIST(), 0, 0, 0); |
537 | 50 | } |
538 | 100 | } |
539 | 1 | ccv_nnc_tensor_variable_t x = ccv_nnc_tensor_variable_new(graph, CPU_TENSOR_NHWC(32F, 1)); |
540 | 1 | ccv_nnc_tensor_from_variable(graph, x)->data.f32[0] = 10; |
541 | 1 | ccv_nnc_tensor_variable_t y = ccv_nnc_tensor_variable_new(graph); |
542 | 1 | ccv_nnc_dynamic_graph_exec(graph, CMD_EWLOG_FORWARD(), ccv_nnc_no_hint, 0, TENSOR_VARIABLE_LIST(x), TENSOR_VARIABLE_LIST(y), 0, 0); |
543 | 1 | ccv_nnc_dynamic_graph_evaluate(graph, linear, 1, TENSOR_VARIABLE_LIST(y), TENSOR_VARIABLE_LIST(y), 0, 0); |
544 | 1 | REQUIRE_EQ_WITH_TOLERANCE(ccv_nnc_tensor_from_variable(graph, y)->data.f32[0], 5, 1e-2, "linear model should be trained to generate the same value as z"); |
545 | 1 | DYNAMIC_GRAPH_GEN(graph, CCV_NNC_LONG_DOT_GRAPH); |
546 | 1 | ccv_nnc_dynamic_graph_free(graph); |
547 | 1 | ccv_cnnp_model_free(linear); |
548 | 1 | } |
549 | | |
550 | | TEST_CASE("dynamic graph to accumulate gradients cross cnnp models") |
551 | 1 | { |
552 | 1 | ccv_nnc_dynamic_graph_t* const graph = ccv_nnc_dynamic_graph_new(); |
553 | 1 | ccv_cnnp_model_t* const linear = ccv_cnnp_dense(1, 1, 0, 1, 0); |
554 | 1 | ccv_nnc_tensor_variable_t a = ccv_nnc_tensor_variable_new(graph, CPU_TENSOR_NHWC(32F, 1)); |
555 | 1 | ccv_nnc_dynamic_graph_exec(graph, CMD_SET_FORWARD(0.2485), ccv_nnc_no_hint, 0, TENSOR_VARIABLE_LIST(), TENSOR_VARIABLE_LIST(a), 0, 0); |
556 | 1 | const ccv_nnc_tensor_param_t input = CPU_TENSOR_NHWC(32F, 1); |
557 | 1 | ccv_cnnp_model_compile(linear, &input, 1, CMD_NOOP(), CMD_NOOP()); |
558 | 1 | ccv_cnnp_model_set_parameter(linear, ccv_cnnp_model_parameters(linear, CCV_CNNP_PARAMETER_SELECT_WEIGHT, 0), ccv_nnc_tensor_from_variable(graph, a)); |
559 | 1 | ccv_nnc_tensor_variable_t a_grad = ccv_nnc_tensor_variable_new(graph); |
560 | 1 | ccv_nnc_tensor_variable_t saved_aux = ccv_nnc_tensor_variable_new(graph, CPU_TENSOR_NHWC(32F, 1)); |
561 | 1 | int i; |
562 | 1.00k | for (i = 0; i < 1000; i++1.00k ) |
563 | 1.00k | { |
564 | 1.00k | ccv_nnc_tensor_variable_t x = ccv_nnc_tensor_variable_new(graph, CPU_TENSOR_NHWC(32F, 1)); |
565 | 1.00k | ccv_nnc_tensor_from_variable(graph, x)->data.f32[0] = i; |
566 | 1.00k | ccv_nnc_tensor_variable_t t = ccv_nnc_tensor_variable_new(graph, CPU_TENSOR_NHWC(32F, 1)); |
567 | 1.00k | ccv_nnc_tensor_from_variable(graph, t)->data.f32[0] = -i; |
568 | 1.00k | ccv_nnc_tensor_variable_t y = ccv_nnc_tensor_variable_new(graph); |
569 | 1.00k | ccv_nnc_dynamic_graph_exec(graph, CMD_MUL_FORWARD(1), ccv_nnc_no_hint, 0, TENSOR_VARIABLE_LIST(x, a), TENSOR_VARIABLE_LIST(y), 0, 0); |
570 | 1.00k | ccv_nnc_tensor_variable_t z = ccv_nnc_tensor_variable_new(graph); |
571 | 1.00k | ccv_nnc_dynamic_graph_evaluate(graph, linear, 0, TENSOR_VARIABLE_LIST(y), TENSOR_VARIABLE_LIST(z), 0, 0); |
572 | 1.00k | ccv_nnc_tensor_variable_t n = ccv_nnc_tensor_variable_new(graph); |
573 | 1.00k | ccv_nnc_dynamic_graph_exec(graph, CMD_ADD_FORWARD(1, -1), ccv_nnc_no_hint, 0, TENSOR_VARIABLE_LIST(z, t), TENSOR_VARIABLE_LIST(n), 0, 0); |
574 | 1.00k | ccv_nnc_tensor_variable_t f = ccv_nnc_tensor_variable_new(graph); |
575 | 1.00k | ccv_nnc_dynamic_graph_exec(graph, CMD_MUL_FORWARD(1), ccv_nnc_no_hint, 0, TENSOR_VARIABLE_LIST(n, n), TENSOR_VARIABLE_LIST(f), 0, 0); |
576 | 1.00k | ccv_nnc_dynamic_graph_backward(graph, TENSOR_VARIABLE_LIST(f), 0, TENSOR_VARIABLE_LIST(a), TENSOR_VARIABLE_LIST(a_grad), 0); |
577 | 1.00k | ccv_nnc_tensor_variable_free(graph, n); |
578 | 1.00k | ccv_nnc_tensor_variable_free(graph, z); |
579 | 1.00k | ccv_nnc_tensor_variable_free(graph, y); |
580 | 1.00k | ccv_nnc_tensor_variable_free(graph, x); |
581 | 1.00k | ccv_nnc_tensor_variable_free(graph, t); |
582 | 1.00k | ccv_nnc_tensor_variable_free(graph, f); |
583 | 1.00k | if (((i + 1) % 5) == 0) |
584 | 200 | { |
585 | 200 | DYNAMIC_GRAPH_GEN(graph, CCV_NNC_LONG_DOT_GRAPH); |
586 | 200 | REQUIRE_EQ(ccv_nnc_dynamic_graph_bookkeeping_count(graph, CCV_NNC_SYMBOL_GRAPH_EXEC), 0, "everything should be freed"); |
587 | 200 | float lr = 0.001; |
588 | 200 | if (i >= 200) |
589 | 160 | lr = 0.0001; |
590 | 40 | else if (i >= 600) |
591 | 0 | lr = 0.00001; |
592 | 200 | ccv_cnnp_model_set_minimizer(linear, CMD_SGD_FORWARD(0, lr, 0.001, 0, 0, 0), 0, 0, 0); |
593 | 200 | ccv_nnc_dynamic_graph_apply_gradients(graph, CMD_SGD_FORWARD(0, lr, 0.001, 0, 0, 0), TENSOR_VARIABLE_LIST(a_grad), TENSOR_VARIABLE_LIST(a), &saved_aux, 0, 0); |
594 | 200 | } |
595 | 1.00k | } |
596 | 1 | ccv_nnc_tensor_variable_t x = ccv_nnc_tensor_variable_new(graph, CPU_TENSOR_NHWC(32F, 1)); |
597 | 1 | ccv_nnc_tensor_from_variable(graph, x)->data.f32[0] = 5; |
598 | 1 | ccv_nnc_tensor_variable_t y = ccv_nnc_tensor_variable_new(graph); |
599 | 1 | ccv_nnc_dynamic_graph_exec(graph, CMD_MUL_FORWARD(1), ccv_nnc_no_hint, 0, TENSOR_VARIABLE_LIST(x, a), TENSOR_VARIABLE_LIST(y), 0, 0); |
600 | 1 | ccv_nnc_tensor_variable_t z = ccv_nnc_tensor_variable_new(graph); |
601 | 1 | ccv_nnc_dynamic_graph_evaluate(graph, linear, 0, TENSOR_VARIABLE_LIST(y), TENSOR_VARIABLE_LIST(z), 0, 0); |
602 | 1 | REQUIRE_EQ_WITH_TOLERANCE(ccv_nnc_tensor_from_variable(graph, z)->data.f32[0], -5, 1e-2, "linear model should be trained to generate the same value as z"); |
603 | 1 | ccv_nnc_dynamic_graph_free(graph); |
604 | 1 | ccv_cnnp_model_free(linear); |
605 | 1 | } |
606 | | |
607 | | TEST_CASE("dynamic graph to accumulate gradients cross cnnp models with aliases") |
608 | 1 | { |
609 | 1 | ccv_nnc_dynamic_graph_t* const graph = ccv_nnc_dynamic_graph_new(); |
610 | 1 | ccv_cnnp_model_t* const linear = ccv_cnnp_sequential_new(MODEL_LIST( |
611 | 1 | ccv_cnnp_dense(1, 1, 0, 1, 0), |
612 | 1 | ccv_cnnp_reshape(0, DIM_ALLOC(1), DIM_ALLOC(), DIM_ALLOC(), 0), |
613 | 1 | ), 1, 0); |
614 | 1 | ccv_nnc_tensor_variable_t a = ccv_nnc_tensor_variable_new(graph, CPU_TENSOR_NHWC(32F, 1)); |
615 | 1 | ccv_nnc_dynamic_graph_exec(graph, CMD_SET_FORWARD(0.2485), ccv_nnc_no_hint, 0, TENSOR_VARIABLE_LIST(), TENSOR_VARIABLE_LIST(a), 0, 0); |
616 | 1 | const ccv_nnc_tensor_param_t input = CPU_TENSOR_NHWC(32F, 1); |
617 | 1 | ccv_cnnp_model_compile(linear, &input, 1, CMD_NOOP(), CMD_NOOP()); |
618 | 1 | ccv_cnnp_model_set_parameter(linear, ccv_cnnp_model_parameters(linear, CCV_CNNP_PARAMETER_SELECT_WEIGHT, 0), ccv_nnc_tensor_from_variable(graph, a)); |
619 | 1 | ccv_nnc_tensor_variable_t a_grad = ccv_nnc_tensor_variable_new(graph); |
620 | 1 | ccv_nnc_tensor_variable_t saved_aux = ccv_nnc_tensor_variable_new(graph, CPU_TENSOR_NHWC(32F, 1)); |
621 | 1 | int i; |
622 | 1.00k | for (i = 0; i < 1000; i++1.00k ) |
623 | 1.00k | { |
624 | 1.00k | ccv_nnc_tensor_variable_t x = ccv_nnc_tensor_variable_new(graph, CPU_TENSOR_NHWC(32F, 1)); |
625 | 1.00k | ccv_nnc_tensor_from_variable(graph, x)->data.f32[0] = i; |
626 | 1.00k | ccv_nnc_tensor_variable_t t = ccv_nnc_tensor_variable_new(graph, CPU_TENSOR_NHWC(32F, 1)); |
627 | 1.00k | ccv_nnc_tensor_from_variable(graph, t)->data.f32[0] = -i; |
628 | 1.00k | ccv_nnc_tensor_variable_t y = ccv_nnc_tensor_variable_new(graph); |
629 | 1.00k | ccv_nnc_dynamic_graph_exec(graph, CMD_MUL_FORWARD(1), ccv_nnc_no_hint, 0, TENSOR_VARIABLE_LIST(x, a), TENSOR_VARIABLE_LIST(y), 0, 0); |
630 | 1.00k | ccv_nnc_tensor_variable_t y_alias = ccv_nnc_tensor_variable_alias_new(graph, y, DIM_ALLOC(), DIM_ALLOC(1), CPU_TENSOR_NHWC(32F, 1)); |
631 | 1.00k | ccv_nnc_tensor_variable_t z = ccv_nnc_tensor_variable_new(graph); |
632 | 1.00k | ccv_nnc_dynamic_graph_evaluate(graph, linear, 0, TENSOR_VARIABLE_LIST(y_alias), TENSOR_VARIABLE_LIST(z), 0, 0); |
633 | 1.00k | ccv_nnc_tensor_variable_t n = ccv_nnc_tensor_variable_new(graph); |
634 | 1.00k | ccv_nnc_dynamic_graph_exec(graph, CMD_ADD_FORWARD(1, -1), ccv_nnc_no_hint, 0, TENSOR_VARIABLE_LIST(z, t), TENSOR_VARIABLE_LIST(n), 0, 0); |
635 | 1.00k | ccv_nnc_tensor_variable_t f = ccv_nnc_tensor_variable_new(graph); |
636 | 1.00k | ccv_nnc_dynamic_graph_exec(graph, CMD_MUL_FORWARD(1), ccv_nnc_no_hint, 0, TENSOR_VARIABLE_LIST(n, n), TENSOR_VARIABLE_LIST(f), 0, 0); |
637 | 1.00k | ccv_nnc_dynamic_graph_backward(graph, TENSOR_VARIABLE_LIST(f), 0, TENSOR_VARIABLE_LIST(a), TENSOR_VARIABLE_LIST(a_grad), 0); |
638 | 1.00k | ccv_nnc_tensor_variable_free(graph, n); |
639 | 1.00k | ccv_nnc_tensor_variable_free(graph, z); |
640 | 1.00k | ccv_nnc_tensor_variable_free(graph, y); |
641 | 1.00k | ccv_nnc_tensor_variable_free(graph, y_alias); |
642 | 1.00k | ccv_nnc_tensor_variable_free(graph, x); |
643 | 1.00k | ccv_nnc_tensor_variable_free(graph, t); |
644 | 1.00k | ccv_nnc_tensor_variable_free(graph, f); |
645 | 1.00k | if (((i + 1) % 5) == 0) |
646 | 200 | { |
647 | 200 | REQUIRE_EQ(ccv_nnc_dynamic_graph_bookkeeping_count(graph, CCV_NNC_SYMBOL_GRAPH_EXEC), 0, "everything should be freed"); |
648 | 200 | DYNAMIC_GRAPH_GEN(graph, CCV_NNC_LONG_DOT_GRAPH); |
649 | 200 | float lr = 0.001; |
650 | 200 | if (i >= 200) |
651 | 160 | lr = 0.0001; |
652 | 40 | else if (i >= 600) |
653 | 0 | lr = 0.00001; |
654 | 200 | ccv_cnnp_model_set_minimizer(linear, CMD_SGD_FORWARD(0, lr, 0.001, 0, 0, 0), 0, 0, 0); |
655 | 200 | ccv_nnc_dynamic_graph_apply_gradients(graph, CMD_SGD_FORWARD(0, lr, 0.001, 0, 0, 0), TENSOR_VARIABLE_LIST(a_grad), TENSOR_VARIABLE_LIST(a), &saved_aux, 0, 0); |
656 | 200 | } |
657 | 1.00k | } |
658 | 1 | ccv_nnc_tensor_variable_t x = ccv_nnc_tensor_variable_new(graph, CPU_TENSOR_NHWC(32F, 1)); |
659 | 1 | ccv_nnc_tensor_from_variable(graph, x)->data.f32[0] = 5; |
660 | 1 | ccv_nnc_tensor_variable_t y = ccv_nnc_tensor_variable_new(graph); |
661 | 1 | ccv_nnc_dynamic_graph_exec(graph, CMD_MUL_FORWARD(1), ccv_nnc_no_hint, 0, TENSOR_VARIABLE_LIST(x, a), TENSOR_VARIABLE_LIST(y), 0, 0); |
662 | 1 | ccv_nnc_tensor_variable_t z = ccv_nnc_tensor_variable_new(graph); |
663 | 1 | ccv_nnc_dynamic_graph_evaluate(graph, linear, 0, TENSOR_VARIABLE_LIST(y), TENSOR_VARIABLE_LIST(z), 0, 0); |
664 | 1 | REQUIRE_EQ_WITH_TOLERANCE(ccv_nnc_tensor_from_variable(graph, z)->data.f32[0], -5, 1e-2, "linear model should be trained to generate the same value as z"); |
665 | 1 | ccv_nnc_dynamic_graph_free(graph); |
666 | 1 | ccv_cnnp_model_free(linear); |
667 | 1 | } |
668 | | |
669 | | TEST_CASE("dynamic graph to use cnnp model for permute and reshape") |
670 | 1 | { |
671 | 1 | ccv_cnnp_model_t* const sequential = ccv_cnnp_sequential_new(MODEL_LIST( |
672 | 1 | ccv_cnnp_reshape(0, DIM_ALLOC(4, 3, 2), DIM_ALLOC(), DIM_ALLOC(), 0), |
673 | 1 | ccv_cnnp_permute(DIM_ALLOC(2, 0, 1), 0), |
674 | 1 | ccv_cnnp_reshape(0, DIM_ALLOC(2 * 4, 3), DIM_ALLOC(), DIM_ALLOC(), 0), |
675 | 1 | ), 1, 0); |
676 | 1 | ccv_nnc_dynamic_graph_t* const graph = ccv_nnc_dynamic_graph_new(); |
677 | 1 | ccv_nnc_tensor_variable_t x = ccv_nnc_tensor_variable_new(graph, CPU_TENSOR_NHWC(32F, 4, 3, 2)); |
678 | 1 | int i; |
679 | 25 | for (i = 0; i < 4 * 3 * 2; i++24 ) |
680 | 24 | ccv_nnc_tensor_from_variable(graph, x)->data.f32[i] = i; |
681 | 1 | ccv_nnc_tensor_variable_t y = ccv_nnc_tensor_variable_new(graph); |
682 | 1 | ccv_nnc_dynamic_graph_evaluate(graph, sequential, 0, TENSOR_VARIABLE_LIST(x), TENSOR_VARIABLE_LIST(y), 0, 0); |
683 | | // 0, 1, |
684 | | // 2, 3, |
685 | | // 4, 5 |
686 | | // |
687 | | // 6, 7, |
688 | | // 8, 9, |
689 | | // 10, 11 |
690 | | // |
691 | | // 12, 13, |
692 | | // 14, 15, |
693 | | // 16, 17 |
694 | | // |
695 | | // 18, 19, |
696 | | // 20, 21, |
697 | | // 22, 23 |
698 | | // |
699 | | // After permute: |
700 | | // 0, 2, 4, |
701 | | // 6, 8, 10, |
702 | | // 12, 14, 16, |
703 | | // 18, 20, 22 |
704 | | // |
705 | | // 1, 3, 5, |
706 | | // 7, 9, 11, |
707 | | // 13, 15, 17, |
708 | | // 19, 21, 23 |
709 | 1 | float btp[] = { |
710 | 1 | 0, 2, 4, |
711 | 1 | 6, 8, 10, |
712 | 1 | 12, 14, 16, |
713 | 1 | 18, 20, 22, |
714 | 1 | 1, 3, 5, |
715 | 1 | 7, 9, 11, |
716 | 1 | 13, 15, 17, |
717 | 1 | 19, 21, 23 |
718 | 1 | }; |
719 | 1 | ccv_nnc_tensor_t bt = ccv_nnc_tensor(btp, CPU_TENSOR_NHWC(32F, 2 * 4, 3), 0); |
720 | 1 | REQUIRE_TENSOR_EQ(ccv_nnc_tensor_from_variable(graph, y), &bt, "should materialize permute before reshape"); |
721 | 1 | ccv_nnc_dynamic_graph_free(graph); |
722 | 1 | ccv_cnnp_model_free(sequential); |
723 | 1 | } |
724 | | |
725 | | TEST_CASE("dynamic graph to compute f(x) = x * log(x) + 1.2 * x, f'(x) and sum on x = 19, 10") |
726 | 1 | { |
727 | 1 | ccv_nnc_dynamic_graph_t* const graph = ccv_nnc_dynamic_graph_new(); |
728 | 1 | ccv_nnc_tensor_variable_t x = ccv_nnc_tensor_variable_new(graph, CPU_TENSOR_NHWC(32F, 1)); |
729 | 1 | ccv_nnc_tensor_from_variable(graph, x)->data.f32[0] = 19; |
730 | 1 | ccv_nnc_tensor_variable_t f = ccv_nnc_tensor_variable_new(graph); |
731 | 1 | ccv_nnc_dynamic_graph_exec(graph, CMD_EWLOG_FORWARD(), ccv_nnc_no_hint, 0, TENSOR_VARIABLE_LIST(x), TENSOR_VARIABLE_LIST(f), 0, 0); |
732 | 1 | ccv_nnc_dynamic_graph_exec(graph, CMD_EWPROD_FORWARD(), ccv_nnc_no_hint, 0, TENSOR_VARIABLE_LIST(x, f), TENSOR_VARIABLE_LIST(f), 0, 0); |
733 | 1 | ccv_nnc_tensor_variable_t y = ccv_nnc_tensor_variable_new(graph, CPU_TENSOR_NHWC(32F, 1)); |
734 | 1 | ccv_nnc_tensor_from_variable(graph, y)->data.f32[0] = 1.2; |
735 | 1 | ccv_nnc_tensor_variable_t z = ccv_nnc_tensor_variable_new(graph); |
736 | 1 | ccv_nnc_dynamic_graph_exec(graph, CMD_EWPROD_FORWARD(), ccv_nnc_no_hint, 0, TENSOR_VARIABLE_LIST(x, y), TENSOR_VARIABLE_LIST(z), 0, 0); |
737 | 1 | ccv_nnc_dynamic_graph_exec(graph, CMD_EWSUM_FORWARD(), ccv_nnc_no_hint, 0, TENSOR_VARIABLE_LIST(f, z), TENSOR_VARIABLE_LIST(f), 0, 0); |
738 | 1 | REQUIRE_EQ_WITH_TOLERANCE(ccv_nnc_tensor_from_variable(graph, f)->data.f32[0], 19 * logf(19) + 1.2 * 19, 1e-5, "f(x) = 1.2 * 19 + 19 * log(19)"); |
739 | | // Do gradient computation multiple times. |
740 | 1 | ccv_nnc_tensor_variable_t dx = ccv_nnc_tensor_variable_new(graph); |
741 | 1 | ccv_nnc_dynamic_graph_backward(graph, TENSOR_VARIABLE_LIST(f), 0, TENSOR_VARIABLE_LIST(x), TENSOR_VARIABLE_LIST(dx), 0); |
742 | 1 | REQUIRE_EQ_WITH_TOLERANCE(ccv_nnc_tensor_from_variable(graph, dx)->data.f32[0], logf(19) + 1 + 1.2, 1e-5, "f'(x) = 1.2 + log(19) + 19 * 1 / 19"); |
743 | 1 | ccv_nnc_tensor_variable_free(graph, x); |
744 | 1 | x = ccv_nnc_tensor_variable_new(graph, CPU_TENSOR_NHWC(32F, 1)); |
745 | 1 | ccv_nnc_tensor_from_variable(graph, x)->data.f32[0] = 10; |
746 | 1 | ccv_nnc_dynamic_graph_exec(graph, CMD_EWLOG_FORWARD(), ccv_nnc_no_hint, 0, TENSOR_VARIABLE_LIST(x), TENSOR_VARIABLE_LIST(f), 0, 0); |
747 | 1 | ccv_nnc_dynamic_graph_exec(graph, CMD_EWPROD_FORWARD(), ccv_nnc_no_hint, 0, TENSOR_VARIABLE_LIST(x, f), TENSOR_VARIABLE_LIST(f), 0, 0); |
748 | 1 | ccv_nnc_tensor_from_variable(graph, y)->data.f32[0] = 1.2; |
749 | 1 | ccv_nnc_dynamic_graph_exec(graph, CMD_EWPROD_FORWARD(), ccv_nnc_no_hint, 0, TENSOR_VARIABLE_LIST(x, y), TENSOR_VARIABLE_LIST(z), 0, 0); |
750 | 1 | ccv_nnc_dynamic_graph_exec(graph, CMD_EWSUM_FORWARD(), ccv_nnc_no_hint, 0, TENSOR_VARIABLE_LIST(f, z), TENSOR_VARIABLE_LIST(f), 0, 0); |
751 | 1 | REQUIRE_EQ_WITH_TOLERANCE(ccv_nnc_tensor_from_variable(graph, f)->data.f32[0], 10 * logf(10) + 1.2 * 10, 1e-5, "f(x) = 1.2 * 10 + 10 * log(10)"); |
752 | 1 | ccv_nnc_dynamic_graph_backward(graph, TENSOR_VARIABLE_LIST(f), 0, TENSOR_VARIABLE_LIST(x), TENSOR_VARIABLE_LIST(dx), 0); |
753 | 1 | REQUIRE_EQ_WITH_TOLERANCE(ccv_nnc_tensor_from_variable(graph, dx)->data.f32[0], logf(19) + 1 + 1.2 + logf(10) + 1 + 1.2, 1e-5, "f'(x) = 1.2 + log(19) + 19 * 1 / 19 + 1.2 + log(19) + 19 * 1 / 19"); |
754 | 1 | DYNAMIC_GRAPH_GEN(graph, CCV_NNC_LONG_DOT_GRAPH); |
755 | 1 | ccv_nnc_dynamic_graph_free(graph); |
756 | 1 | } |
757 | | |
758 | | TEST_CASE("dynamic graph to compute f(x) = x * log(x) + y' where y = 1.2 * x and y' is an alias to y") |
759 | 1 | { |
760 | 1 | ccv_nnc_dynamic_graph_t* const graph = ccv_nnc_dynamic_graph_new(); |
761 | 1 | ccv_nnc_tensor_variable_t x = ccv_nnc_tensor_variable_new(graph, CPU_TENSOR_NHWC(32F, 1)); |
762 | 1 | ccv_nnc_tensor_from_variable(graph, x)->data.f32[0] = 19; |
763 | 1 | ccv_nnc_tensor_variable_t f = ccv_nnc_tensor_variable_new(graph); |
764 | 1 | ccv_nnc_dynamic_graph_exec(graph, CMD_EWLOG_FORWARD(), ccv_nnc_no_hint, 0, TENSOR_VARIABLE_LIST(x), TENSOR_VARIABLE_LIST(f), 0, 0); |
765 | 1 | ccv_nnc_dynamic_graph_exec(graph, CMD_EWPROD_FORWARD(), ccv_nnc_no_hint, 0, TENSOR_VARIABLE_LIST(x, f), TENSOR_VARIABLE_LIST(f), 0, 0); |
766 | 1 | ccv_nnc_tensor_variable_t z = ccv_nnc_tensor_variable_new(graph, CPU_TENSOR_NHWC(32F, 1)); |
767 | 1 | ccv_nnc_tensor_from_variable(graph, z)->data.f32[0] = 1.2; |
768 | 1 | ccv_nnc_tensor_variable_t y = ccv_nnc_tensor_variable_new(graph, CPU_TENSOR_NHWC(32F, 1)); |
769 | 1 | ccv_nnc_tensor_variable_t y_hat = ccv_nnc_tensor_variable_alias_new(graph, y, ccv_nnc_no_ofs, DIM_ALLOC(), CPU_TENSOR_NHWC(32F, 1)); |
770 | 1 | ccv_nnc_dynamic_graph_exec(graph, CMD_EWPROD_FORWARD(), ccv_nnc_no_hint, 0, TENSOR_VARIABLE_LIST(x, z), TENSOR_VARIABLE_LIST(y), 0, 0); |
771 | 1 | ccv_nnc_dynamic_graph_exec(graph, CMD_EWSUM_FORWARD(), ccv_nnc_no_hint, 0, TENSOR_VARIABLE_LIST(f, y_hat), TENSOR_VARIABLE_LIST(f), 0, 0); |
772 | 1 | REQUIRE_EQ_WITH_TOLERANCE(ccv_nnc_tensor_from_variable(graph, f)->data.f32[0], 19 * logf(19) + 1.2 * 19, 1e-5, "f(x) = 1.2 * 19 + 19 * log(19)"); |
773 | 1 | ccv_nnc_tensor_variable_free(graph, y_hat); |
774 | 1 | ccv_nnc_tensor_variable_free(graph, f); |
775 | 1 | ccv_nnc_tensor_variable_free(graph, x); |
776 | 1 | ccv_nnc_tensor_variable_free(graph, z); |
777 | 1 | ccv_nnc_tensor_variable_free(graph, y); |
778 | 1 | REQUIRE_EQ(ccv_nnc_dynamic_graph_bookkeeping_count(graph, CCV_NNC_SYMBOL_TENSOR), 0, "should have no tensor left"); |
779 | 1 | ccv_nnc_dynamic_graph_free(graph); |
780 | 1 | } |
781 | | |
782 | | TEST_CASE("dynamic graph to compute f(x) = x * log(x) + y' where y'' = 1.2 * x and both y'' and y' are aliases to y") |
783 | 1 | { |
784 | 1 | ccv_nnc_dynamic_graph_t* const graph = ccv_nnc_dynamic_graph_new(); |
785 | 1 | ccv_nnc_tensor_variable_t x = ccv_nnc_tensor_variable_new(graph, CPU_TENSOR_NHWC(32F, 1)); |
786 | 1 | ccv_nnc_tensor_from_variable(graph, x)->data.f32[0] = 19; |
787 | 1 | ccv_nnc_tensor_variable_t f = ccv_nnc_tensor_variable_new(graph); |
788 | 1 | ccv_nnc_dynamic_graph_exec(graph, CMD_EWLOG_FORWARD(), ccv_nnc_no_hint, 0, TENSOR_VARIABLE_LIST(x), TENSOR_VARIABLE_LIST(f), 0, 0); |
789 | 1 | ccv_nnc_dynamic_graph_exec(graph, CMD_EWPROD_FORWARD(), ccv_nnc_no_hint, 0, TENSOR_VARIABLE_LIST(x, f), TENSOR_VARIABLE_LIST(f), 0, 0); |
790 | 1 | ccv_nnc_tensor_variable_t z = ccv_nnc_tensor_variable_new(graph, CPU_TENSOR_NHWC(32F, 1)); |
791 | 1 | ccv_nnc_tensor_from_variable(graph, z)->data.f32[0] = 1.2; |
792 | 1 | ccv_nnc_tensor_variable_t y = ccv_nnc_tensor_variable_new(graph, CPU_TENSOR_NHWC(32F, 1)); |
793 | 1 | ccv_nnc_tensor_variable_t y_hat = ccv_nnc_tensor_variable_alias_new(graph, y, ccv_nnc_no_ofs, DIM_ALLOC(), CPU_TENSOR_NHWC(32F, 1)); |
794 | 1 | ccv_nnc_tensor_variable_t y_double_hat = ccv_nnc_tensor_variable_alias_new(graph, y, ccv_nnc_no_ofs, DIM_ALLOC(), CPU_TENSOR_NHWC(32F, 1)); |
795 | 1 | ccv_nnc_dynamic_graph_exec(graph, CMD_EWPROD_FORWARD(), ccv_nnc_no_hint, 0, TENSOR_VARIABLE_LIST(x, z), TENSOR_VARIABLE_LIST(y_double_hat), 0, 0); |
796 | 1 | ccv_nnc_dynamic_graph_exec(graph, CMD_EWSUM_FORWARD(), ccv_nnc_no_hint, 0, TENSOR_VARIABLE_LIST(f, y_hat), TENSOR_VARIABLE_LIST(f), 0, 0); |
797 | 1 | REQUIRE_EQ_WITH_TOLERANCE(ccv_nnc_tensor_from_variable(graph, f)->data.f32[0], 19 * logf(19) + 1.2 * 19, 1e-5, "f(x) = 1.2 * 19 + 19 * log(19)"); |
798 | 1 | ccv_nnc_tensor_variable_free(graph, y_hat); |
799 | 1 | ccv_nnc_tensor_variable_free(graph, y_double_hat); |
800 | 1 | ccv_nnc_tensor_variable_free(graph, f); |
801 | 1 | ccv_nnc_tensor_variable_free(graph, x); |
802 | 1 | ccv_nnc_tensor_variable_free(graph, z); |
803 | 1 | ccv_nnc_tensor_variable_free(graph, y); |
804 | 1 | REQUIRE_EQ(ccv_nnc_dynamic_graph_bookkeeping_count(graph, CCV_NNC_SYMBOL_TENSOR), 0, "should have no tensor left"); |
805 | 1 | ccv_nnc_dynamic_graph_free(graph); |
806 | 1 | } |
807 | | |
808 | | TEST_CASE("dynamic graph to compute f(x) = x * y, where y[0] = 10 * z, y[1] = 2 * z, z = [2], x = [10], should free all variables") |
809 | 1 | { |
810 | 1 | ccv_nnc_dynamic_graph_t* const graph = ccv_nnc_dynamic_graph_new(); |
811 | 1 | ccv_nnc_tensor_variable_t x = ccv_nnc_tensor_variable_new(graph, CPU_TENSOR_NHWC(32F, 1)); |
812 | 1 | ccv_nnc_tensor_variable_t y = ccv_nnc_tensor_variable_new(graph, CPU_TENSOR_NHWC(32F, 2)); |
813 | 1 | ccv_nnc_tensor_variable_t z = ccv_nnc_tensor_variable_new(graph, CPU_TENSOR_NHWC(32F, 1)); |
814 | 1 | ccv_nnc_dynamic_graph_exec(graph, CMD_SET_FORWARD(2), ccv_nnc_no_hint, 0, TENSOR_VARIABLE_LIST(), TENSOR_VARIABLE_LIST(z), 0, 0); |
815 | 1 | ccv_nnc_dynamic_graph_exec(graph, CMD_SET_FORWARD(10), ccv_nnc_no_hint, 0, TENSOR_VARIABLE_LIST(), TENSOR_VARIABLE_LIST(x), 0, 0); |
816 | 1 | ccv_nnc_tensor_variable_t y_1 = ccv_nnc_tensor_variable_alias_new(graph, y, ccv_nnc_no_ofs, DIM_ALLOC(1), CPU_TENSOR_NHWC(32F, 1)); |
817 | 1 | ccv_nnc_tensor_variable_t y_2 = ccv_nnc_tensor_variable_alias_new(graph, y, DIM_ALLOC(1), DIM_ALLOC(1), CPU_TENSOR_NHWC(32F, 1)); |
818 | 1 | ccv_nnc_dynamic_graph_exec(graph, CMD_SCALAR_MUL_FORWARD(10), ccv_nnc_no_hint, 0, TENSOR_VARIABLE_LIST(z), TENSOR_VARIABLE_LIST(y_1), 0, 0); |
819 | 1 | ccv_nnc_dynamic_graph_exec(graph, CMD_SCALAR_MUL_FORWARD(2), ccv_nnc_no_hint, 0, TENSOR_VARIABLE_LIST(z), TENSOR_VARIABLE_LIST(y_2), 0, 0); |
820 | 1 | ccv_nnc_tensor_variable_t f = ccv_nnc_tensor_variable_new(graph); |
821 | 1 | ccv_nnc_dynamic_graph_exec(graph, CMD_MUL_FORWARD(1), ccv_nnc_no_hint, 0, TENSOR_VARIABLE_LIST(x, y), TENSOR_VARIABLE_LIST(f), 0, 0); |
822 | 1 | float gt[] = {10 * 2 * 10, 2 * 2 * 10}; |
823 | 1 | REQUIRE_ARRAY_EQ_WITH_TOLERANCE(float, ccv_nnc_tensor_from_variable(graph, f)->data.f32, gt, 2, 1e-5, "should be equal"); |
824 | 1 | ccv_nnc_tensor_variable_free(graph, z); |
825 | 1 | ccv_nnc_tensor_variable_free(graph, y_1); |
826 | 1 | ccv_nnc_tensor_variable_free(graph, y_2); |
827 | 1 | ccv_nnc_tensor_variable_free(graph, x); |
828 | 1 | ccv_nnc_tensor_variable_free(graph, y); |
829 | 1 | ccv_nnc_tensor_variable_free(graph, f); |
830 | 1 | REQUIRE_EQ(ccv_nnc_dynamic_graph_bookkeeping_count(graph, CCV_NNC_SYMBOL_GRAPH_EXEC), 0, "everything should be freed"); |
831 | 1 | REQUIRE_EQ(ccv_nnc_dynamic_graph_bookkeeping_count(graph, CCV_NNC_SYMBOL_TENSOR), 0, "everything should be freed"); |
832 | 1 | ccv_nnc_dynamic_graph_free(graph); |
833 | 1 | } |
834 | | |
835 | | TEST_CASE("dynamic graph to compute f(x) = x * y, where y[0] = 10 * z, y[1] = 2 * z, z = [2], x = [10], freed y[0], y[1] before use") |
836 | 1 | { |
837 | 1 | ccv_nnc_dynamic_graph_t* const graph = ccv_nnc_dynamic_graph_new(); |
838 | 1 | ccv_nnc_tensor_variable_t x = ccv_nnc_tensor_variable_new(graph, CPU_TENSOR_NHWC(32F, 1)); |
839 | 1 | ccv_nnc_tensor_variable_t y = ccv_nnc_tensor_variable_new(graph, CPU_TENSOR_NHWC(32F, 2)); |
840 | 1 | ccv_nnc_tensor_variable_t z = ccv_nnc_tensor_variable_new(graph, CPU_TENSOR_NHWC(32F, 1)); |
841 | 1 | ccv_nnc_dynamic_graph_exec(graph, CMD_SET_FORWARD(2), ccv_nnc_no_hint, 0, TENSOR_VARIABLE_LIST(), TENSOR_VARIABLE_LIST(z), 0, 0); |
842 | 1 | ccv_nnc_dynamic_graph_exec(graph, CMD_SET_FORWARD(10), ccv_nnc_no_hint, 0, TENSOR_VARIABLE_LIST(), TENSOR_VARIABLE_LIST(x), 0, 0); |
843 | 1 | ccv_nnc_tensor_variable_t y_1 = ccv_nnc_tensor_variable_alias_new(graph, y, ccv_nnc_no_ofs, DIM_ALLOC(1), CPU_TENSOR_NHWC(32F, 1)); |
844 | 1 | ccv_nnc_tensor_variable_t y_2 = ccv_nnc_tensor_variable_alias_new(graph, y, DIM_ALLOC(1), DIM_ALLOC(1), CPU_TENSOR_NHWC(32F, 1)); |
845 | 1 | ccv_nnc_dynamic_graph_exec(graph, CMD_SCALAR_MUL_FORWARD(10), ccv_nnc_no_hint, 0, TENSOR_VARIABLE_LIST(z), TENSOR_VARIABLE_LIST(y_1), 0, 0); |
846 | 1 | ccv_nnc_dynamic_graph_exec(graph, CMD_SCALAR_MUL_FORWARD(2), ccv_nnc_no_hint, 0, TENSOR_VARIABLE_LIST(z), TENSOR_VARIABLE_LIST(y_2), 0, 0); |
847 | 1 | ccv_nnc_tensor_variable_free(graph, y_1); |
848 | 1 | ccv_nnc_tensor_variable_free(graph, y_2); |
849 | 1 | ccv_nnc_tensor_variable_t f = ccv_nnc_tensor_variable_new(graph); |
850 | 1 | ccv_nnc_dynamic_graph_exec(graph, CMD_MUL_FORWARD(1), ccv_nnc_no_hint, 0, TENSOR_VARIABLE_LIST(x, y), TENSOR_VARIABLE_LIST(f), 0, 0); |
851 | 1 | REQUIRE_EQ(ccv_nnc_dynamic_graph_bookkeeping_count(graph, CCV_NNC_SYMBOL_GRAPH_EXEC), 3, "should keep 3 ops"); |
852 | 1 | float gt[] = {10 * 2 * 10, 2 * 2 * 10}; |
853 | 1 | REQUIRE_ARRAY_EQ_WITH_TOLERANCE(float, ccv_nnc_tensor_from_variable(graph, f)->data.f32, gt, 2, 1e-5, "should be equal"); |
854 | 1 | ccv_nnc_tensor_variable_free(graph, z); |
855 | 1 | ccv_nnc_tensor_variable_free(graph, x); |
856 | 1 | ccv_nnc_tensor_variable_free(graph, y); |
857 | 1 | ccv_nnc_tensor_variable_free(graph, f); |
858 | 1 | REQUIRE_EQ(ccv_nnc_dynamic_graph_bookkeeping_count(graph, CCV_NNC_SYMBOL_GRAPH_EXEC), 0, "everything should be freed"); |
859 | 1 | REQUIRE_EQ(ccv_nnc_dynamic_graph_bookkeeping_count(graph, CCV_NNC_SYMBOL_TENSOR), 0, "everything should be freed"); |
860 | 1 | ccv_nnc_dynamic_graph_free(graph); |
861 | 1 | } |
862 | | |
863 | | TEST_CASE("dynamic graph to compute f(x) = x * y', where y[0, 0] = 2, y[0, 1] = 4, x = [10], y' = y^T, gradient against y") |
864 | 1 | { |
865 | 1 | ccv_nnc_dynamic_graph_t* const graph = ccv_nnc_dynamic_graph_new(); |
866 | 1 | ccv_nnc_tensor_variable_t x = ccv_nnc_tensor_variable_new(graph, CPU_TENSOR_NHWC(32F, 1)); |
867 | 1 | ccv_nnc_tensor_variable_t y = ccv_nnc_tensor_variable_new(graph, CPU_TENSOR_NHWC(32F, 2)); |
868 | 1 | ccv_nnc_tensor_from_variable(graph, y)->data.f32[0] = 2; |
869 | 1 | ccv_nnc_tensor_from_variable(graph, y)->data.f32[0] = 4; |
870 | 1 | ccv_nnc_dynamic_graph_exec(graph, CMD_SET_FORWARD(10), ccv_nnc_no_hint, 0, TENSOR_VARIABLE_LIST(), TENSOR_VARIABLE_LIST(x), 0, 0); |
871 | 1 | ccv_nnc_tensor_variable_t y_t = ccv_nnc_tensor_variable_alias_new(graph, y, ccv_nnc_no_ofs, DIM_ALLOC(), CPU_TENSOR_NHWC(32F, 2, 1)); |
872 | 1 | ccv_nnc_tensor_variable_t f = ccv_nnc_tensor_variable_new(graph); |
873 | 1 | ccv_nnc_dynamic_graph_exec(graph, CMD_MUL_FORWARD(1), ccv_nnc_no_hint, 0, TENSOR_VARIABLE_LIST(x, y_t), TENSOR_VARIABLE_LIST(f), 0, 0); |
874 | 1 | ccv_nnc_tensor_variable_t dy_t = ccv_nnc_tensor_variable_new(graph); |
875 | 1 | ccv_nnc_tensor_variable_free(graph, x); // Cannot free y_t, it has to be freed in lock step with y. |
876 | 1 | ccv_nnc_tensor_variable_free(graph, y); |
877 | 1 | ccv_nnc_dynamic_graph_backward(graph, TENSOR_VARIABLE_LIST(f), 0, TENSOR_VARIABLE_LIST(y_t), TENSOR_VARIABLE_LIST(dy_t), 0); |
878 | 1 | float gt[] = {10, 10}; |
879 | 1 | REQUIRE_ARRAY_EQ_WITH_TOLERANCE(float, ccv_nnc_tensor_from_variable(graph, dy_t)->data.f32, gt, 2, 1e-5, "should be equal"); |
880 | 1 | ccv_nnc_tensor_variable_free(graph, y_t); |
881 | 1 | ccv_nnc_tensor_variable_free(graph, f); |
882 | 1 | REQUIRE_EQ(ccv_nnc_dynamic_graph_bookkeeping_count(graph, CCV_NNC_SYMBOL_GRAPH_EXEC), 0, "everything should be freed"); |
883 | 1 | REQUIRE_EQ(ccv_nnc_dynamic_graph_bookkeeping_count(graph, CCV_NNC_SYMBOL_TENSOR), 0, "everything should be freed"); |
884 | 1 | ccv_nnc_dynamic_graph_free(graph); |
885 | 1 | } |
886 | | |
887 | | TEST_CASE("long chain to autograd, simulate random free of unused tensors due to garbage collection") |
888 | 1 | { |
889 | 1 | ccv_nnc_dynamic_graph_t* const graph = ccv_nnc_dynamic_graph_new(); |
890 | 1 | ccv_nnc_tensor_variable_t x[10]; |
891 | 1 | ccv_nnc_tensor_variable_t f[10]; |
892 | 1 | ccv_nnc_tensor_variable_t y = ccv_nnc_tensor_variable_new(graph, CPU_TENSOR_NHWC(32F, 1)); |
893 | 1 | ccv_nnc_tensor_variable_t dy[3]; |
894 | 1 | ccv_nnc_tensor_from_variable(graph, y)->data.f32[0] = 1.3; |
895 | 1 | int i, j; |
896 | 11 | for (i = 0; i < 10; i++10 ) |
897 | 10 | { |
898 | 10 | x[i] = ccv_nnc_tensor_variable_new(graph, CPU_TENSOR_NHWC(32F, 1)); |
899 | 10 | ccv_nnc_tensor_from_variable(graph, x[i])->data.f32[0] = i + 1; |
900 | 10 | f[i] = ccv_nnc_tensor_variable_new(graph); |
901 | 10 | ccv_nnc_dynamic_graph_exec(graph, CMD_MUL_FORWARD(1), ccv_nnc_no_hint, 0, TENSOR_VARIABLE_LIST(x[i], y), TENSOR_VARIABLE_LIST(f[i]), 0, 0); |
902 | 10 | if (i >= 5) |
903 | 5 | { |
904 | 5 | if (i - 5 < 3) |
905 | 3 | dy[i - 5] = ccv_nnc_tensor_variable_new(graph); |
906 | 5 | ccv_nnc_dynamic_graph_backward(graph, TENSOR_VARIABLE_LIST(f[i]), 0, TENSOR_VARIABLE_LIST(y), TENSOR_VARIABLE_LIST(dy[ccv_min(2, i - 5)]), 0); |
907 | 5 | } |
908 | 10 | if (i == 7) |
909 | 6 | for (j = 0; 1 j < 5; j++5 ) |
910 | 5 | ccv_nnc_tensor_variable_free(graph, f[j]); |
911 | 10 | } |
912 | 1 | REQUIRE_EQ_WITH_TOLERANCE(ccv_nnc_tensor_from_variable(graph, f[9])->data.f32[0], 1.3 * 10, 1e-5, "should equal to 1.3 * 10"); |
913 | 1 | REQUIRE_EQ_WITH_TOLERANCE(ccv_nnc_tensor_from_variable(graph, dy[2])->data.f32[0], 8 + 9 + 10, 1e-5, "should equal to sum of the last 3"); |
914 | 11 | for (i = 0; i < 10; i++10 ) |
915 | 10 | ccv_nnc_tensor_variable_free(graph, x[i]); |
916 | 1 | ccv_nnc_dynamic_graph_free(graph); |
917 | 1 | } |
918 | | |
919 | | TEST_CASE("parallel exec with a stream, focus on uma tensors") |
920 | 1 | { |
921 | 1 | ccv_nnc_dynamic_graph_t* const graph = ccv_nnc_dynamic_graph_new(); |
922 | 1 | ccv_nnc_stream_context_t* const stream = ccv_nnc_stream_context_new(CCV_STREAM_CONTEXT_CPU); |
923 | 1 | ccv_nnc_tensor_variable_t x[2]; |
924 | 1 | x[0] = ccv_nnc_tensor_variable_new(graph, CPU_TENSOR_NHWC(32F, 1)); |
925 | 1 | x[1] = ccv_nnc_tensor_variable_new(graph, CPU_TENSOR_NHWC(32F, 1)); |
926 | 1 | ccv_nnc_tensor_from_variable(graph, x[0])->data.f32[0] = 10; |
927 | 1 | ccv_nnc_tensor_from_variable(graph, x[1])->data.f32[0] = 9; |
928 | 1 | ccv_nnc_tensor_variable_t y[2]; |
929 | 1 | y[0] = ccv_nnc_tensor_variable_new(graph); |
930 | 1 | y[1] = ccv_nnc_tensor_variable_new(graph); |
931 | 1 | ccv_nnc_dynamic_graph_exec(graph, CMD_SCALAR_MUL_FORWARD(0.4), ccv_nnc_no_hint, 0, TENSOR_VARIABLE_LIST(x[0], x[1]), TENSOR_VARIABLE_LIST(y[0], y[1]), 2, stream); |
932 | 1 | ccv_nnc_stream_context_wait(stream); |
933 | 1 | ccv_nnc_stream_context_free(stream); |
934 | 1 | REQUIRE_EQ_WITH_TOLERANCE(ccv_nnc_tensor_from_variable(graph, y[0])->data.f32[0], 10 * 0.4, 1e-5, "should match the result"); |
935 | 1 | REQUIRE_EQ_WITH_TOLERANCE(ccv_nnc_tensor_from_variable(graph, y[1])->data.f32[0], 9 * 0.4, 1e-5, "should match the result"); |
936 | 1 | ccv_nnc_dynamic_graph_free(graph); |
937 | 1 | } |
938 | | |
939 | | TEST_CASE("parallel exec with a stream, focus on potential memory issues") |
940 | 1 | { |
941 | 1 | ccv_nnc_dynamic_graph_t* const graph = ccv_nnc_dynamic_graph_new(); |
942 | 1 | ccv_nnc_stream_context_t* const stream = ccv_nnc_stream_context_new(CCV_STREAM_CONTEXT_CPU); |
943 | 1 | ccv_nnc_tensor_variable_t x[2]; |
944 | 1 | x[0] = ccv_nnc_tensor_variable_new(graph, CPU_NUMA_TENSOR_NHWC(000, 32F, 1)); |
945 | 1 | x[1] = ccv_nnc_tensor_variable_new(graph, CPU_NUMA_TENSOR_NHWC(000, 32F, 1)); |
946 | 1 | ccv_nnc_tensor_from_variable(graph, x[0])->data.f32[0] = 10; |
947 | 1 | ccv_nnc_tensor_from_variable(graph, x[1])->data.f32[0] = 9; |
948 | 1 | ccv_nnc_tensor_variable_t y[2]; |
949 | 1 | y[0] = ccv_nnc_tensor_variable_new(graph); |
950 | 1 | y[1] = ccv_nnc_tensor_variable_new(graph); |
951 | 1 | ccv_nnc_dynamic_graph_exec(graph, CMD_SCALAR_MUL_FORWARD(0.4), ccv_nnc_no_hint, 0, TENSOR_VARIABLE_LIST(x[0], x[1]), TENSOR_VARIABLE_LIST(y[0], y[1]), 2, stream); |
952 | 1 | ccv_nnc_stream_context_wait(stream); |
953 | 1 | ccv_nnc_stream_context_free(stream); |
954 | 1 | REQUIRE_EQ_WITH_TOLERANCE(ccv_nnc_tensor_from_variable(graph, y[0])->data.f32[0], 10 * 0.4, 1e-5, "should match the result"); |
955 | 1 | REQUIRE_EQ_WITH_TOLERANCE(ccv_nnc_tensor_from_variable(graph, y[1])->data.f32[0], 9 * 0.4, 1e-5, "should match the result"); |
956 | 1 | ccv_nnc_dynamic_graph_free(graph); |
957 | 1 | } |
958 | | |
959 | | TEST_CASE("query whether a tensor variable depends on another") |
960 | 1 | { |
961 | 1 | ccv_nnc_dynamic_graph_t* const graph = ccv_nnc_dynamic_graph_new(); |
962 | 1 | ccv_nnc_tensor_variable_t a = ccv_nnc_tensor_variable_new(graph, CPU_TENSOR_NHWC(32F, 1)); |
963 | 1 | ccv_nnc_tensor_from_variable(graph, a)->data.f32[0] = 10; |
964 | 1 | ccv_nnc_tensor_variable_t b = ccv_nnc_tensor_variable_new(graph); |
965 | 1 | ccv_nnc_dynamic_graph_exec(graph, CMD_SCALAR_MUL_FORWARD(0.5), ccv_nnc_no_hint, 0, TENSOR_VARIABLE_LIST(a), TENSOR_VARIABLE_LIST(b), 0, 0); |
966 | 1 | ccv_nnc_tensor_variable_t c = ccv_nnc_tensor_variable_new(graph, CPU_TENSOR_NHWC(32F, 1)); |
967 | 1 | ccv_nnc_tensor_from_variable(graph, c)->data.f32[0] = 9; |
968 | 1 | ccv_nnc_tensor_variable_t d = ccv_nnc_tensor_variable_new(graph); |
969 | 1 | ccv_nnc_dynamic_graph_exec(graph, CMD_SCALAR_MUL_FORWARD(0.4), ccv_nnc_no_hint, 0, TENSOR_VARIABLE_LIST(c), TENSOR_VARIABLE_LIST(d), 0, 0); |
970 | 1 | ccv_nnc_tensor_variable_t e = ccv_nnc_tensor_variable_new(graph); |
971 | 1 | ccv_nnc_dynamic_graph_exec(graph, CMD_SCALAR_MUL_FORWARD(2), ccv_nnc_no_hint, 0, TENSOR_VARIABLE_LIST(d), TENSOR_VARIABLE_LIST(e), 0, 0); |
972 | 1 | ccv_nnc_tensor_variable_t f = ccv_nnc_tensor_variable_new(graph); |
973 | 1 | ccv_nnc_dynamic_graph_exec(graph, CMD_EWSUM_FORWARD(), ccv_nnc_no_hint, 0, TENSOR_VARIABLE_LIST(a, c), TENSOR_VARIABLE_LIST(f), 0, 0); |
974 | 1 | ccv_nnc_tensor_variable_t g = ccv_nnc_tensor_variable_new(graph); |
975 | 1 | ccv_nnc_dynamic_graph_exec(graph, CMD_EWSUM_FORWARD(), ccv_nnc_no_hint, 0, TENSOR_VARIABLE_LIST(e, b), TENSOR_VARIABLE_LIST(g), 0, 0); |
976 | 1 | REQUIRE_EQ_WITH_TOLERANCE(ccv_nnc_tensor_from_variable(graph, f)->data.f32[0], 10 + 9, 1e-5, "should match the result"); |
977 | 1 | REQUIRE_EQ_WITH_TOLERANCE(ccv_nnc_tensor_from_variable(graph, g)->data.f32[0], 10 * 0.5 + 9 * 0.4 * 2, 1e-5, "should match the result"); |
978 | 1 | uint64_t bitmask = 1; |
979 | 1 | ccv_nnc_dynamic_graph_has_effect_to_tensor_variables(graph, TENSOR_VARIABLE_LIST(a, b, f), TENSOR_VARIABLE_LIST(g), &bitmask); |
980 | 1 | REQUIRE(bitmask == 3, "a and b should be ancestors to g"); |
981 | 1 | bitmask = 8; |
982 | 1 | ccv_nnc_dynamic_graph_has_effect_to_tensor_variables(graph, TENSOR_VARIABLE_LIST(a, e, c, f), TENSOR_VARIABLE_LIST(g), &bitmask); |
983 | 1 | REQUIRE(bitmask == 7, "a, e and c should be ancestors to g"); |
984 | 1 | bitmask = 2; |
985 | 1 | ccv_nnc_dynamic_graph_has_effect_to_tensor_variables(graph, TENSOR_VARIABLE_LIST(a, e, c), TENSOR_VARIABLE_LIST(f), &bitmask); |
986 | 1 | REQUIRE(bitmask == 5, "a and c should be ancestors to f"); |
987 | 1 | ccv_nnc_dynamic_graph_free(graph); |
988 | 1 | } |
989 | | |
990 | | TEST_CASE("compute f(x) = 0.5 * log(x), detach log(x) means 0.5 * y should be freed") |
991 | 1 | { |
992 | 1 | ccv_nnc_dynamic_graph_t* const graph = ccv_nnc_dynamic_graph_new(); |
993 | 1 | ccv_nnc_tensor_variable_t x = ccv_nnc_tensor_variable_new(graph, CPU_TENSOR_NHWC(32F, 1)); |
994 | 1 | ccv_nnc_tensor_from_variable(graph, x)->data.f32[0] = 10; |
995 | 1 | ccv_nnc_tensor_variable_t y = ccv_nnc_tensor_variable_new(graph); |
996 | 1 | ccv_nnc_dynamic_graph_exec(graph, CMD_EWLOG_FORWARD(), ccv_nnc_no_hint, 0, TENSOR_VARIABLE_LIST(x), TENSOR_VARIABLE_LIST(y), 0, 0); |
997 | 1 | REQUIRE_EQ_WITH_TOLERANCE(ccv_nnc_tensor_from_variable(graph, y)->data.f32[0], log(10), 1e-5, "should match the result"); |
998 | 1 | ccv_nnc_tensor_variable_t z = ccv_nnc_tensor_variable_new(graph); |
999 | 1 | ccv_nnc_dynamic_graph_exec(graph, CMD_SCALAR_MUL_FORWARD(0.5), ccv_nnc_no_hint, 0, TENSOR_VARIABLE_LIST(y), TENSOR_VARIABLE_LIST(z), 0, 0); |
1000 | 1 | REQUIRE_EQ_WITH_TOLERANCE(ccv_nnc_tensor_from_variable(graph, z)->data.f32[0], 0.5 * log(10), 1e-5, "should match the result"); |
1001 | 1 | REQUIRE_EQ(ccv_nnc_dynamic_graph_bookkeeping_count(graph, CCV_NNC_SYMBOL_GRAPH_EXEC), 2, "nothing should be freed"); |
1002 | 1 | REQUIRE_EQ(ccv_nnc_dynamic_graph_bookkeeping_count(graph, CCV_NNC_SYMBOL_TENSOR), 3, "nothing should be freed"); |
1003 | 1 | uint64_t bitmask = 0; |
1004 | 1 | ccv_nnc_dynamic_graph_has_effect_to_tensor_variables(graph, TENSOR_VARIABLE_LIST(x), TENSOR_VARIABLE_LIST(z), &bitmask); |
1005 | 1 | REQUIRE(bitmask == 1, "before detach, it should have effect"); |
1006 | 1 | ccv_nnc_tensor_variable_detach(graph, y); |
1007 | 1 | REQUIRE_EQ(ccv_nnc_dynamic_graph_bookkeeping_count(graph, CCV_NNC_SYMBOL_GRAPH_EXEC), 1, "0.5 * y should be freed"); |
1008 | 1 | REQUIRE_EQ(ccv_nnc_dynamic_graph_bookkeeping_count(graph, CCV_NNC_SYMBOL_TENSOR), 3, "nothing should be freed"); |
1009 | 1 | bitmask = 0; |
1010 | 1 | ccv_nnc_dynamic_graph_has_effect_to_tensor_variables(graph, TENSOR_VARIABLE_LIST(x), TENSOR_VARIABLE_LIST(z), &bitmask); |
1011 | 1 | REQUIRE(bitmask == 0, "after detach, it should have no effect"); |
1012 | 1 | ccv_nnc_tensor_variable_free(graph, x); |
1013 | 1 | REQUIRE_EQ(ccv_nnc_dynamic_graph_bookkeeping_count(graph, CCV_NNC_SYMBOL_GRAPH_EXEC), 0, "everything should be freed"); |
1014 | 1 | REQUIRE_EQ(ccv_nnc_dynamic_graph_bookkeeping_count(graph, CCV_NNC_SYMBOL_TENSOR), 1, "x and y should be freed"); |
1015 | 1 | ccv_nnc_tensor_variable_free(graph, y); |
1016 | 1 | ccv_nnc_tensor_variable_free(graph, z); |
1017 | 1 | REQUIRE_EQ(ccv_nnc_dynamic_graph_bookkeeping_count(graph, CCV_NNC_SYMBOL_TENSOR), 0, "everything should be freed"); |
1018 | 1 | ccv_nnc_dynamic_graph_free(graph); |
1019 | 1 | } |
1020 | | |
1021 | | TEST_CASE("compute f(x) = 0.5 * log(x), detach log(x) after freeing x means all computations on the graph should be freed") |
1022 | 1 | { |
1023 | 1 | ccv_nnc_dynamic_graph_t* const graph = ccv_nnc_dynamic_graph_new(); |
1024 | 1 | ccv_nnc_tensor_variable_t x = ccv_nnc_tensor_variable_new(graph, CPU_TENSOR_NHWC(32F, 1)); |
1025 | 1 | ccv_nnc_tensor_from_variable(graph, x)->data.f32[0] = 10; |
1026 | 1 | ccv_nnc_tensor_variable_t y = ccv_nnc_tensor_variable_new(graph); |
1027 | 1 | ccv_nnc_dynamic_graph_exec(graph, CMD_EWLOG_FORWARD(), ccv_nnc_no_hint, 0, TENSOR_VARIABLE_LIST(x), TENSOR_VARIABLE_LIST(y), 0, 0); |
1028 | 1 | REQUIRE_EQ_WITH_TOLERANCE(ccv_nnc_tensor_from_variable(graph, y)->data.f32[0], log(10), 1e-5, "should match the result"); |
1029 | 1 | ccv_nnc_tensor_variable_t z = ccv_nnc_tensor_variable_new(graph); |
1030 | 1 | ccv_nnc_dynamic_graph_exec(graph, CMD_SCALAR_MUL_FORWARD(0.5), ccv_nnc_no_hint, 0, TENSOR_VARIABLE_LIST(y), TENSOR_VARIABLE_LIST(z), 0, 0); |
1031 | 1 | REQUIRE_EQ_WITH_TOLERANCE(ccv_nnc_tensor_from_variable(graph, z)->data.f32[0], 0.5 * log(10), 1e-5, "should match the result"); |
1032 | 1 | REQUIRE_EQ(ccv_nnc_dynamic_graph_bookkeeping_count(graph, CCV_NNC_SYMBOL_GRAPH_EXEC), 2, "nothing should be freed"); |
1033 | 1 | REQUIRE_EQ(ccv_nnc_dynamic_graph_bookkeeping_count(graph, CCV_NNC_SYMBOL_TENSOR), 3, "nothing should be freed"); |
1034 | | // free x first before detaching y. |
1035 | 1 | ccv_nnc_tensor_variable_free(graph, x); |
1036 | 1 | REQUIRE_EQ(ccv_nnc_dynamic_graph_bookkeeping_count(graph, CCV_NNC_SYMBOL_GRAPH_EXEC), 1, "log(x) should be freed"); |
1037 | 1 | REQUIRE_EQ(ccv_nnc_dynamic_graph_bookkeeping_count(graph, CCV_NNC_SYMBOL_TENSOR), 2, "only x should be freed"); |
1038 | 1 | ccv_nnc_tensor_variable_detach(graph, y); |
1039 | 1 | REQUIRE_EQ(ccv_nnc_dynamic_graph_bookkeeping_count(graph, CCV_NNC_SYMBOL_GRAPH_EXEC), 0, "everything should be freed"); |
1040 | 1 | REQUIRE_EQ(ccv_nnc_dynamic_graph_bookkeeping_count(graph, CCV_NNC_SYMBOL_TENSOR), 1, "x and y should be freed"); |
1041 | 1 | ccv_nnc_tensor_variable_free(graph, y); |
1042 | 1 | ccv_nnc_tensor_variable_free(graph, z); |
1043 | 1 | REQUIRE_EQ(ccv_nnc_dynamic_graph_bookkeeping_count(graph, CCV_NNC_SYMBOL_TENSOR), 0, "everything should be freed"); |
1044 | 1 | ccv_nnc_dynamic_graph_free(graph); |
1045 | 1 | } |
1046 | | |
1047 | | TEST_CASE("compute f(x) = a * log(x), detach log(x) means nothing changed") |
1048 | 1 | { |
1049 | 1 | ccv_nnc_dynamic_graph_t* const graph = ccv_nnc_dynamic_graph_new(); |
1050 | 1 | ccv_nnc_tensor_variable_t x = ccv_nnc_tensor_variable_new(graph, CPU_TENSOR_NHWC(32F, 1)); |
1051 | 1 | ccv_nnc_tensor_from_variable(graph, x)->data.f32[0] = 10; |
1052 | 1 | ccv_nnc_tensor_variable_t a = ccv_nnc_tensor_variable_new(graph, CPU_TENSOR_NHWC(32F, 1)); |
1053 | 1 | ccv_nnc_tensor_from_variable(graph, a)->data.f32[0] = 5; |
1054 | 1 | ccv_nnc_tensor_variable_t y = ccv_nnc_tensor_variable_new(graph); |
1055 | 1 | ccv_nnc_dynamic_graph_exec(graph, CMD_EWLOG_FORWARD(), ccv_nnc_no_hint, 0, TENSOR_VARIABLE_LIST(x), TENSOR_VARIABLE_LIST(y), 0, 0); |
1056 | 1 | REQUIRE_EQ_WITH_TOLERANCE(ccv_nnc_tensor_from_variable(graph, y)->data.f32[0], log(10), 1e-5, "should match the result"); |
1057 | 1 | ccv_nnc_tensor_variable_t z = ccv_nnc_tensor_variable_new(graph); |
1058 | 1 | ccv_nnc_dynamic_graph_exec(graph, CMD_MUL_FORWARD(1), ccv_nnc_no_hint, 0, TENSOR_VARIABLE_LIST(a, y), TENSOR_VARIABLE_LIST(z), 0, 0); |
1059 | 1 | REQUIRE_EQ_WITH_TOLERANCE(ccv_nnc_tensor_from_variable(graph, z)->data.f32[0], 5 * log(10), 1e-5, "should match the result"); |
1060 | 1 | REQUIRE_EQ(ccv_nnc_dynamic_graph_bookkeeping_count(graph, CCV_NNC_SYMBOL_GRAPH_EXEC), 2, "nothing should be freed"); |
1061 | 1 | REQUIRE_EQ(ccv_nnc_dynamic_graph_bookkeeping_count(graph, CCV_NNC_SYMBOL_TENSOR), 4, "nothing should be freed"); |
1062 | 1 | uint64_t bitmask = 0; |
1063 | 1 | ccv_nnc_dynamic_graph_has_effect_to_tensor_variables(graph, TENSOR_VARIABLE_LIST(x), TENSOR_VARIABLE_LIST(z), &bitmask); |
1064 | 1 | REQUIRE(bitmask == 1, "before detach, it should have effect"); |
1065 | 1 | ccv_nnc_tensor_variable_detach(graph, y); |
1066 | 1 | REQUIRE_EQ(ccv_nnc_dynamic_graph_bookkeeping_count(graph, CCV_NNC_SYMBOL_GRAPH_EXEC), 2, "nothing should be freed"); |
1067 | 1 | REQUIRE_EQ(ccv_nnc_dynamic_graph_bookkeeping_count(graph, CCV_NNC_SYMBOL_TENSOR), 4, "nothing should be freed"); |
1068 | 1 | bitmask = 0; |
1069 | 1 | ccv_nnc_dynamic_graph_has_effect_to_tensor_variables(graph, TENSOR_VARIABLE_LIST(x), TENSOR_VARIABLE_LIST(z), &bitmask); |
1070 | 1 | REQUIRE(bitmask == 0, "after detach, it should have no effect"); |
1071 | 1 | ccv_nnc_tensor_variable_free(graph, a); |
1072 | 1 | REQUIRE_EQ(ccv_nnc_dynamic_graph_bookkeeping_count(graph, CCV_NNC_SYMBOL_GRAPH_EXEC), 1, "a * y should be freed"); |
1073 | 1 | REQUIRE_EQ(ccv_nnc_dynamic_graph_bookkeeping_count(graph, CCV_NNC_SYMBOL_TENSOR), 3, "a should be freed"); |
1074 | 1 | ccv_nnc_tensor_variable_free(graph, x); |
1075 | 1 | REQUIRE_EQ(ccv_nnc_dynamic_graph_bookkeeping_count(graph, CCV_NNC_SYMBOL_GRAPH_EXEC), 0, "everything should be freed"); |
1076 | 1 | REQUIRE_EQ(ccv_nnc_dynamic_graph_bookkeeping_count(graph, CCV_NNC_SYMBOL_TENSOR), 1, "x and y should be freed"); |
1077 | 1 | ccv_nnc_tensor_variable_free(graph, y); |
1078 | 1 | ccv_nnc_tensor_variable_free(graph, z); |
1079 | 1 | REQUIRE_EQ(ccv_nnc_dynamic_graph_bookkeeping_count(graph, CCV_NNC_SYMBOL_TENSOR), 0, "everything should be freed"); |
1080 | 1 | ccv_nnc_dynamic_graph_free(graph); |
1081 | 1 | } |
1082 | | |
1083 | | TEST_CASE("compute f(x) = exp(x) * log(x), detach log(x) means nothing changed") |
1084 | 1 | { |
1085 | 1 | ccv_nnc_dynamic_graph_t* const graph = ccv_nnc_dynamic_graph_new(); |
1086 | 1 | ccv_nnc_tensor_variable_t x = ccv_nnc_tensor_variable_new(graph, CPU_TENSOR_NHWC(32F, 1)); |
1087 | 1 | ccv_nnc_tensor_from_variable(graph, x)->data.f32[0] = 10; |
1088 | 1 | ccv_nnc_tensor_variable_t a = ccv_nnc_tensor_variable_new(graph); |
1089 | 1 | ccv_nnc_dynamic_graph_exec(graph, CMD_EWEXP_FORWARD(), ccv_nnc_no_hint, 0, TENSOR_VARIABLE_LIST(x), TENSOR_VARIABLE_LIST(a), 0, 0); |
1090 | 1 | REQUIRE_EQ_WITH_TOLERANCE(ccv_nnc_tensor_from_variable(graph, a)->data.f32[0], exp(10), 1e-2, "should match the result"); |
1091 | 1 | ccv_nnc_tensor_variable_t y = ccv_nnc_tensor_variable_new(graph); |
1092 | 1 | ccv_nnc_dynamic_graph_exec(graph, CMD_EWLOG_FORWARD(), ccv_nnc_no_hint, 0, TENSOR_VARIABLE_LIST(x), TENSOR_VARIABLE_LIST(y), 0, 0); |
1093 | 1 | REQUIRE_EQ_WITH_TOLERANCE(ccv_nnc_tensor_from_variable(graph, y)->data.f32[0], log(10), 1e-5, "should match the result"); |
1094 | 1 | ccv_nnc_tensor_variable_t z = ccv_nnc_tensor_variable_new(graph); |
1095 | 1 | ccv_nnc_dynamic_graph_exec(graph, CMD_MUL_FORWARD(1), ccv_nnc_no_hint, 0, TENSOR_VARIABLE_LIST(a, y), TENSOR_VARIABLE_LIST(z), 0, 0); |
1096 | 1 | REQUIRE_EQ_WITH_TOLERANCE(ccv_nnc_tensor_from_variable(graph, z)->data.f32[0], exp(10) * log(10), 1e-2, "should match the result"); |
1097 | 1 | REQUIRE_EQ(ccv_nnc_dynamic_graph_bookkeeping_count(graph, CCV_NNC_SYMBOL_GRAPH_EXEC), 3, "nothing should be freed"); |
1098 | 1 | REQUIRE_EQ(ccv_nnc_dynamic_graph_bookkeeping_count(graph, CCV_NNC_SYMBOL_TENSOR), 4, "nothing should be freed"); |
1099 | 1 | uint64_t bitmask = 0; |
1100 | 1 | ccv_nnc_dynamic_graph_has_effect_to_tensor_variables(graph, TENSOR_VARIABLE_LIST(x), TENSOR_VARIABLE_LIST(z), &bitmask); |
1101 | 1 | REQUIRE(bitmask == 1, "before detach, it should have effect"); |
1102 | 1 | ccv_nnc_tensor_variable_detach(graph, y); |
1103 | 1 | REQUIRE_EQ(ccv_nnc_dynamic_graph_bookkeeping_count(graph, CCV_NNC_SYMBOL_GRAPH_EXEC), 3, "nothing should be freed"); |
1104 | 1 | REQUIRE_EQ(ccv_nnc_dynamic_graph_bookkeeping_count(graph, CCV_NNC_SYMBOL_TENSOR), 4, "nothing should be freed"); |
1105 | 1 | bitmask = 0; |
1106 | 1 | ccv_nnc_dynamic_graph_has_effect_to_tensor_variables(graph, TENSOR_VARIABLE_LIST(x), TENSOR_VARIABLE_LIST(z), &bitmask); |
1107 | 1 | REQUIRE(bitmask == 1, "after detach, it should have still effect (through exp(x))"); |
1108 | 1 | ccv_nnc_tensor_variable_detach(graph, a); |
1109 | 1 | REQUIRE_EQ(ccv_nnc_dynamic_graph_bookkeeping_count(graph, CCV_NNC_SYMBOL_GRAPH_EXEC), 2, "a * y should be freed"); |
1110 | 1 | REQUIRE_EQ(ccv_nnc_dynamic_graph_bookkeeping_count(graph, CCV_NNC_SYMBOL_TENSOR), 4, "nothing should be freed"); |
1111 | 1 | bitmask = 0; |
1112 | 1 | ccv_nnc_dynamic_graph_has_effect_to_tensor_variables(graph, TENSOR_VARIABLE_LIST(x), TENSOR_VARIABLE_LIST(z), &bitmask); |
1113 | 1 | REQUIRE(bitmask == 0, "after a freed, it should have no effect"); |
1114 | 1 | ccv_nnc_tensor_variable_free(graph, a); |
1115 | 1 | REQUIRE_EQ(ccv_nnc_dynamic_graph_bookkeeping_count(graph, CCV_NNC_SYMBOL_TENSOR), 3, "a should be freed"); |
1116 | 1 | ccv_nnc_tensor_variable_free(graph, x); |
1117 | 1 | REQUIRE_EQ(ccv_nnc_dynamic_graph_bookkeeping_count(graph, CCV_NNC_SYMBOL_GRAPH_EXEC), 0, "everything should be freed"); |
1118 | 1 | REQUIRE_EQ(ccv_nnc_dynamic_graph_bookkeeping_count(graph, CCV_NNC_SYMBOL_TENSOR), 1, "x and y should be freed"); |
1119 | 1 | ccv_nnc_tensor_variable_free(graph, y); |
1120 | 1 | ccv_nnc_tensor_variable_free(graph, z); |
1121 | 1 | REQUIRE_EQ(ccv_nnc_dynamic_graph_bookkeeping_count(graph, CCV_NNC_SYMBOL_TENSOR), 0, "everything should be freed"); |
1122 | 1 | ccv_nnc_dynamic_graph_free(graph); |
1123 | 1 | } |
1124 | | |
1125 | | TEST_CASE("compute f(x) = exp(0.5 * log(x)), detach log(x) won't free exp(_)") |
1126 | 1 | { |
1127 | 1 | ccv_nnc_dynamic_graph_t* const graph = ccv_nnc_dynamic_graph_new(); |
1128 | 1 | ccv_nnc_tensor_variable_t x = ccv_nnc_tensor_variable_new(graph, CPU_TENSOR_NHWC(32F, 1)); |
1129 | 1 | ccv_nnc_tensor_from_variable(graph, x)->data.f32[0] = 10; |
1130 | 1 | ccv_nnc_tensor_variable_t y = ccv_nnc_tensor_variable_new(graph); |
1131 | 1 | ccv_nnc_dynamic_graph_exec(graph, CMD_EWLOG_FORWARD(), ccv_nnc_no_hint, 0, TENSOR_VARIABLE_LIST(x), TENSOR_VARIABLE_LIST(y), 0, 0); |
1132 | 1 | REQUIRE_EQ_WITH_TOLERANCE(ccv_nnc_tensor_from_variable(graph, y)->data.f32[0], log(10), 1e-5, "should match the result"); |
1133 | 1 | ccv_nnc_tensor_variable_t z = ccv_nnc_tensor_variable_new(graph); |
1134 | 1 | ccv_nnc_dynamic_graph_exec(graph, CMD_SCALAR_MUL_FORWARD(0.5), ccv_nnc_no_hint, 0, TENSOR_VARIABLE_LIST(y), TENSOR_VARIABLE_LIST(z), 0, 0); |
1135 | 1 | REQUIRE_EQ_WITH_TOLERANCE(ccv_nnc_tensor_from_variable(graph, z)->data.f32[0], 0.5 * log(10), 1e-5, "should match the result"); |
1136 | 1 | ccv_nnc_tensor_variable_t f = ccv_nnc_tensor_variable_new(graph); |
1137 | 1 | ccv_nnc_dynamic_graph_exec(graph, CMD_EWEXP_FORWARD(), ccv_nnc_no_hint, 0, TENSOR_VARIABLE_LIST(z), TENSOR_VARIABLE_LIST(f), 0, 0); |
1138 | 1 | REQUIRE_EQ_WITH_TOLERANCE(ccv_nnc_tensor_from_variable(graph, f)->data.f32[0], exp(0.5 * log(10)), 1e-5, "should match the result"); |
1139 | 1 | uint64_t bitmask = 0; |
1140 | 1 | ccv_nnc_dynamic_graph_has_effect_to_tensor_variables(graph, TENSOR_VARIABLE_LIST(x), TENSOR_VARIABLE_LIST(f), &bitmask); |
1141 | 1 | REQUIRE(bitmask == 1, "before detach, it should have effect"); |
1142 | 1 | ccv_nnc_tensor_variable_detach(graph, y); |
1143 | 1 | REQUIRE_EQ(ccv_nnc_dynamic_graph_bookkeeping_count(graph, CCV_NNC_SYMBOL_GRAPH_EXEC), 2, "0.5 * y should be freed"); |
1144 | 1 | REQUIRE_EQ(ccv_nnc_dynamic_graph_bookkeeping_count(graph, CCV_NNC_SYMBOL_TENSOR), 4, "nothing should be freed"); |
1145 | 1 | bitmask = 0; |
1146 | 1 | ccv_nnc_dynamic_graph_has_effect_to_tensor_variables(graph, TENSOR_VARIABLE_LIST(x), TENSOR_VARIABLE_LIST(f), &bitmask); |
1147 | 1 | REQUIRE(bitmask == 0, "after detach, it should have no effect"); |
1148 | 1 | ccv_nnc_tensor_variable_free(graph, f); |
1149 | 1 | REQUIRE_EQ(ccv_nnc_dynamic_graph_bookkeeping_count(graph, CCV_NNC_SYMBOL_GRAPH_EXEC), 1, "exp(_) should be freed"); |
1150 | 1 | REQUIRE_EQ(ccv_nnc_dynamic_graph_bookkeeping_count(graph, CCV_NNC_SYMBOL_TENSOR), 3, "f should be freed"); |
1151 | 1 | ccv_nnc_tensor_variable_free(graph, z); |
1152 | 1 | REQUIRE_EQ(ccv_nnc_dynamic_graph_bookkeeping_count(graph, CCV_NNC_SYMBOL_GRAPH_EXEC), 1, "nothing should change"); |
1153 | 1 | REQUIRE_EQ(ccv_nnc_dynamic_graph_bookkeeping_count(graph, CCV_NNC_SYMBOL_TENSOR), 2, "z should be freed"); |
1154 | 1 | ccv_nnc_tensor_variable_free(graph, y); |
1155 | 1 | REQUIRE_EQ(ccv_nnc_dynamic_graph_bookkeeping_count(graph, CCV_NNC_SYMBOL_GRAPH_EXEC), 0, "log(x) should be freed"); |
1156 | 1 | REQUIRE_EQ(ccv_nnc_dynamic_graph_bookkeeping_count(graph, CCV_NNC_SYMBOL_TENSOR), 1, "y should be freed"); |
1157 | 1 | ccv_nnc_tensor_variable_free(graph, x); |
1158 | 1 | REQUIRE_EQ(ccv_nnc_dynamic_graph_bookkeeping_count(graph, CCV_NNC_SYMBOL_TENSOR), 0, "x should be freed"); |
1159 | 1 | ccv_nnc_dynamic_graph_free(graph); |
1160 | 1 | } |
1161 | | |
1162 | | #include "case_main.h" |