Coverage Report

Created: 2021-04-12 01:11

/home/liu/buildslave/linux-x64-runtests/build/test/unit/nnc/minimize.tests.c
Line
Count
Source
1
#include "case.h"
2
#include "ccv_case.h"
3
#include "ccv_nnc_case.h"
4
#include <ccv.h>
5
#include <nnc/ccv_nnc.h>
6
#include <nnc/ccv_nnc_easy.h>
7
#include "3rdparty/dsfmt/dSFMT.h"
8
9
TEST_SETUP()
10
{
11
  ccv_nnc_init();
12
}
13
14
TEST_CASE("solve least square sum with stochastic gradient descent on symbolic graph")
15
1
{
16
1
  ccv_nnc_symbolic_graph_t* const symbolic_graph = ccv_nnc_symbolic_graph_new();
17
1
  ccv_nnc_tensor_symbol_t a = ccv_nnc_tensor_symbol_new(symbolic_graph, CPU_TENSOR_NHWC(32F, 2, 2), "a");
18
1
  ccv_nnc_tensor_symbol_t w = ccv_nnc_tensor_symbol_new(symbolic_graph, CPU_TENSOR_NHWC(32F, 2, 2), "w");
19
1
  ccv_nnc_tensor_symbol_t bias = ccv_nnc_tensor_symbol_new(symbolic_graph, CPU_TENSOR_NHWC(32F, 2), "bias");
20
1
  ccv_nnc_tensor_symbol_t b = ccv_nnc_tensor_symbol_new(symbolic_graph, CPU_TENSOR_NHWC(32F, 2, 2), "b");
21
1
  ccv_nnc_graph_exec_symbol_new(symbolic_graph, CMD_GEMM_FORWARD(NO_TRANSPOSE, TRANSPOSE(0, 1)), TENSOR_SYMBOL_LIST(a, w, bias), TENSOR_SYMBOL_LIST(b), "gemm");
22
1
  ccv_nnc_tensor_symbol_t c = ccv_nnc_tensor_symbol_new(symbolic_graph, CPU_TENSOR_NHWC(32F, 2, 2), "c");
23
1
  ccv_nnc_graph_exec_symbol_new(symbolic_graph, CMD_EWPROD_FORWARD(), TENSOR_SYMBOL_LIST(b, b), TENSOR_SYMBOL_LIST(c), "square");
24
1
  ccv_nnc_tensor_symbol_t s = ccv_nnc_tensor_symbol_new(symbolic_graph, ccv_nnc_tensor_auto, "s");
25
1
  ccv_nnc_graph_exec_symbol_new(symbolic_graph, CMD_REDUCE_SUM_FORWARD(0, 1), TENSOR_SYMBOL_LIST(c), TENSOR_SYMBOL_LIST(s), "sum");
26
1
  ccv_nnc_graph_exec_symbol_autogen(symbolic_graph, 0, 0, CCV_NNC_AUTOGEN_ALL_EXECS | CCV_NNC_AUTOGEN_SOURCES_AND_DESTINATIONS);
27
1
  ccv_nnc_tensor_symbol_t updates[1];
28
1
  ccv_nnc_tensor_symbol_map_t aux[1];
29
1
  ccv_nnc_graph_exec_symbol_t update_execs[1];
30
1
  ccv_nnc_symbolic_graph_minimize(symbolic_graph, CMD_SGD_FORWARD(0, 0.001, 1, 0.995, 0.9, 0.9), TENSOR_SYMBOL_LIST(s), TENSOR_SYMBOL_LIST(w), 0, 0, SYMBOLIC_GRAPH_SOURCES(symbolic_graph), SYMBOLIC_GRAPH_DESTINATIONS(symbolic_graph), 0, updates, aux, update_execs);
31
1
  SYMBOLIC_GRAPH_GEN(symbolic_graph, CCV_NNC_LONG_DOT_GRAPH);
32
1
  ccv_nnc_graph_t* graph;
33
1
  ccv_nnc_tensor_arena_t* tensor_arena;
34
1
  ccv_nnc_graph_exec_arena_t* graph_exec_arena;
35
1
  ccv_nnc_symbolic_graph_compile(symbolic_graph, ccv_nnc_default_compile_params, 0, 0, 0, 0, SYMBOLIC_GRAPH_SOURCES(symbolic_graph), update_execs, 1, &graph, &tensor_arena, &graph_exec_arena);
36
1
  GRAPH_GEN(graph, CCV_NNC_LONG_DOT_GRAPH);
37
1
  // Relies on the inplace ops for SGD set on both updated w / bias, and momentum.
38
1
  ccv_nnc_tensor_t* const w_tensor = ccv_nnc_tensor_from_symbol(tensor_arena, w);
39
1
  ccv_nnc_cmd_exec(CMD_RANDOM_UNIFORM_FORWARD(-0.5, 0.5), ccv_nnc_no_hint, 0, 0, 0, TENSOR_LIST(w_tensor), 0);
40
1
  ccv_nnc_tensor_t* const bias_tensor = ccv_nnc_tensor_from_symbol(tensor_arena, bias);
41
1
  int i;
42
2
  for (i = 0; i < 1; 
i++1
)
43
1
  {
44
1
    ccv_nnc_tensor_t* const aux_tensor = ccv_nnc_tensor_from_symbol(tensor_arena, aux[i].source);
45
1
    ccv_nnc_cmd_exec(CMD_SET_FORWARD(0), ccv_nnc_no_hint, 0, 0, 0, TENSOR_LIST(aux_tensor), 0);
46
1
  }
47
1
  ccv_nnc_tensor_t* const a_tensor = ccv_nnc_tensor_from_symbol(tensor_arena, a);
48
1
  ccv_nnc_tensor_t* const f_tensor = ccv_nnc_tensor_from_symbol(tensor_arena, ccv_nnc_tensor_symbol_for_backward(symbolic_graph, s));
49
1
  ccv_nnc_graph_exec_t sgd = ccv_nnc_graph_exec_from_symbol(graph_exec_arena, update_execs[0]);
50
1.00k
  for (i = 0; i < 1000; 
i++1.00k
)
51
1.00k
  {
52
1.00k
    a_tensor->data.f32[0] = 10;
53
1.00k
    a_tensor->data.f32[1] = 1;
54
1.00k
    a_tensor->data.f32[2] = 3;
55
1.00k
    a_tensor->data.f32[3] = 5;
56
1.00k
    f_tensor->data.f32[0] = 1;
57
1.00k
    bias_tensor->data.f32[0] = 1;
58
1.00k
    bias_tensor->data.f32[1] = -1;
59
1.00k
    if (i == 750)
60
1
      ccv_nnc_graph_exec_set(graph, sgd, CMD_SGD_FORWARD(0, 0.000001, 1, 0.995, 0.9, 0.9));
61
999
    else if (i == 500)
62
1
      ccv_nnc_graph_exec_set(graph, sgd, CMD_SGD_FORWARD(0, 0.00001, 1, 0.995, 0.9, 0.9));
63
998
    else if (i == 250)
64
1
      ccv_nnc_graph_exec_set(graph, sgd, CMD_SGD_FORWARD(0, 0.0001, 1, 0.995, 0.9, 0.9));
65
1.00k
    ccv_nnc_graph_run(graph, 0, TRAVERSE_FULL, 0, 0);
66
1.00k
  }
67
1
  REQUIRE_EQ_WITH_TOLERANCE(a_tensor->data.f32[0] * w_tensor->data.f32[0] + a_tensor->data.f32[1] * w_tensor->data.f32[1], -1, 1e-3, "converge for vector 1");
68
1
  REQUIRE_EQ_WITH_TOLERANCE(a_tensor->data.f32[0] * w_tensor->data.f32[2] + a_tensor->data.f32[1] * w_tensor->data.f32[3], 1, 1e-3, "converge for vector 1");
69
1
  REQUIRE_EQ_WITH_TOLERANCE(a_tensor->data.f32[2] * w_tensor->data.f32[0] + a_tensor->data.f32[3] * w_tensor->data.f32[1], -1, 1e-1, "converge for vector 2");
70
1
  REQUIRE_EQ_WITH_TOLERANCE(a_tensor->data.f32[2] * w_tensor->data.f32[2] + a_tensor->data.f32[3] * w_tensor->data.f32[3], 1, 1e-1, "converge for vector 2");
71
1
  ccv_nnc_symbolic_graph_free(symbolic_graph);
72
1
  ccv_nnc_graph_free(graph);
73
1
  ccv_nnc_tensor_arena_free(tensor_arena);
74
1
  ccv_nnc_graph_exec_arena_free(graph_exec_arena);
75
1
}
76
77
TEST_CASE("solve least square sum with stochastic gradient descent on dynamic graph")
78
1
{
79
1
  ccv_nnc_dynamic_graph_t* const graph = ccv_nnc_dynamic_graph_new();
80
1
  ccv_nnc_tensor_variable_t w = ccv_nnc_tensor_variable_new(graph, CPU_TENSOR_NHWC(32F, 2, 2));
81
1
  ccv_nnc_tensor_variable_t aux = ccv_nnc_tensor_variable_new(graph, CPU_TENSOR_NHWC(32F, 2, 2));
82
1
  ccv_nnc_dynamic_graph_exec(graph, CMD_SET_FORWARD(0), ccv_nnc_no_hint, 0, 0, 0, TENSOR_VARIABLE_LIST(aux), 0, 0);
83
1
  ccv_nnc_dynamic_graph_exec(graph, CMD_RANDOM_UNIFORM_FORWARD(-0.5, 0.5), ccv_nnc_no_hint, 0, 0, 0, TENSOR_VARIABLE_LIST(w), 0, 0);
84
1
  int i;
85
1.00k
  for (i = 0; i < 1000; 
i++1.00k
)
86
1.00k
  {
87
1.00k
    ccv_nnc_tensor_variable_t a = ccv_nnc_tensor_variable_new(graph, CPU_TENSOR_NHWC(32F, 2, 2));
88
1.00k
    ccv_nnc_tensor_t* const a_tensor = ccv_nnc_tensor_from_variable(graph, a);
89
1.00k
    a_tensor->data.f32[0] = 10;
90
1.00k
    a_tensor->data.f32[1] = 1;
91
1.00k
    a_tensor->data.f32[2] = 3;
92
1.00k
    a_tensor->data.f32[3] = 5;
93
1.00k
    ccv_nnc_tensor_variable_t bias = ccv_nnc_tensor_variable_new(graph, CPU_TENSOR_NHWC(32F, 2));
94
1.00k
    ccv_nnc_tensor_t* const bias_tensor = ccv_nnc_tensor_from_variable(graph, bias);
95
1.00k
    bias_tensor->data.f32[0] = 1;
96
1.00k
    bias_tensor->data.f32[1] = -1;
97
1.00k
    ccv_nnc_tensor_variable_t b = ccv_nnc_tensor_variable_new(graph);
98
1.00k
    ccv_nnc_dynamic_graph_exec(graph, CMD_GEMM_FORWARD(NO_TRANSPOSE, TRANSPOSE(0, 1)), ccv_nnc_no_hint, 0, TENSOR_VARIABLE_LIST(a, w, bias), TENSOR_VARIABLE_LIST(b), 0, 0);
99
1.00k
    ccv_nnc_tensor_variable_t c = ccv_nnc_tensor_variable_new(graph);
100
1.00k
    ccv_nnc_dynamic_graph_exec(graph, CMD_EWPROD_FORWARD(), ccv_nnc_no_hint, 0, TENSOR_VARIABLE_LIST(b, b), TENSOR_VARIABLE_LIST(c), 0, 0);
101
1.00k
    ccv_nnc_tensor_variable_t s = ccv_nnc_tensor_variable_new(graph);
102
1.00k
    ccv_nnc_dynamic_graph_exec(graph, CMD_REDUCE_SUM_FORWARD(0, 1), ccv_nnc_no_hint, 0, TENSOR_VARIABLE_LIST(c), TENSOR_VARIABLE_LIST(s), 0, 0);
103
1.00k
    ccv_nnc_dynamic_graph_minimize(graph, CMD_SGD_FORWARD(0, 0.001, 1, 0.995, 0.9, 0.9), TENSOR_VARIABLE_LIST(s), 0, TENSOR_VARIABLE_LIST(w), &aux, 0, 0);
104
1.00k
    ccv_nnc_tensor_variable_free(graph, a);
105
1.00k
    ccv_nnc_tensor_variable_free(graph, b);
106
1.00k
    ccv_nnc_tensor_variable_free(graph, bias);
107
1.00k
    ccv_nnc_tensor_variable_free(graph, c);
108
1.00k
    ccv_nnc_tensor_variable_free(graph, s);
109
1.00k
  }
110
1
  DYNAMIC_GRAPH_GEN(graph, CCV_NNC_LONG_DOT_GRAPH);
111
1
  ccv_nnc_tensor_t* const w_tensor = ccv_nnc_tensor_from_variable(graph, w);
112
1
  REQUIRE_EQ_WITH_TOLERANCE(10 * w_tensor->data.f32[0] + 1 * w_tensor->data.f32[1], -1, 1e-3, "converge for vector 1");
113
1
  REQUIRE_EQ_WITH_TOLERANCE(10 * w_tensor->data.f32[2] + 1 * w_tensor->data.f32[3], 1, 1e-3, "converge for vector 1");
114
1
  REQUIRE_EQ_WITH_TOLERANCE(3 * w_tensor->data.f32[0] + 5 * w_tensor->data.f32[1], -1, 1e-1, "converge for vector 2");
115
1
  REQUIRE_EQ_WITH_TOLERANCE(3 * w_tensor->data.f32[2] + 5 * w_tensor->data.f32[3], 1, 1e-1, "converge for vector 2");
116
1
  ccv_nnc_dynamic_graph_free(graph);
117
1
}
118
119
TEST_CASE("solve least square sum with stochastic gradient descent on dynamic graph, backward & apply gradients")
120
1
{
121
1
  ccv_nnc_dynamic_graph_t* const graph = ccv_nnc_dynamic_graph_new();
122
1
  ccv_nnc_tensor_variable_t w = ccv_nnc_tensor_variable_new(graph, CPU_TENSOR_NHWC(32F, 2, 2));
123
1
  ccv_nnc_tensor_variable_t aux = ccv_nnc_tensor_variable_new(graph, CPU_TENSOR_NHWC(32F, 2, 2));
124
1
  ccv_nnc_dynamic_graph_exec(graph, CMD_SET_FORWARD(0), ccv_nnc_no_hint, 0, 0, 0, TENSOR_VARIABLE_LIST(aux), 0, 0);
125
1
  ccv_nnc_dynamic_graph_exec(graph, CMD_RANDOM_UNIFORM_FORWARD(-0.5, 0.5), ccv_nnc_no_hint, 0, 0, 0, TENSOR_VARIABLE_LIST(w), 0, 0);
126
1
  int i;
127
1.00k
  for (i = 0; i < 1000; 
i++1.00k
)
128
1.00k
  {
129
1.00k
    ccv_nnc_tensor_variable_t a = ccv_nnc_tensor_variable_new(graph, CPU_TENSOR_NHWC(32F, 2, 2));
130
1.00k
    ccv_nnc_tensor_t* const a_tensor = ccv_nnc_tensor_from_variable(graph, a);
131
1.00k
    a_tensor->data.f32[0] = 10;
132
1.00k
    a_tensor->data.f32[1] = 1;
133
1.00k
    a_tensor->data.f32[2] = 3;
134
1.00k
    a_tensor->data.f32[3] = 5;
135
1.00k
    ccv_nnc_tensor_variable_t bias = ccv_nnc_tensor_variable_new(graph, CPU_TENSOR_NHWC(32F, 2));
136
1.00k
    ccv_nnc_tensor_t* const bias_tensor = ccv_nnc_tensor_from_variable(graph, bias);
137
1.00k
    bias_tensor->data.f32[0] = 1;
138
1.00k
    bias_tensor->data.f32[1] = -1;
139
1.00k
    ccv_nnc_tensor_variable_t b = ccv_nnc_tensor_variable_new(graph);
140
1.00k
    ccv_nnc_dynamic_graph_exec(graph, CMD_GEMM_FORWARD(NO_TRANSPOSE, TRANSPOSE(0, 1)), ccv_nnc_no_hint, 0, TENSOR_VARIABLE_LIST(a, w, bias), TENSOR_VARIABLE_LIST(b), 0, 0);
141
1.00k
    ccv_nnc_tensor_variable_t c = ccv_nnc_tensor_variable_new(graph);
142
1.00k
    ccv_nnc_dynamic_graph_exec(graph, CMD_EWPROD_FORWARD(), ccv_nnc_no_hint, 0, TENSOR_VARIABLE_LIST(b, b), TENSOR_VARIABLE_LIST(c), 0, 0);
143
1.00k
    ccv_nnc_tensor_variable_t s = ccv_nnc_tensor_variable_new(graph);
144
1.00k
    ccv_nnc_dynamic_graph_exec(graph, CMD_REDUCE_SUM_FORWARD(0, 1), ccv_nnc_no_hint, 0, TENSOR_VARIABLE_LIST(c), TENSOR_VARIABLE_LIST(s), 0, 0);
145
1.00k
    ccv_nnc_tensor_variable_t g = ccv_nnc_tensor_variable_new(graph);
146
1.00k
    ccv_nnc_dynamic_graph_backward(graph, TENSOR_VARIABLE_LIST(s), 0, TENSOR_VARIABLE_LIST(w), TENSOR_VARIABLE_LIST(g), 0);
147
1.00k
    ccv_nnc_dynamic_graph_apply_gradients(graph, CMD_SGD_FORWARD(0, 0.001, 1, 0.995, 0.9, 0.9), TENSOR_VARIABLE_LIST(g), TENSOR_VARIABLE_LIST(w), &aux, 0, 0);
148
1.00k
    ccv_nnc_tensor_variable_free(graph, a);
149
1.00k
    ccv_nnc_tensor_variable_free(graph, b);
150
1.00k
    ccv_nnc_tensor_variable_free(graph, bias);
151
1.00k
    ccv_nnc_tensor_variable_free(graph, c);
152
1.00k
    ccv_nnc_tensor_variable_free(graph, s);
153
1.00k
    ccv_nnc_tensor_variable_free(graph, g);
154
1.00k
  }
155
1
  DYNAMIC_GRAPH_GEN(graph, CCV_NNC_LONG_DOT_GRAPH);
156
1
  ccv_nnc_tensor_t* const w_tensor = ccv_nnc_tensor_from_variable(graph, w);
157
1
  REQUIRE_EQ_WITH_TOLERANCE(10 * w_tensor->data.f32[0] + 1 * w_tensor->data.f32[1], -1, 1e-3, "converge for vector 1");
158
1
  REQUIRE_EQ_WITH_TOLERANCE(10 * w_tensor->data.f32[2] + 1 * w_tensor->data.f32[3], 1, 1e-3, "converge for vector 1");
159
1
  REQUIRE_EQ_WITH_TOLERANCE(3 * w_tensor->data.f32[0] + 5 * w_tensor->data.f32[1], -1, 1e-1, "converge for vector 2");
160
1
  REQUIRE_EQ_WITH_TOLERANCE(3 * w_tensor->data.f32[2] + 5 * w_tensor->data.f32[3], 1, 1e-1, "converge for vector 2");
161
1
  ccv_nnc_dynamic_graph_free(graph);
162
1
}
163
164
TEST_CASE("solve least square sum with adam on symbolic graph")
165
1
{
166
1
  ccv_nnc_symbolic_graph_t* const symbolic_graph = ccv_nnc_symbolic_graph_new();
167
1
  ccv_nnc_tensor_symbol_t a = ccv_nnc_tensor_symbol_new(symbolic_graph, CPU_TENSOR_NHWC(32F, 2, 2), "a");
168
1
  ccv_nnc_tensor_symbol_t w = ccv_nnc_tensor_symbol_new(symbolic_graph, CPU_TENSOR_NHWC(32F, 2, 2), "w");
169
1
  ccv_nnc_tensor_symbol_t bias = ccv_nnc_tensor_symbol_new(symbolic_graph, CPU_TENSOR_NHWC(32F, 2), "bias");
170
1
  ccv_nnc_tensor_symbol_t b = ccv_nnc_tensor_symbol_new(symbolic_graph, CPU_TENSOR_NHWC(32F, 2, 2), "b");
171
1
  ccv_nnc_graph_exec_symbol_new(symbolic_graph, CMD_GEMM_FORWARD(NO_TRANSPOSE, TRANSPOSE(0, 1)), TENSOR_SYMBOL_LIST(a, w, bias), TENSOR_SYMBOL_LIST(b), "gemm");
172
1
  ccv_nnc_tensor_symbol_t c = ccv_nnc_tensor_symbol_new(symbolic_graph, CPU_TENSOR_NHWC(32F, 2, 2), "c");
173
1
  ccv_nnc_graph_exec_symbol_new(symbolic_graph, CMD_EWPROD_FORWARD(), TENSOR_SYMBOL_LIST(b, b), TENSOR_SYMBOL_LIST(c), "square");
174
1
  ccv_nnc_tensor_symbol_t s = ccv_nnc_tensor_symbol_new(symbolic_graph, ccv_nnc_tensor_auto, "s");
175
1
  ccv_nnc_graph_exec_symbol_new(symbolic_graph, CMD_REDUCE_SUM_FORWARD(0, 1), TENSOR_SYMBOL_LIST(c), TENSOR_SYMBOL_LIST(s), "sum");
176
1
  ccv_nnc_graph_exec_symbol_autogen(symbolic_graph, 0, 0, CCV_NNC_AUTOGEN_ALL_EXECS | CCV_NNC_AUTOGEN_SOURCES_AND_DESTINATIONS);
177
1
  ccv_nnc_tensor_symbol_t updates[1];
178
1
  ccv_nnc_tensor_symbol_map_t aux[2];
179
1
  ccv_nnc_graph_exec_symbol_t update_execs[1];
180
1
  ccv_nnc_symbolic_graph_minimize(symbolic_graph, CMD_ADAM_FORWARD(1, 0.002, 0.9, 0.98, 0, 1e-9), TENSOR_SYMBOL_LIST(s), TENSOR_SYMBOL_LIST(w), 0, 0, SYMBOLIC_GRAPH_SOURCES(symbolic_graph), SYMBOLIC_GRAPH_DESTINATIONS(symbolic_graph), 0, updates, aux, update_execs);
181
1
  SYMBOLIC_GRAPH_GEN(symbolic_graph, CCV_NNC_LONG_DOT_GRAPH);
182
1
  ccv_nnc_graph_t* graph;
183
1
  ccv_nnc_tensor_arena_t* tensor_arena;
184
1
  ccv_nnc_graph_exec_arena_t* graph_exec_arena;
185
1
  ccv_nnc_symbolic_graph_compile(symbolic_graph, ccv_nnc_default_compile_params, 0, 0, 0, 0, SYMBOLIC_GRAPH_SOURCES(symbolic_graph), update_execs, 1, &graph, &tensor_arena, &graph_exec_arena);
186
1
  GRAPH_GEN(graph, CCV_NNC_LONG_DOT_GRAPH);
187
1
  // Relies on the inplace ops for ADAM set on both updated w / bias, and momentum.
188
1
  ccv_nnc_tensor_t* const w_tensor = ccv_nnc_tensor_from_symbol(tensor_arena, w);
189
1
  ccv_nnc_cmd_exec(CMD_RANDOM_UNIFORM_FORWARD(-0.5, 0.5), ccv_nnc_no_hint, 0, 0, 0, TENSOR_LIST(w_tensor), 0);
190
1
  ccv_nnc_tensor_t* const bias_tensor = ccv_nnc_tensor_from_symbol(tensor_arena, bias);
191
1
  int i;
192
3
  for (i = 0; i < 2; 
i++2
)
193
2
  {
194
2
    ccv_nnc_tensor_t* const aux_tensor = ccv_nnc_tensor_from_symbol(tensor_arena, aux[i].source);
195
2
    ccv_nnc_cmd_exec(CMD_SET_FORWARD(0), ccv_nnc_no_hint, 0, 0, 0, TENSOR_LIST(aux_tensor), 0);
196
2
  }
197
1
  ccv_nnc_tensor_t* const a_tensor = ccv_nnc_tensor_from_symbol(tensor_arena, a);
198
1
  ccv_nnc_tensor_t* const f_tensor = ccv_nnc_tensor_from_symbol(tensor_arena, ccv_nnc_tensor_symbol_for_backward(symbolic_graph, s));
199
1
  ccv_nnc_graph_exec_t adam = ccv_nnc_graph_exec_from_symbol(graph_exec_arena, update_execs[0]);
200
1.00k
  for (i = 0; i < 1000; 
i++1.00k
)
201
1.00k
  {
202
1.00k
    a_tensor->data.f32[0] = 10;
203
1.00k
    a_tensor->data.f32[1] = 1;
204
1.00k
    a_tensor->data.f32[2] = 3;
205
1.00k
    a_tensor->data.f32[3] = 5;
206
1.00k
    f_tensor->data.f32[0] = 1;
207
1.00k
    bias_tensor->data.f32[0] = 1;
208
1.00k
    bias_tensor->data.f32[1] = -1;
209
1.00k
    ccv_nnc_graph_exec_set(graph, adam, CMD_ADAM_FORWARD(i + 1, 0.002, 0.9, 0.98, 0, 1e-9));
210
1.00k
    ccv_nnc_graph_run(graph, 0, TRAVERSE_FULL, 0, 0);
211
1.00k
  }
212
1
  REQUIRE_EQ_WITH_TOLERANCE(a_tensor->data.f32[0] * w_tensor->data.f32[0] + a_tensor->data.f32[1] * w_tensor->data.f32[1], -1, 1e-1, "converge for vector 1");
213
1
  REQUIRE_EQ_WITH_TOLERANCE(a_tensor->data.f32[0] * w_tensor->data.f32[2] + a_tensor->data.f32[1] * w_tensor->data.f32[3], 1, 1e-1, "converge for vector 1");
214
1
  REQUIRE_EQ_WITH_TOLERANCE(a_tensor->data.f32[2] * w_tensor->data.f32[0] + a_tensor->data.f32[3] * w_tensor->data.f32[1], -1, 1e-1, "converge for vector 2");
215
1
  REQUIRE_EQ_WITH_TOLERANCE(a_tensor->data.f32[2] * w_tensor->data.f32[2] + a_tensor->data.f32[3] * w_tensor->data.f32[3], 1, 1e-1, "converge for vector 2");
216
1
  ccv_nnc_symbolic_graph_free(symbolic_graph);
217
1
  ccv_nnc_graph_free(graph);
218
1
  ccv_nnc_tensor_arena_free(tensor_arena);
219
1
  ccv_nnc_graph_exec_arena_free(graph_exec_arena);
220
1
}
221
222
TEST_CASE("solve least square sum with rmsprop on symbolic graph")
223
1
{
224
1
  ccv_nnc_symbolic_graph_t* const symbolic_graph = ccv_nnc_symbolic_graph_new();
225
1
  ccv_nnc_tensor_symbol_t a = ccv_nnc_tensor_symbol_new(symbolic_graph, CPU_TENSOR_NHWC(32F, 2, 2), "a");
226
1
  ccv_nnc_tensor_symbol_t w = ccv_nnc_tensor_symbol_new(symbolic_graph, CPU_TENSOR_NHWC(32F, 2, 2), "w");
227
1
  ccv_nnc_tensor_symbol_t bias = ccv_nnc_tensor_symbol_new(symbolic_graph, CPU_TENSOR_NHWC(32F, 2), "bias");
228
1
  ccv_nnc_tensor_symbol_t b = ccv_nnc_tensor_symbol_new(symbolic_graph, CPU_TENSOR_NHWC(32F, 2, 2), "b");
229
1
  ccv_nnc_graph_exec_symbol_new(symbolic_graph, CMD_GEMM_FORWARD(NO_TRANSPOSE, TRANSPOSE(0, 1)), TENSOR_SYMBOL_LIST(a, w, bias), TENSOR_SYMBOL_LIST(b), "gemm");
230
1
  ccv_nnc_tensor_symbol_t c = ccv_nnc_tensor_symbol_new(symbolic_graph, CPU_TENSOR_NHWC(32F, 2, 2), "c");
231
1
  ccv_nnc_graph_exec_symbol_new(symbolic_graph, CMD_EWPROD_FORWARD(), TENSOR_SYMBOL_LIST(b, b), TENSOR_SYMBOL_LIST(c), "square");
232
1
  ccv_nnc_tensor_symbol_t s = ccv_nnc_tensor_symbol_new(symbolic_graph, ccv_nnc_tensor_auto, "s");
233
1
  ccv_nnc_graph_exec_symbol_new(symbolic_graph, CMD_REDUCE_SUM_FORWARD(0, 1), TENSOR_SYMBOL_LIST(c), TENSOR_SYMBOL_LIST(s), "sum");
234
1
  ccv_nnc_graph_exec_symbol_autogen(symbolic_graph, 0, 0, CCV_NNC_AUTOGEN_ALL_EXECS | CCV_NNC_AUTOGEN_SOURCES_AND_DESTINATIONS);
235
1
  ccv_nnc_tensor_symbol_t updates[1];
236
1
  ccv_nnc_tensor_symbol_map_t aux[2];
237
1
  ccv_nnc_graph_exec_symbol_t update_execs[1];
238
1
  ccv_nnc_symbolic_graph_minimize(symbolic_graph, CMD_RMSPROP_FORWARD(0.001, 0.0001, 0.9, 0.9, 1e-9), TENSOR_SYMBOL_LIST(s), TENSOR_SYMBOL_LIST(w), 0, 0, SYMBOLIC_GRAPH_SOURCES(symbolic_graph), SYMBOLIC_GRAPH_DESTINATIONS(symbolic_graph), 0, updates, aux, update_execs);
239
1
  SYMBOLIC_GRAPH_GEN(symbolic_graph, CCV_NNC_LONG_DOT_GRAPH);
240
1
  ccv_nnc_graph_t* graph;
241
1
  ccv_nnc_tensor_arena_t* tensor_arena;
242
1
  ccv_nnc_graph_exec_arena_t* graph_exec_arena;
243
1
  ccv_nnc_symbolic_graph_compile(symbolic_graph, ccv_nnc_default_compile_params, 0, 0, 0, 0, SYMBOLIC_GRAPH_SOURCES(symbolic_graph), update_execs, 1, &graph, &tensor_arena, &graph_exec_arena);
244
1
  GRAPH_GEN(graph, CCV_NNC_LONG_DOT_GRAPH);
245
1
  // Relies on the inplace ops for ADAM set on both updated w / bias, and momentum.
246
1
  ccv_nnc_tensor_t* const w_tensor = ccv_nnc_tensor_from_symbol(tensor_arena, w);
247
1
  ccv_nnc_cmd_exec(CMD_RANDOM_UNIFORM_FORWARD(-0.5, 0.5), ccv_nnc_no_hint, 0, 0, 0, TENSOR_LIST(w_tensor), 0);
248
1
  ccv_nnc_tensor_t* const bias_tensor = ccv_nnc_tensor_from_symbol(tensor_arena, bias);
249
1
  int i;
250
3
  for (i = 0; i < 2; 
i++2
)
251
2
  {
252
2
    ccv_nnc_tensor_t* const aux_tensor = ccv_nnc_tensor_from_symbol(tensor_arena, aux[i].source);
253
2
    ccv_nnc_cmd_exec(CMD_SET_FORWARD(0), ccv_nnc_no_hint, 0, 0, 0, TENSOR_LIST(aux_tensor), 0);
254
2
  }
255
1
  ccv_nnc_tensor_t* const a_tensor = ccv_nnc_tensor_from_symbol(tensor_arena, a);
256
1
  ccv_nnc_tensor_t* const f_tensor = ccv_nnc_tensor_from_symbol(tensor_arena, ccv_nnc_tensor_symbol_for_backward(symbolic_graph, s));
257
1
  ccv_nnc_graph_exec_t adam = ccv_nnc_graph_exec_from_symbol(graph_exec_arena, update_execs[0]);
258
1.00k
  for (i = 0; i < 1000; 
i++1.00k
)
259
1.00k
  {
260
1.00k
    a_tensor->data.f32[0] = 10;
261
1.00k
    a_tensor->data.f32[1] = 1;
262
1.00k
    a_tensor->data.f32[2] = 3;
263
1.00k
    a_tensor->data.f32[3] = 5;
264
1.00k
    f_tensor->data.f32[0] = 1;
265
1.00k
    bias_tensor->data.f32[0] = 1;
266
1.00k
    bias_tensor->data.f32[1] = -1;
267
1.00k
    ccv_nnc_graph_exec_set(graph, adam, CMD_RMSPROP_FORWARD(0.001, 0.0001, 0.9, 0.9, 1e-9));
268
1.00k
    ccv_nnc_graph_run(graph, 0, TRAVERSE_FULL, 0, 0);
269
1.00k
  }
270
1
  REQUIRE_EQ_WITH_TOLERANCE(a_tensor->data.f32[0] * w_tensor->data.f32[0] + a_tensor->data.f32[1] * w_tensor->data.f32[1], -1, 1e-1, "converge for vector 1");
271
1
  REQUIRE_EQ_WITH_TOLERANCE(a_tensor->data.f32[0] * w_tensor->data.f32[2] + a_tensor->data.f32[1] * w_tensor->data.f32[3], 1, 1e-1, "converge for vector 1");
272
1
  REQUIRE_EQ_WITH_TOLERANCE(a_tensor->data.f32[2] * w_tensor->data.f32[0] + a_tensor->data.f32[3] * w_tensor->data.f32[1], -1, 1e-1, "converge for vector 2");
273
1
  REQUIRE_EQ_WITH_TOLERANCE(a_tensor->data.f32[2] * w_tensor->data.f32[2] + a_tensor->data.f32[3] * w_tensor->data.f32[3], 1, 1e-1, "converge for vector 2");
274
1
  ccv_nnc_symbolic_graph_free(symbolic_graph);
275
1
  ccv_nnc_graph_free(graph);
276
1
  ccv_nnc_tensor_arena_free(tensor_arena);
277
1
  ccv_nnc_graph_exec_arena_free(graph_exec_arena);
278
1
}
279
280
#include "case_main.h"