Coverage Report

Created: 2026-05-04 15:30

next uncovered line (L), next uncovered region (R), next uncovered branch (B)
/home/liu/actions-runner/_work/ccv/ccv/test/int/nnc/cublas.tests.c
Line
Count
Source
1
#include "case.h"
2
#include "ccv_case.h"
3
#include "ccv_nnc_case.h"
4
#include <ccv.h>
5
#include <nnc/ccv_nnc.h>
6
#include <nnc/ccv_nnc_easy.h>
7
#include <3rdparty/dsfmt/dSFMT.h>
8
9
TEST_SETUP()
10
{
11
  ccv_nnc_init();
12
}
13
14
TEST_CASE("gemm no transpose")
15
1
{
16
1
  GUARD_ELSE_RETURN(ccv_nnc_cmd_ok(CCV_NNC_GEMM_FORWARD, CCV_NNC_BACKEND_GPU_CUBLAS));
17
1
  float ap[] = {
18
1
    1, 2,
19
1
    3, 4,
20
1
    5, 6,
21
1
    7, 8,
22
1
  };
23
1
  ccv_nnc_tensor_t* const a = ccv_nnc_tensor_new(ap, CPU_TENSOR_NHWC(32F, 4, 2), 0);
24
1
  float bp[] = {
25
1
    7, 8, 9,
26
1
    10, 11, 12,
27
1
  };
28
1
  ccv_nnc_tensor_t* const b = ccv_nnc_tensor_new(bp, CPU_TENSOR_NHWC(32F, 2, 3), 0);
29
1
  ccv_nnc_tensor_t* const c = ccv_nnc_tensor_new(0, CPU_TENSOR_NHWC(32F, 4, 3), 0);
30
1
  ccv_nnc_tensor_t* ga = ccv_nnc_tensor_new(0, GPU_TENSOR_NHWC(000, 32F, 4, 2), 0);
31
1
  ccv_nnc_tensor_t* gb = ccv_nnc_tensor_new(0, GPU_TENSOR_NHWC(000, 32F, 2, 3), 0);
32
1
  ccv_nnc_tensor_t* gc = ccv_nnc_tensor_new(0, GPU_TENSOR_NHWC(000, 32F, 4, 3), 0);
33
1
  ccv_nnc_cmd_exec(CMD_DATA_TRANSFER_FORWARD(), ccv_nnc_no_hint, 0, TENSOR_LIST(a, b), TENSOR_LIST(ga, gb), 0);
34
1
  ccv_nnc_cmd_exec(CMD_GEMM_FORWARD(), ccv_nnc_no_hint, 0, TENSOR_LIST(ga, gb), TENSOR_LIST(gc), 0);
35
1
  ccv_nnc_cmd_exec(CMD_DATA_TRANSFER_FORWARD(), ccv_nnc_no_hint, 0, TENSOR_LIST(gc), TENSOR_LIST(c), 0);
36
1
  float ctp[] = {
37
1
    1 * 7 + 2 * 10, 1 * 8 + 2 * 11, 1 * 9 + 2 * 12,
38
1
    3 * 7 + 4 * 10, 3 * 8 + 4 * 11, 3 * 9 + 4 * 12,
39
1
    5 * 7 + 6 * 10, 5 * 8 + 6 * 11, 5 * 9 + 6 * 12,
40
1
    7 * 7 + 8 * 10, 7 * 8 + 8 * 11, 7 * 9 + 8 * 12,
41
1
  };
42
1
  ccv_nnc_tensor_t ct = ccv_nnc_tensor(ctp, CPU_TENSOR_NHWC(32F, 4, 3), 0);
43
1
  REQUIRE_TENSOR_EQ(c, &ct, "result should be equal");
44
1
  ccv_nnc_tensor_free(a);
45
1
  ccv_nnc_tensor_free(b);
46
1
  ccv_nnc_tensor_free(c);
47
1
  ccv_nnc_tensor_free(ga);
48
1
  ccv_nnc_tensor_free(gb);
49
1
  ccv_nnc_tensor_free(gc);
50
1
}
51
52
TEST_CASE("gemm transpose a")
53
1
{
54
1
  GUARD_ELSE_RETURN(ccv_nnc_cmd_ok(CCV_NNC_GEMM_FORWARD, CCV_NNC_BACKEND_GPU_CUBLAS));
55
1
  float ap[] = {
56
1
    1, 3, 5, 7,
57
1
    2, 4, 6, 8,
58
1
  };
59
1
  ccv_nnc_tensor_t* const a = ccv_nnc_tensor_new(ap, CPU_TENSOR_NHWC(32F, 2, 4), 0);
60
1
  float bp[] = {
61
1
    7, 8, 9,
62
1
    10, 11, 12,
63
1
  };
64
1
  ccv_nnc_tensor_t* const b = ccv_nnc_tensor_new(bp, CPU_TENSOR_NHWC(32F, 2, 3), 0);
65
1
  ccv_nnc_tensor_t* const c = ccv_nnc_tensor_new(0, CPU_TENSOR_NHWC(32F, 4, 3), 0);
66
1
  ccv_nnc_tensor_t* ga = ccv_nnc_tensor_new(0, GPU_TENSOR_NHWC(000, 32F, 2, 4), 0);
67
1
  ccv_nnc_tensor_t* gb = ccv_nnc_tensor_new(0, GPU_TENSOR_NHWC(000, 32F, 2, 3), 0);
68
1
  ccv_nnc_tensor_t* gc = ccv_nnc_tensor_new(0, GPU_TENSOR_NHWC(000, 32F, 4, 3), 0);
69
1
  ccv_nnc_cmd_exec(CMD_DATA_TRANSFER_FORWARD(), ccv_nnc_no_hint, 0, TENSOR_LIST(a, b), TENSOR_LIST(ga, gb), 0);
70
1
  ccv_nnc_cmd_exec(CMD_GEMM_FORWARD(TRANSPOSE(0, 1)), ccv_nnc_no_hint, 0, TENSOR_LIST(ga, gb), TENSOR_LIST(gc), 0);
71
1
  ccv_nnc_cmd_exec(CMD_DATA_TRANSFER_FORWARD(), ccv_nnc_no_hint, 0, TENSOR_LIST(gc), TENSOR_LIST(c), 0);
72
1
  float ctp[] = {
73
1
    1 * 7 + 2 * 10, 1 * 8 + 2 * 11, 1 * 9 + 2 * 12,
74
1
    3 * 7 + 4 * 10, 3 * 8 + 4 * 11, 3 * 9 + 4 * 12,
75
1
    5 * 7 + 6 * 10, 5 * 8 + 6 * 11, 5 * 9 + 6 * 12,
76
1
    7 * 7 + 8 * 10, 7 * 8 + 8 * 11, 7 * 9 + 8 * 12,
77
1
  };
78
1
  ccv_nnc_tensor_t ct = ccv_nnc_tensor(ctp, CPU_TENSOR_NHWC(32F, 4, 3), 0);
79
1
  REQUIRE_TENSOR_EQ(c, &ct, "result should be equal");
80
1
  ccv_nnc_tensor_free(a);
81
1
  ccv_nnc_tensor_free(b);
82
1
  ccv_nnc_tensor_free(c);
83
1
  ccv_nnc_tensor_free(ga);
84
1
  ccv_nnc_tensor_free(gb);
85
1
  ccv_nnc_tensor_free(gc);
86
1
}
87
88
TEST_CASE("gemm transpose b")
89
1
{
90
1
  GUARD_ELSE_RETURN(ccv_nnc_cmd_ok(CCV_NNC_GEMM_FORWARD, CCV_NNC_BACKEND_GPU_CUBLAS));
91
1
  float ap[] = {
92
1
    1, 2,
93
1
    3, 4,
94
1
    5, 6,
95
1
    7, 8,
96
1
  };
97
1
  ccv_nnc_tensor_t* const a = ccv_nnc_tensor_new(ap, CPU_TENSOR_NHWC(32F, 4, 2), 0);
98
1
  float bp[] = {
99
1
    7, 10,
100
1
    8, 11,
101
1
    9, 12,
102
1
  };
103
1
  ccv_nnc_tensor_t* const b = ccv_nnc_tensor_new(bp, CPU_TENSOR_NHWC(32F, 3, 2), 0);
104
1
  ccv_nnc_tensor_t* const c = ccv_nnc_tensor_new(0, CPU_TENSOR_NHWC(32F, 4, 3), 0);
105
1
  ccv_nnc_tensor_t* ga = ccv_nnc_tensor_new(0, GPU_TENSOR_NHWC(000, 32F, 4, 2), 0);
106
1
  ccv_nnc_tensor_t* gb = ccv_nnc_tensor_new(0, GPU_TENSOR_NHWC(000, 32F, 3, 2), 0);
107
1
  ccv_nnc_tensor_t* gc = ccv_nnc_tensor_new(0, GPU_TENSOR_NHWC(000, 32F, 4, 3), 0);
108
1
  ccv_nnc_cmd_exec(CMD_DATA_TRANSFER_FORWARD(), ccv_nnc_no_hint, 0, TENSOR_LIST(a, b), TENSOR_LIST(ga, gb), 0);
109
1
  ccv_nnc_cmd_exec(CMD_GEMM_FORWARD(NO_TRANSPOSE, TRANSPOSE(0, 1)), ccv_nnc_no_hint, 0, TENSOR_LIST(ga, gb), TENSOR_LIST(gc), 0);
110
1
  ccv_nnc_cmd_exec(CMD_DATA_TRANSFER_FORWARD(), ccv_nnc_no_hint, 0, TENSOR_LIST(gc), TENSOR_LIST(c), 0);
111
1
  float ctp[] = {
112
1
    1 * 7 + 2 * 10, 1 * 8 + 2 * 11, 1 * 9 + 2 * 12,
113
1
    3 * 7 + 4 * 10, 3 * 8 + 4 * 11, 3 * 9 + 4 * 12,
114
1
    5 * 7 + 6 * 10, 5 * 8 + 6 * 11, 5 * 9 + 6 * 12,
115
1
    7 * 7 + 8 * 10, 7 * 8 + 8 * 11, 7 * 9 + 8 * 12,
116
1
  };
117
1
  ccv_nnc_tensor_t ct = ccv_nnc_tensor(ctp, CPU_TENSOR_NHWC(32F, 4, 3), 0);
118
1
  REQUIRE_TENSOR_EQ(c, &ct, "result should be equal");
119
1
  ccv_nnc_tensor_free(a);
120
1
  ccv_nnc_tensor_free(b);
121
1
  ccv_nnc_tensor_free(c);
122
1
  ccv_nnc_tensor_free(ga);
123
1
  ccv_nnc_tensor_free(gb);
124
1
  ccv_nnc_tensor_free(gc);
125
1
}
126
127
TEST_CASE("gemm transpose a and b")
128
1
{
129
1
  GUARD_ELSE_RETURN(ccv_nnc_cmd_ok(CCV_NNC_GEMM_FORWARD, CCV_NNC_BACKEND_GPU_CUBLAS));
130
1
  float ap[] = {
131
1
    1, 3, 5, 7,
132
1
    2, 4, 6, 8,
133
1
  };
134
1
  ccv_nnc_tensor_t* const a = ccv_nnc_tensor_new(ap, CPU_TENSOR_NHWC(32F, 2, 4), 0);
135
1
  float bp[] = {
136
1
    7, 10,
137
1
    8, 11,
138
1
    9, 12,
139
1
  };
140
1
  ccv_nnc_tensor_t* const b = ccv_nnc_tensor_new(bp, CPU_TENSOR_NHWC(32F, 3, 2), 0);
141
1
  ccv_nnc_tensor_t* const c = ccv_nnc_tensor_new(0, CPU_TENSOR_NHWC(32F, 4, 3), 0);
142
1
  ccv_nnc_tensor_t* ga = ccv_nnc_tensor_new(0, GPU_TENSOR_NHWC(000, 32F, 2, 4), 0);
143
1
  ccv_nnc_tensor_t* gb = ccv_nnc_tensor_new(0, GPU_TENSOR_NHWC(000, 32F, 3, 2), 0);
144
1
  ccv_nnc_tensor_t* gc = ccv_nnc_tensor_new(0, GPU_TENSOR_NHWC(000, 32F, 4, 3), 0);
145
1
  ccv_nnc_cmd_exec(CMD_DATA_TRANSFER_FORWARD(), ccv_nnc_no_hint, 0, TENSOR_LIST(a, b), TENSOR_LIST(ga, gb), 0);
146
1
  ccv_nnc_cmd_exec(CMD_GEMM_FORWARD(TRANSPOSE(0, 1), TRANSPOSE(0, 1)), ccv_nnc_no_hint, 0, TENSOR_LIST(ga, gb), TENSOR_LIST(gc), 0);
147
1
  ccv_nnc_cmd_exec(CMD_DATA_TRANSFER_FORWARD(), ccv_nnc_no_hint, 0, TENSOR_LIST(gc), TENSOR_LIST(c), 0);
148
1
  float ctp[] = {
149
1
    1 * 7 + 2 * 10, 1 * 8 + 2 * 11, 1 * 9 + 2 * 12,
150
1
    3 * 7 + 4 * 10, 3 * 8 + 4 * 11, 3 * 9 + 4 * 12,
151
1
    5 * 7 + 6 * 10, 5 * 8 + 6 * 11, 5 * 9 + 6 * 12,
152
1
    7 * 7 + 8 * 10, 7 * 8 + 8 * 11, 7 * 9 + 8 * 12,
153
1
  };
154
1
  ccv_nnc_tensor_t ct = ccv_nnc_tensor(ctp, CPU_TENSOR_NHWC(32F, 4, 3), 0);
155
1
  REQUIRE_TENSOR_EQ(c, &ct, "result should be equal");
156
1
  ccv_nnc_tensor_free(a);
157
1
  ccv_nnc_tensor_free(b);
158
1
  ccv_nnc_tensor_free(c);
159
1
  ccv_nnc_tensor_free(ga);
160
1
  ccv_nnc_tensor_free(gb);
161
1
  ccv_nnc_tensor_free(gc);
162
1
}
163
164
TEST_CASE("gemm no transpose with bias")
165
1
{
166
1
  GUARD_ELSE_RETURN(ccv_nnc_cmd_ok(CCV_NNC_GEMM_FORWARD, CCV_NNC_BACKEND_GPU_CUBLAS));
167
1
  float ap[] = {
168
1
    1, 2,
169
1
    3, 4,
170
1
    5, 6,
171
1
    7, 8,
172
1
  };
173
1
  ccv_nnc_tensor_t* const a = ccv_nnc_tensor_new(ap, CPU_TENSOR_NHWC(32F, 4, 2), 0);
174
1
  float bp[] = {
175
1
    7, 8, 9,
176
1
    10, 11, 12,
177
1
  };
178
1
  ccv_nnc_tensor_t* const b = ccv_nnc_tensor_new(bp, CPU_TENSOR_NHWC(32F, 2, 3), 0);
179
1
  float dp[] = {
180
1
    1, -1, 1,
181
1
    1, -1, 1,
182
1
    1, -1, 1,
183
1
    1, -1, 1,
184
1
  };
185
1
  ccv_nnc_tensor_t* const d = ccv_nnc_tensor_new(dp, CPU_TENSOR_NHWC(32F, 4, 3), 0);
186
1
  ccv_nnc_tensor_t* const c = ccv_nnc_tensor_new(0, CPU_TENSOR_NHWC(32F, 4, 3), 0);
187
1
  ccv_nnc_tensor_t* ga = ccv_nnc_tensor_new(0, GPU_TENSOR_NHWC(000, 32F, 4, 2), 0);
188
1
  ccv_nnc_tensor_t* gb = ccv_nnc_tensor_new(0, GPU_TENSOR_NHWC(000, 32F, 2, 3), 0);
189
1
  ccv_nnc_tensor_t* gd = ccv_nnc_tensor_new(0, GPU_TENSOR_NHWC(000, 32F, 4, 3), 0);
190
1
  ccv_nnc_tensor_t* gc = ccv_nnc_tensor_new(0, GPU_TENSOR_NHWC(000, 32F, 4, 3), 0);
191
1
  ccv_nnc_cmd_exec(CMD_DATA_TRANSFER_FORWARD(), ccv_nnc_no_hint, 0, TENSOR_LIST(a, b, d), TENSOR_LIST(ga, gb, gd), 0);
192
1
  ccv_nnc_cmd_exec(CMD_GEMM_FORWARD(), ccv_nnc_no_hint, 0, TENSOR_LIST(ga, gb, gd), TENSOR_LIST(gc), 0);
193
1
  ccv_nnc_cmd_exec(CMD_DATA_TRANSFER_FORWARD(), ccv_nnc_no_hint, 0, TENSOR_LIST(gc), TENSOR_LIST(c), 0);
194
1
  float ctp[] = {
195
1
    1 * 7 + 2 * 10 + 1, 1 * 8 + 2 * 11 - 1, 1 * 9 + 2 * 12 + 1,
196
1
    3 * 7 + 4 * 10 + 1, 3 * 8 + 4 * 11 - 1, 3 * 9 + 4 * 12 + 1,
197
1
    5 * 7 + 6 * 10 + 1, 5 * 8 + 6 * 11 - 1, 5 * 9 + 6 * 12 + 1,
198
1
    7 * 7 + 8 * 10 + 1, 7 * 8 + 8 * 11 - 1, 7 * 9 + 8 * 12 + 1,
199
1
  };
200
1
  ccv_nnc_tensor_t ct = ccv_nnc_tensor(ctp, CPU_TENSOR_NHWC(32F, 4, 3), 0);
201
1
  REQUIRE_TENSOR_EQ(c, &ct, "result should be equal");
202
1
  ccv_nnc_tensor_free(a);
203
1
  ccv_nnc_tensor_free(b);
204
1
  ccv_nnc_tensor_free(c);
205
1
  ccv_nnc_tensor_free(d);
206
1
  ccv_nnc_tensor_free(ga);
207
1
  ccv_nnc_tensor_free(gb);
208
1
  ccv_nnc_tensor_free(gc);
209
1
  ccv_nnc_tensor_free(gd);
210
1
}
211
212
TEST_CASE("gemm no transpose with bias and palettize weights")
213
1
{
214
1
  GUARD_ELSE_RETURN(ccv_nnc_cmd_ok(CCV_NNC_GEMM_FORWARD, CCV_NNC_BACKEND_GPU_CUBLAS));
215
1
  float ap[] = {
216
1
    1, 2,
217
1
    3, 4,
218
1
    5, 6,
219
1
    7, 8,
220
1
  };
221
1
  ccv_nnc_tensor_t* const a = ccv_nnc_tensor_new(ap, CPU_TENSOR_NHWC(32F, 4, 2), 0);
222
1
  float bp[] = {
223
1
    7, 8, 9,
224
1
    10, 11, 12,
225
1
  };
226
1
  ccv_nnc_tensor_t* const b = ccv_nnc_tensor_new(bp, CPU_TENSOR_NHWC(32F, 2, 3), 0);
227
1
  float dp[] = {
228
1
    1, -1, 1,
229
1
    1, -1, 1,
230
1
    1, -1, 1,
231
1
    1, -1, 1,
232
1
  };
233
1
  ccv_nnc_tensor_t* const pb = ccv_nnc_tensor_new(0, ccv_nnc_tensor_palettize(CPU_TENSOR_NHWC(32F, 2, 3), 4, 128), 0);
234
1
  (void)ccv_nnc_palettize(b->data.u8, CCV_32F, CCV_TENSOR_CPU_MEMORY, 6, 4, 128, pb->data.u8, ccv_nnc_tensor_data_size_without_padding(pb->info));
235
1
  ccv_nnc_tensor_t* const d = ccv_nnc_tensor_new(dp, CPU_TENSOR_NHWC(32F, 4, 3), 0);
236
1
  ccv_nnc_tensor_t* const c = ccv_nnc_tensor_new(0, CPU_TENSOR_NHWC(32F, 4, 3), 0);
237
1
  ccv_nnc_tensor_t* ga = ccv_nnc_tensor_new(0, GPU_TENSOR_NHWC(000, 32F, 4, 2), 0);
238
1
  ccv_nnc_tensor_t* gb = ccv_nnc_tensor_new(0, ccv_nnc_tensor_palettize(GPU_TENSOR_NHWC(000, 32F, 2, 3), 4, 128), 0);
239
1
  ccv_nnc_tensor_t* gd = ccv_nnc_tensor_new(0, GPU_TENSOR_NHWC(000, 32F, 4, 3), 0);
240
1
  ccv_nnc_tensor_t* gc = ccv_nnc_tensor_new(0, GPU_TENSOR_NHWC(000, 32F, 4, 3), 0);
241
1
  ccv_nnc_cmd_exec(CMD_DATA_TRANSFER_FORWARD(), ccv_nnc_no_hint, 0, TENSOR_LIST(a, pb, d), TENSOR_LIST(ga, gb, gd), 0);
242
1
  ccv_nnc_cmd_exec(CMD_GEMM_FORWARD(), ccv_nnc_no_hint, 0, TENSOR_LIST(ga, gb, gd), TENSOR_LIST(gc), 0);
243
1
  ccv_nnc_cmd_exec(CMD_DATA_TRANSFER_FORWARD(), ccv_nnc_no_hint, 0, TENSOR_LIST(gc), TENSOR_LIST(c), 0);
244
1
  float ctp[] = {
245
1
    1 * 7 + 2 * 10 + 1, 1 * 8 + 2 * 11 - 1, 1 * 9 + 2 * 12 + 1,
246
1
    3 * 7 + 4 * 10 + 1, 3 * 8 + 4 * 11 - 1, 3 * 9 + 4 * 12 + 1,
247
1
    5 * 7 + 6 * 10 + 1, 5 * 8 + 6 * 11 - 1, 5 * 9 + 6 * 12 + 1,
248
1
    7 * 7 + 8 * 10 + 1, 7 * 8 + 8 * 11 - 1, 7 * 9 + 8 * 12 + 1,
249
1
  };
250
1
  ccv_nnc_tensor_t ct = ccv_nnc_tensor(ctp, CPU_TENSOR_NHWC(32F, 4, 3), 0);
251
1
  REQUIRE_TENSOR_EQ(c, &ct, "result should be equal");
252
1
  ccv_nnc_tensor_free(a);
253
1
  ccv_nnc_tensor_free(b);
254
1
  ccv_nnc_tensor_free(pb);
255
1
  ccv_nnc_tensor_free(c);
256
1
  ccv_nnc_tensor_free(d);
257
1
  ccv_nnc_tensor_free(ga);
258
1
  ccv_nnc_tensor_free(gb);
259
1
  ccv_nnc_tensor_free(gc);
260
1
  ccv_nnc_tensor_free(gd);
261
1
}
262
263
TEST_CASE("gemm no transpose with bias and row-wise int8 weights")
264
1
{
265
1
  GUARD_ELSE_RETURN(ccv_nnc_cmd_ok(CCV_NNC_GEMM_FORWARD, CCV_NNC_BACKEND_GPU_CUBLAS));
266
1
  float ap[] = {
267
1
    1, 2,
268
1
    3, 4,
269
1
    5, 6,
270
1
    7, 8,
271
1
  };
272
1
  ccv_nnc_tensor_t* const a = ccv_nnc_tensor_new(ap, CPU_TENSOR_NHWC(32F, 4, 2), 0);
273
1
  float bp[] = {
274
1
    7, 8, 9,
275
1
    10, 11, 12,
276
1
  };
277
1
  ccv_nnc_tensor_t* const b = ccv_nnc_tensor_new(bp, CPU_TENSOR_NHWC(32F, 2, 3), 0);
278
1
  ccv_nnc_tensor_t* const qb = ccv_nnc_tensor_new(0, ccv_nnc_tensor_8i_rowwise(CPU_TENSOR_NHWC(32F, 2, 3)), 0);
279
1
  const size_t qsize = ccv_nnc_quantize_8i_rowwise(b->data.u8, CCV_32F, CCV_TENSOR_CPU_MEMORY, 6, 3, qb->data.u8, ccv_nnc_tensor_data_size_without_padding(qb->info));
280
1
  ccv_nnc_tensor_t* const dq_b = ccv_nnc_tensor_new(0, CPU_TENSOR_NHWC(32F, 2, 3), 0);
281
1
  ccv_nnc_dequantize_8i_rowwise(qb->data.u8, CCV_32F, CCV_TENSOR_CPU_MEMORY, qsize, 3, dq_b->data.u8, 6);
282
1
  float dp[] = {
283
1
    1, -1, 1,
284
1
    1, -1, 1,
285
1
    1, -1, 1,
286
1
    1, -1, 1,
287
1
  };
288
1
  ccv_nnc_tensor_t* const d = ccv_nnc_tensor_new(dp, CPU_TENSOR_NHWC(32F, 4, 3), 0);
289
1
  ccv_nnc_tensor_t* const ct = ccv_nnc_tensor_new(0, CPU_TENSOR_NHWC(32F, 4, 3), 0);
290
1
  ccv_nnc_cmd_t cmd = CMD_GEMM_FORWARD();
291
1
  cmd.backend = CCV_NNC_BACKEND_CPU_REF;
292
1
  assert(cmd.backend >= 0);
293
1
  ccv_nnc_cmd_exec(cmd, ccv_nnc_no_hint, 0, TENSOR_LIST(a, dq_b, d), TENSOR_LIST(ct), 0);
294
1
  ccv_nnc_tensor_t* const c = ccv_nnc_tensor_new(0, CPU_TENSOR_NHWC(32F, 4, 3), 0);
295
1
  ccv_nnc_tensor_t* ga = ccv_nnc_tensor_new(0, GPU_TENSOR_NHWC(000, 32F, 4, 2), 0);
296
1
  ccv_nnc_tensor_t* gqb = ccv_nnc_tensor_new(0, ccv_nnc_tensor_8i_rowwise(GPU_TENSOR_NHWC(000, 32F, 2, 3)), 0);
297
1
  ccv_nnc_tensor_t* gd = ccv_nnc_tensor_new(0, GPU_TENSOR_NHWC(000, 32F, 4, 3), 0);
298
1
  ccv_nnc_tensor_t* gc = ccv_nnc_tensor_new(0, GPU_TENSOR_NHWC(000, 32F, 4, 3), 0);
299
1
  ccv_nnc_cmd_exec(CMD_DATA_TRANSFER_FORWARD(), ccv_nnc_no_hint, 0, TENSOR_LIST(a, qb, d), TENSOR_LIST(ga, gqb, gd), 0);
300
1
  ccv_nnc_cmd_exec(CMD_GEMM_FORWARD(), ccv_nnc_no_hint, 0, TENSOR_LIST(ga, gqb, gd), TENSOR_LIST(gc), 0);
301
1
  ccv_nnc_cmd_exec(CMD_DATA_TRANSFER_FORWARD(), ccv_nnc_no_hint, 0, TENSOR_LIST(gc), TENSOR_LIST(c), 0);
302
1
  REQUIRE_ARRAY_EQ_WITH_TOLERANCE(float, ct->data.f32, c->data.f32, 12, 1e-5, "result should match CPU with dequantized row-wise weights");
303
1
  ccv_nnc_tensor_free(a);
304
1
  ccv_nnc_tensor_free(b);
305
1
  ccv_nnc_tensor_free(qb);
306
1
  ccv_nnc_tensor_free(dq_b);
307
1
  ccv_nnc_tensor_free(d);
308
1
  ccv_nnc_tensor_free(ct);
309
1
  ccv_nnc_tensor_free(c);
310
1
  ccv_nnc_tensor_free(ga);
311
1
  ccv_nnc_tensor_free(gqb);
312
1
  ccv_nnc_tensor_free(gd);
313
1
  ccv_nnc_tensor_free(gc);
314
1
}
315
316
TEST_CASE("backward gemm with no transpose")
317
1
{
318
1
  GUARD_ELSE_RETURN(ccv_nnc_cmd_ok(CCV_NNC_GEMM_FORWARD, CCV_NNC_BACKEND_GPU_CUBLAS) &&
319
1
    ccv_nnc_cmd_ok(CCV_NNC_GEMM_BACKWARD, CCV_NNC_BACKEND_GPU_CUBLAS));
320
1
  float gp[] = {
321
1
    1, 2, 3,
322
1
    4, 5, 6,
323
1
    7, 8, 9,
324
1
    10, 11, 12,
325
1
  };
326
1
  ccv_nnc_tensor_t* const g = ccv_nnc_tensor_new(gp, CPU_TENSOR_NHWC(32F, 4, 3), 0);
327
1
  float ap[] = {
328
1
    13, 14,
329
1
    15, 16,
330
1
    17, 18,
331
1
    19, 20,
332
1
  };
333
1
  ccv_nnc_tensor_t* const a = ccv_nnc_tensor_new(ap, CPU_TENSOR_NHWC(32F, 4, 2), 0);
334
1
  float bp[] = {
335
1
    21, 22, 23,
336
1
    24, 25, 26,
337
1
  };
338
1
  ccv_nnc_tensor_t* const b = ccv_nnc_tensor_new(bp, CPU_TENSOR_NHWC(32F, 2, 3), 0);
339
1
  ccv_nnc_tensor_t* const h = ccv_nnc_tensor_new(0, CPU_TENSOR_NHWC(32F, 4, 2), 0);
340
1
  ccv_nnc_tensor_t* const db = ccv_nnc_tensor_new(0, CPU_TENSOR_NHWC(32F, 2, 3), 0);
341
1
  ccv_nnc_tensor_t* const dbias = ccv_nnc_tensor_new(0, CPU_TENSOR_NHWC(32F, 3), 0);
342
1
  ccv_nnc_tensor_t* gg = ccv_nnc_tensor_new(0, GPU_TENSOR_NHWC(000, 32F, 4, 3), 0);
343
1
  ccv_nnc_tensor_t* ga = ccv_nnc_tensor_new(0, GPU_TENSOR_NHWC(000, 32F, 4, 2), 0);
344
1
  ccv_nnc_tensor_t* gb = ccv_nnc_tensor_new(0, GPU_TENSOR_NHWC(000, 32F, 2, 3), 0);
345
1
  ccv_nnc_tensor_t* gh = ccv_nnc_tensor_new(0, GPU_TENSOR_NHWC(000, 32F, 4, 2), 0);
346
1
  ccv_nnc_tensor_t* gdb = ccv_nnc_tensor_new(0, GPU_TENSOR_NHWC(000, 32F, 2, 3), 0);
347
1
  ccv_nnc_tensor_t* gdbias = ccv_nnc_tensor_new(0, GPU_TENSOR_NHWC(000, 32F, 3), 0);
348
1
  ccv_nnc_cmd_exec(CMD_DATA_TRANSFER_FORWARD(), ccv_nnc_no_hint, 0, TENSOR_LIST(g, a, b), TENSOR_LIST(gg, ga, gb), 0);
349
1
  ccv_nnc_cmd_exec(CMD_GEMM_BACKWARD(), ccv_nnc_no_hint, 0, TENSOR_LIST(gg, ga, gb), TENSOR_LIST(gh, gdb, gdbias), 0);
350
1
  ccv_nnc_cmd_exec(CMD_DATA_TRANSFER_FORWARD(), ccv_nnc_no_hint, 0, TENSOR_LIST(gh, gdb, gdbias), TENSOR_LIST(h, db, dbias), 0);
351
1
  float dbiastp[] = {
352
1
    22, 26, 30,
353
1
  };
354
1
  ccv_nnc_tensor_t dbiast = ccv_nnc_tensor(dbiastp, CPU_TENSOR_NHWC(32F, 3), 0);
355
1
  REQUIRE_TENSOR_EQ(dbias, &dbiast, "bias should be equal");
356
1
  float htp[] = {
357
1
    1 * 21 + 2 * 22 + 3 * 23, 1 * 24 + 2 * 25 + 3 * 26,
358
1
    4 * 21 + 5 * 22 + 6 * 23, 4 * 24 + 5 * 25 + 6 * 26,
359
1
    7 * 21 + 8 * 22 + 9 * 23, 7 * 24 + 8 * 25 + 9 * 26,
360
1
    10 * 21 + 11 * 22 + 12 * 23, 10 * 24 + 11 * 25 + 12 * 26,
361
1
  };
362
1
  ccv_nnc_tensor_t ht = ccv_nnc_tensor(htp, CPU_TENSOR_NHWC(32F, 4, 2), 0);
363
1
  REQUIRE_TENSOR_EQ(h, &ht, "h should be equal");
364
1
  float dbtp[] = {
365
1
    1 * 13 + 4 * 15 + 7 * 17 + 10 * 19, 2 * 13 + 5 * 15 + 8 * 17 + 11 * 19, 3 * 13 + 6 * 15 + 9 * 17 + 12 * 19,
366
1
    1 * 14 + 4 * 16 + 7 * 18 + 10 * 20, 2 * 14 + 5 * 16 + 8 * 18 + 11 * 20, 3 * 14 + 6 * 16 + 9 * 18 + 12 * 20,
367
1
  };
368
1
  ccv_nnc_tensor_t dbt = ccv_nnc_tensor(dbtp, CPU_TENSOR_NHWC(32F, 2, 3), 0);
369
1
  REQUIRE_TENSOR_EQ(db, &dbt, "db should be equal");
370
1
  ccv_nnc_tensor_free(g);
371
1
  ccv_nnc_tensor_free(a);
372
1
  ccv_nnc_tensor_free(b);
373
1
  ccv_nnc_tensor_free(h);
374
1
  ccv_nnc_tensor_free(db);
375
1
  ccv_nnc_tensor_free(dbias);
376
1
  ccv_nnc_tensor_free(gg);
377
1
  ccv_nnc_tensor_free(ga);
378
1
  ccv_nnc_tensor_free(gb);
379
1
  ccv_nnc_tensor_free(gh);
380
1
  ccv_nnc_tensor_free(gdb);
381
1
  ccv_nnc_tensor_free(gdbias);
382
1
}
383
384
TEST_CASE("backward gemm with no transpose and palettize weights")
385
1
{
386
1
  GUARD_ELSE_RETURN(ccv_nnc_cmd_ok(CCV_NNC_GEMM_FORWARD, CCV_NNC_BACKEND_GPU_CUBLAS) &&
387
1
    ccv_nnc_cmd_ok(CCV_NNC_GEMM_BACKWARD, CCV_NNC_BACKEND_GPU_CUBLAS));
388
1
  float gp[] = {
389
1
    1, 2, 3,
390
1
    4, 5, 6,
391
1
    7, 8, 9,
392
1
    10, 11, 12,
393
1
  };
394
1
  ccv_nnc_tensor_t* const g = ccv_nnc_tensor_new(gp, CPU_TENSOR_NHWC(32F, 4, 3), 0);
395
1
  float ap[] = {
396
1
    13, 14,
397
1
    15, 16,
398
1
    17, 18,
399
1
    19, 20,
400
1
  };
401
1
  ccv_nnc_tensor_t* const a = ccv_nnc_tensor_new(ap, CPU_TENSOR_NHWC(32F, 4, 2), 0);
402
1
  float bp[] = {
403
1
    21, 22, 23,
404
1
    24, 25, 26,
405
1
  };
406
1
  ccv_nnc_tensor_t* const b = ccv_nnc_tensor_new(bp, CPU_TENSOR_NHWC(32F, 2, 3), 0);
407
1
  ccv_nnc_tensor_t* const pb = ccv_nnc_tensor_new(0, ccv_nnc_tensor_palettize(CPU_TENSOR_NHWC(32F, 2, 3), 4, 128), 0);
408
1
  (void)ccv_nnc_palettize(b->data.u8, CCV_32F, CCV_TENSOR_CPU_MEMORY, 6, 4, 128, pb->data.u8, ccv_nnc_tensor_data_size_without_padding(pb->info));
409
1
  ccv_nnc_tensor_t* const h = ccv_nnc_tensor_new(0, CPU_TENSOR_NHWC(32F, 4, 2), 0);
410
1
  ccv_nnc_tensor_t* const db = ccv_nnc_tensor_new(0, CPU_TENSOR_NHWC(32F, 2, 3), 0);
411
1
  ccv_nnc_tensor_t* const dbias = ccv_nnc_tensor_new(0, CPU_TENSOR_NHWC(32F, 3), 0);
412
1
  ccv_nnc_tensor_t* gg = ccv_nnc_tensor_new(0, GPU_TENSOR_NHWC(000, 32F, 4, 3), 0);
413
1
  ccv_nnc_tensor_t* ga = ccv_nnc_tensor_new(0, GPU_TENSOR_NHWC(000, 32F, 4, 2), 0);
414
1
  ccv_nnc_tensor_t* gb = ccv_nnc_tensor_new(0, ccv_nnc_tensor_palettize(GPU_TENSOR_NHWC(000, 32F, 2, 3), 4, 128), 0);
415
1
  ccv_nnc_tensor_t* gh = ccv_nnc_tensor_new(0, GPU_TENSOR_NHWC(000, 32F, 4, 2), 0);
416
1
  ccv_nnc_tensor_t* gdb = ccv_nnc_tensor_new(0, GPU_TENSOR_NHWC(000, 32F, 2, 3), 0);
417
1
  ccv_nnc_tensor_t* gdbias = ccv_nnc_tensor_new(0, GPU_TENSOR_NHWC(000, 32F, 3), 0);
418
1
  ccv_nnc_cmd_exec(CMD_DATA_TRANSFER_FORWARD(), ccv_nnc_no_hint, 0, TENSOR_LIST(g, a, pb), TENSOR_LIST(gg, ga, gb), 0);
419
1
  ccv_nnc_cmd_exec(CMD_GEMM_BACKWARD(), ccv_nnc_no_hint, 0, TENSOR_LIST(gg, ga, gb), TENSOR_LIST(gh, gdb, gdbias), 0);
420
1
  ccv_nnc_cmd_exec(CMD_DATA_TRANSFER_FORWARD(), ccv_nnc_no_hint, 0, TENSOR_LIST(gh, gdb, gdbias), TENSOR_LIST(h, db, dbias), 0);
421
1
  float dbiastp[] = {
422
1
    22, 26, 30,
423
1
  };
424
1
  ccv_nnc_tensor_t dbiast = ccv_nnc_tensor(dbiastp, CPU_TENSOR_NHWC(32F, 3), 0);
425
1
  REQUIRE_TENSOR_EQ(dbias, &dbiast, "bias should be equal");
426
1
  float htp[] = {
427
1
    1 * 21 + 2 * 22 + 3 * 23, 1 * 24 + 2 * 25 + 3 * 26,
428
1
    4 * 21 + 5 * 22 + 6 * 23, 4 * 24 + 5 * 25 + 6 * 26,
429
1
    7 * 21 + 8 * 22 + 9 * 23, 7 * 24 + 8 * 25 + 9 * 26,
430
1
    10 * 21 + 11 * 22 + 12 * 23, 10 * 24 + 11 * 25 + 12 * 26,
431
1
  };
432
1
  ccv_nnc_tensor_t ht = ccv_nnc_tensor(htp, CPU_TENSOR_NHWC(32F, 4, 2), 0);
433
1
  REQUIRE_TENSOR_EQ(h, &ht, "h should be equal");
434
1
  float dbtp[] = {
435
1
    1 * 13 + 4 * 15 + 7 * 17 + 10 * 19, 2 * 13 + 5 * 15 + 8 * 17 + 11 * 19, 3 * 13 + 6 * 15 + 9 * 17 + 12 * 19,
436
1
    1 * 14 + 4 * 16 + 7 * 18 + 10 * 20, 2 * 14 + 5 * 16 + 8 * 18 + 11 * 20, 3 * 14 + 6 * 16 + 9 * 18 + 12 * 20,
437
1
  };
438
1
  ccv_nnc_tensor_t dbt = ccv_nnc_tensor(dbtp, CPU_TENSOR_NHWC(32F, 2, 3), 0);
439
1
  REQUIRE_TENSOR_EQ(db, &dbt, "db should be equal");
440
1
  ccv_nnc_tensor_free(g);
441
1
  ccv_nnc_tensor_free(a);
442
1
  ccv_nnc_tensor_free(b);
443
1
  ccv_nnc_tensor_free(pb);
444
1
  ccv_nnc_tensor_free(h);
445
1
  ccv_nnc_tensor_free(db);
446
1
  ccv_nnc_tensor_free(dbias);
447
1
  ccv_nnc_tensor_free(gg);
448
1
  ccv_nnc_tensor_free(ga);
449
1
  ccv_nnc_tensor_free(gb);
450
1
  ccv_nnc_tensor_free(gh);
451
1
  ccv_nnc_tensor_free(gdb);
452
1
  ccv_nnc_tensor_free(gdbias);
453
1
}
454
455
TEST_CASE("backward gemm with no transpose and row-wise int8 weights")
456
1
{
457
1
  GUARD_ELSE_RETURN(ccv_nnc_cmd_ok(CCV_NNC_GEMM_FORWARD, CCV_NNC_BACKEND_GPU_CUBLAS) &&
458
1
    ccv_nnc_cmd_ok(CCV_NNC_GEMM_BACKWARD, CCV_NNC_BACKEND_GPU_CUBLAS));
459
1
  float gp[] = {
460
1
    1, 2, 3,
461
1
    4, 5, 6,
462
1
    7, 8, 9,
463
1
    10, 11, 12,
464
1
  };
465
1
  ccv_nnc_tensor_t* const g = ccv_nnc_tensor_new(gp, CPU_TENSOR_NHWC(32F, 4, 3), 0);
466
1
  float ap[] = {
467
1
    13, 14,
468
1
    15, 16,
469
1
    17, 18,
470
1
    19, 20,
471
1
  };
472
1
  ccv_nnc_tensor_t* const a = ccv_nnc_tensor_new(ap, CPU_TENSOR_NHWC(32F, 4, 2), 0);
473
1
  float bp[] = {
474
1
    21, 22, 23,
475
1
    24, 25, 26,
476
1
  };
477
1
  ccv_nnc_tensor_t* const b = ccv_nnc_tensor_new(bp, CPU_TENSOR_NHWC(32F, 2, 3), 0);
478
1
  ccv_nnc_tensor_t* const qb = ccv_nnc_tensor_new(0, ccv_nnc_tensor_8i_rowwise(CPU_TENSOR_NHWC(32F, 2, 3)), 0);
479
1
  const size_t qsize = ccv_nnc_quantize_8i_rowwise(b->data.u8, CCV_32F, CCV_TENSOR_CPU_MEMORY, 6, 3, qb->data.u8, ccv_nnc_tensor_data_size_without_padding(qb->info));
480
1
  ccv_nnc_tensor_t* const dq_b = ccv_nnc_tensor_new(0, CPU_TENSOR_NHWC(32F, 2, 3), 0);
481
1
  ccv_nnc_dequantize_8i_rowwise(qb->data.u8, CCV_32F, CCV_TENSOR_CPU_MEMORY, qsize, 3, dq_b->data.u8, 6);
482
1
  ccv_nnc_tensor_t* const ht = ccv_nnc_tensor_new(0, CPU_TENSOR_NHWC(32F, 4, 2), 0);
483
1
  ccv_nnc_tensor_t* const dbt = ccv_nnc_tensor_new(0, CPU_TENSOR_NHWC(32F, 2, 3), 0);
484
1
  ccv_nnc_tensor_t* const dbiast = ccv_nnc_tensor_new(0, CPU_TENSOR_NHWC(32F, 3), 0);
485
1
  ccv_nnc_cmd_t cmd = CMD_GEMM_BACKWARD();
486
1
  cmd.backend = CCV_NNC_BACKEND_CPU_REF;
487
1
  assert(cmd.backend >= 0);
488
1
  ccv_nnc_cmd_exec(cmd, ccv_nnc_no_hint, 0, TENSOR_LIST(g, a, dq_b), TENSOR_LIST(ht, dbt, dbiast), 0);
489
1
  ccv_nnc_tensor_t* const h = ccv_nnc_tensor_new(0, CPU_TENSOR_NHWC(32F, 4, 2), 0);
490
1
  ccv_nnc_tensor_t* const db = ccv_nnc_tensor_new(0, CPU_TENSOR_NHWC(32F, 2, 3), 0);
491
1
  ccv_nnc_tensor_t* const dbias = ccv_nnc_tensor_new(0, CPU_TENSOR_NHWC(32F, 3), 0);
492
1
  ccv_nnc_tensor_t* gg = ccv_nnc_tensor_new(0, GPU_TENSOR_NHWC(000, 32F, 4, 3), 0);
493
1
  ccv_nnc_tensor_t* ga = ccv_nnc_tensor_new(0, GPU_TENSOR_NHWC(000, 32F, 4, 2), 0);
494
1
  ccv_nnc_tensor_t* gqb = ccv_nnc_tensor_new(0, ccv_nnc_tensor_8i_rowwise(GPU_TENSOR_NHWC(000, 32F, 2, 3)), 0);
495
1
  ccv_nnc_tensor_t* gh = ccv_nnc_tensor_new(0, GPU_TENSOR_NHWC(000, 32F, 4, 2), 0);
496
1
  ccv_nnc_tensor_t* gdb = ccv_nnc_tensor_new(0, GPU_TENSOR_NHWC(000, 32F, 2, 3), 0);
497
1
  ccv_nnc_tensor_t* gdbias = ccv_nnc_tensor_new(0, GPU_TENSOR_NHWC(000, 32F, 3), 0);
498
1
  ccv_nnc_cmd_exec(CMD_DATA_TRANSFER_FORWARD(), ccv_nnc_no_hint, 0, TENSOR_LIST(g, a, qb), TENSOR_LIST(gg, ga, gqb), 0);
499
1
  ccv_nnc_cmd_exec(CMD_GEMM_BACKWARD(), ccv_nnc_no_hint, 0, TENSOR_LIST(gg, ga, gqb), TENSOR_LIST(gh, gdb, gdbias), 0);
500
1
  ccv_nnc_cmd_exec(CMD_DATA_TRANSFER_FORWARD(), ccv_nnc_no_hint, 0, TENSOR_LIST(gh, gdb, gdbias), TENSOR_LIST(h, db, dbias), 0);
501
1
  REQUIRE_ARRAY_EQ_WITH_TOLERANCE(float, ht->data.f32, h->data.f32, 8, 1e-5, "h should match CPU with dequantized row-wise weights");
502
1
  REQUIRE_ARRAY_EQ_WITH_TOLERANCE(float, dbt->data.f32, db->data.f32, 6, 1e-5, "db should match CPU with dequantized row-wise weights");
503
1
  REQUIRE_ARRAY_EQ_WITH_TOLERANCE(float, dbiast->data.f32, dbias->data.f32, 3, 1e-5, "dbias should match CPU with dequantized row-wise weights");
504
1
  ccv_nnc_tensor_free(g);
505
1
  ccv_nnc_tensor_free(a);
506
1
  ccv_nnc_tensor_free(b);
507
1
  ccv_nnc_tensor_free(qb);
508
1
  ccv_nnc_tensor_free(dq_b);
509
1
  ccv_nnc_tensor_free(ht);
510
1
  ccv_nnc_tensor_free(dbt);
511
1
  ccv_nnc_tensor_free(dbiast);
512
1
  ccv_nnc_tensor_free(h);
513
1
  ccv_nnc_tensor_free(db);
514
1
  ccv_nnc_tensor_free(dbias);
515
1
  ccv_nnc_tensor_free(gg);
516
1
  ccv_nnc_tensor_free(ga);
517
1
  ccv_nnc_tensor_free(gqb);
518
1
  ccv_nnc_tensor_free(gh);
519
1
  ccv_nnc_tensor_free(gdb);
520
1
  ccv_nnc_tensor_free(gdbias);
521
1
}
522
523
TEST_CASE("backward gemm with transpose a")
524
1
{
525
1
  GUARD_ELSE_RETURN(ccv_nnc_cmd_ok(CCV_NNC_GEMM_FORWARD, CCV_NNC_BACKEND_GPU_CUBLAS) &&
526
1
    ccv_nnc_cmd_ok(CCV_NNC_GEMM_BACKWARD, CCV_NNC_BACKEND_GPU_CUBLAS));
527
1
  float gp[] = {
528
1
    1, 2, 3,
529
1
    4, 5, 6,
530
1
    7, 8, 9,
531
1
    10, 11, 12,
532
1
  };
533
1
  ccv_nnc_tensor_t* const g = ccv_nnc_tensor_new(gp, CPU_TENSOR_NHWC(32F, 4, 3), 0);
534
1
  float ap[] = {
535
1
    13, 15, 17, 19,
536
1
    14, 16, 18, 20,
537
1
  };
538
1
  ccv_nnc_tensor_t* const a = ccv_nnc_tensor_new(ap, CPU_TENSOR_NHWC(32F, 2, 4), 0);
539
1
  float bp[] = {
540
1
    21, 22, 23,
541
1
    24, 25, 26,
542
1
  };
543
1
  ccv_nnc_tensor_t* const b = ccv_nnc_tensor_new(bp, CPU_TENSOR_NHWC(32F, 2, 3), 0);
544
1
  ccv_nnc_tensor_t* const h = ccv_nnc_tensor_new(0, CPU_TENSOR_NHWC(32F, 2, 4), 0);
545
1
  ccv_nnc_tensor_t* const db = ccv_nnc_tensor_new(0, CPU_TENSOR_NHWC(32F, 2, 3), 0);
546
1
  ccv_nnc_tensor_t* const dbias = ccv_nnc_tensor_new(0, CPU_TENSOR_NHWC(32F, 3), 0);
547
1
  ccv_nnc_tensor_t* gg = ccv_nnc_tensor_new(0, GPU_TENSOR_NHWC(000, 32F, 4, 3), 0);
548
1
  ccv_nnc_tensor_t* ga = ccv_nnc_tensor_new(0, GPU_TENSOR_NHWC(000, 32F, 2, 4), 0);
549
1
  ccv_nnc_tensor_t* gb = ccv_nnc_tensor_new(0, GPU_TENSOR_NHWC(000, 32F, 2, 3), 0);
550
1
  ccv_nnc_tensor_t* gh = ccv_nnc_tensor_new(0, GPU_TENSOR_NHWC(000, 32F, 2, 4), 0);
551
1
  ccv_nnc_tensor_t* gdb = ccv_nnc_tensor_new(0, GPU_TENSOR_NHWC(000, 32F, 2, 3), 0);
552
1
  ccv_nnc_tensor_t* gdbias = ccv_nnc_tensor_new(0, GPU_TENSOR_NHWC(000, 32F, 3), 0);
553
1
  ccv_nnc_cmd_exec(CMD_DATA_TRANSFER_FORWARD(), ccv_nnc_no_hint, 0, TENSOR_LIST(g, a, b), TENSOR_LIST(gg, ga, gb), 0);
554
1
  ccv_nnc_cmd_exec(CMD_GEMM_BACKWARD(TRANSPOSE(0, 1)), ccv_nnc_no_hint, 0, TENSOR_LIST(gg, ga, gb), TENSOR_LIST(gh, gdb, gdbias), 0);
555
1
  ccv_nnc_cmd_exec(CMD_DATA_TRANSFER_FORWARD(), ccv_nnc_no_hint, 0, TENSOR_LIST(gh, gdb, gdbias), TENSOR_LIST(h, db, dbias), 0);
556
1
  float dbiastp[] = {
557
1
    22, 26, 30,
558
1
  };
559
1
  ccv_nnc_tensor_t dbiast = ccv_nnc_tensor(dbiastp, CPU_TENSOR_NHWC(32F, 3), 0);
560
1
  REQUIRE_TENSOR_EQ(dbias, &dbiast, "bias should be equal");
561
1
  float htp[] = {
562
1
    1 * 21 + 2 * 22 + 3 * 23, 4 * 21 + 5 * 22 + 6 * 23, 7 * 21 + 8 * 22 + 9 * 23, 10 * 21 + 11 * 22 + 12 * 23,
563
1
    1 * 24 + 2 * 25 + 3 * 26, 4 * 24 + 5 * 25 + 6 * 26, 7 * 24 + 8 * 25 + 9 * 26, 10 * 24 + 11 * 25 + 12 * 26,
564
1
  };
565
1
  ccv_nnc_tensor_t ht = ccv_nnc_tensor(htp, CPU_TENSOR_NHWC(32F, 2, 4), 0);
566
1
  REQUIRE_TENSOR_EQ(h, &ht, "h should be equal");
567
1
  float dbtp[] = {
568
1
    1 * 13 + 4 * 15 + 7 * 17 + 10 * 19, 2 * 13 + 5 * 15 + 8 * 17 + 11 * 19, 3 * 13 + 6 * 15 + 9 * 17 + 12 * 19,
569
1
    1 * 14 + 4 * 16 + 7 * 18 + 10 * 20, 2 * 14 + 5 * 16 + 8 * 18 + 11 * 20, 3 * 14 + 6 * 16 + 9 * 18 + 12 * 20,
570
1
  };
571
1
  ccv_nnc_tensor_t dbt = ccv_nnc_tensor(dbtp, CPU_TENSOR_NHWC(32F, 2, 3), 0);
572
1
  REQUIRE_TENSOR_EQ(db, &dbt, "db should be equal");
573
1
  ccv_nnc_tensor_free(g);
574
1
  ccv_nnc_tensor_free(a);
575
1
  ccv_nnc_tensor_free(b);
576
1
  ccv_nnc_tensor_free(h);
577
1
  ccv_nnc_tensor_free(db);
578
1
  ccv_nnc_tensor_free(dbias);
579
1
  ccv_nnc_tensor_free(gg);
580
1
  ccv_nnc_tensor_free(ga);
581
1
  ccv_nnc_tensor_free(gb);
582
1
  ccv_nnc_tensor_free(gh);
583
1
  ccv_nnc_tensor_free(gdb);
584
1
  ccv_nnc_tensor_free(gdbias);
585
1
}
586
587
TEST_CASE("backward gemm with transpose b")
588
1
{
589
1
  GUARD_ELSE_RETURN(ccv_nnc_cmd_ok(CCV_NNC_GEMM_FORWARD, CCV_NNC_BACKEND_GPU_CUBLAS) &&
590
1
    ccv_nnc_cmd_ok(CCV_NNC_GEMM_BACKWARD, CCV_NNC_BACKEND_GPU_CUBLAS));
591
1
  float gp[] = {
592
1
    1, 2, 3,
593
1
    4, 5, 6,
594
1
    7, 8, 9,
595
1
    10, 11, 12,
596
1
  };
597
1
  ccv_nnc_tensor_t* const g = ccv_nnc_tensor_new(gp, CPU_TENSOR_NHWC(32F, 4, 3), 0);
598
1
  float ap[] = {
599
1
    13, 14,
600
1
    15, 16,
601
1
    17, 18,
602
1
    19, 20,
603
1
  };
604
1
  ccv_nnc_tensor_t* const a = ccv_nnc_tensor_new(ap, CPU_TENSOR_NHWC(32F, 4, 2), 0);
605
1
  float bp[] = {
606
1
    21, 24,
607
1
    22, 25,
608
1
    23, 26,
609
1
  };
610
1
  ccv_nnc_tensor_t* const b = ccv_nnc_tensor_new(bp, CPU_TENSOR_NHWC(32F, 3, 2), 0);
611
1
  ccv_nnc_tensor_t* const h = ccv_nnc_tensor_new(0, CPU_TENSOR_NHWC(32F, 4, 2), 0);
612
1
  ccv_nnc_tensor_t* const db = ccv_nnc_tensor_new(0, CPU_TENSOR_NHWC(32F, 3, 2), 0);
613
1
  ccv_nnc_tensor_t* const dbias = ccv_nnc_tensor_new(0, CPU_TENSOR_NHWC(32F, 3), 0);
614
1
  ccv_nnc_tensor_t* gg = ccv_nnc_tensor_new(0, GPU_TENSOR_NHWC(000, 32F, 4, 3), 0);
615
1
  ccv_nnc_tensor_t* ga = ccv_nnc_tensor_new(0, GPU_TENSOR_NHWC(000, 32F, 4, 2), 0);
616
1
  ccv_nnc_tensor_t* gb = ccv_nnc_tensor_new(0, GPU_TENSOR_NHWC(000, 32F, 3, 2), 0);
617
1
  ccv_nnc_tensor_t* gh = ccv_nnc_tensor_new(0, GPU_TENSOR_NHWC(000, 32F, 4, 2), 0);
618
1
  ccv_nnc_tensor_t* gdb = ccv_nnc_tensor_new(0, GPU_TENSOR_NHWC(000, 32F, 3, 2), 0);
619
1
  ccv_nnc_tensor_t* gdbias = ccv_nnc_tensor_new(0, GPU_TENSOR_NHWC(000, 32F, 3), 0);
620
1
  ccv_nnc_cmd_exec(CMD_DATA_TRANSFER_FORWARD(), ccv_nnc_no_hint, 0, TENSOR_LIST(g, a, b), TENSOR_LIST(gg, ga, gb), 0);
621
1
  ccv_nnc_cmd_exec(CMD_GEMM_BACKWARD(NO_TRANSPOSE, TRANSPOSE(0, 1)), ccv_nnc_no_hint, 0, TENSOR_LIST(gg, ga, gb), TENSOR_LIST(gh, gdb, gdbias), 0);
622
1
  ccv_nnc_cmd_exec(CMD_DATA_TRANSFER_FORWARD(), ccv_nnc_no_hint, 0, TENSOR_LIST(gh, gdb, gdbias), TENSOR_LIST(h, db, dbias), 0);
623
1
  float dbiastp[] = {
624
1
    22, 26, 30,
625
1
  };
626
1
  ccv_nnc_tensor_t dbiast = ccv_nnc_tensor(dbiastp, CPU_TENSOR_NHWC(32F, 3), 0);
627
1
  REQUIRE_TENSOR_EQ(dbias, &dbiast, "bias should be equal");
628
1
  float htp[] = {
629
1
    1 * 21 + 2 * 22 + 3 * 23, 1 * 24 + 2 * 25 + 3 * 26,
630
1
    4 * 21 + 5 * 22 + 6 * 23, 4 * 24 + 5 * 25 + 6 * 26,
631
1
    7 * 21 + 8 * 22 + 9 * 23, 7 * 24 + 8 * 25 + 9 * 26,
632
1
    10 * 21 + 11 * 22 + 12 * 23, 10 * 24 + 11 * 25 + 12 * 26,
633
1
  };
634
1
  ccv_nnc_tensor_t ht = ccv_nnc_tensor(htp, CPU_TENSOR_NHWC(32F, 4, 2), 0);
635
1
  REQUIRE_TENSOR_EQ(h, &ht, "h should be equal");
636
1
  float dbtp[] = {
637
1
    1 * 13 + 4 * 15 + 7 * 17 + 10 * 19, 1 * 14 + 4 * 16 + 7 * 18 + 10 * 20,
638
1
    2 * 13 + 5 * 15 + 8 * 17 + 11 * 19, 2 * 14 + 5 * 16 + 8 * 18 + 11 * 20,
639
1
    3 * 13 + 6 * 15 + 9 * 17 + 12 * 19, 3 * 14 + 6 * 16 + 9 * 18 + 12 * 20,
640
1
  };
641
1
  ccv_nnc_tensor_t dbt = ccv_nnc_tensor(dbtp, CPU_TENSOR_NHWC(32F, 3, 2), 0);
642
1
  REQUIRE_TENSOR_EQ(db, &dbt, "db should be equal");
643
1
  ccv_nnc_tensor_free(g);
644
1
  ccv_nnc_tensor_free(a);
645
1
  ccv_nnc_tensor_free(b);
646
1
  ccv_nnc_tensor_free(h);
647
1
  ccv_nnc_tensor_free(db);
648
1
  ccv_nnc_tensor_free(dbias);
649
1
  ccv_nnc_tensor_free(gg);
650
1
  ccv_nnc_tensor_free(ga);
651
1
  ccv_nnc_tensor_free(gb);
652
1
  ccv_nnc_tensor_free(gh);
653
1
  ccv_nnc_tensor_free(gdb);
654
1
  ccv_nnc_tensor_free(gdbias);
655
1
}
656
657
TEST_CASE("backward gemm with transpose a and b")
658
1
{
659
1
  GUARD_ELSE_RETURN(ccv_nnc_cmd_ok(CCV_NNC_GEMM_FORWARD, CCV_NNC_BACKEND_GPU_CUBLAS) &&
660
1
    ccv_nnc_cmd_ok(CCV_NNC_GEMM_BACKWARD, CCV_NNC_BACKEND_GPU_CUBLAS));
661
1
  float gp[] = {
662
1
    1, 2, 3,
663
1
    4, 5, 6,
664
1
    7, 8, 9,
665
1
    10, 11, 12,
666
1
  };
667
1
  ccv_nnc_tensor_t* const g = ccv_nnc_tensor_new(gp, CPU_TENSOR_NHWC(32F, 4, 3), 0);
668
1
  float ap[] = {
669
1
    13, 15, 17, 19,
670
1
    14, 16, 18, 20,
671
1
  };
672
1
  ccv_nnc_tensor_t* const a = ccv_nnc_tensor_new(ap, CPU_TENSOR_NHWC(32F, 2, 4), 0);
673
1
  float bp[] = {
674
1
    21, 24,
675
1
    22, 25,
676
1
    23, 26,
677
1
  };
678
1
  ccv_nnc_tensor_t* const b = ccv_nnc_tensor_new(bp, CPU_TENSOR_NHWC(32F, 3, 2), 0);
679
1
  ccv_nnc_tensor_t* const h = ccv_nnc_tensor_new(0, CPU_TENSOR_NHWC(32F, 2, 4), 0);
680
1
  ccv_nnc_tensor_t* const db = ccv_nnc_tensor_new(0, CPU_TENSOR_NHWC(32F, 3, 2), 0);
681
1
  ccv_nnc_tensor_t* const dbias = ccv_nnc_tensor_new(0, CPU_TENSOR_NHWC(32F, 3), 0);
682
1
  ccv_nnc_tensor_t* gg = ccv_nnc_tensor_new(0, GPU_TENSOR_NHWC(000, 32F, 4, 3), 0);
683
1
  ccv_nnc_tensor_t* ga = ccv_nnc_tensor_new(0, GPU_TENSOR_NHWC(000, 32F, 2, 4), 0);
684
1
  ccv_nnc_tensor_t* gb = ccv_nnc_tensor_new(0, GPU_TENSOR_NHWC(000, 32F, 3, 2), 0);
685
1
  ccv_nnc_tensor_t* gh = ccv_nnc_tensor_new(0, GPU_TENSOR_NHWC(000, 32F, 2, 4), 0);
686
1
  ccv_nnc_tensor_t* gdb = ccv_nnc_tensor_new(0, GPU_TENSOR_NHWC(000, 32F, 3, 2), 0);
687
1
  ccv_nnc_tensor_t* gdbias = ccv_nnc_tensor_new(0, GPU_TENSOR_NHWC(000, 32F, 3), 0);
688
1
  ccv_nnc_cmd_exec(CMD_DATA_TRANSFER_FORWARD(), ccv_nnc_no_hint, 0, TENSOR_LIST(g, a, b), TENSOR_LIST(gg, ga, gb), 0);
689
1
  ccv_nnc_cmd_exec(CMD_GEMM_BACKWARD(TRANSPOSE(0, 1), TRANSPOSE(0, 1)), ccv_nnc_no_hint, 0, TENSOR_LIST(gg, ga, gb), TENSOR_LIST(gh, gdb, gdbias), 0);
690
1
  ccv_nnc_cmd_exec(CMD_DATA_TRANSFER_FORWARD(), ccv_nnc_no_hint, 0, TENSOR_LIST(gh, gdb, gdbias), TENSOR_LIST(h, db, dbias), 0);
691
1
  float dbiastp[] = {
692
1
    22, 26, 30,
693
1
  };
694
1
  ccv_nnc_tensor_t dbiast = ccv_nnc_tensor(dbiastp, CPU_TENSOR_NHWC(32F, 3), 0);
695
1
  REQUIRE_TENSOR_EQ(dbias, &dbiast, "bias should be equal");
696
1
  float htp[] = {
697
1
    1 * 21 + 2 * 22 + 3 * 23, 4 * 21 + 5 * 22 + 6 * 23, 7 * 21 + 8 * 22 + 9 * 23, 10 * 21 + 11 * 22 + 12 * 23,
698
1
    1 * 24 + 2 * 25 + 3 * 26, 4 * 24 + 5 * 25 + 6 * 26, 7 * 24 + 8 * 25 + 9 * 26, 10 * 24 + 11 * 25 + 12 * 26,
699
1
  };
700
1
  ccv_nnc_tensor_t ht = ccv_nnc_tensor(htp, CPU_TENSOR_NHWC(32F, 2, 4), 0);
701
1
  REQUIRE_TENSOR_EQ(h, &ht, "h should be equal");
702
1
  float dbtp[] = {
703
1
    1 * 13 + 4 * 15 + 7 * 17 + 10 * 19, 1 * 14 + 4 * 16 + 7 * 18 + 10 * 20,
704
1
    2 * 13 + 5 * 15 + 8 * 17 + 11 * 19, 2 * 14 + 5 * 16 + 8 * 18 + 11 * 20,
705
1
    3 * 13 + 6 * 15 + 9 * 17 + 12 * 19, 3 * 14 + 6 * 16 + 9 * 18 + 12 * 20,
706
1
  };
707
1
  ccv_nnc_tensor_t dbt = ccv_nnc_tensor(dbtp, CPU_TENSOR_NHWC(32F, 3, 2), 0);
708
1
  REQUIRE_TENSOR_EQ(db, &dbt, "db should be equal");
709
1
  ccv_nnc_tensor_free(g);
710
1
  ccv_nnc_tensor_free(a);
711
1
  ccv_nnc_tensor_free(b);
712
1
  ccv_nnc_tensor_free(h);
713
1
  ccv_nnc_tensor_free(db);
714
1
  ccv_nnc_tensor_free(dbias);
715
1
  ccv_nnc_tensor_free(gg);
716
1
  ccv_nnc_tensor_free(ga);
717
1
  ccv_nnc_tensor_free(gb);
718
1
  ccv_nnc_tensor_free(gh);
719
1
  ccv_nnc_tensor_free(gdb);
720
1
  ccv_nnc_tensor_free(gdbias);
721
1
}
722
723
TEST_CASE("gemm no transpose batch 2")
724
1
{
725
1
  GUARD_ELSE_RETURN(ccv_nnc_cmd_ok(CCV_NNC_GEMM_FORWARD, CCV_NNC_BACKEND_GPU_CUBLAS));
726
1
  float ap[] = {
727
1
    1, 2,
728
1
    3, 4,
729
1
    5, 6,
730
1
    7, 8,
731
1
    2, 3,
732
1
    4, 5,
733
1
    6, 7,
734
1
    8, 9
735
1
  };
736
1
  ccv_nnc_tensor_t* const a = ccv_nnc_tensor_new(ap, CPU_TENSOR_NHWC(32F, 2, 4, 2), 0);
737
1
  float bp[] = {
738
1
    7, 8, 9,
739
1
    10, 11, 12,
740
1
  };
741
1
  ccv_nnc_tensor_t* const b = ccv_nnc_tensor_new(bp, CPU_TENSOR_NHWC(32F, 2, 3), 0);
742
1
  ccv_nnc_tensor_t* const c = ccv_nnc_tensor_new(0, CPU_TENSOR_NHWC(32F, 2, 4, 3), 0);
743
1
  ccv_nnc_tensor_t* ga = ccv_nnc_tensor_new(0, GPU_TENSOR_NHWC(000, 32F, 2, 4, 2), 0);
744
1
  ccv_nnc_tensor_t* gb = ccv_nnc_tensor_new(0, GPU_TENSOR_NHWC(000, 32F, 2, 3), 0);
745
1
  ccv_nnc_tensor_t* gc = ccv_nnc_tensor_new(0, GPU_TENSOR_NHWC(000, 32F, 2, 4, 3), 0);
746
1
  ccv_nnc_cmd_exec(CMD_DATA_TRANSFER_FORWARD(), ccv_nnc_no_hint, 0, TENSOR_LIST(a, b), TENSOR_LIST(ga, gb), 0);
747
1
  ccv_nnc_cmd_exec(CMD_GEMM_FORWARD(), ccv_nnc_no_hint, 0, TENSOR_LIST(ga, gb), TENSOR_LIST(gc), 0);
748
1
  ccv_nnc_cmd_exec(CMD_DATA_TRANSFER_FORWARD(), ccv_nnc_no_hint, 0, TENSOR_LIST(gc), TENSOR_LIST(c), 0);
749
1
  float ctp[] = {
750
1
    1 * 7 + 2 * 10, 1 * 8 + 2 * 11, 1 * 9 + 2 * 12,
751
1
    3 * 7 + 4 * 10, 3 * 8 + 4 * 11, 3 * 9 + 4 * 12,
752
1
    5 * 7 + 6 * 10, 5 * 8 + 6 * 11, 5 * 9 + 6 * 12,
753
1
    7 * 7 + 8 * 10, 7 * 8 + 8 * 11, 7 * 9 + 8 * 12,
754
1
    2 * 7 + 3 * 10, 2 * 8 + 3 * 11, 2 * 9 + 3 * 12,
755
1
    4 * 7 + 5 * 10, 4 * 8 + 5 * 11, 4 * 9 + 5 * 12,
756
1
    6 * 7 + 7 * 10, 6 * 8 + 7 * 11, 6 * 9 + 7 * 12,
757
1
    8 * 7 + 9 * 10, 8 * 8 + 9 * 11, 8 * 9 + 9 * 12,
758
1
  };
759
1
  ccv_nnc_tensor_t ct = ccv_nnc_tensor(ctp, CPU_TENSOR_NHWC(32F, 2, 4, 3), 0);
760
1
  REQUIRE_TENSOR_EQ(c, &ct, "result should be equal");
761
1
  ccv_nnc_tensor_free(a);
762
1
  ccv_nnc_tensor_free(b);
763
1
  ccv_nnc_tensor_free(c);
764
1
  ccv_nnc_tensor_free(ga);
765
1
  ccv_nnc_tensor_free(gb);
766
1
  ccv_nnc_tensor_free(gc);
767
1
}
768
769
TEST_CASE("gemm transpose a batch 2")
770
1
{
771
1
  GUARD_ELSE_RETURN(ccv_nnc_cmd_ok(CCV_NNC_GEMM_FORWARD, CCV_NNC_BACKEND_GPU_CUBLAS));
772
1
  float ap[] = {
773
1
    1, 3, 5, 7,
774
1
    2, 4, 6, 8,
775
1
    2, 4, 6, 8,
776
1
    3, 5, 7, 9,
777
1
  };
778
1
  ccv_nnc_tensor_t* const a = ccv_nnc_tensor_new(ap, CPU_TENSOR_NHWC(32F, 2, 2, 4), 0);
779
1
  float bp[] = {
780
1
    7, 8, 9,
781
1
    10, 11, 12,
782
1
  };
783
1
  ccv_nnc_tensor_t* const b = ccv_nnc_tensor_new(bp, CPU_TENSOR_NHWC(32F, 2, 3), 0);
784
1
  float dp[] = {
785
1
    -1, 0, 1,
786
1
  };
787
1
  ccv_nnc_tensor_t* const d = ccv_nnc_tensor_new(dp, CPU_TENSOR_NHWC(32F, 3), 0);
788
1
  ccv_nnc_tensor_t* const c = ccv_nnc_tensor_new(0, CPU_TENSOR_NHWC(32F, 2, 4, 3), 0);
789
1
  ccv_nnc_tensor_t* ga = ccv_nnc_tensor_new(0, GPU_TENSOR_NHWC(000, 32F, 2, 2, 4), 0);
790
1
  ccv_nnc_tensor_t* gb = ccv_nnc_tensor_new(0, GPU_TENSOR_NHWC(000, 32F, 2, 3), 0);
791
1
  ccv_nnc_tensor_t* gc = ccv_nnc_tensor_new(0, GPU_TENSOR_NHWC(000, 32F, 2, 4, 3), 0);
792
1
  ccv_nnc_tensor_t* gd = ccv_nnc_tensor_new(0, GPU_TENSOR_NHWC(000, 32F, 3), 0);
793
1
  ccv_nnc_cmd_exec(CMD_DATA_TRANSFER_FORWARD(), ccv_nnc_no_hint, 0, TENSOR_LIST(a, b, d), TENSOR_LIST(ga, gb, gd), 0);
794
1
  ccv_nnc_cmd_exec(CMD_GEMM_FORWARD(TRANSPOSE(1, 2)), ccv_nnc_no_hint, 0, TENSOR_LIST(ga, gb, gd), TENSOR_LIST(gc), 0);
795
1
  ccv_nnc_cmd_exec(CMD_DATA_TRANSFER_FORWARD(), ccv_nnc_no_hint, 0, TENSOR_LIST(gc), TENSOR_LIST(c), 0);
796
1
  float ctp[] = {
797
1
    1 * 7 + 2 * 10 - 1, 1 * 8 + 2 * 11, 1 * 9 + 2 * 12 + 1,
798
1
    3 * 7 + 4 * 10 - 1, 3 * 8 + 4 * 11, 3 * 9 + 4 * 12 + 1,
799
1
    5 * 7 + 6 * 10 - 1, 5 * 8 + 6 * 11, 5 * 9 + 6 * 12 + 1,
800
1
    7 * 7 + 8 * 10 - 1, 7 * 8 + 8 * 11, 7 * 9 + 8 * 12 + 1,
801
1
    2 * 7 + 3 * 10 - 1, 2 * 8 + 3 * 11, 2 * 9 + 3 * 12 + 1,
802
1
    4 * 7 + 5 * 10 - 1, 4 * 8 + 5 * 11, 4 * 9 + 5 * 12 + 1,
803
1
    6 * 7 + 7 * 10 - 1, 6 * 8 + 7 * 11, 6 * 9 + 7 * 12 + 1,
804
1
    8 * 7 + 9 * 10 - 1, 8 * 8 + 9 * 11, 8 * 9 + 9 * 12 + 1,
805
1
  };
806
1
  ccv_nnc_tensor_t ct = ccv_nnc_tensor(ctp, CPU_TENSOR_NHWC(32F, 2, 4, 3), 0);
807
1
  REQUIRE_TENSOR_EQ(c, &ct, "result should be equal");
808
1
  ccv_nnc_tensor_free(a);
809
1
  ccv_nnc_tensor_free(b);
810
1
  ccv_nnc_tensor_free(c);
811
1
  ccv_nnc_tensor_free(d);
812
1
  ccv_nnc_tensor_free(ga);
813
1
  ccv_nnc_tensor_free(gb);
814
1
  ccv_nnc_tensor_free(gc);
815
1
  ccv_nnc_tensor_free(gd);
816
1
}
817
818
TEST_CASE("gemm transpose b batch 2")
819
1
{
820
1
  GUARD_ELSE_RETURN(ccv_nnc_cmd_ok(CCV_NNC_GEMM_FORWARD, CCV_NNC_BACKEND_GPU_CUBLAS));
821
1
  float ap[] = {
822
1
    1, 2,
823
1
    3, 4,
824
1
    5, 6,
825
1
    7, 8,
826
1
    2, 3,
827
1
    4, 5,
828
1
    6, 7,
829
1
    8, 9
830
1
  };
831
1
  ccv_nnc_tensor_t* const a = ccv_nnc_tensor_new(ap, CPU_TENSOR_NHWC(32F, 2, 4, 2), 0);
832
1
  float bp[] = {
833
1
    7, 10,
834
1
    8, 11,
835
1
    9, 12,
836
1
    80, 110,
837
1
    90, 120,
838
1
    10, 13,
839
1
  };
840
1
  ccv_nnc_tensor_t* const b = ccv_nnc_tensor_new(bp, CPU_TENSOR_NHWC(32F, 2, 3, 2), 0);
841
1
  float dp[] = {
842
1
    -1, 0, 1,
843
1
    2, 3, -4,
844
1
  };
845
1
  ccv_nnc_tensor_t* const d = ccv_nnc_tensor_new(dp, CPU_TENSOR_NHWC(32F, 2, 1, 3), 0);
846
1
  ccv_nnc_tensor_t* const c = ccv_nnc_tensor_new(0, CPU_TENSOR_NHWC(32F, 2, 4, 3), 0);
847
1
  ccv_nnc_tensor_t* ga = ccv_nnc_tensor_new(0, GPU_TENSOR_NHWC(000, 32F, 2, 4, 2), 0);
848
1
  ccv_nnc_tensor_t* gb = ccv_nnc_tensor_new(0, GPU_TENSOR_NHWC(000, 32F, 2, 3, 2), 0);
849
1
  ccv_nnc_tensor_t* gc = ccv_nnc_tensor_new(0, GPU_TENSOR_NHWC(000, 32F, 2, 4, 3), 0);
850
1
  ccv_nnc_tensor_t* gd = ccv_nnc_tensor_new(0, GPU_TENSOR_NHWC(000, 32F, 2, 1, 3), 0);
851
1
  ccv_nnc_cmd_exec(CMD_DATA_TRANSFER_FORWARD(), ccv_nnc_no_hint, 0, TENSOR_LIST(a, b, d), TENSOR_LIST(ga, gb, gd), 0);
852
1
  ccv_nnc_cmd_exec(CMD_GEMM_FORWARD(NO_TRANSPOSE, TRANSPOSE(1, 2)), ccv_nnc_no_hint, 0, TENSOR_LIST(ga, gb, gd), TENSOR_LIST(gc), 0);
853
1
  ccv_nnc_cmd_exec(CMD_DATA_TRANSFER_FORWARD(), ccv_nnc_no_hint, 0, TENSOR_LIST(gc), TENSOR_LIST(c), 0);
854
1
  float ctp[] = {
855
1
    1 * 7 + 2 * 10 - 1, 1 * 8 + 2 * 11, 1 * 9 + 2 * 12 + 1,
856
1
    3 * 7 + 4 * 10 - 1, 3 * 8 + 4 * 11, 3 * 9 + 4 * 12 + 1,
857
1
    5 * 7 + 6 * 10 - 1, 5 * 8 + 6 * 11, 5 * 9 + 6 * 12 + 1,
858
1
    7 * 7 + 8 * 10 - 1, 7 * 8 + 8 * 11, 7 * 9 + 8 * 12 + 1,
859
1
    2 * 80 + 3 * 110 + 2, 2 * 90 + 3 * 120 + 3, 2 * 10 + 3 * 13 - 4,
860
1
    4 * 80 + 5 * 110 + 2, 4 * 90 + 5 * 120 + 3, 4 * 10 + 5 * 13 - 4,
861
1
    6 * 80 + 7 * 110 + 2, 6 * 90 + 7 * 120 + 3, 6 * 10 + 7 * 13 - 4,
862
1
    8 * 80 + 9 * 110 + 2, 8 * 90 + 9 * 120 + 3, 8 * 10 + 9 * 13 - 4,
863
1
  };
864
1
  ccv_nnc_tensor_t ct = ccv_nnc_tensor(ctp, CPU_TENSOR_NHWC(32F, 2, 4, 3), 0);
865
1
  REQUIRE_TENSOR_EQ(c, &ct, "result should be equal");
866
1
  ccv_nnc_tensor_free(a);
867
1
  ccv_nnc_tensor_free(b);
868
1
  ccv_nnc_tensor_free(c);
869
1
  ccv_nnc_tensor_free(d);
870
1
  ccv_nnc_tensor_free(ga);
871
1
  ccv_nnc_tensor_free(gb);
872
1
  ccv_nnc_tensor_free(gc);
873
1
  ccv_nnc_tensor_free(gd);
874
1
}
875
876
TEST_CASE("backward gemm with no transpose batch 2, same b")
877
1
{
878
1
  GUARD_ELSE_RETURN(ccv_nnc_cmd_ok(CCV_NNC_GEMM_FORWARD, CCV_NNC_BACKEND_GPU_CUBLAS) &&
879
1
    ccv_nnc_cmd_ok(CCV_NNC_GEMM_BACKWARD, CCV_NNC_BACKEND_GPU_CUBLAS));
880
1
  float gp[] = {
881
1
    1, 2, 3,
882
1
    4, 5, 6,
883
1
    7, 8, 9,
884
1
    10, 11, 12,
885
1
    10, 20, 30,
886
1
    40, 50, 60,
887
1
    70, 80, 90,
888
1
    100, 110, 120,
889
1
  };
890
1
  ccv_nnc_tensor_t* const g = ccv_nnc_tensor_new(gp, CPU_TENSOR_NHWC(32F, 2, 4, 3), 0);
891
1
  float ap[] = {
892
1
    13, 14,
893
1
    15, 16,
894
1
    17, 18,
895
1
    19, 20,
896
1
    131, 141,
897
1
    151, 161,
898
1
    171, 181,
899
1
    191, 201,
900
1
  };
901
1
  ccv_nnc_tensor_t* const a = ccv_nnc_tensor_new(ap, CPU_TENSOR_NHWC(32F, 2, 4, 2), 0);
902
1
  float bp[] = {
903
1
    21, 22, 23,
904
1
    24, 25, 26,
905
1
  };
906
1
  ccv_nnc_tensor_t* const b = ccv_nnc_tensor_new(bp, CPU_TENSOR_NHWC(32F, 2, 3), 0);
907
1
  ccv_nnc_tensor_t* const h = ccv_nnc_tensor_new(0, CPU_TENSOR_NHWC(32F, 2, 4, 2), 0);
908
1
  ccv_nnc_tensor_t* const db = ccv_nnc_tensor_new(0, CPU_TENSOR_NHWC(32F, 2, 3), 0);
909
1
  ccv_nnc_tensor_t* const dbias = ccv_nnc_tensor_new(0, CPU_TENSOR_NHWC(32F, 3), 0);
910
1
  ccv_nnc_tensor_t* gg = ccv_nnc_tensor_new(0, GPU_TENSOR_NHWC(000, 32F, 2, 4, 3), 0);
911
1
  ccv_nnc_tensor_t* ga = ccv_nnc_tensor_new(0, GPU_TENSOR_NHWC(000, 32F, 2, 4, 2), 0);
912
1
  ccv_nnc_tensor_t* gb = ccv_nnc_tensor_new(0, GPU_TENSOR_NHWC(000, 32F, 2, 3), 0);
913
1
  ccv_nnc_tensor_t* gh = ccv_nnc_tensor_new(0, GPU_TENSOR_NHWC(000, 32F, 2, 4, 2), 0);
914
1
  ccv_nnc_tensor_t* gdb = ccv_nnc_tensor_new(0, GPU_TENSOR_NHWC(000, 32F, 2, 3), 0);
915
1
  ccv_nnc_tensor_t* gdbias = ccv_nnc_tensor_new(0, GPU_TENSOR_NHWC(000, 32F, 3), 0);
916
1
  ccv_nnc_cmd_exec(CMD_DATA_TRANSFER_FORWARD(), ccv_nnc_no_hint, 0, TENSOR_LIST(g, a, b), TENSOR_LIST(gg, ga, gb), 0);
917
1
  ccv_nnc_cmd_exec(CMD_GEMM_BACKWARD(), ccv_nnc_no_hint, 0, TENSOR_LIST(gg, ga, gb), TENSOR_LIST(gh, gdb, gdbias), 0);
918
1
  ccv_nnc_cmd_exec(CMD_DATA_TRANSFER_FORWARD(), ccv_nnc_no_hint, 0, TENSOR_LIST(gh, gdb, gdbias), TENSOR_LIST(h, db, dbias), 0);
919
1
  float dbiastp[] = {
920
1
    22 + 220, 26 + 260, 30 + 300,
921
1
  };
922
1
  ccv_nnc_tensor_t dbiast = ccv_nnc_tensor(dbiastp, CPU_TENSOR_NHWC(32F, 3), 0);
923
1
  REQUIRE_TENSOR_EQ(dbias, &dbiast, "bias should be equal");
924
1
  float htp[] = {
925
1
    1 * 21 + 2 * 22 + 3 * 23, 1 * 24 + 2 * 25 + 3 * 26,
926
1
    4 * 21 + 5 * 22 + 6 * 23, 4 * 24 + 5 * 25 + 6 * 26,
927
1
    7 * 21 + 8 * 22 + 9 * 23, 7 * 24 + 8 * 25 + 9 * 26,
928
1
    10 * 21 + 11 * 22 + 12 * 23, 10 * 24 + 11 * 25 + 12 * 26,
929
1
    10 * 21 + 20 * 22 + 30 * 23, 10 * 24 + 20 * 25 + 30 * 26,
930
1
    40 * 21 + 50 * 22 + 60 * 23, 40 * 24 + 50 * 25 + 60 * 26,
931
1
    70 * 21 + 80 * 22 + 90 * 23, 70 * 24 + 80 * 25 + 90 * 26,
932
1
    100 * 21 + 110 * 22 + 120 * 23, 100 * 24 + 110 * 25 + 120 * 26,
933
1
  };
934
1
  ccv_nnc_tensor_t ht = ccv_nnc_tensor(htp, CPU_TENSOR_NHWC(32F, 2, 4, 2), 0);
935
1
  REQUIRE_TENSOR_EQ(h, &ht, "h should be equal");
936
1
  float dbtp[] = {
937
1
    1 * 13 + 4 * 15 + 7 * 17 + 10 * 19 + 10 * 131 + 40 * 151 + 70 * 171 + 100 * 191, 2 * 13 + 5 * 15 + 8 * 17 + 11 * 19 + 20 * 131 + 50 * 151 + 80 * 171 + 110 * 191, 3 * 13 + 6 * 15 + 9 * 17 + 12 * 19 + 30 * 131 + 60 * 151 + 90 * 171 + 120 * 191,
938
1
    1 * 14 + 4 * 16 + 7 * 18 + 10 * 20 + 10 * 141 + 40 * 161 + 70 * 181 + 100 * 201, 2 * 14 + 5 * 16 + 8 * 18 + 11 * 20 + 20 * 141 + 50 * 161 + 80 * 181 + 110 * 201, 3 * 14 + 6 * 16 + 9 * 18 + 12 * 20 + 30 * 141 + 60 * 161 + 90 * 181 + 120 * 201,
939
1
  };
940
1
  ccv_nnc_tensor_t dbt = ccv_nnc_tensor(dbtp, CPU_TENSOR_NHWC(32F, 2, 3), 0);
941
1
  REQUIRE_TENSOR_EQ(db, &dbt, "db should be equal");
942
1
  ccv_nnc_tensor_free(g);
943
1
  ccv_nnc_tensor_free(a);
944
1
  ccv_nnc_tensor_free(b);
945
1
  ccv_nnc_tensor_free(h);
946
1
  ccv_nnc_tensor_free(db);
947
1
  ccv_nnc_tensor_free(dbias);
948
1
  ccv_nnc_tensor_free(gg);
949
1
  ccv_nnc_tensor_free(ga);
950
1
  ccv_nnc_tensor_free(gb);
951
1
  ccv_nnc_tensor_free(gh);
952
1
  ccv_nnc_tensor_free(gdb);
953
1
  ccv_nnc_tensor_free(gdbias);
954
1
}
955
956
TEST_CASE("backward gemm with no transpose batch 2, batched b")
957
1
{
958
1
  GUARD_ELSE_RETURN(ccv_nnc_cmd_ok(CCV_NNC_GEMM_FORWARD, CCV_NNC_BACKEND_GPU_CUBLAS) &&
959
1
    ccv_nnc_cmd_ok(CCV_NNC_GEMM_BACKWARD, CCV_NNC_BACKEND_GPU_CUBLAS));
960
1
  float gp[] = {
961
1
    1, 2, 3,
962
1
    4, 5, 6,
963
1
    7, 8, 9,
964
1
    10, 11, 12,
965
1
    10, 20, 30,
966
1
    40, 50, 60,
967
1
    70, 80, 90,
968
1
    100, 110, 120,
969
1
  };
970
1
  ccv_nnc_tensor_t* const g = ccv_nnc_tensor_new(gp, CPU_TENSOR_NHWC(32F, 2, 4, 3), 0);
971
1
  float ap[] = {
972
1
    13, 14,
973
1
    15, 16,
974
1
    17, 18,
975
1
    19, 20,
976
1
    131, 141,
977
1
    151, 161,
978
1
    171, 181,
979
1
    191, 201,
980
1
  };
981
1
  ccv_nnc_tensor_t* const a = ccv_nnc_tensor_new(ap, CPU_TENSOR_NHWC(32F, 2, 4, 2), 0);
982
1
  float bp[] = {
983
1
    21, 22, 23,
984
1
    24, 25, 26,
985
1
    212, 222, 232,
986
1
    242, 252, 262,
987
1
  };
988
1
  ccv_nnc_tensor_t* const b = ccv_nnc_tensor_new(bp, CPU_TENSOR_NHWC(32F, 2, 2, 3), 0);
989
1
  ccv_nnc_tensor_t* const h = ccv_nnc_tensor_new(0, CPU_TENSOR_NHWC(32F, 2, 4, 2), 0);
990
1
  ccv_nnc_tensor_t* const db = ccv_nnc_tensor_new(0, CPU_TENSOR_NHWC(32F, 2, 2, 3), 0);
991
1
  ccv_nnc_tensor_t* const dbias = ccv_nnc_tensor_new(0, CPU_TENSOR_NHWC(32F, 2, 1, 3), 0);
992
1
  ccv_nnc_tensor_t* const gg = ccv_nnc_tensor_new(0, GPU_TENSOR_NHWC(000, 32F, 2, 4, 3), 0);
993
1
  ccv_nnc_tensor_t* const ga = ccv_nnc_tensor_new(0, GPU_TENSOR_NHWC(000, 32F, 2, 4, 2), 0);
994
1
  ccv_nnc_tensor_t* const gb = ccv_nnc_tensor_new(0, GPU_TENSOR_NHWC(000, 32F, 2, 2, 3), 0);
995
1
  ccv_nnc_tensor_t* const gh = ccv_nnc_tensor_new(0, GPU_TENSOR_NHWC(000, 32F, 2, 4, 2), 0);
996
1
  ccv_nnc_tensor_t* const gdb = ccv_nnc_tensor_new(0, GPU_TENSOR_NHWC(000, 32F, 2, 2, 3), 0);
997
1
  ccv_nnc_tensor_t* const gdbias = ccv_nnc_tensor_new(0, GPU_TENSOR_NHWC(000, 32F, 2, 1, 3), 0);
998
1
  ccv_nnc_cmd_exec(CMD_DATA_TRANSFER_FORWARD(), ccv_nnc_no_hint, 0, TENSOR_LIST(g, a, b), TENSOR_LIST(gg, ga, gb), 0);
999
1
  ccv_nnc_cmd_exec(CMD_GEMM_BACKWARD(), ccv_nnc_no_hint, 0, TENSOR_LIST(gg, ga, gb), TENSOR_LIST(gh, gdb, gdbias), 0);
1000
1
  ccv_nnc_cmd_exec(CMD_DATA_TRANSFER_FORWARD(), ccv_nnc_no_hint, 0, TENSOR_LIST(gh, gdb, gdbias), TENSOR_LIST(h, db, dbias), 0);
1001
1
  float dbiastp[] = {
1002
1
    22, 26, 30,
1003
1
    220, 260, 300,
1004
1
  };
1005
1
  ccv_nnc_tensor_t dbiast = ccv_nnc_tensor(dbiastp, CPU_TENSOR_NHWC(32F, 2, 1, 3), 0);
1006
1
  REQUIRE_TENSOR_EQ(dbias, &dbiast, "bias should be equal");
1007
1
  float htp[] = {
1008
1
    1 * 21 + 2 * 22 + 3 * 23, 1 * 24 + 2 * 25 + 3 * 26,
1009
1
    4 * 21 + 5 * 22 + 6 * 23, 4 * 24 + 5 * 25 + 6 * 26,
1010
1
    7 * 21 + 8 * 22 + 9 * 23, 7 * 24 + 8 * 25 + 9 * 26,
1011
1
    10 * 21 + 11 * 22 + 12 * 23, 10 * 24 + 11 * 25 + 12 * 26,
1012
1
    10 * 212 + 20 * 222 + 30 * 232, 10 * 242 + 20 * 252 + 30 * 262,
1013
1
    40 * 212 + 50 * 222 + 60 * 232, 40 * 242 + 50 * 252 + 60 * 262,
1014
1
    70 * 212 + 80 * 222 + 90 * 232, 70 * 242 + 80 * 252 + 90 * 262,
1015
1
    100 * 212 + 110 * 222 + 120 * 232, 100 * 242 + 110 * 252 + 120 * 262,
1016
1
  };
1017
1
  ccv_nnc_tensor_t ht = ccv_nnc_tensor(htp, CPU_TENSOR_NHWC(32F, 2, 4, 2), 0);
1018
1
  REQUIRE_TENSOR_EQ(h, &ht, "h should be equal");
1019
1
  float dbtp[] = {
1020
1
    1 * 13 + 4 * 15 + 7 * 17 + 10 * 19, 2 * 13 + 5 * 15 + 8 * 17 + 11 * 19, 3 * 13 + 6 * 15 + 9 * 17 + 12 * 19,
1021
1
    1 * 14 + 4 * 16 + 7 * 18 + 10 * 20, 2 * 14 + 5 * 16 + 8 * 18 + 11 * 20, 3 * 14 + 6 * 16 + 9 * 18 + 12 * 20,
1022
1
    10 * 131 + 40 * 151 + 70 * 171 + 100 * 191, 20 * 131 + 50 * 151 + 80 * 171 + 110 * 191, 30 * 131 + 60 * 151 + 90 * 171 + 120 * 191,
1023
1
    10 * 141 + 40 * 161 + 70 * 181 + 100 * 201, 20 * 141 + 50 * 161 + 80 * 181 + 110 * 201, 30 * 141 + 60 * 161 + 90 * 181 + 120 * 201,
1024
1
  };
1025
1
  ccv_nnc_tensor_t dbt = ccv_nnc_tensor(dbtp, CPU_TENSOR_NHWC(32F, 2, 2, 3), 0);
1026
1
  REQUIRE_TENSOR_EQ(db, &dbt, "db should be equal");
1027
1
  ccv_nnc_tensor_free(g);
1028
1
  ccv_nnc_tensor_free(a);
1029
1
  ccv_nnc_tensor_free(b);
1030
1
  ccv_nnc_tensor_free(h);
1031
1
  ccv_nnc_tensor_free(db);
1032
1
  ccv_nnc_tensor_free(dbias);
1033
1
  ccv_nnc_tensor_free(gg);
1034
1
  ccv_nnc_tensor_free(ga);
1035
1
  ccv_nnc_tensor_free(gb);
1036
1
  ccv_nnc_tensor_free(gh);
1037
1
  ccv_nnc_tensor_free(gdb);
1038
1
  ccv_nnc_tensor_free(gdbias);
1039
1
}
1040
1041
TEST_CASE("backward gemm with transpose a batch 2, same b")
1042
1
{
1043
1
  GUARD_ELSE_RETURN(ccv_nnc_cmd_ok(CCV_NNC_GEMM_FORWARD, CCV_NNC_BACKEND_GPU_CUBLAS) &&
1044
1
    ccv_nnc_cmd_ok(CCV_NNC_GEMM_BACKWARD, CCV_NNC_BACKEND_GPU_CUBLAS));
1045
1
  float gp[] = {
1046
1
    1, 2, 3,
1047
1
    4, 5, 6,
1048
1
    7, 8, 9,
1049
1
    10, 11, 12,
1050
1
    10, 20, 30,
1051
1
    40, 50, 60,
1052
1
    70, 80, 90,
1053
1
    100, 110, 120,
1054
1
  };
1055
1
  ccv_nnc_tensor_t* const g = ccv_nnc_tensor_new(gp, CPU_TENSOR_NHWC(32F, 2, 4, 3), 0);
1056
1
  float ap[] = {
1057
1
    13, 15, 17, 19,
1058
1
    14, 16, 18, 20,
1059
1
    131, 151, 171, 191,
1060
1
    141, 161, 181, 201,
1061
1
  };
1062
1
  ccv_nnc_tensor_t* const a = ccv_nnc_tensor_new(ap, CPU_TENSOR_NHWC(32F, 2, 2, 4), 0);
1063
1
  float bp[] = {
1064
1
    21, 22, 23,
1065
1
    24, 25, 26,
1066
1
  };
1067
1
  ccv_nnc_tensor_t* const b = ccv_nnc_tensor_new(bp, CPU_TENSOR_NHWC(32F, 2, 3), 0);
1068
1
  ccv_nnc_tensor_t* const h = ccv_nnc_tensor_new(0, CPU_TENSOR_NHWC(32F, 2, 2, 4), 0);
1069
1
  ccv_nnc_tensor_t* const db = ccv_nnc_tensor_new(0, CPU_TENSOR_NHWC(32F, 2, 3), 0);
1070
1
  ccv_nnc_tensor_t* const dbias = ccv_nnc_tensor_new(0, CPU_TENSOR_NHWC(32F, 3), 0);
1071
1
  ccv_nnc_tensor_t* const gg = ccv_nnc_tensor_new(0, GPU_TENSOR_NHWC(000, 32F, 2, 4, 3), 0);
1072
1
  ccv_nnc_tensor_t* const ga = ccv_nnc_tensor_new(0, GPU_TENSOR_NHWC(000, 32F, 2, 2, 4), 0);
1073
1
  ccv_nnc_tensor_t* const gb = ccv_nnc_tensor_new(0, GPU_TENSOR_NHWC(000, 32F, 2, 3), 0);
1074
1
  ccv_nnc_tensor_t* const gh = ccv_nnc_tensor_new(0, GPU_TENSOR_NHWC(000, 32F, 2, 2, 4), 0);
1075
1
  ccv_nnc_tensor_t* const gdb = ccv_nnc_tensor_new(0, GPU_TENSOR_NHWC(000, 32F, 2, 3), 0);
1076
1
  ccv_nnc_tensor_t* const gdbias = ccv_nnc_tensor_new(0, GPU_TENSOR_NHWC(000, 32F, 3), 0);
1077
1
  ccv_nnc_cmd_exec(CMD_DATA_TRANSFER_FORWARD(), ccv_nnc_no_hint, 0, TENSOR_LIST(g, a, b), TENSOR_LIST(gg, ga, gb), 0);
1078
1
  ccv_nnc_cmd_exec(CMD_GEMM_BACKWARD(TRANSPOSE(1, 2)), ccv_nnc_no_hint, 0, TENSOR_LIST(gg, ga, gb), TENSOR_LIST(gh, gdb, gdbias), 0);
1079
1
  ccv_nnc_cmd_exec(CMD_DATA_TRANSFER_FORWARD(), ccv_nnc_no_hint, 0, TENSOR_LIST(gh, gdb, gdbias), TENSOR_LIST(h, db, dbias), 0);
1080
1
  float dbiastp[] = {
1081
1
    22 + 220, 26 + 260, 30 + 300,
1082
1
  };
1083
1
  ccv_nnc_tensor_t dbiast = ccv_nnc_tensor(dbiastp, CPU_TENSOR_NHWC(32F, 3), 0);
1084
1
  REQUIRE_TENSOR_EQ(dbias, &dbiast, "bias should be equal");
1085
1
  float htp[] = {
1086
1
    1 * 21 + 2 * 22 + 3 * 23, 4 * 21 + 5 * 22 + 6 * 23, 7 * 21 + 8 * 22 + 9 * 23, 10 * 21 + 11 * 22 + 12 * 23,
1087
1
    1 * 24 + 2 * 25 + 3 * 26, 4 * 24 + 5 * 25 + 6 * 26, 7 * 24 + 8 * 25 + 9 * 26, 10 * 24 + 11 * 25 + 12 * 26,
1088
1
    10 * 21 + 20 * 22 + 30 * 23, 40 * 21 + 50 * 22 + 60 * 23, 70 * 21 + 80 * 22 + 90 * 23, 100 * 21 + 110 * 22 + 120 * 23,
1089
1
    10 * 24 + 20 * 25 + 30 * 26, 40 * 24 + 50 * 25 + 60 * 26, 70 * 24 + 80 * 25 + 90 * 26, 100 * 24 + 110 * 25 + 120 * 26,
1090
1
  };
1091
1
  ccv_nnc_tensor_t ht = ccv_nnc_tensor(htp, CPU_TENSOR_NHWC(32F, 2, 2, 4), 0);
1092
1
  REQUIRE_TENSOR_EQ(h, &ht, "h should be equal");
1093
1
  float dbtp[] = {
1094
1
    1 * 13 + 4 * 15 + 7 * 17 + 10 * 19 + 10 * 131 + 40 * 151 + 70 * 171 + 100 * 191, 2 * 13 + 5 * 15 + 8 * 17 + 11 * 19 + 20 * 131 + 50 * 151 + 80 * 171 + 110 * 191, 3 * 13 + 6 * 15 + 9 * 17 + 12 * 19 + 30 * 131 + 60 * 151 + 90 * 171 + 120 * 191,
1095
1
    1 * 14 + 4 * 16 + 7 * 18 + 10 * 20 + 10 * 141 + 40 * 161 + 70 * 181 + 100 * 201, 2 * 14 + 5 * 16 + 8 * 18 + 11 * 20 + 20 * 141 + 50 * 161 + 80 * 181 + 110 * 201, 3 * 14 + 6 * 16 + 9 * 18 + 12 * 20 + 30 * 141 + 60 * 161 + 90 * 181 + 120 * 201,
1096
1
  };
1097
1
  ccv_nnc_tensor_t dbt = ccv_nnc_tensor(dbtp, CPU_TENSOR_NHWC(32F, 2, 3), 0);
1098
1
  REQUIRE_TENSOR_EQ(db, &dbt, "db should be equal");
1099
1
  ccv_nnc_tensor_free(g);
1100
1
  ccv_nnc_tensor_free(a);
1101
1
  ccv_nnc_tensor_free(b);
1102
1
  ccv_nnc_tensor_free(h);
1103
1
  ccv_nnc_tensor_free(db);
1104
1
  ccv_nnc_tensor_free(dbias);
1105
1
  ccv_nnc_tensor_free(gg);
1106
1
  ccv_nnc_tensor_free(ga);
1107
1
  ccv_nnc_tensor_free(gb);
1108
1
  ccv_nnc_tensor_free(gh);
1109
1
  ccv_nnc_tensor_free(gdb);
1110
1
  ccv_nnc_tensor_free(gdbias);
1111
1
}
1112
1113
TEST_CASE("backward gemm with transpose b batch 2, batched b")
1114
1
{
1115
1
  GUARD_ELSE_RETURN(ccv_nnc_cmd_ok(CCV_NNC_GEMM_FORWARD, CCV_NNC_BACKEND_GPU_CUBLAS) &&
1116
1
    ccv_nnc_cmd_ok(CCV_NNC_GEMM_BACKWARD, CCV_NNC_BACKEND_GPU_CUBLAS));
1117
1
  float gp[] = {
1118
1
    1, 2, 3,
1119
1
    4, 5, 6,
1120
1
    7, 8, 9,
1121
1
    10, 11, 12,
1122
1
    10, 20, 30,
1123
1
    40, 50, 60,
1124
1
    70, 80, 90,
1125
1
    100, 110, 120,
1126
1
  };
1127
1
  ccv_nnc_tensor_t* const g = ccv_nnc_tensor_new(gp, CPU_TENSOR_NHWC(32F, 2, 4, 3), 0);
1128
1
  float ap[] = {
1129
1
    13, 14,
1130
1
    15, 16,
1131
1
    17, 18,
1132
1
    19, 20,
1133
1
    131, 141,
1134
1
    151, 161,
1135
1
    171, 181,
1136
1
    191, 201,
1137
1
  };
1138
1
  ccv_nnc_tensor_t* const a = ccv_nnc_tensor_new(ap, CPU_TENSOR_NHWC(32F, 2, 4, 2), 0);
1139
1
  float bp[] = {
1140
1
    21, 24,
1141
1
    22, 25,
1142
1
    23, 26,
1143
1
    212, 242,
1144
1
    222, 252,
1145
1
    232, 262,
1146
1
  };
1147
1
  ccv_nnc_tensor_t* const b = ccv_nnc_tensor_new(bp, CPU_TENSOR_NHWC(32F, 2, 3, 2), 0);
1148
1
  ccv_nnc_tensor_t* const h = ccv_nnc_tensor_new(0, CPU_TENSOR_NHWC(32F, 2, 4, 2), 0);
1149
1
  ccv_nnc_tensor_t* const db = ccv_nnc_tensor_new(0, CPU_TENSOR_NHWC(32F, 2, 3, 2), 0);
1150
1
  ccv_nnc_tensor_t* const dbias = ccv_nnc_tensor_new(0, CPU_TENSOR_NHWC(32F, 2, 1, 3), 0);
1151
1
  ccv_nnc_tensor_t* const gg = ccv_nnc_tensor_new(0, GPU_TENSOR_NHWC(000, 32F, 2, 4, 3), 0);
1152
1
  ccv_nnc_tensor_t* const ga = ccv_nnc_tensor_new(0, GPU_TENSOR_NHWC(000, 32F, 2, 4, 2), 0);
1153
1
  ccv_nnc_tensor_t* const gb = ccv_nnc_tensor_new(0, GPU_TENSOR_NHWC(000, 32F, 2, 3, 2), 0);
1154
1
  ccv_nnc_tensor_t* const gh = ccv_nnc_tensor_new(0, GPU_TENSOR_NHWC(000, 32F, 2, 4, 2), 0);
1155
1
  ccv_nnc_tensor_t* const gdb = ccv_nnc_tensor_new(0, GPU_TENSOR_NHWC(000, 32F, 2, 3, 2), 0);
1156
1
  ccv_nnc_tensor_t* const gdbias = ccv_nnc_tensor_new(0, GPU_TENSOR_NHWC(000, 32F, 2, 1, 3), 0);
1157
1
  ccv_nnc_cmd_exec(CMD_DATA_TRANSFER_FORWARD(), ccv_nnc_no_hint, 0, TENSOR_LIST(g, a, b), TENSOR_LIST(gg, ga, gb), 0);
1158
1
  ccv_nnc_cmd_exec(CMD_GEMM_BACKWARD(NO_TRANSPOSE, TRANSPOSE(1, 2)), ccv_nnc_no_hint, 0, TENSOR_LIST(gg, ga, gb), TENSOR_LIST(gh, gdb, gdbias), 0);
1159
1
  ccv_nnc_cmd_exec(CMD_DATA_TRANSFER_FORWARD(), ccv_nnc_no_hint, 0, TENSOR_LIST(gh, gdb, gdbias), TENSOR_LIST(h, db, dbias), 0);
1160
1
  float dbiastp[] = {
1161
1
    22, 26, 30,
1162
1
    220, 260, 300,
1163
1
  };
1164
1
  ccv_nnc_tensor_t dbiast = ccv_nnc_tensor(dbiastp, CPU_TENSOR_NHWC(32F, 2, 1, 3), 0);
1165
1
  REQUIRE_TENSOR_EQ(dbias, &dbiast, "bias should be equal");
1166
1
  float htp[] = {
1167
1
    1 * 21 + 2 * 22 + 3 * 23, 1 * 24 + 2 * 25 + 3 * 26,
1168
1
    4 * 21 + 5 * 22 + 6 * 23, 4 * 24 + 5 * 25 + 6 * 26,
1169
1
    7 * 21 + 8 * 22 + 9 * 23, 7 * 24 + 8 * 25 + 9 * 26,
1170
1
    10 * 21 + 11 * 22 + 12 * 23, 10 * 24 + 11 * 25 + 12 * 26,
1171
1
    10 * 212 + 20 * 222 + 30 * 232, 10 * 242 + 20 * 252 + 30 * 262,
1172
1
    40 * 212 + 50 * 222 + 60 * 232, 40 * 242 + 50 * 252 + 60 * 262,
1173
1
    70 * 212 + 80 * 222 + 90 * 232, 70 * 242 + 80 * 252 + 90 * 262,
1174
1
    100 * 212 + 110 * 222 + 120 * 232, 100 * 242 + 110 * 252 + 120 * 262,
1175
1
  };
1176
1
  ccv_nnc_tensor_t ht = ccv_nnc_tensor(htp, CPU_TENSOR_NHWC(32F, 2, 4, 2), 0);
1177
1
  REQUIRE_TENSOR_EQ(h, &ht, "h should be equal");
1178
1
  float dbtp[] = {
1179
1
    1 * 13 + 4 * 15 + 7 * 17 + 10 * 19, 1 * 14 + 4 * 16 + 7 * 18 + 10 * 20,
1180
1
    2 * 13 + 5 * 15 + 8 * 17 + 11 * 19, 2 * 14 + 5 * 16 + 8 * 18 + 11 * 20,
1181
1
    3 * 13 + 6 * 15 + 9 * 17 + 12 * 19, 3 * 14 + 6 * 16 + 9 * 18 + 12 * 20,
1182
1
    10 * 131 + 40 * 151 + 70 * 171 + 100 * 191, 10 * 141 + 40 * 161 + 70 * 181 + 100 * 201,
1183
1
    20 * 131 + 50 * 151 + 80 * 171 + 110 * 191, 20 * 141 + 50 * 161 + 80 * 181 + 110 * 201,
1184
1
    30 * 131 + 60 * 151 + 90 * 171 + 120 * 191, 30 * 141 + 60 * 161 + 90 * 181 + 120 * 201,
1185
1
  };
1186
1
  ccv_nnc_tensor_t dbt = ccv_nnc_tensor(dbtp, CPU_TENSOR_NHWC(32F, 2, 3, 2), 0);
1187
1
  REQUIRE_TENSOR_EQ(db, &dbt, "db should be equal");
1188
1
  ccv_nnc_tensor_free(g);
1189
1
  ccv_nnc_tensor_free(a);
1190
1
  ccv_nnc_tensor_free(b);
1191
1
  ccv_nnc_tensor_free(h);
1192
1
  ccv_nnc_tensor_free(db);
1193
1
  ccv_nnc_tensor_free(dbias);
1194
1
  ccv_nnc_tensor_free(gg);
1195
1
  ccv_nnc_tensor_free(ga);
1196
1
  ccv_nnc_tensor_free(gb);
1197
1
  ccv_nnc_tensor_free(gh);
1198
1
  ccv_nnc_tensor_free(gdb);
1199
1
  ccv_nnc_tensor_free(gdbias);
1200
1
}
1201
1202
TEST_CASE("backward gemm with transpose a and b batch 2, same b")
1203
1
{
1204
1
  GUARD_ELSE_RETURN(ccv_nnc_cmd_ok(CCV_NNC_GEMM_FORWARD, CCV_NNC_BACKEND_GPU_CUBLAS) &&
1205
1
    ccv_nnc_cmd_ok(CCV_NNC_GEMM_BACKWARD, CCV_NNC_BACKEND_GPU_CUBLAS));
1206
1
  float gp[] = {
1207
1
    1, 2, 3,
1208
1
    4, 5, 6,
1209
1
    7, 8, 9,
1210
1
    10, 11, 12,
1211
1
    10, 20, 30,
1212
1
    40, 50, 60,
1213
1
    70, 80, 90,
1214
1
    100, 110, 120,
1215
1
  };
1216
1
  ccv_nnc_tensor_t* const g = ccv_nnc_tensor_new(gp, CPU_TENSOR_NHWC(32F, 2, 4, 3), 0);
1217
1
  float ap[] = {
1218
1
    13, 15, 17, 19,
1219
1
    14, 16, 18, 20,
1220
1
    131, 151, 171, 191,
1221
1
    141, 161, 181, 201,
1222
1
  };
1223
1
  ccv_nnc_tensor_t* const a = ccv_nnc_tensor_new(ap, CPU_TENSOR_NHWC(32F, 2, 2, 4), 0);
1224
1
  float bp[] = {
1225
1
    21, 24,
1226
1
    22, 25,
1227
1
    23, 26,
1228
1
  };
1229
1
  ccv_nnc_tensor_t* const b = ccv_nnc_tensor_new(bp, CPU_TENSOR_NHWC(32F, 3, 2), 0);
1230
1
  ccv_nnc_tensor_t* const h = ccv_nnc_tensor_new(0, CPU_TENSOR_NHWC(32F, 2, 2, 4), 0);
1231
1
  ccv_nnc_tensor_t* const db = ccv_nnc_tensor_new(0, CPU_TENSOR_NHWC(32F, 3, 2), 0);
1232
1
  ccv_nnc_tensor_t* const dbias = ccv_nnc_tensor_new(0, CPU_TENSOR_NHWC(32F, 3), 0);
1233
1
  ccv_nnc_tensor_t* const gg = ccv_nnc_tensor_new(0, GPU_TENSOR_NHWC(000, 32F, 2, 4, 3), 0);
1234
1
  ccv_nnc_tensor_t* const ga = ccv_nnc_tensor_new(0, GPU_TENSOR_NHWC(000, 32F, 2, 2, 4), 0);
1235
1
  ccv_nnc_tensor_t* const gb = ccv_nnc_tensor_new(0, GPU_TENSOR_NHWC(000, 32F, 3, 2), 0);
1236
1
  ccv_nnc_tensor_t* const gh = ccv_nnc_tensor_new(0, GPU_TENSOR_NHWC(000, 32F, 2, 2, 4), 0);
1237
1
  ccv_nnc_tensor_t* const gdb = ccv_nnc_tensor_new(0, GPU_TENSOR_NHWC(000, 32F, 3, 2), 0);
1238
1
  ccv_nnc_tensor_t* const gdbias = ccv_nnc_tensor_new(0, GPU_TENSOR_NHWC(000, 32F, 3), 0);
1239
1
  ccv_nnc_cmd_exec(CMD_DATA_TRANSFER_FORWARD(), ccv_nnc_no_hint, 0, TENSOR_LIST(g, a, b), TENSOR_LIST(gg, ga, gb), 0);
1240
1
  ccv_nnc_cmd_exec(CMD_GEMM_BACKWARD(TRANSPOSE(1, 2), TRANSPOSE(0, 1)), ccv_nnc_no_hint, 0, TENSOR_LIST(gg, ga, gb), TENSOR_LIST(gh, gdb, gdbias), 0);
1241
1
  ccv_nnc_cmd_exec(CMD_DATA_TRANSFER_FORWARD(), ccv_nnc_no_hint, 0, TENSOR_LIST(gh, gdb, gdbias), TENSOR_LIST(h, db, dbias), 0);
1242
1
  float dbiastp[] = {
1243
1
    22 + 220, 26 + 260, 30 + 300,
1244
1
  };
1245
1
  ccv_nnc_tensor_t dbiast = ccv_nnc_tensor(dbiastp, CPU_TENSOR_NHWC(32F, 3), 0);
1246
1
  REQUIRE_TENSOR_EQ(dbias, &dbiast, "bias should be equal");
1247
1
  float htp[] = {
1248
1
    1 * 21 + 2 * 22 + 3 * 23, 4 * 21 + 5 * 22 + 6 * 23, 7 * 21 + 8 * 22 + 9 * 23, 10 * 21 + 11 * 22 + 12 * 23,
1249
1
    1 * 24 + 2 * 25 + 3 * 26, 4 * 24 + 5 * 25 + 6 * 26, 7 * 24 + 8 * 25 + 9 * 26, 10 * 24 + 11 * 25 + 12 * 26,
1250
1
    10 * 21 + 20 * 22 + 30 * 23, 40 * 21 + 50 * 22 + 60 * 23, 70 * 21 + 80 * 22 + 90 * 23, 100 * 21 + 110 * 22 + 120 * 23,
1251
1
    10 * 24 + 20 * 25 + 30 * 26, 40 * 24 + 50 * 25 + 60 * 26, 70 * 24 + 80 * 25 + 90 * 26, 100 * 24 + 110 * 25 + 120 * 26,
1252
1
  };
1253
1
  ccv_nnc_tensor_t ht = ccv_nnc_tensor(htp, CPU_TENSOR_NHWC(32F, 2, 2, 4), 0);
1254
1
  REQUIRE_TENSOR_EQ(h, &ht, "h should be equal");
1255
1
  float dbtp[] = {
1256
1
    1 * 13 + 4 * 15 + 7 * 17 + 10 * 19 + 10 * 131 + 40 * 151 + 70 * 171 + 100 * 191, 1 * 14 + 4 * 16 + 7 * 18 + 10 * 20 + 10 * 141 + 40 * 161 + 70 * 181 + 100 * 201,
1257
1
    2 * 13 + 5 * 15 + 8 * 17 + 11 * 19 + 20 * 131 + 50 * 151 + 80 * 171 + 110 * 191, 2 * 14 + 5 * 16 + 8 * 18 + 11 * 20 + 20 * 141 + 50 * 161 + 80 * 181 + 110 * 201,
1258
1
    3 * 13 + 6 * 15 + 9 * 17 + 12 * 19 + 30 * 131 + 60 * 151 + 90 * 171 + 120 * 191, 3 * 14 + 6 * 16 + 9 * 18 + 12 * 20 + 30 * 141 + 60 * 161 + 90 * 181 + 120 * 201,
1259
1
  };
1260
1
  ccv_nnc_tensor_t dbt = ccv_nnc_tensor(dbtp, CPU_TENSOR_NHWC(32F, 3, 2), 0);
1261
1
  REQUIRE_TENSOR_EQ(db, &dbt, "db should be equal");
1262
1
  ccv_nnc_tensor_free(g);
1263
1
  ccv_nnc_tensor_free(a);
1264
1
  ccv_nnc_tensor_free(b);
1265
1
  ccv_nnc_tensor_free(h);
1266
1
  ccv_nnc_tensor_free(db);
1267
1
  ccv_nnc_tensor_free(dbias);
1268
1
  ccv_nnc_tensor_free(gg);
1269
1
  ccv_nnc_tensor_free(ga);
1270
1
  ccv_nnc_tensor_free(gb);
1271
1
  ccv_nnc_tensor_free(gh);
1272
1
  ccv_nnc_tensor_free(gdb);
1273
1
  ccv_nnc_tensor_free(gdbias);
1274
1
}
1275
1276
TEST_CASE("cublas forward gemm")
1277
1
{
1278
1
  GUARD_ELSE_RETURN(ccv_nnc_cmd_ok(CCV_NNC_GEMM_FORWARD, CCV_NNC_BACKEND_GPU_CUBLAS));
1279
1
  dsfmt_t dsfmt;
1280
1
  dsfmt_init_gen_rand(&dsfmt, 0);
1281
1
  ccv_nnc_tensor_t* a = ccv_nnc_tensor_new(0, GPU_TENSOR_NHWC(000, 32F, 10, 128), 0);
1282
1
  ccv_nnc_tensor_t* w = ccv_nnc_tensor_new(0, GPU_TENSOR_NHWC(000, 32F, 64, 128), 0);
1283
1
  ccv_nnc_tensor_t* bias = ccv_nnc_tensor_new(0, GPU_TENSOR_NHWC(000, 32F, 64), 0);
1284
1
  ccv_nnc_tensor_t* b = ccv_nnc_tensor_new(0, GPU_TENSOR_NHWC(000, 32F, 10, 64), 0);
1285
1286
1
  ccv_nnc_tensor_t* ha = ccv_nnc_tensor_new(0, CPU_TENSOR_NHWC(32F, 1, 128), 0);
1287
1
  ccv_nnc_tensor_t* hw = ccv_nnc_tensor_new(0, CPU_TENSOR_NHWC(32F, 64, 128), 0);
1288
1
  ccv_nnc_tensor_t* hbias = ccv_nnc_tensor_new(0, CPU_TENSOR_NHWC(32F, 64), 0);
1289
1
  ccv_nnc_tensor_t* hb = ccv_nnc_tensor_new(0, CPU_TENSOR_NHWC(32F, 1, 64), 0);
1290
1
  int i;
1291
8.19k
  for (i = 0; i < 64 * 128; 
i++8.19k
)
1292
8.19k
    hw->data.f32[i] = dsfmt_genrand_open_close(&dsfmt) / (64 * 128);
1293
65
  for (i = 0; i < 64; 
i++64
)
1294
64
    hbias->data.f32[i] = dsfmt_genrand_open_close(&dsfmt);
1295
1
  ccv_nnc_tensor_t* ha1 = ccv_nnc_tensor_new(0, CPU_TENSOR_NHWC(32F, 10, 128), 0);
1296
1.28k
  for (i = 0; i < 10 * 128; 
i++1.28k
)
1297
1.28k
    ha1->data.f32[i] = dsfmt_genrand_open_close(&dsfmt);
1298
129
  for (i = 0; i < 128; 
i++128
)
1299
128
    ha->data.f32[i] = ha1->data.f32[i];
1300
1
  ccv_nnc_cmd_exec(CMD_DATA_TRANSFER_FORWARD(), ccv_nnc_no_hint, 0, TENSOR_LIST(ha1, hw, hbias), TENSOR_LIST(a, w, bias), 0);
1301
1
  ccv_nnc_cmd_exec(CMD_GEMM_FORWARD(NO_TRANSPOSE, TRANSPOSE(0, 1)), ccv_nnc_no_hint, 0, TENSOR_LIST(ha, hw, hbias), TENSOR_LIST(hb), 0);
1302
1
  ccv_nnc_cmd_exec(CMD_GEMM_FORWARD(NO_TRANSPOSE, TRANSPOSE(0, 1)), ccv_nnc_no_hint, 0, TENSOR_LIST(a, w, bias), TENSOR_LIST(b), 0);
1303
1
  ccv_nnc_tensor_t* tb = ccv_nnc_tensor_new(0, CPU_TENSOR_NHWC(32F, 10, 64), 0);
1304
1
  ccv_nnc_cmd_exec(CMD_DATA_TRANSFER_FORWARD(), ccv_nnc_no_hint, 0, TENSOR_LIST(b), TENSOR_LIST(tb), 0);
1305
1
  ccv_nnc_tensor_t* tb1 = ccv_nnc_tensor_new(0, CPU_TENSOR_NHWC(32F, 1, 64), 0);
1306
65
  for (i = 0; i < 64; 
i++64
)
1307
64
    tb1->data.f32[i] = tb->data.f32[i];
1308
1
  REQUIRE_TENSOR_EQ(tb1, hb, "GPU computed output should be the same as CPU computed ones");
1309
1
  ccv_nnc_tensor_free(a);
1310
1
  ccv_nnc_tensor_free(w);
1311
1
  ccv_nnc_tensor_free(bias);
1312
1
  ccv_nnc_tensor_free(tb);
1313
1
  ccv_nnc_tensor_free(b);
1314
1
  ccv_nnc_tensor_free(ha);
1315
1
  ccv_nnc_tensor_free(ha1);
1316
1
  ccv_nnc_tensor_free(tb1);
1317
1
  ccv_nnc_tensor_free(hw);
1318
1
  ccv_nnc_tensor_free(hbias);
1319
1
  ccv_nnc_tensor_free(hb);
1320
1
}
1321
1322
TEST_CASE("cublas forward gemm in half precision")
1323
1
{
1324
1
  GUARD_ELSE_RETURN(ccv_nnc_cmd_ok(CCV_NNC_GEMM_FORWARD, CCV_NNC_BACKEND_GPU_CUBLAS));
1325
1
  dsfmt_t dsfmt;
1326
1
  dsfmt_init_gen_rand(&dsfmt, 0);
1327
1
  ccv_nnc_tensor_t* a = ccv_nnc_tensor_new(0, GPU_TENSOR_NHWC(000, 16F, 10, 128), 0);
1328
1
  ccv_nnc_tensor_t* w = ccv_nnc_tensor_new(0, GPU_TENSOR_NHWC(000, 16F, 64, 128), 0);
1329
1
  ccv_nnc_tensor_t* bias = ccv_nnc_tensor_new(0, GPU_TENSOR_NHWC(000, 16F, 64), 0);
1330
1
  ccv_nnc_tensor_t* b = ccv_nnc_tensor_new(0, GPU_TENSOR_NHWC(000, 16F, 10, 64), 0);
1331
1332
1
  ccv_nnc_tensor_t* ha = ccv_nnc_tensor_new(0, CPU_TENSOR_NHWC(32F, 1, 128), 0);
1333
1
  ccv_nnc_tensor_t* hw = ccv_nnc_tensor_new(0, CPU_TENSOR_NHWC(32F, 64, 128), 0);
1334
1
  ccv_nnc_tensor_t* hbias = ccv_nnc_tensor_new(0, CPU_TENSOR_NHWC(32F, 64), 0);
1335
1
  ccv_nnc_tensor_t* hb = ccv_nnc_tensor_new(0, CPU_TENSOR_NHWC(32F, 1, 64), 0);
1336
1
  int i;
1337
8.19k
  for (i = 0; i < 64 * 128; 
i++8.19k
)
1338
8.19k
    hw->data.f32[i] = dsfmt_genrand_open_close(&dsfmt) / (64 * 128);
1339
65
  for (i = 0; i < 64; 
i++64
)
1340
64
    hbias->data.f32[i] = dsfmt_genrand_open_close(&dsfmt);
1341
1
  ccv_nnc_tensor_t* ha1 = ccv_nnc_tensor_new(0, CPU_TENSOR_NHWC(32F, 10, 128), 0);
1342
1.28k
  for (i = 0; i < 10 * 128; 
i++1.28k
)
1343
1.28k
    ha1->data.f32[i] = dsfmt_genrand_open_close(&dsfmt);
1344
129
  for (i = 0; i < 128; 
i++128
)
1345
128
    ha->data.f32[i] = ha1->data.f32[i];
1346
1
  ccv_nnc_tensor_t* ha2 = ccv_nnc_tensor_new(0, CPU_TENSOR_NHWC(16F, 10, 128), 0);
1347
1
  ccv_nnc_tensor_t* hw2 = ccv_nnc_tensor_new(0, CPU_TENSOR_NHWC(16F, 64, 128), 0);
1348
1
  ccv_nnc_tensor_t* hbias2 = ccv_nnc_tensor_new(0, CPU_TENSOR_NHWC(16F, 64), 0);
1349
1
  ccv_nnc_cmd_exec(CMD_DATATYPE_CONVERSION_FORWARD(), ccv_nnc_no_hint, 0, TENSOR_LIST(ha1, hw, hbias), TENSOR_LIST(ha2, hw2, hbias2), 0);
1350
1
  ccv_nnc_cmd_exec(CMD_DATA_TRANSFER_FORWARD(), ccv_nnc_no_hint, 0, TENSOR_LIST(ha2, hw2, hbias2), TENSOR_LIST(a, w, bias), 0);
1351
1
  ccv_nnc_cmd_exec(CMD_GEMM_FORWARD(NO_TRANSPOSE, TRANSPOSE(0, 1)), ccv_nnc_no_hint, 0, TENSOR_LIST(ha, hw, hbias), TENSOR_LIST(hb), 0);
1352
1
  ccv_nnc_cmd_exec(CMD_GEMM_FORWARD(NO_TRANSPOSE, TRANSPOSE(0, 1)), ccv_nnc_no_hint, 0, TENSOR_LIST(a, w, bias), TENSOR_LIST(b), 0);
1353
1
  ccv_nnc_tensor_t* tb = ccv_nnc_tensor_new(0, CPU_TENSOR_NHWC(16F, 10, 64), 0);
1354
1
  ccv_nnc_cmd_exec(CMD_DATA_TRANSFER_FORWARD(), ccv_nnc_no_hint, 0, TENSOR_LIST(b), TENSOR_LIST(tb), 0);
1355
1
  ccv_nnc_tensor_t* tb1 = ccv_nnc_tensor_new(0, CPU_TENSOR_NHWC(32F, 10, 64), 0);
1356
1
  ccv_nnc_cmd_exec(CMD_DATATYPE_CONVERSION_FORWARD(), ccv_nnc_no_hint, 0, TENSOR_LIST(tb), TENSOR_LIST(tb1), 0);
1357
1
  REQUIRE_ARRAY_EQ_WITH_TOLERANCE(float, tb1->data.f32, hb->data.f32, 64, 1e-3, "GPU computed output should be the same as CPU computed ones");
1358
1
  ccv_nnc_tensor_free(a);
1359
1
  ccv_nnc_tensor_free(w);
1360
1
  ccv_nnc_tensor_free(bias);
1361
1
  ccv_nnc_tensor_free(b);
1362
1
  ccv_nnc_tensor_free(tb);
1363
1
  ccv_nnc_tensor_free(ha);
1364
1
  ccv_nnc_tensor_free(ha1);
1365
1
  ccv_nnc_tensor_free(tb1);
1366
1
  ccv_nnc_tensor_free(hw);
1367
1
  ccv_nnc_tensor_free(hbias);
1368
1
  ccv_nnc_tensor_free(hb);
1369
1
  ccv_nnc_tensor_free(ha2);
1370
1
  ccv_nnc_tensor_free(hw2);
1371
1
  ccv_nnc_tensor_free(hbias2);
1372
1
}
1373
1374
TEST_CASE("cublas forward gemm in bfloat precision")
1375
1
{
1376
1
  GUARD_ELSE_RETURN(ccv_nnc_cmd_ok(CCV_NNC_GEMM_FORWARD, CCV_NNC_BACKEND_GPU_CUBLAS));
1377
1
  dsfmt_t dsfmt;
1378
1
  dsfmt_init_gen_rand(&dsfmt, 0);
1379
1
  ccv_nnc_tensor_t* a = ccv_nnc_tensor_new(0, GPU_TENSOR_NHWC(000, 16BF, 10, 128), 0);
1380
1
  ccv_nnc_tensor_t* w = ccv_nnc_tensor_new(0, GPU_TENSOR_NHWC(000, 16BF, 64, 128), 0);
1381
1
  ccv_nnc_tensor_t* bias = ccv_nnc_tensor_new(0, GPU_TENSOR_NHWC(000, 16BF, 64), 0);
1382
1
  ccv_nnc_tensor_t* b = ccv_nnc_tensor_new(0, GPU_TENSOR_NHWC(000, 16BF, 10, 64), 0);
1383
1384
1
  ccv_nnc_tensor_t* ha = ccv_nnc_tensor_new(0, CPU_TENSOR_NHWC(32F, 1, 128), 0);
1385
1
  ccv_nnc_tensor_t* hw = ccv_nnc_tensor_new(0, CPU_TENSOR_NHWC(32F, 64, 128), 0);
1386
1
  ccv_nnc_tensor_t* hbias = ccv_nnc_tensor_new(0, CPU_TENSOR_NHWC(32F, 64), 0);
1387
1
  ccv_nnc_tensor_t* hb = ccv_nnc_tensor_new(0, CPU_TENSOR_NHWC(32F, 1, 64), 0);
1388
1
  int i;
1389
8.19k
  for (i = 0; i < 64 * 128; 
i++8.19k
)
1390
8.19k
    hw->data.f32[i] = dsfmt_genrand_open_close(&dsfmt) / (64 * 128);
1391
65
  for (i = 0; i < 64; 
i++64
)
1392
64
    hbias->data.f32[i] = dsfmt_genrand_open_close(&dsfmt);
1393
1
  ccv_nnc_tensor_t* ha1 = ccv_nnc_tensor_new(0, CPU_TENSOR_NHWC(32F, 10, 128), 0);
1394
1.28k
  for (i = 0; i < 10 * 128; 
i++1.28k
)
1395
1.28k
    ha1->data.f32[i] = dsfmt_genrand_open_close(&dsfmt);
1396
129
  for (i = 0; i < 128; 
i++128
)
1397
128
    ha->data.f32[i] = ha1->data.f32[i];
1398
1
  ccv_nnc_tensor_t* ha2 = ccv_nnc_tensor_new(0, CPU_TENSOR_NHWC(16BF, 10, 128), 0);
1399
1
  ccv_nnc_tensor_t* hw2 = ccv_nnc_tensor_new(0, CPU_TENSOR_NHWC(16BF, 64, 128), 0);
1400
1
  ccv_nnc_tensor_t* hbias2 = ccv_nnc_tensor_new(0, CPU_TENSOR_NHWC(16BF, 64), 0);
1401
1
  ccv_nnc_cmd_exec(CMD_DATATYPE_CONVERSION_FORWARD(), ccv_nnc_no_hint, 0, TENSOR_LIST(ha1, hw, hbias), TENSOR_LIST(ha2, hw2, hbias2), 0);
1402
1
  ccv_nnc_cmd_exec(CMD_DATA_TRANSFER_FORWARD(), ccv_nnc_no_hint, 0, TENSOR_LIST(ha2, hw2, hbias2), TENSOR_LIST(a, w, bias), 0);
1403
1
  ccv_nnc_cmd_exec(CMD_GEMM_FORWARD(NO_TRANSPOSE, TRANSPOSE(0, 1)), ccv_nnc_no_hint, 0, TENSOR_LIST(ha, hw, hbias), TENSOR_LIST(hb), 0);
1404
1
  ccv_nnc_cmd_exec(CMD_GEMM_FORWARD(NO_TRANSPOSE, TRANSPOSE(0, 1)), ccv_nnc_no_hint, 0, TENSOR_LIST(a, w, bias), TENSOR_LIST(b), 0);
1405
1
  ccv_nnc_tensor_t* tb = ccv_nnc_tensor_new(0, CPU_TENSOR_NHWC(16BF, 10, 64), 0);
1406
1
  ccv_nnc_cmd_exec(CMD_DATA_TRANSFER_FORWARD(), ccv_nnc_no_hint, 0, TENSOR_LIST(b), TENSOR_LIST(tb), 0);
1407
1
  ccv_nnc_tensor_t* tb1 = ccv_nnc_tensor_new(0, CPU_TENSOR_NHWC(32F, 10, 64), 0);
1408
1
  ccv_nnc_cmd_exec(CMD_DATATYPE_CONVERSION_FORWARD(), ccv_nnc_no_hint, 0, TENSOR_LIST(tb), TENSOR_LIST(tb1), 0);
1409
1
  REQUIRE_ARRAY_EQ_WITH_TOLERANCE(float, tb1->data.f32, hb->data.f32, 64, 1e-2, "GPU computed output should be the same as CPU computed ones");
1410
1
  ccv_nnc_tensor_free(a);
1411
1
  ccv_nnc_tensor_free(w);
1412
1
  ccv_nnc_tensor_free(bias);
1413
1
  ccv_nnc_tensor_free(b);
1414
1
  ccv_nnc_tensor_free(tb);
1415
1
  ccv_nnc_tensor_free(ha);
1416
1
  ccv_nnc_tensor_free(ha1);
1417
1
  ccv_nnc_tensor_free(tb1);
1418
1
  ccv_nnc_tensor_free(hw);
1419
1
  ccv_nnc_tensor_free(hbias);
1420
1
  ccv_nnc_tensor_free(hb);
1421
1
  ccv_nnc_tensor_free(ha2);
1422
1
  ccv_nnc_tensor_free(hw2);
1423
1
  ccv_nnc_tensor_free(hbias2);
1424
1
}
1425
1426
TEST_CASE("cublas forward gemv in half precision, variant 1")
1427
1
{
1428
1
  GUARD_ELSE_RETURN(ccv_nnc_cmd_ok(CCV_NNC_GEMM_FORWARD, CCV_NNC_BACKEND_GPU_CUBLAS));
1429
1
  dsfmt_t dsfmt;
1430
1
  dsfmt_init_gen_rand(&dsfmt, 0);
1431
1
  ccv_nnc_tensor_t* a = ccv_nnc_tensor_new(0, GPU_TENSOR_NHWC(000, 16F, 1, 128), 0);
1432
1
  ccv_nnc_tensor_t* w = ccv_nnc_tensor_new(0, GPU_TENSOR_NHWC(000, 16F, 64, 128), 0);
1433
1
  ccv_nnc_tensor_t* bias = ccv_nnc_tensor_new(0, GPU_TENSOR_NHWC(000, 16F, 64), 0);
1434
1
  ccv_nnc_tensor_t* b = ccv_nnc_tensor_new(0, GPU_TENSOR_NHWC(000, 16F, 1, 64), 0);
1435
1436
1
  ccv_nnc_tensor_t* ha = ccv_nnc_tensor_new(0, CPU_TENSOR_NHWC(32F, 1, 128), 0);
1437
1
  ccv_nnc_tensor_t* hw = ccv_nnc_tensor_new(0, CPU_TENSOR_NHWC(32F, 64, 128), 0);
1438
1
  ccv_nnc_tensor_t* hbias = ccv_nnc_tensor_new(0, CPU_TENSOR_NHWC(32F, 64), 0);
1439
1
  ccv_nnc_tensor_t* hb = ccv_nnc_tensor_new(0, CPU_TENSOR_NHWC(32F, 1, 64), 0);
1440
1
  int i;
1441
8.19k
  for (i = 0; i < 64 * 128; 
i++8.19k
)
1442
8.19k
    hw->data.f32[i] = dsfmt_genrand_open_close(&dsfmt) / (64 * 128);
1443
65
  for (i = 0; i < 64; 
i++64
)
1444
64
    hbias->data.f32[i] = dsfmt_genrand_open_close(&dsfmt);
1445
1
  ccv_nnc_tensor_t* ha1 = ccv_nnc_tensor_new(0, CPU_TENSOR_NHWC(32F, 1, 128), 0);
1446
129
  for (i = 0; i < 128; 
i++128
)
1447
128
    ha1->data.f32[i] = dsfmt_genrand_open_close(&dsfmt);
1448
129
  for (i = 0; i < 128; 
i++128
)
1449
128
    ha->data.f32[i] = ha1->data.f32[i];
1450
1
  ccv_nnc_tensor_t* ha2 = ccv_nnc_tensor_new(0, CPU_TENSOR_NHWC(16F, 1, 128), 0);
1451
1
  ccv_nnc_tensor_t* hw2 = ccv_nnc_tensor_new(0, CPU_TENSOR_NHWC(16F, 64, 128), 0);
1452
1
  ccv_nnc_tensor_t* hbias2 = ccv_nnc_tensor_new(0, CPU_TENSOR_NHWC(16F, 64), 0);
1453
1
  ccv_nnc_cmd_exec(CMD_DATATYPE_CONVERSION_FORWARD(), ccv_nnc_no_hint, 0, TENSOR_LIST(ha1, hw, hbias), TENSOR_LIST(ha2, hw2, hbias2), 0);
1454
1
  ccv_nnc_cmd_exec(CMD_DATA_TRANSFER_FORWARD(), ccv_nnc_no_hint, 0, TENSOR_LIST(ha2, hw2, hbias2), TENSOR_LIST(a, w, bias), 0);
1455
1
  ccv_nnc_cmd_exec(CMD_GEMM_FORWARD(NO_TRANSPOSE, TRANSPOSE(0, 1)), ccv_nnc_no_hint, 0, TENSOR_LIST(ha, hw, hbias), TENSOR_LIST(hb), 0);
1456
1
  ccv_nnc_cmd_exec(CMD_GEMM_FORWARD(NO_TRANSPOSE, TRANSPOSE(0, 1)), ccv_nnc_no_hint, 0, TENSOR_LIST(a, w, bias), TENSOR_LIST(b), 0);
1457
1
  ccv_nnc_tensor_t* tb = ccv_nnc_tensor_new(0, CPU_TENSOR_NHWC(16F, 1, 64), 0);
1458
1
  ccv_nnc_cmd_exec(CMD_DATA_TRANSFER_FORWARD(), ccv_nnc_no_hint, 0, TENSOR_LIST(b), TENSOR_LIST(tb), 0);
1459
1
  ccv_nnc_tensor_t* tb1 = ccv_nnc_tensor_new(0, CPU_TENSOR_NHWC(32F, 1, 64), 0);
1460
1
  ccv_nnc_cmd_exec(CMD_DATATYPE_CONVERSION_FORWARD(), ccv_nnc_no_hint, 0, TENSOR_LIST(tb), TENSOR_LIST(tb1), 0);
1461
1
  REQUIRE_ARRAY_EQ_WITH_TOLERANCE(float, tb1->data.f32, hb->data.f32, 64, 1e-3, "GPU computed output should be the same as CPU computed ones");
1462
1
  ccv_nnc_tensor_free(a);
1463
1
  ccv_nnc_tensor_free(w);
1464
1
  ccv_nnc_tensor_free(bias);
1465
1
  ccv_nnc_tensor_free(b);
1466
1
  ccv_nnc_tensor_free(tb);
1467
1
  ccv_nnc_tensor_free(ha);
1468
1
  ccv_nnc_tensor_free(ha1);
1469
1
  ccv_nnc_tensor_free(tb1);
1470
1
  ccv_nnc_tensor_free(hw);
1471
1
  ccv_nnc_tensor_free(hbias);
1472
1
  ccv_nnc_tensor_free(hb);
1473
1
  ccv_nnc_tensor_free(ha2);
1474
1
  ccv_nnc_tensor_free(hw2);
1475
1
  ccv_nnc_tensor_free(hbias2);
1476
1
}
1477
1478
TEST_CASE("cublas forward gemv in bfloat precision, variant 1")
1479
1
{
1480
1
  GUARD_ELSE_RETURN(ccv_nnc_cmd_ok(CCV_NNC_GEMM_FORWARD, CCV_NNC_BACKEND_GPU_CUBLAS));
1481
1
  dsfmt_t dsfmt;
1482
1
  dsfmt_init_gen_rand(&dsfmt, 0);
1483
1
  ccv_nnc_tensor_t* a = ccv_nnc_tensor_new(0, GPU_TENSOR_NHWC(000, 16BF, 1, 128), 0);
1484
1
  ccv_nnc_tensor_t* w = ccv_nnc_tensor_new(0, GPU_TENSOR_NHWC(000, 16BF, 64, 128), 0);
1485
1
  ccv_nnc_tensor_t* bias = ccv_nnc_tensor_new(0, GPU_TENSOR_NHWC(000, 16BF, 64), 0);
1486
1
  ccv_nnc_tensor_t* b = ccv_nnc_tensor_new(0, GPU_TENSOR_NHWC(000, 16BF, 1, 64), 0);
1487
1488
1
  ccv_nnc_tensor_t* ha = ccv_nnc_tensor_new(0, CPU_TENSOR_NHWC(32F, 1, 128), 0);
1489
1
  ccv_nnc_tensor_t* hw = ccv_nnc_tensor_new(0, CPU_TENSOR_NHWC(32F, 64, 128), 0);
1490
1
  ccv_nnc_tensor_t* hbias = ccv_nnc_tensor_new(0, CPU_TENSOR_NHWC(32F, 64), 0);
1491
1
  ccv_nnc_tensor_t* hb = ccv_nnc_tensor_new(0, CPU_TENSOR_NHWC(32F, 1, 64), 0);
1492
1
  int i;
1493
8.19k
  for (i = 0; i < 64 * 128; 
i++8.19k
)
1494
8.19k
    hw->data.f32[i] = dsfmt_genrand_open_close(&dsfmt) / (64 * 128);
1495
65
  for (i = 0; i < 64; 
i++64
)
1496
64
    hbias->data.f32[i] = dsfmt_genrand_open_close(&dsfmt);
1497
1
  ccv_nnc_tensor_t* ha1 = ccv_nnc_tensor_new(0, CPU_TENSOR_NHWC(32F, 1, 128), 0);
1498
129
  for (i = 0; i < 128; 
i++128
)
1499
128
    ha1->data.f32[i] = dsfmt_genrand_open_close(&dsfmt);
1500
129
  for (i = 0; i < 128; 
i++128
)
1501
128
    ha->data.f32[i] = ha1->data.f32[i];
1502
1
  ccv_nnc_tensor_t* ha2 = ccv_nnc_tensor_new(0, CPU_TENSOR_NHWC(16BF, 1, 128), 0);
1503
1
  ccv_nnc_tensor_t* hw2 = ccv_nnc_tensor_new(0, CPU_TENSOR_NHWC(16BF, 64, 128), 0);
1504
1
  ccv_nnc_tensor_t* hbias2 = ccv_nnc_tensor_new(0, CPU_TENSOR_NHWC(16BF, 64), 0);
1505
1
  ccv_nnc_cmd_exec(CMD_DATATYPE_CONVERSION_FORWARD(), ccv_nnc_no_hint, 0, TENSOR_LIST(ha1, hw, hbias), TENSOR_LIST(ha2, hw2, hbias2), 0);
1506
1
  ccv_nnc_cmd_exec(CMD_DATA_TRANSFER_FORWARD(), ccv_nnc_no_hint, 0, TENSOR_LIST(ha2, hw2, hbias2), TENSOR_LIST(a, w, bias), 0);
1507
1
  ccv_nnc_cmd_exec(CMD_GEMM_FORWARD(NO_TRANSPOSE, TRANSPOSE(0, 1)), ccv_nnc_no_hint, 0, TENSOR_LIST(ha, hw, hbias), TENSOR_LIST(hb), 0);
1508
1
  ccv_nnc_cmd_exec(CMD_GEMM_FORWARD(NO_TRANSPOSE, TRANSPOSE(0, 1)), ccv_nnc_no_hint, 0, TENSOR_LIST(a, w, bias), TENSOR_LIST(b), 0);
1509
1
  ccv_nnc_tensor_t* tb = ccv_nnc_tensor_new(0, CPU_TENSOR_NHWC(16BF, 1, 64), 0);
1510
1
  ccv_nnc_cmd_exec(CMD_DATA_TRANSFER_FORWARD(), ccv_nnc_no_hint, 0, TENSOR_LIST(b), TENSOR_LIST(tb), 0);
1511
1
  ccv_nnc_tensor_t* tb1 = ccv_nnc_tensor_new(0, CPU_TENSOR_NHWC(32F, 1, 64), 0);
1512
1
  ccv_nnc_cmd_exec(CMD_DATATYPE_CONVERSION_FORWARD(), ccv_nnc_no_hint, 0, TENSOR_LIST(tb), TENSOR_LIST(tb1), 0);
1513
1
  REQUIRE_ARRAY_EQ_WITH_TOLERANCE(float, tb1->data.f32, hb->data.f32, 64, 1e-2, "GPU computed output should be the same as CPU computed ones");
1514
1
  ccv_nnc_tensor_free(a);
1515
1
  ccv_nnc_tensor_free(w);
1516
1
  ccv_nnc_tensor_free(bias);
1517
1
  ccv_nnc_tensor_free(b);
1518
1
  ccv_nnc_tensor_free(tb);
1519
1
  ccv_nnc_tensor_free(ha);
1520
1
  ccv_nnc_tensor_free(ha1);
1521
1
  ccv_nnc_tensor_free(tb1);
1522
1
  ccv_nnc_tensor_free(hw);
1523
1
  ccv_nnc_tensor_free(hbias);
1524
1
  ccv_nnc_tensor_free(hb);
1525
1
  ccv_nnc_tensor_free(ha2);
1526
1
  ccv_nnc_tensor_free(hw2);
1527
1
  ccv_nnc_tensor_free(hbias2);
1528
1
}
1529
1530
TEST_CASE("cublas forward gemm no bias")
1531
1
{
1532
1
  GUARD_ELSE_RETURN(ccv_nnc_cmd_ok(CCV_NNC_GEMM_FORWARD, CCV_NNC_BACKEND_GPU_CUBLAS));
1533
1
  dsfmt_t dsfmt;
1534
1
  dsfmt_init_gen_rand(&dsfmt, 0);
1535
1
  ccv_nnc_tensor_t* a = ccv_nnc_tensor_new(0, GPU_TENSOR_NHWC(000, 32F, 10, 128), 0);
1536
1
  ccv_nnc_tensor_t* w = ccv_nnc_tensor_new(0, GPU_TENSOR_NHWC(000, 32F, 64, 128), 0);
1537
1
  ccv_nnc_tensor_t* b = ccv_nnc_tensor_new(0, GPU_TENSOR_NHWC(000, 32F, 10, 64), 0);
1538
1539
1
  ccv_nnc_tensor_t* ha = ccv_nnc_tensor_new(0, CPU_TENSOR_NHWC(32F, 1, 128), 0);
1540
1
  ccv_nnc_tensor_t* hw = ccv_nnc_tensor_new(0, CPU_TENSOR_NHWC(32F, 64, 128), 0);
1541
1
  ccv_nnc_tensor_t* hb = ccv_nnc_tensor_new(0, CPU_TENSOR_NHWC(32F, 1, 64), 0);
1542
1
  int i;
1543
8.19k
  for (i = 0; i < 64 * 128; 
i++8.19k
)
1544
8.19k
    hw->data.f32[i] = dsfmt_genrand_open_close(&dsfmt) / (64 * 128);
1545
1
  ccv_nnc_tensor_t* ha1 = ccv_nnc_tensor_new(0, CPU_TENSOR_NHWC(32F, 10, 128), 0);
1546
1.28k
  for (i = 0; i < 10 * 128; 
i++1.28k
)
1547
1.28k
    ha1->data.f32[i] = dsfmt_genrand_open_close(&dsfmt);
1548
129
  for (i = 0; i < 128; 
i++128
)
1549
128
    ha->data.f32[i] = ha1->data.f32[i];
1550
1
  ccv_nnc_cmd_exec(CMD_DATA_TRANSFER_FORWARD(), ccv_nnc_no_hint, 0, TENSOR_LIST(ha1, hw), TENSOR_LIST(a, w), 0);
1551
1
  ccv_nnc_cmd_exec(CMD_GEMM_FORWARD(NO_TRANSPOSE, TRANSPOSE(0, 1)), ccv_nnc_no_hint, 0, TENSOR_LIST(ha, hw), TENSOR_LIST(hb), 0);
1552
1
  ccv_nnc_cmd_exec(CMD_GEMM_FORWARD(NO_TRANSPOSE, TRANSPOSE(0, 1)), ccv_nnc_no_hint, 0, TENSOR_LIST(a, w), TENSOR_LIST(b), 0);
1553
1
  ccv_nnc_tensor_t* tb = ccv_nnc_tensor_new(0, CPU_TENSOR_NHWC(32F, 10, 64), 0);
1554
1
  ccv_nnc_cmd_exec(CMD_DATA_TRANSFER_FORWARD(), ccv_nnc_no_hint, 0, TENSOR_LIST(b), TENSOR_LIST(tb), 0);
1555
1
  ccv_nnc_tensor_t* tb1 = ccv_nnc_tensor_new(0, CPU_TENSOR_NHWC(32F, 1, 64), 0);
1556
65
  for (i = 0; i < 64; 
i++64
)
1557
64
    tb1->data.f32[i] = tb->data.f32[i];
1558
1
  REQUIRE_TENSOR_EQ(tb1, hb, "GPU computed output should be the same as CPU computed ones");
1559
1
  ccv_nnc_tensor_free(a);
1560
1
  ccv_nnc_tensor_free(w);
1561
1
  ccv_nnc_tensor_free(b);
1562
1
  ccv_nnc_tensor_free(tb);
1563
1
  ccv_nnc_tensor_free(ha);
1564
1
  ccv_nnc_tensor_free(ha1);
1565
1
  ccv_nnc_tensor_free(tb1);
1566
1
  ccv_nnc_tensor_free(hw);
1567
1
  ccv_nnc_tensor_free(hb);
1568
1
}
1569
1570
TEST_CASE("cublas forward gemm no bias in half precision")
1571
1
{
1572
1
  GUARD_ELSE_RETURN(ccv_nnc_cmd_ok(CCV_NNC_GEMM_FORWARD, CCV_NNC_BACKEND_GPU_CUBLAS));
1573
1
  dsfmt_t dsfmt;
1574
1
  dsfmt_init_gen_rand(&dsfmt, 0);
1575
1
  ccv_nnc_tensor_t* a = ccv_nnc_tensor_new(0, GPU_TENSOR_NHWC(000, 16F, 10, 128), 0);
1576
1
  ccv_nnc_tensor_t* w = ccv_nnc_tensor_new(0, GPU_TENSOR_NHWC(000, 16F, 64, 128), 0);
1577
1
  ccv_nnc_tensor_t* b = ccv_nnc_tensor_new(0, GPU_TENSOR_NHWC(000, 16F, 10, 64), 0);
1578
1579
1
  ccv_nnc_tensor_t* ha = ccv_nnc_tensor_new(0, CPU_TENSOR_NHWC(32F, 1, 128), 0);
1580
1
  ccv_nnc_tensor_t* hw = ccv_nnc_tensor_new(0, CPU_TENSOR_NHWC(32F, 64, 128), 0);
1581
1
  ccv_nnc_tensor_t* hb = ccv_nnc_tensor_new(0, CPU_TENSOR_NHWC(32F, 1, 64), 0);
1582
1
  int i;
1583
8.19k
  for (i = 0; i < 64 * 128; 
i++8.19k
)
1584
8.19k
    hw->data.f32[i] = dsfmt_genrand_open_close(&dsfmt) / (64 * 128);
1585
1
  ccv_nnc_tensor_t* ha1 = ccv_nnc_tensor_new(0, CPU_TENSOR_NHWC(32F, 10, 128), 0);
1586
1.28k
  for (i = 0; i < 10 * 128; 
i++1.28k
)
1587
1.28k
    ha1->data.f32[i] = dsfmt_genrand_open_close(&dsfmt);
1588
129
  for (i = 0; i < 128; 
i++128
)
1589
128
    ha->data.f32[i] = ha1->data.f32[i];
1590
1
  ccv_nnc_tensor_t* ha2 = ccv_nnc_tensor_new(0, CPU_TENSOR_NHWC(16F, 10, 128), 0);
1591
1
  ccv_nnc_tensor_t* hw2 = ccv_nnc_tensor_new(0, CPU_TENSOR_NHWC(16F, 64, 128), 0);
1592
1
  ccv_nnc_cmd_exec(CMD_DATATYPE_CONVERSION_FORWARD(), ccv_nnc_no_hint, 0, TENSOR_LIST(ha1, hw), TENSOR_LIST(ha2, hw2), 0);
1593
1
  ccv_nnc_cmd_exec(CMD_DATA_TRANSFER_FORWARD(), ccv_nnc_no_hint, 0, TENSOR_LIST(ha2, hw2), TENSOR_LIST(a, w), 0);
1594
1
  ccv_nnc_cmd_exec(CMD_GEMM_FORWARD(NO_TRANSPOSE, TRANSPOSE(0, 1)), ccv_nnc_no_hint, 0, TENSOR_LIST(ha, hw), TENSOR_LIST(hb), 0);
1595
1
  ccv_nnc_cmd_exec(CMD_GEMM_FORWARD(NO_TRANSPOSE, TRANSPOSE(0, 1)), ccv_nnc_no_hint, 0, TENSOR_LIST(a, w), TENSOR_LIST(b), 0);
1596
1
  ccv_nnc_tensor_t* tb = ccv_nnc_tensor_new(0, CPU_TENSOR_NHWC(16F, 10, 64), 0);
1597
1
  ccv_nnc_cmd_exec(CMD_DATA_TRANSFER_FORWARD(), ccv_nnc_no_hint, 0, TENSOR_LIST(b), TENSOR_LIST(tb), 0);
1598
1
  ccv_nnc_tensor_t* tb1 = ccv_nnc_tensor_new(0, CPU_TENSOR_NHWC(32F, 10, 64), 0);
1599
1
  ccv_nnc_cmd_exec(CMD_DATATYPE_CONVERSION_FORWARD(), ccv_nnc_no_hint, 0, TENSOR_LIST(tb), TENSOR_LIST(tb1), 0);
1600
1
  REQUIRE_ARRAY_EQ_WITH_TOLERANCE(float, tb1->data.f32, hb->data.f32, 64, 1e-3, "GPU computed output should be the same as CPU computed ones");
1601
1
  ccv_nnc_tensor_free(a);
1602
1
  ccv_nnc_tensor_free(w);
1603
1
  ccv_nnc_tensor_free(b);
1604
1
  ccv_nnc_tensor_free(tb);
1605
1
  ccv_nnc_tensor_free(ha);
1606
1
  ccv_nnc_tensor_free(ha1);
1607
1
  ccv_nnc_tensor_free(tb1);
1608
1
  ccv_nnc_tensor_free(hw);
1609
1
  ccv_nnc_tensor_free(hb);
1610
1
  ccv_nnc_tensor_free(ha2);
1611
1
  ccv_nnc_tensor_free(hw2);
1612
1
}
1613
1614
TEST_CASE("cublas forward gemv in half precision no bias, variant 1")
1615
1
{
1616
1
  GUARD_ELSE_RETURN(ccv_nnc_cmd_ok(CCV_NNC_GEMM_FORWARD, CCV_NNC_BACKEND_GPU_CUBLAS));
1617
1
  dsfmt_t dsfmt;
1618
1
  dsfmt_init_gen_rand(&dsfmt, 0);
1619
1
  ccv_nnc_tensor_t* a = ccv_nnc_tensor_new(0, GPU_TENSOR_NHWC(000, 16F, 1, 128), 0);
1620
1
  ccv_nnc_tensor_t* w = ccv_nnc_tensor_new(0, GPU_TENSOR_NHWC(000, 16F, 64, 128), 0);
1621
1
  ccv_nnc_tensor_t* b = ccv_nnc_tensor_new(0, GPU_TENSOR_NHWC(000, 16F, 1, 64), 0);
1622
1623
1
  ccv_nnc_tensor_t* ha = ccv_nnc_tensor_new(0, CPU_TENSOR_NHWC(32F, 1, 128), 0);
1624
1
  ccv_nnc_tensor_t* hw = ccv_nnc_tensor_new(0, CPU_TENSOR_NHWC(32F, 64, 128), 0);
1625
1
  ccv_nnc_tensor_t* hb = ccv_nnc_tensor_new(0, CPU_TENSOR_NHWC(32F, 1, 64), 0);
1626
1
  int i;
1627
8.19k
  for (i = 0; i < 64 * 128; 
i++8.19k
)
1628
8.19k
    hw->data.f32[i] = dsfmt_genrand_open_close(&dsfmt) / (64 * 128);
1629
1
  ccv_nnc_tensor_t* ha1 = ccv_nnc_tensor_new(0, CPU_TENSOR_NHWC(32F, 1, 128), 0);
1630
129
  for (i = 0; i < 128; 
i++128
)
1631
128
    ha1->data.f32[i] = dsfmt_genrand_open_close(&dsfmt);
1632
129
  for (i = 0; i < 128; 
i++128
)
1633
128
    ha->data.f32[i] = ha1->data.f32[i];
1634
1
  ccv_nnc_tensor_t* ha2 = ccv_nnc_tensor_new(0, CPU_TENSOR_NHWC(16F, 1, 128), 0);
1635
1
  ccv_nnc_tensor_t* hw2 = ccv_nnc_tensor_new(0, CPU_TENSOR_NHWC(16F, 64, 128), 0);
1636
1
  ccv_nnc_cmd_exec(CMD_DATATYPE_CONVERSION_FORWARD(), ccv_nnc_no_hint, 0, TENSOR_LIST(ha1, hw), TENSOR_LIST(ha2, hw2), 0);
1637
1
  ccv_nnc_cmd_exec(CMD_DATA_TRANSFER_FORWARD(), ccv_nnc_no_hint, 0, TENSOR_LIST(ha2, hw2), TENSOR_LIST(a, w), 0);
1638
1
  ccv_nnc_cmd_exec(CMD_GEMM_FORWARD(NO_TRANSPOSE, TRANSPOSE(0, 1)), ccv_nnc_no_hint, 0, TENSOR_LIST(ha, hw), TENSOR_LIST(hb), 0);
1639
1
  ccv_nnc_cmd_exec(CMD_GEMM_FORWARD(NO_TRANSPOSE, TRANSPOSE(0, 1)), ccv_nnc_no_hint, 0, TENSOR_LIST(a, w), TENSOR_LIST(b), 0);
1640
1
  ccv_nnc_tensor_t* tb = ccv_nnc_tensor_new(0, CPU_TENSOR_NHWC(16F, 1, 64), 0);
1641
1
  ccv_nnc_cmd_exec(CMD_DATA_TRANSFER_FORWARD(), ccv_nnc_no_hint, 0, TENSOR_LIST(b), TENSOR_LIST(tb), 0);
1642
1
  ccv_nnc_tensor_t* tb1 = ccv_nnc_tensor_new(0, CPU_TENSOR_NHWC(32F, 1, 64), 0);
1643
1
  ccv_nnc_cmd_exec(CMD_DATATYPE_CONVERSION_FORWARD(), ccv_nnc_no_hint, 0, TENSOR_LIST(tb), TENSOR_LIST(tb1), 0);
1644
1
  REQUIRE_ARRAY_EQ_WITH_TOLERANCE(float, tb1->data.f32, hb->data.f32, 64, 1e-3, "GPU computed output should be the same as CPU computed ones");
1645
1
  ccv_nnc_tensor_free(a);
1646
1
  ccv_nnc_tensor_free(w);
1647
1
  ccv_nnc_tensor_free(b);
1648
1
  ccv_nnc_tensor_free(tb);
1649
1
  ccv_nnc_tensor_free(ha);
1650
1
  ccv_nnc_tensor_free(ha1);
1651
1
  ccv_nnc_tensor_free(tb1);
1652
1
  ccv_nnc_tensor_free(hw);
1653
1
  ccv_nnc_tensor_free(hb);
1654
1
  ccv_nnc_tensor_free(ha2);
1655
1
  ccv_nnc_tensor_free(hw2);
1656
1
}
1657
1658
TEST_CASE("cublas forward gemv in half precision no bias, variant 2")
1659
1
{
1660
1
  GUARD_ELSE_RETURN(ccv_nnc_cmd_ok(CCV_NNC_GEMM_FORWARD, CCV_NNC_BACKEND_GPU_CUBLAS));
1661
1
  dsfmt_t dsfmt;
1662
1
  dsfmt_init_gen_rand(&dsfmt, 0);
1663
1
  ccv_nnc_tensor_t* w = ccv_nnc_tensor_new(0, GPU_TENSOR_NHWC(000, 16F, 64, 128), 0);
1664
1
  ccv_nnc_tensor_t* a = ccv_nnc_tensor_new(0, GPU_TENSOR_NHWC(000, 16F, 128, 1), 0);
1665
1
  ccv_nnc_tensor_t* b = ccv_nnc_tensor_new(0, GPU_TENSOR_NHWC(000, 16F, 64, 1), 0);
1666
1667
1
  ccv_nnc_tensor_t* hw = ccv_nnc_tensor_new(0, CPU_TENSOR_NHWC(32F, 64, 128), 0);
1668
1
  ccv_nnc_tensor_t* ha = ccv_nnc_tensor_new(0, CPU_TENSOR_NHWC(32F, 128, 1), 0);
1669
1
  ccv_nnc_tensor_t* hb = ccv_nnc_tensor_new(0, CPU_TENSOR_NHWC(32F, 64, 1), 0);
1670
1
  int i;
1671
8.19k
  for (i = 0; i < 64 * 128; 
i++8.19k
)
1672
8.19k
    hw->data.f32[i] = dsfmt_genrand_open_close(&dsfmt) / (64 * 128);
1673
1
  ccv_nnc_tensor_t* ha1 = ccv_nnc_tensor_new(0, CPU_TENSOR_NHWC(32F, 128, 1), 0);
1674
129
  for (i = 0; i < 128; 
i++128
)
1675
128
    ha1->data.f32[i] = dsfmt_genrand_open_close(&dsfmt);
1676
129
  for (i = 0; i < 128; 
i++128
)
1677
128
    ha->data.f32[i] = ha1->data.f32[i];
1678
1
  ccv_nnc_tensor_t* hw2 = ccv_nnc_tensor_new(0, CPU_TENSOR_NHWC(16F, 64, 128), 0);
1679
1
  ccv_nnc_tensor_t* ha2 = ccv_nnc_tensor_new(0, CPU_TENSOR_NHWC(16F, 128, 1), 0);
1680
1
  ccv_nnc_cmd_exec(CMD_DATATYPE_CONVERSION_FORWARD(), ccv_nnc_no_hint, 0, TENSOR_LIST(ha1, hw), TENSOR_LIST(ha2, hw2), 0);
1681
1
  ccv_nnc_cmd_exec(CMD_DATA_TRANSFER_FORWARD(), ccv_nnc_no_hint, 0, TENSOR_LIST(ha2, hw2), TENSOR_LIST(a, w), 0);
1682
1
  ccv_nnc_cmd_exec(CMD_GEMM_FORWARD(NO_TRANSPOSE, NO_TRANSPOSE), ccv_nnc_no_hint, 0, TENSOR_LIST(hw, ha), TENSOR_LIST(hb), 0);
1683
1
  ccv_nnc_cmd_exec(CMD_GEMM_FORWARD(NO_TRANSPOSE, NO_TRANSPOSE), ccv_nnc_no_hint, 0, TENSOR_LIST(w, a), TENSOR_LIST(b), 0);
1684
1
  ccv_nnc_tensor_t* tb = ccv_nnc_tensor_new(0, CPU_TENSOR_NHWC(16F, 64, 1), 0);
1685
1
  ccv_nnc_cmd_exec(CMD_DATA_TRANSFER_FORWARD(), ccv_nnc_no_hint, 0, TENSOR_LIST(b), TENSOR_LIST(tb), 0);
1686
1
  ccv_nnc_tensor_t* tb1 = ccv_nnc_tensor_new(0, CPU_TENSOR_NHWC(32F, 64, 1), 0);
1687
1
  ccv_nnc_cmd_exec(CMD_DATATYPE_CONVERSION_FORWARD(), ccv_nnc_no_hint, 0, TENSOR_LIST(tb), TENSOR_LIST(tb1), 0);
1688
1
  REQUIRE_ARRAY_EQ_WITH_TOLERANCE(float, tb1->data.f32, hb->data.f32, 64, 1e-3, "GPU computed output should be the same as CPU computed ones");
1689
1
  ccv_nnc_tensor_free(a);
1690
1
  ccv_nnc_tensor_free(w);
1691
1
  ccv_nnc_tensor_free(b);
1692
1
  ccv_nnc_tensor_free(tb);
1693
1
  ccv_nnc_tensor_free(ha);
1694
1
  ccv_nnc_tensor_free(ha1);
1695
1
  ccv_nnc_tensor_free(tb1);
1696
1
  ccv_nnc_tensor_free(hw);
1697
1
  ccv_nnc_tensor_free(hb);
1698
1
  ccv_nnc_tensor_free(ha2);
1699
1
  ccv_nnc_tensor_free(hw2);
1700
1
}
1701
1702
TEST_CASE("cublas backward gemm")
1703
1
{
1704
1
  GUARD_ELSE_RETURN(ccv_nnc_cmd_ok(CCV_NNC_GEMM_FORWARD, CCV_NNC_BACKEND_GPU_CUBLAS) &&
1705
1
    ccv_nnc_cmd_ok(CCV_NNC_GEMM_BACKWARD, CCV_NNC_BACKEND_GPU_CUBLAS));
1706
1
  dsfmt_t dsfmt;
1707
1
  dsfmt_init_gen_rand(&dsfmt, 0);
1708
1
  ccv_nnc_tensor_t* a = ccv_nnc_tensor_new(0, GPU_TENSOR_NHWC(000, 32F, 10, 128), 0);
1709
1
  ccv_nnc_tensor_t* w = ccv_nnc_tensor_new(0, GPU_TENSOR_NHWC(000, 32F, 64, 128), 0);
1710
1
  ccv_nnc_tensor_t* bias = ccv_nnc_tensor_new(0, GPU_TENSOR_NHWC(000, 32F, 64), 0);
1711
1
  ccv_nnc_tensor_t* b = ccv_nnc_tensor_new(0, GPU_TENSOR_NHWC(000, 32F, 10, 64), 0);
1712
1
  ccv_nnc_tensor_t* g = ccv_nnc_tensor_new(0, GPU_TENSOR_NHWC(000, 32F, 10, 64), 0);
1713
1
  ccv_nnc_tensor_t* dw = ccv_nnc_tensor_new(0, GPU_TENSOR_NHWC(000, 32F, 64, 128), 0);
1714
1
  ccv_nnc_tensor_t* dbias = ccv_nnc_tensor_new(0, GPU_TENSOR_NHWC(000, 32F, 64), 0);
1715
1
  ccv_nnc_tensor_t* h = ccv_nnc_tensor_new(0, GPU_TENSOR_NHWC(000, 32F, 10, 128), 0);
1716
1717
1
  ccv_nnc_tensor_t* ha = ccv_nnc_tensor_new(0, CPU_TENSOR_NHWC(32F, 10, 128), 0);
1718
1
  ccv_nnc_tensor_t* hw = ccv_nnc_tensor_new(0, CPU_TENSOR_NHWC(32F, 64, 128), 0);
1719
1
  ccv_nnc_tensor_t* hbias = ccv_nnc_tensor_new(0, CPU_TENSOR_NHWC(32F, 64), 0);
1720
1
  ccv_nnc_tensor_t* hb = ccv_nnc_tensor_new(0, CPU_TENSOR_NHWC(32F, 10, 64), 0);
1721
1
  ccv_nnc_tensor_t* hg = ccv_nnc_tensor_new(0, CPU_TENSOR_NHWC(32F, 10, 64), 0);
1722
1
  ccv_nnc_tensor_t* hdw = ccv_nnc_tensor_new(0, CPU_TENSOR_NHWC(32F, 64, 128), 0);
1723
1
  ccv_nnc_tensor_t* hdbias = ccv_nnc_tensor_new(0, CPU_TENSOR_NHWC(32F, 64), 0);
1724
1
  ccv_nnc_tensor_t* hh = ccv_nnc_tensor_new(0, CPU_TENSOR_NHWC(32F, 10, 128), 0);
1725
1
  int i;
1726
8.19k
  for (i = 0; i < 64 * 128; 
i++8.19k
)
1727
8.19k
    hw->data.f32[i] = dsfmt_genrand_open_close(&dsfmt) / (64 * 128);
1728
65
  for (i = 0; i < 64; 
i++64
)
1729
64
    hbias->data.f32[i] = dsfmt_genrand_open_close(&dsfmt);
1730
1.28k
  for (i = 0; i < 10 * 128; 
i++1.28k
)
1731
1.28k
    ha->data.f32[i] = dsfmt_genrand_open_close(&dsfmt);
1732
641
  for (i = 0; i < 10 * 64; 
i++640
)
1733
640
    hg->data.f32[i] = dsfmt_genrand_open_close(&dsfmt);
1734
1
  ccv_nnc_cmd_exec(CMD_DATA_TRANSFER_FORWARD(), ccv_nnc_no_hint, 0, TENSOR_LIST(ha, hw, hbias, hg), TENSOR_LIST(a, w, bias, g), 0);
1735
1
  ccv_nnc_cmd_exec(CMD_GEMM_FORWARD(NO_TRANSPOSE, TRANSPOSE(0, 1)), ccv_nnc_no_hint, 0, TENSOR_LIST(ha, hw, hbias), TENSOR_LIST(hb), 0);
1736
1
  ccv_nnc_cmd_exec(CMD_GEMM_BACKWARD(NO_TRANSPOSE, TRANSPOSE(0, 1)), ccv_nnc_no_hint, 0, TENSOR_LIST(hg, ha, hw, 0), TENSOR_LIST(hh, hdw, hdbias), 0);
1737
1
  ccv_nnc_cmd_exec(CMD_GEMM_FORWARD(NO_TRANSPOSE, TRANSPOSE(0, 1)), ccv_nnc_no_hint, 0, TENSOR_LIST(a, w, bias), TENSOR_LIST(b), 0);
1738
1
  ccv_nnc_cmd_exec(CMD_GEMM_BACKWARD(NO_TRANSPOSE, TRANSPOSE(0, 1)), ccv_nnc_no_hint, 0, TENSOR_LIST(g, a, w, 0), TENSOR_LIST(h, dw, dbias), 0);
1739
1
  ccv_nnc_tensor_t* tb = ccv_nnc_tensor_new(0, CPU_TENSOR_NHWC(32F, 10, 64), 0);
1740
1
  ccv_nnc_tensor_t* tdw = ccv_nnc_tensor_new(0, CPU_TENSOR_NHWC(32F, 64, 128), 0);
1741
1
  ccv_nnc_tensor_t* tdbias = ccv_nnc_tensor_new(0, CPU_TENSOR_NHWC(32F, 64), 0);
1742
1
  ccv_nnc_tensor_t* th = ccv_nnc_tensor_new(0, CPU_TENSOR_NHWC(32F, 10, 128), 0);
1743
1
  ccv_nnc_cmd_exec(CMD_DATA_TRANSFER_FORWARD(), ccv_nnc_no_hint, 0, TENSOR_LIST(b, dw, dbias, h), TENSOR_LIST(tb, tdw, tdbias, th), 0);
1744
1
  REQUIRE_TENSOR_EQ(tb, hb, "GPU computed output should be the same as CPU computed ones");
1745
1
  REQUIRE_TENSOR_EQ(tdw, hdw, "GPU computed output should be the same as CPU computed ones");
1746
1
  REQUIRE_TENSOR_EQ(tdbias, hdbias, "GPU computed output should be the same as CPU computed ones");
1747
1
  REQUIRE_TENSOR_EQ(th, hh, "GPU computed output should be the same as CPU computed ones");
1748
1
  ccv_nnc_tensor_free(a);
1749
1
  ccv_nnc_tensor_free(w);
1750
1
  ccv_nnc_tensor_free(bias);
1751
1
  ccv_nnc_tensor_free(b);
1752
1
  ccv_nnc_tensor_free(g);
1753
1
  ccv_nnc_tensor_free(dw);
1754
1
  ccv_nnc_tensor_free(dbias);
1755
1
  ccv_nnc_tensor_free(h);
1756
1
  ccv_nnc_tensor_free(ha);
1757
1
  ccv_nnc_tensor_free(hw);
1758
1
  ccv_nnc_tensor_free(hbias);
1759
1
  ccv_nnc_tensor_free(hb);
1760
1
  ccv_nnc_tensor_free(hg);
1761
1
  ccv_nnc_tensor_free(hdw);
1762
1
  ccv_nnc_tensor_free(hdbias);
1763
1
  ccv_nnc_tensor_free(hh);
1764
1
  ccv_nnc_tensor_free(tb);
1765
1
  ccv_nnc_tensor_free(th);
1766
1
  ccv_nnc_tensor_free(tdw);
1767
1
  ccv_nnc_tensor_free(tdbias);
1768
1
}
1769
1770
TEST_CASE("cublas backward gemm in half precision")
1771
1
{
1772
1
  GUARD_ELSE_RETURN(ccv_nnc_cmd_ok(CCV_NNC_GEMM_FORWARD, CCV_NNC_BACKEND_GPU_CUBLAS) &&
1773
1
    ccv_nnc_cmd_ok(CCV_NNC_GEMM_BACKWARD, CCV_NNC_BACKEND_GPU_CUBLAS));
1774
1
  dsfmt_t dsfmt;
1775
1
  dsfmt_init_gen_rand(&dsfmt, 0);
1776
1
  ccv_nnc_tensor_t* a = ccv_nnc_tensor_new(0, GPU_TENSOR_NHWC(000, 16F, 10, 128), 0);
1777
1
  ccv_nnc_tensor_t* w = ccv_nnc_tensor_new(0, GPU_TENSOR_NHWC(000, 16F, 64, 128), 0);
1778
1
  ccv_nnc_tensor_t* bias = ccv_nnc_tensor_new(0, GPU_TENSOR_NHWC(000, 16F, 64), 0);
1779
1
  ccv_nnc_tensor_t* b = ccv_nnc_tensor_new(0, GPU_TENSOR_NHWC(000, 16F, 10, 64), 0);
1780
1
  ccv_nnc_tensor_t* g = ccv_nnc_tensor_new(0, GPU_TENSOR_NHWC(000, 16F, 10, 64), 0);
1781
1
  ccv_nnc_tensor_t* dw = ccv_nnc_tensor_new(0, GPU_TENSOR_NHWC(000, 16F, 64, 128), 0);
1782
1
  ccv_nnc_tensor_t* dbias = ccv_nnc_tensor_new(0, GPU_TENSOR_NHWC(000, 16F, 64), 0);
1783
1
  ccv_nnc_tensor_t* h = ccv_nnc_tensor_new(0, GPU_TENSOR_NHWC(000, 16F, 10, 128), 0);
1784
1785
1
  ccv_nnc_tensor_t* ha = ccv_nnc_tensor_new(0, CPU_TENSOR_NHWC(32F, 10, 128), 0);
1786
1
  ccv_nnc_tensor_t* hw = ccv_nnc_tensor_new(0, CPU_TENSOR_NHWC(32F, 64, 128), 0);
1787
1
  ccv_nnc_tensor_t* hbias = ccv_nnc_tensor_new(0, CPU_TENSOR_NHWC(32F, 64), 0);
1788
1
  ccv_nnc_tensor_t* hb = ccv_nnc_tensor_new(0, CPU_TENSOR_NHWC(32F, 10, 64), 0);
1789
1
  ccv_nnc_tensor_t* hg = ccv_nnc_tensor_new(0, CPU_TENSOR_NHWC(32F, 10, 64), 0);
1790
1
  ccv_nnc_tensor_t* hdw = ccv_nnc_tensor_new(0, CPU_TENSOR_NHWC(32F, 64, 128), 0);
1791
1
  ccv_nnc_tensor_t* hdbias = ccv_nnc_tensor_new(0, CPU_TENSOR_NHWC(32F, 64), 0);
1792
1
  ccv_nnc_tensor_t* hh = ccv_nnc_tensor_new(0, CPU_TENSOR_NHWC(32F, 10, 128), 0);
1793
1
  int i;
1794
8.19k
  for (i = 0; i < 64 * 128; 
i++8.19k
)
1795
8.19k
    hw->data.f32[i] = dsfmt_genrand_open_close(&dsfmt) / (64 * 128);
1796
65
  for (i = 0; i < 64; 
i++64
)
1797
64
    hbias->data.f32[i] = dsfmt_genrand_open_close(&dsfmt);
1798
1.28k
  for (i = 0; i < 10 * 128; 
i++1.28k
)
1799
1.28k
    ha->data.f32[i] = dsfmt_genrand_open_close(&dsfmt);
1800
641
  for (i = 0; i < 10 * 64; 
i++640
)
1801
640
    hg->data.f32[i] = dsfmt_genrand_open_close(&dsfmt);
1802
1
  ccv_nnc_tensor_t* ha2 = ccv_nnc_tensor_new(0, CPU_TENSOR_NHWC(16F, 10, 128), 0);
1803
1
  ccv_nnc_tensor_t* hw2 = ccv_nnc_tensor_new(0, CPU_TENSOR_NHWC(16F, 64, 128), 0);
1804
1
  ccv_nnc_tensor_t* hbias2 = ccv_nnc_tensor_new(0, CPU_TENSOR_NHWC(16F, 64), 0);
1805
1
  ccv_nnc_tensor_t* hg2 = ccv_nnc_tensor_new(0, CPU_TENSOR_NHWC(16F, 10, 64), 0);
1806
1
  ccv_nnc_cmd_exec(CMD_DATATYPE_CONVERSION_FORWARD(), ccv_nnc_no_hint, 0, TENSOR_LIST(ha, hw, hbias, hg), TENSOR_LIST(ha2, hw2, hbias2, hg2), 0);
1807
1
  ccv_nnc_cmd_exec(CMD_DATA_TRANSFER_FORWARD(), ccv_nnc_no_hint, 0, TENSOR_LIST(ha2, hw2, hbias2, hg2), TENSOR_LIST(a, w, bias, g), 0);
1808
1
  ccv_nnc_cmd_exec(CMD_GEMM_FORWARD(NO_TRANSPOSE, TRANSPOSE(0, 1)), ccv_nnc_no_hint, 0, TENSOR_LIST(ha, hw, hbias), TENSOR_LIST(hb), 0);
1809
1
  ccv_nnc_cmd_exec(CMD_GEMM_BACKWARD(NO_TRANSPOSE, TRANSPOSE(0, 1)), ccv_nnc_no_hint, 0, TENSOR_LIST(hg, ha, hw, 0), TENSOR_LIST(hh, hdw, hdbias), 0);
1810
1
  ccv_nnc_cmd_exec(CMD_GEMM_FORWARD(NO_TRANSPOSE, TRANSPOSE(0, 1)), ccv_nnc_no_hint, 0, TENSOR_LIST(a, w, bias), TENSOR_LIST(b), 0);
1811
1
  ccv_nnc_cmd_exec(CMD_GEMM_BACKWARD(NO_TRANSPOSE, TRANSPOSE(0, 1)), ccv_nnc_no_hint, 0, TENSOR_LIST(g, a, w, 0), TENSOR_LIST(h, dw, dbias), 0);
1812
1
  ccv_nnc_tensor_t* tb = ccv_nnc_tensor_new(0, CPU_TENSOR_NHWC(16F, 10, 64), 0);
1813
1
  ccv_nnc_tensor_t* tdw = ccv_nnc_tensor_new(0, CPU_TENSOR_NHWC(16F, 64, 128), 0);
1814
1
  ccv_nnc_tensor_t* tdbias = ccv_nnc_tensor_new(0, CPU_TENSOR_NHWC(16F, 64), 0);
1815
1
  ccv_nnc_tensor_t* th = ccv_nnc_tensor_new(0, CPU_TENSOR_NHWC(16F, 10, 128), 0);
1816
1
  ccv_nnc_cmd_exec(CMD_DATA_TRANSFER_FORWARD(), ccv_nnc_no_hint, 0, TENSOR_LIST(b, dw, dbias, h), TENSOR_LIST(tb, tdw, tdbias, th), 0);
1817
1
  ccv_nnc_tensor_t* tb1 = ccv_nnc_tensor_new(0, CPU_TENSOR_NHWC(32F, 10, 64), 0);
1818
1
  ccv_nnc_tensor_t* tdw1 = ccv_nnc_tensor_new(0, CPU_TENSOR_NHWC(32F, 64, 128), 0);
1819
1
  ccv_nnc_tensor_t* tdbias1 = ccv_nnc_tensor_new(0, CPU_TENSOR_NHWC(32F, 64), 0);
1820
1
  ccv_nnc_tensor_t* th1 = ccv_nnc_tensor_new(0, CPU_TENSOR_NHWC(32F, 10, 128), 0);
1821
1
  ccv_nnc_cmd_exec(CMD_DATATYPE_CONVERSION_FORWARD(), ccv_nnc_no_hint, 0, TENSOR_LIST(tb, tdw, tdbias, th), TENSOR_LIST(tb1, tdw1, tdbias1, th1), 0);
1822
1
  REQUIRE_ARRAY_EQ_WITH_TOLERANCE(float, tb1->data.f32, hb->data.f32, 10 * 64, 1e-3, "GPU computed output should be the same as CPU computed ones");
1823
1
  REQUIRE_ARRAY_EQ_WITH_TOLERANCE(float, tdw1->data.f32, hdw->data.f32, 64 * 128, 1e-2, "GPU computed output should be the same as CPU computed ones");
1824
1
  REQUIRE_ARRAY_EQ_WITH_TOLERANCE(float, tdbias1->data.f32, hdbias->data.f32, 64, 1e-2, "GPU computed output should be the same as CPU computed ones");
1825
1
  REQUIRE_ARRAY_EQ_WITH_TOLERANCE(float, th1->data.f32, hh->data.f32, 10 * 128, 1e-3, "GPU computed output should be the same as CPU computed ones");
1826
1
  ccv_nnc_tensor_free(a);
1827
1
  ccv_nnc_tensor_free(w);
1828
1
  ccv_nnc_tensor_free(bias);
1829
1
  ccv_nnc_tensor_free(b);
1830
1
  ccv_nnc_tensor_free(g);
1831
1
  ccv_nnc_tensor_free(dw);
1832
1
  ccv_nnc_tensor_free(dbias);
1833
1
  ccv_nnc_tensor_free(h);
1834
1
  ccv_nnc_tensor_free(ha);
1835
1
  ccv_nnc_tensor_free(hw);
1836
1
  ccv_nnc_tensor_free(hbias);
1837
1
  ccv_nnc_tensor_free(hb);
1838
1
  ccv_nnc_tensor_free(hg);
1839
1
  ccv_nnc_tensor_free(hdw);
1840
1
  ccv_nnc_tensor_free(hdbias);
1841
1
  ccv_nnc_tensor_free(hh);
1842
1
  ccv_nnc_tensor_free(tb);
1843
1
  ccv_nnc_tensor_free(th);
1844
1
  ccv_nnc_tensor_free(tdw);
1845
1
  ccv_nnc_tensor_free(tdbias);
1846
1
  ccv_nnc_tensor_free(ha2);
1847
1
  ccv_nnc_tensor_free(hw2);
1848
1
  ccv_nnc_tensor_free(hbias2);
1849
1
  ccv_nnc_tensor_free(hg2);
1850
1
  ccv_nnc_tensor_free(tb1);
1851
1
  ccv_nnc_tensor_free(tdw1);
1852
1
  ccv_nnc_tensor_free(tdbias1);
1853
1
  ccv_nnc_tensor_free(th1);
1854
1
}
1855
1856
TEST_CASE("cublas backward gemm no bias")
1857
1
{
1858
1
  GUARD_ELSE_RETURN(ccv_nnc_cmd_ok(CCV_NNC_GEMM_FORWARD, CCV_NNC_BACKEND_GPU_CUBLAS) &&
1859
1
    ccv_nnc_cmd_ok(CCV_NNC_GEMM_BACKWARD, CCV_NNC_BACKEND_GPU_CUBLAS));
1860
1
  dsfmt_t dsfmt;
1861
1
  dsfmt_init_gen_rand(&dsfmt, 0);
1862
1
  ccv_nnc_tensor_t* a = ccv_nnc_tensor_new(0, GPU_TENSOR_NHWC(000, 32F, 10, 128), 0);
1863
1
  ccv_nnc_tensor_t* w = ccv_nnc_tensor_new(0, GPU_TENSOR_NHWC(000, 32F, 64, 128), 0);
1864
1
  ccv_nnc_tensor_t* b = ccv_nnc_tensor_new(0, GPU_TENSOR_NHWC(000, 32F, 10, 64), 0);
1865
1
  ccv_nnc_tensor_t* g = ccv_nnc_tensor_new(0, GPU_TENSOR_NHWC(000, 32F, 10, 64), 0);
1866
1
  ccv_nnc_tensor_t* dw = ccv_nnc_tensor_new(0, GPU_TENSOR_NHWC(000, 32F, 64, 128), 0);
1867
1
  ccv_nnc_tensor_t* h = ccv_nnc_tensor_new(0, GPU_TENSOR_NHWC(000, 32F, 10, 128), 0);
1868
1869
1
  ccv_nnc_tensor_t* ha = ccv_nnc_tensor_new(0, CPU_TENSOR_NHWC(32F, 10, 128), 0);
1870
1
  ccv_nnc_tensor_t* hw = ccv_nnc_tensor_new(0, CPU_TENSOR_NHWC(32F, 64, 128), 0);
1871
1
  ccv_nnc_tensor_t* hb = ccv_nnc_tensor_new(0, CPU_TENSOR_NHWC(32F, 10, 64), 0);
1872
1
  ccv_nnc_tensor_t* hg = ccv_nnc_tensor_new(0, CPU_TENSOR_NHWC(32F, 10, 64), 0);
1873
1
  ccv_nnc_tensor_t* hdw = ccv_nnc_tensor_new(0, CPU_TENSOR_NHWC(32F, 64, 128), 0);
1874
1
  ccv_nnc_tensor_t* hh = ccv_nnc_tensor_new(0, CPU_TENSOR_NHWC(32F, 10, 128), 0);
1875
1
  int i;
1876
8.19k
  for (i = 0; i < 64 * 128; 
i++8.19k
)
1877
8.19k
    hw->data.f32[i] = dsfmt_genrand_open_close(&dsfmt) / (64 * 128);
1878
1.28k
  for (i = 0; i < 10 * 128; 
i++1.28k
)
1879
1.28k
    ha->data.f32[i] = dsfmt_genrand_open_close(&dsfmt);
1880
641
  for (i = 0; i < 10 * 64; 
i++640
)
1881
640
    hg->data.f32[i] = dsfmt_genrand_open_close(&dsfmt);
1882
1
  ccv_nnc_cmd_exec(CMD_DATA_TRANSFER_FORWARD(), ccv_nnc_no_hint, 0, TENSOR_LIST(ha, hw, hg), TENSOR_LIST(a, w, g), 0);
1883
1
  ccv_nnc_cmd_exec(CMD_GEMM_FORWARD(NO_TRANSPOSE, TRANSPOSE(0, 1)), ccv_nnc_no_hint, 0, TENSOR_LIST(ha, hw), TENSOR_LIST(hb), 0);
1884
1
  ccv_nnc_cmd_exec(CMD_GEMM_BACKWARD(NO_TRANSPOSE, TRANSPOSE(0, 1)), ccv_nnc_no_hint, 0, TENSOR_LIST(hg, ha, hw, 0), TENSOR_LIST(hh, hdw, 0), 0);
1885
1
  ccv_nnc_cmd_exec(CMD_GEMM_FORWARD(NO_TRANSPOSE, TRANSPOSE(0, 1)), ccv_nnc_no_hint, 0, TENSOR_LIST(a, w), TENSOR_LIST(b), 0);
1886
1
  ccv_nnc_cmd_exec(CMD_GEMM_BACKWARD(NO_TRANSPOSE, TRANSPOSE(0, 1)), ccv_nnc_no_hint, 0, TENSOR_LIST(g, a, w, 0), TENSOR_LIST(h, dw, 0), 0);
1887
1
  ccv_nnc_tensor_t* tb = ccv_nnc_tensor_new(0, CPU_TENSOR_NHWC(32F, 10, 64), 0);
1888
1
  ccv_nnc_tensor_t* tdw = ccv_nnc_tensor_new(0, CPU_TENSOR_NHWC(32F, 64, 128), 0);
1889
1
  ccv_nnc_tensor_t* th = ccv_nnc_tensor_new(0, CPU_TENSOR_NHWC(32F, 10, 128), 0);
1890
1
  ccv_nnc_cmd_exec(CMD_DATA_TRANSFER_FORWARD(), ccv_nnc_no_hint, 0, TENSOR_LIST(b, dw, h), TENSOR_LIST(tb, tdw, th), 0);
1891
1
  REQUIRE_TENSOR_EQ(tb, hb, "GPU computed output should be the same as CPU computed ones");
1892
1
  REQUIRE_TENSOR_EQ(tdw, hdw, "GPU computed output should be the same as CPU computed ones");
1893
1
  REQUIRE_TENSOR_EQ(th, hh, "GPU computed output should be the same as CPU computed ones");
1894
1
  ccv_nnc_tensor_free(a);
1895
1
  ccv_nnc_tensor_free(w);
1896
1
  ccv_nnc_tensor_free(b);
1897
1
  ccv_nnc_tensor_free(g);
1898
1
  ccv_nnc_tensor_free(dw);
1899
1
  ccv_nnc_tensor_free(h);
1900
1
  ccv_nnc_tensor_free(ha);
1901
1
  ccv_nnc_tensor_free(hw);
1902
1
  ccv_nnc_tensor_free(hb);
1903
1
  ccv_nnc_tensor_free(hg);
1904
1
  ccv_nnc_tensor_free(hdw);
1905
1
  ccv_nnc_tensor_free(hh);
1906
1
  ccv_nnc_tensor_free(tb);
1907
1
  ccv_nnc_tensor_free(th);
1908
1
  ccv_nnc_tensor_free(tdw);
1909
1
}
1910
1911
TEST_CASE("cublas backward gemm no bias in half precision")
1912
1
{
1913
1
  GUARD_ELSE_RETURN(ccv_nnc_cmd_ok(CCV_NNC_GEMM_FORWARD, CCV_NNC_BACKEND_GPU_CUBLAS) &&
1914
1
    ccv_nnc_cmd_ok(CCV_NNC_GEMM_BACKWARD, CCV_NNC_BACKEND_GPU_CUBLAS));
1915
1
  dsfmt_t dsfmt;
1916
1
  dsfmt_init_gen_rand(&dsfmt, 0);
1917
1
  ccv_nnc_tensor_t* a = ccv_nnc_tensor_new(0, GPU_TENSOR_NHWC(000, 16F, 10, 128), 0);
1918
1
  ccv_nnc_tensor_t* w = ccv_nnc_tensor_new(0, GPU_TENSOR_NHWC(000, 16F, 64, 128), 0);
1919
1
  ccv_nnc_tensor_t* b = ccv_nnc_tensor_new(0, GPU_TENSOR_NHWC(000, 16F, 10, 64), 0);
1920
1
  ccv_nnc_tensor_t* g = ccv_nnc_tensor_new(0, GPU_TENSOR_NHWC(000, 16F, 10, 64), 0);
1921
1
  ccv_nnc_tensor_t* dw = ccv_nnc_tensor_new(0, GPU_TENSOR_NHWC(000, 16F, 64, 128), 0);
1922
1
  ccv_nnc_tensor_t* h = ccv_nnc_tensor_new(0, GPU_TENSOR_NHWC(000, 16F, 10, 128), 0);
1923
1924
1
  ccv_nnc_tensor_t* ha = ccv_nnc_tensor_new(0, CPU_TENSOR_NHWC(32F, 10, 128), 0);
1925
1
  ccv_nnc_tensor_t* hw = ccv_nnc_tensor_new(0, CPU_TENSOR_NHWC(32F, 64, 128), 0);
1926
1
  ccv_nnc_tensor_t* hb = ccv_nnc_tensor_new(0, CPU_TENSOR_NHWC(32F, 10, 64), 0);
1927
1
  ccv_nnc_tensor_t* hg = ccv_nnc_tensor_new(0, CPU_TENSOR_NHWC(32F, 10, 64), 0);
1928
1
  ccv_nnc_tensor_t* hdw = ccv_nnc_tensor_new(0, CPU_TENSOR_NHWC(32F, 64, 128), 0);
1929
1
  ccv_nnc_tensor_t* hh = ccv_nnc_tensor_new(0, CPU_TENSOR_NHWC(32F, 10, 128), 0);
1930
1
  int i;
1931
8.19k
  for (i = 0; i < 64 * 128; 
i++8.19k
)
1932
8.19k
    hw->data.f32[i] = dsfmt_genrand_open_close(&dsfmt) / (64 * 128);
1933
1.28k
  for (i = 0; i < 10 * 128; 
i++1.28k
)
1934
1.28k
    ha->data.f32[i] = dsfmt_genrand_open_close(&dsfmt);
1935
641
  for (i = 0; i < 10 * 64; 
i++640
)
1936
640
    hg->data.f32[i] = dsfmt_genrand_open_close(&dsfmt);
1937
1
  ccv_nnc_tensor_t* ha2 = ccv_nnc_tensor_new(0, CPU_TENSOR_NHWC(16F, 10, 128), 0);
1938
1
  ccv_nnc_tensor_t* hw2 = ccv_nnc_tensor_new(0, CPU_TENSOR_NHWC(16F, 64, 128), 0);
1939
1
  ccv_nnc_tensor_t* hg2 = ccv_nnc_tensor_new(0, CPU_TENSOR_NHWC(16F, 10, 64), 0);
1940
1
  ccv_nnc_cmd_exec(CMD_DATATYPE_CONVERSION_FORWARD(), ccv_nnc_no_hint, 0, TENSOR_LIST(ha, hw, hg), TENSOR_LIST(ha2, hw2, hg2), 0);
1941
1
  ccv_nnc_cmd_exec(CMD_DATA_TRANSFER_FORWARD(), ccv_nnc_no_hint, 0, TENSOR_LIST(ha2, hw2, hg2), TENSOR_LIST(a, w, g), 0);
1942
1
  ccv_nnc_cmd_exec(CMD_GEMM_FORWARD(NO_TRANSPOSE, TRANSPOSE(0, 1)), ccv_nnc_no_hint, 0, TENSOR_LIST(ha, hw), TENSOR_LIST(hb), 0);
1943
1
  ccv_nnc_cmd_exec(CMD_GEMM_BACKWARD(NO_TRANSPOSE, TRANSPOSE(0, 1)), ccv_nnc_no_hint, 0, TENSOR_LIST(hg, ha, hw, 0), TENSOR_LIST(hh, hdw, 0), 0);
1944
1
  ccv_nnc_cmd_exec(CMD_GEMM_FORWARD(NO_TRANSPOSE, TRANSPOSE(0, 1)), ccv_nnc_no_hint, 0, TENSOR_LIST(a, w), TENSOR_LIST(b), 0);
1945
1
  ccv_nnc_cmd_exec(CMD_GEMM_BACKWARD(NO_TRANSPOSE, TRANSPOSE(0, 1)), ccv_nnc_no_hint, 0, TENSOR_LIST(g, a, w, 0), TENSOR_LIST(h, dw, 0), 0);
1946
1
  ccv_nnc_tensor_t* tb = ccv_nnc_tensor_new(0, CPU_TENSOR_NHWC(16F, 10, 64), 0);
1947
1
  ccv_nnc_tensor_t* tdw = ccv_nnc_tensor_new(0, CPU_TENSOR_NHWC(16F, 64, 128), 0);
1948
1
  ccv_nnc_tensor_t* th = ccv_nnc_tensor_new(0, CPU_TENSOR_NHWC(16F, 10, 128), 0);
1949
1
  ccv_nnc_tensor_t* tb1 = ccv_nnc_tensor_new(0, CPU_TENSOR_NHWC(32F, 10, 64), 0);
1950
1
  ccv_nnc_tensor_t* tdw1 = ccv_nnc_tensor_new(0, CPU_TENSOR_NHWC(32F, 64, 128), 0);
1951
1
  ccv_nnc_tensor_t* th1 = ccv_nnc_tensor_new(0, CPU_TENSOR_NHWC(32F, 10, 128), 0);
1952
1
  ccv_nnc_cmd_exec(CMD_DATA_TRANSFER_FORWARD(), ccv_nnc_no_hint, 0, TENSOR_LIST(b, dw, h), TENSOR_LIST(tb, tdw, th), 0);
1953
1
  ccv_nnc_cmd_exec(CMD_DATATYPE_CONVERSION_FORWARD(), ccv_nnc_no_hint, 0, TENSOR_LIST(tb, tdw, th), TENSOR_LIST(tb1, tdw1, th1), 0);
1954
1
  REQUIRE_ARRAY_EQ_WITH_TOLERANCE(float, tb1->data.f32, hb->data.f32, 10 * 64, 1e-3, "GPU computed output should be the same as CPU computed ones");
1955
1
  REQUIRE_ARRAY_EQ_WITH_TOLERANCE(float, tdw1->data.f32, hdw->data.f32, 64 * 128, 1e-2, "GPU computed output should be the same as CPU computed ones");
1956
1
  REQUIRE_ARRAY_EQ_WITH_TOLERANCE(float, th1->data.f32, hh->data.f32, 10 * 128, 1e-3, "GPU computed output should be the same as CPU computed ones");
1957
1
  ccv_nnc_tensor_free(a);
1958
1
  ccv_nnc_tensor_free(w);
1959
1
  ccv_nnc_tensor_free(b);
1960
1
  ccv_nnc_tensor_free(g);
1961
1
  ccv_nnc_tensor_free(dw);
1962
1
  ccv_nnc_tensor_free(h);
1963
1
  ccv_nnc_tensor_free(ha);
1964
1
  ccv_nnc_tensor_free(hw);
1965
1
  ccv_nnc_tensor_free(hb);
1966
1
  ccv_nnc_tensor_free(hg);
1967
1
  ccv_nnc_tensor_free(hdw);
1968
1
  ccv_nnc_tensor_free(hh);
1969
1
  ccv_nnc_tensor_free(tb);
1970
1
  ccv_nnc_tensor_free(th);
1971
1
  ccv_nnc_tensor_free(tdw);
1972
1
  ccv_nnc_tensor_free(ha2);
1973
1
  ccv_nnc_tensor_free(hw2);
1974
1
  ccv_nnc_tensor_free(hg2);
1975
1
  ccv_nnc_tensor_free(tb1);
1976
1
  ccv_nnc_tensor_free(tdw1);
1977
1
  ccv_nnc_tensor_free(th1);
1978
1
}
1979
1980
TEST_CASE("cublas handle permute")
1981
1
{
1982
1
  GUARD_ELSE_RETURN(ccv_nnc_cmd_ok(CCV_NNC_GEMM_FORWARD, CCV_NNC_BACKEND_GPU_CUBLAS));
1983
1
  dsfmt_t dsfmt;
1984
1
  dsfmt_init_gen_rand(&dsfmt, 0);
1985
1
  ccv_nnc_tensor_t* a = ccv_nnc_tensor_new(0, GPU_TENSOR_NHWC(000, 32F, 10, 2, 128), 0);
1986
1
  ccv_nnc_tensor_t* w = ccv_nnc_tensor_new(0, GPU_TENSOR_NHWC(000, 32F, 64, 2, 128), 0);
1987
1
  ccv_nnc_tensor_t* ha = ccv_nnc_tensor_new(0, CPU_TENSOR_NHWC(32F, 10, 2, 128), 0);
1988
1
  ccv_nnc_tensor_t* hw = ccv_nnc_tensor_new(0, CPU_TENSOR_NHWC(32F, 64, 2, 128), 0);
1989
1
  ccv_nnc_tensor_t* b = ccv_nnc_tensor_new(0, GPU_TENSOR_NHWC(000, 32F, 2, 10, 64), 0);
1990
1991
1
  ccv_nnc_tensor_t* at = ccv_nnc_tensor_new(0, GPU_TENSOR_NHWC(000, 32F, 2, 10, 128), 0);
1992
1
  ccv_nnc_tensor_t* wt = ccv_nnc_tensor_new(0, GPU_TENSOR_NHWC(000, 32F, 2, 64, 128), 0);
1993
1
  ccv_nnc_tensor_t* bt = ccv_nnc_tensor_new(0, GPU_TENSOR_NHWC(000, 32F, 2, 10, 64), 0);
1994
1
  int i;
1995
16.3k
  for (i = 0; i < 2 * 64 * 128; 
i++16.3k
)
1996
16.3k
    hw->data.f32[i] = dsfmt_genrand_open_close(&dsfmt) / (64 * 128);
1997
2.56k
  for (i = 0; i < 2 * 10 * 128; 
i++2.56k
)
1998
2.56k
    ha->data.f32[i] = dsfmt_genrand_open_close(&dsfmt);
1999
1
  ccv_nnc_cmd_exec(CMD_DATA_TRANSFER_FORWARD(), ccv_nnc_no_hint, 0, TENSOR_LIST(ha, hw), TENSOR_LIST(a, w), 0);
2000
1
  ccv_nnc_cmd_exec(CMD_TRANSPOSE_FORWARD(0, 1), ccv_nnc_no_hint, 0, TENSOR_LIST(a), TENSOR_LIST(at), 0);
2001
1
  ccv_nnc_cmd_exec(CMD_TRANSPOSE_FORWARD(0, 1), ccv_nnc_no_hint, 0, TENSOR_LIST(w), TENSOR_LIST(wt), 0);
2002
1
  ccv_nnc_cmd_exec(CMD_GEMM_FORWARD(NO_TRANSPOSE, TRANSPOSE(1, 2)), ccv_nnc_no_hint, 0, TENSOR_LIST(at, wt), TENSOR_LIST(bt), 0);
2003
1
  ccv_nnc_tensor_view_t* av = ccv_nnc_tensor_view_new(a, GPU_TENSOR_NHWC(000, 32F, 2, 10, 128), ccv_nnc_no_ofs, DIM_ALLOC(128, 2 * 128, 1));
2004
1
  ccv_nnc_tensor_view_t* wv = ccv_nnc_tensor_view_new(w, GPU_TENSOR_NHWC(000, 32F, 2, 64, 128), ccv_nnc_no_ofs, DIM_ALLOC(128, 2 * 128, 1));
2005
1
  ccv_nnc_cmd_exec(CMD_GEMM_FORWARD(NO_TRANSPOSE, TRANSPOSE(1, 2)), ccv_nnc_no_hint, 0, TENSOR_LIST((ccv_nnc_tensor_t*)av, (ccv_nnc_tensor_t*)wv), TENSOR_LIST(b), 0);
2006
1
  ccv_nnc_tensor_t* hb = ccv_nnc_tensor_new(0, CPU_TENSOR_NHWC(32F, 2, 10, 64), 0);
2007
1
  ccv_nnc_tensor_t* hbt = ccv_nnc_tensor_new(0, CPU_TENSOR_NHWC(32F, 2, 10, 64), 0);
2008
1
  ccv_nnc_cmd_exec(CMD_DATA_TRANSFER_FORWARD(), ccv_nnc_no_hint, 0, TENSOR_LIST(b, bt), TENSOR_LIST(hb, hbt), 0);
2009
1
  REQUIRE_TENSOR_EQ(hb, hbt, "permute computed output should be the same as non-permute computed ones");
2010
1
  ccv_nnc_tensor_free(ha);
2011
1
  ccv_nnc_tensor_free(hw);
2012
1
  ccv_nnc_tensor_free(a);
2013
1
  ccv_nnc_tensor_free(w);
2014
1
  ccv_nnc_tensor_free(b);
2015
1
  ccv_nnc_tensor_view_free(av);
2016
1
  ccv_nnc_tensor_view_free(wv);
2017
1
  ccv_nnc_tensor_free(at);
2018
1
  ccv_nnc_tensor_free(wt);
2019
1
  ccv_nnc_tensor_free(bt);
2020
1
  ccv_nnc_tensor_free(hb);
2021
1
  ccv_nnc_tensor_free(hbt);
2022
1
}
2023
2024
TEST_CASE("generalized batched gemm with batch (2, 4) compare cublas")
2025
1
{
2026
1
  GUARD_ELSE_RETURN(ccv_nnc_cmd_ok(CCV_NNC_GEMM_FORWARD, CCV_NNC_BACKEND_GPU_CUBLAS));
2027
  // This is a particular batched gemm which treat every dimensions other than the last two as batching.
2028
1
  dsfmt_t dsfmt;
2029
1
  dsfmt_init_gen_rand(&dsfmt, 0);
2030
1
  ccv_nnc_tensor_t* ha = ccv_nnc_tensor_new(0, CPU_TENSOR_NHWC(32F, 2, 10, 4, 128), 0);
2031
1
  ccv_nnc_tensor_t* hw = ccv_nnc_tensor_new(0, CPU_TENSOR_NHWC(32F, 2, 64, 4, 128), 0);
2032
1
  ccv_nnc_tensor_t* hb = ccv_nnc_tensor_new(0, CPU_TENSOR_NHWC(32F, 2, 4, 10, 64), 0);
2033
1
  ccv_nnc_tensor_t* a = ccv_nnc_tensor_new(0, GPU_TENSOR_NHWC(000, 32F, 2, 10, 4, 128), 0);
2034
1
  ccv_nnc_tensor_t* w = ccv_nnc_tensor_new(0, GPU_TENSOR_NHWC(000, 32F, 2, 64, 4, 128), 0);
2035
1
  ccv_nnc_tensor_t* b = ccv_nnc_tensor_new(0, GPU_TENSOR_NHWC(000, 32F, 2, 4, 10, 64), 0);
2036
2037
1
  ccv_nnc_tensor_t* at = ccv_nnc_tensor_new(0, CPU_TENSOR_NHWC(32F, 2, 4, 10, 128), 0);
2038
1
  ccv_nnc_tensor_t* wt = ccv_nnc_tensor_new(0, CPU_TENSOR_NHWC(32F, 2, 4, 64, 128), 0);
2039
1
  ccv_nnc_tensor_t* bt = ccv_nnc_tensor_new(0, CPU_TENSOR_NHWC(32F, 2, 4, 10, 64), 0);
2040
1
  int i;
2041
65.5k
  for (i = 0; i < 8 * 64 * 128; 
i++65.5k
)
2042
65.5k
    hw->data.f32[i] = dsfmt_genrand_open_close(&dsfmt) / (64 * 128);
2043
10.2k
  for (i = 0; i < 8 * 10 * 128; 
i++10.2k
)
2044
10.2k
    ha->data.f32[i] = dsfmt_genrand_open_close(&dsfmt);
2045
1
  ccv_nnc_cmd_exec(CMD_TRANSPOSE_FORWARD(1, 2), ccv_nnc_no_hint, 0, TENSOR_LIST(ha), TENSOR_LIST(at), 0);
2046
1
  ccv_nnc_cmd_exec(CMD_TRANSPOSE_FORWARD(1, 2), ccv_nnc_no_hint, 0, TENSOR_LIST(hw), TENSOR_LIST(wt), 0);
2047
1
  ccv_nnc_cmd_exec(CMD_DATA_TRANSFER_FORWARD(), ccv_nnc_no_hint, 0, TENSOR_LIST(ha, hw), TENSOR_LIST(a, w), 0);
2048
1
  ccv_nnc_tensor_view_t* av = ccv_nnc_tensor_view_new(a, GPU_TENSOR_NHWC(000, 32F, 2, 4, 10, 128), ccv_nnc_no_ofs, DIM_ALLOC(10 * 4 * 128, 128, 4 * 128, 1));
2049
1
  ccv_nnc_tensor_view_t* wv = ccv_nnc_tensor_view_new(w, GPU_TENSOR_NHWC(000, 32F, 2, 4, 64, 128), ccv_nnc_no_ofs, DIM_ALLOC(64 * 4 * 128, 128, 4 * 128, 1));
2050
1
  ccv_nnc_cmd_exec(CMD_GEMM_FORWARD(NO_TRANSPOSE, TRANSPOSE(2, 3)), ccv_nnc_no_hint, 0, TENSOR_LIST((ccv_nnc_tensor_t*)av, (ccv_nnc_tensor_t*)wv), TENSOR_LIST(b), 0);
2051
1
  ccv_nnc_cmd_exec(CMD_GEMM_FORWARD(NO_TRANSPOSE, TRANSPOSE(2, 3)), ccv_nnc_no_hint, 0, TENSOR_LIST(at, wt), TENSOR_LIST(bt), 0);
2052
1
  ccv_nnc_cmd_exec(CMD_DATA_TRANSFER_FORWARD(), ccv_nnc_no_hint, 0, TENSOR_LIST(b), TENSOR_LIST(hb), 0);
2053
1
  REQUIRE_TENSOR_EQ(hb, bt, "permute computed output should be the same as non-permute computed ones");
2054
1
  ccv_nnc_tensor_free(ha);
2055
1
  ccv_nnc_tensor_free(hw);
2056
1
  ccv_nnc_tensor_free(hb);
2057
1
  ccv_nnc_tensor_free(a);
2058
1
  ccv_nnc_tensor_free(w);
2059
1
  ccv_nnc_tensor_free(b);
2060
1
  ccv_nnc_tensor_view_free(av);
2061
1
  ccv_nnc_tensor_view_free(wv);
2062
1
  ccv_nnc_tensor_free(at);
2063
1
  ccv_nnc_tensor_free(wt);
2064
1
  ccv_nnc_tensor_free(bt);
2065
1
}
2066
2067
TEST_CASE("generalized batched gemm with batch (2, 4) and broadcast compare cublas")
2068
1
{
2069
1
  GUARD_ELSE_RETURN(ccv_nnc_cmd_ok(CCV_NNC_GEMM_FORWARD, CCV_NNC_BACKEND_GPU_CUBLAS));
2070
  // This is a particular batched gemm which treat every dimensions other than the last two as batching.
2071
1
  dsfmt_t dsfmt;
2072
1
  dsfmt_init_gen_rand(&dsfmt, 0);
2073
1
  ccv_nnc_tensor_t* ha = ccv_nnc_tensor_new(0, CPU_TENSOR_NHWC(32F, 2, 10, 4, 128), 0);
2074
1
  ccv_nnc_tensor_t* hw = ccv_nnc_tensor_new(0, CPU_TENSOR_NHWC(32F, 64, 128), 0);
2075
1
  ccv_nnc_tensor_t* hb = ccv_nnc_tensor_new(0, CPU_TENSOR_NHWC(32F, 2, 4, 10, 64), 0);
2076
1
  ccv_nnc_tensor_t* a = ccv_nnc_tensor_new(0, GPU_TENSOR_NHWC(000, 32F, 2, 10, 4, 128), 0);
2077
1
  ccv_nnc_tensor_t* w = ccv_nnc_tensor_new(0, GPU_TENSOR_NHWC(000, 32F, 64, 128), 0);
2078
1
  ccv_nnc_tensor_t* b = ccv_nnc_tensor_new(0, GPU_TENSOR_NHWC(000, 32F, 2, 4, 10, 64), 0);
2079
2080
1
  ccv_nnc_tensor_t* at = ccv_nnc_tensor_new(0, CPU_TENSOR_NHWC(32F, 2, 4, 10, 128), 0);
2081
1
  ccv_nnc_tensor_t* bt = ccv_nnc_tensor_new(0, CPU_TENSOR_NHWC(32F, 2, 4, 10, 64), 0);
2082
1
  int i;
2083
8.19k
  for (i = 0; i < 64 * 128; 
i++8.19k
)
2084
8.19k
    hw->data.f32[i] = dsfmt_genrand_open_close(&dsfmt) / (64 * 128);
2085
10.2k
  for (i = 0; i < 8 * 10 * 128; 
i++10.2k
)
2086
10.2k
    ha->data.f32[i] = dsfmt_genrand_open_close(&dsfmt);
2087
1
  ccv_nnc_cmd_exec(CMD_TRANSPOSE_FORWARD(1, 2), ccv_nnc_no_hint, 0, TENSOR_LIST(ha), TENSOR_LIST(at), 0);
2088
1
  ccv_nnc_cmd_exec(CMD_DATA_TRANSFER_FORWARD(), ccv_nnc_no_hint, 0, TENSOR_LIST(ha, hw), TENSOR_LIST(a, w), 0);
2089
1
  ccv_nnc_tensor_view_t* av = ccv_nnc_tensor_view_new(a, GPU_TENSOR_NHWC(000, 32F, 2, 4, 10, 128), ccv_nnc_no_ofs, DIM_ALLOC(10 * 4 * 128, 128, 4 * 128, 1));
2090
1
  ccv_nnc_cmd_exec(CMD_GEMM_FORWARD(NO_TRANSPOSE, TRANSPOSE(0, 1)), ccv_nnc_no_hint, 0, TENSOR_LIST((ccv_nnc_tensor_t*)av, w), TENSOR_LIST(b), 0);
2091
1
  ccv_nnc_cmd_exec(CMD_GEMM_FORWARD(NO_TRANSPOSE, TRANSPOSE(0, 1)), ccv_nnc_no_hint, 0, TENSOR_LIST(at, hw), TENSOR_LIST(bt), 0);
2092
1
  ccv_nnc_cmd_exec(CMD_DATA_TRANSFER_FORWARD(), ccv_nnc_no_hint, 0, TENSOR_LIST(b), TENSOR_LIST(hb), 0);
2093
1
  REQUIRE_TENSOR_EQ(hb, bt, "permute computed output should be the same as non-permute computed ones");
2094
1
  ccv_nnc_tensor_free(ha);
2095
1
  ccv_nnc_tensor_free(hw);
2096
1
  ccv_nnc_tensor_free(hb);
2097
1
  ccv_nnc_tensor_free(a);
2098
1
  ccv_nnc_tensor_free(w);
2099
1
  ccv_nnc_tensor_free(b);
2100
1
  ccv_nnc_tensor_view_free(av);
2101
1
  ccv_nnc_tensor_free(at);
2102
1
  ccv_nnc_tensor_free(bt);
2103
1
}
2104
2105
TEST_CASE("generalized batched gemm with batch (2, 4) with bias compare cublas")
2106
1
{
2107
1
  GUARD_ELSE_RETURN(ccv_nnc_cmd_ok(CCV_NNC_GEMM_FORWARD, CCV_NNC_BACKEND_GPU_CUBLAS));
2108
  // This is a particular batched gemm which treat every dimensions other than the last two as batching.
2109
1
  dsfmt_t dsfmt;
2110
1
  dsfmt_init_gen_rand(&dsfmt, 0);
2111
1
  ccv_nnc_tensor_t* ha = ccv_nnc_tensor_new(0, CPU_TENSOR_NHWC(32F, 2, 10, 4, 128), 0);
2112
1
  ccv_nnc_tensor_t* hw = ccv_nnc_tensor_new(0, CPU_TENSOR_NHWC(32F, 2, 64, 4, 128), 0);
2113
1
  ccv_nnc_tensor_t* hbias = ccv_nnc_tensor_new(0, CPU_TENSOR_NHWC(32F, 64), 0);
2114
1
  ccv_nnc_tensor_t* hb = ccv_nnc_tensor_new(0, CPU_TENSOR_NHWC(32F, 2, 4, 10, 64), 0);
2115
1
  ccv_nnc_tensor_t* a = ccv_nnc_tensor_new(0, GPU_TENSOR_NHWC(000, 32F, 2, 10, 4, 128), 0);
2116
1
  ccv_nnc_tensor_t* w = ccv_nnc_tensor_new(0, GPU_TENSOR_NHWC(000, 32F, 2, 64, 4, 128), 0);
2117
1
  ccv_nnc_tensor_t* bias = ccv_nnc_tensor_new(0, GPU_TENSOR_NHWC(000, 32F, 64), 0);
2118
1
  ccv_nnc_tensor_t* b = ccv_nnc_tensor_new(0, GPU_TENSOR_NHWC(000, 32F, 2, 4, 10, 64), 0);
2119
2120
1
  ccv_nnc_tensor_t* at = ccv_nnc_tensor_new(0, CPU_TENSOR_NHWC(32F, 2, 4, 10, 128), 0);
2121
1
  ccv_nnc_tensor_t* wt = ccv_nnc_tensor_new(0, CPU_TENSOR_NHWC(32F, 2, 4, 64, 128), 0);
2122
1
  ccv_nnc_tensor_t* bt = ccv_nnc_tensor_new(0, CPU_TENSOR_NHWC(32F, 2, 4, 10, 64), 0);
2123
1
  int i;
2124
65.5k
  for (i = 0; i < 8 * 64 * 128; 
i++65.5k
)
2125
65.5k
    hw->data.f32[i] = dsfmt_genrand_open_close(&dsfmt) / (64 * 128);
2126
65
  for (i = 0; i < 64; 
i++64
)
2127
64
    hbias->data.f32[i] = dsfmt_genrand_open_close(&dsfmt) / 64;
2128
10.2k
  for (i = 0; i < 8 * 10 * 128; 
i++10.2k
)
2129
10.2k
    ha->data.f32[i] = dsfmt_genrand_open_close(&dsfmt);
2130
1
  ccv_nnc_cmd_exec(CMD_TRANSPOSE_FORWARD(1, 2), ccv_nnc_no_hint, 0, TENSOR_LIST(ha), TENSOR_LIST(at), 0);
2131
1
  ccv_nnc_cmd_exec(CMD_TRANSPOSE_FORWARD(1, 2), ccv_nnc_no_hint, 0, TENSOR_LIST(hw), TENSOR_LIST(wt), 0);
2132
1
  ccv_nnc_cmd_exec(CMD_DATA_TRANSFER_FORWARD(), ccv_nnc_no_hint, 0, TENSOR_LIST(ha, hw, hbias), TENSOR_LIST(a, w, bias), 0);
2133
1
  ccv_nnc_tensor_view_t* av = ccv_nnc_tensor_view_new(a, GPU_TENSOR_NHWC(000, 32F, 2, 4, 10, 128), ccv_nnc_no_ofs, DIM_ALLOC(10 * 4 * 128, 128, 4 * 128, 1));
2134
1
  ccv_nnc_tensor_view_t* wv = ccv_nnc_tensor_view_new(w, GPU_TENSOR_NHWC(000, 32F, 2, 4, 64, 128), ccv_nnc_no_ofs, DIM_ALLOC(64 * 4 * 128, 128, 4 * 128, 1));
2135
1
  ccv_nnc_cmd_exec(CMD_GEMM_FORWARD(NO_TRANSPOSE, TRANSPOSE(2, 3)), ccv_nnc_no_hint, 0, TENSOR_LIST((ccv_nnc_tensor_t*)av, (ccv_nnc_tensor_t*)wv, bias), TENSOR_LIST(b), 0);
2136
1
  ccv_nnc_cmd_exec(CMD_GEMM_FORWARD(NO_TRANSPOSE, TRANSPOSE(2, 3)), ccv_nnc_no_hint, 0, TENSOR_LIST(at, wt, hbias), TENSOR_LIST(bt), 0);
2137
1
  ccv_nnc_cmd_exec(CMD_DATA_TRANSFER_FORWARD(), ccv_nnc_no_hint, 0, TENSOR_LIST(b), TENSOR_LIST(hb), 0);
2138
1
  REQUIRE_TENSOR_EQ(hb, bt, "permute computed output should be the same as non-permute computed ones");
2139
1
  ccv_nnc_tensor_free(ha);
2140
1
  ccv_nnc_tensor_free(hw);
2141
1
  ccv_nnc_tensor_free(hbias);
2142
1
  ccv_nnc_tensor_free(hb);
2143
1
  ccv_nnc_tensor_free(a);
2144
1
  ccv_nnc_tensor_free(w);
2145
1
  ccv_nnc_tensor_free(bias);
2146
1
  ccv_nnc_tensor_free(b);
2147
1
  ccv_nnc_tensor_view_free(av);
2148
1
  ccv_nnc_tensor_view_free(wv);
2149
1
  ccv_nnc_tensor_free(at);
2150
1
  ccv_nnc_tensor_free(wt);
2151
1
  ccv_nnc_tensor_free(bt);
2152
1
}
2153
2154
TEST_CASE("generalized batched gemm with batch (2, 4) with bias and broadcast compare cublas")
2155
1
{
2156
1
  GUARD_ELSE_RETURN(ccv_nnc_cmd_ok(CCV_NNC_GEMM_FORWARD, CCV_NNC_BACKEND_GPU_CUBLAS));
2157
  // This is a particular batched gemm which treat every dimensions other than the last two as batching.
2158
1
  dsfmt_t dsfmt;
2159
1
  dsfmt_init_gen_rand(&dsfmt, 0);
2160
1
  ccv_nnc_tensor_t* ha = ccv_nnc_tensor_new(0, CPU_TENSOR_NHWC(32F, 2, 10, 4, 128), 0);
2161
1
  ccv_nnc_tensor_t* hw = ccv_nnc_tensor_new(0, CPU_TENSOR_NHWC(32F, 64, 128), 0);
2162
1
  ccv_nnc_tensor_t* hbias = ccv_nnc_tensor_new(0, CPU_TENSOR_NHWC(32F, 64), 0);
2163
1
  ccv_nnc_tensor_t* hb = ccv_nnc_tensor_new(0, CPU_TENSOR_NHWC(32F, 2, 4, 10, 64), 0);
2164
1
  ccv_nnc_tensor_t* a = ccv_nnc_tensor_new(0, GPU_TENSOR_NHWC(000, 32F, 2, 10, 4, 128), 0);
2165
1
  ccv_nnc_tensor_t* w = ccv_nnc_tensor_new(0, GPU_TENSOR_NHWC(000, 32F, 64, 128), 0);
2166
1
  ccv_nnc_tensor_t* bias = ccv_nnc_tensor_new(0, GPU_TENSOR_NHWC(000, 32F, 64), 0);
2167
1
  ccv_nnc_tensor_t* b = ccv_nnc_tensor_new(0, GPU_TENSOR_NHWC(000, 32F, 2, 4, 10, 64), 0);
2168
2169
1
  ccv_nnc_tensor_t* at = ccv_nnc_tensor_new(0, CPU_TENSOR_NHWC(32F, 2, 4, 10, 128), 0);
2170
1
  ccv_nnc_tensor_t* bt = ccv_nnc_tensor_new(0, CPU_TENSOR_NHWC(32F, 2, 4, 10, 64), 0);
2171
1
  int i;
2172
8.19k
  for (i = 0; i < 64 * 128; 
i++8.19k
)
2173
8.19k
    hw->data.f32[i] = dsfmt_genrand_open_close(&dsfmt) / (64 * 128);
2174
65
  for (i = 0; i < 64; 
i++64
)
2175
64
    hbias->data.f32[i] = dsfmt_genrand_open_close(&dsfmt) / 64;
2176
10.2k
  for (i = 0; i < 8 * 10 * 128; 
i++10.2k
)
2177
10.2k
    ha->data.f32[i] = dsfmt_genrand_open_close(&dsfmt);
2178
1
  ccv_nnc_cmd_exec(CMD_TRANSPOSE_FORWARD(1, 2), ccv_nnc_no_hint, 0, TENSOR_LIST(ha), TENSOR_LIST(at), 0);
2179
1
  ccv_nnc_cmd_exec(CMD_DATA_TRANSFER_FORWARD(), ccv_nnc_no_hint, 0, TENSOR_LIST(ha, hw, hbias), TENSOR_LIST(a, w, bias), 0);
2180
1
  ccv_nnc_tensor_view_t* av = ccv_nnc_tensor_view_new(a, GPU_TENSOR_NHWC(000, 32F, 2, 4, 10, 128), ccv_nnc_no_ofs, DIM_ALLOC(10 * 4 * 128, 128, 4 * 128, 1));
2181
1
  ccv_nnc_cmd_exec(CMD_GEMM_FORWARD(NO_TRANSPOSE, TRANSPOSE(0, 1)), ccv_nnc_no_hint, 0, TENSOR_LIST((ccv_nnc_tensor_t*)av, w, bias), TENSOR_LIST(b), 0);
2182
1
  ccv_nnc_cmd_exec(CMD_GEMM_FORWARD(NO_TRANSPOSE, TRANSPOSE(0, 1)), ccv_nnc_no_hint, 0, TENSOR_LIST(at, hw, hbias), TENSOR_LIST(bt), 0);
2183
1
  ccv_nnc_cmd_exec(CMD_DATA_TRANSFER_FORWARD(), ccv_nnc_no_hint, 0, TENSOR_LIST(b), TENSOR_LIST(hb), 0);
2184
1
  REQUIRE_TENSOR_EQ(hb, bt, "permute computed output should be the same as non-permute computed ones");
2185
1
  ccv_nnc_tensor_free(ha);
2186
1
  ccv_nnc_tensor_free(hw);
2187
1
  ccv_nnc_tensor_free(hbias);
2188
1
  ccv_nnc_tensor_free(hb);
2189
1
  ccv_nnc_tensor_free(a);
2190
1
  ccv_nnc_tensor_free(w);
2191
1
  ccv_nnc_tensor_free(bias);
2192
1
  ccv_nnc_tensor_free(b);
2193
1
  ccv_nnc_tensor_view_free(av);
2194
1
  ccv_nnc_tensor_free(at);
2195
1
  ccv_nnc_tensor_free(bt);
2196
1
}
2197
2198
TEST_CASE("generalized batched backward gemm with batch (2, 4) compare cublas")
2199
1
{
2200
1
  GUARD_ELSE_RETURN(ccv_nnc_cmd_ok(CCV_NNC_GEMM_BACKWARD, CCV_NNC_BACKEND_GPU_CUBLAS));
2201
  // This is a particular batched gemm which treat every dimensions other than the last two as batching.
2202
1
  dsfmt_t dsfmt;
2203
1
  dsfmt_init_gen_rand(&dsfmt, 0);
2204
1
  ccv_nnc_tensor_t* ha = ccv_nnc_tensor_new(0, CPU_TENSOR_NHWC(32F, 2, 10, 4, 128), 0);
2205
1
  ccv_nnc_tensor_t* hw = ccv_nnc_tensor_new(0, CPU_TENSOR_NHWC(32F, 2, 64, 4, 128), 0);
2206
1
  ccv_nnc_tensor_t* hda = ccv_nnc_tensor_new(0, CPU_TENSOR_NHWC(32F, 2, 10, 4, 128), 0);
2207
1
  ccv_nnc_tensor_t* hdw = ccv_nnc_tensor_new(0, CPU_TENSOR_NHWC(32F, 2, 64, 4, 128), 0);
2208
1
  ccv_nnc_tensor_t* hb = ccv_nnc_tensor_new(0, CPU_TENSOR_NHWC(32F, 2, 4, 10, 64), 0);
2209
1
  ccv_nnc_tensor_t* a = ccv_nnc_tensor_new(0, GPU_TENSOR_NHWC(000, 32F, 2, 10, 4, 128), 0);
2210
1
  ccv_nnc_tensor_t* w = ccv_nnc_tensor_new(0, GPU_TENSOR_NHWC(000, 32F, 2, 64, 4, 128), 0);
2211
1
  ccv_nnc_tensor_t* da = ccv_nnc_tensor_new(0, GPU_TENSOR_NHWC(000, 32F, 2, 10, 4, 128), 0);
2212
1
  ccv_nnc_tensor_t* dw = ccv_nnc_tensor_new(0, GPU_TENSOR_NHWC(000, 32F, 2, 64, 4, 128), 0);
2213
1
  ccv_nnc_tensor_t* b = ccv_nnc_tensor_new(0, GPU_TENSOR_NHWC(000, 32F, 2, 4, 10, 64), 0);
2214
2215
1
  ccv_nnc_tensor_t* at = ccv_nnc_tensor_new(0, CPU_TENSOR_NHWC(32F, 2, 4, 10, 128), 0);
2216
1
  ccv_nnc_tensor_t* wt = ccv_nnc_tensor_new(0, CPU_TENSOR_NHWC(32F, 2, 4, 64, 128), 0);
2217
1
  ccv_nnc_tensor_t* dat = ccv_nnc_tensor_new(0, CPU_TENSOR_NHWC(32F, 2, 4, 10, 128), 0);
2218
1
  ccv_nnc_tensor_t* dwt = ccv_nnc_tensor_new(0, CPU_TENSOR_NHWC(32F, 2, 4, 64, 128), 0);
2219
1
  ccv_nnc_tensor_t* tda = ccv_nnc_tensor_new(0, CPU_TENSOR_NHWC(32F, 2, 10, 4, 128), 0);
2220
1
  ccv_nnc_tensor_t* tdw = ccv_nnc_tensor_new(0, CPU_TENSOR_NHWC(32F, 2, 64, 4, 128), 0);
2221
1
  int i;
2222
65.5k
  for (i = 0; i < 8 * 64 * 128; 
i++65.5k
)
2223
65.5k
    hw->data.f32[i] = dsfmt_genrand_open_close(&dsfmt) / (64 * 128);
2224
10.2k
  for (i = 0; i < 8 * 10 * 128; 
i++10.2k
)
2225
10.2k
    ha->data.f32[i] = dsfmt_genrand_open_close(&dsfmt);
2226
5.12k
  for (i = 0; i < 2 * 4 * 10 * 64; 
i++5.12k
)
2227
5.12k
    hb->data.f32[i] = dsfmt_genrand_open_close(&dsfmt);
2228
1
  ccv_nnc_cmd_exec(CMD_TRANSPOSE_FORWARD(1, 2), ccv_nnc_no_hint, 0, TENSOR_LIST(ha), TENSOR_LIST(at), 0);
2229
1
  ccv_nnc_cmd_exec(CMD_TRANSPOSE_FORWARD(1, 2), ccv_nnc_no_hint, 0, TENSOR_LIST(hw), TENSOR_LIST(wt), 0);
2230
1
  ccv_nnc_cmd_exec(CMD_DATA_TRANSFER_FORWARD(), ccv_nnc_no_hint, 0, TENSOR_LIST(ha, hw, hb), TENSOR_LIST(a, w, b), 0);
2231
1
  ccv_nnc_tensor_view_t* av = ccv_nnc_tensor_view_new(a, GPU_TENSOR_NHWC(000, 32F, 2, 4, 10, 128), ccv_nnc_no_ofs, DIM_ALLOC(10 * 4 * 128, 128, 4 * 128, 1));
2232
1
  ccv_nnc_tensor_view_t* wv = ccv_nnc_tensor_view_new(w, GPU_TENSOR_NHWC(000, 32F, 2, 4, 64, 128), ccv_nnc_no_ofs, DIM_ALLOC(64 * 4 * 128, 128, 4 * 128, 1));
2233
1
  ccv_nnc_tensor_view_t* dav = ccv_nnc_tensor_view_new(da, GPU_TENSOR_NHWC(000, 32F, 2, 4, 10, 128), ccv_nnc_no_ofs, DIM_ALLOC(10 * 4 * 128, 128, 4 * 128, 1));
2234
1
  ccv_nnc_tensor_view_t* dwv = ccv_nnc_tensor_view_new(dw, GPU_TENSOR_NHWC(000, 32F, 2, 4, 64, 128), ccv_nnc_no_ofs, DIM_ALLOC(64 * 4 * 128, 128, 4 * 128, 1));
2235
1
  ccv_nnc_cmd_exec(CMD_GEMM_BACKWARD(NO_TRANSPOSE, TRANSPOSE(2, 3)), ccv_nnc_no_hint, 0, TENSOR_LIST(b, (ccv_nnc_tensor_t*)av, (ccv_nnc_tensor_t*)wv), TENSOR_LIST((ccv_nnc_tensor_t*)dav, (ccv_nnc_tensor_t*)dwv), 0);
2236
1
  ccv_nnc_cmd_exec(CMD_GEMM_BACKWARD(NO_TRANSPOSE, TRANSPOSE(2, 3)), ccv_nnc_no_hint, 0, TENSOR_LIST(hb, at, wt), TENSOR_LIST(dat, dwt), 0);
2237
1
  ccv_nnc_cmd_exec(CMD_TRANSPOSE_FORWARD(1, 2), ccv_nnc_no_hint, 0, TENSOR_LIST(dat), TENSOR_LIST(tda), 0);
2238
1
  ccv_nnc_cmd_exec(CMD_TRANSPOSE_FORWARD(1, 2), ccv_nnc_no_hint, 0, TENSOR_LIST(dwt), TENSOR_LIST(tdw), 0);
2239
1
  ccv_nnc_cmd_exec(CMD_DATA_TRANSFER_FORWARD(), ccv_nnc_no_hint, 0, TENSOR_LIST(da, dw), TENSOR_LIST(hda, hdw), 0);
2240
1
  REQUIRE_TENSOR_EQ(hda, tda, "permute computed output should be the same as non-permute computed ones");
2241
1
  REQUIRE_TENSOR_EQ(hdw, tdw, "permute computed output should be the same as non-permute computed ones");
2242
1
  ccv_nnc_tensor_free(ha);
2243
1
  ccv_nnc_tensor_free(hw);
2244
1
  ccv_nnc_tensor_free(hda);
2245
1
  ccv_nnc_tensor_free(hdw);
2246
1
  ccv_nnc_tensor_free(hb);
2247
1
  ccv_nnc_tensor_free(a);
2248
1
  ccv_nnc_tensor_free(w);
2249
1
  ccv_nnc_tensor_free(da);
2250
1
  ccv_nnc_tensor_free(dw);
2251
1
  ccv_nnc_tensor_free(b);
2252
1
  ccv_nnc_tensor_view_free(av);
2253
1
  ccv_nnc_tensor_view_free(wv);
2254
1
  ccv_nnc_tensor_view_free(dav);
2255
1
  ccv_nnc_tensor_view_free(dwv);
2256
1
  ccv_nnc_tensor_free(at);
2257
1
  ccv_nnc_tensor_free(wt);
2258
1
  ccv_nnc_tensor_free(dat);
2259
1
  ccv_nnc_tensor_free(tda);
2260
1
  ccv_nnc_tensor_free(dwt);
2261
1
  ccv_nnc_tensor_free(tdw);
2262
1
}
2263
2264
TEST_CASE("generalized batched backward gemm with batch (2, 4) and broadcast compare cublas")
2265
1
{
2266
1
  GUARD_ELSE_RETURN(ccv_nnc_cmd_ok(CCV_NNC_GEMM_BACKWARD, CCV_NNC_BACKEND_GPU_CUBLAS));
2267
  // This is a particular batched gemm which treat every dimensions other than the last two as batching.
2268
1
  dsfmt_t dsfmt;
2269
1
  dsfmt_init_gen_rand(&dsfmt, 0);
2270
1
  ccv_nnc_tensor_t* ha = ccv_nnc_tensor_new(0, CPU_TENSOR_NHWC(32F, 2, 10, 4, 128), 0);
2271
1
  ccv_nnc_tensor_t* hw = ccv_nnc_tensor_new(0, CPU_TENSOR_NHWC(32F, 64, 128), 0);
2272
1
  ccv_nnc_tensor_t* hda = ccv_nnc_tensor_new(0, CPU_TENSOR_NHWC(32F, 2, 10, 4, 128), 0);
2273
1
  ccv_nnc_tensor_t* hdw = ccv_nnc_tensor_new(0, CPU_TENSOR_NHWC(32F, 64, 128), 0);
2274
1
  ccv_nnc_tensor_t* hb = ccv_nnc_tensor_new(0, CPU_TENSOR_NHWC(32F, 2, 4, 10, 64), 0);
2275
1
  ccv_nnc_tensor_t* a = ccv_nnc_tensor_new(0, GPU_TENSOR_NHWC(000, 32F, 2, 10, 4, 128), 0);
2276
1
  ccv_nnc_tensor_t* w = ccv_nnc_tensor_new(0, GPU_TENSOR_NHWC(000, 32F, 64, 128), 0);
2277
1
  ccv_nnc_tensor_t* da = ccv_nnc_tensor_new(0, GPU_TENSOR_NHWC(000, 32F, 2, 10, 4, 128), 0);
2278
1
  ccv_nnc_tensor_t* dw = ccv_nnc_tensor_new(0, GPU_TENSOR_NHWC(000, 32F, 64, 128), 0);
2279
1
  ccv_nnc_tensor_t* b = ccv_nnc_tensor_new(0, GPU_TENSOR_NHWC(000, 32F, 2, 4, 10, 64), 0);
2280
2281
1
  ccv_nnc_tensor_t* at = ccv_nnc_tensor_new(0, CPU_TENSOR_NHWC(32F, 2, 4, 10, 128), 0);
2282
1
  ccv_nnc_tensor_t* dat = ccv_nnc_tensor_new(0, CPU_TENSOR_NHWC(32F, 2, 4, 10, 128), 0);
2283
1
  ccv_nnc_tensor_t* tda = ccv_nnc_tensor_new(0, CPU_TENSOR_NHWC(32F, 2, 10, 4, 128), 0);
2284
1
  ccv_nnc_tensor_t* tdw = ccv_nnc_tensor_new(0, CPU_TENSOR_NHWC(32F, 64, 128), 0);
2285
1
  int i;
2286
8.19k
  for (i = 0; i < 64 * 128; 
i++8.19k
)
2287
8.19k
    hw->data.f32[i] = dsfmt_genrand_open_close(&dsfmt) / (64 * 128);
2288
10.2k
  for (i = 0; i < 8 * 10 * 128; 
i++10.2k
)
2289
10.2k
    ha->data.f32[i] = dsfmt_genrand_open_close(&dsfmt);
2290
5.12k
  for (i = 0; i < 2 * 4 * 10 * 64; 
i++5.12k
)
2291
5.12k
    hb->data.f32[i] = dsfmt_genrand_open_close(&dsfmt);
2292
1
  ccv_nnc_cmd_exec(CMD_TRANSPOSE_FORWARD(1, 2), ccv_nnc_no_hint, 0, TENSOR_LIST(ha), TENSOR_LIST(at), 0);
2293
1
  ccv_nnc_cmd_exec(CMD_DATA_TRANSFER_FORWARD(), ccv_nnc_no_hint, 0, TENSOR_LIST(ha, hw, hb), TENSOR_LIST(a, w, b), 0);
2294
1
  ccv_nnc_tensor_view_t* av = ccv_nnc_tensor_view_new(a, GPU_TENSOR_NHWC(000, 32F, 2, 4, 10, 128), ccv_nnc_no_ofs, DIM_ALLOC(10 * 4 * 128, 128, 4 * 128, 1));
2295
1
  ccv_nnc_tensor_view_t* dav = ccv_nnc_tensor_view_new(da, GPU_TENSOR_NHWC(000, 32F, 2, 4, 10, 128), ccv_nnc_no_ofs, DIM_ALLOC(10 * 4 * 128, 128, 4 * 128, 1));
2296
1
  ccv_nnc_cmd_exec(CMD_GEMM_BACKWARD(NO_TRANSPOSE, TRANSPOSE(0, 1)), ccv_nnc_no_hint, 0, TENSOR_LIST(b, (ccv_nnc_tensor_t*)av, w), TENSOR_LIST((ccv_nnc_tensor_t*)dav, dw), 0);
2297
1
  ccv_nnc_cmd_exec(CMD_GEMM_BACKWARD(NO_TRANSPOSE, TRANSPOSE(0, 1)), ccv_nnc_no_hint, 0, TENSOR_LIST(hb, at, hw), TENSOR_LIST(dat, tdw), 0);
2298
1
  ccv_nnc_cmd_exec(CMD_TRANSPOSE_FORWARD(1, 2), ccv_nnc_no_hint, 0, TENSOR_LIST(dat), TENSOR_LIST(tda), 0);
2299
1
  ccv_nnc_cmd_exec(CMD_DATA_TRANSFER_FORWARD(), ccv_nnc_no_hint, 0, TENSOR_LIST(da, dw), TENSOR_LIST(hda, hdw), 0);
2300
1
  REQUIRE_TENSOR_EQ(hda, tda, "permute computed output should be the same as non-permute computed ones");
2301
1
  REQUIRE_TENSOR_EQ(hdw, tdw, "permute computed output should be the same as non-permute computed ones");
2302
1
  ccv_nnc_tensor_free(ha);
2303
1
  ccv_nnc_tensor_free(hw);
2304
1
  ccv_nnc_tensor_free(hda);
2305
1
  ccv_nnc_tensor_free(hdw);
2306
1
  ccv_nnc_tensor_free(hb);
2307
1
  ccv_nnc_tensor_free(a);
2308
1
  ccv_nnc_tensor_free(w);
2309
1
  ccv_nnc_tensor_free(da);
2310
1
  ccv_nnc_tensor_free(dw);
2311
1
  ccv_nnc_tensor_free(b);
2312
1
  ccv_nnc_tensor_view_free(av);
2313
1
  ccv_nnc_tensor_view_free(dav);
2314
1
  ccv_nnc_tensor_free(at);
2315
1
  ccv_nnc_tensor_free(dat);
2316
1
  ccv_nnc_tensor_free(tda);
2317
1
  ccv_nnc_tensor_free(tdw);
2318
1
}
2319
2320
TEST_CASE("generalized batched backward gemm with batch (2, 4) with bias compare cublas")
2321
1
{
2322
1
  GUARD_ELSE_RETURN(ccv_nnc_cmd_ok(CCV_NNC_GEMM_BACKWARD, CCV_NNC_BACKEND_GPU_CUBLAS));
2323
  // This is a particular batched gemm which treat every dimensions other than the last two as batching.
2324
1
  dsfmt_t dsfmt;
2325
1
  dsfmt_init_gen_rand(&dsfmt, 0);
2326
1
  ccv_nnc_tensor_t* ha = ccv_nnc_tensor_new(0, CPU_TENSOR_NHWC(32F, 2, 10, 4, 128), 0);
2327
1
  ccv_nnc_tensor_t* hw = ccv_nnc_tensor_new(0, CPU_TENSOR_NHWC(32F, 2, 64, 4, 128), 0);
2328
1
  ccv_nnc_tensor_t* hda = ccv_nnc_tensor_new(0, CPU_TENSOR_NHWC(32F, 2, 10, 4, 128), 0);
2329
1
  ccv_nnc_tensor_t* hdw = ccv_nnc_tensor_new(0, CPU_TENSOR_NHWC(32F, 2, 64, 4, 128), 0);
2330
1
  ccv_nnc_tensor_t* hdbias = ccv_nnc_tensor_new(0, CPU_TENSOR_NHWC(32F, 64), 0);
2331
1
  ccv_nnc_tensor_t* hb = ccv_nnc_tensor_new(0, CPU_TENSOR_NHWC(32F, 2, 4, 10, 64), 0);
2332
1
  ccv_nnc_tensor_t* a = ccv_nnc_tensor_new(0, GPU_TENSOR_NHWC(000, 32F, 2, 10, 4, 128), 0);
2333
1
  ccv_nnc_tensor_t* w = ccv_nnc_tensor_new(0, GPU_TENSOR_NHWC(000, 32F, 2, 64, 4, 128), 0);
2334
1
  ccv_nnc_tensor_t* da = ccv_nnc_tensor_new(0, GPU_TENSOR_NHWC(000, 32F, 2, 10, 4, 128), 0);
2335
1
  ccv_nnc_tensor_t* dw = ccv_nnc_tensor_new(0, GPU_TENSOR_NHWC(000, 32F, 2, 64, 4, 128), 0);
2336
1
  ccv_nnc_tensor_t* dbias = ccv_nnc_tensor_new(0, GPU_TENSOR_NHWC(000, 32F, 64), 0);
2337
1
  ccv_nnc_tensor_t* b = ccv_nnc_tensor_new(0, GPU_TENSOR_NHWC(000, 32F, 2, 4, 10, 64), 0);
2338
2339
1
  ccv_nnc_tensor_t* at = ccv_nnc_tensor_new(0, CPU_TENSOR_NHWC(32F, 2, 4, 10, 128), 0);
2340
1
  ccv_nnc_tensor_t* wt = ccv_nnc_tensor_new(0, CPU_TENSOR_NHWC(32F, 2, 4, 64, 128), 0);
2341
1
  ccv_nnc_tensor_t* dat = ccv_nnc_tensor_new(0, CPU_TENSOR_NHWC(32F, 2, 4, 10, 128), 0);
2342
1
  ccv_nnc_tensor_t* dwt = ccv_nnc_tensor_new(0, CPU_TENSOR_NHWC(32F, 2, 4, 64, 128), 0);
2343
1
  ccv_nnc_tensor_t* tda = ccv_nnc_tensor_new(0, CPU_TENSOR_NHWC(32F, 2, 10, 4, 128), 0);
2344
1
  ccv_nnc_tensor_t* tdw = ccv_nnc_tensor_new(0, CPU_TENSOR_NHWC(32F, 2, 64, 4, 128), 0);
2345
1
  ccv_nnc_tensor_t* tdbias = ccv_nnc_tensor_new(0, CPU_TENSOR_NHWC(32F, 64), 0);
2346
1
  int i;
2347
65.5k
  for (i = 0; i < 8 * 64 * 128; 
i++65.5k
)
2348
65.5k
    hw->data.f32[i] = dsfmt_genrand_open_close(&dsfmt) / (64 * 128);
2349
10.2k
  for (i = 0; i < 8 * 10 * 128; 
i++10.2k
)
2350
10.2k
    ha->data.f32[i] = dsfmt_genrand_open_close(&dsfmt);
2351
5.12k
  for (i = 0; i < 2 * 4 * 10 * 64; 
i++5.12k
)
2352
5.12k
    hb->data.f32[i] = dsfmt_genrand_open_close(&dsfmt);
2353
1
  ccv_nnc_cmd_exec(CMD_TRANSPOSE_FORWARD(1, 2), ccv_nnc_no_hint, 0, TENSOR_LIST(ha), TENSOR_LIST(at), 0);
2354
1
  ccv_nnc_cmd_exec(CMD_TRANSPOSE_FORWARD(1, 2), ccv_nnc_no_hint, 0, TENSOR_LIST(hw), TENSOR_LIST(wt), 0);
2355
1
  ccv_nnc_cmd_exec(CMD_DATA_TRANSFER_FORWARD(), ccv_nnc_no_hint, 0, TENSOR_LIST(ha, hw, hb), TENSOR_LIST(a, w, b), 0);
2356
1
  ccv_nnc_tensor_view_t* av = ccv_nnc_tensor_view_new(a, GPU_TENSOR_NHWC(000, 32F, 2, 4, 10, 128), ccv_nnc_no_ofs, DIM_ALLOC(10 * 4 * 128, 128, 4 * 128, 1));
2357
1
  ccv_nnc_tensor_view_t* wv = ccv_nnc_tensor_view_new(w, GPU_TENSOR_NHWC(000, 32F, 2, 4, 64, 128), ccv_nnc_no_ofs, DIM_ALLOC(64 * 4 * 128, 128, 4 * 128, 1));
2358
1
  ccv_nnc_tensor_view_t* dav = ccv_nnc_tensor_view_new(da, GPU_TENSOR_NHWC(000, 32F, 2, 4, 10, 128), ccv_nnc_no_ofs, DIM_ALLOC(10 * 4 * 128, 128, 4 * 128, 1));
2359
1
  ccv_nnc_tensor_view_t* dwv = ccv_nnc_tensor_view_new(dw, GPU_TENSOR_NHWC(000, 32F, 2, 4, 64, 128), ccv_nnc_no_ofs, DIM_ALLOC(64 * 4 * 128, 128, 4 * 128, 1));
2360
1
  ccv_nnc_cmd_exec(CMD_GEMM_BACKWARD(NO_TRANSPOSE, TRANSPOSE(2, 3)), ccv_nnc_no_hint, 0, TENSOR_LIST(b, (ccv_nnc_tensor_t*)av, (ccv_nnc_tensor_t*)wv), TENSOR_LIST((ccv_nnc_tensor_t*)dav, (ccv_nnc_tensor_t*)dwv, dbias), 0);
2361
1
  ccv_nnc_cmd_exec(CMD_GEMM_BACKWARD(NO_TRANSPOSE, TRANSPOSE(2, 3)), ccv_nnc_no_hint, 0, TENSOR_LIST(hb, at, wt), TENSOR_LIST(dat, dwt, tdbias), 0);
2362
1
  ccv_nnc_cmd_exec(CMD_DATA_TRANSFER_FORWARD(), ccv_nnc_no_hint, 0, TENSOR_LIST(da, dw, dbias), TENSOR_LIST(hda, hdw, hdbias), 0);
2363
1
  ccv_nnc_cmd_exec(CMD_TRANSPOSE_FORWARD(1, 2), ccv_nnc_no_hint, 0, TENSOR_LIST(dat), TENSOR_LIST(tda), 0);
2364
1
  ccv_nnc_cmd_exec(CMD_TRANSPOSE_FORWARD(1, 2), ccv_nnc_no_hint, 0, TENSOR_LIST(dwt), TENSOR_LIST(tdw), 0);
2365
1
  REQUIRE_TENSOR_EQ(hda, tda, "permute computed output should be the same as non-permute computed ones");
2366
1
  REQUIRE_TENSOR_EQ(hdw, tdw, "permute computed output should be the same as non-permute computed ones");
2367
1
  REQUIRE_TENSOR_EQ(hdbias, tdbias, "permute computed output should be the same as non-permute computed ones");
2368
1
  ccv_nnc_tensor_free(ha);
2369
1
  ccv_nnc_tensor_free(hw);
2370
1
  ccv_nnc_tensor_free(hda);
2371
1
  ccv_nnc_tensor_free(hdw);
2372
1
  ccv_nnc_tensor_free(hdbias);
2373
1
  ccv_nnc_tensor_free(hb);
2374
1
  ccv_nnc_tensor_free(a);
2375
1
  ccv_nnc_tensor_free(w);
2376
1
  ccv_nnc_tensor_free(da);
2377
1
  ccv_nnc_tensor_free(dw);
2378
1
  ccv_nnc_tensor_free(dbias);
2379
1
  ccv_nnc_tensor_free(b);
2380
1
  ccv_nnc_tensor_view_free(av);
2381
1
  ccv_nnc_tensor_view_free(wv);
2382
1
  ccv_nnc_tensor_view_free(dav);
2383
1
  ccv_nnc_tensor_view_free(dwv);
2384
1
  ccv_nnc_tensor_free(at);
2385
1
  ccv_nnc_tensor_free(wt);
2386
1
  ccv_nnc_tensor_free(dat);
2387
1
  ccv_nnc_tensor_free(dwt);
2388
1
  ccv_nnc_tensor_free(tda);
2389
1
  ccv_nnc_tensor_free(tdw);
2390
1
  ccv_nnc_tensor_free(tdbias);
2391
1
}
2392
2393
TEST_CASE("generalized batched backward gemm with batch (2, 4) with bias and broadcast compare cublas")
2394
1
{
2395
1
  GUARD_ELSE_RETURN(ccv_nnc_cmd_ok(CCV_NNC_GEMM_BACKWARD, CCV_NNC_BACKEND_GPU_CUBLAS));
2396
  // This is a particular batched gemm which treat every dimensions other than the last two as batching.
2397
1
  dsfmt_t dsfmt;
2398
1
  dsfmt_init_gen_rand(&dsfmt, 0);
2399
1
  ccv_nnc_tensor_t* ha = ccv_nnc_tensor_new(0, CPU_TENSOR_NHWC(32F, 2, 10, 4, 128), 0);
2400
1
  ccv_nnc_tensor_t* hw = ccv_nnc_tensor_new(0, CPU_TENSOR_NHWC(32F, 64, 128), 0);
2401
1
  ccv_nnc_tensor_t* hda = ccv_nnc_tensor_new(0, CPU_TENSOR_NHWC(32F, 2, 10, 4, 128), 0);
2402
1
  ccv_nnc_tensor_t* hdw = ccv_nnc_tensor_new(0, CPU_TENSOR_NHWC(32F, 64, 128), 0);
2403
1
  ccv_nnc_tensor_t* hdbias = ccv_nnc_tensor_new(0, CPU_TENSOR_NHWC(32F, 64), 0);
2404
1
  ccv_nnc_tensor_t* hb = ccv_nnc_tensor_new(0, CPU_TENSOR_NHWC(32F, 2, 4, 10, 64), 0);
2405
1
  ccv_nnc_tensor_t* a = ccv_nnc_tensor_new(0, GPU_TENSOR_NHWC(000, 32F, 2, 10, 4, 128), 0);
2406
1
  ccv_nnc_tensor_t* w = ccv_nnc_tensor_new(0, GPU_TENSOR_NHWC(000, 32F, 64, 128), 0);
2407
1
  ccv_nnc_tensor_t* da = ccv_nnc_tensor_new(0, GPU_TENSOR_NHWC(000, 32F, 2, 10, 4, 128), 0);
2408
1
  ccv_nnc_tensor_t* dw = ccv_nnc_tensor_new(0, GPU_TENSOR_NHWC(000, 32F, 64, 128), 0);
2409
1
  ccv_nnc_tensor_t* dbias = ccv_nnc_tensor_new(0, GPU_TENSOR_NHWC(000, 32F, 64), 0);
2410
1
  ccv_nnc_tensor_t* b = ccv_nnc_tensor_new(0, GPU_TENSOR_NHWC(000, 32F, 2, 4, 10, 64), 0);
2411
2412
1
  ccv_nnc_tensor_t* at = ccv_nnc_tensor_new(0, CPU_TENSOR_NHWC(32F, 2, 4, 10, 128), 0);
2413
1
  ccv_nnc_tensor_t* dat = ccv_nnc_tensor_new(0, CPU_TENSOR_NHWC(32F, 2, 4, 10, 128), 0);
2414
1
  ccv_nnc_tensor_t* tda = ccv_nnc_tensor_new(0, CPU_TENSOR_NHWC(32F, 2, 10, 4, 128), 0);
2415
1
  ccv_nnc_tensor_t* tdw = ccv_nnc_tensor_new(0, CPU_TENSOR_NHWC(32F, 64, 128), 0);
2416
1
  ccv_nnc_tensor_t* tdbias = ccv_nnc_tensor_new(0, CPU_TENSOR_NHWC(32F, 64), 0);
2417
1
  int i;
2418
8.19k
  for (i = 0; i < 64 * 128; 
i++8.19k
)
2419
8.19k
    hw->data.f32[i] = dsfmt_genrand_open_close(&dsfmt) / (64 * 128);
2420
10.2k
  for (i = 0; i < 8 * 10 * 128; 
i++10.2k
)
2421
10.2k
    ha->data.f32[i] = dsfmt_genrand_open_close(&dsfmt);
2422
5.12k
  for (i = 0; i < 2 * 4 * 10 * 64; 
i++5.12k
)
2423
5.12k
    hb->data.f32[i] = dsfmt_genrand_open_close(&dsfmt);
2424
1
  ccv_nnc_cmd_exec(CMD_TRANSPOSE_FORWARD(1, 2), ccv_nnc_no_hint, 0, TENSOR_LIST(ha), TENSOR_LIST(at), 0);
2425
1
  ccv_nnc_cmd_exec(CMD_DATA_TRANSFER_FORWARD(), ccv_nnc_no_hint, 0, TENSOR_LIST(ha, hw, hb), TENSOR_LIST(a, w, b), 0);
2426
1
  ccv_nnc_tensor_view_t* av = ccv_nnc_tensor_view_new(a, GPU_TENSOR_NHWC(000, 32F, 2, 4, 10, 128), ccv_nnc_no_ofs, DIM_ALLOC(10 * 4 * 128, 128, 4 * 128, 1));
2427
1
  ccv_nnc_tensor_view_t* dav = ccv_nnc_tensor_view_new(da, GPU_TENSOR_NHWC(000, 32F, 2, 4, 10, 128), ccv_nnc_no_ofs, DIM_ALLOC(10 * 4 * 128, 128, 4 * 128, 1));
2428
1
  ccv_nnc_cmd_exec(CMD_GEMM_BACKWARD(NO_TRANSPOSE, TRANSPOSE(0, 1)), ccv_nnc_no_hint, 0, TENSOR_LIST(b, (ccv_nnc_tensor_t*)av, w, dbias), TENSOR_LIST((ccv_nnc_tensor_t*)dav, dw, dbias), 0);
2429
1
  ccv_nnc_cmd_exec(CMD_GEMM_BACKWARD(NO_TRANSPOSE, TRANSPOSE(0, 1)), ccv_nnc_no_hint, 0, TENSOR_LIST(hb, at, hw, hdbias), TENSOR_LIST(dat, tdw, tdbias), 0);
2430
1
  ccv_nnc_cmd_exec(CMD_DATA_TRANSFER_FORWARD(), ccv_nnc_no_hint, 0, TENSOR_LIST(da, dw, dbias), TENSOR_LIST(hda, hdw, hdbias), 0);
2431
1
  ccv_nnc_cmd_exec(CMD_TRANSPOSE_FORWARD(1, 2), ccv_nnc_no_hint, 0, TENSOR_LIST(dat), TENSOR_LIST(tda), 0);
2432
1
  REQUIRE_TENSOR_EQ(hda, tda, "permute computed output should be the same as non-permute computed ones");
2433
1
  REQUIRE_TENSOR_EQ(hdw, tdw, "permute computed output should be the same as non-permute computed ones");
2434
1
  REQUIRE_TENSOR_EQ(hdbias, tdbias, "permute computed output should be the same as non-permute computed ones");
2435
1
  ccv_nnc_tensor_free(ha);
2436
1
  ccv_nnc_tensor_free(hw);
2437
1
  ccv_nnc_tensor_free(hda);
2438
1
  ccv_nnc_tensor_free(hdw);
2439
1
  ccv_nnc_tensor_free(hdbias);
2440
1
  ccv_nnc_tensor_free(hb);
2441
1
  ccv_nnc_tensor_free(a);
2442
1
  ccv_nnc_tensor_free(w);
2443
1
  ccv_nnc_tensor_free(da);
2444
1
  ccv_nnc_tensor_free(dw);
2445
1
  ccv_nnc_tensor_free(dbias);
2446
1
  ccv_nnc_tensor_free(b);
2447
1
  ccv_nnc_tensor_view_free(av);
2448
1
  ccv_nnc_tensor_view_free(dav);
2449
1
  ccv_nnc_tensor_free(at);
2450
1
  ccv_nnc_tensor_free(dat);
2451
1
  ccv_nnc_tensor_free(tdw);
2452
1
  ccv_nnc_tensor_free(tdbias);
2453
1
}
2454
2455
TEST_CASE("ewdiv forward with reciprocal")
2456
1
{
2457
1
  GUARD_ELSE_RETURN(ccv_nnc_cmd_ok(CCV_NNC_EWDIV_FORWARD, CCV_NNC_BACKEND_GPU_REF));
2458
1
  ccv_nnc_tensor_t* a = ccv_nnc_tensor_new(0, GPU_TENSOR_NCHW(000, 32F, 10, 100), 0);
2459
1
  ccv_nnc_tensor_t* b = ccv_nnc_tensor_new(0, GPU_TENSOR_NCHW(000, 32F, 10, 100), 0);
2460
1
  ccv_nnc_tensor_t* ha = ccv_nnc_tensor_new(0, CPU_TENSOR_NCHW(32F, 10, 100), 0);
2461
1
  ccv_nnc_tensor_t* hb = ccv_nnc_tensor_new(0, CPU_TENSOR_NCHW(32F, 10, 100), 0);
2462
1
  ccv_nnc_tensor_t* bt = ccv_nnc_tensor_new(0, CPU_TENSOR_NCHW(32F, 10, 100), 0);
2463
1
  dsfmt_t dsfmt;
2464
1
  dsfmt_init_gen_rand(&dsfmt, 0);
2465
1
  int i;
2466
1.00k
  for (i = 0; i < 1000; 
i++1.00k
)
2467
1.00k
    ha->data.f32[i] = dsfmt_genrand_open_close(&dsfmt) * 0.01;
2468
1
  ccv_nnc_cmd_exec(CMD_DATA_TRANSFER_FORWARD(), ccv_nnc_no_hint, 0, TENSOR_LIST(ha), TENSOR_LIST(a), 0);
2469
1
  ccv_nnc_cmd_exec(CMD_EWDIV_FORWARD(), ccv_nnc_no_hint, 0, TENSOR_LIST(0, a), TENSOR_LIST(b), 0);
2470
1
  ccv_nnc_cmd_exec(CMD_EWDIV_FORWARD(), ccv_nnc_no_hint, 0, TENSOR_LIST(0, ha), TENSOR_LIST(bt), 0);
2471
1
  ccv_nnc_cmd_exec(CMD_DATA_TRANSFER_FORWARD(), ccv_nnc_no_hint, 0, TENSOR_LIST(b), TENSOR_LIST(hb), 0);
2472
1
  REQUIRE_TENSOR_EQ(bt, hb, "GPU computed output should be the same as CPU computed ones");
2473
1
  ccv_nnc_tensor_free(a);
2474
1
  ccv_nnc_tensor_free(b);
2475
1
  ccv_nnc_tensor_free(ha);
2476
1
  ccv_nnc_tensor_free(hb);
2477
1
  ccv_nnc_tensor_free(bt);
2478
1
}
2479
2480
TEST_CASE("ewdiv forward")
2481
1
{
2482
1
  GUARD_ELSE_RETURN(ccv_nnc_cmd_ok(CCV_NNC_EWDIV_FORWARD, CCV_NNC_BACKEND_GPU_REF));
2483
1
  ccv_nnc_tensor_t* a = ccv_nnc_tensor_new(0, GPU_TENSOR_NCHW(000, 32F, 10, 100), 0);
2484
1
  ccv_nnc_tensor_t* b = ccv_nnc_tensor_new(0, GPU_TENSOR_NCHW(000, 32F, 10, 100), 0);
2485
1
  ccv_nnc_tensor_t* c = ccv_nnc_tensor_new(0, GPU_TENSOR_NCHW(000, 32F, 10, 100), 0);
2486
1
  ccv_nnc_tensor_t* ha = ccv_nnc_tensor_new(0, CPU_TENSOR_NCHW(32F, 10, 100), 0);
2487
1
  ccv_nnc_tensor_t* hb = ccv_nnc_tensor_new(0, CPU_TENSOR_NCHW(32F, 10, 100), 0);
2488
1
  ccv_nnc_tensor_t* hc = ccv_nnc_tensor_new(0, CPU_TENSOR_NCHW(32F, 10, 100), 0);
2489
1
  ccv_nnc_tensor_t* ct = ccv_nnc_tensor_new(0, CPU_TENSOR_NCHW(32F, 10, 100), 0);
2490
1
  dsfmt_t dsfmt;
2491
1
  dsfmt_init_gen_rand(&dsfmt, 0);
2492
1
  int i;
2493
1.00k
  for (i = 0; i < 1000; 
i++1.00k
)
2494
1.00k
    ha->data.f32[i] = dsfmt_genrand_open_close(&dsfmt) * 0.01;
2495
1.00k
  for (i = 0; i < 1000; 
i++1.00k
)
2496
1.00k
    hb->data.f32[i] = dsfmt_genrand_open_close(&dsfmt) * 0.01;
2497
1
  ccv_nnc_cmd_exec(CMD_DATA_TRANSFER_FORWARD(), ccv_nnc_no_hint, 0, TENSOR_LIST(ha, hb), TENSOR_LIST(a, b), 0);
2498
1
  ccv_nnc_cmd_exec(CMD_EWDIV_FORWARD(), ccv_nnc_no_hint, 0, TENSOR_LIST(a, b), TENSOR_LIST(c), 0);
2499
1
  ccv_nnc_cmd_exec(CMD_EWDIV_FORWARD(), ccv_nnc_no_hint, 0, TENSOR_LIST(ha, hb), TENSOR_LIST(ct), 0);
2500
1
  ccv_nnc_cmd_exec(CMD_DATA_TRANSFER_FORWARD(), ccv_nnc_no_hint, 0, TENSOR_LIST(c), TENSOR_LIST(hc), 0);
2501
1
  REQUIRE_TENSOR_EQ(ct, hc, "GPU computed output should be the same as CPU computed ones");
2502
1
  ccv_nnc_tensor_free(a);
2503
1
  ccv_nnc_tensor_free(b);
2504
1
  ccv_nnc_tensor_free(c);
2505
1
  ccv_nnc_tensor_free(ha);
2506
1
  ccv_nnc_tensor_free(hb);
2507
1
  ccv_nnc_tensor_free(hc);
2508
1
  ccv_nnc_tensor_free(ct);
2509
1
}
2510
2511
TEST_CASE("ewdiv backward with output 1")
2512
1
{
2513
1
  GUARD_ELSE_RETURN(ccv_nnc_cmd_ok(CCV_NNC_EWDIV_FORWARD, CCV_NNC_BACKEND_GPU_REF) &&
2514
1
    ccv_nnc_cmd_ok(CCV_NNC_EWDIV_BACKWARD, CCV_NNC_BACKEND_GPU_REF));
2515
1
  ccv_nnc_tensor_t* a = ccv_nnc_tensor_new(0, GPU_TENSOR_NCHW(000, 32F, 10, 100), 0);
2516
1
  ccv_nnc_tensor_t* b = ccv_nnc_tensor_new(0, GPU_TENSOR_NCHW(000, 32F, 10, 100), 0);
2517
1
  ccv_nnc_tensor_t* c = ccv_nnc_tensor_new(0, GPU_TENSOR_NCHW(000, 32F, 10, 100), 0);
2518
1
  ccv_nnc_tensor_t* g = ccv_nnc_tensor_new(0, GPU_TENSOR_NCHW(000, 32F, 10, 100), 0);
2519
1
  ccv_nnc_tensor_t* da = ccv_nnc_tensor_new(0, GPU_TENSOR_NCHW(000, 32F, 10, 100), 0);
2520
1
  ccv_nnc_tensor_t* ha = ccv_nnc_tensor_new(0, CPU_TENSOR_NCHW(32F, 10, 100), 0);
2521
1
  ccv_nnc_tensor_t* hda = ccv_nnc_tensor_new(0, CPU_TENSOR_NCHW(32F, 10, 100), 0);
2522
1
  ccv_nnc_tensor_t* hb = ccv_nnc_tensor_new(0, CPU_TENSOR_NCHW(32F, 10, 100), 0);
2523
1
  ccv_nnc_tensor_t* hc = ccv_nnc_tensor_new(0, CPU_TENSOR_NCHW(32F, 10, 100), 0);
2524
1
  ccv_nnc_tensor_t* hg = ccv_nnc_tensor_new(0, CPU_TENSOR_NCHW(32F, 10, 100), 0);
2525
1
  ccv_nnc_tensor_t* dat = ccv_nnc_tensor_new(0, CPU_TENSOR_NCHW(32F, 10, 100), 0);
2526
1
  dsfmt_t dsfmt;
2527
1
  dsfmt_init_gen_rand(&dsfmt, 0);
2528
1
  int i;
2529
1.00k
  for (i = 0; i < 1000; 
i++1.00k
)
2530
1.00k
    ha->data.f32[i] = dsfmt_genrand_open_close(&dsfmt) * 0.01;
2531
1.00k
  for (i = 0; i < 1000; 
i++1.00k
)
2532
1.00k
    hb->data.f32[i] = dsfmt_genrand_open_close(&dsfmt) * 0.01;
2533
1.00k
  for (i = 0; i < 1000; 
i++1.00k
)
2534
1.00k
    hg->data.f32[i] = dsfmt_genrand_open_close(&dsfmt);
2535
1
  ccv_nnc_cmd_exec(CMD_DATA_TRANSFER_FORWARD(), ccv_nnc_no_hint, 0, TENSOR_LIST(ha, hb, hg), TENSOR_LIST(a, b, g), 0);
2536
1
  ccv_nnc_cmd_exec(CMD_EWDIV_FORWARD(), ccv_nnc_no_hint, 0, TENSOR_LIST(a, b), TENSOR_LIST(c), 0);
2537
1
  ccv_nnc_cmd_exec(CMD_EWDIV_BACKWARD(), ccv_nnc_no_hint, 0, TENSOR_LIST(g, 0, b), TENSOR_LIST(da), 0);
2538
1
  ccv_nnc_cmd_exec(CMD_EWDIV_FORWARD(), ccv_nnc_no_hint, 0, TENSOR_LIST(ha, hb), TENSOR_LIST(hc), 0);
2539
1
  ccv_nnc_cmd_exec(CMD_EWDIV_BACKWARD(), ccv_nnc_no_hint, 0, TENSOR_LIST(hg, 0, hb), TENSOR_LIST(dat), 0);
2540
1
  ccv_nnc_cmd_exec(CMD_DATA_TRANSFER_FORWARD(), ccv_nnc_no_hint, 0, TENSOR_LIST(da), TENSOR_LIST(hda), 0);
2541
1
  REQUIRE_TENSOR_EQ(dat, hda, "GPU computed output should be the same as CPU computed ones");
2542
1
  ccv_nnc_tensor_free(a);
2543
1
  ccv_nnc_tensor_free(b);
2544
1
  ccv_nnc_tensor_free(c);
2545
1
  ccv_nnc_tensor_free(g);
2546
1
  ccv_nnc_tensor_free(da);
2547
1
  ccv_nnc_tensor_free(ha);
2548
1
  ccv_nnc_tensor_free(hb);
2549
1
  ccv_nnc_tensor_free(hc);
2550
1
  ccv_nnc_tensor_free(hg);
2551
1
  ccv_nnc_tensor_free(hda);
2552
1
  ccv_nnc_tensor_free(dat);
2553
1
}
2554
2555
TEST_CASE("ewdiv backward with output 2")
2556
1
{
2557
1
  GUARD_ELSE_RETURN(ccv_nnc_cmd_ok(CCV_NNC_EWDIV_FORWARD, CCV_NNC_BACKEND_GPU_REF) &&
2558
1
    ccv_nnc_cmd_ok(CCV_NNC_EWDIV_BACKWARD, CCV_NNC_BACKEND_GPU_REF));
2559
1
  ccv_nnc_tensor_t* a = ccv_nnc_tensor_new(0, GPU_TENSOR_NCHW(000, 32F, 10, 100), 0);
2560
1
  ccv_nnc_tensor_t* b = ccv_nnc_tensor_new(0, GPU_TENSOR_NCHW(000, 32F, 10, 100), 0);
2561
1
  ccv_nnc_tensor_t* c = ccv_nnc_tensor_new(0, GPU_TENSOR_NCHW(000, 32F, 10, 100), 0);
2562
1
  ccv_nnc_tensor_t* g = ccv_nnc_tensor_new(0, GPU_TENSOR_NCHW(000, 32F, 10, 100), 0);
2563
1
  ccv_nnc_tensor_t* db = ccv_nnc_tensor_new(0, GPU_TENSOR_NCHW(000, 32F, 10, 100), 0);
2564
1
  ccv_nnc_tensor_t* ha = ccv_nnc_tensor_new(0, CPU_TENSOR_NCHW(32F, 10, 100), 0);
2565
1
  ccv_nnc_tensor_t* hdb = ccv_nnc_tensor_new(0, CPU_TENSOR_NCHW(32F, 10, 100), 0);
2566
1
  ccv_nnc_tensor_t* hb = ccv_nnc_tensor_new(0, CPU_TENSOR_NCHW(32F, 10, 100), 0);
2567
1
  ccv_nnc_tensor_t* hc = ccv_nnc_tensor_new(0, CPU_TENSOR_NCHW(32F, 10, 100), 0);
2568
1
  ccv_nnc_tensor_t* hg = ccv_nnc_tensor_new(0, CPU_TENSOR_NCHW(32F, 10, 100), 0);
2569
1
  ccv_nnc_tensor_t* dbt = ccv_nnc_tensor_new(0, CPU_TENSOR_NCHW(32F, 10, 100), 0);
2570
1
  dsfmt_t dsfmt;
2571
1
  dsfmt_init_gen_rand(&dsfmt, 0);
2572
1
  int i;
2573
1.00k
  for (i = 0; i < 1000; 
i++1.00k
)
2574
1.00k
    ha->data.f32[i] = dsfmt_genrand_open_close(&dsfmt) * 0.01;
2575
1.00k
  for (i = 0; i < 1000; 
i++1.00k
)
2576
1.00k
    hb->data.f32[i] = dsfmt_genrand_open_close(&dsfmt) * 0.01;
2577
1.00k
  for (i = 0; i < 1000; 
i++1.00k
)
2578
1.00k
    hg->data.f32[i] = dsfmt_genrand_open_close(&dsfmt);
2579
1
  ccv_nnc_cmd_exec(CMD_DATA_TRANSFER_FORWARD(), ccv_nnc_no_hint, 0, TENSOR_LIST(ha, hb, hg), TENSOR_LIST(a, b, g), 0);
2580
1
  ccv_nnc_cmd_exec(CMD_EWDIV_FORWARD(), ccv_nnc_no_hint, 0, TENSOR_LIST(a, b), TENSOR_LIST(c), 0);
2581
1
  ccv_nnc_cmd_exec(CMD_EWDIV_BACKWARD(), ccv_nnc_no_hint, 0, TENSOR_LIST(g, 0, b, c), TENSOR_LIST(0, db), 0);
2582
1
  ccv_nnc_cmd_exec(CMD_EWDIV_FORWARD(), ccv_nnc_no_hint, 0, TENSOR_LIST(ha, hb), TENSOR_LIST(hc), 0);
2583
1
  ccv_nnc_cmd_exec(CMD_EWDIV_BACKWARD(), ccv_nnc_no_hint, 0, TENSOR_LIST(hg, 0, hb, hc), TENSOR_LIST(0, dbt), 0);
2584
1
  ccv_nnc_cmd_exec(CMD_DATA_TRANSFER_FORWARD(), ccv_nnc_no_hint, 0, TENSOR_LIST(db), TENSOR_LIST(hdb), 0);
2585
1
  REQUIRE_TENSOR_EQ(dbt, hdb, "GPU computed output should be the same as CPU computed ones");
2586
1
  ccv_nnc_tensor_free(a);
2587
1
  ccv_nnc_tensor_free(b);
2588
1
  ccv_nnc_tensor_free(c);
2589
1
  ccv_nnc_tensor_free(g);
2590
1
  ccv_nnc_tensor_free(db);
2591
1
  ccv_nnc_tensor_free(ha);
2592
1
  ccv_nnc_tensor_free(hb);
2593
1
  ccv_nnc_tensor_free(hc);
2594
1
  ccv_nnc_tensor_free(hg);
2595
1
  ccv_nnc_tensor_free(hdb);
2596
1
  ccv_nnc_tensor_free(dbt);
2597
1
}
2598
2599
static ccv_nnc_tensor_param_t _ccv_nnc_ew_cpu_nchw_datatype(const int datatype)
2600
20
{
2601
20
  ccv_nnc_tensor_param_t params = CPU_TENSOR_NCHW(32F, 10, 100);
2602
20
  params.datatype = datatype;
2603
20
  return params;
2604
20
}
2605
2606
static ccv_nnc_tensor_param_t _ccv_nnc_ew_gpu_nchw_datatype(const int datatype)
2607
16
{
2608
16
  ccv_nnc_tensor_param_t params = GPU_TENSOR_NCHW(000, 32F, 10, 100);
2609
16
  params.datatype = datatype;
2610
16
  return params;
2611
16
}
2612
2613
static void _ccv_nnc_ewexp_gpu_ref_datatype(const int datatype, float* const forward_max_diff, float* const backward_max_diff)
2614
2
{
2615
2
  ccv_nnc_tensor_t* const ha32 = ccv_nnc_tensor_new(0, CPU_TENSOR_NCHW(32F, 10, 100), 0);
2616
2
  ccv_nnc_tensor_t* const hg32 = ccv_nnc_tensor_new(0, CPU_TENSOR_NCHW(32F, 10, 100), 0);
2617
2
  ccv_nnc_tensor_t* const ha = ccv_nnc_tensor_new(0, _ccv_nnc_ew_cpu_nchw_datatype(datatype), 0);
2618
2
  ccv_nnc_tensor_t* const hg = ccv_nnc_tensor_new(0, _ccv_nnc_ew_cpu_nchw_datatype(datatype), 0);
2619
2
  ccv_nnc_tensor_t* const har = ccv_nnc_tensor_new(0, CPU_TENSOR_NCHW(32F, 10, 100), 0);
2620
2
  ccv_nnc_tensor_t* const hgr = ccv_nnc_tensor_new(0, CPU_TENSOR_NCHW(32F, 10, 100), 0);
2621
2
  ccv_nnc_tensor_t* const hb = ccv_nnc_tensor_new(0, _ccv_nnc_ew_cpu_nchw_datatype(datatype), 0);
2622
2
  ccv_nnc_tensor_t* const hb32 = ccv_nnc_tensor_new(0, CPU_TENSOR_NCHW(32F, 10, 100), 0);
2623
2
  ccv_nnc_tensor_t* const hbr = ccv_nnc_tensor_new(0, CPU_TENSOR_NCHW(32F, 10, 100), 0);
2624
2
  ccv_nnc_tensor_t* const hda = ccv_nnc_tensor_new(0, _ccv_nnc_ew_cpu_nchw_datatype(datatype), 0);
2625
2
  ccv_nnc_tensor_t* const hda32 = ccv_nnc_tensor_new(0, CPU_TENSOR_NCHW(32F, 10, 100), 0);
2626
2
  ccv_nnc_tensor_t* const dat = ccv_nnc_tensor_new(0, _ccv_nnc_ew_cpu_nchw_datatype(datatype), 0);
2627
2
  ccv_nnc_tensor_t* const dat32 = ccv_nnc_tensor_new(0, CPU_TENSOR_NCHW(32F, 10, 100), 0);
2628
2
  ccv_nnc_tensor_t* const datr = ccv_nnc_tensor_new(0, CPU_TENSOR_NCHW(32F, 10, 100), 0);
2629
2
  ccv_nnc_tensor_t* const a = ccv_nnc_tensor_new(0, _ccv_nnc_ew_gpu_nchw_datatype(datatype), 0);
2630
2
  ccv_nnc_tensor_t* const b = ccv_nnc_tensor_new(0, _ccv_nnc_ew_gpu_nchw_datatype(datatype), 0);
2631
2
  ccv_nnc_tensor_t* const g = ccv_nnc_tensor_new(0, _ccv_nnc_ew_gpu_nchw_datatype(datatype), 0);
2632
2
  ccv_nnc_tensor_t* const da = ccv_nnc_tensor_new(0, _ccv_nnc_ew_gpu_nchw_datatype(datatype), 0);
2633
2
  dsfmt_t dsfmt;
2634
2
  dsfmt_init_gen_rand(&dsfmt, 0);
2635
2
  int i;
2636
2.00k
  for (i = 0; i < 1000; 
i++2.00k
)
2637
2.00k
    ha32->data.f32[i] = dsfmt_genrand_open_close(&dsfmt) * 6 - 3;
2638
2.00k
  for (i = 0; i < 1000; 
i++2.00k
)
2639
2.00k
    hg32->data.f32[i] = dsfmt_genrand_open_close(&dsfmt);
2640
2
  ccv_nnc_cmd_exec(CMD_DATATYPE_CONVERSION_FORWARD(), ccv_nnc_no_hint, 0, TENSOR_LIST(ha32, hg32), TENSOR_LIST(ha, hg), 0);
2641
2
  ccv_nnc_cmd_exec(CMD_DATATYPE_CONVERSION_FORWARD(), ccv_nnc_no_hint, 0, TENSOR_LIST(ha, hg), TENSOR_LIST(har, hgr), 0);
2642
2
  ccv_nnc_cmd_exec(CMD_DATA_TRANSFER_FORWARD(), ccv_nnc_no_hint, 0, TENSOR_LIST(ha, hg), TENSOR_LIST(a, g), 0);
2643
2
  ccv_nnc_cmd_exec(CMD_EWEXP_FORWARD(), ccv_nnc_no_hint, 0, TENSOR_LIST(a), TENSOR_LIST(b), 0);
2644
2
  ccv_nnc_cmd_exec(CMD_EWEXP_FORWARD(), ccv_nnc_no_hint, 0, TENSOR_LIST(har), TENSOR_LIST(hb32), 0);
2645
2
  ccv_nnc_cmd_exec(CMD_DATATYPE_CONVERSION_FORWARD(), ccv_nnc_no_hint, 0, TENSOR_LIST(hb32), TENSOR_LIST(hb), 0);
2646
2
  ccv_nnc_cmd_exec(CMD_DATATYPE_CONVERSION_FORWARD(), ccv_nnc_no_hint, 0, TENSOR_LIST(hb), TENSOR_LIST(hbr), 0);
2647
2
  ccv_nnc_cmd_exec(CMD_DATA_TRANSFER_FORWARD(), ccv_nnc_no_hint, 0, TENSOR_LIST(b), TENSOR_LIST(hb), 0);
2648
2
  ccv_nnc_cmd_exec(CMD_DATATYPE_CONVERSION_FORWARD(), ccv_nnc_no_hint, 0, TENSOR_LIST(hb), TENSOR_LIST(hb32), 0);
2649
2
  *forward_max_diff = 0;
2650
2.00k
  for (i = 0; i < 1000; 
i++2.00k
)
2651
2.00k
  {
2652
2.00k
    const float max_val = ccv_max(ccv_max(fabsf(hbr->data.f32[i]), fabsf(hb32->data.f32[i])), 1);
2653
2.00k
    *forward_max_diff = ccv_max(*forward_max_diff, fabsf(hbr->data.f32[i] - hb32->data.f32[i]) / max_val);
2654
2.00k
  }
2655
2
  ccv_nnc_cmd_exec(CMD_EWEXP_BACKWARD(), ccv_nnc_no_hint, 0, TENSOR_LIST(g, 0, b), TENSOR_LIST(da), 0);
2656
2
  ccv_nnc_cmd_exec(CMD_EWEXP_BACKWARD(), ccv_nnc_no_hint, 0, TENSOR_LIST(hgr, 0, hbr), TENSOR_LIST(dat32), 0);
2657
2
  ccv_nnc_cmd_exec(CMD_DATATYPE_CONVERSION_FORWARD(), ccv_nnc_no_hint, 0, TENSOR_LIST(dat32), TENSOR_LIST(dat), 0);
2658
2
  ccv_nnc_cmd_exec(CMD_DATATYPE_CONVERSION_FORWARD(), ccv_nnc_no_hint, 0, TENSOR_LIST(dat), TENSOR_LIST(datr), 0);
2659
2
  ccv_nnc_cmd_exec(CMD_DATA_TRANSFER_FORWARD(), ccv_nnc_no_hint, 0, TENSOR_LIST(da), TENSOR_LIST(hda), 0);
2660
2
  ccv_nnc_cmd_exec(CMD_DATATYPE_CONVERSION_FORWARD(), ccv_nnc_no_hint, 0, TENSOR_LIST(hda), TENSOR_LIST(hda32), 0);
2661
2
  *backward_max_diff = 0;
2662
2.00k
  for (i = 0; i < 1000; 
i++2.00k
)
2663
2.00k
  {
2664
2.00k
    const float max_val = ccv_max(ccv_max(fabsf(datr->data.f32[i]), fabsf(hda32->data.f32[i])), 1);
2665
2.00k
    *backward_max_diff = ccv_max(*backward_max_diff, fabsf(datr->data.f32[i] - hda32->data.f32[i]) / max_val);
2666
2.00k
  }
2667
2
  ccv_nnc_tensor_free(ha32);
2668
2
  ccv_nnc_tensor_free(hg32);
2669
2
  ccv_nnc_tensor_free(ha);
2670
2
  ccv_nnc_tensor_free(hg);
2671
2
  ccv_nnc_tensor_free(har);
2672
2
  ccv_nnc_tensor_free(hgr);
2673
2
  ccv_nnc_tensor_free(hb);
2674
2
  ccv_nnc_tensor_free(hb32);
2675
2
  ccv_nnc_tensor_free(hbr);
2676
2
  ccv_nnc_tensor_free(hda);
2677
2
  ccv_nnc_tensor_free(hda32);
2678
2
  ccv_nnc_tensor_free(dat);
2679
2
  ccv_nnc_tensor_free(dat32);
2680
2
  ccv_nnc_tensor_free(datr);
2681
2
  ccv_nnc_tensor_free(a);
2682
2
  ccv_nnc_tensor_free(b);
2683
2
  ccv_nnc_tensor_free(g);
2684
2
  ccv_nnc_tensor_free(da);
2685
2
}
2686
2687
static void _ccv_nnc_ewsoftplus_gpu_ref_datatype(const int datatype, float* const forward_max_diff, float* const backward_max_diff)
2688
2
{
2689
2
  ccv_nnc_tensor_t* const ha32 = ccv_nnc_tensor_new(0, CPU_TENSOR_NCHW(32F, 10, 100), 0);
2690
2
  ccv_nnc_tensor_t* const hg32 = ccv_nnc_tensor_new(0, CPU_TENSOR_NCHW(32F, 10, 100), 0);
2691
2
  ccv_nnc_tensor_t* const ha = ccv_nnc_tensor_new(0, _ccv_nnc_ew_cpu_nchw_datatype(datatype), 0);
2692
2
  ccv_nnc_tensor_t* const hg = ccv_nnc_tensor_new(0, _ccv_nnc_ew_cpu_nchw_datatype(datatype), 0);
2693
2
  ccv_nnc_tensor_t* const har = ccv_nnc_tensor_new(0, CPU_TENSOR_NCHW(32F, 10, 100), 0);
2694
2
  ccv_nnc_tensor_t* const hgr = ccv_nnc_tensor_new(0, CPU_TENSOR_NCHW(32F, 10, 100), 0);
2695
2
  ccv_nnc_tensor_t* const hb = ccv_nnc_tensor_new(0, _ccv_nnc_ew_cpu_nchw_datatype(datatype), 0);
2696
2
  ccv_nnc_tensor_t* const hb32 = ccv_nnc_tensor_new(0, CPU_TENSOR_NCHW(32F, 10, 100), 0);
2697
2
  ccv_nnc_tensor_t* const hbr = ccv_nnc_tensor_new(0, CPU_TENSOR_NCHW(32F, 10, 100), 0);
2698
2
  ccv_nnc_tensor_t* const hda = ccv_nnc_tensor_new(0, _ccv_nnc_ew_cpu_nchw_datatype(datatype), 0);
2699
2
  ccv_nnc_tensor_t* const hda32 = ccv_nnc_tensor_new(0, CPU_TENSOR_NCHW(32F, 10, 100), 0);
2700
2
  ccv_nnc_tensor_t* const dat = ccv_nnc_tensor_new(0, _ccv_nnc_ew_cpu_nchw_datatype(datatype), 0);
2701
2
  ccv_nnc_tensor_t* const dat32 = ccv_nnc_tensor_new(0, CPU_TENSOR_NCHW(32F, 10, 100), 0);
2702
2
  ccv_nnc_tensor_t* const datr = ccv_nnc_tensor_new(0, CPU_TENSOR_NCHW(32F, 10, 100), 0);
2703
2
  ccv_nnc_tensor_t* const a = ccv_nnc_tensor_new(0, _ccv_nnc_ew_gpu_nchw_datatype(datatype), 0);
2704
2
  ccv_nnc_tensor_t* const b = ccv_nnc_tensor_new(0, _ccv_nnc_ew_gpu_nchw_datatype(datatype), 0);
2705
2
  ccv_nnc_tensor_t* const g = ccv_nnc_tensor_new(0, _ccv_nnc_ew_gpu_nchw_datatype(datatype), 0);
2706
2
  ccv_nnc_tensor_t* const da = ccv_nnc_tensor_new(0, _ccv_nnc_ew_gpu_nchw_datatype(datatype), 0);
2707
2
  dsfmt_t dsfmt;
2708
2
  dsfmt_init_gen_rand(&dsfmt, 0);
2709
2
  int i;
2710
2.00k
  for (i = 0; i < 1000; 
i++2.00k
)
2711
2.00k
    ha32->data.f32[i] = dsfmt_genrand_open_close(&dsfmt) * 16 - 8;
2712
2.00k
  for (i = 0; i < 1000; 
i++2.00k
)
2713
2.00k
    hg32->data.f32[i] = dsfmt_genrand_open_close(&dsfmt);
2714
2
  ccv_nnc_cmd_exec(CMD_DATATYPE_CONVERSION_FORWARD(), ccv_nnc_no_hint, 0, TENSOR_LIST(ha32, hg32), TENSOR_LIST(ha, hg), 0);
2715
2
  ccv_nnc_cmd_exec(CMD_DATATYPE_CONVERSION_FORWARD(), ccv_nnc_no_hint, 0, TENSOR_LIST(ha, hg), TENSOR_LIST(har, hgr), 0);
2716
2
  ccv_nnc_cmd_exec(CMD_DATA_TRANSFER_FORWARD(), ccv_nnc_no_hint, 0, TENSOR_LIST(ha, hg), TENSOR_LIST(a, g), 0);
2717
2
  ccv_nnc_cmd_exec(CMD_EWSOFTPLUS_FORWARD(), ccv_nnc_no_hint, 0, TENSOR_LIST(a), TENSOR_LIST(b), 0);
2718
2
  ccv_nnc_cmd_exec(CMD_EWSOFTPLUS_FORWARD(), ccv_nnc_no_hint, 0, TENSOR_LIST(har), TENSOR_LIST(hb32), 0);
2719
2
  ccv_nnc_cmd_exec(CMD_DATATYPE_CONVERSION_FORWARD(), ccv_nnc_no_hint, 0, TENSOR_LIST(hb32), TENSOR_LIST(hb), 0);
2720
2
  ccv_nnc_cmd_exec(CMD_DATATYPE_CONVERSION_FORWARD(), ccv_nnc_no_hint, 0, TENSOR_LIST(hb), TENSOR_LIST(hbr), 0);
2721
2
  ccv_nnc_cmd_exec(CMD_DATA_TRANSFER_FORWARD(), ccv_nnc_no_hint, 0, TENSOR_LIST(b), TENSOR_LIST(hb), 0);
2722
2
  ccv_nnc_cmd_exec(CMD_DATATYPE_CONVERSION_FORWARD(), ccv_nnc_no_hint, 0, TENSOR_LIST(hb), TENSOR_LIST(hb32), 0);
2723
2
  *forward_max_diff = 0;
2724
2.00k
  for (i = 0; i < 1000; 
i++2.00k
)
2725
2.00k
  {
2726
2.00k
    const float max_val = ccv_max(ccv_max(fabsf(hbr->data.f32[i]), fabsf(hb32->data.f32[i])), 1);
2727
2.00k
    *forward_max_diff = ccv_max(*forward_max_diff, fabsf(hbr->data.f32[i] - hb32->data.f32[i]) / max_val);
2728
2.00k
  }
2729
2
  ccv_nnc_cmd_exec(CMD_EWSOFTPLUS_BACKWARD(), ccv_nnc_no_hint, 0, TENSOR_LIST(g, a), TENSOR_LIST(da), 0);
2730
2
  ccv_nnc_cmd_exec(CMD_EWSOFTPLUS_BACKWARD(), ccv_nnc_no_hint, 0, TENSOR_LIST(hgr, har), TENSOR_LIST(dat32), 0);
2731
2
  ccv_nnc_cmd_exec(CMD_DATATYPE_CONVERSION_FORWARD(), ccv_nnc_no_hint, 0, TENSOR_LIST(dat32), TENSOR_LIST(dat), 0);
2732
2
  ccv_nnc_cmd_exec(CMD_DATATYPE_CONVERSION_FORWARD(), ccv_nnc_no_hint, 0, TENSOR_LIST(dat), TENSOR_LIST(datr), 0);
2733
2
  ccv_nnc_cmd_exec(CMD_DATA_TRANSFER_FORWARD(), ccv_nnc_no_hint, 0, TENSOR_LIST(da), TENSOR_LIST(hda), 0);
2734
2
  ccv_nnc_cmd_exec(CMD_DATATYPE_CONVERSION_FORWARD(), ccv_nnc_no_hint, 0, TENSOR_LIST(hda), TENSOR_LIST(hda32), 0);
2735
2
  *backward_max_diff = 0;
2736
2.00k
  for (i = 0; i < 1000; 
i++2.00k
)
2737
2.00k
  {
2738
2.00k
    const float max_val = ccv_max(ccv_max(fabsf(datr->data.f32[i]), fabsf(hda32->data.f32[i])), 1);
2739
2.00k
    *backward_max_diff = ccv_max(*backward_max_diff, fabsf(datr->data.f32[i] - hda32->data.f32[i]) / max_val);
2740
2.00k
  }
2741
2
  ccv_nnc_tensor_free(ha32);
2742
2
  ccv_nnc_tensor_free(hg32);
2743
2
  ccv_nnc_tensor_free(ha);
2744
2
  ccv_nnc_tensor_free(hg);
2745
2
  ccv_nnc_tensor_free(har);
2746
2
  ccv_nnc_tensor_free(hgr);
2747
2
  ccv_nnc_tensor_free(hb);
2748
2
  ccv_nnc_tensor_free(hb32);
2749
2
  ccv_nnc_tensor_free(hbr);
2750
2
  ccv_nnc_tensor_free(hda);
2751
2
  ccv_nnc_tensor_free(hda32);
2752
2
  ccv_nnc_tensor_free(dat);
2753
2
  ccv_nnc_tensor_free(dat32);
2754
2
  ccv_nnc_tensor_free(datr);
2755
2
  ccv_nnc_tensor_free(a);
2756
2
  ccv_nnc_tensor_free(b);
2757
2
  ccv_nnc_tensor_free(g);
2758
2
  ccv_nnc_tensor_free(da);
2759
2
}
2760
2761
TEST_CASE("exp forward")
2762
1
{
2763
1
  GUARD_ELSE_RETURN(ccv_nnc_cmd_ok(CCV_NNC_EWEXP_FORWARD, CCV_NNC_BACKEND_GPU_REF));
2764
1
  ccv_nnc_tensor_t* a = ccv_nnc_tensor_new(0, GPU_TENSOR_NCHW(000, 32F, 10, 100), 0);
2765
1
  ccv_nnc_tensor_t* b = ccv_nnc_tensor_new(0, GPU_TENSOR_NCHW(000, 32F, 10, 100), 0);
2766
1
  ccv_nnc_tensor_t* ha = ccv_nnc_tensor_new(0, CPU_TENSOR_NCHW(32F, 10, 100), 0);
2767
1
  ccv_nnc_tensor_t* hb = ccv_nnc_tensor_new(0, CPU_TENSOR_NCHW(32F, 10, 100), 0);
2768
1
  ccv_nnc_tensor_t* bt = ccv_nnc_tensor_new(0, CPU_TENSOR_NCHW(32F, 10, 100), 0);
2769
1
  dsfmt_t dsfmt;
2770
1
  dsfmt_init_gen_rand(&dsfmt, 0);
2771
1
  int i;
2772
1.00k
  for (i = 0; i < 1000; 
i++1.00k
)
2773
1.00k
    ha->data.f32[i] = dsfmt_genrand_open_close(&dsfmt) * 10 - 1;
2774
1
  ccv_nnc_cmd_exec(CMD_DATA_TRANSFER_FORWARD(), ccv_nnc_no_hint, 0, TENSOR_LIST(ha), TENSOR_LIST(a), 0);
2775
1
  ccv_nnc_cmd_exec(CMD_EWEXP_FORWARD(), ccv_nnc_no_hint, 0, TENSOR_LIST(a), TENSOR_LIST(b), 0);
2776
1
  ccv_nnc_cmd_exec(CMD_EWEXP_FORWARD(), ccv_nnc_no_hint, 0, TENSOR_LIST(ha), TENSOR_LIST(bt), 0);
2777
1
  ccv_nnc_cmd_exec(CMD_DATA_TRANSFER_FORWARD(), ccv_nnc_no_hint, 0, TENSOR_LIST(b), TENSOR_LIST(hb), 0);
2778
1
  REQUIRE_TENSOR_EQ(bt, hb, "GPU computed output should be the same as CPU computed ones");
2779
1
  ccv_nnc_tensor_free(a);
2780
1
  ccv_nnc_tensor_free(b);
2781
1
  ccv_nnc_tensor_free(ha);
2782
1
  ccv_nnc_tensor_free(hb);
2783
1
  ccv_nnc_tensor_free(bt);
2784
1
}
2785
2786
TEST_CASE("ewexp backward")
2787
1
{
2788
1
  GUARD_ELSE_RETURN(ccv_nnc_cmd_ok(CCV_NNC_EWEXP_FORWARD, CCV_NNC_BACKEND_GPU_REF) &&
2789
1
    ccv_nnc_cmd_ok(CCV_NNC_EWEXP_BACKWARD, CCV_NNC_BACKEND_GPU_REF));
2790
1
  ccv_nnc_tensor_t* a = ccv_nnc_tensor_new(0, GPU_TENSOR_NCHW(000, 32F, 10, 100), 0);
2791
1
  ccv_nnc_tensor_t* b = ccv_nnc_tensor_new(0, GPU_TENSOR_NCHW(000, 32F, 10, 100), 0);
2792
1
  ccv_nnc_tensor_t* g = ccv_nnc_tensor_new(0, GPU_TENSOR_NCHW(000, 32F, 10, 100), 0);
2793
1
  ccv_nnc_tensor_t* da = ccv_nnc_tensor_new(0, GPU_TENSOR_NCHW(000, 32F, 10, 100), 0);
2794
1
  ccv_nnc_tensor_t* ha = ccv_nnc_tensor_new(0, CPU_TENSOR_NCHW(32F, 10, 100), 0);
2795
1
  ccv_nnc_tensor_t* hda = ccv_nnc_tensor_new(0, CPU_TENSOR_NCHW(32F, 10, 100), 0);
2796
1
  ccv_nnc_tensor_t* hb = ccv_nnc_tensor_new(0, CPU_TENSOR_NCHW(32F, 10, 100), 0);
2797
1
  ccv_nnc_tensor_t* hg = ccv_nnc_tensor_new(0, CPU_TENSOR_NCHW(32F, 10, 100), 0);
2798
1
  ccv_nnc_tensor_t* dat = ccv_nnc_tensor_new(0, CPU_TENSOR_NCHW(32F, 10, 100), 0);
2799
1
  dsfmt_t dsfmt;
2800
1
  dsfmt_init_gen_rand(&dsfmt, 0);
2801
1
  int i;
2802
1.00k
  for (i = 0; i < 1000; 
i++1.00k
)
2803
1.00k
    ha->data.f32[i] = dsfmt_genrand_open_close(&dsfmt) * 10;
2804
1.00k
  for (i = 0; i < 1000; 
i++1.00k
)
2805
1.00k
    hg->data.f32[i] = dsfmt_genrand_open_close(&dsfmt);
2806
1
  ccv_nnc_cmd_exec(CMD_DATA_TRANSFER_FORWARD(), ccv_nnc_no_hint, 0, TENSOR_LIST(ha, hg), TENSOR_LIST(a, g), 0);
2807
1
  ccv_nnc_cmd_exec(CMD_EWEXP_FORWARD(), ccv_nnc_no_hint, 0, TENSOR_LIST(a), TENSOR_LIST(b), 0);
2808
1
  ccv_nnc_cmd_exec(CMD_EWEXP_BACKWARD(), ccv_nnc_no_hint, 0, TENSOR_LIST(g, 0, b), TENSOR_LIST(da), 0);
2809
1
  ccv_nnc_cmd_exec(CMD_EWEXP_FORWARD(), ccv_nnc_no_hint, 0, TENSOR_LIST(ha), TENSOR_LIST(hb), 0);
2810
1
  ccv_nnc_cmd_exec(CMD_EWEXP_BACKWARD(), ccv_nnc_no_hint, 0, TENSOR_LIST(hg, 0, hb), TENSOR_LIST(dat), 0);
2811
1
  ccv_nnc_cmd_exec(CMD_DATA_TRANSFER_FORWARD(), ccv_nnc_no_hint, 0, TENSOR_LIST(da), TENSOR_LIST(hda), 0);
2812
1
  REQUIRE_TENSOR_EQ(dat, hda, "GPU computed output should be the same as CPU computed ones");
2813
1
  ccv_nnc_tensor_free(a);
2814
1
  ccv_nnc_tensor_free(b);
2815
1
  ccv_nnc_tensor_free(g);
2816
1
  ccv_nnc_tensor_free(da);
2817
1
  ccv_nnc_tensor_free(ha);
2818
1
  ccv_nnc_tensor_free(hb);
2819
1
  ccv_nnc_tensor_free(hg);
2820
1
  ccv_nnc_tensor_free(hda);
2821
1
  ccv_nnc_tensor_free(dat);
2822
1
}
2823
2824
TEST_CASE("ewexp half precision")
2825
1
{
2826
1
  GUARD_ELSE_RETURN(ccv_nnc_cmd_ok(CCV_NNC_EWEXP_FORWARD, CCV_NNC_BACKEND_GPU_REF) &&
2827
1
    ccv_nnc_cmd_ok(CCV_NNC_EWEXP_BACKWARD, CCV_NNC_BACKEND_GPU_REF));
2828
1
  float forward_max_diff, backward_max_diff;
2829
1
  _ccv_nnc_ewexp_gpu_ref_datatype(CCV_16F, &forward_max_diff, &backward_max_diff);
2830
1
  REQUIRE(forward_max_diff <= 5e-3, "GPU computed output should be close to CPU computed ones after 16F rounding");
2831
1
  REQUIRE(backward_max_diff <= 5e-3, "GPU computed gradient should be close to CPU computed ones after 16F rounding");
2832
1
}
2833
2834
TEST_CASE("ewexp bfloat precision")
2835
1
{
2836
1
  GUARD_ELSE_RETURN(ccv_nnc_cmd_ok(CCV_NNC_EWEXP_FORWARD, CCV_NNC_BACKEND_GPU_REF) &&
2837
1
    ccv_nnc_cmd_ok(CCV_NNC_EWEXP_BACKWARD, CCV_NNC_BACKEND_GPU_REF));
2838
1
  float forward_max_diff, backward_max_diff;
2839
1
  _ccv_nnc_ewexp_gpu_ref_datatype(CCV_16BF, &forward_max_diff, &backward_max_diff);
2840
1
  REQUIRE(forward_max_diff <= 2e-2, "GPU computed output should be close to CPU computed ones after 16BF rounding");
2841
1
  REQUIRE(backward_max_diff <= 2e-2, "GPU computed gradient should be close to CPU computed ones after 16BF rounding");
2842
1
}
2843
2844
TEST_CASE("ewsoftplus forward")
2845
1
{
2846
1
  GUARD_ELSE_RETURN(ccv_nnc_cmd_ok(CCV_NNC_EWSOFTPLUS_FORWARD, CCV_NNC_BACKEND_GPU_REF));
2847
1
  ccv_nnc_tensor_t* a = ccv_nnc_tensor_new(0, GPU_TENSOR_NCHW(000, 32F, 10, 100), 0);
2848
1
  ccv_nnc_tensor_t* b = ccv_nnc_tensor_new(0, GPU_TENSOR_NCHW(000, 32F, 10, 100), 0);
2849
1
  ccv_nnc_tensor_t* ha = ccv_nnc_tensor_new(0, CPU_TENSOR_NCHW(32F, 10, 100), 0);
2850
1
  ccv_nnc_tensor_t* hb = ccv_nnc_tensor_new(0, CPU_TENSOR_NCHW(32F, 10, 100), 0);
2851
1
  ccv_nnc_tensor_t* bt = ccv_nnc_tensor_new(0, CPU_TENSOR_NCHW(32F, 10, 100), 0);
2852
1
  dsfmt_t dsfmt;
2853
1
  dsfmt_init_gen_rand(&dsfmt, 0);
2854
1
  int i;
2855
1.00k
  for (i = 0; i < 1000; 
i++1.00k
)
2856
1.00k
    ha->data.f32[i] = dsfmt_genrand_open_close(&dsfmt) * 40 - 20;
2857
1
  ccv_nnc_cmd_exec(CMD_DATA_TRANSFER_FORWARD(), ccv_nnc_no_hint, 0, TENSOR_LIST(ha), TENSOR_LIST(a), 0);
2858
1
  ccv_nnc_cmd_exec(CMD_EWSOFTPLUS_FORWARD(), ccv_nnc_no_hint, 0, TENSOR_LIST(a), TENSOR_LIST(b), 0);
2859
1
  ccv_nnc_cmd_exec(CMD_EWSOFTPLUS_FORWARD(), ccv_nnc_no_hint, 0, TENSOR_LIST(ha), TENSOR_LIST(bt), 0);
2860
1
  ccv_nnc_cmd_exec(CMD_DATA_TRANSFER_FORWARD(), ccv_nnc_no_hint, 0, TENSOR_LIST(b), TENSOR_LIST(hb), 0);
2861
1
  REQUIRE_ARRAY_EQ_WITH_TOLERANCE(float, bt->data.f32, hb->data.f32, 1000, 1e-5, "GPU computed output should be the same as CPU computed ones");
2862
1
  ccv_nnc_tensor_free(a);
2863
1
  ccv_nnc_tensor_free(b);
2864
1
  ccv_nnc_tensor_free(ha);
2865
1
  ccv_nnc_tensor_free(hb);
2866
1
  ccv_nnc_tensor_free(bt);
2867
1
}
2868
2869
TEST_CASE("ewsoftplus backward")
2870
1
{
2871
1
  GUARD_ELSE_RETURN(ccv_nnc_cmd_ok(CCV_NNC_EWSOFTPLUS_FORWARD, CCV_NNC_BACKEND_GPU_REF) &&
2872
1
    ccv_nnc_cmd_ok(CCV_NNC_EWSOFTPLUS_BACKWARD, CCV_NNC_BACKEND_GPU_REF));
2873
1
  ccv_nnc_tensor_t* a = ccv_nnc_tensor_new(0, GPU_TENSOR_NCHW(000, 32F, 10, 100), 0);
2874
1
  ccv_nnc_tensor_t* b = ccv_nnc_tensor_new(0, GPU_TENSOR_NCHW(000, 32F, 10, 100), 0);
2875
1
  ccv_nnc_tensor_t* g = ccv_nnc_tensor_new(0, GPU_TENSOR_NCHW(000, 32F, 10, 100), 0);
2876
1
  ccv_nnc_tensor_t* da = ccv_nnc_tensor_new(0, GPU_TENSOR_NCHW(000, 32F, 10, 100), 0);
2877
1
  ccv_nnc_tensor_t* ha = ccv_nnc_tensor_new(0, CPU_TENSOR_NCHW(32F, 10, 100), 0);
2878
1
  ccv_nnc_tensor_t* hda = ccv_nnc_tensor_new(0, CPU_TENSOR_NCHW(32F, 10, 100), 0);
2879
1
  ccv_nnc_tensor_t* hb = ccv_nnc_tensor_new(0, CPU_TENSOR_NCHW(32F, 10, 100), 0);
2880
1
  ccv_nnc_tensor_t* hg = ccv_nnc_tensor_new(0, CPU_TENSOR_NCHW(32F, 10, 100), 0);
2881
1
  ccv_nnc_tensor_t* dat = ccv_nnc_tensor_new(0, CPU_TENSOR_NCHW(32F, 10, 100), 0);
2882
1
  dsfmt_t dsfmt;
2883
1
  dsfmt_init_gen_rand(&dsfmt, 0);
2884
1
  int i;
2885
1.00k
  for (i = 0; i < 1000; 
i++1.00k
)
2886
1.00k
    ha->data.f32[i] = dsfmt_genrand_open_close(&dsfmt) * 40 - 20;
2887
1.00k
  for (i = 0; i < 1000; 
i++1.00k
)
2888
1.00k
    hg->data.f32[i] = dsfmt_genrand_open_close(&dsfmt);
2889
1
  ccv_nnc_cmd_exec(CMD_DATA_TRANSFER_FORWARD(), ccv_nnc_no_hint, 0, TENSOR_LIST(ha, hg), TENSOR_LIST(a, g), 0);
2890
1
  ccv_nnc_cmd_exec(CMD_EWSOFTPLUS_FORWARD(), ccv_nnc_no_hint, 0, TENSOR_LIST(a), TENSOR_LIST(b), 0);
2891
1
  ccv_nnc_cmd_exec(CMD_EWSOFTPLUS_BACKWARD(), ccv_nnc_no_hint, 0, TENSOR_LIST(g, a), TENSOR_LIST(da), 0);
2892
1
  ccv_nnc_cmd_exec(CMD_EWSOFTPLUS_FORWARD(), ccv_nnc_no_hint, 0, TENSOR_LIST(ha), TENSOR_LIST(hb), 0);
2893
1
  ccv_nnc_cmd_exec(CMD_EWSOFTPLUS_BACKWARD(), ccv_nnc_no_hint, 0, TENSOR_LIST(hg, ha), TENSOR_LIST(dat), 0);
2894
1
  ccv_nnc_cmd_exec(CMD_DATA_TRANSFER_FORWARD(), ccv_nnc_no_hint, 0, TENSOR_LIST(da), TENSOR_LIST(hda), 0);
2895
1
  REQUIRE_ARRAY_EQ_WITH_TOLERANCE(float, dat->data.f32, hda->data.f32, 1000, 1e-5, "GPU computed output should be the same as CPU computed ones");
2896
1
  ccv_nnc_tensor_free(a);
2897
1
  ccv_nnc_tensor_free(b);
2898
1
  ccv_nnc_tensor_free(g);
2899
1
  ccv_nnc_tensor_free(da);
2900
1
  ccv_nnc_tensor_free(ha);
2901
1
  ccv_nnc_tensor_free(hb);
2902
1
  ccv_nnc_tensor_free(hg);
2903
1
  ccv_nnc_tensor_free(hda);
2904
1
  ccv_nnc_tensor_free(dat);
2905
1
}
2906
2907
TEST_CASE("ewsoftplus half precision")
2908
1
{
2909
1
  GUARD_ELSE_RETURN(ccv_nnc_cmd_ok(CCV_NNC_EWSOFTPLUS_FORWARD, CCV_NNC_BACKEND_GPU_REF) &&
2910
1
    ccv_nnc_cmd_ok(CCV_NNC_EWSOFTPLUS_BACKWARD, CCV_NNC_BACKEND_GPU_REF));
2911
1
  float forward_max_diff, backward_max_diff;
2912
1
  _ccv_nnc_ewsoftplus_gpu_ref_datatype(CCV_16F, &forward_max_diff, &backward_max_diff);
2913
1
  REQUIRE(forward_max_diff <= 5e-3, "GPU computed output should be close to CPU computed ones after 16F rounding");
2914
1
  REQUIRE(backward_max_diff <= 5e-3, "GPU computed gradient should be close to CPU computed ones after 16F rounding");
2915
1
}
2916
2917
TEST_CASE("ewsoftplus bfloat precision")
2918
1
{
2919
1
  GUARD_ELSE_RETURN(ccv_nnc_cmd_ok(CCV_NNC_EWSOFTPLUS_FORWARD, CCV_NNC_BACKEND_GPU_REF) &&
2920
1
    ccv_nnc_cmd_ok(CCV_NNC_EWSOFTPLUS_BACKWARD, CCV_NNC_BACKEND_GPU_REF));
2921
1
  float forward_max_diff, backward_max_diff;
2922
1
  _ccv_nnc_ewsoftplus_gpu_ref_datatype(CCV_16BF, &forward_max_diff, &backward_max_diff);
2923
1
  REQUIRE(forward_max_diff <= 2e-2, "GPU computed output should be close to CPU computed ones after 16BF rounding");
2924
1
  REQUIRE(backward_max_diff <= 2e-2, "GPU computed gradient should be close to CPU computed ones after 16BF rounding");
2925
1
}
2926
2927
TEST_CASE("ewpow forward")
2928
1
{
2929
1
  GUARD_ELSE_RETURN(ccv_nnc_cmd_ok(CCV_NNC_EWPOW_FORWARD, CCV_NNC_BACKEND_GPU_REF));
2930
1
  ccv_nnc_tensor_t* a = ccv_nnc_tensor_new(0, GPU_TENSOR_NCHW(000, 32F, 10, 100), 0);
2931
1
  ccv_nnc_tensor_t* c = ccv_nnc_tensor_new(0, GPU_TENSOR_NCHW(000, 32F, 10, 100), 0);
2932
1
  ccv_nnc_tensor_t* ha = ccv_nnc_tensor_new(0, CPU_TENSOR_NCHW(32F, 10, 100), 0);
2933
1
  ccv_nnc_tensor_t* hc = ccv_nnc_tensor_new(0, CPU_TENSOR_NCHW(32F, 10, 100), 0);
2934
1
  ccv_nnc_tensor_t* ct = ccv_nnc_tensor_new(0, CPU_TENSOR_NCHW(32F, 10, 100), 0);
2935
1
  dsfmt_t dsfmt;
2936
1
  dsfmt_init_gen_rand(&dsfmt, 0);
2937
1
  int i;
2938
1.00k
  for (i = 0; i < 1000; 
i++1.00k
)
2939
1.00k
    ha->data.f32[i] = dsfmt_genrand_open_close(&dsfmt) * 2 + 0.1;
2940
1
  ccv_nnc_cmd_exec(CMD_DATA_TRANSFER_FORWARD(), ccv_nnc_no_hint, 0, TENSOR_LIST(ha), TENSOR_LIST(a), 0);
2941
1
  ccv_nnc_cmd_exec(CMD_EWPOW_FORWARD(3), ccv_nnc_no_hint, 0, TENSOR_LIST(a), TENSOR_LIST(c), 0);
2942
1
  ccv_nnc_cmd_exec(CMD_EWPOW_FORWARD(3), ccv_nnc_no_hint, 0, TENSOR_LIST(ha), TENSOR_LIST(ct), 0);
2943
1
  ccv_nnc_cmd_exec(CMD_DATA_TRANSFER_FORWARD(), ccv_nnc_no_hint, 0, TENSOR_LIST(c), TENSOR_LIST(hc), 0);
2944
1
  REQUIRE_TENSOR_EQ(ct, hc, "GPU computed output should be the same as CPU computed ones");
2945
1
  ccv_nnc_tensor_free(a);
2946
1
  ccv_nnc_tensor_free(c);
2947
1
  ccv_nnc_tensor_free(ha);
2948
1
  ccv_nnc_tensor_free(hc);
2949
1
  ccv_nnc_tensor_free(ct);
2950
1
}
2951
2952
TEST_CASE("ewpow backward")
2953
1
{
2954
1
  GUARD_ELSE_RETURN(ccv_nnc_cmd_ok(CCV_NNC_EWPOW_FORWARD, CCV_NNC_BACKEND_GPU_REF) &&
2955
1
    ccv_nnc_cmd_ok(CCV_NNC_EWPOW_BACKWARD, CCV_NNC_BACKEND_GPU_REF));
2956
1
  ccv_nnc_tensor_t* a = ccv_nnc_tensor_new(0, GPU_TENSOR_NCHW(000, 32F, 10, 100), 0);
2957
1
  ccv_nnc_tensor_t* c = ccv_nnc_tensor_new(0, GPU_TENSOR_NCHW(000, 32F, 10, 100), 0);
2958
1
  ccv_nnc_tensor_t* g = ccv_nnc_tensor_new(0, GPU_TENSOR_NCHW(000, 32F, 10, 100), 0);
2959
1
  ccv_nnc_tensor_t* da = ccv_nnc_tensor_new(0, GPU_TENSOR_NCHW(000, 32F, 10, 100), 0);
2960
1
  ccv_nnc_tensor_t* ha = ccv_nnc_tensor_new(0, CPU_TENSOR_NCHW(32F, 10, 100), 0);
2961
1
  ccv_nnc_tensor_t* hc = ccv_nnc_tensor_new(0, CPU_TENSOR_NCHW(32F, 10, 100), 0);
2962
1
  ccv_nnc_tensor_t* hg = ccv_nnc_tensor_new(0, CPU_TENSOR_NCHW(32F, 10, 100), 0);
2963
1
  ccv_nnc_tensor_t* hda = ccv_nnc_tensor_new(0, CPU_TENSOR_NCHW(32F, 10, 100), 0);
2964
1
  ccv_nnc_tensor_t* dat = ccv_nnc_tensor_new(0, CPU_TENSOR_NCHW(32F, 10, 100), 0);
2965
1
  dsfmt_t dsfmt;
2966
1
  dsfmt_init_gen_rand(&dsfmt, 0);
2967
1
  int i;
2968
1.00k
  for (i = 0; i < 1000; 
i++1.00k
)
2969
1.00k
    ha->data.f32[i] = dsfmt_genrand_open_close(&dsfmt) * 2 + 0.1;
2970
1.00k
  for (i = 0; i < 1000; 
i++1.00k
)
2971
1.00k
    hg->data.f32[i] = dsfmt_genrand_open_close(&dsfmt) * 2 - 1;
2972
1
  ccv_nnc_cmd_exec(CMD_DATA_TRANSFER_FORWARD(), ccv_nnc_no_hint, 0, TENSOR_LIST(ha, hg), TENSOR_LIST(a, g), 0);
2973
1
  ccv_nnc_cmd_exec(CMD_EWPOW_FORWARD(3), ccv_nnc_no_hint, 0, TENSOR_LIST(a), TENSOR_LIST(c), 0);
2974
1
  ccv_nnc_cmd_exec(CMD_EWPOW_BACKWARD(3), ccv_nnc_no_hint, 0, TENSOR_LIST(g, a), TENSOR_LIST(da), 0);
2975
1
  ccv_nnc_cmd_exec(CMD_EWPOW_FORWARD(3), ccv_nnc_no_hint, 0, TENSOR_LIST(ha), TENSOR_LIST(hc), 0);
2976
1
  ccv_nnc_cmd_exec(CMD_EWPOW_BACKWARD(3), ccv_nnc_no_hint, 0, TENSOR_LIST(hg, ha), TENSOR_LIST(dat), 0);
2977
1
  ccv_nnc_cmd_exec(CMD_DATA_TRANSFER_FORWARD(), ccv_nnc_no_hint, 0, TENSOR_LIST(da), TENSOR_LIST(hda), 0);
2978
1
  REQUIRE_TENSOR_EQ(dat, hda, "GPU computed dA should be the same as CPU computed ones");
2979
1
  ccv_nnc_tensor_free(a);
2980
1
  ccv_nnc_tensor_free(c);
2981
1
  ccv_nnc_tensor_free(g);
2982
1
  ccv_nnc_tensor_free(da);
2983
1
  ccv_nnc_tensor_free(ha);
2984
1
  ccv_nnc_tensor_free(hc);
2985
1
  ccv_nnc_tensor_free(hg);
2986
1
  ccv_nnc_tensor_free(hda);
2987
1
  ccv_nnc_tensor_free(dat);
2988
1
}
2989
2990
TEST_CASE("ewsin forward")
2991
1
{
2992
1
  GUARD_ELSE_RETURN(ccv_nnc_cmd_ok(CCV_NNC_EWSIN_FORWARD, CCV_NNC_BACKEND_GPU_REF));
2993
1
  ccv_nnc_tensor_t* a = ccv_nnc_tensor_new(0, GPU_TENSOR_NCHW(000, 32F, 10, 100), 0);
2994
1
  ccv_nnc_tensor_t* b = ccv_nnc_tensor_new(0, GPU_TENSOR_NCHW(000, 32F, 10, 100), 0);
2995
1
  ccv_nnc_tensor_t* ha = ccv_nnc_tensor_new(0, CPU_TENSOR_NCHW(32F, 10, 100), 0);
2996
1
  ccv_nnc_tensor_t* hb = ccv_nnc_tensor_new(0, CPU_TENSOR_NCHW(32F, 10, 100), 0);
2997
1
  ccv_nnc_tensor_t* bt = ccv_nnc_tensor_new(0, CPU_TENSOR_NCHW(32F, 10, 100), 0);
2998
1
  dsfmt_t dsfmt;
2999
1
  dsfmt_init_gen_rand(&dsfmt, 0);
3000
1
  int i;
3001
1.00k
  for (i = 0; i < 1000; 
i++1.00k
)
3002
1.00k
    ha->data.f32[i] = dsfmt_genrand_open_close(&dsfmt) * 10 - 5;
3003
1
  ccv_nnc_cmd_exec(CMD_DATA_TRANSFER_FORWARD(), ccv_nnc_no_hint, 0, TENSOR_LIST(ha), TENSOR_LIST(a), 0);
3004
1
  ccv_nnc_cmd_exec(CMD_EWSIN_FORWARD(), ccv_nnc_no_hint, 0, TENSOR_LIST(a), TENSOR_LIST(b), 0);
3005
1
  ccv_nnc_cmd_exec(CMD_EWSIN_FORWARD(), ccv_nnc_no_hint, 0, TENSOR_LIST(ha), TENSOR_LIST(bt), 0);
3006
1
  ccv_nnc_cmd_exec(CMD_DATA_TRANSFER_FORWARD(), ccv_nnc_no_hint, 0, TENSOR_LIST(b), TENSOR_LIST(hb), 0);
3007
1
  REQUIRE_ARRAY_EQ_WITH_TOLERANCE(float, bt->data.f32, hb->data.f32, 10 * 100, 1e-3, "GPU computed output should be the same as CPU computed ones");
3008
1
  ccv_nnc_tensor_free(a);
3009
1
  ccv_nnc_tensor_free(b);
3010
1
  ccv_nnc_tensor_free(ha);
3011
1
  ccv_nnc_tensor_free(hb);
3012
1
  ccv_nnc_tensor_free(bt);
3013
1
}
3014
3015
TEST_CASE("ewsin backward")
3016
1
{
3017
1
  GUARD_ELSE_RETURN(ccv_nnc_cmd_ok(CCV_NNC_EWSIN_FORWARD, CCV_NNC_BACKEND_GPU_REF) &&
3018
1
    ccv_nnc_cmd_ok(CCV_NNC_EWSIN_BACKWARD, CCV_NNC_BACKEND_GPU_REF));
3019
1
  ccv_nnc_tensor_t* a = ccv_nnc_tensor_new(0, GPU_TENSOR_NCHW(000, 32F, 10, 100), 0);
3020
1
  ccv_nnc_tensor_t* b = ccv_nnc_tensor_new(0, GPU_TENSOR_NCHW(000, 32F, 10, 100), 0);
3021
1
  ccv_nnc_tensor_t* g = ccv_nnc_tensor_new(0, GPU_TENSOR_NCHW(000, 32F, 10, 100), 0);
3022
1
  ccv_nnc_tensor_t* da = ccv_nnc_tensor_new(0, GPU_TENSOR_NCHW(000, 32F, 10, 100), 0);
3023
1
  ccv_nnc_tensor_t* ha = ccv_nnc_tensor_new(0, CPU_TENSOR_NCHW(32F, 10, 100), 0);
3024
1
  ccv_nnc_tensor_t* hda = ccv_nnc_tensor_new(0, CPU_TENSOR_NCHW(32F, 10, 100), 0);
3025
1
  ccv_nnc_tensor_t* hb = ccv_nnc_tensor_new(0, CPU_TENSOR_NCHW(32F, 10, 100), 0);
3026
1
  ccv_nnc_tensor_t* hg = ccv_nnc_tensor_new(0, CPU_TENSOR_NCHW(32F, 10, 100), 0);
3027
1
  ccv_nnc_tensor_t* dat = ccv_nnc_tensor_new(0, CPU_TENSOR_NCHW(32F, 10, 100), 0);
3028
1
  dsfmt_t dsfmt;
3029
1
  dsfmt_init_gen_rand(&dsfmt, 0);
3030
1
  int i;
3031
1.00k
  for (i = 0; i < 1000; 
i++1.00k
)
3032
1.00k
    ha->data.f32[i] = dsfmt_genrand_open_close(&dsfmt) * 10 - 5;
3033
1.00k
  for (i = 0; i < 1000; 
i++1.00k
)
3034
1.00k
    hg->data.f32[i] = dsfmt_genrand_open_close(&dsfmt) * 2 - 1;
3035
1
  ccv_nnc_cmd_exec(CMD_DATA_TRANSFER_FORWARD(), ccv_nnc_no_hint, 0, TENSOR_LIST(ha, hg), TENSOR_LIST(a, g), 0);
3036
1
  ccv_nnc_cmd_exec(CMD_EWSIN_FORWARD(), ccv_nnc_no_hint, 0, TENSOR_LIST(a), TENSOR_LIST(b), 0);
3037
1
  ccv_nnc_cmd_exec(CMD_EWSIN_BACKWARD(), ccv_nnc_no_hint, 0, TENSOR_LIST(g, a), TENSOR_LIST(da), 0);
3038
1
  ccv_nnc_cmd_exec(CMD_EWSIN_FORWARD(), ccv_nnc_no_hint, 0, TENSOR_LIST(ha), TENSOR_LIST(hb), 0);
3039
1
  ccv_nnc_cmd_exec(CMD_EWSIN_BACKWARD(), ccv_nnc_no_hint, 0, TENSOR_LIST(hg, ha), TENSOR_LIST(dat), 0);
3040
1
  ccv_nnc_cmd_exec(CMD_DATA_TRANSFER_FORWARD(), ccv_nnc_no_hint, 0, TENSOR_LIST(da), TENSOR_LIST(hda), 0);
3041
1
  REQUIRE_ARRAY_EQ_WITH_TOLERANCE(float, dat->data.f32, hda->data.f32, 10 * 100, 1e-3, "GPU computed output should be the same as CPU computed ones");
3042
1
  ccv_nnc_tensor_free(a);
3043
1
  ccv_nnc_tensor_free(b);
3044
1
  ccv_nnc_tensor_free(g);
3045
1
  ccv_nnc_tensor_free(da);
3046
1
  ccv_nnc_tensor_free(ha);
3047
1
  ccv_nnc_tensor_free(hb);
3048
1
  ccv_nnc_tensor_free(hg);
3049
1
  ccv_nnc_tensor_free(hda);
3050
1
  ccv_nnc_tensor_free(dat);
3051
1
}
3052
3053
TEST_CASE("ewcos forward")
3054
1
{
3055
1
  GUARD_ELSE_RETURN(ccv_nnc_cmd_ok(CCV_NNC_EWCOS_FORWARD, CCV_NNC_BACKEND_GPU_REF));
3056
1
  ccv_nnc_tensor_t* a = ccv_nnc_tensor_new(0, GPU_TENSOR_NCHW(000, 32F, 10, 100), 0);
3057
1
  ccv_nnc_tensor_t* b = ccv_nnc_tensor_new(0, GPU_TENSOR_NCHW(000, 32F, 10, 100), 0);
3058
1
  ccv_nnc_tensor_t* ha = ccv_nnc_tensor_new(0, CPU_TENSOR_NCHW(32F, 10, 100), 0);
3059
1
  ccv_nnc_tensor_t* hb = ccv_nnc_tensor_new(0, CPU_TENSOR_NCHW(32F, 10, 100), 0);
3060
1
  ccv_nnc_tensor_t* bt = ccv_nnc_tensor_new(0, CPU_TENSOR_NCHW(32F, 10, 100), 0);
3061
1
  dsfmt_t dsfmt;
3062
1
  dsfmt_init_gen_rand(&dsfmt, 0);
3063
1
  int i;
3064
1.00k
  for (i = 0; i < 1000; 
i++1.00k
)
3065
1.00k
    ha->data.f32[i] = dsfmt_genrand_open_close(&dsfmt) * 10 - 5;
3066
1
  ccv_nnc_cmd_exec(CMD_DATA_TRANSFER_FORWARD(), ccv_nnc_no_hint, 0, TENSOR_LIST(ha), TENSOR_LIST(a), 0);
3067
1
  ccv_nnc_cmd_exec(CMD_EWCOS_FORWARD(), ccv_nnc_no_hint, 0, TENSOR_LIST(a), TENSOR_LIST(b), 0);
3068
1
  ccv_nnc_cmd_exec(CMD_EWCOS_FORWARD(), ccv_nnc_no_hint, 0, TENSOR_LIST(ha), TENSOR_LIST(bt), 0);
3069
1
  ccv_nnc_cmd_exec(CMD_DATA_TRANSFER_FORWARD(), ccv_nnc_no_hint, 0, TENSOR_LIST(b), TENSOR_LIST(hb), 0);
3070
1
  REQUIRE_ARRAY_EQ_WITH_TOLERANCE(float, bt->data.f32, hb->data.f32, 10 * 100, 1e-3, "GPU computed output should be the same as CPU computed ones");
3071
1
  ccv_nnc_tensor_free(a);
3072
1
  ccv_nnc_tensor_free(b);
3073
1
  ccv_nnc_tensor_free(ha);
3074
1
  ccv_nnc_tensor_free(hb);
3075
1
  ccv_nnc_tensor_free(bt);
3076
1
}
3077
3078
TEST_CASE("ewcos backward")
3079
1
{
3080
1
  GUARD_ELSE_RETURN(ccv_nnc_cmd_ok(CCV_NNC_EWCOS_FORWARD, CCV_NNC_BACKEND_GPU_REF) &&
3081
1
    ccv_nnc_cmd_ok(CCV_NNC_EWCOS_BACKWARD, CCV_NNC_BACKEND_GPU_REF));
3082
1
  ccv_nnc_tensor_t* a = ccv_nnc_tensor_new(0, GPU_TENSOR_NCHW(000, 32F, 10, 100), 0);
3083
1
  ccv_nnc_tensor_t* b = ccv_nnc_tensor_new(0, GPU_TENSOR_NCHW(000, 32F, 10, 100), 0);
3084
1
  ccv_nnc_tensor_t* g = ccv_nnc_tensor_new(0, GPU_TENSOR_NCHW(000, 32F, 10, 100), 0);
3085
1
  ccv_nnc_tensor_t* da = ccv_nnc_tensor_new(0, GPU_TENSOR_NCHW(000, 32F, 10, 100), 0);
3086
1
  ccv_nnc_tensor_t* ha = ccv_nnc_tensor_new(0, CPU_TENSOR_NCHW(32F, 10, 100), 0);
3087
1
  ccv_nnc_tensor_t* hda = ccv_nnc_tensor_new(0, CPU_TENSOR_NCHW(32F, 10, 100), 0);
3088
1
  ccv_nnc_tensor_t* hb = ccv_nnc_tensor_new(0, CPU_TENSOR_NCHW(32F, 10, 100), 0);
3089
1
  ccv_nnc_tensor_t* hg = ccv_nnc_tensor_new(0, CPU_TENSOR_NCHW(32F, 10, 100), 0);
3090
1
  ccv_nnc_tensor_t* dat = ccv_nnc_tensor_new(0, CPU_TENSOR_NCHW(32F, 10, 100), 0);
3091
1
  dsfmt_t dsfmt;
3092
1
  dsfmt_init_gen_rand(&dsfmt, 0);
3093
1
  int i;
3094
1.00k
  for (i = 0; i < 1000; 
i++1.00k
)
3095
1.00k
    ha->data.f32[i] = dsfmt_genrand_open_close(&dsfmt) * 10 - 5;
3096
1.00k
  for (i = 0; i < 1000; 
i++1.00k
)
3097
1.00k
    hg->data.f32[i] = dsfmt_genrand_open_close(&dsfmt) * 2 - 1;
3098
1
  ccv_nnc_cmd_exec(CMD_DATA_TRANSFER_FORWARD(), ccv_nnc_no_hint, 0, TENSOR_LIST(ha, hg), TENSOR_LIST(a, g), 0);
3099
1
  ccv_nnc_cmd_exec(CMD_EWCOS_FORWARD(), ccv_nnc_no_hint, 0, TENSOR_LIST(a), TENSOR_LIST(b), 0);
3100
1
  ccv_nnc_cmd_exec(CMD_EWCOS_BACKWARD(), ccv_nnc_no_hint, 0, TENSOR_LIST(g, a), TENSOR_LIST(da), 0);
3101
1
  ccv_nnc_cmd_exec(CMD_EWCOS_FORWARD(), ccv_nnc_no_hint, 0, TENSOR_LIST(ha), TENSOR_LIST(hb), 0);
3102
1
  ccv_nnc_cmd_exec(CMD_EWCOS_BACKWARD(), ccv_nnc_no_hint, 0, TENSOR_LIST(hg, ha), TENSOR_LIST(dat), 0);
3103
1
  ccv_nnc_cmd_exec(CMD_DATA_TRANSFER_FORWARD(), ccv_nnc_no_hint, 0, TENSOR_LIST(da), TENSOR_LIST(hda), 0);
3104
1
  REQUIRE_ARRAY_EQ_WITH_TOLERANCE(float, dat->data.f32, hda->data.f32, 10 * 100, 1e-3, "GPU computed output should be the same as CPU computed ones");
3105
1
  ccv_nnc_tensor_free(a);
3106
1
  ccv_nnc_tensor_free(b);
3107
1
  ccv_nnc_tensor_free(g);
3108
1
  ccv_nnc_tensor_free(da);
3109
1
  ccv_nnc_tensor_free(ha);
3110
1
  ccv_nnc_tensor_free(hb);
3111
1
  ccv_nnc_tensor_free(hg);
3112
1
  ccv_nnc_tensor_free(hda);
3113
1
  ccv_nnc_tensor_free(dat);
3114
1
}
3115
3116
TEST_CASE("ewlog forward")
3117
1
{
3118
1
  GUARD_ELSE_RETURN(ccv_nnc_cmd_ok(CCV_NNC_EWLOG_FORWARD, CCV_NNC_BACKEND_GPU_REF));
3119
1
  ccv_nnc_tensor_t* a = ccv_nnc_tensor_new(0, GPU_TENSOR_NCHW(000, 32F, 10, 100), 0);
3120
1
  ccv_nnc_tensor_t* b = ccv_nnc_tensor_new(0, GPU_TENSOR_NCHW(000, 32F, 10, 100), 0);
3121
1
  ccv_nnc_tensor_t* ha = ccv_nnc_tensor_new(0, CPU_TENSOR_NCHW(32F, 10, 100), 0);
3122
1
  ccv_nnc_tensor_t* hb = ccv_nnc_tensor_new(0, CPU_TENSOR_NCHW(32F, 10, 100), 0);
3123
1
  ccv_nnc_tensor_t* bt = ccv_nnc_tensor_new(0, CPU_TENSOR_NCHW(32F, 10, 100), 0);
3124
1
  dsfmt_t dsfmt;
3125
1
  dsfmt_init_gen_rand(&dsfmt, 0);
3126
1
  int i;
3127
1.00k
  for (i = 0; i < 1000; 
i++1.00k
)
3128
1.00k
    ha->data.f32[i] = dsfmt_genrand_open_close(&dsfmt) * 10 + 0.0001;
3129
1
  ccv_nnc_cmd_exec(CMD_DATA_TRANSFER_FORWARD(), ccv_nnc_no_hint, 0, TENSOR_LIST(ha), TENSOR_LIST(a), 0);
3130
1
  ccv_nnc_cmd_exec(CMD_EWLOG_FORWARD(), ccv_nnc_no_hint, 0, TENSOR_LIST(a), TENSOR_LIST(b), 0);
3131
1
  ccv_nnc_cmd_exec(CMD_EWLOG_FORWARD(), ccv_nnc_no_hint, 0, TENSOR_LIST(ha), TENSOR_LIST(bt), 0);
3132
1
  ccv_nnc_cmd_exec(CMD_DATA_TRANSFER_FORWARD(), ccv_nnc_no_hint, 0, TENSOR_LIST(b), TENSOR_LIST(hb), 0);
3133
1
  REQUIRE_TENSOR_EQ(bt, hb, "GPU computed output should be the same as CPU computed ones");
3134
1
  ccv_nnc_tensor_free(a);
3135
1
  ccv_nnc_tensor_free(b);
3136
1
  ccv_nnc_tensor_free(ha);
3137
1
  ccv_nnc_tensor_free(hb);
3138
1
  ccv_nnc_tensor_free(bt);
3139
1
}
3140
3141
TEST_CASE("ewlog backward")
3142
1
{
3143
1
  GUARD_ELSE_RETURN(ccv_nnc_cmd_ok(CCV_NNC_EWLOG_FORWARD, CCV_NNC_BACKEND_GPU_REF) &&
3144
1
    ccv_nnc_cmd_ok(CCV_NNC_EWLOG_BACKWARD, CCV_NNC_BACKEND_GPU_REF));
3145
1
  ccv_nnc_tensor_t* a = ccv_nnc_tensor_new(0, GPU_TENSOR_NCHW(000, 32F, 10, 100), 0);
3146
1
  ccv_nnc_tensor_t* b = ccv_nnc_tensor_new(0, GPU_TENSOR_NCHW(000, 32F, 10, 100), 0);
3147
1
  ccv_nnc_tensor_t* g = ccv_nnc_tensor_new(0, GPU_TENSOR_NCHW(000, 32F, 10, 100), 0);
3148
1
  ccv_nnc_tensor_t* da = ccv_nnc_tensor_new(0, GPU_TENSOR_NCHW(000, 32F, 10, 100), 0);
3149
1
  ccv_nnc_tensor_t* ha = ccv_nnc_tensor_new(0, CPU_TENSOR_NCHW(32F, 10, 100), 0);
3150
1
  ccv_nnc_tensor_t* hda = ccv_nnc_tensor_new(0, CPU_TENSOR_NCHW(32F, 10, 100), 0);
3151
1
  ccv_nnc_tensor_t* hb = ccv_nnc_tensor_new(0, CPU_TENSOR_NCHW(32F, 10, 100), 0);
3152
1
  ccv_nnc_tensor_t* hg = ccv_nnc_tensor_new(0, CPU_TENSOR_NCHW(32F, 10, 100), 0);
3153
1
  ccv_nnc_tensor_t* dat = ccv_nnc_tensor_new(0, CPU_TENSOR_NCHW(32F, 10, 100), 0);
3154
1
  dsfmt_t dsfmt;
3155
1
  dsfmt_init_gen_rand(&dsfmt, 0);
3156
1
  int i;
3157
1.00k
  for (i = 0; i < 1000; 
i++1.00k
)
3158
1.00k
    ha->data.f32[i] = dsfmt_genrand_open_close(&dsfmt) * 10;
3159
1.00k
  for (i = 0; i < 1000; 
i++1.00k
)
3160
1.00k
    hg->data.f32[i] = dsfmt_genrand_open_close(&dsfmt);
3161
1
  ccv_nnc_cmd_exec(CMD_DATA_TRANSFER_FORWARD(), ccv_nnc_no_hint, 0, TENSOR_LIST(ha, hg), TENSOR_LIST(a, g), 0);
3162
1
  ccv_nnc_cmd_exec(CMD_EWLOG_FORWARD(), ccv_nnc_no_hint, 0, TENSOR_LIST(a), TENSOR_LIST(b), 0);
3163
1
  ccv_nnc_cmd_exec(CMD_EWLOG_BACKWARD(), ccv_nnc_no_hint, 0, TENSOR_LIST(g, a), TENSOR_LIST(da), 0);
3164
1
  ccv_nnc_cmd_exec(CMD_EWLOG_FORWARD(), ccv_nnc_no_hint, 0, TENSOR_LIST(ha), TENSOR_LIST(hb), 0);
3165
1
  ccv_nnc_cmd_exec(CMD_EWLOG_BACKWARD(), ccv_nnc_no_hint, 0, TENSOR_LIST(hg, ha), TENSOR_LIST(dat), 0);
3166
1
  ccv_nnc_cmd_exec(CMD_DATA_TRANSFER_FORWARD(), ccv_nnc_no_hint, 0, TENSOR_LIST(da), TENSOR_LIST(hda), 0);
3167
1
  REQUIRE_TENSOR_EQ(dat, hda, "GPU computed output should be the same as CPU computed ones");
3168
1
  ccv_nnc_tensor_free(a);
3169
1
  ccv_nnc_tensor_free(b);
3170
1
  ccv_nnc_tensor_free(g);
3171
1
  ccv_nnc_tensor_free(da);
3172
1
  ccv_nnc_tensor_free(ha);
3173
1
  ccv_nnc_tensor_free(hb);
3174
1
  ccv_nnc_tensor_free(hg);
3175
1
  ccv_nnc_tensor_free(hda);
3176
1
  ccv_nnc_tensor_free(dat);
3177
1
}
3178
3179
TEST_CASE("ewsqrt forward")
3180
1
{
3181
1
  GUARD_ELSE_RETURN(ccv_nnc_cmd_ok(CCV_NNC_EWSQRT_FORWARD, CCV_NNC_BACKEND_GPU_REF));
3182
1
  ccv_nnc_tensor_t* a = ccv_nnc_tensor_new(0, GPU_TENSOR_NCHW(000, 32F, 10, 100), 0);
3183
1
  ccv_nnc_tensor_t* b = ccv_nnc_tensor_new(0, GPU_TENSOR_NCHW(000, 32F, 10, 100), 0);
3184
1
  ccv_nnc_tensor_t* ha = ccv_nnc_tensor_new(0, CPU_TENSOR_NCHW(32F, 10, 100), 0);
3185
1
  ccv_nnc_tensor_t* hb = ccv_nnc_tensor_new(0, CPU_TENSOR_NCHW(32F, 10, 100), 0);
3186
1
  ccv_nnc_tensor_t* bt = ccv_nnc_tensor_new(0, CPU_TENSOR_NCHW(32F, 10, 100), 0);
3187
1
  dsfmt_t dsfmt;
3188
1
  dsfmt_init_gen_rand(&dsfmt, 0);
3189
1
  int i;
3190
1.00k
  for (i = 0; i < 1000; 
i++1.00k
)
3191
1.00k
    ha->data.f32[i] = dsfmt_genrand_open_close(&dsfmt) * 10 + 0.0001;
3192
1
  ccv_nnc_cmd_exec(CMD_DATA_TRANSFER_FORWARD(), ccv_nnc_no_hint, 0, TENSOR_LIST(ha), TENSOR_LIST(a), 0);
3193
1
  ccv_nnc_cmd_exec(CMD_EWSQRT_FORWARD(), ccv_nnc_no_hint, 0, TENSOR_LIST(a), TENSOR_LIST(b), 0);
3194
1
  ccv_nnc_cmd_exec(CMD_EWSQRT_FORWARD(), ccv_nnc_no_hint, 0, TENSOR_LIST(ha), TENSOR_LIST(bt), 0);
3195
1
  ccv_nnc_cmd_exec(CMD_DATA_TRANSFER_FORWARD(), ccv_nnc_no_hint, 0, TENSOR_LIST(b), TENSOR_LIST(hb), 0);
3196
1
  REQUIRE_TENSOR_EQ(bt, hb, "GPU computed output should be the same as CPU computed ones");
3197
1
  ccv_nnc_tensor_free(a);
3198
1
  ccv_nnc_tensor_free(b);
3199
1
  ccv_nnc_tensor_free(ha);
3200
1
  ccv_nnc_tensor_free(hb);
3201
1
  ccv_nnc_tensor_free(bt);
3202
1
}
3203
3204
TEST_CASE("ewsqrt backward")
3205
1
{
3206
1
  GUARD_ELSE_RETURN(ccv_nnc_cmd_ok(CCV_NNC_EWSQRT_FORWARD, CCV_NNC_BACKEND_GPU_REF) &&
3207
1
    ccv_nnc_cmd_ok(CCV_NNC_EWSQRT_BACKWARD, CCV_NNC_BACKEND_GPU_REF));
3208
1
  ccv_nnc_tensor_t* a = ccv_nnc_tensor_new(0, GPU_TENSOR_NCHW(000, 32F, 10, 100), 0);
3209
1
  ccv_nnc_tensor_t* b = ccv_nnc_tensor_new(0, GPU_TENSOR_NCHW(000, 32F, 10, 100), 0);
3210
1
  ccv_nnc_tensor_t* g = ccv_nnc_tensor_new(0, GPU_TENSOR_NCHW(000, 32F, 10, 100), 0);
3211
1
  ccv_nnc_tensor_t* da = ccv_nnc_tensor_new(0, GPU_TENSOR_NCHW(000, 32F, 10, 100), 0);
3212
1
  ccv_nnc_tensor_t* ha = ccv_nnc_tensor_new(0, CPU_TENSOR_NCHW(32F, 10, 100), 0);
3213
1
  ccv_nnc_tensor_t* hda = ccv_nnc_tensor_new(0, CPU_TENSOR_NCHW(32F, 10, 100), 0);
3214
1
  ccv_nnc_tensor_t* hb = ccv_nnc_tensor_new(0, CPU_TENSOR_NCHW(32F, 10, 100), 0);
3215
1
  ccv_nnc_tensor_t* hg = ccv_nnc_tensor_new(0, CPU_TENSOR_NCHW(32F, 10, 100), 0);
3216
1
  ccv_nnc_tensor_t* dat = ccv_nnc_tensor_new(0, CPU_TENSOR_NCHW(32F, 10, 100), 0);
3217
1
  dsfmt_t dsfmt;
3218
1
  dsfmt_init_gen_rand(&dsfmt, 0);
3219
1
  int i;
3220
1.00k
  for (i = 0; i < 1000; 
i++1.00k
)
3221
1.00k
    ha->data.f32[i] = dsfmt_genrand_open_close(&dsfmt) * 10;
3222
1.00k
  for (i = 0; i < 1000; 
i++1.00k
)
3223
1.00k
    hg->data.f32[i] = dsfmt_genrand_open_close(&dsfmt);
3224
1
  ccv_nnc_cmd_exec(CMD_DATA_TRANSFER_FORWARD(), ccv_nnc_no_hint, 0, TENSOR_LIST(ha, hg), TENSOR_LIST(a, g), 0);
3225
1
  ccv_nnc_cmd_exec(CMD_EWSQRT_FORWARD(), ccv_nnc_no_hint, 0, TENSOR_LIST(a), TENSOR_LIST(b), 0);
3226
1
  ccv_nnc_cmd_exec(CMD_EWSQRT_BACKWARD(), ccv_nnc_no_hint, 0, TENSOR_LIST(g, 0, b), TENSOR_LIST(da), 0);
3227
1
  ccv_nnc_cmd_exec(CMD_EWSQRT_FORWARD(), ccv_nnc_no_hint, 0, TENSOR_LIST(ha), TENSOR_LIST(hb), 0);
3228
1
  ccv_nnc_cmd_exec(CMD_EWSQRT_BACKWARD(), ccv_nnc_no_hint, 0, TENSOR_LIST(hg, 0, hb), TENSOR_LIST(dat), 0);
3229
1
  ccv_nnc_cmd_exec(CMD_DATA_TRANSFER_FORWARD(), ccv_nnc_no_hint, 0, TENSOR_LIST(da), TENSOR_LIST(hda), 0);
3230
1
  REQUIRE_TENSOR_EQ(dat, hda, "GPU computed output should be the same as CPU computed ones");
3231
1
  ccv_nnc_tensor_free(a);
3232
1
  ccv_nnc_tensor_free(b);
3233
1
  ccv_nnc_tensor_free(g);
3234
1
  ccv_nnc_tensor_free(da);
3235
1
  ccv_nnc_tensor_free(ha);
3236
1
  ccv_nnc_tensor_free(hb);
3237
1
  ccv_nnc_tensor_free(hg);
3238
1
  ccv_nnc_tensor_free(hda);
3239
1
  ccv_nnc_tensor_free(dat);
3240
1
}
3241
3242
TEST_CASE("ewabs forward")
3243
1
{
3244
1
  GUARD_ELSE_RETURN(ccv_nnc_cmd_ok(CCV_NNC_EWABS_FORWARD, CCV_NNC_BACKEND_GPU_REF));
3245
1
  ccv_nnc_tensor_t* a = ccv_nnc_tensor_new(0, GPU_TENSOR_NCHW(000, 32F, 10, 100), 0);
3246
1
  ccv_nnc_tensor_t* b = ccv_nnc_tensor_new(0, GPU_TENSOR_NCHW(000, 32F, 10, 100), 0);
3247
1
  ccv_nnc_tensor_t* ha = ccv_nnc_tensor_new(0, CPU_TENSOR_NCHW(32F, 10, 100), 0);
3248
1
  ccv_nnc_tensor_t* hb = ccv_nnc_tensor_new(0, CPU_TENSOR_NCHW(32F, 10, 100), 0);
3249
1
  ccv_nnc_tensor_t* bt = ccv_nnc_tensor_new(0, CPU_TENSOR_NCHW(32F, 10, 100), 0);
3250
1
  dsfmt_t dsfmt;
3251
1
  dsfmt_init_gen_rand(&dsfmt, 0);
3252
1
  int i;
3253
1.00k
  for (i = 0; i < 1000; 
i++1.00k
)
3254
1.00k
    ha->data.f32[i] = dsfmt_genrand_open_close(&dsfmt) * 10 - 5 + 0.0001;
3255
1
  ccv_nnc_cmd_exec(CMD_DATA_TRANSFER_FORWARD(), ccv_nnc_no_hint, 0, TENSOR_LIST(ha), TENSOR_LIST(a), 0);
3256
1
  ccv_nnc_cmd_exec(CMD_EWABS_FORWARD(), ccv_nnc_no_hint, 0, TENSOR_LIST(a), TENSOR_LIST(b), 0);
3257
1
  ccv_nnc_cmd_exec(CMD_EWABS_FORWARD(), ccv_nnc_no_hint, 0, TENSOR_LIST(ha), TENSOR_LIST(bt), 0);
3258
1
  ccv_nnc_cmd_exec(CMD_DATA_TRANSFER_FORWARD(), ccv_nnc_no_hint, 0, TENSOR_LIST(b), TENSOR_LIST(hb), 0);
3259
1
  REQUIRE_TENSOR_EQ(bt, hb, "GPU computed output should be the same as CPU computed ones");
3260
1
  ccv_nnc_tensor_free(a);
3261
1
  ccv_nnc_tensor_free(b);
3262
1
  ccv_nnc_tensor_free(ha);
3263
1
  ccv_nnc_tensor_free(hb);
3264
1
  ccv_nnc_tensor_free(bt);
3265
1
}
3266
3267
TEST_CASE("ewabs backward")
3268
1
{
3269
1
  GUARD_ELSE_RETURN(ccv_nnc_cmd_ok(CCV_NNC_EWABS_FORWARD, CCV_NNC_BACKEND_GPU_REF) &&
3270
1
    ccv_nnc_cmd_ok(CCV_NNC_EWABS_BACKWARD, CCV_NNC_BACKEND_GPU_REF));
3271
1
  ccv_nnc_tensor_t* a = ccv_nnc_tensor_new(0, GPU_TENSOR_NCHW(000, 32F, 10, 100), 0);
3272
1
  ccv_nnc_tensor_t* b = ccv_nnc_tensor_new(0, GPU_TENSOR_NCHW(000, 32F, 10, 100), 0);
3273
1
  ccv_nnc_tensor_t* g = ccv_nnc_tensor_new(0, GPU_TENSOR_NCHW(000, 32F, 10, 100), 0);
3274
1
  ccv_nnc_tensor_t* da = ccv_nnc_tensor_new(0, GPU_TENSOR_NCHW(000, 32F, 10, 100), 0);
3275
1
  ccv_nnc_tensor_t* ha = ccv_nnc_tensor_new(0, CPU_TENSOR_NCHW(32F, 10, 100), 0);
3276
1
  ccv_nnc_tensor_t* hda = ccv_nnc_tensor_new(0, CPU_TENSOR_NCHW(32F, 10, 100), 0);
3277
1
  ccv_nnc_tensor_t* hb = ccv_nnc_tensor_new(0, CPU_TENSOR_NCHW(32F, 10, 100), 0);
3278
1
  ccv_nnc_tensor_t* hg = ccv_nnc_tensor_new(0, CPU_TENSOR_NCHW(32F, 10, 100), 0);
3279
1
  ccv_nnc_tensor_t* dat = ccv_nnc_tensor_new(0, CPU_TENSOR_NCHW(32F, 10, 100), 0);
3280
1
  dsfmt_t dsfmt;
3281
1
  dsfmt_init_gen_rand(&dsfmt, 0);
3282
1
  int i;
3283
1.00k
  for (i = 0; i < 1000; 
i++1.00k
)
3284
1.00k
    ha->data.f32[i] = dsfmt_genrand_open_close(&dsfmt) * 10;
3285
1.00k
  for (i = 0; i < 1000; 
i++1.00k
)
3286
1.00k
    hg->data.f32[i] = dsfmt_genrand_open_close(&dsfmt);
3287
1
  ccv_nnc_cmd_exec(CMD_DATA_TRANSFER_FORWARD(), ccv_nnc_no_hint, 0, TENSOR_LIST(ha, hg), TENSOR_LIST(a, g), 0);
3288
1
  ccv_nnc_cmd_exec(CMD_EWABS_FORWARD(), ccv_nnc_no_hint, 0, TENSOR_LIST(a), TENSOR_LIST(b), 0);
3289
1
  ccv_nnc_cmd_exec(CMD_EWABS_BACKWARD(), ccv_nnc_no_hint, 0, TENSOR_LIST(g, a), TENSOR_LIST(da), 0);
3290
1
  ccv_nnc_cmd_exec(CMD_EWABS_FORWARD(), ccv_nnc_no_hint, 0, TENSOR_LIST(ha), TENSOR_LIST(hb), 0);
3291
1
  ccv_nnc_cmd_exec(CMD_EWABS_BACKWARD(), ccv_nnc_no_hint, 0, TENSOR_LIST(hg, ha), TENSOR_LIST(dat), 0);
3292
1
  ccv_nnc_cmd_exec(CMD_DATA_TRANSFER_FORWARD(), ccv_nnc_no_hint, 0, TENSOR_LIST(da), TENSOR_LIST(hda), 0);
3293
1
  REQUIRE_TENSOR_EQ(dat, hda, "GPU computed output should be the same as CPU computed ones");
3294
1
  ccv_nnc_tensor_free(a);
3295
1
  ccv_nnc_tensor_free(b);
3296
1
  ccv_nnc_tensor_free(g);
3297
1
  ccv_nnc_tensor_free(da);
3298
1
  ccv_nnc_tensor_free(ha);
3299
1
  ccv_nnc_tensor_free(hb);
3300
1
  ccv_nnc_tensor_free(hg);
3301
1
  ccv_nnc_tensor_free(hda);
3302
1
  ccv_nnc_tensor_free(dat);
3303
1
}
3304
3305
TEST_CASE("clamp forward")
3306
1
{
3307
1
  GUARD_ELSE_RETURN(ccv_nnc_cmd_ok(CCV_NNC_CLAMP_FORWARD, CCV_NNC_BACKEND_GPU_REF));
3308
1
  ccv_nnc_tensor_t* a = ccv_nnc_tensor_new(0, GPU_TENSOR_NCHW(000, 32F, 10, 100), 0);
3309
1
  ccv_nnc_tensor_t* b = ccv_nnc_tensor_new(0, GPU_TENSOR_NCHW(000, 32F, 10, 100), 0);
3310
1
  ccv_nnc_tensor_t* ha = ccv_nnc_tensor_new(0, CPU_TENSOR_NCHW(32F, 10, 100), 0);
3311
1
  ccv_nnc_tensor_t* hb = ccv_nnc_tensor_new(0, CPU_TENSOR_NCHW(32F, 10, 100), 0);
3312
1
  ccv_nnc_tensor_t* bt = ccv_nnc_tensor_new(0, CPU_TENSOR_NCHW(32F, 10, 100), 0);
3313
1
  dsfmt_t dsfmt;
3314
1
  dsfmt_init_gen_rand(&dsfmt, 0);
3315
1
  int i;
3316
1.00k
  for (i = 0; i < 1000; 
i++1.00k
)
3317
1.00k
    ha->data.f32[i] = dsfmt_genrand_open_close(&dsfmt) * 10 - 1;
3318
1
  ccv_nnc_cmd_exec(CMD_DATA_TRANSFER_FORWARD(), ccv_nnc_no_hint, 0, TENSOR_LIST(ha), TENSOR_LIST(a), 0);
3319
1
  ccv_nnc_cmd_exec(CMD_CLAMP_FORWARD(0, 6), ccv_nnc_no_hint, 0, TENSOR_LIST(a), TENSOR_LIST(b), 0);
3320
1
  ccv_nnc_cmd_exec(CMD_CLAMP_FORWARD(0, 6), ccv_nnc_no_hint, 0, TENSOR_LIST(ha), TENSOR_LIST(bt), 0);
3321
1
  ccv_nnc_cmd_exec(CMD_DATA_TRANSFER_FORWARD(), ccv_nnc_no_hint, 0, TENSOR_LIST(b), TENSOR_LIST(hb), 0);
3322
1
  REQUIRE_TENSOR_EQ(bt, hb, "GPU computed output should be the same as CPU computed ones");
3323
1
  ccv_nnc_tensor_free(a);
3324
1
  ccv_nnc_tensor_free(b);
3325
1
  ccv_nnc_tensor_free(ha);
3326
1
  ccv_nnc_tensor_free(hb);
3327
1
  ccv_nnc_tensor_free(bt);
3328
1
}
3329
3330
TEST_CASE("clamp backward")
3331
1
{
3332
1
  GUARD_ELSE_RETURN(ccv_nnc_cmd_ok(CCV_NNC_CLAMP_FORWARD, CCV_NNC_BACKEND_GPU_REF) &&
3333
1
    ccv_nnc_cmd_ok(CCV_NNC_CLAMP_BACKWARD, CCV_NNC_BACKEND_GPU_REF));
3334
1
  ccv_nnc_tensor_t* a = ccv_nnc_tensor_new(0, GPU_TENSOR_NCHW(000, 32F, 10, 100), 0);
3335
1
  ccv_nnc_tensor_t* b = ccv_nnc_tensor_new(0, GPU_TENSOR_NCHW(000, 32F, 10, 100), 0);
3336
1
  ccv_nnc_tensor_t* g = ccv_nnc_tensor_new(0, GPU_TENSOR_NCHW(000, 32F, 10, 100), 0);
3337
1
  ccv_nnc_tensor_t* da = ccv_nnc_tensor_new(0, GPU_TENSOR_NCHW(000, 32F, 10, 100), 0);
3338
1
  ccv_nnc_tensor_t* ha = ccv_nnc_tensor_new(0, CPU_TENSOR_NCHW(32F, 10, 100), 0);
3339
1
  ccv_nnc_tensor_t* hda = ccv_nnc_tensor_new(0, CPU_TENSOR_NCHW(32F, 10, 100), 0);
3340
1
  ccv_nnc_tensor_t* hb = ccv_nnc_tensor_new(0, CPU_TENSOR_NCHW(32F, 10, 100), 0);
3341
1
  ccv_nnc_tensor_t* hg = ccv_nnc_tensor_new(0, CPU_TENSOR_NCHW(32F, 10, 100), 0);
3342
1
  ccv_nnc_tensor_t* dat = ccv_nnc_tensor_new(0, CPU_TENSOR_NCHW(32F, 10, 100), 0);
3343
1
  dsfmt_t dsfmt;
3344
1
  dsfmt_init_gen_rand(&dsfmt, 0);
3345
1
  int i;
3346
1.00k
  for (i = 0; i < 1000; 
i++1.00k
)
3347
1.00k
    ha->data.f32[i] = dsfmt_genrand_open_close(&dsfmt) * 10;
3348
1.00k
  for (i = 0; i < 1000; 
i++1.00k
)
3349
1.00k
    hg->data.f32[i] = dsfmt_genrand_open_close(&dsfmt);
3350
1
  ccv_nnc_cmd_exec(CMD_DATA_TRANSFER_FORWARD(), ccv_nnc_no_hint, 0, TENSOR_LIST(ha, hg), TENSOR_LIST(a, g), 0);
3351
1
  ccv_nnc_cmd_exec(CMD_CLAMP_FORWARD(0, 5), ccv_nnc_no_hint, 0, TENSOR_LIST(a), TENSOR_LIST(b), 0);
3352
1
  ccv_nnc_cmd_exec(CMD_CLAMP_BACKWARD(0, 5), ccv_nnc_no_hint, 0, TENSOR_LIST(g, 0, b), TENSOR_LIST(da), 0);
3353
1
  ccv_nnc_cmd_exec(CMD_CLAMP_FORWARD(0, 5), ccv_nnc_no_hint, 0, TENSOR_LIST(ha), TENSOR_LIST(hb), 0);
3354
1
  ccv_nnc_cmd_exec(CMD_CLAMP_BACKWARD(0, 5), ccv_nnc_no_hint, 0, TENSOR_LIST(hg, 0, hb), TENSOR_LIST(dat), 0);
3355
1
  ccv_nnc_cmd_exec(CMD_DATA_TRANSFER_FORWARD(), ccv_nnc_no_hint, 0, TENSOR_LIST(da), TENSOR_LIST(hda), 0);
3356
1
  REQUIRE_TENSOR_EQ(dat, hda, "GPU computed output should be the same as CPU computed ones");
3357
1
  ccv_nnc_tensor_free(a);
3358
1
  ccv_nnc_tensor_free(b);
3359
1
  ccv_nnc_tensor_free(g);
3360
1
  ccv_nnc_tensor_free(da);
3361
1
  ccv_nnc_tensor_free(ha);
3362
1
  ccv_nnc_tensor_free(hb);
3363
1
  ccv_nnc_tensor_free(hg);
3364
1
  ccv_nnc_tensor_free(hda);
3365
1
  ccv_nnc_tensor_free(dat);
3366
1
}
3367
3368
TEST_CASE("clamp forward with only max")
3369
1
{
3370
1
  GUARD_ELSE_RETURN(ccv_nnc_cmd_ok(CCV_NNC_CLAMP_FORWARD, CCV_NNC_BACKEND_GPU_REF));
3371
1
  ccv_nnc_tensor_t* a = ccv_nnc_tensor_new(0, GPU_TENSOR_NCHW(000, 32F, 10, 100), 0);
3372
1
  ccv_nnc_tensor_t* b = ccv_nnc_tensor_new(0, GPU_TENSOR_NCHW(000, 32F, 10, 100), 0);
3373
1
  ccv_nnc_tensor_t* ha = ccv_nnc_tensor_new(0, CPU_TENSOR_NCHW(32F, 10, 100), 0);
3374
1
  ccv_nnc_tensor_t* hb = ccv_nnc_tensor_new(0, CPU_TENSOR_NCHW(32F, 10, 100), 0);
3375
1
  ccv_nnc_tensor_t* bt = ccv_nnc_tensor_new(0, CPU_TENSOR_NCHW(32F, 10, 100), 0);
3376
1
  dsfmt_t dsfmt;
3377
1
  dsfmt_init_gen_rand(&dsfmt, 0);
3378
1
  int i;
3379
1.00k
  for (i = 0; i < 1000; 
i++1.00k
)
3380
1.00k
    ha->data.f32[i] = dsfmt_genrand_open_close(&dsfmt) * 10 - 1;
3381
1
  ccv_nnc_cmd_exec(CMD_DATA_TRANSFER_FORWARD(), ccv_nnc_no_hint, 0, TENSOR_LIST(ha), TENSOR_LIST(a), 0);
3382
1
  ccv_nnc_cmd_exec(CMD_CLAMP_FORWARD(NAN, 6), ccv_nnc_no_hint, 0, TENSOR_LIST(a), TENSOR_LIST(b), 0);
3383
1
  ccv_nnc_cmd_exec(CMD_CLAMP_FORWARD(NAN, 6), ccv_nnc_no_hint, 0, TENSOR_LIST(ha), TENSOR_LIST(bt), 0);
3384
1
  ccv_nnc_cmd_exec(CMD_DATA_TRANSFER_FORWARD(), ccv_nnc_no_hint, 0, TENSOR_LIST(b), TENSOR_LIST(hb), 0);
3385
1
  REQUIRE_TENSOR_EQ(bt, hb, "GPU computed output should be the same as CPU computed ones");
3386
1
  ccv_nnc_tensor_free(a);
3387
1
  ccv_nnc_tensor_free(b);
3388
1
  ccv_nnc_tensor_free(ha);
3389
1
  ccv_nnc_tensor_free(hb);
3390
1
  ccv_nnc_tensor_free(bt);
3391
1
}
3392
3393
TEST_CASE("clamp backward with only max")
3394
1
{
3395
1
  GUARD_ELSE_RETURN(ccv_nnc_cmd_ok(CCV_NNC_CLAMP_FORWARD, CCV_NNC_BACKEND_GPU_REF) &&
3396
1
    ccv_nnc_cmd_ok(CCV_NNC_CLAMP_BACKWARD, CCV_NNC_BACKEND_GPU_REF));
3397
1
  ccv_nnc_tensor_t* a = ccv_nnc_tensor_new(0, GPU_TENSOR_NCHW(000, 32F, 10, 100), 0);
3398
1
  ccv_nnc_tensor_t* b = ccv_nnc_tensor_new(0, GPU_TENSOR_NCHW(000, 32F, 10, 100), 0);
3399
1
  ccv_nnc_tensor_t* g = ccv_nnc_tensor_new(0, GPU_TENSOR_NCHW(000, 32F, 10, 100), 0);
3400
1
  ccv_nnc_tensor_t* da = ccv_nnc_tensor_new(0, GPU_TENSOR_NCHW(000, 32F, 10, 100), 0);
3401
1
  ccv_nnc_tensor_t* ha = ccv_nnc_tensor_new(0, CPU_TENSOR_NCHW(32F, 10, 100), 0);
3402
1
  ccv_nnc_tensor_t* hda = ccv_nnc_tensor_new(0, CPU_TENSOR_NCHW(32F, 10, 100), 0);
3403
1
  ccv_nnc_tensor_t* hb = ccv_nnc_tensor_new(0, CPU_TENSOR_NCHW(32F, 10, 100), 0);
3404
1
  ccv_nnc_tensor_t* hg = ccv_nnc_tensor_new(0, CPU_TENSOR_NCHW(32F, 10, 100), 0);
3405
1
  ccv_nnc_tensor_t* dat = ccv_nnc_tensor_new(0, CPU_TENSOR_NCHW(32F, 10, 100), 0);
3406
1
  dsfmt_t dsfmt;
3407
1
  dsfmt_init_gen_rand(&dsfmt, 0);
3408
1
  int i;
3409
1.00k
  for (i = 0; i < 1000; 
i++1.00k
)
3410
1.00k
    ha->data.f32[i] = dsfmt_genrand_open_close(&dsfmt) * 10;
3411
1.00k
  for (i = 0; i < 1000; 
i++1.00k
)
3412
1.00k
    hg->data.f32[i] = dsfmt_genrand_open_close(&dsfmt);
3413
1
  ccv_nnc_cmd_exec(CMD_DATA_TRANSFER_FORWARD(), ccv_nnc_no_hint, 0, TENSOR_LIST(ha, hg), TENSOR_LIST(a, g), 0);
3414
1
  ccv_nnc_cmd_exec(CMD_CLAMP_FORWARD(NAN, 5), ccv_nnc_no_hint, 0, TENSOR_LIST(a), TENSOR_LIST(b), 0);
3415
1
  ccv_nnc_cmd_exec(CMD_CLAMP_BACKWARD(NAN, 5), ccv_nnc_no_hint, 0, TENSOR_LIST(g, 0, b), TENSOR_LIST(da), 0);
3416
1
  ccv_nnc_cmd_exec(CMD_CLAMP_FORWARD(NAN, 5), ccv_nnc_no_hint, 0, TENSOR_LIST(ha), TENSOR_LIST(hb), 0);
3417
1
  ccv_nnc_cmd_exec(CMD_CLAMP_BACKWARD(NAN, 5), ccv_nnc_no_hint, 0, TENSOR_LIST(hg, 0, hb), TENSOR_LIST(dat), 0);
3418
1
  ccv_nnc_cmd_exec(CMD_DATA_TRANSFER_FORWARD(), ccv_nnc_no_hint, 0, TENSOR_LIST(da), TENSOR_LIST(hda), 0);
3419
1
  REQUIRE_TENSOR_EQ(dat, hda, "GPU computed output should be the same as CPU computed ones");
3420
1
  ccv_nnc_tensor_free(a);
3421
1
  ccv_nnc_tensor_free(b);
3422
1
  ccv_nnc_tensor_free(g);
3423
1
  ccv_nnc_tensor_free(da);
3424
1
  ccv_nnc_tensor_free(ha);
3425
1
  ccv_nnc_tensor_free(hb);
3426
1
  ccv_nnc_tensor_free(hg);
3427
1
  ccv_nnc_tensor_free(hda);
3428
1
  ccv_nnc_tensor_free(dat);
3429
1
}
3430
3431
TEST_CASE("clamp forward with only min")
3432
1
{
3433
1
  GUARD_ELSE_RETURN(ccv_nnc_cmd_ok(CCV_NNC_CLAMP_FORWARD, CCV_NNC_BACKEND_GPU_REF));
3434
1
  ccv_nnc_tensor_t* a = ccv_nnc_tensor_new(0, GPU_TENSOR_NCHW(000, 32F, 10, 100), 0);
3435
1
  ccv_nnc_tensor_t* b = ccv_nnc_tensor_new(0, GPU_TENSOR_NCHW(000, 32F, 10, 100), 0);
3436
1
  ccv_nnc_tensor_t* ha = ccv_nnc_tensor_new(0, CPU_TENSOR_NCHW(32F, 10, 100), 0);
3437
1
  ccv_nnc_tensor_t* hb = ccv_nnc_tensor_new(0, CPU_TENSOR_NCHW(32F, 10, 100), 0);
3438
1
  ccv_nnc_tensor_t* bt = ccv_nnc_tensor_new(0, CPU_TENSOR_NCHW(32F, 10, 100), 0);
3439
1
  dsfmt_t dsfmt;
3440
1
  dsfmt_init_gen_rand(&dsfmt, 0);
3441
1
  int i;
3442
1.00k
  for (i = 0; i < 1000; 
i++1.00k
)
3443
1.00k
    ha->data.f32[i] = dsfmt_genrand_open_close(&dsfmt) * 10 - 1;
3444
1
  ccv_nnc_cmd_exec(CMD_DATA_TRANSFER_FORWARD(), ccv_nnc_no_hint, 0, TENSOR_LIST(ha), TENSOR_LIST(a), 0);
3445
1
  ccv_nnc_cmd_exec(CMD_CLAMP_FORWARD(0, NAN), ccv_nnc_no_hint, 0, TENSOR_LIST(a), TENSOR_LIST(b), 0);
3446
1
  ccv_nnc_cmd_exec(CMD_CLAMP_FORWARD(0, NAN), ccv_nnc_no_hint, 0, TENSOR_LIST(ha), TENSOR_LIST(bt), 0);
3447
1
  ccv_nnc_cmd_exec(CMD_DATA_TRANSFER_FORWARD(), ccv_nnc_no_hint, 0, TENSOR_LIST(b), TENSOR_LIST(hb), 0);
3448
1
  REQUIRE_TENSOR_EQ(bt, hb, "GPU computed output should be the same as CPU computed ones");
3449
1
  ccv_nnc_tensor_free(a);
3450
1
  ccv_nnc_tensor_free(b);
3451
1
  ccv_nnc_tensor_free(ha);
3452
1
  ccv_nnc_tensor_free(hb);
3453
1
  ccv_nnc_tensor_free(bt);
3454
1
}
3455
3456
TEST_CASE("clamp backward with only min")
3457
1
{
3458
1
  GUARD_ELSE_RETURN(ccv_nnc_cmd_ok(CCV_NNC_CLAMP_FORWARD, CCV_NNC_BACKEND_GPU_REF) &&
3459
1
    ccv_nnc_cmd_ok(CCV_NNC_CLAMP_BACKWARD, CCV_NNC_BACKEND_GPU_REF));
3460
1
  ccv_nnc_tensor_t* a = ccv_nnc_tensor_new(0, GPU_TENSOR_NCHW(000, 32F, 10, 100), 0);
3461
1
  ccv_nnc_tensor_t* b = ccv_nnc_tensor_new(0, GPU_TENSOR_NCHW(000, 32F, 10, 100), 0);
3462
1
  ccv_nnc_tensor_t* g = ccv_nnc_tensor_new(0, GPU_TENSOR_NCHW(000, 32F, 10, 100), 0);
3463
1
  ccv_nnc_tensor_t* da = ccv_nnc_tensor_new(0, GPU_TENSOR_NCHW(000, 32F, 10, 100), 0);
3464
1
  ccv_nnc_tensor_t* ha = ccv_nnc_tensor_new(0, CPU_TENSOR_NCHW(32F, 10, 100), 0);
3465
1
  ccv_nnc_tensor_t* hda = ccv_nnc_tensor_new(0, CPU_TENSOR_NCHW(32F, 10, 100), 0);
3466
1
  ccv_nnc_tensor_t* hb = ccv_nnc_tensor_new(0, CPU_TENSOR_NCHW(32F, 10, 100), 0);
3467
1
  ccv_nnc_tensor_t* hg = ccv_nnc_tensor_new(0, CPU_TENSOR_NCHW(32F, 10, 100), 0);
3468
1
  ccv_nnc_tensor_t* dat = ccv_nnc_tensor_new(0, CPU_TENSOR_NCHW(32F, 10, 100), 0);
3469
1
  dsfmt_t dsfmt;
3470
1
  dsfmt_init_gen_rand(&dsfmt, 0);
3471
1
  int i;
3472
1.00k
  for (i = 0; i < 1000; 
i++1.00k
)
3473
1.00k
    ha->data.f32[i] = dsfmt_genrand_open_close(&dsfmt) * 10;
3474
1.00k
  for (i = 0; i < 1000; 
i++1.00k
)
3475
1.00k
    hg->data.f32[i] = dsfmt_genrand_open_close(&dsfmt);
3476
1
  ccv_nnc_cmd_exec(CMD_DATA_TRANSFER_FORWARD(), ccv_nnc_no_hint, 0, TENSOR_LIST(ha, hg), TENSOR_LIST(a, g), 0);
3477
1
  ccv_nnc_cmd_exec(CMD_CLAMP_FORWARD(0, NAN), ccv_nnc_no_hint, 0, TENSOR_LIST(a), TENSOR_LIST(b), 0);
3478
1
  ccv_nnc_cmd_exec(CMD_CLAMP_BACKWARD(0, NAN), ccv_nnc_no_hint, 0, TENSOR_LIST(g, 0, b), TENSOR_LIST(da), 0);
3479
1
  ccv_nnc_cmd_exec(CMD_CLAMP_FORWARD(0, NAN), ccv_nnc_no_hint, 0, TENSOR_LIST(ha), TENSOR_LIST(hb), 0);
3480
1
  ccv_nnc_cmd_exec(CMD_CLAMP_BACKWARD(0, NAN), ccv_nnc_no_hint, 0, TENSOR_LIST(hg, 0, hb), TENSOR_LIST(dat), 0);
3481
1
  ccv_nnc_cmd_exec(CMD_DATA_TRANSFER_FORWARD(), ccv_nnc_no_hint, 0, TENSOR_LIST(da), TENSOR_LIST(hda), 0);
3482
1
  REQUIRE_TENSOR_EQ(dat, hda, "GPU computed output should be the same as CPU computed ones");
3483
1
  ccv_nnc_tensor_free(a);
3484
1
  ccv_nnc_tensor_free(b);
3485
1
  ccv_nnc_tensor_free(g);
3486
1
  ccv_nnc_tensor_free(da);
3487
1
  ccv_nnc_tensor_free(ha);
3488
1
  ccv_nnc_tensor_free(hb);
3489
1
  ccv_nnc_tensor_free(hg);
3490
1
  ccv_nnc_tensor_free(hda);
3491
1
  ccv_nnc_tensor_free(dat);
3492
1
}
3493
3494
TEST_CASE("scaled dot product attention with flash_attn")
3495
1
{
3496
1
  GUARD_ELSE_RETURN(ccv_nnc_cmd_ok(CCV_NNC_SCALED_DOT_PRODUCT_ATTENTION_FORWARD, CCV_NNC_BACKEND_GPU_REF));
3497
  // Bypass error: variable-sized object may not be initialized
3498
0
#define num_long_trials 4
3499
0
#define num_short_trials 2
3500
0
#define num_trials (num_long_trials + num_short_trials)
3501
3502
0
  for (int trial = 0; trial < num_trials; ++trial) {
3503
0
    int B_candidates[num_trials] = {  32,   12, 16, 1, 2, 1 };
3504
0
    int R_candidates[num_trials] = { 160,  256, 128, 77, 77, 5 };
3505
0
    int C_candidates[num_trials] = { 128,  128, 128, 128, 128, 5 };
3506
0
    int Hq_candidates[num_trials] = {   8,  8, 8, 8, 8, 32 };
3507
0
    int Hk_candidates[num_trials] = {   8,  8, 8, 8, 2, 8 };
3508
0
    int D_candidates[num_trials] = {  64, 40, 160, 224, 224, 128 };
3509
0
    int is_causal_candidates[num_trials] = {  1, 0, 1, 1, 0, 1 };
3510
3511
0
    int B = B_candidates[trial];
3512
0
    int R = R_candidates[trial];
3513
0
    int C = C_candidates[trial];
3514
0
    int Hq = Hq_candidates[trial];
3515
0
    int Hk = Hk_candidates[trial];
3516
0
    int D = D_candidates[trial];
3517
0
    int is_causal = is_causal_candidates[trial];
3518
0
    float scale = 1.0 / sqrt((float)D);
3519
3520
0
    GUARD_ELSE_RETURN(ccv_nnc_cmd_ok(CCV_NNC_SCALED_DOT_PRODUCT_ATTENTION_FORWARD, CCV_NNC_BACKEND_GPU_REF));
3521
0
    ccv_nnc_tensor_t* const q_tensor = ccv_nnc_tensor_new(0, CPU_TENSOR_NHWC(32F, B, R, Hq, D), 0);
3522
0
    ccv_nnc_tensor_t* const k_tensor = ccv_nnc_tensor_new(0, CPU_TENSOR_NHWC(32F, B, C, Hk, D), 0);
3523
0
    ccv_nnc_tensor_t* const v_tensor = ccv_nnc_tensor_new(0, CPU_TENSOR_NHWC(32F, B, C, Hk, D), 0);
3524
3525
0
    for (int i = 0; i < B * R * Hq * D; ++i) {
3526
0
      q_tensor->data.f32[i] = (float)(i) / (float)(B * R * Hq * D);
3527
0
    }
3528
0
    for (int i = 0; i < B * C * Hk * D; ++i) {
3529
0
      k_tensor->data.f32[i] = (float)(i) / (float)(B * C * Hk * D);
3530
0
    }
3531
0
    for (int i = 0; i < B * C * Hk * D; ++i) {
3532
0
      v_tensor->data.f32[i] = (float)(i) / (float)(B * C * Hk * D);
3533
0
    }
3534
3535
0
    ccv_nnc_tensor_t* const o_tensor = ccv_nnc_tensor_new(0, CPU_TENSOR_NHWC(32F, B, R, Hq, D), 0);
3536
0
    ccv_nnc_cmd_exec(CMD_SCALED_DOT_PRODUCT_ATTENTION_FORWARD(scale, is_causal), ccv_nnc_no_hint, 0, TENSOR_LIST(q_tensor, k_tensor, v_tensor, NULL, NULL, NULL), TENSOR_LIST(o_tensor, NULL), 0);
3537
0
    ccv_nnc_tensor_t* const q_tensor_f16 = ccv_nnc_tensor_new(0, CPU_TENSOR_NHWC(16F, B, R, Hq, D), 0);
3538
0
    ccv_nnc_tensor_t* const k_tensor_f16 = ccv_nnc_tensor_new(0, CPU_TENSOR_NHWC(16F, B, C, Hk, D), 0);
3539
0
    ccv_nnc_tensor_t* const v_tensor_f16 = ccv_nnc_tensor_new(0, CPU_TENSOR_NHWC(16F, B, C, Hk, D), 0);
3540
0
    ccv_nnc_cmd_exec(CMD_DATATYPE_CONVERSION_FORWARD(), ccv_nnc_no_hint, 0, TENSOR_LIST(q_tensor, k_tensor, v_tensor), TENSOR_LIST(q_tensor_f16, k_tensor_f16, v_tensor_f16), 0);
3541
3542
    // Why it there 000 in the beginning of the argument list for GPU_TENSOR_NHWC?
3543
0
    ccv_nnc_tensor_t* const gpu_q_tensor = ccv_nnc_tensor_new(0, GPU_TENSOR_NHWC(000, 16F, B, R, Hq, D), 0);
3544
0
    ccv_nnc_tensor_t* const gpu_k_tensor = ccv_nnc_tensor_new(0, GPU_TENSOR_NHWC(000, 16F, B, C, Hk, D), 0);
3545
0
    ccv_nnc_tensor_t* const gpu_v_tensor = ccv_nnc_tensor_new(0, GPU_TENSOR_NHWC(000, 16F, B, C, Hk, D), 0);
3546
0
    ccv_nnc_tensor_t* const gpu_o_tensor = ccv_nnc_tensor_new(0, GPU_TENSOR_NHWC(000, 16F, B, R, Hq, D), 0);
3547
0
    ccv_nnc_cmd_exec(CMD_DATA_TRANSFER_FORWARD(), ccv_nnc_no_hint, 0, TENSOR_LIST(q_tensor_f16, k_tensor_f16, v_tensor_f16), TENSOR_LIST(gpu_q_tensor, gpu_k_tensor, gpu_v_tensor), 0);
3548
3549
0
    ccv_nnc_cmd_exec(CMD_SCALED_DOT_PRODUCT_ATTENTION_FORWARD(scale, is_causal), ccv_nnc_no_hint, 0, TENSOR_LIST(gpu_q_tensor, gpu_k_tensor, gpu_v_tensor, NULL, NULL, NULL), TENSOR_LIST(gpu_o_tensor, NULL), 0);
3550
3551
0
    ccv_nnc_tensor_t* const copy_of_gpu_o_tensor_f16 = ccv_nnc_tensor_new(0, CPU_TENSOR_NHWC(16F, B, R, Hq, D), 0);
3552
0
    ccv_nnc_cmd_exec(CMD_DATA_TRANSFER_FORWARD(), ccv_nnc_no_hint, 0, TENSOR_LIST(gpu_o_tensor), TENSOR_LIST(copy_of_gpu_o_tensor_f16), 0);
3553
0
    ccv_nnc_tensor_t* const copy_of_gpu_o_tensor = ccv_nnc_tensor_new(0, CPU_TENSOR_NHWC(32F, B, R, Hq, D), 0);
3554
0
    ccv_nnc_cmd_exec(CMD_DATATYPE_CONVERSION_FORWARD(), ccv_nnc_no_hint, 0, TENSOR_LIST(copy_of_gpu_o_tensor_f16), TENSOR_LIST(copy_of_gpu_o_tensor), 0);
3555
3556
0
    REQUIRE_ARRAY_EQ_WITH_TOLERANCE(float, copy_of_gpu_o_tensor->data.f32, o_tensor->data.f32, B * R * Hq * D, 3e-3, "GPU computed output should be the same as CPU computed ones");
3557
3558
0
    ccv_nnc_tensor_free(o_tensor);
3559
0
    ccv_nnc_tensor_free(gpu_o_tensor);
3560
0
    ccv_nnc_tensor_free(copy_of_gpu_o_tensor);
3561
0
    ccv_nnc_tensor_free(copy_of_gpu_o_tensor_f16);
3562
0
    ccv_nnc_tensor_free(q_tensor);
3563
0
    ccv_nnc_tensor_free(k_tensor);
3564
0
    ccv_nnc_tensor_free(v_tensor);
3565
0
    ccv_nnc_tensor_free(q_tensor_f16);
3566
0
    ccv_nnc_tensor_free(k_tensor_f16);
3567
0
    ccv_nnc_tensor_free(v_tensor_f16);
3568
0
    ccv_nnc_tensor_free(gpu_q_tensor);
3569
0
    ccv_nnc_tensor_free(gpu_k_tensor);
3570
0
    ccv_nnc_tensor_free(gpu_v_tensor);
3571
0
  }
3572
0
#undef num_long_trials
3573
0
#undef num_short_trials
3574
0
#undef num_trials
3575
0
}
3576
3577
TEST_CASE("scaled dot product attention with flash_attn varlen")
3578
1
{
3579
1
  GUARD_ELSE_RETURN(ccv_nnc_cmd_ok(CCV_NNC_SCALED_DOT_PRODUCT_ATTENTION_FORWARD, CCV_NNC_BACKEND_GPU_REF));
3580
0
  const int B = 4;
3581
0
  const int Hq = 4;
3582
0
  const int Hk = 2;
3583
0
  const int D = 64;
3584
0
  const int q_offsets[2][5] = {
3585
0
    { 0, 5, 8, 15, 17 },
3586
0
    { 0, 4, 10, 13, 20 },
3587
0
  };
3588
0
  const int kv_offsets[2][5] = {
3589
0
    { 0, 6, 13, 16, 22 },
3590
0
    { 0, 4, 11, 15, 21 },
3591
0
  };
3592
0
  const int max_seqlen_q[2] = { 7, 7 };
3593
0
  const int max_seqlen_kv[2] = { 7, 7 };
3594
0
  for (int trial = 0; trial < 2; ++trial)
3595
0
  {
3596
0
    const int is_causal = trial;
3597
0
    const int total_q = q_offsets[trial][B];
3598
0
    const int total_k = kv_offsets[trial][B];
3599
0
    const float scale = 1.0 / sqrt((float)D);
3600
0
    ccv_nnc_tensor_t* const q_tensor = ccv_nnc_tensor_new(0, CPU_TENSOR_NHWC(32F, 1, total_q, Hq, D), 0);
3601
0
    ccv_nnc_tensor_t* const k_tensor = ccv_nnc_tensor_new(0, CPU_TENSOR_NHWC(32F, 1, total_k, Hk, D), 0);
3602
0
    ccv_nnc_tensor_t* const v_tensor = ccv_nnc_tensor_new(0, CPU_TENSOR_NHWC(32F, 1, total_k, Hk, D), 0);
3603
0
    for (int i = 0; i < total_q * Hq * D; ++i)
3604
0
      q_tensor->data.f32[i] = (float)((i * 17) % 97 - 48) / 256;
3605
0
    for (int i = 0; i < total_k * Hk * D; ++i)
3606
0
    {
3607
0
      k_tensor->data.f32[i] = (float)((i * 13) % 89 - 44) / 256;
3608
0
      v_tensor->data.f32[i] = (float)((i * 19) % 101 - 50) / 128;
3609
0
    }
3610
0
    ccv_nnc_tensor_t* const o_tensor_ref = ccv_nnc_tensor_new(0, CPU_TENSOR_NHWC(32F, 1, total_q, Hq, D), 0);
3611
0
    for (int b = 0; b < B; ++b)
3612
0
    {
3613
0
      const int q_start = q_offsets[trial][b];
3614
0
      const int k_start = kv_offsets[trial][b];
3615
0
      const int R = q_offsets[trial][b + 1] - q_start;
3616
0
      const int C = kv_offsets[trial][b + 1] - k_start;
3617
0
      ccv_nnc_tensor_t* const q_seq = ccv_nnc_tensor_new(0, CPU_TENSOR_NHWC(32F, 1, R, Hq, D), 0);
3618
0
      ccv_nnc_tensor_t* const k_seq = ccv_nnc_tensor_new(0, CPU_TENSOR_NHWC(32F, 1, C, Hk, D), 0);
3619
0
      ccv_nnc_tensor_t* const v_seq = ccv_nnc_tensor_new(0, CPU_TENSOR_NHWC(32F, 1, C, Hk, D), 0);
3620
0
      ccv_nnc_tensor_t* const o_seq = ccv_nnc_tensor_new(0, CPU_TENSOR_NHWC(32F, 1, R, Hq, D), 0);
3621
0
      memcpy(q_seq->data.f32, q_tensor->data.f32 + q_start * Hq * D, sizeof(float) * R * Hq * D);
3622
0
      memcpy(k_seq->data.f32, k_tensor->data.f32 + k_start * Hk * D, sizeof(float) * C * Hk * D);
3623
0
      memcpy(v_seq->data.f32, v_tensor->data.f32 + k_start * Hk * D, sizeof(float) * C * Hk * D);
3624
0
      ccv_nnc_cmd_exec(CMD_SCALED_DOT_PRODUCT_ATTENTION_FORWARD(scale, is_causal), ccv_nnc_no_hint, 0, TENSOR_LIST(q_seq, k_seq, v_seq), TENSOR_LIST(o_seq), 0);
3625
0
      memcpy(o_tensor_ref->data.f32 + q_start * Hq * D, o_seq->data.f32, sizeof(float) * R * Hq * D);
3626
0
      ccv_nnc_tensor_free(q_seq);
3627
0
      ccv_nnc_tensor_free(k_seq);
3628
0
      ccv_nnc_tensor_free(v_seq);
3629
0
      ccv_nnc_tensor_free(o_seq);
3630
0
    }
3631
0
    ccv_nnc_tensor_t* const h_q_seq_offsets = ccv_nnc_tensor_new(0, CPU_TENSOR_NHWC(32S, B + 1), 0);
3632
0
    ccv_nnc_tensor_t* const h_kv_seq_offsets = ccv_nnc_tensor_new(0, CPU_TENSOR_NHWC(32S, B + 1), 0);
3633
0
    for (int i = 0; i < B + 1; ++i)
3634
0
    {
3635
0
      h_q_seq_offsets->data.i32[i] = q_offsets[trial][i];
3636
0
      h_kv_seq_offsets->data.i32[i] = kv_offsets[trial][i];
3637
0
    }
3638
0
    ccv_nnc_cmd_t cmd = CMD_SCALED_DOT_PRODUCT_ATTENTION_FORWARD(scale, is_causal);
3639
0
    cmd.info.scaled_dot_product_attention.is_varlen = 1;
3640
0
    cmd.info.scaled_dot_product_attention.max_seqlen_q = max_seqlen_q[trial];
3641
0
    cmd.info.scaled_dot_product_attention.max_seqlen_kv = max_seqlen_kv[trial];
3642
0
    ccv_nnc_tensor_t* const o_tensor = ccv_nnc_tensor_new(0, CPU_TENSOR_NHWC(32F, 1, total_q, Hq, D), 0);
3643
0
    ccv_nnc_cmd_exec(cmd, ccv_nnc_no_hint, 0, TENSOR_LIST(q_tensor, k_tensor, v_tensor, NULL, NULL, NULL, h_q_seq_offsets, h_kv_seq_offsets), TENSOR_LIST(o_tensor, NULL), 0);
3644
0
    REQUIRE_ARRAY_EQ_WITH_TOLERANCE(float, o_tensor->data.f32, o_tensor_ref->data.f32, total_q * Hq * D, 1e-5, "varlen CPU computed output should match per-sequence CPU output when causal=%d", is_causal);
3645
0
    ccv_nnc_tensor_t* const q_tensor_f16 = ccv_nnc_tensor_new(0, CPU_TENSOR_NHWC(16F, 1, total_q, Hq, D), 0);
3646
0
    ccv_nnc_tensor_t* const k_tensor_f16 = ccv_nnc_tensor_new(0, CPU_TENSOR_NHWC(16F, 1, total_k, Hk, D), 0);
3647
0
    ccv_nnc_tensor_t* const v_tensor_f16 = ccv_nnc_tensor_new(0, CPU_TENSOR_NHWC(16F, 1, total_k, Hk, D), 0);
3648
0
    ccv_nnc_cmd_exec(CMD_DATATYPE_CONVERSION_FORWARD(), ccv_nnc_no_hint, 0, TENSOR_LIST(q_tensor, k_tensor, v_tensor), TENSOR_LIST(q_tensor_f16, k_tensor_f16, v_tensor_f16), 0);
3649
0
    ccv_nnc_tensor_t* const gpu_q_tensor = ccv_nnc_tensor_new(0, GPU_TENSOR_NHWC(000, 16F, 1, total_q, Hq, D), 0);
3650
0
    ccv_nnc_tensor_t* const gpu_k_tensor = ccv_nnc_tensor_new(0, GPU_TENSOR_NHWC(000, 16F, 1, total_k, Hk, D), 0);
3651
0
    ccv_nnc_tensor_t* const gpu_v_tensor = ccv_nnc_tensor_new(0, GPU_TENSOR_NHWC(000, 16F, 1, total_k, Hk, D), 0);
3652
0
    ccv_nnc_tensor_t* const gpu_q_seq_offsets = ccv_nnc_tensor_new(0, GPU_TENSOR_NHWC(000, 32S, B + 1), 0);
3653
0
    ccv_nnc_tensor_t* const gpu_kv_seq_offsets = ccv_nnc_tensor_new(0, GPU_TENSOR_NHWC(000, 32S, B + 1), 0);
3654
0
    ccv_nnc_tensor_t* const gpu_o_tensor = ccv_nnc_tensor_new(0, GPU_TENSOR_NHWC(000, 16F, 1, total_q, Hq, D), 0);
3655
0
    ccv_nnc_cmd_exec(CMD_DATA_TRANSFER_FORWARD(), ccv_nnc_no_hint, 0, TENSOR_LIST(q_tensor_f16, k_tensor_f16, v_tensor_f16, h_q_seq_offsets, h_kv_seq_offsets), TENSOR_LIST(gpu_q_tensor, gpu_k_tensor, gpu_v_tensor, gpu_q_seq_offsets, gpu_kv_seq_offsets), 0);
3656
0
    ccv_nnc_cmd_exec(cmd, ccv_nnc_no_hint, 0, TENSOR_LIST(gpu_q_tensor, gpu_k_tensor, gpu_v_tensor, NULL, NULL, NULL, gpu_q_seq_offsets, gpu_kv_seq_offsets), TENSOR_LIST(gpu_o_tensor, NULL), 0);
3657
0
    ccv_nnc_tensor_t* const copy_of_gpu_o_tensor_f16 = ccv_nnc_tensor_new(0, CPU_TENSOR_NHWC(16F, 1, total_q, Hq, D), 0);
3658
0
    ccv_nnc_cmd_exec(CMD_DATA_TRANSFER_FORWARD(), ccv_nnc_no_hint, 0, TENSOR_LIST(gpu_o_tensor), TENSOR_LIST(copy_of_gpu_o_tensor_f16), 0);
3659
0
    ccv_nnc_tensor_t* const copy_of_gpu_o_tensor = ccv_nnc_tensor_new(0, CPU_TENSOR_NHWC(32F, 1, total_q, Hq, D), 0);
3660
0
    ccv_nnc_cmd_exec(CMD_DATATYPE_CONVERSION_FORWARD(), ccv_nnc_no_hint, 0, TENSOR_LIST(copy_of_gpu_o_tensor_f16), TENSOR_LIST(copy_of_gpu_o_tensor), 0);
3661
0
    REQUIRE_ARRAY_EQ_WITH_TOLERANCE(float, copy_of_gpu_o_tensor->data.f32, o_tensor->data.f32, total_q * Hq * D, 5e-3, "varlen GPU computed output should match packed CPU output when causal=%d", is_causal);
3662
0
    ccv_nnc_tensor_free(q_tensor);
3663
0
    ccv_nnc_tensor_free(k_tensor);
3664
0
    ccv_nnc_tensor_free(v_tensor);
3665
0
    ccv_nnc_tensor_free(o_tensor_ref);
3666
0
    ccv_nnc_tensor_free(o_tensor);
3667
0
    ccv_nnc_tensor_free(q_tensor_f16);
3668
0
    ccv_nnc_tensor_free(k_tensor_f16);
3669
0
    ccv_nnc_tensor_free(v_tensor_f16);
3670
0
    ccv_nnc_tensor_free(h_q_seq_offsets);
3671
0
    ccv_nnc_tensor_free(h_kv_seq_offsets);
3672
0
    ccv_nnc_tensor_free(gpu_q_tensor);
3673
0
    ccv_nnc_tensor_free(gpu_k_tensor);
3674
0
    ccv_nnc_tensor_free(gpu_v_tensor);
3675
0
    ccv_nnc_tensor_free(gpu_q_seq_offsets);
3676
0
    ccv_nnc_tensor_free(gpu_kv_seq_offsets);
3677
0
    ccv_nnc_tensor_free(gpu_o_tensor);
3678
0
    ccv_nnc_tensor_free(copy_of_gpu_o_tensor_f16);
3679
0
    ccv_nnc_tensor_free(copy_of_gpu_o_tensor);
3680
0
  }
3681
0
}
3682
3683
TEST_CASE("scaled dot product attention with flash_attn in bfloat")
3684
1
{
3685
1
  GUARD_ELSE_RETURN(ccv_nnc_cmd_ok(CCV_NNC_SCALED_DOT_PRODUCT_ATTENTION_FORWARD, CCV_NNC_BACKEND_GPU_REF));
3686
  // Bypass error: variable-sized object may not be initialized
3687
0
#define num_long_trials 4
3688
0
#define num_short_trials 2
3689
0
#define num_trials (num_long_trials + num_short_trials)
3690
3691
0
  for (int trial = 0; trial < num_trials; ++trial) {
3692
0
    int B_candidates[num_trials] = {  32,   12, 16, 1, 2, 1 };
3693
0
    int R_candidates[num_trials] = { 160,  256, 128, 77, 77, 5 };
3694
0
    int C_candidates[num_trials] = { 128,  128, 128, 128, 128, 5 };
3695
0
    int Hq_candidates[num_trials] = {   8,  8, 8, 8, 8, 32 };
3696
0
    int Hk_candidates[num_trials] = {   8,  8, 8, 8, 2, 8 };
3697
0
    int D_candidates[num_trials] = {  64, 40, 160, 224, 224, 128 };
3698
0
    int is_causal_candidates[num_trials] = {  1, 0, 1, 1, 0, 1 };
3699
3700
0
    int B = B_candidates[trial];
3701
0
    int R = R_candidates[trial];
3702
0
    int C = C_candidates[trial];
3703
0
    int Hq = Hq_candidates[trial];
3704
0
    int Hk = Hk_candidates[trial];
3705
0
    int D = D_candidates[trial];
3706
0
    int is_causal = is_causal_candidates[trial];
3707
0
    float scale = 1.0 / sqrt((float)D);
3708
3709
0
    GUARD_ELSE_RETURN(ccv_nnc_cmd_ok(CCV_NNC_SCALED_DOT_PRODUCT_ATTENTION_FORWARD, CCV_NNC_BACKEND_GPU_REF));
3710
0
    ccv_nnc_tensor_t* const q_tensor = ccv_nnc_tensor_new(0, CPU_TENSOR_NHWC(32F, B, R, Hq, D), 0);
3711
0
    ccv_nnc_tensor_t* const k_tensor = ccv_nnc_tensor_new(0, CPU_TENSOR_NHWC(32F, B, C, Hk, D), 0);
3712
0
    ccv_nnc_tensor_t* const v_tensor = ccv_nnc_tensor_new(0, CPU_TENSOR_NHWC(32F, B, C, Hk, D), 0);
3713
3714
0
    for (int i = 0; i < B * R * Hq * D; ++i) {
3715
0
      q_tensor->data.f32[i] = (float)(i) / (float)(B * R * Hq * D);
3716
0
    }
3717
0
    for (int i = 0; i < B * C * Hk * D; ++i) {
3718
0
      k_tensor->data.f32[i] = (float)(i) / (float)(B * C * Hk * D);
3719
0
    }
3720
0
    for (int i = 0; i < B * C * Hk * D; ++i) {
3721
0
      v_tensor->data.f32[i] = (float)(i) / (float)(B * C * Hk * D);
3722
0
    }
3723
3724
0
    ccv_nnc_tensor_t* const o_tensor = ccv_nnc_tensor_new(0, CPU_TENSOR_NHWC(32F, B, R, Hq, D), 0);
3725
0
    ccv_nnc_cmd_exec(CMD_SCALED_DOT_PRODUCT_ATTENTION_FORWARD(scale, is_causal), ccv_nnc_no_hint, 0, TENSOR_LIST(q_tensor, k_tensor, v_tensor, NULL, NULL, NULL), TENSOR_LIST(o_tensor, NULL), 0);
3726
0
    ccv_nnc_tensor_t* const q_tensor_f16 = ccv_nnc_tensor_new(0, CPU_TENSOR_NHWC(16BF, B, R, Hq, D), 0);
3727
0
    ccv_nnc_tensor_t* const k_tensor_f16 = ccv_nnc_tensor_new(0, CPU_TENSOR_NHWC(16BF, B, C, Hk, D), 0);
3728
0
    ccv_nnc_tensor_t* const v_tensor_f16 = ccv_nnc_tensor_new(0, CPU_TENSOR_NHWC(16BF, B, C, Hk, D), 0);
3729
0
    ccv_nnc_cmd_exec(CMD_DATATYPE_CONVERSION_FORWARD(), ccv_nnc_no_hint, 0, TENSOR_LIST(q_tensor, k_tensor, v_tensor), TENSOR_LIST(q_tensor_f16, k_tensor_f16, v_tensor_f16), 0);
3730
3731
    // Why it there 000 in the beginning of the argument list for GPU_TENSOR_NHWC?
3732
0
    ccv_nnc_tensor_t* const gpu_q_tensor = ccv_nnc_tensor_new(0, GPU_TENSOR_NHWC(000, 16BF, B, R, Hq, D), 0);
3733
0
    ccv_nnc_tensor_t* const gpu_k_tensor = ccv_nnc_tensor_new(0, GPU_TENSOR_NHWC(000, 16BF, B, C, Hk, D), 0);
3734
0
    ccv_nnc_tensor_t* const gpu_v_tensor = ccv_nnc_tensor_new(0, GPU_TENSOR_NHWC(000, 16BF, B, C, Hk, D), 0);
3735
0
    ccv_nnc_tensor_t* const gpu_o_tensor = ccv_nnc_tensor_new(0, GPU_TENSOR_NHWC(000, 16BF, B, R, Hq, D), 0);
3736
0
    ccv_nnc_cmd_exec(CMD_DATA_TRANSFER_FORWARD(), ccv_nnc_no_hint, 0, TENSOR_LIST(q_tensor_f16, k_tensor_f16, v_tensor_f16), TENSOR_LIST(gpu_q_tensor, gpu_k_tensor, gpu_v_tensor), 0);
3737
3738
0
    ccv_nnc_cmd_exec(CMD_SCALED_DOT_PRODUCT_ATTENTION_FORWARD(scale, is_causal), ccv_nnc_no_hint, 0, TENSOR_LIST(gpu_q_tensor, gpu_k_tensor, gpu_v_tensor, NULL, NULL, NULL), TENSOR_LIST(gpu_o_tensor, NULL), 0);
3739
3740
0
    ccv_nnc_tensor_t* const copy_of_gpu_o_tensor_f16 = ccv_nnc_tensor_new(0, CPU_TENSOR_NHWC(16BF, B, R, Hq, D), 0);
3741
0
    ccv_nnc_cmd_exec(CMD_DATA_TRANSFER_FORWARD(), ccv_nnc_no_hint, 0, TENSOR_LIST(gpu_o_tensor), TENSOR_LIST(copy_of_gpu_o_tensor_f16), 0);
3742
0
    ccv_nnc_tensor_t* const copy_of_gpu_o_tensor = ccv_nnc_tensor_new(0, CPU_TENSOR_NHWC(32F, B, R, Hq, D), 0);
3743
0
    ccv_nnc_cmd_exec(CMD_DATATYPE_CONVERSION_FORWARD(), ccv_nnc_no_hint, 0, TENSOR_LIST(copy_of_gpu_o_tensor_f16), TENSOR_LIST(copy_of_gpu_o_tensor), 0);
3744
3745
0
    REQUIRE_ARRAY_EQ_WITH_TOLERANCE(float, copy_of_gpu_o_tensor->data.f32, o_tensor->data.f32, B * R * Hq * D, 1e-2, "GPU computed output should be the same as CPU computed ones");
3746
3747
0
    ccv_nnc_tensor_free(o_tensor);
3748
0
    ccv_nnc_tensor_free(gpu_o_tensor);
3749
0
    ccv_nnc_tensor_free(copy_of_gpu_o_tensor);
3750
0
    ccv_nnc_tensor_free(copy_of_gpu_o_tensor_f16);
3751
0
    ccv_nnc_tensor_free(q_tensor);
3752
0
    ccv_nnc_tensor_free(k_tensor);
3753
0
    ccv_nnc_tensor_free(v_tensor);
3754
0
    ccv_nnc_tensor_free(q_tensor_f16);
3755
0
    ccv_nnc_tensor_free(k_tensor_f16);
3756
0
    ccv_nnc_tensor_free(v_tensor_f16);
3757
0
    ccv_nnc_tensor_free(gpu_q_tensor);
3758
0
    ccv_nnc_tensor_free(gpu_k_tensor);
3759
0
    ccv_nnc_tensor_free(gpu_v_tensor);
3760
0
  }
3761
0
#undef num_long_trials
3762
0
#undef num_short_trials
3763
0
#undef num_trials
3764
0
}
3765
3766
TEST_CASE("scaled dot product attention + unify head with flash_attn")
3767
1
{
3768
1
  GUARD_ELSE_RETURN(ccv_nnc_cmd_ok(CCV_NNC_SCALED_DOT_PRODUCT_ATTENTION_FORWARD, CCV_NNC_BACKEND_GPU_REF));
3769
0
  ccv_nnc_symbolic_graph_t* const sdp_symbolic_graph = ccv_nnc_symbolic_graph_new();
3770
0
  ccv_nnc_tensor_symbol_t q = ccv_nnc_tensor_symbol_new(sdp_symbolic_graph, CPU_TENSOR_NHWC(32F, 32, 128, 8, 64), "q");
3771
0
  ccv_nnc_tensor_symbol_t k = ccv_nnc_tensor_symbol_new(sdp_symbolic_graph, CPU_TENSOR_NHWC(32F, 32, 128, 8, 64), "k");
3772
0
  ccv_nnc_tensor_symbol_t v = ccv_nnc_tensor_symbol_new(sdp_symbolic_graph, CPU_TENSOR_NHWC(32F, 32, 128, 8, 64), "v");
3773
0
  ccv_nnc_tensor_symbol_t w = ccv_nnc_tensor_symbol_new(sdp_symbolic_graph, CPU_TENSOR_NHWC(32F, 512, 512), "w");
3774
0
  ccv_nnc_tensor_symbol_t bias = ccv_nnc_tensor_symbol_new(sdp_symbolic_graph, CPU_TENSOR_NHWC(32F, 512), "bias");
3775
0
  ccv_nnc_tensor_symbol_t c = ccv_nnc_tensor_symbol_new(sdp_symbolic_graph, CPU_TENSOR_NHWC(32F, 32, 128, 8, 64), "c");
3776
0
  ccv_nnc_tensor_symbol_t r = ccv_nnc_tensor_symbol_new(sdp_symbolic_graph, CPU_TENSOR_NHWC(32F, 32, 128, 512), "r");
3777
0
  ccv_nnc_graph_exec_symbol_new(sdp_symbolic_graph, CMD_SCALED_DOT_PRODUCT_ATTENTION_FORWARD(1.0 / 8, 0), TENSOR_SYMBOL_LIST(q, k, v, NO_TENSOR_SYMBOL, w, bias), TENSOR_SYMBOL_LIST(r, NO_TENSOR_SYMBOL, c), "scaled_dot_product_attention");
3778
0
  ccv_nnc_graph_exec_symbol_autogen(sdp_symbolic_graph, 0, 0, CCV_NNC_AUTOGEN_ALL_EXECS | CCV_NNC_AUTOGEN_SOURCES_AND_DESTINATIONS);
3779
0
  ccv_nnc_graph_t* sdp_graph = 0;
3780
0
  ccv_nnc_tensor_arena_t* sdp_tensor_arena = 0;
3781
0
  ccv_nnc_graph_exec_arena_t* sdp_graph_exec_arena = 0;
3782
0
  ccv_nnc_symbolic_graph_compile(sdp_symbolic_graph, ccv_nnc_default_compile_params, 0, 0, 0, 0, SYMBOLIC_GRAPH_SOURCES(sdp_symbolic_graph), SYMBOLIC_GRAPH_DESTINATIONS(sdp_symbolic_graph), &sdp_graph, &sdp_tensor_arena, &sdp_graph_exec_arena);
3783
0
  ccv_nnc_tensor_t* const q_tensor = ccv_nnc_tensor_from_symbol(sdp_tensor_arena, q);
3784
0
  ccv_nnc_tensor_t* const k_tensor = ccv_nnc_tensor_from_symbol(sdp_tensor_arena, k);
3785
0
  ccv_nnc_tensor_t* const v_tensor = ccv_nnc_tensor_from_symbol(sdp_tensor_arena, v);
3786
0
  ccv_nnc_tensor_t* const w_tensor = ccv_nnc_tensor_from_symbol(sdp_tensor_arena, w);
3787
0
  ccv_nnc_tensor_t* const bias_tensor = ccv_nnc_tensor_from_symbol(sdp_tensor_arena, bias);
3788
0
  dsfmt_t dsfmt;
3789
0
  int i;
3790
0
  dsfmt_init_gen_rand(&dsfmt, 1);
3791
0
  for (i = 0; i < 32 * 8 * 128 * 64; i++)
3792
0
    q_tensor->data.f32[i] = dsfmt_genrand_open_close(&dsfmt);
3793
0
  for (i = 0; i < 32 * 8 * 128 * 64; i++)
3794
0
    k_tensor->data.f32[i] = dsfmt_genrand_open_close(&dsfmt);
3795
0
  for (i = 0; i < 32 * 8 * 128 * 64; i++)
3796
0
    v_tensor->data.f32[i] = dsfmt_genrand_open_close(&dsfmt);
3797
0
  for (i = 0; i < 512 * 512; i++)
3798
0
    w_tensor->data.f32[i] = dsfmt_genrand_open_close(&dsfmt) / sqrtf(512);
3799
0
  for (i = 0; i < 512; i++)
3800
0
    bias_tensor->data.f32[i] = dsfmt_genrand_open_close(&dsfmt);
3801
0
  ccv_nnc_tensor_t* const q_tensor_f16 = ccv_nnc_tensor_new(0, CPU_TENSOR_NHWC(16F, 32, 128, 8, 64), 0);
3802
0
  ccv_nnc_tensor_t* const k_tensor_f16 = ccv_nnc_tensor_new(0, CPU_TENSOR_NHWC(16F, 32, 128, 8, 64), 0);
3803
0
  ccv_nnc_tensor_t* const v_tensor_f16 = ccv_nnc_tensor_new(0, CPU_TENSOR_NHWC(16F, 32, 128, 8, 64), 0);
3804
0
  ccv_nnc_tensor_t* const w_tensor_f16 = ccv_nnc_tensor_new(0, CPU_TENSOR_NHWC(16F, 512, 512), 0);
3805
0
  ccv_nnc_tensor_t* const bias_tensor_f16 = ccv_nnc_tensor_new(0, CPU_TENSOR_NHWC(16F, 512), 0);
3806
0
  ccv_nnc_cmd_exec(CMD_DATATYPE_CONVERSION_FORWARD(), ccv_nnc_no_hint, 0, TENSOR_LIST(q_tensor, k_tensor, v_tensor, w_tensor, bias_tensor), TENSOR_LIST(q_tensor_f16, k_tensor_f16, v_tensor_f16, w_tensor_f16, bias_tensor_f16), 0);
3807
0
  ccv_nnc_symbolic_graph_t* const g_symbolic_graph = ccv_nnc_symbolic_graph_new();
3808
0
  ccv_nnc_tensor_symbol_t gq = ccv_nnc_tensor_symbol_new(g_symbolic_graph, GPU_TENSOR_NHWC(000, 16F, 32, 128, 8, 64), "q");
3809
0
  ccv_nnc_tensor_symbol_t gk = ccv_nnc_tensor_symbol_new(g_symbolic_graph, GPU_TENSOR_NHWC(000, 16F, 32, 128, 8, 64), "k");
3810
0
  ccv_nnc_tensor_symbol_t gv = ccv_nnc_tensor_symbol_new(g_symbolic_graph, GPU_TENSOR_NHWC(000, 16F, 32, 128, 8, 64), "v");
3811
0
  ccv_nnc_tensor_symbol_t gw = ccv_nnc_tensor_symbol_new(g_symbolic_graph, GPU_TENSOR_NHWC(000, 16F, 512, 512), "w");
3812
0
  ccv_nnc_tensor_symbol_t gbias = ccv_nnc_tensor_symbol_new(g_symbolic_graph, GPU_TENSOR_NHWC(000, 16F, 512), "bias");
3813
0
  ccv_nnc_tensor_symbol_t gc = ccv_nnc_tensor_symbol_new(g_symbolic_graph, GPU_TENSOR_NHWC(000, 16F, 32, 128, 8, 64), "c");
3814
0
  ccv_nnc_tensor_symbol_t gr = ccv_nnc_tensor_symbol_new(g_symbolic_graph, GPU_TENSOR_NHWC(000, 16F, 32, 128, 512), "r");
3815
0
  ccv_nnc_graph_exec_symbol_new(g_symbolic_graph, CMD_SCALED_DOT_PRODUCT_ATTENTION_FORWARD(1.0 / 8, 0), TENSOR_SYMBOL_LIST(gq, gk, gv, NO_TENSOR_SYMBOL, gw, gbias), TENSOR_SYMBOL_LIST(gr, NO_TENSOR_SYMBOL, gc), "scaled_dot_product_attention");
3816
0
  ccv_nnc_graph_exec_symbol_autogen(g_symbolic_graph, 0, 0, CCV_NNC_AUTOGEN_ALL_EXECS | CCV_NNC_AUTOGEN_SOURCES_AND_DESTINATIONS);
3817
0
  ccv_nnc_graph_t* g_graph = 0;
3818
0
  ccv_nnc_tensor_arena_t* g_tensor_arena = 0;
3819
0
  ccv_nnc_graph_exec_arena_t* g_graph_exec_arena = 0;
3820
0
  ccv_nnc_symbolic_graph_compile(g_symbolic_graph, ccv_nnc_default_compile_params, 0, 0, 0, 0, SYMBOLIC_GRAPH_SOURCES(g_symbolic_graph), SYMBOLIC_GRAPH_DESTINATIONS(g_symbolic_graph), &g_graph, &g_tensor_arena, &g_graph_exec_arena);
3821
0
  ccv_nnc_tensor_t* const gq_tensor = ccv_nnc_tensor_from_symbol(g_tensor_arena, gq);
3822
0
  ccv_nnc_tensor_t* const gk_tensor = ccv_nnc_tensor_from_symbol(g_tensor_arena, gk);
3823
0
  ccv_nnc_tensor_t* const gv_tensor = ccv_nnc_tensor_from_symbol(g_tensor_arena, gv);
3824
0
  ccv_nnc_tensor_t* const gw_tensor = ccv_nnc_tensor_from_symbol(g_tensor_arena, gw);
3825
0
  ccv_nnc_tensor_t* const gbias_tensor = ccv_nnc_tensor_from_symbol(g_tensor_arena, gbias);
3826
0
  ccv_nnc_cmd_exec(CMD_DATA_TRANSFER_FORWARD(), ccv_nnc_no_hint, 0, TENSOR_LIST(q_tensor_f16, k_tensor_f16, v_tensor_f16, w_tensor_f16, bias_tensor_f16), TENSOR_LIST(gq_tensor, gk_tensor, gv_tensor, gw_tensor, gbias_tensor), 0);
3827
0
  ccv_nnc_graph_run(sdp_graph, 0, TRAVERSE_FULL, 0, 0);
3828
0
  ccv_nnc_graph_run(g_graph, 0, TRAVERSE_FULL, 0, 0);
3829
0
  ccv_nnc_tensor_t* const r_tensor = ccv_nnc_tensor_from_symbol(sdp_tensor_arena, r);
3830
0
  ccv_nnc_tensor_t* const o_tensor = ccv_nnc_tensor_from_symbol(sdp_tensor_arena, c);
3831
0
  ccv_nnc_tensor_t* const gc_tensor = ccv_nnc_tensor_from_symbol(g_tensor_arena, gc);
3832
0
  ccv_nnc_tensor_t* const gr_tensor = ccv_nnc_tensor_from_symbol(g_tensor_arena, gr);
3833
0
  ccv_nnc_tensor_t* const ho_f16 = ccv_nnc_tensor_new(0, CPU_TENSOR_NHWC(16F, 32, 128, 8, 64), 0);
3834
0
  ccv_nnc_tensor_t* const hr_f16 = ccv_nnc_tensor_new(0, CPU_TENSOR_NHWC(16F, 32, 128, 512), 0);
3835
0
  ccv_nnc_cmd_exec(CMD_DATA_TRANSFER_FORWARD(), ccv_nnc_no_hint, 0, TENSOR_LIST(gc_tensor, gr_tensor), TENSOR_LIST(ho_f16, hr_f16), 0);
3836
0
  ccv_nnc_tensor_t* const ho = ccv_nnc_tensor_new(0, CPU_TENSOR_NHWC(32F, 32, 128, 8, 64), 0);
3837
0
  ccv_nnc_tensor_t* const hr = ccv_nnc_tensor_new(0, CPU_TENSOR_NHWC(32F, 32, 128, 512), 0);
3838
0
  ccv_nnc_cmd_exec(CMD_DATATYPE_CONVERSION_FORWARD(), ccv_nnc_no_hint, 0, TENSOR_LIST(ho_f16, hr_f16), TENSOR_LIST(ho, hr), 0);
3839
0
  REQUIRE_ARRAY_EQ_WITH_TOLERANCE(float, o_tensor->data.f32, ho->data.f32, 32 * 128 * 8 * 64, 3e-3, "graph computed result should match scaled dot product attention op result");
3840
0
  REQUIRE_ARRAY_EQ_WITH_TOLERANCE(float, r_tensor->data.f32, hr->data.f32, 32 * 128 * 512, 3e-2, "graph computed result should match scaled dot product attention op result");
3841
0
  ccv_nnc_symbolic_graph_free(sdp_symbolic_graph);
3842
0
  ccv_nnc_tensor_arena_free(sdp_tensor_arena);
3843
0
  ccv_nnc_graph_exec_arena_free(sdp_graph_exec_arena);
3844
0
  ccv_nnc_graph_free(sdp_graph);
3845
0
  ccv_nnc_symbolic_graph_free(g_symbolic_graph);
3846
0
  ccv_nnc_tensor_arena_free(g_tensor_arena);
3847
0
  ccv_nnc_graph_exec_arena_free(g_graph_exec_arena);
3848
0
  ccv_nnc_graph_free(g_graph);
3849
0
  ccv_nnc_tensor_free(ho);
3850
0
  ccv_nnc_tensor_free(hr);
3851
0
  ccv_nnc_tensor_free(ho_f16);
3852
0
  ccv_nnc_tensor_free(hr_f16);
3853
0
  ccv_nnc_tensor_free(q_tensor_f16);
3854
0
  ccv_nnc_tensor_free(k_tensor_f16);
3855
0
  ccv_nnc_tensor_free(v_tensor_f16);
3856
0
  ccv_nnc_tensor_free(w_tensor_f16);
3857
0
  ccv_nnc_tensor_free(bias_tensor_f16);
3858
0
}
3859
3860
TEST_CASE("scaled dot product attention + unify head with flash_attn in bfloat")
3861
1
{
3862
1
  GUARD_ELSE_RETURN(ccv_nnc_cmd_ok(CCV_NNC_SCALED_DOT_PRODUCT_ATTENTION_FORWARD, CCV_NNC_BACKEND_GPU_REF));
3863
0
  ccv_nnc_symbolic_graph_t* const sdp_symbolic_graph = ccv_nnc_symbolic_graph_new();
3864
0
  ccv_nnc_tensor_symbol_t q = ccv_nnc_tensor_symbol_new(sdp_symbolic_graph, CPU_TENSOR_NHWC(32F, 32, 128, 8, 64), "q");
3865
0
  ccv_nnc_tensor_symbol_t k = ccv_nnc_tensor_symbol_new(sdp_symbolic_graph, CPU_TENSOR_NHWC(32F, 32, 128, 8, 64), "k");
3866
0
  ccv_nnc_tensor_symbol_t v = ccv_nnc_tensor_symbol_new(sdp_symbolic_graph, CPU_TENSOR_NHWC(32F, 32, 128, 8, 64), "v");
3867
0
  ccv_nnc_tensor_symbol_t w = ccv_nnc_tensor_symbol_new(sdp_symbolic_graph, CPU_TENSOR_NHWC(32F, 512, 512), "w");
3868
0
  ccv_nnc_tensor_symbol_t bias = ccv_nnc_tensor_symbol_new(sdp_symbolic_graph, CPU_TENSOR_NHWC(32F, 512), "bias");
3869
0
  ccv_nnc_tensor_symbol_t c = ccv_nnc_tensor_symbol_new(sdp_symbolic_graph, CPU_TENSOR_NHWC(32F, 32, 128, 8, 64), "c");
3870
0
  ccv_nnc_tensor_symbol_t r = ccv_nnc_tensor_symbol_new(sdp_symbolic_graph, CPU_TENSOR_NHWC(32F, 32, 128, 512), "r");
3871
0
  ccv_nnc_graph_exec_symbol_new(sdp_symbolic_graph, CMD_SCALED_DOT_PRODUCT_ATTENTION_FORWARD(1.0 / 8, 0), TENSOR_SYMBOL_LIST(q, k, v, NO_TENSOR_SYMBOL, w, bias), TENSOR_SYMBOL_LIST(r, NO_TENSOR_SYMBOL, c), "scaled_dot_product_attention");
3872
0
  ccv_nnc_graph_exec_symbol_autogen(sdp_symbolic_graph, 0, 0, CCV_NNC_AUTOGEN_ALL_EXECS | CCV_NNC_AUTOGEN_SOURCES_AND_DESTINATIONS);
3873
0
  ccv_nnc_graph_t* sdp_graph = 0;
3874
0
  ccv_nnc_tensor_arena_t* sdp_tensor_arena = 0;
3875
0
  ccv_nnc_graph_exec_arena_t* sdp_graph_exec_arena = 0;
3876
0
  ccv_nnc_symbolic_graph_compile(sdp_symbolic_graph, ccv_nnc_default_compile_params, 0, 0, 0, 0, SYMBOLIC_GRAPH_SOURCES(sdp_symbolic_graph), SYMBOLIC_GRAPH_DESTINATIONS(sdp_symbolic_graph), &sdp_graph, &sdp_tensor_arena, &sdp_graph_exec_arena);
3877
0
  ccv_nnc_tensor_t* const q_tensor = ccv_nnc_tensor_from_symbol(sdp_tensor_arena, q);
3878
0
  ccv_nnc_tensor_t* const k_tensor = ccv_nnc_tensor_from_symbol(sdp_tensor_arena, k);
3879
0
  ccv_nnc_tensor_t* const v_tensor = ccv_nnc_tensor_from_symbol(sdp_tensor_arena, v);
3880
0
  ccv_nnc_tensor_t* const w_tensor = ccv_nnc_tensor_from_symbol(sdp_tensor_arena, w);
3881
0
  ccv_nnc_tensor_t* const bias_tensor = ccv_nnc_tensor_from_symbol(sdp_tensor_arena, bias);
3882
0
  dsfmt_t dsfmt;
3883
0
  int i;
3884
0
  dsfmt_init_gen_rand(&dsfmt, 1);
3885
0
  for (i = 0; i < 32 * 8 * 128 * 64; i++)
3886
0
    q_tensor->data.f32[i] = dsfmt_genrand_open_close(&dsfmt);
3887
0
  for (i = 0; i < 32 * 8 * 128 * 64; i++)
3888
0
    k_tensor->data.f32[i] = dsfmt_genrand_open_close(&dsfmt);
3889
0
  for (i = 0; i < 32 * 8 * 128 * 64; i++)
3890
0
    v_tensor->data.f32[i] = dsfmt_genrand_open_close(&dsfmt);
3891
0
  for (i = 0; i < 512 * 512; i++)
3892
0
    w_tensor->data.f32[i] = dsfmt_genrand_open_close(&dsfmt) / sqrtf(512);
3893
0
  for (i = 0; i < 512; i++)
3894
0
    bias_tensor->data.f32[i] = dsfmt_genrand_open_close(&dsfmt);
3895
0
  ccv_nnc_tensor_t* const q_tensor_f16 = ccv_nnc_tensor_new(0, CPU_TENSOR_NHWC(16BF, 32, 128, 8, 64), 0);
3896
0
  ccv_nnc_tensor_t* const k_tensor_f16 = ccv_nnc_tensor_new(0, CPU_TENSOR_NHWC(16BF, 32, 128, 8, 64), 0);
3897
0
  ccv_nnc_tensor_t* const v_tensor_f16 = ccv_nnc_tensor_new(0, CPU_TENSOR_NHWC(16BF, 32, 128, 8, 64), 0);
3898
0
  ccv_nnc_tensor_t* const w_tensor_f16 = ccv_nnc_tensor_new(0, CPU_TENSOR_NHWC(16BF, 512, 512), 0);
3899
0
  ccv_nnc_tensor_t* const bias_tensor_f16 = ccv_nnc_tensor_new(0, CPU_TENSOR_NHWC(16BF, 512), 0);
3900
0
  ccv_nnc_cmd_exec(CMD_DATATYPE_CONVERSION_FORWARD(), ccv_nnc_no_hint, 0, TENSOR_LIST(q_tensor, k_tensor, v_tensor, w_tensor, bias_tensor), TENSOR_LIST(q_tensor_f16, k_tensor_f16, v_tensor_f16, w_tensor_f16, bias_tensor_f16), 0);
3901
0
  ccv_nnc_symbolic_graph_t* const g_symbolic_graph = ccv_nnc_symbolic_graph_new();
3902
0
  ccv_nnc_tensor_symbol_t gq = ccv_nnc_tensor_symbol_new(g_symbolic_graph, GPU_TENSOR_NHWC(000, 16BF, 32, 128, 8, 64), "q");
3903
0
  ccv_nnc_tensor_symbol_t gk = ccv_nnc_tensor_symbol_new(g_symbolic_graph, GPU_TENSOR_NHWC(000, 16BF, 32, 128, 8, 64), "k");
3904
0
  ccv_nnc_tensor_symbol_t gv = ccv_nnc_tensor_symbol_new(g_symbolic_graph, GPU_TENSOR_NHWC(000, 16BF, 32, 128, 8, 64), "v");
3905
0
  ccv_nnc_tensor_symbol_t gw = ccv_nnc_tensor_symbol_new(g_symbolic_graph, GPU_TENSOR_NHWC(000, 16BF, 512, 512), "w");
3906
0
  ccv_nnc_tensor_symbol_t gbias = ccv_nnc_tensor_symbol_new(g_symbolic_graph, GPU_TENSOR_NHWC(000, 16BF, 512), "bias");
3907
0
  ccv_nnc_tensor_symbol_t gc = ccv_nnc_tensor_symbol_new(g_symbolic_graph, GPU_TENSOR_NHWC(000, 16BF, 32, 128, 8, 64), "c");
3908
0
  ccv_nnc_tensor_symbol_t gr = ccv_nnc_tensor_symbol_new(g_symbolic_graph, GPU_TENSOR_NHWC(000, 16BF, 32, 128, 512), "r");
3909
0
  ccv_nnc_graph_exec_symbol_new(g_symbolic_graph, CMD_SCALED_DOT_PRODUCT_ATTENTION_FORWARD(1.0 / 8, 0), TENSOR_SYMBOL_LIST(gq, gk, gv, NO_TENSOR_SYMBOL, gw, gbias), TENSOR_SYMBOL_LIST(gr, NO_TENSOR_SYMBOL, gc), "scaled_dot_product_attention");
3910
0
  ccv_nnc_graph_exec_symbol_autogen(g_symbolic_graph, 0, 0, CCV_NNC_AUTOGEN_ALL_EXECS | CCV_NNC_AUTOGEN_SOURCES_AND_DESTINATIONS);
3911
0
  ccv_nnc_graph_t* g_graph = 0;
3912
0
  ccv_nnc_tensor_arena_t* g_tensor_arena = 0;
3913
0
  ccv_nnc_graph_exec_arena_t* g_graph_exec_arena = 0;
3914
0
  ccv_nnc_symbolic_graph_compile(g_symbolic_graph, ccv_nnc_default_compile_params, 0, 0, 0, 0, SYMBOLIC_GRAPH_SOURCES(g_symbolic_graph), SYMBOLIC_GRAPH_DESTINATIONS(g_symbolic_graph), &g_graph, &g_tensor_arena, &g_graph_exec_arena);
3915
0
  ccv_nnc_tensor_t* const gq_tensor = ccv_nnc_tensor_from_symbol(g_tensor_arena, gq);
3916
0
  ccv_nnc_tensor_t* const gk_tensor = ccv_nnc_tensor_from_symbol(g_tensor_arena, gk);
3917
0
  ccv_nnc_tensor_t* const gv_tensor = ccv_nnc_tensor_from_symbol(g_tensor_arena, gv);
3918
0
  ccv_nnc_tensor_t* const gw_tensor = ccv_nnc_tensor_from_symbol(g_tensor_arena, gw);
3919
0
  ccv_nnc_tensor_t* const gbias_tensor = ccv_nnc_tensor_from_symbol(g_tensor_arena, gbias);
3920
0
  ccv_nnc_cmd_exec(CMD_DATA_TRANSFER_FORWARD(), ccv_nnc_no_hint, 0, TENSOR_LIST(q_tensor_f16, k_tensor_f16, v_tensor_f16, w_tensor_f16, bias_tensor_f16), TENSOR_LIST(gq_tensor, gk_tensor, gv_tensor, gw_tensor, gbias_tensor), 0);
3921
0
  ccv_nnc_graph_run(sdp_graph, 0, TRAVERSE_FULL, 0, 0);
3922
0
  ccv_nnc_graph_run(g_graph, 0, TRAVERSE_FULL, 0, 0);
3923
0
  ccv_nnc_tensor_t* const r_tensor = ccv_nnc_tensor_from_symbol(sdp_tensor_arena, r);
3924
0
  ccv_nnc_tensor_t* const o_tensor = ccv_nnc_tensor_from_symbol(sdp_tensor_arena, c);
3925
0
  ccv_nnc_tensor_t* const gc_tensor = ccv_nnc_tensor_from_symbol(g_tensor_arena, gc);
3926
0
  ccv_nnc_tensor_t* const gr_tensor = ccv_nnc_tensor_from_symbol(g_tensor_arena, gr);
3927
0
  ccv_nnc_tensor_t* const ho_f16 = ccv_nnc_tensor_new(0, CPU_TENSOR_NHWC(16BF, 32, 128, 8, 64), 0);
3928
0
  ccv_nnc_tensor_t* const hr_f16 = ccv_nnc_tensor_new(0, CPU_TENSOR_NHWC(16BF, 32, 128, 512), 0);
3929
0
  ccv_nnc_cmd_exec(CMD_DATA_TRANSFER_FORWARD(), ccv_nnc_no_hint, 0, TENSOR_LIST(gc_tensor, gr_tensor), TENSOR_LIST(ho_f16, hr_f16), 0);
3930
0
  ccv_nnc_tensor_t* const ho = ccv_nnc_tensor_new(0, CPU_TENSOR_NHWC(32F, 32, 128, 8, 64), 0);
3931
0
  ccv_nnc_tensor_t* const hr = ccv_nnc_tensor_new(0, CPU_TENSOR_NHWC(32F, 32, 128, 512), 0);
3932
0
  ccv_nnc_cmd_exec(CMD_DATATYPE_CONVERSION_FORWARD(), ccv_nnc_no_hint, 0, TENSOR_LIST(ho_f16, hr_f16), TENSOR_LIST(ho, hr), 0);
3933
0
  REQUIRE_ARRAY_EQ_WITH_TOLERANCE(float, o_tensor->data.f32, ho->data.f32, 32 * 128 * 8 * 64, 1e-2, "graph computed result should match scaled dot product attention op result");
3934
0
  REQUIRE_ARRAY_EQ_WITH_TOLERANCE(float, r_tensor->data.f32, hr->data.f32, 32 * 128 * 512, 1e-1, "graph computed result should match scaled dot product attention op result");
3935
0
  ccv_nnc_symbolic_graph_free(sdp_symbolic_graph);
3936
0
  ccv_nnc_tensor_arena_free(sdp_tensor_arena);
3937
0
  ccv_nnc_graph_exec_arena_free(sdp_graph_exec_arena);
3938
0
  ccv_nnc_graph_free(sdp_graph);
3939
0
  ccv_nnc_symbolic_graph_free(g_symbolic_graph);
3940
0
  ccv_nnc_tensor_arena_free(g_tensor_arena);
3941
0
  ccv_nnc_graph_exec_arena_free(g_graph_exec_arena);
3942
0
  ccv_nnc_graph_free(g_graph);
3943
0
  ccv_nnc_tensor_free(ho);
3944
0
  ccv_nnc_tensor_free(hr);
3945
0
  ccv_nnc_tensor_free(ho_f16);
3946
0
  ccv_nnc_tensor_free(hr_f16);
3947
0
  ccv_nnc_tensor_free(q_tensor_f16);
3948
0
  ccv_nnc_tensor_free(k_tensor_f16);
3949
0
  ccv_nnc_tensor_free(v_tensor_f16);
3950
0
  ccv_nnc_tensor_free(w_tensor_f16);
3951
0
  ccv_nnc_tensor_free(bias_tensor_f16);
3952
0
}
3953
3954
TEST_CASE("scaled dot product attention gradient with flash_attn")
3955
1
{
3956
1
  GUARD_ELSE_RETURN(ccv_nnc_cmd_ok(CCV_NNC_SCALED_DOT_PRODUCT_ATTENTION_FORWARD, CCV_NNC_BACKEND_GPU_REF) &&
3957
1
    ccv_nnc_cmd_ok(CCV_NNC_SCALED_DOT_PRODUCT_ATTENTION_BACKWARD, CCV_NNC_BACKEND_GPU_REF));
3958
0
#define num_long_trials 8
3959
0
#define num_short_trials 4
3960
0
#define num_trials (num_long_trials + num_short_trials)
3961
3962
0
  dsfmt_t dsfmt;
3963
0
  dsfmt_init_gen_rand(&dsfmt, 10);
3964
0
  for (int trial = 0; trial < num_trials; ++trial) {
3965
0
    const int B_candidates[num_trials] = {  32,   12, 16, 1, 2, 1, 32,   12, 16, 1, 2, 1 };
3966
0
    const int R_candidates[num_trials] = { 160,  256, 128, 77, 77, 5, 160,  256, 128, 77, 77, 5 };
3967
0
    const int C_candidates[num_trials] = { 128,  128, 128, 128, 128, 5, 128,  128, 128, 128, 128, 5 };
3968
0
    const int Hq_candidates[num_trials] = {   8,  8, 8, 8, 8, 32, 8,  8, 8, 8, 8, 32 };
3969
0
    const int Hk_candidates[num_trials] = {   8,  8, 8, 8, 2, 8, 8,  8, 8, 8, 2, 8 };
3970
0
    const int D_candidates[num_trials] = {  64, 40, 160, 192, 256, 128, 64, 40, 160, 192, 256, 128 };
3971
0
    const int is_causal_candidates[num_trials] = {  1, 0, 1, 1, 0, 1, 1, 0, 1, 1, 0, 1 };
3972
0
    const int deterministic_candidates[num_trials] = {  0, 0, 0, 0, 0, 0, 1, 1, 1, 1, 1, 1 };
3973
3974
0
    const int B = B_candidates[trial];
3975
0
    const int R = R_candidates[trial];
3976
0
    const int C = C_candidates[trial];
3977
0
    const int Hq = Hq_candidates[trial];
3978
0
    const int Hk = Hk_candidates[trial];
3979
0
    const int D = D_candidates[trial];
3980
0
    const int is_causal = is_causal_candidates[trial];
3981
0
    const int deterministic = deterministic_candidates[trial];
3982
0
    const float scale = 1.0 / sqrt((float)D);
3983
3984
0
    ccv_nnc_tensor_t* const q_tensor = ccv_nnc_tensor_new(0, CPU_TENSOR_NHWC(32F, B, R, Hq, D), 0);
3985
0
    ccv_nnc_tensor_t* const k_tensor = ccv_nnc_tensor_new(0, CPU_TENSOR_NHWC(32F, B, C, Hk, D), 0);
3986
0
    ccv_nnc_tensor_t* const v_tensor = ccv_nnc_tensor_new(0, CPU_TENSOR_NHWC(32F, B, C, Hk, D), 0);
3987
0
    ccv_nnc_tensor_t* const dq_tensor = ccv_nnc_tensor_new(0, CPU_TENSOR_NHWC(32F, B, R, Hq, D), 0);
3988
0
    ccv_nnc_tensor_t* const dk_tensor = ccv_nnc_tensor_new(0, CPU_TENSOR_NHWC(32F, B, C, Hk, D), 0);
3989
0
    ccv_nnc_tensor_t* const dv_tensor = ccv_nnc_tensor_new(0, CPU_TENSOR_NHWC(32F, B, C, Hk, D), 0);
3990
3991
0
    for (int i = 0; i < B * R * Hq * D; ++i) {
3992
0
      q_tensor->data.f32[i] = dsfmt_genrand_open_close(&dsfmt);
3993
0
    }
3994
0
    for (int i = 0; i < B * C * Hk * D; ++i) {
3995
0
      k_tensor->data.f32[i] = dsfmt_genrand_open_close(&dsfmt);
3996
0
    }
3997
0
    for (int i = 0; i < B * C * Hk * D; ++i) {
3998
0
      v_tensor->data.f32[i] = dsfmt_genrand_open_close(&dsfmt);
3999
0
    }
4000
4001
0
    ccv_nnc_tensor_t* const do_tensor = ccv_nnc_tensor_new(0, CPU_TENSOR_NHWC(32F, B, R, Hq, D), 0);
4002
0
    for (int i = 0; i < B * R * Hq * D; ++i) {
4003
0
      do_tensor->data.f32[i] = dsfmt_genrand_open_close(&dsfmt);
4004
0
    }
4005
0
    ccv_nnc_cmd_exec(CMD_SCALED_DOT_PRODUCT_ATTENTION_BACKWARD(scale, is_causal), ccv_nnc_no_hint, 0, TENSOR_LIST(do_tensor, 0, 0, q_tensor, k_tensor, v_tensor), TENSOR_LIST(dq_tensor, dk_tensor, dv_tensor), 0);
4006
0
    ccv_nnc_tensor_t* const q_tensor_f16 = ccv_nnc_tensor_new(0, CPU_TENSOR_NHWC(16F, B, R, Hq, D), 0);
4007
0
    ccv_nnc_tensor_t* const k_tensor_f16 = ccv_nnc_tensor_new(0, CPU_TENSOR_NHWC(16F, B, C, Hk, D), 0);
4008
0
    ccv_nnc_tensor_t* const v_tensor_f16 = ccv_nnc_tensor_new(0, CPU_TENSOR_NHWC(16F, B, C, Hk, D), 0);
4009
0
    ccv_nnc_tensor_t* const do_tensor_f16 = ccv_nnc_tensor_new(0, CPU_TENSOR_NHWC(16F, B, R, Hq, D), 0);
4010
0
    ccv_nnc_cmd_exec(CMD_DATATYPE_CONVERSION_FORWARD(), ccv_nnc_no_hint, 0, TENSOR_LIST(q_tensor, k_tensor, v_tensor, do_tensor), TENSOR_LIST(q_tensor_f16, k_tensor_f16, v_tensor_f16, do_tensor_f16), 0);
4011
4012
0
    ccv_nnc_tensor_t* const gpu_q_tensor = ccv_nnc_tensor_new(0, GPU_TENSOR_NHWC(000, 16F, B, R, Hq, D), 0);
4013
0
    ccv_nnc_tensor_t* const gpu_k_tensor = ccv_nnc_tensor_new(0, GPU_TENSOR_NHWC(000, 16F, B, C, Hk, D), 0);
4014
0
    ccv_nnc_tensor_t* const gpu_v_tensor = ccv_nnc_tensor_new(0, GPU_TENSOR_NHWC(000, 16F, B, C, Hk, D), 0);
4015
0
    ccv_nnc_tensor_t* const gpu_o_tensor = ccv_nnc_tensor_new(0, GPU_TENSOR_NHWC(000, 16F, B, R, Hq, D), 0);
4016
0
    ccv_nnc_tensor_t* const gpu_do_tensor = ccv_nnc_tensor_new(0, GPU_TENSOR_NHWC(000, 16F, B, R, Hq, D), 0);
4017
0
    ccv_nnc_tensor_t* const gpu_dq_tensor = ccv_nnc_tensor_new(0, GPU_TENSOR_NHWC(000, 16F, B, R, Hq, D), 0);
4018
0
    ccv_nnc_tensor_t* const gpu_dk_tensor = ccv_nnc_tensor_new(0, GPU_TENSOR_NHWC(000, 16F, B, C, Hk, D), 0);
4019
0
    ccv_nnc_tensor_t* const gpu_dv_tensor = ccv_nnc_tensor_new(0, GPU_TENSOR_NHWC(000, 16F, B, C, Hk, D), 0);
4020
0
    ccv_nnc_cmd_exec(CMD_DATA_TRANSFER_FORWARD(), ccv_nnc_no_hint, 0, TENSOR_LIST(q_tensor_f16, k_tensor_f16, v_tensor_f16, do_tensor_f16), TENSOR_LIST(gpu_q_tensor, gpu_k_tensor, gpu_v_tensor, gpu_do_tensor), 0);
4021
4022
0
    ccv_nnc_tensor_t* const gpu_softmax_lse = ccv_nnc_tensor_new(0, GPU_TENSOR_NHWC(000, 32F, B, Hq, R), 0);
4023
0
    ccv_nnc_cmd_exec(CMD_SCALED_DOT_PRODUCT_ATTENTION_FORWARD(scale, is_causal), ccv_nnc_no_hint, 0, TENSOR_LIST(gpu_q_tensor, gpu_k_tensor, gpu_v_tensor, NULL, NULL, NULL), TENSOR_LIST(gpu_o_tensor, gpu_softmax_lse), 0);
4024
4025
0
    ccv_nnc_cmd_t cmd = CMD_SCALED_DOT_PRODUCT_ATTENTION_BACKWARD(scale, is_causal);
4026
0
    cmd.info.scaled_dot_product_attention.deterministic = deterministic;
4027
0
    ccv_nnc_cmd_exec(cmd, ccv_nnc_no_hint, 0, TENSOR_LIST(gpu_do_tensor, 0, 0, gpu_q_tensor, gpu_k_tensor, gpu_v_tensor, 0, 0, 0, gpu_o_tensor, gpu_softmax_lse), TENSOR_LIST(gpu_dq_tensor, gpu_dk_tensor, gpu_dv_tensor), 0);
4028
4029
0
    ccv_nnc_tensor_t* const copy_of_gpu_dq_tensor_f16 = ccv_nnc_tensor_new(0, CPU_TENSOR_NHWC(16F, B, R, Hq, D), 0);
4030
0
    ccv_nnc_tensor_t* const copy_of_gpu_dk_tensor_f16 = ccv_nnc_tensor_new(0, CPU_TENSOR_NHWC(16F, B, C, Hk, D), 0);
4031
0
    ccv_nnc_tensor_t* const copy_of_gpu_dv_tensor_f16 = ccv_nnc_tensor_new(0, CPU_TENSOR_NHWC(16F, B, C, Hk, D), 0);
4032
0
    ccv_nnc_cmd_exec(CMD_DATA_TRANSFER_FORWARD(), ccv_nnc_no_hint, 0, TENSOR_LIST(gpu_dq_tensor, gpu_dk_tensor, gpu_dv_tensor), TENSOR_LIST(copy_of_gpu_dq_tensor_f16, copy_of_gpu_dk_tensor_f16, copy_of_gpu_dv_tensor_f16), 0);
4033
4034
0
    ccv_nnc_tensor_t* const copy_of_gpu_dq_tensor = ccv_nnc_tensor_new(0, CPU_TENSOR_NHWC(32F, B, R, Hq, D), 0);
4035
0
    ccv_nnc_tensor_t* const copy_of_gpu_dk_tensor = ccv_nnc_tensor_new(0, CPU_TENSOR_NHWC(32F, B, C, Hk, D), 0);
4036
0
    ccv_nnc_tensor_t* const copy_of_gpu_dv_tensor = ccv_nnc_tensor_new(0, CPU_TENSOR_NHWC(32F, B, C, Hk, D), 0);
4037
0
    ccv_nnc_cmd_exec(CMD_DATATYPE_CONVERSION_FORWARD(), ccv_nnc_no_hint, 0, TENSOR_LIST(copy_of_gpu_dq_tensor_f16, copy_of_gpu_dk_tensor_f16, copy_of_gpu_dv_tensor_f16), TENSOR_LIST(copy_of_gpu_dq_tensor, copy_of_gpu_dk_tensor, copy_of_gpu_dv_tensor), 0);
4038
4039
0
    REQUIRE_ARRAY_EQ_WITH_TOLERANCE(float, copy_of_gpu_dq_tensor->data.f32, dq_tensor->data.f32, B * R * Hq * D, 1e-3, "scaled dot product attention result should be the same");
4040
0
    REQUIRE_ARRAY_EQ_WITH_TOLERANCE(float, copy_of_gpu_dk_tensor->data.f32, dk_tensor->data.f32, B * C * Hk * D, 3e-3, "scaled dot product attention result should be the same");
4041
0
    REQUIRE_ARRAY_EQ_WITH_TOLERANCE(float, copy_of_gpu_dv_tensor->data.f32, dv_tensor->data.f32, B * C * Hk * D, 6e-3, "GPU computed output should be the same as CPU computed ones");
4042
4043
0
    ccv_nnc_tensor_free(do_tensor);
4044
0
    ccv_nnc_tensor_free(gpu_do_tensor);
4045
0
    ccv_nnc_tensor_free(gpu_o_tensor);
4046
0
    ccv_nnc_tensor_free(copy_of_gpu_dq_tensor_f16);
4047
0
    ccv_nnc_tensor_free(copy_of_gpu_dk_tensor_f16);
4048
0
    ccv_nnc_tensor_free(copy_of_gpu_dv_tensor_f16);
4049
0
    ccv_nnc_tensor_free(copy_of_gpu_dq_tensor);
4050
0
    ccv_nnc_tensor_free(copy_of_gpu_dk_tensor);
4051
0
    ccv_nnc_tensor_free(copy_of_gpu_dv_tensor);
4052
0
    ccv_nnc_tensor_free(q_tensor);
4053
0
    ccv_nnc_tensor_free(k_tensor);
4054
0
    ccv_nnc_tensor_free(v_tensor);
4055
0
    ccv_nnc_tensor_free(q_tensor_f16);
4056
0
    ccv_nnc_tensor_free(k_tensor_f16);
4057
0
    ccv_nnc_tensor_free(v_tensor_f16);
4058
0
    ccv_nnc_tensor_free(do_tensor_f16);
4059
0
    ccv_nnc_tensor_free(gpu_q_tensor);
4060
0
    ccv_nnc_tensor_free(gpu_k_tensor);
4061
0
    ccv_nnc_tensor_free(gpu_v_tensor);
4062
0
    ccv_nnc_tensor_free(dq_tensor);
4063
0
    ccv_nnc_tensor_free(dk_tensor);
4064
0
    ccv_nnc_tensor_free(dv_tensor);
4065
0
    ccv_nnc_tensor_free(gpu_dq_tensor);
4066
0
    ccv_nnc_tensor_free(gpu_dk_tensor);
4067
0
    ccv_nnc_tensor_free(gpu_dv_tensor);
4068
0
    ccv_nnc_tensor_free(gpu_softmax_lse);
4069
0
  }
4070
0
#undef num_long_trials
4071
0
#undef num_short_trials
4072
0
#undef num_trials
4073
0
}
4074
4075
static ccv_nnc_tensor_param_t _ccv_nnc_cmul_mixed_params(const int memory, const int datatype)
4076
38
{
4077
38
  return (ccv_nnc_tensor_param_t){
4078
38
    .type = memory,
4079
38
    .format = CCV_TENSOR_FORMAT_NCHW,
4080
38
    .datatype = datatype,
4081
38
    .dim = { 2, 3, 10, 0 },
4082
38
  };
4083
38
}
4084
4085
static int _ccv_nnc_cmul_round_to_datatype(const ccv_nnc_tensor_t* const source, ccv_nnc_tensor_t* const typed, ccv_nnc_tensor_t* const rounded, const int count)
4086
14
{
4087
14
  int i;
4088
14
  switch (typed->info.datatype)
4089
14
  {
4090
1
    case CCV_32F:
4091
61
      for (i = 0; i < count; 
i++60
)
4092
60
      {
4093
60
        typed->data.f32[i] = source->data.f32[i];
4094
60
        rounded->data.f32[i] = source->data.f32[i];
4095
60
      }
4096
1
      return CCV_NNC_EXEC_SUCCESS;
4097
7
    case CCV_16F: {
4098
7
      int status = ccv_nnc_cmd_exec(CMD_DATATYPE_CONVERSION_FORWARD(), ccv_nnc_no_hint, 0, TENSOR_LIST((ccv_nnc_tensor_t*)source), TENSOR_LIST(typed), 0);
4099
7
      if (status != CCV_NNC_EXEC_SUCCESS)
4100
0
        return status;
4101
7
      return ccv_nnc_cmd_exec(CMD_DATATYPE_CONVERSION_FORWARD(), ccv_nnc_no_hint, 0, TENSOR_LIST(typed), TENSOR_LIST(rounded), 0);
4102
7
    }
4103
6
    case CCV_16BF:
4104
6
      ccv_float_to_bfloat(source->data.f32, (uint16_t*)typed->data.f16, count);
4105
6
      ccv_bfloat_to_float((uint16_t*)typed->data.f16, rounded->data.f32, count);
4106
6
      return CCV_NNC_EXEC_SUCCESS;
4107
14
  }
4108
0
  return CCV_NNC_EXEC_INVALID;
4109
14
}
4110
4111
static int _ccv_nnc_cmul_convert_to_float(const ccv_nnc_tensor_t* const typed, ccv_nnc_tensor_t* const rounded, const int count)
4112
5
{
4113
5
  int i;
4114
5
  switch (typed->info.datatype)
4115
5
  {
4116
1
    case CCV_32F:
4117
61
      for (i = 0; i < count; 
i++60
)
4118
60
        rounded->data.f32[i] = typed->data.f32[i];
4119
1
      return CCV_NNC_EXEC_SUCCESS;
4120
2
    case CCV_16F:
4121
2
      return ccv_nnc_cmd_exec(CMD_DATATYPE_CONVERSION_FORWARD(), ccv_nnc_no_hint, 0, TENSOR_LIST((ccv_nnc_tensor_t*)typed), TENSOR_LIST(rounded), 0);
4122
2
    case CCV_16BF:
4123
2
      ccv_bfloat_to_float((uint16_t*)typed->data.f16, rounded->data.f32, count);
4124
2
      return CCV_NNC_EXEC_SUCCESS;
4125
5
  }
4126
0
  return CCV_NNC_EXEC_INVALID;
4127
5
}
4128
4129
static int _ccv_nnc_cmul_mixed_precision_case(const int backend, const int a_datatype, const int b_datatype, const int c_datatype, float* const max_diff)
4130
3
{
4131
3
  const int count = 2 * 3 * 10;
4132
3
  ccv_nnc_tensor_param_t ha32_params = _ccv_nnc_cmul_mixed_params(CCV_TENSOR_CPU_MEMORY, CCV_32F);
4133
3
  ccv_nnc_tensor_param_t hb32_params = _ccv_nnc_cmul_mixed_params(CCV_TENSOR_CPU_MEMORY, CCV_32F);
4134
3
  ccv_nnc_tensor_param_t hc32_params = _ccv_nnc_cmul_mixed_params(CCV_TENSOR_CPU_MEMORY, CCV_32F);
4135
3
  ccv_nnc_tensor_param_t ha_params = _ccv_nnc_cmul_mixed_params(CCV_TENSOR_CPU_MEMORY, a_datatype);
4136
3
  ccv_nnc_tensor_param_t hb_params = _ccv_nnc_cmul_mixed_params(CCV_TENSOR_CPU_MEMORY, b_datatype);
4137
3
  ccv_nnc_tensor_param_t hc_params = _ccv_nnc_cmul_mixed_params(CCV_TENSOR_CPU_MEMORY, c_datatype);
4138
3
  ccv_nnc_tensor_param_t ga_params = _ccv_nnc_cmul_mixed_params(CCV_TENSOR_GPU_MEMORY | 000, a_datatype);
4139
3
  ccv_nnc_tensor_param_t gb_params = _ccv_nnc_cmul_mixed_params(CCV_TENSOR_GPU_MEMORY | 000, b_datatype);
4140
3
  ccv_nnc_tensor_param_t gc_params = _ccv_nnc_cmul_mixed_params(CCV_TENSOR_GPU_MEMORY | 000, c_datatype);
4141
3
  ccv_nnc_tensor_t* const ha32 = ccv_nnc_tensor_new(0, ha32_params, 0);
4142
3
  ccv_nnc_tensor_t* const hb32 = ccv_nnc_tensor_new(0, hb32_params, 0);
4143
3
  ccv_nnc_tensor_t* const ha = ccv_nnc_tensor_new(0, ha_params, 0);
4144
3
  ccv_nnc_tensor_t* const hb = ccv_nnc_tensor_new(0, hb_params, 0);
4145
3
  ccv_nnc_tensor_t* const hc = ccv_nnc_tensor_new(0, hc_params, 0);
4146
3
  ccv_nnc_tensor_t* const ha_rounded = ccv_nnc_tensor_new(0, ha32_params, 0);
4147
3
  ccv_nnc_tensor_t* const hb_rounded = ccv_nnc_tensor_new(0, hb32_params, 0);
4148
3
  ccv_nnc_tensor_t* const expected = ccv_nnc_tensor_new(0, hc32_params, 0);
4149
3
  ccv_nnc_tensor_t* const expected_rounded = ccv_nnc_tensor_new(0, hc32_params, 0);
4150
3
  ccv_nnc_tensor_t* const actual = ccv_nnc_tensor_new(0, hc32_params, 0);
4151
3
  ccv_nnc_tensor_t* const expected_typed = ccv_nnc_tensor_new(0, hc_params, 0);
4152
3
  ccv_nnc_tensor_t* const ga = ccv_nnc_tensor_new(0, ga_params, 0);
4153
3
  ccv_nnc_tensor_t* const gb = ccv_nnc_tensor_new(0, gb_params, 0);
4154
3
  ccv_nnc_tensor_t* const gc = ccv_nnc_tensor_new(0, gc_params, 0);
4155
3
  int i;
4156
183
  for (i = 0; i < count; 
i++180
)
4157
180
  {
4158
180
    ha32->data.f32[i] = (float)((i % 13) - 6) * 0.07f;
4159
180
    hb32->data.f32[i] = (float)(((i * 5 + 3) % 17) - 8) * 0.05f;
4160
180
  }
4161
3
  int status = _ccv_nnc_cmul_round_to_datatype(ha32, ha, ha_rounded, count);
4162
3
  if (status == CCV_NNC_EXEC_SUCCESS)
4163
3
    status = _ccv_nnc_cmul_round_to_datatype(hb32, hb, hb_rounded, count);
4164
3
  if (status == CCV_NNC_EXEC_SUCCESS)
4165
3
    status = ccv_nnc_cmd_exec(CMD_DATA_TRANSFER_FORWARD(), ccv_nnc_no_hint, 0, TENSOR_LIST(ha, hb), TENSOR_LIST(ga, gb), 0);
4166
3
  if (status == CCV_NNC_EXEC_SUCCESS)
4167
3
  {
4168
3
    ccv_nnc_cmd_t cmd = CMD_CMUL_FORWARD();
4169
3
    cmd.backend = backend;
4170
3
    status = ccv_nnc_cmd_exec(cmd, ccv_nnc_no_hint, 0, TENSOR_LIST(ga, gb), TENSOR_LIST(gc), 0);
4171
3
  }
4172
3
  if (status == CCV_NNC_EXEC_SUCCESS)
4173
3
    status = ccv_nnc_cmd_exec(CMD_DATA_TRANSFER_FORWARD(), ccv_nnc_no_hint, 0, TENSOR_LIST(gc), TENSOR_LIST(hc), 0);
4174
3
  if (status == CCV_NNC_EXEC_SUCCESS)
4175
3
    status = _ccv_nnc_cmul_convert_to_float(hc, actual, count);
4176
3
  if (status == CCV_NNC_EXEC_SUCCESS)
4177
3
  {
4178
93
    for (i = 0; i < count; 
i += 290
)
4179
90
    {
4180
90
      const float a0 = ha_rounded->data.f32[i];
4181
90
      const float a1 = ha_rounded->data.f32[i + 1];
4182
90
      const float b0 = hb_rounded->data.f32[i];
4183
90
      const float b1 = hb_rounded->data.f32[i + 1];
4184
90
      expected->data.f32[i] = a0 * b0 - a1 * b1;
4185
90
      expected->data.f32[i + 1] = a0 * b1 + a1 * b0;
4186
90
    }
4187
3
    status = _ccv_nnc_cmul_round_to_datatype(expected, expected_typed, expected_rounded, count);
4188
3
  }
4189
3
  *max_diff = 0;
4190
3
  if (status == CCV_NNC_EXEC_SUCCESS)
4191
183
    
for (i = 0; 3
i < count; 
i++180
)
4192
180
      *max_diff = ccv_max(*max_diff, fabsf(actual->data.f32[i] - expected_rounded->data.f32[i]));
4193
3
  ccv_nnc_tensor_free(ha32);
4194
3
  ccv_nnc_tensor_free(hb32);
4195
3
  ccv_nnc_tensor_free(ha);
4196
3
  ccv_nnc_tensor_free(hb);
4197
3
  ccv_nnc_tensor_free(hc);
4198
3
  ccv_nnc_tensor_free(ha_rounded);
4199
3
  ccv_nnc_tensor_free(hb_rounded);
4200
3
  ccv_nnc_tensor_free(expected);
4201
3
  ccv_nnc_tensor_free(expected_rounded);
4202
3
  ccv_nnc_tensor_free(actual);
4203
3
  ccv_nnc_tensor_free(expected_typed);
4204
3
  ccv_nnc_tensor_free(ga);
4205
3
  ccv_nnc_tensor_free(gb);
4206
3
  ccv_nnc_tensor_free(gc);
4207
3
  return status;
4208
3
}
4209
4210
static int _ccv_nnc_cmul_mixed_precision_backward_case(const int backend, const int g_datatype, const int a_datatype, const int b_datatype, const int c_datatype, const int d_datatype, float* const max_diff_c, float* const max_diff_d)
4211
1
{
4212
1
  const int count = 2 * 3 * 10;
4213
1
  ccv_nnc_tensor_param_t h32_params = _ccv_nnc_cmul_mixed_params(CCV_TENSOR_CPU_MEMORY, CCV_32F);
4214
1
  ccv_nnc_tensor_param_t hg_params = _ccv_nnc_cmul_mixed_params(CCV_TENSOR_CPU_MEMORY, g_datatype);
4215
1
  ccv_nnc_tensor_param_t ha_params = _ccv_nnc_cmul_mixed_params(CCV_TENSOR_CPU_MEMORY, a_datatype);
4216
1
  ccv_nnc_tensor_param_t hb_params = _ccv_nnc_cmul_mixed_params(CCV_TENSOR_CPU_MEMORY, b_datatype);
4217
1
  ccv_nnc_tensor_param_t hc_params = _ccv_nnc_cmul_mixed_params(CCV_TENSOR_CPU_MEMORY, c_datatype);
4218
1
  ccv_nnc_tensor_param_t hd_params = _ccv_nnc_cmul_mixed_params(CCV_TENSOR_CPU_MEMORY, d_datatype);
4219
1
  ccv_nnc_tensor_param_t gg_params = _ccv_nnc_cmul_mixed_params(CCV_TENSOR_GPU_MEMORY | 000, g_datatype);
4220
1
  ccv_nnc_tensor_param_t ga_params = _ccv_nnc_cmul_mixed_params(CCV_TENSOR_GPU_MEMORY | 000, a_datatype);
4221
1
  ccv_nnc_tensor_param_t gb_params = _ccv_nnc_cmul_mixed_params(CCV_TENSOR_GPU_MEMORY | 000, b_datatype);
4222
1
  ccv_nnc_tensor_param_t gc_params = _ccv_nnc_cmul_mixed_params(CCV_TENSOR_GPU_MEMORY | 000, c_datatype);
4223
1
  ccv_nnc_tensor_param_t gd_params = _ccv_nnc_cmul_mixed_params(CCV_TENSOR_GPU_MEMORY | 000, d_datatype);
4224
1
  ccv_nnc_tensor_t* const hg32 = ccv_nnc_tensor_new(0, h32_params, 0);
4225
1
  ccv_nnc_tensor_t* const ha32 = ccv_nnc_tensor_new(0, h32_params, 0);
4226
1
  ccv_nnc_tensor_t* const hb32 = ccv_nnc_tensor_new(0, h32_params, 0);
4227
1
  ccv_nnc_tensor_t* const hg = ccv_nnc_tensor_new(0, hg_params, 0);
4228
1
  ccv_nnc_tensor_t* const ha = ccv_nnc_tensor_new(0, ha_params, 0);
4229
1
  ccv_nnc_tensor_t* const hb = ccv_nnc_tensor_new(0, hb_params, 0);
4230
1
  ccv_nnc_tensor_t* const hc = ccv_nnc_tensor_new(0, hc_params, 0);
4231
1
  ccv_nnc_tensor_t* const hd = ccv_nnc_tensor_new(0, hd_params, 0);
4232
1
  ccv_nnc_tensor_t* const hg_rounded = ccv_nnc_tensor_new(0, h32_params, 0);
4233
1
  ccv_nnc_tensor_t* const ha_rounded = ccv_nnc_tensor_new(0, h32_params, 0);
4234
1
  ccv_nnc_tensor_t* const hb_rounded = ccv_nnc_tensor_new(0, h32_params, 0);
4235
1
  ccv_nnc_tensor_t* const expected_c = ccv_nnc_tensor_new(0, h32_params, 0);
4236
1
  ccv_nnc_tensor_t* const expected_d = ccv_nnc_tensor_new(0, h32_params, 0);
4237
1
  ccv_nnc_tensor_t* const expected_c_rounded = ccv_nnc_tensor_new(0, h32_params, 0);
4238
1
  ccv_nnc_tensor_t* const expected_d_rounded = ccv_nnc_tensor_new(0, h32_params, 0);
4239
1
  ccv_nnc_tensor_t* const actual_c = ccv_nnc_tensor_new(0, h32_params, 0);
4240
1
  ccv_nnc_tensor_t* const actual_d = ccv_nnc_tensor_new(0, h32_params, 0);
4241
1
  ccv_nnc_tensor_t* const expected_c_typed = ccv_nnc_tensor_new(0, hc_params, 0);
4242
1
  ccv_nnc_tensor_t* const expected_d_typed = ccv_nnc_tensor_new(0, hd_params, 0);
4243
1
  ccv_nnc_tensor_t* const gg = ccv_nnc_tensor_new(0, gg_params, 0);
4244
1
  ccv_nnc_tensor_t* const ga = ccv_nnc_tensor_new(0, ga_params, 0);
4245
1
  ccv_nnc_tensor_t* const gb = ccv_nnc_tensor_new(0, gb_params, 0);
4246
1
  ccv_nnc_tensor_t* const gc = ccv_nnc_tensor_new(0, gc_params, 0);
4247
1
  ccv_nnc_tensor_t* const gd = ccv_nnc_tensor_new(0, gd_params, 0);
4248
1
  int i;
4249
61
  for (i = 0; i < count; 
i++60
)
4250
60
  {
4251
60
    hg32->data.f32[i] = (float)(((i * 7 + 1) % 19) - 9) * 0.04f;
4252
60
    ha32->data.f32[i] = (float)((i % 13) - 6) * 0.07f;
4253
60
    hb32->data.f32[i] = (float)(((i * 5 + 3) % 17) - 8) * 0.05f;
4254
60
  }
4255
1
  int status = _ccv_nnc_cmul_round_to_datatype(hg32, hg, hg_rounded, count);
4256
1
  if (status == CCV_NNC_EXEC_SUCCESS)
4257
1
    status = _ccv_nnc_cmul_round_to_datatype(ha32, ha, ha_rounded, count);
4258
1
  if (status == CCV_NNC_EXEC_SUCCESS)
4259
1
    status = _ccv_nnc_cmul_round_to_datatype(hb32, hb, hb_rounded, count);
4260
1
  if (status == CCV_NNC_EXEC_SUCCESS)
4261
1
    status = ccv_nnc_cmd_exec(CMD_DATA_TRANSFER_FORWARD(), ccv_nnc_no_hint, 0, TENSOR_LIST(hg, ha, hb), TENSOR_LIST(gg, ga, gb), 0);
4262
1
  if (status == CCV_NNC_EXEC_SUCCESS)
4263
1
  {
4264
1
    ccv_nnc_cmd_t cmd = CMD_CMUL_BACKWARD();
4265
1
    cmd.backend = backend;
4266
1
    status = ccv_nnc_cmd_exec(cmd, ccv_nnc_no_hint, 0, TENSOR_LIST(gg, ga, gb), TENSOR_LIST(gc, gd), 0);
4267
1
  }
4268
1
  if (status == CCV_NNC_EXEC_SUCCESS)
4269
1
    status = ccv_nnc_cmd_exec(CMD_DATA_TRANSFER_FORWARD(), ccv_nnc_no_hint, 0, TENSOR_LIST(gc, gd), TENSOR_LIST(hc, hd), 0);
4270
1
  if (status == CCV_NNC_EXEC_SUCCESS)
4271
1
    status = _ccv_nnc_cmul_convert_to_float(hc, actual_c, count);
4272
1
  if (status == CCV_NNC_EXEC_SUCCESS)
4273
1
    status = _ccv_nnc_cmul_convert_to_float(hd, actual_d, count);
4274
1
  if (status == CCV_NNC_EXEC_SUCCESS)
4275
1
  {
4276
31
    for (i = 0; i < count; 
i += 230
)
4277
30
    {
4278
30
      const float g0 = hg_rounded->data.f32[i];
4279
30
      const float g1 = hg_rounded->data.f32[i + 1];
4280
30
      const float a0 = ha_rounded->data.f32[i];
4281
30
      const float a1 = ha_rounded->data.f32[i + 1];
4282
30
      const float b0 = hb_rounded->data.f32[i];
4283
30
      const float b1 = hb_rounded->data.f32[i + 1];
4284
30
      expected_c->data.f32[i] = g0 * b0 + g1 * b1;
4285
30
      expected_c->data.f32[i + 1] = -g0 * b1 + g1 * b0;
4286
30
      expected_d->data.f32[i] = g0 * a0 + g1 * a1;
4287
30
      expected_d->data.f32[i + 1] = -g0 * a1 + g1 * a0;
4288
30
    }
4289
1
    status = _ccv_nnc_cmul_round_to_datatype(expected_c, expected_c_typed, expected_c_rounded, count);
4290
1
  }
4291
1
  if (status == CCV_NNC_EXEC_SUCCESS)
4292
1
    status = _ccv_nnc_cmul_round_to_datatype(expected_d, expected_d_typed, expected_d_rounded, count);
4293
1
  *max_diff_c = 0;
4294
1
  *max_diff_d = 0;
4295
1
  if (status == CCV_NNC_EXEC_SUCCESS)
4296
61
    
for (i = 0; 1
i < count; 
i++60
)
4297
60
    {
4298
60
      *max_diff_c = ccv_max(*max_diff_c, fabsf(actual_c->data.f32[i] - expected_c_rounded->data.f32[i]));
4299
60
      *max_diff_d = ccv_max(*max_diff_d, fabsf(actual_d->data.f32[i] - expected_d_rounded->data.f32[i]));
4300
60
    }
4301
1
  ccv_nnc_tensor_free(hg32);
4302
1
  ccv_nnc_tensor_free(ha32);
4303
1
  ccv_nnc_tensor_free(hb32);
4304
1
  ccv_nnc_tensor_free(hg);
4305
1
  ccv_nnc_tensor_free(ha);
4306
1
  ccv_nnc_tensor_free(hb);
4307
1
  ccv_nnc_tensor_free(hc);
4308
1
  ccv_nnc_tensor_free(hd);
4309
1
  ccv_nnc_tensor_free(hg_rounded);
4310
1
  ccv_nnc_tensor_free(ha_rounded);
4311
1
  ccv_nnc_tensor_free(hb_rounded);
4312
1
  ccv_nnc_tensor_free(expected_c);
4313
1
  ccv_nnc_tensor_free(expected_d);
4314
1
  ccv_nnc_tensor_free(expected_c_rounded);
4315
1
  ccv_nnc_tensor_free(expected_d_rounded);
4316
1
  ccv_nnc_tensor_free(actual_c);
4317
1
  ccv_nnc_tensor_free(actual_d);
4318
1
  ccv_nnc_tensor_free(expected_c_typed);
4319
1
  ccv_nnc_tensor_free(expected_d_typed);
4320
1
  ccv_nnc_tensor_free(gg);
4321
1
  ccv_nnc_tensor_free(ga);
4322
1
  ccv_nnc_tensor_free(gb);
4323
1
  ccv_nnc_tensor_free(gc);
4324
1
  ccv_nnc_tensor_free(gd);
4325
1
  return status;
4326
1
}
4327
4328
TEST_CASE("cmul mixed half and bfloat precision with mps")
4329
1
{
4330
1
  GUARD_ELSE_RETURN(ccv_nnc_cmd_ok(CCV_NNC_CMUL_FORWARD, CCV_NNC_BACKEND_MPS));
4331
0
  const uint64_t old_flags = ccv_nnc_flags();
4332
0
  if (old_flags & CCV_NNC_DISABLE_MFA)
4333
0
    ccv_nnc_disable_flag(CCV_NNC_DISABLE_MFA);
4334
0
  float max_diff_mfa_0 = 1e30f;
4335
0
  float max_diff_mfa_1 = 1e30f;
4336
0
  float max_diff_mfa_2 = 1e30f;
4337
0
  const int status_mfa_0 = _ccv_nnc_cmul_mixed_precision_case(CCV_NNC_BACKEND_MPS, CCV_16F, CCV_16BF, CCV_16F, &max_diff_mfa_0);
4338
0
  const int status_mfa_1 = _ccv_nnc_cmul_mixed_precision_case(CCV_NNC_BACKEND_MPS, CCV_16BF, CCV_16F, CCV_16BF, &max_diff_mfa_1);
4339
0
  const int status_mfa_2 = _ccv_nnc_cmul_mixed_precision_case(CCV_NNC_BACKEND_MPS, CCV_16F, CCV_16BF, CCV_32F, &max_diff_mfa_2);
4340
0
  ccv_nnc_enable_flag(CCV_NNC_DISABLE_MFA);
4341
0
  float max_diff_graph_0 = 1e30f;
4342
0
  float max_diff_graph_1 = 1e30f;
4343
0
  float max_diff_graph_2 = 1e30f;
4344
0
  const int status_graph_0 = _ccv_nnc_cmul_mixed_precision_case(CCV_NNC_BACKEND_MPS, CCV_16F, CCV_16BF, CCV_16F, &max_diff_graph_0);
4345
0
  const int status_graph_1 = _ccv_nnc_cmul_mixed_precision_case(CCV_NNC_BACKEND_MPS, CCV_16BF, CCV_16F, CCV_16BF, &max_diff_graph_1);
4346
0
  const int status_graph_2 = _ccv_nnc_cmul_mixed_precision_case(CCV_NNC_BACKEND_MPS, CCV_16F, CCV_16BF, CCV_32F, &max_diff_graph_2);
4347
0
  if (old_flags & CCV_NNC_DISABLE_MFA)
4348
0
    ccv_nnc_enable_flag(CCV_NNC_DISABLE_MFA);
4349
0
  else
4350
0
    ccv_nnc_disable_flag(CCV_NNC_DISABLE_MFA);
4351
0
  REQUIRE_EQ(CCV_NNC_EXEC_SUCCESS, status_mfa_0, "mixed half / bfloat cmul through mfa should run");
4352
0
  REQUIRE_EQ(CCV_NNC_EXEC_SUCCESS, status_mfa_1, "mixed bfloat / half cmul through mfa should run");
4353
0
  REQUIRE_EQ(CCV_NNC_EXEC_SUCCESS, status_mfa_2, "mixed half / bfloat cmul to float through mfa should run");
4354
0
  REQUIRE_EQ(CCV_NNC_EXEC_SUCCESS, status_graph_0, "mixed half / bfloat cmul through graph fallback should run");
4355
0
  REQUIRE_EQ(CCV_NNC_EXEC_SUCCESS, status_graph_1, "mixed bfloat / half cmul through graph fallback should run");
4356
0
  REQUIRE_EQ(CCV_NNC_EXEC_SUCCESS, status_graph_2, "mixed half / bfloat cmul to float through graph fallback should run");
4357
0
  REQUIRE(max_diff_mfa_0 <= 3e-2, "mixed half / bfloat cmul through mfa should match fp32 reference rounded to output dtype");
4358
0
  REQUIRE(max_diff_mfa_1 <= 3e-2, "mixed bfloat / half cmul through mfa should match fp32 reference rounded to output dtype");
4359
0
  REQUIRE(max_diff_mfa_2 <= 3e-2, "mixed half / bfloat cmul to float through mfa should match fp32 reference");
4360
0
  REQUIRE(max_diff_graph_0 <= 3e-2, "mixed half / bfloat cmul through graph fallback should match fp32 reference rounded to output dtype");
4361
0
  REQUIRE(max_diff_graph_1 <= 3e-2, "mixed bfloat / half cmul through graph fallback should match fp32 reference rounded to output dtype");
4362
0
  REQUIRE(max_diff_graph_2 <= 3e-2, "mixed half / bfloat cmul to float through graph fallback should match fp32 reference");
4363
0
}
4364
4365
TEST_CASE("cmul gradient mixed half and bfloat precision with mps")
4366
1
{
4367
1
  GUARD_ELSE_RETURN(ccv_nnc_cmd_ok(CCV_NNC_CMUL_BACKWARD, CCV_NNC_BACKEND_MPS));
4368
0
  const uint64_t old_flags = ccv_nnc_flags();
4369
0
  if (old_flags & CCV_NNC_DISABLE_MFA)
4370
0
    ccv_nnc_disable_flag(CCV_NNC_DISABLE_MFA);
4371
0
  float max_diff_mfa_c = 1e30f;
4372
0
  float max_diff_mfa_d = 1e30f;
4373
0
  const int status_mfa = _ccv_nnc_cmul_mixed_precision_backward_case(CCV_NNC_BACKEND_MPS, CCV_16F, CCV_16BF, CCV_16F, CCV_16BF, CCV_16F, &max_diff_mfa_c, &max_diff_mfa_d);
4374
0
  ccv_nnc_enable_flag(CCV_NNC_DISABLE_MFA);
4375
0
  float max_diff_graph_c = 1e30f;
4376
0
  float max_diff_graph_d = 1e30f;
4377
0
  const int status_graph = _ccv_nnc_cmul_mixed_precision_backward_case(CCV_NNC_BACKEND_MPS, CCV_16F, CCV_16BF, CCV_16F, CCV_16BF, CCV_16F, &max_diff_graph_c, &max_diff_graph_d);
4378
0
  if (old_flags & CCV_NNC_DISABLE_MFA)
4379
0
    ccv_nnc_enable_flag(CCV_NNC_DISABLE_MFA);
4380
0
  else
4381
0
    ccv_nnc_disable_flag(CCV_NNC_DISABLE_MFA);
4382
0
  REQUIRE_EQ(CCV_NNC_EXEC_SUCCESS, status_mfa, "mixed half / bfloat cmul gradient through mfa should run");
4383
0
  REQUIRE_EQ(CCV_NNC_EXEC_SUCCESS, status_graph, "mixed half / bfloat cmul gradient through graph fallback should run");
4384
0
  REQUIRE(max_diff_mfa_c <= 3e-2, "mixed half / bfloat cmul gradient first output through mfa should match fp32 reference rounded to output dtype");
4385
0
  REQUIRE(max_diff_mfa_d <= 3e-2, "mixed half / bfloat cmul gradient second output through mfa should match fp32 reference rounded to output dtype");
4386
0
  REQUIRE(max_diff_graph_c <= 3e-2, "mixed half / bfloat cmul gradient first output through graph fallback should match fp32 reference rounded to output dtype");
4387
0
  REQUIRE(max_diff_graph_d <= 3e-2, "mixed half / bfloat cmul gradient second output through graph fallback should match fp32 reference rounded to output dtype");
4388
0
}
4389
4390
TEST_CASE("cmul mixed half and bfloat precision with cuda")
4391
1
{
4392
1
  GUARD_ELSE_RETURN(ccv_nnc_cmd_ok(CCV_NNC_CMUL_FORWARD, CCV_NNC_BACKEND_GPU_REF));
4393
1
  float max_diff_0 = 1e30f;
4394
1
  float max_diff_1 = 1e30f;
4395
1
  float max_diff_2 = 1e30f;
4396
1
  const int status_0 = _ccv_nnc_cmul_mixed_precision_case(CCV_NNC_BACKEND_GPU_REF, CCV_16F, CCV_16BF, CCV_16F, &max_diff_0);
4397
1
  const int status_1 = _ccv_nnc_cmul_mixed_precision_case(CCV_NNC_BACKEND_GPU_REF, CCV_16BF, CCV_16F, CCV_16BF, &max_diff_1);
4398
1
  const int status_2 = _ccv_nnc_cmul_mixed_precision_case(CCV_NNC_BACKEND_GPU_REF, CCV_16F, CCV_16BF, CCV_32F, &max_diff_2);
4399
1
  REQUIRE_EQ(CCV_NNC_EXEC_SUCCESS, status_0, "mixed half / bfloat cmul through cuda should run");
4400
1
  REQUIRE_EQ(CCV_NNC_EXEC_SUCCESS, status_1, "mixed bfloat / half cmul through cuda should run");
4401
1
  REQUIRE_EQ(CCV_NNC_EXEC_SUCCESS, status_2, "mixed half / bfloat cmul to float through cuda should run");
4402
1
  REQUIRE(max_diff_0 <= 3e-2, "mixed half / bfloat cmul through cuda should match fp32 reference rounded to output dtype");
4403
1
  REQUIRE(max_diff_1 <= 3e-2, "mixed bfloat / half cmul through cuda should match fp32 reference rounded to output dtype");
4404
1
  REQUIRE(max_diff_2 <= 3e-2, "mixed half / bfloat cmul to float through cuda should match fp32 reference");
4405
1
}
4406
4407
TEST_CASE("cmul gradient mixed half and bfloat precision with cuda")
4408
1
{
4409
1
  GUARD_ELSE_RETURN(ccv_nnc_cmd_ok(CCV_NNC_CMUL_BACKWARD, CCV_NNC_BACKEND_GPU_REF));
4410
1
  float max_diff_c = 1e30f;
4411
1
  float max_diff_d = 1e30f;
4412
1
  const int status = _ccv_nnc_cmul_mixed_precision_backward_case(CCV_NNC_BACKEND_GPU_REF, CCV_16F, CCV_16BF, CCV_16F, CCV_16BF, CCV_16F, &max_diff_c, &max_diff_d);
4413
1
  REQUIRE_EQ(CCV_NNC_EXEC_SUCCESS, status, "mixed half / bfloat cmul gradient through cuda should run");
4414
1
  REQUIRE(max_diff_c <= 3e-2, "mixed half / bfloat cmul gradient first output through cuda should match fp32 reference rounded to output dtype");
4415
1
  REQUIRE(max_diff_d <= 3e-2, "mixed half / bfloat cmul gradient second output through cuda should match fp32 reference rounded to output dtype");
4416
1
}
4417
4418
TEST_CASE("cmul in float")
4419
1
{
4420
1
  GUARD_ELSE_RETURN(ccv_nnc_cmd_ok(CCV_NNC_CMUL_FORWARD, CCV_NNC_BACKEND_GPU_REF) || ccv_nnc_cmd_ok(CCV_NNC_CMUL_FORWARD, CCV_NNC_BACKEND_MPS));
4421
1
  ccv_nnc_symbolic_graph_t* const symbolic_graph = ccv_nnc_symbolic_graph_new();
4422
1
  ccv_nnc_tensor_symbol_t a = ccv_nnc_tensor_symbol_new(symbolic_graph, GPU_TENSOR_NCHW(000, 32F, 20, 10), "a");
4423
1
  ccv_nnc_tensor_symbol_t b = ccv_nnc_tensor_symbol_new(symbolic_graph, GPU_TENSOR_NCHW(000, 32F, 20, 10), "b");
4424
1
  ccv_nnc_tensor_symbol_t c = ccv_nnc_tensor_symbol_new(symbolic_graph, GPU_TENSOR_NCHW(000, 32F, 20, 10), "c");
4425
1
  ccv_nnc_graph_exec_symbol_new(symbolic_graph, CMD_CMUL_FORWARD(), TENSOR_SYMBOL_LIST(a, b), TENSOR_SYMBOL_LIST(c), "cmul");
4426
1
  ccv_nnc_graph_exec_symbol_autogen(symbolic_graph, 0, 0, CCV_NNC_AUTOGEN_ALL_EXECS | CCV_NNC_AUTOGEN_SOURCES_AND_DESTINATIONS);
4427
1
  SYMBOLIC_GRAPH_GEN(symbolic_graph, CCV_NNC_LONG_DOT_GRAPH);
4428
1
  ccv_nnc_graph_t* graph = 0;
4429
1
  ccv_nnc_tensor_arena_t* tensor_arena = 0;
4430
1
  ccv_nnc_graph_exec_arena_t* graph_exec_arena = 0;
4431
1
  ccv_nnc_symbolic_graph_compile(symbolic_graph, ccv_nnc_default_compile_params, 0, 0, 0, 0, SYMBOLIC_GRAPH_SOURCES(symbolic_graph), SYMBOLIC_GRAPH_DESTINATIONS(symbolic_graph), &graph, &tensor_arena, &graph_exec_arena);
4432
1
  GRAPH_GEN(graph, CCV_NNC_LONG_DOT_GRAPH);
4433
1
  ccv_nnc_tensor_t* const x_tensor = ccv_nnc_tensor_new(0, CPU_TENSOR_NCHW(32F, 20, 10), 0);
4434
1
  ccv_nnc_tensor_t* const y_tensor = ccv_nnc_tensor_new(0, CPU_TENSOR_NCHW(32F, 20, 10), 0);
4435
1
  dsfmt_t dsfmt;
4436
1
  dsfmt_init_gen_rand(&dsfmt, 0);
4437
1
  int i;
4438
201
  for (i = 0; i < 20 * 10; 
i++200
)
4439
200
    x_tensor->data.f32[i] = dsfmt_genrand_open_close(&dsfmt);
4440
201
  for (i = 0; i < 20 * 10; 
i++200
)
4441
200
    y_tensor->data.f32[i] = dsfmt_genrand_open_close(&dsfmt);
4442
1
  ccv_nnc_tensor_t* const a_tensor = ccv_nnc_tensor_from_symbol(tensor_arena, a);
4443
1
  ccv_nnc_cmd_exec(CMD_DATA_TRANSFER_FORWARD(), ccv_nnc_no_hint, 0, TENSOR_LIST(x_tensor), TENSOR_LIST(a_tensor), 0);
4444
1
  ccv_nnc_tensor_t* const b_tensor = ccv_nnc_tensor_from_symbol(tensor_arena, b);
4445
1
  ccv_nnc_cmd_exec(CMD_DATA_TRANSFER_FORWARD(), ccv_nnc_no_hint, 0, TENSOR_LIST(y_tensor), TENSOR_LIST(b_tensor), 0);
4446
1
  ccv_nnc_graph_run(graph, 0, TRAVERSE_FULL, 0, 0);
4447
1
  ccv_nnc_tensor_t* const z_tensor = ccv_nnc_tensor_new(0, CPU_TENSOR_NCHW(32F, 20, 10), 0);
4448
1
  ccv_nnc_tensor_t* const c_tensor = ccv_nnc_tensor_from_symbol(tensor_arena, c);
4449
1
  ccv_nnc_cmd_exec(CMD_DATA_TRANSFER_FORWARD(), ccv_nnc_no_hint, 0, TENSOR_LIST(c_tensor), TENSOR_LIST(z_tensor), 0);
4450
1
  ccv_nnc_tensor_t* const tz = ccv_nnc_tensor_new(0, CPU_TENSOR_NCHW(32F, 20, 10), 0);
4451
1
  ccv_nnc_cmd_exec(CMD_CMUL_FORWARD(), ccv_nnc_no_hint, 0, TENSOR_LIST(x_tensor, y_tensor), TENSOR_LIST(tz), 0);
4452
1
  REQUIRE_TENSOR_EQ(tz, z_tensor, "gelu from cudnn should match from CPU");
4453
1
  ccv_nnc_tensor_free(x_tensor);
4454
1
  ccv_nnc_tensor_free(y_tensor);
4455
1
  ccv_nnc_tensor_free(z_tensor);
4456
1
  ccv_nnc_tensor_free(tz);
4457
1
  ccv_nnc_graph_free(graph);
4458
1
  ccv_nnc_tensor_arena_free(tensor_arena);
4459
1
  ccv_nnc_graph_exec_arena_free(graph_exec_arena);
4460
1
  ccv_nnc_symbolic_graph_free(symbolic_graph);
4461
1
}
4462
4463
TEST_CASE("cmul in half precision")
4464
1
{
4465
1
  GUARD_ELSE_RETURN(ccv_nnc_cmd_ok(CCV_NNC_CMUL_FORWARD, CCV_NNC_BACKEND_GPU_REF) || ccv_nnc_cmd_ok(CCV_NNC_CMUL_FORWARD, CCV_NNC_BACKEND_MPS));
4466
1
  ccv_nnc_symbolic_graph_t* const symbolic_graph = ccv_nnc_symbolic_graph_new();
4467
1
  ccv_nnc_tensor_symbol_t a = ccv_nnc_tensor_symbol_new(symbolic_graph, GPU_TENSOR_NCHW(000, 16F, 20, 10), "a");
4468
1
  ccv_nnc_tensor_symbol_t b = ccv_nnc_tensor_symbol_new(symbolic_graph, GPU_TENSOR_NCHW(000, 16F, 20, 10), "b");
4469
1
  ccv_nnc_tensor_symbol_t c = ccv_nnc_tensor_symbol_new(symbolic_graph, GPU_TENSOR_NCHW(000, 16F, 20, 10), "c");
4470
1
  ccv_nnc_graph_exec_symbol_new(symbolic_graph, CMD_CMUL_FORWARD(), TENSOR_SYMBOL_LIST(a, b), TENSOR_SYMBOL_LIST(c), "cmul");
4471
1
  ccv_nnc_graph_exec_symbol_autogen(symbolic_graph, 0, 0, CCV_NNC_AUTOGEN_ALL_EXECS | CCV_NNC_AUTOGEN_SOURCES_AND_DESTINATIONS);
4472
1
  SYMBOLIC_GRAPH_GEN(symbolic_graph, CCV_NNC_LONG_DOT_GRAPH);
4473
1
  ccv_nnc_graph_t* graph = 0;
4474
1
  ccv_nnc_tensor_arena_t* tensor_arena = 0;
4475
1
  ccv_nnc_graph_exec_arena_t* graph_exec_arena = 0;
4476
1
  ccv_nnc_symbolic_graph_compile(symbolic_graph, ccv_nnc_default_compile_params, 0, 0, 0, 0, SYMBOLIC_GRAPH_SOURCES(symbolic_graph), SYMBOLIC_GRAPH_DESTINATIONS(symbolic_graph), &graph, &tensor_arena, &graph_exec_arena);
4477
1
  GRAPH_GEN(graph, CCV_NNC_LONG_DOT_GRAPH);
4478
1
  ccv_nnc_tensor_t* const x_tensor = ccv_nnc_tensor_new(0, CPU_TENSOR_NCHW(32F, 20, 10), 0);
4479
1
  ccv_nnc_tensor_t* const y_tensor = ccv_nnc_tensor_new(0, CPU_TENSOR_NCHW(32F, 20, 10), 0);
4480
1
  dsfmt_t dsfmt;
4481
1
  dsfmt_init_gen_rand(&dsfmt, 0);
4482
1
  int i;
4483
201
  for (i = 0; i < 20 * 10; 
i++200
)
4484
200
    x_tensor->data.f32[i] = dsfmt_genrand_open_close(&dsfmt);
4485
201
  for (i = 0; i < 20 * 10; 
i++200
)
4486
200
    y_tensor->data.f32[i] = dsfmt_genrand_open_close(&dsfmt);
4487
1
  ccv_nnc_tensor_t* const a_tensor = ccv_nnc_tensor_from_symbol(tensor_arena, a);
4488
1
  ccv_nnc_tensor_t* const x16_tensor = ccv_nnc_tensor_new(0, CPU_TENSOR_NCHW(16F, 20, 10), 0);
4489
1
  ccv_nnc_cmd_exec(CMD_DATATYPE_CONVERSION_FORWARD(), ccv_nnc_no_hint, 0, TENSOR_LIST(x_tensor), TENSOR_LIST(x16_tensor), 0);
4490
1
  ccv_nnc_cmd_exec(CMD_DATA_TRANSFER_FORWARD(), ccv_nnc_no_hint, 0, TENSOR_LIST(x16_tensor), TENSOR_LIST(a_tensor), 0);
4491
1
  ccv_nnc_tensor_t* const b_tensor = ccv_nnc_tensor_from_symbol(tensor_arena, b);
4492
1
  ccv_nnc_tensor_t* const y16_tensor = ccv_nnc_tensor_new(0, CPU_TENSOR_NCHW(16F, 20, 10), 0);
4493
1
  ccv_nnc_cmd_exec(CMD_DATATYPE_CONVERSION_FORWARD(), ccv_nnc_no_hint, 0, TENSOR_LIST(y_tensor), TENSOR_LIST(y16_tensor), 0);
4494
1
  ccv_nnc_cmd_exec(CMD_DATA_TRANSFER_FORWARD(), ccv_nnc_no_hint, 0, TENSOR_LIST(y16_tensor), TENSOR_LIST(b_tensor), 0);
4495
1
  ccv_nnc_graph_run(graph, 0, TRAVERSE_FULL, 0, 0);
4496
1
  ccv_nnc_tensor_t* const z16_tensor = ccv_nnc_tensor_new(0, CPU_TENSOR_NCHW(16F, 20, 10), 0);
4497
1
  ccv_nnc_tensor_t* const z_tensor = ccv_nnc_tensor_new(0, CPU_TENSOR_NCHW(32F, 20, 10), 0);
4498
1
  ccv_nnc_tensor_t* const c_tensor = ccv_nnc_tensor_from_symbol(tensor_arena, c);
4499
1
  ccv_nnc_cmd_exec(CMD_DATA_TRANSFER_FORWARD(), ccv_nnc_no_hint, 0, TENSOR_LIST(c_tensor), TENSOR_LIST(z16_tensor), 0);
4500
1
  ccv_nnc_cmd_exec(CMD_DATATYPE_CONVERSION_FORWARD(), ccv_nnc_no_hint, 0, TENSOR_LIST(z16_tensor), TENSOR_LIST(z_tensor), 0);
4501
1
  ccv_nnc_tensor_t* const tz = ccv_nnc_tensor_new(0, CPU_TENSOR_NCHW(32F, 20, 10), 0);
4502
1
  ccv_nnc_cmd_exec(CMD_CMUL_FORWARD(), ccv_nnc_no_hint, 0, TENSOR_LIST(x_tensor, y_tensor), TENSOR_LIST(tz), 0);
4503
1
  REQUIRE_ARRAY_EQ_WITH_TOLERANCE(float, tz->data.f32, z_tensor->data.f32, 20 * 10, 2e-3, "gelu from cudnn should match from CPU");
4504
1
  ccv_nnc_tensor_free(x_tensor);
4505
1
  ccv_nnc_tensor_free(x16_tensor);
4506
1
  ccv_nnc_tensor_free(y16_tensor);
4507
1
  ccv_nnc_tensor_free(y_tensor);
4508
1
  ccv_nnc_tensor_free(z16_tensor);
4509
1
  ccv_nnc_tensor_free(z_tensor);
4510
1
  ccv_nnc_tensor_free(tz);
4511
1
  ccv_nnc_graph_free(graph);
4512
1
  ccv_nnc_tensor_arena_free(tensor_arena);
4513
1
  ccv_nnc_graph_exec_arena_free(graph_exec_arena);
4514
1
  ccv_nnc_symbolic_graph_free(symbolic_graph);
4515
1
}
4516
4517
TEST_CASE("cmul in float, broadcast semantics")
4518
1
{
4519
1
  GUARD_ELSE_RETURN(ccv_nnc_cmd_ok(CCV_NNC_CMUL_FORWARD, CCV_NNC_BACKEND_GPU_REF) || ccv_nnc_cmd_ok(CCV_NNC_CMUL_FORWARD, CCV_NNC_BACKEND_MPS));
4520
1
  ccv_nnc_symbolic_graph_t* const symbolic_graph = ccv_nnc_symbolic_graph_new();
4521
1
  ccv_nnc_tensor_symbol_t a = ccv_nnc_tensor_symbol_new(symbolic_graph, GPU_TENSOR_NCHW(000, 32F, 1, 5, 8, 128), "a");
4522
1
  ccv_nnc_tensor_symbol_t b = ccv_nnc_tensor_symbol_new(symbolic_graph, GPU_TENSOR_NCHW(000, 32F, 1, 5, 1, 128), "b");
4523
1
  ccv_nnc_tensor_symbol_t c = ccv_nnc_tensor_symbol_new(symbolic_graph, GPU_TENSOR_NCHW(000, 32F, 1, 5, 8, 128), "c");
4524
1
  ccv_nnc_graph_exec_symbol_new(symbolic_graph, CMD_CMUL_FORWARD(), TENSOR_SYMBOL_LIST(a, b), TENSOR_SYMBOL_LIST(c), "cmul");
4525
1
  ccv_nnc_graph_exec_symbol_autogen(symbolic_graph, 0, 0, CCV_NNC_AUTOGEN_ALL_EXECS | CCV_NNC_AUTOGEN_SOURCES_AND_DESTINATIONS);
4526
1
  SYMBOLIC_GRAPH_GEN(symbolic_graph, CCV_NNC_LONG_DOT_GRAPH);
4527
1
  ccv_nnc_graph_t* graph = 0;
4528
1
  ccv_nnc_tensor_arena_t* tensor_arena = 0;
4529
1
  ccv_nnc_graph_exec_arena_t* graph_exec_arena = 0;
4530
1
  ccv_nnc_symbolic_graph_compile(symbolic_graph, ccv_nnc_default_compile_params, 0, 0, 0, 0, SYMBOLIC_GRAPH_SOURCES(symbolic_graph), SYMBOLIC_GRAPH_DESTINATIONS(symbolic_graph), &graph, &tensor_arena, &graph_exec_arena);
4531
1
  GRAPH_GEN(graph, CCV_NNC_LONG_DOT_GRAPH);
4532
1
  ccv_nnc_tensor_t* const x_tensor = ccv_nnc_tensor_new(0, CPU_TENSOR_NCHW(32F, 1, 5, 8, 128), 0);
4533
1
  ccv_nnc_tensor_t* const y_tensor = ccv_nnc_tensor_new(0, CPU_TENSOR_NCHW(32F, 1, 5, 1, 128), 0);
4534
1
  dsfmt_t dsfmt;
4535
1
  dsfmt_init_gen_rand(&dsfmt, 0);
4536
1
  int i;
4537
5.12k
  for (i = 0; i < 1 * 5 * 8 * 128; 
i++5.12k
)
4538
5.12k
    x_tensor->data.f32[i] = dsfmt_genrand_open_close(&dsfmt);
4539
641
  for (i = 0; i < 1 * 5 * 1 * 128; 
i++640
)
4540
640
    y_tensor->data.f32[i] = dsfmt_genrand_open_close(&dsfmt);
4541
1
  ccv_nnc_tensor_t* const a_tensor = ccv_nnc_tensor_from_symbol(tensor_arena, a);
4542
1
  ccv_nnc_cmd_exec(CMD_DATA_TRANSFER_FORWARD(), ccv_nnc_no_hint, 0, TENSOR_LIST(x_tensor), TENSOR_LIST(a_tensor), 0);
4543
1
  ccv_nnc_tensor_t* const b_tensor = ccv_nnc_tensor_from_symbol(tensor_arena, b);
4544
1
  ccv_nnc_cmd_exec(CMD_DATA_TRANSFER_FORWARD(), ccv_nnc_no_hint, 0, TENSOR_LIST(y_tensor), TENSOR_LIST(b_tensor), 0);
4545
1
  ccv_nnc_graph_run(graph, 0, TRAVERSE_FULL, 0, 0);
4546
1
  ccv_nnc_tensor_t* const z_tensor = ccv_nnc_tensor_new(0, CPU_TENSOR_NCHW(32F, 1, 5, 8, 128), 0);
4547
1
  ccv_nnc_tensor_t* const c_tensor = ccv_nnc_tensor_from_symbol(tensor_arena, c);
4548
1
  ccv_nnc_cmd_exec(CMD_DATA_TRANSFER_FORWARD(), ccv_nnc_no_hint, 0, TENSOR_LIST(c_tensor), TENSOR_LIST(z_tensor), 0);
4549
1
  ccv_nnc_tensor_t* const tz = ccv_nnc_tensor_new(0, CPU_TENSOR_NCHW(32F, 1, 5, 8, 128), 0);
4550
1
  ccv_nnc_cmd_exec(CMD_CMUL_FORWARD(), ccv_nnc_no_hint, 0, TENSOR_LIST(x_tensor, y_tensor), TENSOR_LIST(tz), 0);
4551
1
  REQUIRE_TENSOR_EQ(tz, z_tensor, "gelu from cudnn should match from CPU");
4552
1
  ccv_nnc_tensor_free(x_tensor);
4553
1
  ccv_nnc_tensor_free(y_tensor);
4554
1
  ccv_nnc_tensor_free(z_tensor);
4555
1
  ccv_nnc_tensor_free(tz);
4556
1
  ccv_nnc_graph_free(graph);
4557
1
  ccv_nnc_tensor_arena_free(tensor_arena);
4558
1
  ccv_nnc_graph_exec_arena_free(graph_exec_arena);
4559
1
  ccv_nnc_symbolic_graph_free(symbolic_graph);
4560
1
}
4561
4562
TEST_CASE("cmul in float, broadcast semantics with longer than 65535 sequence")
4563
1
{
4564
1
  GUARD_ELSE_RETURN(ccv_nnc_cmd_ok(CCV_NNC_CMUL_FORWARD, CCV_NNC_BACKEND_GPU_REF) || ccv_nnc_cmd_ok(CCV_NNC_CMUL_FORWARD, CCV_NNC_BACKEND_MPS));
4565
1
  ccv_nnc_symbolic_graph_t* const symbolic_graph = ccv_nnc_symbolic_graph_new();
4566
1
  ccv_nnc_tensor_symbol_t a = ccv_nnc_tensor_symbol_new(symbolic_graph, GPU_TENSOR_NCHW(000, 32F, 1, 70000, 8, 16), "a");
4567
1
  ccv_nnc_tensor_symbol_t b = ccv_nnc_tensor_symbol_new(symbolic_graph, GPU_TENSOR_NCHW(000, 32F, 1, 70000, 1, 16), "b");
4568
1
  ccv_nnc_tensor_symbol_t c = ccv_nnc_tensor_symbol_new(symbolic_graph, GPU_TENSOR_NCHW(000, 32F, 1, 70000, 8, 16), "c");
4569
1
  ccv_nnc_graph_exec_symbol_new(symbolic_graph, CMD_CMUL_FORWARD(), TENSOR_SYMBOL_LIST(a, b), TENSOR_SYMBOL_LIST(c), "cmul");
4570
1
  ccv_nnc_graph_exec_symbol_autogen(symbolic_graph, 0, 0, CCV_NNC_AUTOGEN_ALL_EXECS | CCV_NNC_AUTOGEN_SOURCES_AND_DESTINATIONS);
4571
1
  SYMBOLIC_GRAPH_GEN(symbolic_graph, CCV_NNC_LONG_DOT_GRAPH);
4572
1
  ccv_nnc_graph_t* graph = 0;
4573
1
  ccv_nnc_tensor_arena_t* tensor_arena = 0;
4574
1
  ccv_nnc_graph_exec_arena_t* graph_exec_arena = 0;
4575
1
  ccv_nnc_symbolic_graph_compile(symbolic_graph, ccv_nnc_default_compile_params, 0, 0, 0, 0, SYMBOLIC_GRAPH_SOURCES(symbolic_graph), SYMBOLIC_GRAPH_DESTINATIONS(symbolic_graph), &graph, &tensor_arena, &graph_exec_arena);
4576
1
  GRAPH_GEN(graph, CCV_NNC_LONG_DOT_GRAPH);
4577
1
  ccv_nnc_tensor_t* const x_tensor = ccv_nnc_tensor_new(0, CPU_TENSOR_NCHW(32F, 1, 70000, 8, 16), 0);
4578
1
  ccv_nnc_tensor_t* const y_tensor = ccv_nnc_tensor_new(0, CPU_TENSOR_NCHW(32F, 1, 70000, 1, 16), 0);
4579
1
  dsfmt_t dsfmt;
4580
1
  dsfmt_init_gen_rand(&dsfmt, 0);
4581
1
  int i;
4582
8.96M
  for (i = 0; i < 1 * 70000 * 8 * 16; 
i++8.96M
)
4583
8.96M
    x_tensor->data.f32[i] = dsfmt_genrand_open_close(&dsfmt);
4584
1.12M
  for (i = 0; i < 1 * 70000 * 1 * 16; 
i++1.12M
)
4585
1.12M
    y_tensor->data.f32[i] = dsfmt_genrand_open_close(&dsfmt);
4586
1
  ccv_nnc_tensor_t* const a_tensor = ccv_nnc_tensor_from_symbol(tensor_arena, a);
4587
1
  ccv_nnc_cmd_exec(CMD_DATA_TRANSFER_FORWARD(), ccv_nnc_no_hint, 0, TENSOR_LIST(x_tensor), TENSOR_LIST(a_tensor), 0);
4588
1
  ccv_nnc_tensor_t* const b_tensor = ccv_nnc_tensor_from_symbol(tensor_arena, b);
4589
1
  ccv_nnc_cmd_exec(CMD_DATA_TRANSFER_FORWARD(), ccv_nnc_no_hint, 0, TENSOR_LIST(y_tensor), TENSOR_LIST(b_tensor), 0);
4590
1
  ccv_nnc_graph_run(graph, 0, TRAVERSE_FULL, 0, 0);
4591
1
  ccv_nnc_tensor_t* const z_tensor = ccv_nnc_tensor_new(0, CPU_TENSOR_NCHW(32F, 1, 70000, 8, 16), 0);
4592
1
  ccv_nnc_tensor_t* const c_tensor = ccv_nnc_tensor_from_symbol(tensor_arena, c);
4593
1
  ccv_nnc_cmd_exec(CMD_DATA_TRANSFER_FORWARD(), ccv_nnc_no_hint, 0, TENSOR_LIST(c_tensor), TENSOR_LIST(z_tensor), 0);
4594
1
  ccv_nnc_tensor_t* const tz = ccv_nnc_tensor_new(0, CPU_TENSOR_NCHW(32F, 1, 70000, 8, 16), 0);
4595
1
  ccv_nnc_cmd_exec(CMD_CMUL_FORWARD(), ccv_nnc_no_hint, 0, TENSOR_LIST(x_tensor, y_tensor), TENSOR_LIST(tz), 0);
4596
1
  REQUIRE_TENSOR_EQ(tz, z_tensor, "gelu from cudnn should match from CPU");
4597
1
  ccv_nnc_tensor_free(x_tensor);
4598
1
  ccv_nnc_tensor_free(y_tensor);
4599
1
  ccv_nnc_tensor_free(z_tensor);
4600
1
  ccv_nnc_tensor_free(tz);
4601
1
  ccv_nnc_graph_free(graph);
4602
1
  ccv_nnc_tensor_arena_free(tensor_arena);
4603
1
  ccv_nnc_graph_exec_arena_free(graph_exec_arena);
4604
1
  ccv_nnc_symbolic_graph_free(symbolic_graph);
4605
1
}
4606
4607
TEST_CASE("cmul in float, broadcast semantics with longer than 65535 sequence and more than 1 batch size")
4608
1
{
4609
1
  GUARD_ELSE_RETURN(ccv_nnc_cmd_ok(CCV_NNC_CMUL_FORWARD, CCV_NNC_BACKEND_GPU_REF) || ccv_nnc_cmd_ok(CCV_NNC_CMUL_FORWARD, CCV_NNC_BACKEND_MPS));
4610
1
  ccv_nnc_symbolic_graph_t* const symbolic_graph = ccv_nnc_symbolic_graph_new();
4611
1
  ccv_nnc_tensor_symbol_t a = ccv_nnc_tensor_symbol_new(symbolic_graph, GPU_TENSOR_NCHW(000, 32F, 2, 40000, 8, 16), "a");
4612
1
  ccv_nnc_tensor_symbol_t b = ccv_nnc_tensor_symbol_new(symbolic_graph, GPU_TENSOR_NCHW(000, 32F, 1, 40000, 1, 16), "b");
4613
1
  ccv_nnc_tensor_symbol_t c = ccv_nnc_tensor_symbol_new(symbolic_graph, GPU_TENSOR_NCHW(000, 32F, 2, 40000, 8, 16), "c");
4614
1
  ccv_nnc_graph_exec_symbol_new(symbolic_graph, CMD_CMUL_FORWARD(), TENSOR_SYMBOL_LIST(a, b), TENSOR_SYMBOL_LIST(c), "cmul");
4615
1
  ccv_nnc_graph_exec_symbol_autogen(symbolic_graph, 0, 0, CCV_NNC_AUTOGEN_ALL_EXECS | CCV_NNC_AUTOGEN_SOURCES_AND_DESTINATIONS);
4616
1
  SYMBOLIC_GRAPH_GEN(symbolic_graph, CCV_NNC_LONG_DOT_GRAPH);
4617
1
  ccv_nnc_graph_t* graph = 0;
4618
1
  ccv_nnc_tensor_arena_t* tensor_arena = 0;
4619
1
  ccv_nnc_graph_exec_arena_t* graph_exec_arena = 0;
4620
1
  ccv_nnc_symbolic_graph_compile(symbolic_graph, ccv_nnc_default_compile_params, 0, 0, 0, 0, SYMBOLIC_GRAPH_SOURCES(symbolic_graph), SYMBOLIC_GRAPH_DESTINATIONS(symbolic_graph), &graph, &tensor_arena, &graph_exec_arena);
4621
1
  GRAPH_GEN(graph, CCV_NNC_LONG_DOT_GRAPH);
4622
1
  ccv_nnc_tensor_t* const x_tensor = ccv_nnc_tensor_new(0, CPU_TENSOR_NCHW(32F, 2, 40000, 8, 16), 0);
4623
1
  ccv_nnc_tensor_t* const y_tensor = ccv_nnc_tensor_new(0, CPU_TENSOR_NCHW(32F, 1, 40000, 1, 16), 0);
4624
1
  dsfmt_t dsfmt;
4625
1
  dsfmt_init_gen_rand(&dsfmt, 0);
4626
1
  int i;
4627
10.2M
  for (i = 0; i < 2 * 40000 * 8 * 16; 
i++10.2M
)
4628
10.2M
    x_tensor->data.f32[i] = dsfmt_genrand_open_close(&dsfmt);
4629
640k
  for (i = 0; i < 1 * 40000 * 1 * 16; 
i++640k
)
4630
640k
    y_tensor->data.f32[i] = dsfmt_genrand_open_close(&dsfmt);
4631
1
  ccv_nnc_tensor_t* const a_tensor = ccv_nnc_tensor_from_symbol(tensor_arena, a);
4632
1
  ccv_nnc_cmd_exec(CMD_DATA_TRANSFER_FORWARD(), ccv_nnc_no_hint, 0, TENSOR_LIST(x_tensor), TENSOR_LIST(a_tensor), 0);
4633
1
  ccv_nnc_tensor_t* const b_tensor = ccv_nnc_tensor_from_symbol(tensor_arena, b);
4634
1
  ccv_nnc_cmd_exec(CMD_DATA_TRANSFER_FORWARD(), ccv_nnc_no_hint, 0, TENSOR_LIST(y_tensor), TENSOR_LIST(b_tensor), 0);
4635
1
  ccv_nnc_graph_run(graph, 0, TRAVERSE_FULL, 0, 0);
4636
1
  ccv_nnc_tensor_t* const z_tensor = ccv_nnc_tensor_new(0, CPU_TENSOR_NCHW(32F, 2, 40000, 8, 16), 0);
4637
1
  ccv_nnc_tensor_t* const c_tensor = ccv_nnc_tensor_from_symbol(tensor_arena, c);
4638
1
  ccv_nnc_cmd_exec(CMD_DATA_TRANSFER_FORWARD(), ccv_nnc_no_hint, 0, TENSOR_LIST(c_tensor), TENSOR_LIST(z_tensor), 0);
4639
1
  ccv_nnc_tensor_t* const tz = ccv_nnc_tensor_new(0, CPU_TENSOR_NCHW(32F, 2, 40000, 8, 16), 0);
4640
1
  ccv_nnc_cmd_exec(CMD_CMUL_FORWARD(), ccv_nnc_no_hint, 0, TENSOR_LIST(x_tensor, y_tensor), TENSOR_LIST(tz), 0);
4641
1
  REQUIRE_TENSOR_EQ(tz, z_tensor, "gelu from cudnn should match from CPU");
4642
1
  ccv_nnc_tensor_free(x_tensor);
4643
1
  ccv_nnc_tensor_free(y_tensor);
4644
1
  ccv_nnc_tensor_free(z_tensor);
4645
1
  ccv_nnc_tensor_free(tz);
4646
1
  ccv_nnc_graph_free(graph);
4647
1
  ccv_nnc_tensor_arena_free(tensor_arena);
4648
1
  ccv_nnc_graph_exec_arena_free(graph_exec_arena);
4649
1
  ccv_nnc_symbolic_graph_free(symbolic_graph);
4650
1
}
4651
4652
TEST_CASE("cmul gradient in float")
4653
1
{
4654
1
  GUARD_ELSE_RETURN(ccv_nnc_cmd_ok(CCV_NNC_CMUL_BACKWARD, CCV_NNC_BACKEND_GPU_REF) || ccv_nnc_cmd_ok(CCV_NNC_CMUL_BACKWARD, CCV_NNC_BACKEND_MPS));
4655
1
  ccv_nnc_symbolic_graph_t* const symbolic_graph = ccv_nnc_symbolic_graph_new();
4656
1
  ccv_nnc_tensor_symbol_t a = ccv_nnc_tensor_symbol_new(symbolic_graph, GPU_TENSOR_NCHW(000, 32F, 20, 10), "a");
4657
1
  ccv_nnc_tensor_symbol_t b = ccv_nnc_tensor_symbol_new(symbolic_graph, GPU_TENSOR_NCHW(000, 32F, 20, 10), "b");
4658
1
  ccv_nnc_tensor_symbol_t c = ccv_nnc_tensor_symbol_new(symbolic_graph, GPU_TENSOR_NCHW(000, 32F, 20, 10), "c");
4659
1
  ccv_nnc_tensor_symbol_t d = ccv_nnc_tensor_symbol_new(symbolic_graph, GPU_TENSOR_NCHW(000, 32F, 20, 10), "d");
4660
1
  ccv_nnc_tensor_symbol_t e = ccv_nnc_tensor_symbol_new(symbolic_graph, GPU_TENSOR_NCHW(000, 32F, 20, 10), "e");
4661
1
  ccv_nnc_graph_exec_symbol_new(symbolic_graph, CMD_CMUL_BACKWARD(), TENSOR_SYMBOL_LIST(a, b, c), TENSOR_SYMBOL_LIST(d, e), "cmul");
4662
1
  ccv_nnc_graph_exec_symbol_autogen(symbolic_graph, 0, 0, CCV_NNC_AUTOGEN_ALL_EXECS | CCV_NNC_AUTOGEN_SOURCES_AND_DESTINATIONS);
4663
1
  SYMBOLIC_GRAPH_GEN(symbolic_graph, CCV_NNC_LONG_DOT_GRAPH);
4664
1
  ccv_nnc_graph_t* graph = 0;
4665
1
  ccv_nnc_tensor_arena_t* tensor_arena = 0;
4666
1
  ccv_nnc_graph_exec_arena_t* graph_exec_arena = 0;
4667
1
  ccv_nnc_symbolic_graph_compile(symbolic_graph, ccv_nnc_default_compile_params, 0, 0, 0, 0, SYMBOLIC_GRAPH_SOURCES(symbolic_graph), SYMBOLIC_GRAPH_DESTINATIONS(symbolic_graph), &graph, &tensor_arena, &graph_exec_arena);
4668
1
  GRAPH_GEN(graph, CCV_NNC_LONG_DOT_GRAPH);
4669
1
  ccv_nnc_tensor_t* const x_tensor = ccv_nnc_tensor_new(0, CPU_TENSOR_NCHW(32F, 20, 10), 0);
4670
1
  ccv_nnc_tensor_t* const y_tensor = ccv_nnc_tensor_new(0, CPU_TENSOR_NCHW(32F, 20, 10), 0);
4671
1
  ccv_nnc_tensor_t* const z_tensor = ccv_nnc_tensor_new(0, CPU_TENSOR_NCHW(32F, 20, 10), 0);
4672
1
  dsfmt_t dsfmt;
4673
1
  dsfmt_init_gen_rand(&dsfmt, 0);
4674
1
  int i;
4675
201
  for (i = 0; i < 20 * 10; 
i++200
)
4676
200
    x_tensor->data.f32[i] = dsfmt_genrand_open_close(&dsfmt);
4677
201
  for (i = 0; i < 20 * 10; 
i++200
)
4678
200
    y_tensor->data.f32[i] = dsfmt_genrand_open_close(&dsfmt);
4679
201
  for (i = 0; i < 20 * 10; 
i++200
)
4680
200
    z_tensor->data.f32[i] = dsfmt_genrand_open_close(&dsfmt);
4681
1
  ccv_nnc_tensor_t* const a_tensor = ccv_nnc_tensor_from_symbol(tensor_arena, a);
4682
1
  ccv_nnc_cmd_exec(CMD_DATA_TRANSFER_FORWARD(), ccv_nnc_no_hint, 0, TENSOR_LIST(x_tensor), TENSOR_LIST(a_tensor), 0);
4683
1
  ccv_nnc_tensor_t* const b_tensor = ccv_nnc_tensor_from_symbol(tensor_arena, b);
4684
1
  ccv_nnc_cmd_exec(CMD_DATA_TRANSFER_FORWARD(), ccv_nnc_no_hint, 0, TENSOR_LIST(y_tensor), TENSOR_LIST(b_tensor), 0);
4685
1
  ccv_nnc_tensor_t* const c_tensor = ccv_nnc_tensor_from_symbol(tensor_arena, c);
4686
1
  ccv_nnc_cmd_exec(CMD_DATA_TRANSFER_FORWARD(), ccv_nnc_no_hint, 0, TENSOR_LIST(z_tensor), TENSOR_LIST(c_tensor), 0);
4687
1
  ccv_nnc_graph_run(graph, 0, TRAVERSE_FULL, 0, 0);
4688
1
  ccv_nnc_tensor_t* const od_tensor = ccv_nnc_tensor_new(0, CPU_TENSOR_NCHW(32F, 20, 10), 0);
4689
1
  ccv_nnc_tensor_t* const d_tensor = ccv_nnc_tensor_from_symbol(tensor_arena, d);
4690
1
  ccv_nnc_tensor_t* const oe_tensor = ccv_nnc_tensor_new(0, CPU_TENSOR_NCHW(32F, 20, 10), 0);
4691
1
  ccv_nnc_tensor_t* const e_tensor = ccv_nnc_tensor_from_symbol(tensor_arena, e);
4692
1
  ccv_nnc_cmd_exec(CMD_DATA_TRANSFER_FORWARD(), ccv_nnc_no_hint, 0, TENSOR_LIST(d_tensor, e_tensor), TENSOR_LIST(od_tensor, oe_tensor), 0);
4693
1
  ccv_nnc_tensor_t* const td = ccv_nnc_tensor_new(0, CPU_TENSOR_NCHW(32F, 20, 10), 0);
4694
1
  ccv_nnc_tensor_t* const te = ccv_nnc_tensor_new(0, CPU_TENSOR_NCHW(32F, 20, 10), 0);
4695
1
  ccv_nnc_cmd_exec(CMD_CMUL_BACKWARD(), ccv_nnc_no_hint, 0, TENSOR_LIST(x_tensor, y_tensor, z_tensor), TENSOR_LIST(td, te), 0);
4696
1
  REQUIRE_TENSOR_EQ(td, od_tensor, "cmul gradient from cudnn should match from CPU");
4697
1
  REQUIRE_TENSOR_EQ(te, oe_tensor, "cmul gradient from cudnn should match from CPU");
4698
1
  ccv_nnc_tensor_free(x_tensor);
4699
1
  ccv_nnc_tensor_free(y_tensor);
4700
1
  ccv_nnc_tensor_free(z_tensor);
4701
1
  ccv_nnc_tensor_free(od_tensor);
4702
1
  ccv_nnc_tensor_free(oe_tensor);
4703
1
  ccv_nnc_tensor_free(td);
4704
1
  ccv_nnc_tensor_free(te);
4705
1
  ccv_nnc_graph_free(graph);
4706
1
  ccv_nnc_tensor_arena_free(tensor_arena);
4707
1
  ccv_nnc_graph_exec_arena_free(graph_exec_arena);
4708
1
  ccv_nnc_symbolic_graph_free(symbolic_graph);
4709
1
}
4710
4711
TEST_CASE("cmul gradient in half precision")
4712
1
{
4713
1
  GUARD_ELSE_RETURN(ccv_nnc_cmd_ok(CCV_NNC_CMUL_BACKWARD, CCV_NNC_BACKEND_GPU_REF) || ccv_nnc_cmd_ok(CCV_NNC_CMUL_BACKWARD, CCV_NNC_BACKEND_MPS));
4714
1
  ccv_nnc_symbolic_graph_t* const symbolic_graph = ccv_nnc_symbolic_graph_new();
4715
1
  ccv_nnc_tensor_symbol_t a = ccv_nnc_tensor_symbol_new(symbolic_graph, GPU_TENSOR_NCHW(000, 16F, 20, 10), "a");
4716
1
  ccv_nnc_tensor_symbol_t b = ccv_nnc_tensor_symbol_new(symbolic_graph, GPU_TENSOR_NCHW(000, 16F, 20, 10), "b");
4717
1
  ccv_nnc_tensor_symbol_t c = ccv_nnc_tensor_symbol_new(symbolic_graph, GPU_TENSOR_NCHW(000, 16F, 20, 10), "c");
4718
1
  ccv_nnc_tensor_symbol_t d = ccv_nnc_tensor_symbol_new(symbolic_graph, GPU_TENSOR_NCHW(000, 16F, 20, 10), "c");
4719
1
  ccv_nnc_tensor_symbol_t e = ccv_nnc_tensor_symbol_new(symbolic_graph, GPU_TENSOR_NCHW(000, 16F, 20, 10), "c");
4720
1
  ccv_nnc_graph_exec_symbol_new(symbolic_graph, CMD_CMUL_BACKWARD(), TENSOR_SYMBOL_LIST(a, b, c), TENSOR_SYMBOL_LIST(d, e), "cmul");
4721
1
  ccv_nnc_graph_exec_symbol_autogen(symbolic_graph, 0, 0, CCV_NNC_AUTOGEN_ALL_EXECS | CCV_NNC_AUTOGEN_SOURCES_AND_DESTINATIONS);
4722
1
  SYMBOLIC_GRAPH_GEN(symbolic_graph, CCV_NNC_LONG_DOT_GRAPH);
4723
1
  ccv_nnc_graph_t* graph = 0;
4724
1
  ccv_nnc_tensor_arena_t* tensor_arena = 0;
4725
1
  ccv_nnc_graph_exec_arena_t* graph_exec_arena = 0;
4726
1
  ccv_nnc_symbolic_graph_compile(symbolic_graph, ccv_nnc_default_compile_params, 0, 0, 0, 0, SYMBOLIC_GRAPH_SOURCES(symbolic_graph), SYMBOLIC_GRAPH_DESTINATIONS(symbolic_graph), &graph, &tensor_arena, &graph_exec_arena);
4727
1
  GRAPH_GEN(graph, CCV_NNC_LONG_DOT_GRAPH);
4728
1
  ccv_nnc_tensor_t* const x_tensor = ccv_nnc_tensor_new(0, CPU_TENSOR_NCHW(32F, 20, 10), 0);
4729
1
  ccv_nnc_tensor_t* const y_tensor = ccv_nnc_tensor_new(0, CPU_TENSOR_NCHW(32F, 20, 10), 0);
4730
1
  ccv_nnc_tensor_t* const z_tensor = ccv_nnc_tensor_new(0, CPU_TENSOR_NCHW(32F, 20, 10), 0);
4731
1
  dsfmt_t dsfmt;
4732
1
  dsfmt_init_gen_rand(&dsfmt, 0);
4733
1
  int i;
4734
201
  for (i = 0; i < 20 * 10; 
i++200
)
4735
200
    x_tensor->data.f32[i] = dsfmt_genrand_open_close(&dsfmt);
4736
201
  for (i = 0; i < 20 * 10; 
i++200
)
4737
200
    y_tensor->data.f32[i] = dsfmt_genrand_open_close(&dsfmt);
4738
201
  for (i = 0; i < 20 * 10; 
i++200
)
4739
200
    z_tensor->data.f32[i] = dsfmt_genrand_open_close(&dsfmt);
4740
1
  ccv_nnc_tensor_t* const a_tensor = ccv_nnc_tensor_from_symbol(tensor_arena, a);
4741
1
  ccv_nnc_tensor_t* const x16_tensor = ccv_nnc_tensor_new(0, CPU_TENSOR_NCHW(16F, 20, 10), 0);
4742
1
  ccv_nnc_cmd_exec(CMD_DATATYPE_CONVERSION_FORWARD(), ccv_nnc_no_hint, 0, TENSOR_LIST(x_tensor), TENSOR_LIST(x16_tensor), 0);
4743
1
  ccv_nnc_cmd_exec(CMD_DATA_TRANSFER_FORWARD(), ccv_nnc_no_hint, 0, TENSOR_LIST(x16_tensor), TENSOR_LIST(a_tensor), 0);
4744
1
  ccv_nnc_tensor_t* const b_tensor = ccv_nnc_tensor_from_symbol(tensor_arena, b);
4745
1
  ccv_nnc_tensor_t* const y16_tensor = ccv_nnc_tensor_new(0, CPU_TENSOR_NCHW(16F, 20, 10), 0);
4746
1
  ccv_nnc_cmd_exec(CMD_DATATYPE_CONVERSION_FORWARD(), ccv_nnc_no_hint, 0, TENSOR_LIST(y_tensor), TENSOR_LIST(y16_tensor), 0);
4747
1
  ccv_nnc_cmd_exec(CMD_DATA_TRANSFER_FORWARD(), ccv_nnc_no_hint, 0, TENSOR_LIST(y16_tensor), TENSOR_LIST(b_tensor), 0);
4748
1
  ccv_nnc_tensor_t* const c_tensor = ccv_nnc_tensor_from_symbol(tensor_arena, c);
4749
1
  ccv_nnc_tensor_t* const z16_tensor = ccv_nnc_tensor_new(0, CPU_TENSOR_NCHW(16F, 20, 10), 0);
4750
1
  ccv_nnc_cmd_exec(CMD_DATATYPE_CONVERSION_FORWARD(), ccv_nnc_no_hint, 0, TENSOR_LIST(z_tensor), TENSOR_LIST(z16_tensor), 0);
4751
1
  ccv_nnc_cmd_exec(CMD_DATA_TRANSFER_FORWARD(), ccv_nnc_no_hint, 0, TENSOR_LIST(z16_tensor), TENSOR_LIST(c_tensor), 0);
4752
1
  ccv_nnc_graph_run(graph, 0, TRAVERSE_FULL, 0, 0);
4753
1
  ccv_nnc_tensor_t* const od16_tensor = ccv_nnc_tensor_new(0, CPU_TENSOR_NCHW(16F, 20, 10), 0);
4754
1
  ccv_nnc_tensor_t* const od_tensor = ccv_nnc_tensor_new(0, CPU_TENSOR_NCHW(32F, 20, 10), 0);
4755
1
  ccv_nnc_tensor_t* const d_tensor = ccv_nnc_tensor_from_symbol(tensor_arena, d);
4756
1
  ccv_nnc_cmd_exec(CMD_DATA_TRANSFER_FORWARD(), ccv_nnc_no_hint, 0, TENSOR_LIST(d_tensor), TENSOR_LIST(od16_tensor), 0);
4757
1
  ccv_nnc_cmd_exec(CMD_DATATYPE_CONVERSION_FORWARD(), ccv_nnc_no_hint, 0, TENSOR_LIST(od16_tensor), TENSOR_LIST(od_tensor), 0);
4758
1
  ccv_nnc_tensor_t* const oe16_tensor = ccv_nnc_tensor_new(0, CPU_TENSOR_NCHW(16F, 20, 10), 0);
4759
1
  ccv_nnc_tensor_t* const oe_tensor = ccv_nnc_tensor_new(0, CPU_TENSOR_NCHW(32F, 20, 10), 0);
4760
1
  ccv_nnc_tensor_t* const e_tensor = ccv_nnc_tensor_from_symbol(tensor_arena, e);
4761
1
  ccv_nnc_cmd_exec(CMD_DATA_TRANSFER_FORWARD(), ccv_nnc_no_hint, 0, TENSOR_LIST(e_tensor), TENSOR_LIST(oe16_tensor), 0);
4762
1
  ccv_nnc_cmd_exec(CMD_DATATYPE_CONVERSION_FORWARD(), ccv_nnc_no_hint, 0, TENSOR_LIST(oe16_tensor), TENSOR_LIST(oe_tensor), 0);
4763
1
  ccv_nnc_tensor_t* const td = ccv_nnc_tensor_new(0, CPU_TENSOR_NCHW(32F, 20, 10), 0);
4764
1
  ccv_nnc_tensor_t* const te = ccv_nnc_tensor_new(0, CPU_TENSOR_NCHW(32F, 20, 10), 0);
4765
1
  ccv_nnc_cmd_exec(CMD_CMUL_BACKWARD(), ccv_nnc_no_hint, 0, TENSOR_LIST(x_tensor, y_tensor, z_tensor), TENSOR_LIST(td, te), 0);
4766
1
  REQUIRE_ARRAY_EQ_WITH_TOLERANCE(float, td->data.f32, od_tensor->data.f32, 20 * 10, 2e-3, "gelu from cudnn should match from CPU");
4767
1
  REQUIRE_ARRAY_EQ_WITH_TOLERANCE(float, te->data.f32, oe_tensor->data.f32, 20 * 10, 2e-3, "gelu from cudnn should match from CPU");
4768
1
  ccv_nnc_tensor_free(x_tensor);
4769
1
  ccv_nnc_tensor_free(x16_tensor);
4770
1
  ccv_nnc_tensor_free(y_tensor);
4771
1
  ccv_nnc_tensor_free(y16_tensor);
4772
1
  ccv_nnc_tensor_free(z_tensor);
4773
1
  ccv_nnc_tensor_free(z16_tensor);
4774
1
  ccv_nnc_tensor_free(od_tensor);
4775
1
  ccv_nnc_tensor_free(od16_tensor);
4776
1
  ccv_nnc_tensor_free(td);
4777
1
  ccv_nnc_tensor_free(oe_tensor);
4778
1
  ccv_nnc_tensor_free(oe16_tensor);
4779
1
  ccv_nnc_tensor_free(te);
4780
1
  ccv_nnc_graph_free(graph);
4781
1
  ccv_nnc_tensor_arena_free(tensor_arena);
4782
1
  ccv_nnc_graph_exec_arena_free(graph_exec_arena);
4783
1
  ccv_nnc_symbolic_graph_free(symbolic_graph);
4784
1
}
4785
4786
TEST_CASE("segmented gemm")
4787
1
{
4788
1
  GUARD_ELSE_RETURN(ccv_nnc_cmd_ok(CCV_NNC_SEGMENTED_GEMM_FORWARD, CCV_NNC_BACKEND_GPU_CUBLAS) || ccv_nnc_cmd_ok(CCV_NNC_SEGMENTED_GEMM_FORWARD, CCV_NNC_BACKEND_MPS));
4789
1
  dsfmt_t dsfmt;
4790
1
  dsfmt_init_gen_rand(&dsfmt, 0);
4791
1
  ccv_nnc_tensor_t* ha = ccv_nnc_tensor_new(0, CPU_TENSOR_NHWC(32F, 80, 128), 0);
4792
1
  ccv_nnc_tensor_t* hindices = ccv_nnc_tensor_new(0, CPU_TENSOR_NHWC(32S, 3), 0);
4793
1
  hindices->data.i32[0] = 0;
4794
1
  hindices->data.i32[1] = 2;
4795
1
  hindices->data.i32[2] = 1;
4796
1
  ccv_nnc_tensor_t* hcounts = ccv_nnc_tensor_new(0, CPU_TENSOR_NHWC(32S, 3), 0);
4797
1
  hcounts->data.i32[0] = 20;
4798
1
  hcounts->data.i32[1] = 25;
4799
1
  hcounts->data.i32[2] = 35;
4800
1
  ccv_nnc_tensor_t* hw = ccv_nnc_tensor_new(0, CPU_TENSOR_NHWC(32F, 3, 64, 128), 0);
4801
1
  ccv_nnc_tensor_t* hb = ccv_nnc_tensor_new(0, CPU_TENSOR_NHWC(32F, 80, 64), 0);
4802
1
  ccv_nnc_tensor_t* bt = ccv_nnc_tensor_new(0, CPU_TENSOR_NHWC(32F, 80, 64), 0);
4803
1
  int i;
4804
24.5k
  for (i = 0; i < 3 * 64 * 128; 
i++24.5k
)
4805
24.5k
    hw->data.f32[i] = dsfmt_genrand_open_close(&dsfmt) / (64 * 128);
4806
10.2k
  for (i = 0; i < 80 * 128; 
i++10.2k
)
4807
10.2k
    ha->data.f32[i] = dsfmt_genrand_open_close(&dsfmt);
4808
1
  ccv_nnc_tensor_t* a = ccv_nnc_tensor_new(0, GPU_TENSOR_NHWC(000, 32F, 80, 128), 0);
4809
1
  ccv_nnc_tensor_t* indices = ccv_nnc_tensor_new(0, GPU_TENSOR_NHWC(000, 32S, 3), 0);
4810
1
  ccv_nnc_tensor_t* counts = ccv_nnc_tensor_new(0, GPU_TENSOR_NHWC(000, 32S, 3), 0);
4811
1
  ccv_nnc_tensor_t* w = ccv_nnc_tensor_new(0, GPU_TENSOR_NHWC(000, 32F, 3, 64, 128), 0);
4812
1
  ccv_nnc_tensor_t* b = ccv_nnc_tensor_new(0, GPU_TENSOR_NHWC(000, 32F, 80, 64), 0);
4813
1
  ccv_nnc_cmd_exec(CMD_DATA_TRANSFER_FORWARD(), ccv_nnc_no_hint, 0, TENSOR_LIST(ha, hindices, hcounts, hw), TENSOR_LIST(a, indices, counts, w), 0);
4814
1
  ccv_nnc_cmd_exec(CMD_SEGMENTED_GEMM_FORWARD(NO_TRANSPOSE, TRANSPOSE(1, 2)), ccv_nnc_no_hint, 0, TENSOR_LIST(a, indices, counts, w), TENSOR_LIST(b), 0);
4815
1
  ccv_nnc_cmd_exec(CMD_DATA_TRANSFER_FORWARD(), ccv_nnc_no_hint, 0, TENSOR_LIST(b), TENSOR_LIST(hb), 0);
4816
1
  ccv_nnc_cmd_exec(CMD_SEGMENTED_GEMM_FORWARD(NO_TRANSPOSE, TRANSPOSE(1, 2)), ccv_nnc_no_hint, 0, TENSOR_LIST(ha, hindices, hcounts, hw), TENSOR_LIST(bt), 0);
4817
1
  REQUIRE_TENSOR_EQ(hb, bt, "should match from CPU");
4818
1
  ccv_nnc_tensor_free(a);
4819
1
  ccv_nnc_tensor_free(indices);
4820
1
  ccv_nnc_tensor_free(counts);
4821
1
  ccv_nnc_tensor_free(w);
4822
1
  ccv_nnc_tensor_free(b);
4823
1
  ccv_nnc_tensor_free(ha);
4824
1
  ccv_nnc_tensor_free(hindices);
4825
1
  ccv_nnc_tensor_free(hcounts);
4826
1
  ccv_nnc_tensor_free(hw);
4827
1
  ccv_nnc_tensor_free(hb);
4828
1
  ccv_nnc_tensor_free(bt);
4829
1
}
4830
4831
TEST_CASE("segmented gemm with bias")
4832
1
{
4833
1
  GUARD_ELSE_RETURN(ccv_nnc_cmd_ok(CCV_NNC_SEGMENTED_GEMM_FORWARD, CCV_NNC_BACKEND_GPU_CUBLAS) || ccv_nnc_cmd_ok(CCV_NNC_SEGMENTED_GEMM_FORWARD, CCV_NNC_BACKEND_MPS));
4834
1
  dsfmt_t dsfmt;
4835
1
  dsfmt_init_gen_rand(&dsfmt, 0);
4836
1
  ccv_nnc_tensor_t* ha = ccv_nnc_tensor_new(0, CPU_TENSOR_NHWC(32F, 80, 128), 0);
4837
1
  ccv_nnc_tensor_t* hindices = ccv_nnc_tensor_new(0, CPU_TENSOR_NHWC(32S, 3), 0);
4838
1
  hindices->data.i32[0] = 0;
4839
1
  hindices->data.i32[1] = 1;
4840
1
  hindices->data.i32[2] = 2;
4841
1
  ccv_nnc_tensor_t* hcounts = ccv_nnc_tensor_new(0, CPU_TENSOR_NHWC(32S, 3), 0);
4842
1
  hcounts->data.i32[0] = 20;
4843
1
  hcounts->data.i32[1] = 25;
4844
1
  hcounts->data.i32[2] = 35;
4845
1
  ccv_nnc_tensor_t* hw = ccv_nnc_tensor_new(0, CPU_TENSOR_NHWC(32F, 3, 64, 128), 0);
4846
1
  ccv_nnc_tensor_t* hbias = ccv_nnc_tensor_new(0, CPU_TENSOR_NHWC(32F, 3, 64), 0);
4847
1
  ccv_nnc_tensor_t* hb = ccv_nnc_tensor_new(0, CPU_TENSOR_NHWC(32F, 80, 64), 0);
4848
1
  ccv_nnc_tensor_t* bt = ccv_nnc_tensor_new(0, CPU_TENSOR_NHWC(32F, 80, 64), 0);
4849
1
  int i;
4850
24.5k
  for (i = 0; i < 3 * 64 * 128; 
i++24.5k
)
4851
24.5k
    hw->data.f32[i] = dsfmt_genrand_open_close(&dsfmt) / (64 * 128);
4852
193
  for (i = 0; i < 3 * 64; 
i++192
)
4853
192
    hbias->data.f32[i] = dsfmt_genrand_open_close(&dsfmt) / 64;
4854
10.2k
  for (i = 0; i < 80 * 128; 
i++10.2k
)
4855
10.2k
    ha->data.f32[i] = dsfmt_genrand_open_close(&dsfmt);
4856
1
  ccv_nnc_tensor_t* a = ccv_nnc_tensor_new(0, GPU_TENSOR_NHWC(000, 32F, 80, 128), 0);
4857
1
  ccv_nnc_tensor_t* indices = ccv_nnc_tensor_new(0, GPU_TENSOR_NHWC(000, 32S, 3), 0);
4858
1
  ccv_nnc_tensor_t* counts = ccv_nnc_tensor_new(0, GPU_TENSOR_NHWC(000, 32S, 3), 0);
4859
1
  ccv_nnc_tensor_t* w = ccv_nnc_tensor_new(0, GPU_TENSOR_NHWC(000, 32F, 3, 64, 128), 0);
4860
1
  ccv_nnc_tensor_t* bias = ccv_nnc_tensor_new(0, GPU_TENSOR_NHWC(000, 32F, 3, 64), 0);
4861
1
  ccv_nnc_tensor_t* b = ccv_nnc_tensor_new(0, GPU_TENSOR_NHWC(000, 32F, 80, 64), 0);
4862
1
  ccv_nnc_cmd_exec(CMD_DATA_TRANSFER_FORWARD(), ccv_nnc_no_hint, 0, TENSOR_LIST(ha, hindices, hcounts, hw, hbias), TENSOR_LIST(a, indices, counts, w, bias), 0);
4863
1
  ccv_nnc_cmd_exec(CMD_SEGMENTED_GEMM_FORWARD(NO_TRANSPOSE, TRANSPOSE(1, 2)), ccv_nnc_no_hint, 0, TENSOR_LIST(a, indices, counts, w, bias), TENSOR_LIST(b), 0);
4864
1
  ccv_nnc_cmd_exec(CMD_DATA_TRANSFER_FORWARD(), ccv_nnc_no_hint, 0, TENSOR_LIST(b), TENSOR_LIST(hb), 0);
4865
1
  ccv_nnc_cmd_exec(CMD_SEGMENTED_GEMM_FORWARD(NO_TRANSPOSE, TRANSPOSE(1, 2)), ccv_nnc_no_hint, 0, TENSOR_LIST(ha, hindices, hcounts, hw, hbias), TENSOR_LIST(bt), 0);
4866
1
  REQUIRE_TENSOR_EQ(hb, bt, "should match from CPU");
4867
1
  ccv_nnc_tensor_free(a);
4868
1
  ccv_nnc_tensor_free(indices);
4869
1
  ccv_nnc_tensor_free(counts);
4870
1
  ccv_nnc_tensor_free(w);
4871
1
  ccv_nnc_tensor_free(bias);
4872
1
  ccv_nnc_tensor_free(b);
4873
1
  ccv_nnc_tensor_free(ha);
4874
1
  ccv_nnc_tensor_free(hindices);
4875
1
  ccv_nnc_tensor_free(hcounts);
4876
1
  ccv_nnc_tensor_free(hw);
4877
1
  ccv_nnc_tensor_free(hbias);
4878
1
  ccv_nnc_tensor_free(hb);
4879
1
  ccv_nnc_tensor_free(bt);
4880
1
}
4881
4882
TEST_CASE("segmented gemm in half precision")
4883
1
{
4884
1
  GUARD_ELSE_RETURN(ccv_nnc_cmd_ok(CCV_NNC_SEGMENTED_GEMM_FORWARD, CCV_NNC_BACKEND_GPU_CUBLAS) || ccv_nnc_cmd_ok(CCV_NNC_SEGMENTED_GEMM_FORWARD, CCV_NNC_BACKEND_MPS));
4885
1
  dsfmt_t dsfmt;
4886
1
  dsfmt_init_gen_rand(&dsfmt, 0);
4887
1
  ccv_nnc_tensor_t* ha = ccv_nnc_tensor_new(0, CPU_TENSOR_NHWC(32F, 80, 128), 0);
4888
1
  ccv_nnc_tensor_t* hindices = ccv_nnc_tensor_new(0, CPU_TENSOR_NHWC(32S, 3), 0);
4889
1
  hindices->data.i32[0] = 0;
4890
1
  hindices->data.i32[1] = 2;
4891
1
  hindices->data.i32[2] = 1;
4892
1
  ccv_nnc_tensor_t* hcounts = ccv_nnc_tensor_new(0, CPU_TENSOR_NHWC(32S, 3), 0);
4893
1
  hcounts->data.i32[0] = 20;
4894
1
  hcounts->data.i32[1] = 25;
4895
1
  hcounts->data.i32[2] = 35;
4896
1
  ccv_nnc_tensor_t* hw = ccv_nnc_tensor_new(0, CPU_TENSOR_NHWC(32F, 3, 64, 128), 0);
4897
1
  ccv_nnc_tensor_t* hb = ccv_nnc_tensor_new(0, CPU_TENSOR_NHWC(32F, 80, 64), 0);
4898
1
  ccv_nnc_tensor_t* hb16 = ccv_nnc_tensor_new(0, CPU_TENSOR_NHWC(16F, 80, 64), 0);
4899
1
  ccv_nnc_tensor_t* bt = ccv_nnc_tensor_new(0, CPU_TENSOR_NHWC(32F, 80, 64), 0);
4900
1
  int i;
4901
24.5k
  for (i = 0; i < 3 * 64 * 128; 
i++24.5k
)
4902
24.5k
    hw->data.f32[i] = dsfmt_genrand_open_close(&dsfmt) / (64 * 128);
4903
10.2k
  for (i = 0; i < 80 * 128; 
i++10.2k
)
4904
10.2k
    ha->data.f32[i] = dsfmt_genrand_open_close(&dsfmt);
4905
1
  ccv_nnc_tensor_t* ha16 = ccv_nnc_tensor_new(0, CPU_TENSOR_NHWC(16F, 80, 128), 0);
4906
1
  ccv_nnc_tensor_t* hw16 = ccv_nnc_tensor_new(0, CPU_TENSOR_NHWC(16F, 3, 64, 128), 0);
4907
1
  ccv_nnc_cmd_exec(CMD_DATATYPE_CONVERSION_FORWARD(), ccv_nnc_no_hint, 0, TENSOR_LIST(ha, hw), TENSOR_LIST(ha16, hw16), 0);
4908
1
  ccv_nnc_tensor_t* a = ccv_nnc_tensor_new(0, GPU_TENSOR_NHWC(000, 16F, 80, 128), 0);
4909
1
  ccv_nnc_tensor_t* indices = ccv_nnc_tensor_new(0, GPU_TENSOR_NHWC(000, 32S, 3), 0);
4910
1
  ccv_nnc_tensor_t* counts = ccv_nnc_tensor_new(0, GPU_TENSOR_NHWC(000, 32S, 3), 0);
4911
1
  ccv_nnc_tensor_t* w = ccv_nnc_tensor_new(0, GPU_TENSOR_NHWC(000, 16F, 3, 64, 128), 0);
4912
1
  ccv_nnc_tensor_t* b = ccv_nnc_tensor_new(0, GPU_TENSOR_NHWC(000, 16F, 80, 64), 0);
4913
1
  ccv_nnc_cmd_exec(CMD_DATA_TRANSFER_FORWARD(), ccv_nnc_no_hint, 0, TENSOR_LIST(ha16, hindices, hcounts, hw16), TENSOR_LIST(a, indices, counts, w), 0);
4914
1
  ccv_nnc_cmd_exec(CMD_SEGMENTED_GEMM_FORWARD(NO_TRANSPOSE, TRANSPOSE(1, 2)), ccv_nnc_no_hint, 0, TENSOR_LIST(a, indices, counts, w), TENSOR_LIST(b), 0);
4915
1
  ccv_nnc_cmd_exec(CMD_DATA_TRANSFER_FORWARD(), ccv_nnc_no_hint, 0, TENSOR_LIST(b), TENSOR_LIST(hb16), 0);
4916
1
  ccv_nnc_cmd_exec(CMD_DATATYPE_CONVERSION_FORWARD(), ccv_nnc_no_hint, 0, TENSOR_LIST(hb16), TENSOR_LIST(hb), 0);
4917
1
  ccv_nnc_cmd_exec(CMD_SEGMENTED_GEMM_FORWARD(NO_TRANSPOSE, TRANSPOSE(1, 2)), ccv_nnc_no_hint, 0, TENSOR_LIST(ha, hindices, hcounts, hw), TENSOR_LIST(bt), 0);
4918
1
  REQUIRE_ARRAY_EQ_WITH_TOLERANCE(float, hb->data.f32, bt->data.f32, 80 * 64, 1e-3, "should match from CPU");
4919
1
  ccv_nnc_tensor_free(a);
4920
1
  ccv_nnc_tensor_free(indices);
4921
1
  ccv_nnc_tensor_free(counts);
4922
1
  ccv_nnc_tensor_free(w);
4923
1
  ccv_nnc_tensor_free(b);
4924
1
  ccv_nnc_tensor_free(ha);
4925
1
  ccv_nnc_tensor_free(hindices);
4926
1
  ccv_nnc_tensor_free(hcounts);
4927
1
  ccv_nnc_tensor_free(hw);
4928
1
  ccv_nnc_tensor_free(hb);
4929
1
  ccv_nnc_tensor_free(ha16);
4930
1
  ccv_nnc_tensor_free(hw16);
4931
1
  ccv_nnc_tensor_free(hb16);
4932
1
  ccv_nnc_tensor_free(bt);
4933
1
}
4934
4935
TEST_CASE("segmented gemm with bias in half precision")
4936
1
{
4937
1
  GUARD_ELSE_RETURN(ccv_nnc_cmd_ok(CCV_NNC_SEGMENTED_GEMM_FORWARD, CCV_NNC_BACKEND_GPU_CUBLAS) || ccv_nnc_cmd_ok(CCV_NNC_SEGMENTED_GEMM_FORWARD, CCV_NNC_BACKEND_MPS));
4938
1
  dsfmt_t dsfmt;
4939
1
  dsfmt_init_gen_rand(&dsfmt, 0);
4940
1
  ccv_nnc_tensor_t* ha = ccv_nnc_tensor_new(0, CPU_TENSOR_NHWC(32F, 80, 128), 0);
4941
1
  ccv_nnc_tensor_t* hindices = ccv_nnc_tensor_new(0, CPU_TENSOR_NHWC(32S, 3), 0);
4942
1
  hindices->data.i32[0] = 0;
4943
1
  hindices->data.i32[1] = 1;
4944
1
  hindices->data.i32[2] = 2;
4945
1
  ccv_nnc_tensor_t* hcounts = ccv_nnc_tensor_new(0, CPU_TENSOR_NHWC(32S, 3), 0);
4946
1
  hcounts->data.i32[0] = 20;
4947
1
  hcounts->data.i32[1] = 25;
4948
1
  hcounts->data.i32[2] = 35;
4949
1
  ccv_nnc_tensor_t* hw = ccv_nnc_tensor_new(0, CPU_TENSOR_NHWC(32F, 3, 64, 128), 0);
4950
1
  ccv_nnc_tensor_t* hbias = ccv_nnc_tensor_new(0, CPU_TENSOR_NHWC(32F, 3, 64), 0);
4951
1
  ccv_nnc_tensor_t* hb = ccv_nnc_tensor_new(0, CPU_TENSOR_NHWC(32F, 80, 64), 0);
4952
1
  ccv_nnc_tensor_t* hb16 = ccv_nnc_tensor_new(0, CPU_TENSOR_NHWC(16F, 80, 64), 0);
4953
1
  ccv_nnc_tensor_t* bt = ccv_nnc_tensor_new(0, CPU_TENSOR_NHWC(32F, 80, 64), 0);
4954
1
  int i;
4955
24.5k
  for (i = 0; i < 3 * 64 * 128; 
i++24.5k
)
4956
24.5k
    hw->data.f32[i] = dsfmt_genrand_open_close(&dsfmt) / (64 * 128);
4957
193
  for (i = 0; i < 3 * 64; 
i++192
)
4958
192
    hbias->data.f32[i] = dsfmt_genrand_open_close(&dsfmt) / 64;
4959
10.2k
  for (i = 0; i < 80 * 128; 
i++10.2k
)
4960
10.2k
    ha->data.f32[i] = dsfmt_genrand_open_close(&dsfmt);
4961
1
  ccv_nnc_tensor_t* ha16 = ccv_nnc_tensor_new(0, CPU_TENSOR_NHWC(16F, 80, 128), 0);
4962
1
  ccv_nnc_tensor_t* hw16 = ccv_nnc_tensor_new(0, CPU_TENSOR_NHWC(16F, 3, 64, 128), 0);
4963
1
  ccv_nnc_tensor_t* hbias16 = ccv_nnc_tensor_new(0, CPU_TENSOR_NHWC(16F, 3, 64), 0);
4964
1
  ccv_nnc_cmd_exec(CMD_DATATYPE_CONVERSION_FORWARD(), ccv_nnc_no_hint, 0, TENSOR_LIST(ha, hw, hbias), TENSOR_LIST(ha16, hw16, hbias16), 0);
4965
1
  ccv_nnc_tensor_t* a = ccv_nnc_tensor_new(0, GPU_TENSOR_NHWC(000, 16F, 80, 128), 0);
4966
1
  ccv_nnc_tensor_t* indices = ccv_nnc_tensor_new(0, GPU_TENSOR_NHWC(000, 32S, 3), 0);
4967
1
  ccv_nnc_tensor_t* counts = ccv_nnc_tensor_new(0, GPU_TENSOR_NHWC(000, 32S, 3), 0);
4968
1
  ccv_nnc_tensor_t* w = ccv_nnc_tensor_new(0, GPU_TENSOR_NHWC(000, 16F, 3, 64, 128), 0);
4969
1
  ccv_nnc_tensor_t* bias = ccv_nnc_tensor_new(0, GPU_TENSOR_NHWC(000, 16F, 3, 64), 0);
4970
1
  ccv_nnc_tensor_t* b = ccv_nnc_tensor_new(0, GPU_TENSOR_NHWC(000, 16F, 80, 64), 0);
4971
1
  ccv_nnc_cmd_exec(CMD_DATA_TRANSFER_FORWARD(), ccv_nnc_no_hint, 0, TENSOR_LIST(ha16, hindices, hcounts, hw16, hbias16), TENSOR_LIST(a, indices, counts, w, bias), 0);
4972
1
  ccv_nnc_cmd_exec(CMD_SEGMENTED_GEMM_FORWARD(NO_TRANSPOSE, TRANSPOSE(1, 2)), ccv_nnc_no_hint, 0, TENSOR_LIST(a, indices, counts, w, bias), TENSOR_LIST(b), 0);
4973
1
  ccv_nnc_cmd_exec(CMD_DATA_TRANSFER_FORWARD(), ccv_nnc_no_hint, 0, TENSOR_LIST(b), TENSOR_LIST(hb16), 0);
4974
1
  ccv_nnc_cmd_exec(CMD_DATATYPE_CONVERSION_FORWARD(), ccv_nnc_no_hint, 0, TENSOR_LIST(hb16), TENSOR_LIST(hb), 0);
4975
1
  ccv_nnc_cmd_exec(CMD_SEGMENTED_GEMM_FORWARD(NO_TRANSPOSE, TRANSPOSE(1, 2)), ccv_nnc_no_hint, 0, TENSOR_LIST(ha, hindices, hcounts, hw, hbias), TENSOR_LIST(bt), 0);
4976
1
  REQUIRE_ARRAY_EQ_WITH_TOLERANCE(float, hb->data.f32, bt->data.f32, 80 * 64, 1e-3, "should match from CPU");
4977
1
  ccv_nnc_tensor_free(a);
4978
1
  ccv_nnc_tensor_free(indices);
4979
1
  ccv_nnc_tensor_free(counts);
4980
1
  ccv_nnc_tensor_free(w);
4981
1
  ccv_nnc_tensor_free(bias);
4982
1
  ccv_nnc_tensor_free(b);
4983
1
  ccv_nnc_tensor_free(ha);
4984
1
  ccv_nnc_tensor_free(hindices);
4985
1
  ccv_nnc_tensor_free(hcounts);
4986
1
  ccv_nnc_tensor_free(hw);
4987
1
  ccv_nnc_tensor_free(hbias);
4988
1
  ccv_nnc_tensor_free(hb);
4989
1
  ccv_nnc_tensor_free(ha16);
4990
1
  ccv_nnc_tensor_free(hw16);
4991
1
  ccv_nnc_tensor_free(hbias16);
4992
1
  ccv_nnc_tensor_free(hb16);
4993
1
  ccv_nnc_tensor_free(bt);
4994
1
}
4995
4996
TEST_CASE("segmented gemm, reuse")
4997
1
{
4998
1
  GUARD_ELSE_RETURN(ccv_nnc_cmd_ok(CCV_NNC_SEGMENTED_GEMM_FORWARD, CCV_NNC_BACKEND_GPU_CUBLAS) || ccv_nnc_cmd_ok(CCV_NNC_SEGMENTED_GEMM_FORWARD, CCV_NNC_BACKEND_MPS));
4999
1
  dsfmt_t dsfmt;
5000
1
  dsfmt_init_gen_rand(&dsfmt, 0);
5001
1
  ccv_nnc_tensor_t* ha = ccv_nnc_tensor_new(0, CPU_TENSOR_NHWC(32F, 80, 128), 0);
5002
1
  ccv_nnc_tensor_t* hindices = ccv_nnc_tensor_new(0, CPU_TENSOR_NHWC(32S, 3), 0);
5003
1
  hindices->data.i32[0] = 0;
5004
1
  hindices->data.i32[1] = 1;
5005
1
  hindices->data.i32[2] = 2;
5006
1
  ccv_nnc_tensor_t* hcounts = ccv_nnc_tensor_new(0, CPU_TENSOR_NHWC(32S, 3), 0);
5007
1
  hcounts->data.i32[0] = 20;
5008
1
  hcounts->data.i32[1] = 30;
5009
1
  hcounts->data.i32[2] = 30;
5010
1
  ccv_nnc_tensor_t* hw = ccv_nnc_tensor_new(0, CPU_TENSOR_NHWC(32F, 3, 64, 128), 0);
5011
1
  ccv_nnc_tensor_t* hb = ccv_nnc_tensor_new(0, CPU_TENSOR_NHWC(32F, 80, 64), 0);
5012
1
  ccv_nnc_tensor_t* bt = ccv_nnc_tensor_new(0, CPU_TENSOR_NHWC(32F, 80, 64), 0);
5013
1
  int i;
5014
24.5k
  for (i = 0; i < 3 * 64 * 128; 
i++24.5k
)
5015
24.5k
    hw->data.f32[i] = dsfmt_genrand_open_close(&dsfmt) / (64 * 128);
5016
10.2k
  for (i = 0; i < 80 * 128; 
i++10.2k
)
5017
10.2k
    ha->data.f32[i] = dsfmt_genrand_open_close(&dsfmt);
5018
1
  ccv_nnc_tensor_t* a = ccv_nnc_tensor_new(0, GPU_TENSOR_NHWC(000, 32F, 80, 128), 0);
5019
1
  ccv_nnc_tensor_t* indices = ccv_nnc_tensor_new(0, GPU_TENSOR_NHWC(000, 32S, 3), 0);
5020
1
  ccv_nnc_tensor_t* counts = ccv_nnc_tensor_new(0, GPU_TENSOR_NHWC(000, 32S, 3), 0);
5021
1
  ccv_nnc_tensor_t* w = ccv_nnc_tensor_new(0, GPU_TENSOR_NHWC(000, 32F, 3, 64, 128), 0);
5022
1
  ccv_nnc_tensor_t* b = ccv_nnc_tensor_new(0, GPU_TENSOR_NHWC(000, 32F, 80, 64), 0);
5023
1
  ccv_nnc_cmd_exec(CMD_DATA_TRANSFER_FORWARD(), ccv_nnc_no_hint, 0, TENSOR_LIST(ha, hindices, hcounts, hw), TENSOR_LIST(a, indices, counts, w), 0);
5024
1
  ccv_nnc_cmd_exec(CMD_SEGMENTED_GEMM_FORWARD(NO_TRANSPOSE, TRANSPOSE(1, 2)), ccv_nnc_no_hint, 0, TENSOR_LIST(a, indices, counts, w), TENSOR_LIST(b), 0);
5025
1
  ccv_nnc_cmd_exec(CMD_DATA_TRANSFER_FORWARD(), ccv_nnc_no_hint, 0, TENSOR_LIST(b), TENSOR_LIST(hb), 0);
5026
1
  ccv_nnc_cmd_exec(CMD_SEGMENTED_GEMM_FORWARD(NO_TRANSPOSE, TRANSPOSE(1, 2)), ccv_nnc_no_hint, 0, TENSOR_LIST(ha, hindices, hcounts, hw), TENSOR_LIST(bt), 0);
5027
1
  REQUIRE_TENSOR_EQ(hb, bt, "should match from CPU");
5028
1
  ccv_nnc_tensor_free(a);
5029
1
  ccv_nnc_tensor_free(indices);
5030
1
  ccv_nnc_tensor_free(counts);
5031
1
  ccv_nnc_tensor_free(w);
5032
1
  ccv_nnc_tensor_free(b);
5033
1
  ccv_nnc_tensor_free(ha);
5034
1
  ccv_nnc_tensor_free(hindices);
5035
1
  ccv_nnc_tensor_free(hcounts);
5036
1
  ccv_nnc_tensor_free(hw);
5037
1
  ccv_nnc_tensor_free(hb);
5038
1
  ccv_nnc_tensor_free(bt);
5039
1
}
5040
5041
TEST_CASE("segmented gemm, large k")
5042
1
{
5043
1
  GUARD_ELSE_RETURN(ccv_nnc_cmd_ok(CCV_NNC_SEGMENTED_GEMM_FORWARD, CCV_NNC_BACKEND_GPU_CUBLAS) || ccv_nnc_cmd_ok(CCV_NNC_SEGMENTED_GEMM_FORWARD, CCV_NNC_BACKEND_MPS));
5044
1
  dsfmt_t dsfmt;
5045
1
  dsfmt_init_gen_rand(&dsfmt, 0);
5046
1
  ccv_nnc_tensor_t* ha = ccv_nnc_tensor_new(0, CPU_TENSOR_NHWC(32F, 80, 5120), 0);
5047
1
  ccv_nnc_tensor_t* hindices = ccv_nnc_tensor_new(0, CPU_TENSOR_NHWC(32S, 3), 0);
5048
1
  hindices->data.i32[0] = 0;
5049
1
  hindices->data.i32[1] = 1;
5050
1
  hindices->data.i32[2] = 2;
5051
1
  ccv_nnc_tensor_t* hcounts = ccv_nnc_tensor_new(0, CPU_TENSOR_NHWC(32S, 3), 0);
5052
1
  hcounts->data.i32[0] = 20;
5053
1
  hcounts->data.i32[1] = 30;
5054
1
  hcounts->data.i32[2] = 30;
5055
1
  ccv_nnc_tensor_t* hw = ccv_nnc_tensor_new(0, CPU_TENSOR_NHWC(32F, 3, 2560, 5120), 0);
5056
1
  ccv_nnc_tensor_t* hb = ccv_nnc_tensor_new(0, CPU_TENSOR_NHWC(32F, 80, 2560), 0);
5057
1
  ccv_nnc_tensor_t* bt = ccv_nnc_tensor_new(0, CPU_TENSOR_NHWC(32F, 80, 2560), 0);
5058
1
  int i;
5059
39.3M
  for (i = 0; i < 3 * 2560 * 5120; 
i++39.3M
)
5060
39.3M
    hw->data.f32[i] = dsfmt_genrand_open_close(&dsfmt) / 5120;
5061
409k
  for (i = 0; i < 80 * 5120; 
i++409k
)
5062
409k
    ha->data.f32[i] = dsfmt_genrand_open_close(&dsfmt);
5063
1
  ccv_nnc_tensor_t* a = ccv_nnc_tensor_new(0, GPU_TENSOR_NHWC(000, 32F, 80, 5120), 0);
5064
1
  ccv_nnc_tensor_t* indices = ccv_nnc_tensor_new(0, GPU_TENSOR_NHWC(000, 32S, 3), 0);
5065
1
  ccv_nnc_tensor_t* counts = ccv_nnc_tensor_new(0, GPU_TENSOR_NHWC(000, 32S, 3), 0);
5066
1
  ccv_nnc_tensor_t* w = ccv_nnc_tensor_new(0, GPU_TENSOR_NHWC(000, 32F, 3, 2560, 5120), 0);
5067
1
  ccv_nnc_tensor_t* b = ccv_nnc_tensor_new(0, GPU_TENSOR_NHWC(000, 32F, 80, 2560), 0);
5068
1
  ccv_nnc_cmd_exec(CMD_DATA_TRANSFER_FORWARD(), ccv_nnc_no_hint, 0, TENSOR_LIST(ha, hindices, hcounts, hw), TENSOR_LIST(a, indices, counts, w), 0);
5069
1
  ccv_nnc_cmd_exec(CMD_SEGMENTED_GEMM_FORWARD(NO_TRANSPOSE, TRANSPOSE(1, 2)), ccv_nnc_no_hint, 0, TENSOR_LIST(a, indices, counts, w), TENSOR_LIST(b), 0);
5070
1
  ccv_nnc_cmd_exec(CMD_DATA_TRANSFER_FORWARD(), ccv_nnc_no_hint, 0, TENSOR_LIST(b), TENSOR_LIST(hb), 0);
5071
1
  ccv_nnc_cmd_exec(CMD_SEGMENTED_GEMM_FORWARD(NO_TRANSPOSE, TRANSPOSE(1, 2)), ccv_nnc_no_hint, 0, TENSOR_LIST(ha, hindices, hcounts, hw), TENSOR_LIST(bt), 0);
5072
1
  REQUIRE_TENSOR_EQ(hb, bt, "should match from CPU");
5073
1
  ccv_nnc_tensor_free(a);
5074
1
  ccv_nnc_tensor_free(indices);
5075
1
  ccv_nnc_tensor_free(counts);
5076
1
  ccv_nnc_tensor_free(w);
5077
1
  ccv_nnc_tensor_free(b);
5078
1
  ccv_nnc_tensor_free(ha);
5079
1
  ccv_nnc_tensor_free(hindices);
5080
1
  ccv_nnc_tensor_free(hcounts);
5081
1
  ccv_nnc_tensor_free(hw);
5082
1
  ccv_nnc_tensor_free(hb);
5083
1
  ccv_nnc_tensor_free(bt);
5084
1
}
5085
5086
#include "case_main.h"