Coverage Report

Created: 2021-04-14 04:30

/home/liu/buildslave/linux-x64-runtests/build/lib/nnc/cmd/util/ccv_nnc_util.c
Line
Count
Source (jump to first uncovered line)
1
#include "ccv.h"
2
#include "nnc/ccv_nnc.h"
3
#include "nnc/ccv_nnc_internal.h"
4
5
static int _ccv_nnc_set_bitmask(const int input_size, const int output_size, const uint64_t* const input_bitmasks, const int input_bitmask_size, const uint64_t* const output_bitmasks, const int output_bitmask_size)
6
1
{
7
1
  int i, j, flag = 0;
8
1
  int output_bitcount = 0;
9
2
  for (i = 0; i < output_bitmask_size; 
i++1
)
10
1
  {
11
2
    for (j = 0; j < 64; 
j++1
)
12
2
      if (output_bitmasks[i] & (uint64_t)1 << j)
13
1
      {
14
1
        if (flag)
15
0
          return 0;
16
1
      } else
17
1
        break;
18
1
    output_bitcount += j;
19
1
    // Trailing zero even if it is not the end of input_bitmask_size, mark flag,
20
1
    // if we encounter additional 1, return invalid.
21
1
    if (j < 64)
22
1
      flag = 1;
23
1
    // Always like 1111100000, no 1110010101
24
64
    for (; j < 64; 
j++63
)
25
63
      if (output_bitmasks[i] & (uint64_t)1 << j)
26
0
        return 0;
27
1
  }
28
1
  return output_size == output_bitcount;
29
1
}
30
31
REGISTER_COMMAND(CCV_NNC_SET_FORWARD)(ccv_nnc_cmd_registry_t* const registry)
32
  FIND_BACKEND(ccv_nnc_util_cpu_ref.c, gpu/ccv_nnc_util_gpu_cudnn.cu)
33
1
{
34
1
  registry->bitmask = _ccv_nnc_set_bitmask;
35
1
  registry->tensor_auto = ccv_nnc_hint_tensor_auto_forward_from_inputs;
36
1
}
37
38
REGISTER_COMMAND(CCV_NNC_SET_BACKWARD)(ccv_nnc_cmd_registry_t* const registry)
39
  FIND_BACKEND(ccv_nnc_util_cpu_ref.c, gpu/ccv_nnc_util_gpu_cudnn.cu)
40
1
{
41
1
  registry->bitmask = _ccv_nnc_set_bitmask;
42
1
  registry->tensor_auto = ccv_nnc_hint_tensor_auto_forward_from_inputs;
43
1
}
44
45
//@REGISTER_EASY_COMMAND_MACRO(CCV_NNC_SET_FORWARD)
46
#define CMD_SET_FORWARD(_val) ccv_nnc_cmd(CCV_NNC_SET_FORWARD, 0, (ccv_nnc_cmd_param_t){.size={.dim={1,1,1}},.blas={.a={_val,}}}, 0)
47
//@REGISTER_EASY_COMMAND_MACRO(CCV_NNC_SET_BACKWARD)
48
#define CMD_SET_BACKWARD(_val) ccv_nnc_cmd(CCV_NNC_SET_BACKWARD, 0, (ccv_nnc_cmd_param_t){.size={.dim={1,1,1}},.blas={.a={_val,}}}, 0)
49
50
static int _ccv_nnc_masked_fill_forw_bitmask(const int input_size, const int output_size, const uint64_t* const input_bitmasks, const int input_bitmask_size, const uint64_t* const output_bitmasks, const int output_bitmask_size)
51
36
{
52
36
  if (input_size == 2 && (input_bitmasks[0] & 3u) == ((1u << 0) | (1u << 1)) && 
output_bitmasks[0] == 1u12
)
53
12
    return 1;
54
24
  return 0;
55
24
}
56
57
static int _ccv_nnc_masked_fill_back_bitmask(const int input_size, const int output_size, const uint64_t* const input_bitmasks, const int input_bitmask_size, const uint64_t* const output_bitmasks, const int output_bitmask_size)
58
48
{
59
48
  if ((input_bitmasks[0] & 5u) == ((1u << 0) | (0u << 1) | (1u << 2)) && 
output_bitmasks[0] == ((1u << 0) | (1u << 1))24
)
60
6
    return 1;
61
42
  if ((input_bitmasks[0] & 5u) == ((1u << 0) | (0u << 1) | (1u << 2)) && 
output_bitmasks[0] == (1u << 0)18
)
62
18
    return 1;
63
24
  return 0;
64
24
}
65
66
REGISTER_COMMAND(CCV_NNC_MASKED_FILL_FORWARD)(ccv_nnc_cmd_registry_t* const registry)
67
  FIND_BACKEND(ccv_nnc_util_cpu_ref.c, gpu/ccv_nnc_util_gpu_ref.cu)
68
1
{
69
1
  registry->bitmask = _ccv_nnc_masked_fill_forw_bitmask;
70
1
  registry->tensor_auto = ccv_nnc_hint_tensor_auto_forward_from_inputs;
71
1
}
72
73
REGISTER_COMMAND(CCV_NNC_MASKED_FILL_BACKWARD)(ccv_nnc_cmd_registry_t* const registry)
74
  FIND_BACKEND(ccv_nnc_util_cpu_ref.c, gpu/ccv_nnc_util_gpu_ref.cu)
75
1
{
76
1
  registry->bitmask = _ccv_nnc_masked_fill_back_bitmask;
77
1
  registry->tensor_auto = ccv_nnc_hint_tensor_auto_backward_from_gradient_and_inputs;
78
1
}
79
80
//@REGISTER_EASY_COMMAND_MACRO(CCV_NNC_MASKED_FILL_FORWARD)
81
#define CMD_MASKED_FILL_FORWARD(_eq, _fill) ccv_nnc_cmd(CCV_NNC_MASKED_FILL_FORWARD, 0, (ccv_nnc_cmd_param_t){.size={.dim={1,1,1}},.blas={.a={_eq, _fill}}}, 0)
82
//@REGISTER_EASY_COMMAND_MACRO(CCV_NNC_MASKED_FILL_BACKWARD)
83
#define CMD_MASKED_FILL_BACKWARD(_eq, _fill) ccv_nnc_cmd(CCV_NNC_MASKED_FILL_BACKWARD, 0, (ccv_nnc_cmd_param_t){.size={.dim={1,1,1}},.blas={.a={_eq, _fill}}}, 0)
84
85
static int _ccv_nnc_data_transfer_forw_bitmask(const int input_size, const int output_size, const uint64_t* const input_bitmasks, const int input_bitmask_size, const uint64_t* const output_bitmasks, const int output_bitmask_size)
86
174
{
87
174
  int i, j;
88
174
  int input_flag = 0;
89
174
  int input_bitcount = 0;
90
332
  for (i = 0; i < input_bitmask_size; 
i++158
)
91
174
  {
92
285
    for (j = 0; j < 64; 
j++111
)
93
285
      if (input_bitmasks[i] & (uint64_t)1 << j)
94
111
      {
95
111
        if (input_flag)
96
0
          return 0;
97
174
      } else
98
174
        break;
99
174
    input_bitcount += j;
100
174
    if (j < 64)
101
174
      input_flag = 1;
102
174
    // Always like 1111100000, no 1110010101
103
10.1k
    for (; j < 64; 
j++10.0k
)
104
10.0k
      if (input_bitmasks[i] & (uint64_t)1 << j)
105
16
        return 0;
106
174
  }
107
174
  int output_flag = 0;
108
158
  int output_bitcount = 0;
109
316
  for (i = 0; i < output_bitmask_size; 
i++158
)
110
158
  {
111
348
    for (j = 0; j < 64; 
j++190
)
112
348
      if (output_bitmasks[i] & (uint64_t)1 << j)
113
190
      {
114
190
        if (output_flag)
115
0
          return 0;
116
158
      } else
117
158
        break;
118
158
    output_bitcount += j;
119
158
    if (j < 64)
120
158
      output_flag = 1;
121
10.0k
    for (; j < 64; 
j++9.92k
)
122
9.92k
      if (output_bitmasks[i] & (uint64_t)1 << j)
123
0
        return 0;
124
158
  }
125
158
  return output_bitcount == input_bitcount && 
input_size == output_size79
&&
input_size == input_bitcount79
;
126
158
}
127
128
static int _ccv_nnc_data_transfer_back_bitmask(const int input_size, const int output_size, const uint64_t* const input_bitmasks, const int input_bitmask_size, const uint64_t* const output_bitmasks, const int output_bitmask_size)
129
308
{
130
308
  int i, j;
131
308
  int input_flag = 0;
132
308
  int input_bitcount = 0;
133
471
  for (i = 0; i < input_bitmask_size; 
i++163
)
134
308
  {
135
674
    for (j = 0; j < 64; 
j++366
)
136
674
      if (input_bitmasks[i] & (uint64_t)1 << j)
137
366
      {
138
366
        if (input_flag)
139
0
          return 0;
140
308
      } else
141
308
        break;
142
308
    input_bitcount += j;
143
308
    if (j < 64)
144
308
      input_flag = 1;
145
308
    // Always like 1111100000, no 1110010101
146
10.6k
    for (; j < 64; 
j++10.3k
)
147
10.4k
      if (input_bitmasks[i] & (uint64_t)1 << j)
148
145
        return 0;
149
308
  }
150
308
  int output_flag = 0;
151
163
  int output_bitcount = 0;
152
326
  for (i = 0; i < output_bitmask_size; 
i++163
)
153
163
  {
154
344
    for (j = 0; j < 64; 
j++181
)
155
344
      if (output_bitmasks[i] & (uint64_t)1 << j)
156
181
      {
157
181
        if (output_flag)
158
0
          return 0;
159
163
      } else
160
163
        break;
161
163
    output_bitcount += j;
162
163
    if (j < 64)
163
163
      output_flag = 1;
164
10.4k
    for (; j < 64; 
j++10.2k
)
165
10.2k
      if (output_bitmasks[i] & (uint64_t)1 << j)
166
0
        return 0;
167
163
  }
168
163
  return output_bitcount <= input_bitcount && 
output_bitcount == output_size122
;
169
163
}
170
171
REGISTER_COMMAND(CCV_NNC_DATA_TRANSFER_FORWARD)(ccv_nnc_cmd_registry_t* const registry)
172
  FIND_BACKEND(ccv_nnc_util_cpu_ref.c, gpu/ccv_nnc_util_gpu_ref.cu)
173
1
{
174
1
  registry->bitmask = _ccv_nnc_data_transfer_forw_bitmask;
175
1
  registry->tensor_auto = ccv_nnc_hint_tensor_auto_forward_from_inputs;
176
1
}
177
178
REGISTER_COMMAND(CCV_NNC_DATA_TRANSFER_BACKWARD)(ccv_nnc_cmd_registry_t* const registry)
179
  FIND_BACKEND(ccv_nnc_util_cpu_ref.c, gpu/ccv_nnc_util_gpu_ref.cu)
180
1
{
181
1
  registry->bitmask = _ccv_nnc_data_transfer_back_bitmask;
182
1
  registry->tensor_auto = ccv_nnc_hint_tensor_auto_forward_from_inputs;
183
1
}
184
185
//@REGISTER_EASY_COMMAND_MACRO(CCV_NNC_DATA_TRANSFER_FORWARD)
186
#define CMD_DATA_TRANSFER_FORWARD() ccv_nnc_cmd(CCV_NNC_DATA_TRANSFER_FORWARD, 0, ccv_nnc_cmd_auto, 0)
187
//@REGISTER_EASY_COMMAND_MACRO(CCV_NNC_DATA_TRANSFER_BACKWARD)
188
#define CMD_DATA_TRANSFER_BACKWARD() ccv_nnc_cmd(CCV_NNC_DATA_TRANSFER_BACKWARD, 0, ccv_nnc_cmd_auto, 0)
189
190
REGISTER_COMMAND(CCV_NNC_FORMAT_TRANSFORM_FORWARD)(ccv_nnc_cmd_registry_t* const registry)
191
  FIND_BACKEND(ccv_nnc_util_cpu_ref.c, gpu/ccv_nnc_util_gpu_cudnn.cu)
192
1
{
193
1
  registry->bitmask = _ccv_nnc_data_transfer_forw_bitmask;
194
1
  registry->tensor_auto = ccv_nnc_hint_tensor_auto_forward_from_inputs;
195
1
}
196
197
REGISTER_COMMAND(CCV_NNC_FORMAT_TRANSFORM_BACKWARD)(ccv_nnc_cmd_registry_t* const registry)
198
  FIND_BACKEND(ccv_nnc_util_cpu_ref.c, gpu/ccv_nnc_util_gpu_cudnn.cu)
199
1
{
200
1
  registry->bitmask = _ccv_nnc_data_transfer_back_bitmask;
201
1
  registry->tensor_auto = ccv_nnc_hint_tensor_auto_forward_from_inputs;
202
1
}
203
204
//@REGISTER_EASY_COMMAND_MACRO(CCV_NNC_FORMAT_TRANSFORM_FORWARD)
205
#define CMD_FORMAT_TRANSFORM_FORWARD() ccv_nnc_cmd(CCV_NNC_FORMAT_TRANSFORM_FORWARD, 0, ccv_nnc_cmd_auto, 0)
206
//@REGISTER_EASY_COMMAND_MACRO(CCV_NNC_FORMAT_TRANSFORM_BACKWARD)
207
#define CMD_FORMAT_TRANSFORM_BACKWARD() ccv_nnc_cmd(CCV_NNC_FORMAT_TRANSFORM_BACKWARD, 0, ccv_nnc_cmd_auto, 0)
208
209
static void _ccv_nnc_transpose_tensor_auto(const ccv_nnc_cmd_param_t cmd, const ccv_nnc_tensor_param_t* const inputs, const int input_size, const ccv_nnc_hint_t hint, ccv_nnc_tensor_param_t* const outputs, const int output_size)
210
2.09k
{
211
2.09k
  int i;
212
4.18k
  for (i = 0; i < output_size; 
i++2.09k
)
213
2.09k
  {
214
2.09k
    outputs[i] = inputs[i];
215
2.09k
    int t;
216
2.09k
    CCV_SWAP(outputs[i].dim[cmd.transpose.axis[0]], outputs[i].dim[cmd.transpose.axis[1]], t);
217
2.09k
  }
218
2.09k
}
219
220
REGISTER_COMMAND(CCV_NNC_TRANSPOSE_FORWARD)(ccv_nnc_cmd_registry_t* const registry)
221
  FIND_BACKEND(ccv_nnc_util_cpu_ref.c, gpu/ccv_nnc_util_gpu_cudnn.cu)
222
1
{
223
1
  registry->bitmask = _ccv_nnc_data_transfer_forw_bitmask;
224
1
  registry->tensor_auto = _ccv_nnc_transpose_tensor_auto;
225
1
}
226
227
REGISTER_COMMAND(CCV_NNC_TRANSPOSE_BACKWARD)(ccv_nnc_cmd_registry_t* const registry)
228
  FIND_BACKEND(ccv_nnc_util_cpu_ref.c, gpu/ccv_nnc_util_gpu_cudnn.cu)
229
1
{
230
1
  registry->bitmask = _ccv_nnc_data_transfer_back_bitmask;
231
1
  registry->tensor_auto = _ccv_nnc_transpose_tensor_auto;
232
1
}
233
234
//@REGISTER_EASY_COMMAND_MACRO(CCV_NNC_TRANSPOSE_FORWARD)
235
#define CMD_TRANSPOSE_FORWARD(_axis_a, _axis_b) ccv_nnc_cmd(CCV_NNC_TRANSPOSE_FORWARD, 0, ((ccv_nnc_cmd_param_t){.size={.dim={1,1,1}},.transpose={.axis={_axis_a, _axis_b}}}), 0)
236
//@REGISTER_EASY_COMMAND_MACRO(CCV_NNC_TRANSPOSE_BACKWARD)
237
#define CMD_TRANSPOSE_BACKWARD(_axis_a, _axis_b) ccv_nnc_cmd(CCV_NNC_TRANSPOSE_BACKWARD, 0, ((ccv_nnc_cmd_param_t){.size={.dim={1,1,1}},.transpose={.axis={_axis_a, _axis_b}}}), 0)
238
239
REGISTER_COMMAND(CCV_NNC_DATATYPE_CONVERSION_FORWARD)(ccv_nnc_cmd_registry_t* const registry)
240
  FIND_BACKEND(ccv_nnc_util_cpu_ref.c, gpu/ccv_nnc_util_gpu_ref.cu)
241
1
{
242
1
  registry->bitmask = _ccv_nnc_data_transfer_forw_bitmask;
243
1
  registry->tensor_auto = ccv_nnc_hint_tensor_auto_forward_from_inputs;
244
1
}
245
246
REGISTER_COMMAND(CCV_NNC_DATATYPE_CONVERSION_BACKWARD)(ccv_nnc_cmd_registry_t* const registry)
247
  FIND_BACKEND(ccv_nnc_util_cpu_ref.c, gpu/ccv_nnc_util_gpu_ref.cu)
248
1
{
249
1
  registry->bitmask = _ccv_nnc_data_transfer_back_bitmask;
250
1
  registry->tensor_auto = ccv_nnc_hint_tensor_auto_forward_from_inputs;
251
1
}
252
253
//@REGISTER_EASY_COMMAND_MACRO(CCV_NNC_DATATYPE_CONVERSION_FORWARD)
254
#define CMD_DATATYPE_CONVERSION_FORWARD() ccv_nnc_cmd(CCV_NNC_DATATYPE_CONVERSION_FORWARD, 0, ccv_nnc_cmd_auto, 0)
255
//@REGISTER_EASY_COMMAND_MACRO(CCV_NNC_DATATYPE_CONVERSION_BACKWARD)
256
#define CMD_DATATYPE_CONVERSION_BACKWARD() ccv_nnc_cmd(CCV_NNC_DATATYPE_CONVERSION_BACKWARD, 0, ccv_nnc_cmd_auto, 0)