Coverage Report

Created: 2021-09-30 21:42

/home/liu/buildslave/linux-x64-runtests/build/lib/nnc/cmd/blas/ccv_nnc_add_cpu_ref.c
Line
Count
Source (jump to first uncovered line)
1
#include "ccv.h"
2
#include "ccv_internal.h"
3
#include "nnc/ccv_nnc.h"
4
#include "nnc/ccv_nnc_easy.h"
5
#include "nnc/ccv_nnc_internal.h"
6
#ifdef USE_OPENMP
7
#include <omp.h>
8
#endif
9
#ifdef USE_DISPATCH
10
#include <dispatch/dispatch.h>
11
#endif
12
13
// Shared methods.
14
#include "../_ccv_nnc_cpu_ref.h"
15
16
void _ccv_nnc_add_forw_cpu_ref(const float p, const float q, ccv_nnc_tensor_view_t* const a, ccv_nnc_tensor_view_t* const b, ccv_nnc_tensor_view_t* const c)
17
28.8k
{
18
28.8k
  if (b == 0)
19
16.9k
  {
20
16.9k
    // It cannot be set otherwise we have trouble.
21
16.9k
    assert(q == 0);
22
16.9k
    if (p == 1)
23
11.8k
    {
24
11.8k
      _ccv_nnc_tensor_transfer_cpu_ref_f32(a, c);
25
11.8k
      return;
26
11.8k
    } else 
if (5.17k
p == 05.17k
) {
27
0
      ccv_nnc_tensor_zero(c);
28
0
      return;
29
0
    }
30
5.17k
    // Assuming this is float 32.
31
5.17k
    int dim[CCV_NNC_MAX_DIM_ALLOC];
32
5.17k
    int ainc[CCV_NNC_MAX_DIM_ALLOC];
33
5.17k
    int cinc[CCV_NNC_MAX_DIM_ALLOC];
34
5.17k
    assert(ccv_nnc_tensor_nd(a->info.dim) <= CCV_NNC_MAX_DIM + 2);
35
5.17k
    assert(ccv_nnc_tensor_nd(c->info.dim) <= CCV_NNC_MAX_DIM + 2);
36
5.17k
    ccv_nnc_tensor_view_get_dim(a, dim);
37
5.17k
    assert(ccv_nnc_tensor_view_check_dim(c, dim));
38
5.17k
    int x;
39
5.17k
    if (!CCV_IS_TENSOR_VIEW(a) && !CCV_IS_TENSOR_VIEW(c))
40
5.17k
    {
41
5.17k
      // Super optimal case, just do one for-loop for sum.
42
5.17k
      const int tensor_count = ccv_nnc_tensor_count(a->info);
43
11.8k
      for (x = 0; x < tensor_count; 
x++6.67k
)
44
6.67k
        c->data.f32[x] = p * a->data.f32[x];
45
5.17k
      return;
46
5.17k
    }
47
0
    assert(CCV_NNC_MAX_DIM == 2); // Need to change this logic for CCV_NNC_MAX_DIM == other number.
48
0
    ccv_nnc_tensor_view_get_inc(a, ainc);
49
0
    ccv_nnc_tensor_view_get_inc(c, cinc);
50
0
    int i[CCV_NNC_MAX_DIM + 2];
51
0
    float* ap = a->data.f32;
52
0
    float* cp = c->data.f32;
53
0
    const int count = dim[2] * dim[3];
54
0
    if (ainc[3] == dim[3] && cinc[3] == dim[3])
55
0
    {
56
0
      // Special casing if the ainc[3] is the same as dim[3]
57
0
      for (i[0] = 0; i[0] < dim[0]; i[0]++)
58
0
      {
59
0
        for (i[1] = 0; i[1] < dim[1]; i[1]++)
60
0
        {
61
0
          for (x = 0; x < count; x++)
62
0
            cp[x] = p * ap[x];
63
0
          ap += ainc[2] * ainc[3];
64
0
          cp += cinc[2] * cinc[3];
65
0
        }
66
0
        ap += (ainc[1] - dim[1]) * ainc[2] * ainc[3];
67
0
        cp += (cinc[1] - dim[1]) * cinc[2] * cinc[3];
68
0
      }
69
0
      return;
70
0
    }
71
0
    // Non-optimal case, need to do skip copy.
72
0
    for (i[0] = 0; i[0] < dim[0]; i[0]++)
73
0
    {
74
0
      for (i[1] = 0; i[1] < dim[1]; i[1]++)
75
0
      {
76
0
        for (i[2] = 0; i[2] < dim[2]; i[2]++)
77
0
        {
78
0
          for (x = 0; x < dim[3]; x++)
79
0
            cp[x] = p * ap[x];
80
0
          ap += ainc[3];
81
0
          cp += cinc[3];
82
0
        }
83
0
        ap += (ainc[2] - dim[2]) * ainc[3];
84
0
        cp += (cinc[2] - dim[2]) * cinc[3];
85
0
      }
86
0
      ap += (ainc[1] - dim[1]) * ainc[2] * ainc[3];
87
0
      cp += (cinc[1] - dim[1]) * cinc[2] * cinc[3];
88
0
    }
89
0
    return;
90
0
  }
91
11.8k
  int cdim[CCV_NNC_MAX_DIM_ALLOC];
92
11.8k
  assert(ccv_nnc_tensor_nd(a->info.dim) <= CCV_NNC_MAX_DIM + 2);
93
11.8k
  assert(ccv_nnc_tensor_nd(b->info.dim) <= CCV_NNC_MAX_DIM + 2);
94
11.8k
  ccv_nnc_tensor_view_get_dim(a, cdim); // Fill in cdim first.
95
11.8k
  ccv_nnc_tensor_view_get_broadcast_dim(b, cdim);
96
11.8k
  assert(ccv_nnc_tensor_view_check_broadcast_dim(a, cdim));
97
11.8k
  assert(ccv_nnc_tensor_view_check_broadcast_dim(b, cdim));
98
11.8k
  const int a_check_dim = ccv_nnc_tensor_view_check_dim(a, cdim);
99
11.8k
  const int b_check_dim = ccv_nnc_tensor_view_check_dim(b, cdim);
100
11.8k
  if (p == 1 && 
q == 111.8k
&&
a_check_dim4
&&
b_check_dim3
)
101
1
  {
102
1
    _ccv_nnc_ewsum_forw_cpu_ref((ccv_nnc_tensor_view_t*[]){
103
1
      a, b
104
1
    }, 2, &c, 1);
105
1
    return;
106
11.8k
  } else if (p == 1 && 
q == 011.8k
&&
a_check_dim0
) {
107
0
    _ccv_nnc_tensor_transfer_cpu_ref_f32(a, c);
108
0
    return;
109
11.8k
  } else if (p == 0 && 
q == 10
&&
b_check_dim0
) {
110
0
    _ccv_nnc_tensor_transfer_cpu_ref_f32(b, c);
111
0
    return;
112
11.8k
  } else if (p == 0 && 
q == 00
) {
113
0
    ccv_nnc_tensor_zero(c);
114
0
    return;
115
0
  }
116
11.8k
  // Assuming this is float 32.
117
11.8k
  int adim[CCV_NNC_MAX_DIM_ALLOC];
118
11.8k
  int bdim[CCV_NNC_MAX_DIM_ALLOC];
119
11.8k
  ccv_nnc_tensor_view_get_dim(a, adim);
120
11.8k
  ccv_nnc_tensor_view_get_dim(b, bdim);
121
11.8k
  int ainc[CCV_NNC_MAX_DIM_ALLOC];
122
11.8k
  int binc[CCV_NNC_MAX_DIM_ALLOC];
123
11.8k
  int cinc[CCV_NNC_MAX_DIM_ALLOC];
124
11.8k
  assert(ccv_nnc_tensor_nd(c->info.dim) <= CCV_NNC_MAX_DIM + 2);
125
11.8k
  assert(ccv_nnc_tensor_view_check_dim(c, cdim));
126
11.8k
  int x;
127
11.8k
  if (!CCV_IS_TENSOR_VIEW(a) && !CCV_IS_TENSOR_VIEW(b) && !CCV_IS_TENSOR_VIEW(c) && a_check_dim && 
b_check_dim11.8k
)
128
11.7k
  {
129
11.7k
    const int tensor_count = ccv_nnc_tensor_count(a->info);
130
11.7k
    // Super optimal case, just do one for-loop for sum.
131
23.9k
    for (x = 0; x < tensor_count; 
x++12.2k
)
132
12.2k
      c->data.f32[x] = p * a->data.f32[x] + q * b->data.f32[x];
133
11.7k
    return;
134
11.7k
  }
135
112
  assert(CCV_NNC_MAX_DIM == 2); // Need to change this logic for CCV_NNC_MAX_DIM == other number.
136
112
  ccv_nnc_tensor_view_get_inc(a, ainc);
137
112
  ccv_nnc_tensor_view_get_inc(b, binc);
138
112
  ccv_nnc_tensor_view_get_inc(c, cinc);
139
112
  int i[CCV_NNC_MAX_DIM + 2];
140
112
  float* ap = a->data.f32;
141
112
  float* bp = b->data.f32;
142
112
  float* cp = c->data.f32;
143
112
  const int count = cdim[2] * cdim[3];
144
112
  if (ainc[3] == cdim[3] && 
binc[3] == cdim[3]111
&&
cinc[3] == cdim[3]107
&&
adim[2] == cdim[2]107
&&
bdim[2] == cdim[2]107
)
145
0
  {
146
0
    // Special casing if the ainc[3] is the same as dim[3]
147
0
    for (i[0] = 0; i[0] < cdim[0]; i[0]++)
148
0
    {
149
0
      float* const ap0 = adim[0] == 1 ? ap : ap + i[0] * ainc[1] * ainc[2] * ainc[3];
150
0
      float* const bp0 = bdim[0] == 1 ? bp : bp + i[0] * binc[1] * binc[2] * binc[3];
151
0
      for (i[1] = 0; i[1] < cdim[1]; i[1]++)
152
0
      {
153
0
        float* const ap1 = adim[1] == 1 ? ap0 : ap0 + i[1] * ainc[2] * ainc[3];
154
0
        float* const bp1 = bdim[1] == 1 ? bp0 : bp0 + i[1] * binc[2] * binc[3];
155
0
        for (x = 0; x < count; x++)
156
0
          cp[x] = p * ap1[x] + q * bp1[x];
157
0
        cp += cinc[2] * cinc[3];
158
0
      }
159
0
      cp += (cinc[1] - cdim[1]) * cinc[2] * cinc[3];
160
0
    }
161
0
    return;
162
0
  }
163
112
  // Non-optimal case, need to do skip copy and handle broadcasting.
164
282
  
for (i[0] = 0; 112
i[0] < cdim[0];
i[0]++170
)
165
170
  {
166
170
    float* const ap0 = adim[0] == 1 ? 
ap104
:
ap + i[0] * ainc[1] * ainc[2] * ainc[3]66
;
167
170
    float* const bp0 = bdim[0] == 1 ? 
bp114
:
bp + i[0] * binc[1] * binc[2] * binc[3]56
;
168
574
    for (i[1] = 0; i[1] < cdim[1]; 
i[1]++404
)
169
404
    {
170
404
      float* const ap1 = adim[1] == 1 ? 
ap0104
:
ap0 + i[1] * ainc[2] * ainc[3]300
;
171
404
      float* const bp1 = bdim[1] == 1 ? 
bp0204
:
bp0 + i[1] * binc[2] * binc[3]200
;
172
2.00k
      for (i[2] = 0; i[2] < cdim[2]; 
i[2]++1.60k
)
173
1.60k
      {
174
1.60k
        float* const ap2 = adim[2] == 1 ? 
ap12
:
ap1 + i[2] * ainc[3]1.59k
;
175
1.60k
        float* const bp2 = bdim[2] == 1 ? bp1 : 
bp1 + i[2] * binc[3]0
;
176
1.60k
        if (adim[3] == 1)
177
412
          
for (x = 0; 204
x < cdim[3];
x++208
)
178
208
            cp[x] = p * ap2[0] + q * bp2[x];
179
1.39k
        else if (bdim[3] == 1)
180
2.92k
          
for (x = 0; 258
x < cdim[3];
x++2.66k
)
181
2.66k
            cp[x] = p * ap2[x] + q * bp2[0];
182
1.13k
        else
183
5.50k
          
for (x = 0; 1.13k
x < cdim[3];
x++4.36k
)
184
4.36k
            cp[x] = p * ap2[x] + q * bp2[x];
185
1.60k
        cp += cinc[3];
186
1.60k
      }
187
404
      cp += (cinc[2] - cdim[2]) * cinc[3];
188
404
    }
189
170
    cp += (cinc[1] - cdim[1]) * cinc[2] * cinc[3];
190
170
  }
191
112
}
192
193
static int _ccv_nnc_add_forw(const ccv_nnc_cmd_t cmd, const ccv_nnc_hint_t hint, const int flags, ccv_nnc_tensor_t* const* const inputs, const int input_size, ccv_nnc_tensor_t* const* const outputs, const int output_size, ccv_nnc_stream_context_t* const stream_context)
194
11.8k
{
195
11.8k
  assert(input_size == 2);
196
11.8k
  _ccv_nnc_add_forw_cpu_ref(cmd.info.blas.a[0], cmd.info.blas.a[1], (ccv_nnc_tensor_view_t*)inputs[0], (ccv_nnc_tensor_view_t*)inputs[1], (ccv_nnc_tensor_view_t*)outputs[0]);
197
11.8k
  return CCV_NNC_EXEC_SUCCESS;
198
11.8k
}
199
200
static int _ccv_nnc_add_back(const ccv_nnc_cmd_t cmd, const ccv_nnc_hint_t hint, const int flags, ccv_nnc_tensor_t* const* const inputs, const int input_size, ccv_nnc_tensor_t* const* const outputs, const int output_size, ccv_nnc_stream_context_t* const stream_context)
201
11.8k
{
202
11.8k
  if (inputs[0] == 0)
203
0
  {
204
0
    if (outputs[0])
205
0
      _ccv_nnc_tensor_set_cpu_ref((ccv_nnc_tensor_view_t*)outputs[0], cmd.info.blas.a[0]);
206
0
    if (output_size > 1 && outputs[1])
207
0
      _ccv_nnc_tensor_set_cpu_ref((ccv_nnc_tensor_view_t*)outputs[1], cmd.info.blas.a[1]);
208
0
    return CCV_NNC_EXEC_SUCCESS;
209
0
  }
210
11.8k
  int gdim[CCV_NNC_MAX_DIM_ALLOC];
211
11.8k
  int ginc[CCV_NNC_MAX_DIM_ALLOC];
212
11.8k
  ccv_nnc_tensor_view_t* const g = (ccv_nnc_tensor_view_t*)inputs[0];
213
11.8k
  ccv_nnc_tensor_view_get_dim(g, gdim);
214
11.8k
  ccv_nnc_tensor_view_get_inc(g, ginc);
215
11.8k
  if (outputs[0])
216
11.8k
  {
217
11.8k
    ccv_nnc_tensor_view_t* const a = (ccv_nnc_tensor_view_t*)outputs[0];
218
11.8k
    if (ccv_nnc_tensor_view_check_dim(a, gdim))
219
11.8k
      _ccv_nnc_add_forw_cpu_ref(cmd.info.blas.a[0], 0, (ccv_nnc_tensor_view_t*)inputs[0], 0, (ccv_nnc_tensor_view_t*)outputs[0]);
220
1
    else {
221
1
      assert(CCV_NNC_MAX_DIM == 2); // Need to change this logic for CCV_NNC_MAX_DIM == other number.
222
1
      const float p = cmd.info.blas.a[0];
223
1
      int adim[CCV_NNC_MAX_DIM_ALLOC];
224
1
      int ainc[CCV_NNC_MAX_DIM_ALLOC];
225
1
      ccv_nnc_tensor_view_get_dim(a, adim);
226
1
      ccv_nnc_tensor_view_get_inc(a, ainc);
227
1
      int i[CCV_NNC_MAX_DIM + 2];
228
1
      int x;
229
1
      float* const ap = a->data.f32;
230
1
      float* gp = g->data.f32;
231
1
      // zeroing out so that we can accumulate.
232
1
      ccv_nnc_tensor_zero(a);
233
1
      // Non-optimal case, need to do skip copy and handle broadcasting.
234
2
      for (i[0] = 0; i[0] < gdim[0]; 
i[0]++1
)
235
1
      {
236
1
        float* const ap0 = adim[0] == 1 ? ap : 
ap + i[0] * ainc[1] * ainc[2] * ainc[3]0
;
237
2
        for (i[1] = 0; i[1] < gdim[1]; 
i[1]++1
)
238
1
        {
239
1
          float* const ap1 = adim[1] == 1 ? ap0 : 
ap0 + i[1] * ainc[2] * ainc[3]0
;
240
5
          for (i[2] = 0; i[2] < gdim[2]; 
i[2]++4
)
241
4
          {
242
4
            float* const ap2 = adim[2] == 1 ? 
ap10
: ap1 + i[2] * ainc[3];
243
4
            if (adim[3] == 1)
244
12
              
for (x = 0; 4
x < gdim[3];
x++8
)
245
8
                ap2[0] += p * gp[x];
246
0
            else
247
0
              for (x = 0; x < gdim[3]; x++)
248
0
                ap2[x] += p * gp[x];
249
4
            gp += ginc[3];
250
4
          }
251
1
          gp += (ginc[2] - gdim[2]) * ginc[3];
252
1
        }
253
1
        gp += (ginc[1] - gdim[1]) * ginc[2] * ginc[3];
254
1
      }
255
1
    }
256
11.8k
  }
257
11.8k
  if (output_size > 1 && outputs[1])
258
5.17k
  {
259
5.17k
    ccv_nnc_tensor_view_t* const a = (ccv_nnc_tensor_view_t*)outputs[1];
260
5.17k
    if (ccv_nnc_tensor_view_check_dim(a, gdim))
261
5.17k
      _ccv_nnc_add_forw_cpu_ref(cmd.info.blas.a[1], 0, (ccv_nnc_tensor_view_t*)inputs[0], 0, (ccv_nnc_tensor_view_t*)outputs[1]);
262
6
    else {
263
6
      assert(CCV_NNC_MAX_DIM == 2); // Need to change this logic for CCV_NNC_MAX_DIM == other number.
264
6
      const float p = cmd.info.blas.a[1];
265
6
      int adim[CCV_NNC_MAX_DIM_ALLOC];
266
6
      int ainc[CCV_NNC_MAX_DIM_ALLOC];
267
6
      ccv_nnc_tensor_view_get_dim(a, adim);
268
6
      ccv_nnc_tensor_view_get_inc(a, ainc);
269
6
      int i[CCV_NNC_MAX_DIM + 2];
270
6
      int x;
271
6
      float* const ap = a->data.f32;
272
6
      float* gp = g->data.f32;
273
6
      // zeroing out so that we can accumulate.
274
6
      ccv_nnc_tensor_zero(a);
275
6
      // Non-optimal case, need to do skip copy and handle broadcasting.
276
38
      for (i[0] = 0; i[0] < gdim[0]; 
i[0]++32
)
277
32
      {
278
32
        float* const ap0 = adim[0] == 1 ? 
ap4
:
ap + i[0] * ainc[1] * ainc[2] * ainc[3]28
;
279
170
        for (i[1] = 0; i[1] < gdim[1]; 
i[1]++138
)
280
138
        {
281
138
          float* const ap1 = adim[1] == 1 ? 
ap038
:
ap0 + i[1] * ainc[2] * ainc[3]100
;
282
779
          for (i[2] = 0; i[2] < gdim[2]; 
i[2]++641
)
283
641
          {
284
641
            float* const ap2 = adim[2] == 1 ? ap1 : 
ap1 + i[2] * ainc[3]0
;
285
641
            if (adim[3] == 1)
286
1.50k
              
for (x = 0; 129
x < gdim[3];
x++1.38k
)
287
1.38k
                ap2[0] += p * gp[x];
288
512
            else
289
2.10k
              
for (x = 0; 512
x < gdim[3];
x++1.58k
)
290
1.58k
                ap2[x] += p * gp[x];
291
641
            gp += ginc[3];
292
641
          }
293
138
          gp += (ginc[2] - gdim[2]) * ginc[3];
294
138
        }
295
32
        gp += (ginc[1] - gdim[1]) * ginc[2] * ginc[3];
296
32
      }
297
6
    }
298
5.17k
  }
299
11.8k
  return CCV_NNC_EXEC_SUCCESS;
300
11.8k
}
301
302
REGISTER_COMMAND_BACKEND(CCV_NNC_ADD_FORWARD, CCV_NNC_BACKEND_CPU_REF)(ccv_nnc_cmd_backend_registry_t* const registry)
303
1
{
304
1
  registry->tensor_formats = CCV_TENSOR_FORMAT_NHWC | CCV_TENSOR_FORMAT_NCHW | CCV_TENSOR_FORMAT_CHWN;
305
1
  registry->tensor_datatypes = CCV_32F;
306
1
  registry->tensor_memory = CCV_TENSOR_CPU_MEMORY;
307
1
  registry->algorithms = 1;
308
1
  registry->exec = _ccv_nnc_add_forw;
309
1
}
310
311
REGISTER_COMMAND_BACKEND(CCV_NNC_ADD_BACKWARD, CCV_NNC_BACKEND_CPU_REF)(ccv_nnc_cmd_backend_registry_t* const registry)
312
1
{
313
1
  registry->tensor_formats = CCV_TENSOR_FORMAT_NHWC | CCV_TENSOR_FORMAT_NCHW | CCV_TENSOR_FORMAT_CHWN;
314
1
  registry->tensor_datatypes = CCV_32F;
315
1
  registry->tensor_memory = CCV_TENSOR_CPU_MEMORY;
316
1
  registry->algorithms = 1;
317
1
  registry->exec = _ccv_nnc_add_back;
318
1
}