Coverage Report

Created: 2019-07-03 22:50

/home/liu/buildslave/linux-x64-runtests/build/lib/nnc/ccv_nnc_tensor.c
Line
Count
Source (jump to first uncovered line)
1
#include "ccv_nnc.h"
2
#include "ccv_nnc_easy.h"
3
#include "ccv_nnc_internal.h"
4
#ifdef HAVE_CUDA
5
#include "gpu/ccv_nnc_compat.h"
6
#endif
7
8
const int ccv_nnc_no_ofs[CCV_NNC_MAX_DIM_ALLOC] = {0};
9
10
ccv_nnc_tensor_t* ccv_nnc_tensor_new(const void* const ptr, const ccv_nnc_tensor_param_t params, const int flags)
11
17.2k
{
12
17.2k
  ccv_nnc_tensor_t* tensor;
13
17.2k
  // this specific form can be toll-free bridging to ccv_dense_matrix_t (On CPU, and 3 dims (channels, rows, cols), and channels is smaller than max channels of ccv_dense_matrix_t).
14
17.2k
  int tfb = (CCV_TENSOR_GET_MEMORY(params.type) == CCV_TENSOR_CPU_MEMORY && 
params.format == CCV_TENSOR_FORMAT_NHWC13.2k
&&
params.dim[2] > 010.5k
&&
params.dim[2] <= 3.67k
CCV_MAX_CHANNEL3.67k
&&
params.dim[0] > 03.67k
&&
params.dim[1] > 03.67k
&&
params.dim[3] == 03.67k
);
15
17.2k
  if (ptr)
16
227
  {
17
227
    tensor = (ccv_nnc_tensor_t*)ccmalloc(sizeof(ccv_nnc_tensor_t));
18
227
    tensor->alias_ref = 0;
19
227
    tensor->sig = 0;
20
227
    tensor->refcount = 1;
21
227
    tensor->info = params;
22
227
    if (tfb)
23
0
    {
24
0
      tensor->type = CCV_NO_DATA_ALLOC | CCV_MATRIX_DENSE | params.datatype | params.dim[2];
25
0
      // This corresponding to mat->step
26
0
      tensor->info.dim[4] = CCV_GET_STEP(params.dim[1], (params.datatype | params.dim[2]));
27
0
    } else // This won't be recognized by ccv_dense_matrix_t
28
227
      tensor->type = CCV_NO_DATA_ALLOC | CCV_MATRIX_DENSE | params.datatype;
29
227
    tensor->data.u8 = (uint8_t*)ptr;
30
227
    return tensor;
31
227
  }
32
17.0k
  if (flags & CCV_TENSOR_CPU_MEMORY)
33
0
  {
34
0
    assert(CCV_TENSOR_GET_MEMORY(params.type) == CCV_TENSOR_CPU_MEMORY);
35
17.0k
  } else if (flags & CCV_TENSOR_GPU_MEMORY) {
36
0
    assert(CCV_TENSOR_GET_MEMORY(params.type) == CCV_TENSOR_GPU_MEMORY);
37
0
  }
38
17.0k
  const size_t tensor_hdr_size = (sizeof(ccv_nnc_tensor_t) + 15) & -16;
39
17.0k
  const size_t size = ccv_nnc_tensor_data_size(params);
40
17.0k
#ifdef HAVE_CUDA
41
17.0k
  if (CCV_TENSOR_GET_MEMORY(params.type) == CCV_TENSOR_GPU_MEMORY)
42
3.57k
  {
43
3.57k
    tensor = (ccv_nnc_tensor_t*)ccmalloc(sizeof(ccv_nnc_tensor_t));
44
3.57k
    assert(CCV_TENSOR_GET_DEVICE(params.type) != CCV_COMPUTE_DEVICE_ANY);
45
3.57k
    tensor->data.u8 = (uint8_t*)cumalloc(CCV_TENSOR_GET_DEVICE_ID(params.type), size);
46
13.4k
  } else {
47
13.4k
    assert(CCV_TENSOR_GET_MEMORY(params.type) == CCV_TENSOR_CPU_MEMORY);
48
13.4k
    
ccmemalign13.0k
((void **)&tensor, 16, tensor_hdr_size + size);
49
13.0k
    tensor->data.u8 = (uint8_t*)tensor + tensor_hdr_size;
50
13.0k
  }
51
#else
52
  assert(CCV_TENSOR_GET_MEMORY(params.type) == CCV_TENSOR_CPU_MEMORY);
53
  ccmemalign((void **)&tensor, 16, tensor_hdr_size + size);
54
  tensor->data.u8 = (uint8_t*)tensor + tensor_hdr_size;
55
#endif
56
16.6k
  tensor->alias_ref = 0;
57
16.6k
  tensor->sig = 0;
58
16.6k
  tensor->refcount = 1;
59
16.6k
  tensor->info = params;
60
16.6k
  if (tfb)
61
3.30k
  {
62
3.30k
    tensor->type = CCV_UNMANAGED | CCV_MATRIX_DENSE | params.datatype | params.dim[2];
63
3.30k
    // This corresponding to mat->step
64
3.30k
    tensor->info.dim[4] = CCV_GET_STEP(params.dim[1], (params.datatype | params.dim[2]));
65
3.30k
  } else
66
13.3k
    tensor->type = CCV_UNMANAGED | CCV_MATRIX_DENSE | params.datatype;
67
16.6k
  return tensor;
68
17.0k
}
69
70
ccv_nnc_tensor_t ccv_nnc_tensor(const void* const ptr, const ccv_nnc_tensor_param_t params, const int flags)
71
45.6k
{
72
45.6k
  // this specific form can be toll-free bridging to ccv_dense_matrix_t
73
45.6k
  int tfb = (CCV_TENSOR_GET_MEMORY(params.type) == CCV_TENSOR_CPU_MEMORY && 
params.format == CCV_TENSOR_FORMAT_NHWC16.2k
&&
params.dim[2] > 016.1k
&&
params.dim[2] <= 318
CCV_MAX_CHANNEL318
&&
params.dim[0] > 0318
&&
params.dim[1] > 0318
&&
params.dim[3] == 0318
);
74
45.6k
  ccv_nnc_tensor_t tensor;
75
45.6k
  tensor.alias_ref = 0;
76
45.6k
  tensor.sig = 0;
77
45.6k
  tensor.refcount = 1;
78
45.6k
  tensor.info = params;
79
45.6k
  if (flags & CCV_TENSOR_CPU_MEMORY)
80
0
  {
81
0
    assert(CCV_TENSOR_GET_MEMORY(params.type) == CCV_TENSOR_CPU_MEMORY);
82
45.6k
  } else if (flags & CCV_TENSOR_GPU_MEMORY) {
83
0
    assert(CCV_TENSOR_GET_MEMORY(params.type) == CCV_TENSOR_GPU_MEMORY);
84
0
  }
85
45.6k
  if (tfb)
86
89
  {
87
89
    tensor.type = CCV_NO_DATA_ALLOC | CCV_UNMANAGED | CCV_MATRIX_DENSE | params.datatype | params.dim[2];
88
89
    // This corresponding to mat->step
89
89
    tensor.info.dim[4] = CCV_GET_STEP(params.dim[1], (params.datatype | params.dim[2]));
90
89
  } else // This won't be recognized by ccv_dense_matrix_t
91
45.5k
    tensor.type = CCV_NO_DATA_ALLOC | CCV_UNMANAGED | CCV_MATRIX_DENSE | params.datatype;
92
45.6k
  tensor.data.u8 = (uint8_t*)ptr;
93
45.6k
  return tensor;
94
45.6k
}
95
96
int ccv_nnc_tensor_pin_memory(ccv_nnc_tensor_t* const tensor)
97
27.5k
{
98
27.5k
#ifdef HAVE_CUDA
99
27.5k
  if (!(tensor->type & CCV_PINNED_MEM))
100
86
  {
101
86
    const int success = curegister(tensor->data.u8, ccv_nnc_tensor_data_size(tensor->info));
102
86
    if (success)
103
86
      tensor->type |= CCV_PINNED_MEM;
104
86
    return success ? 0 : 
-10
;
105
86
  }
106
27.4k
#endif
107
27.4k
  return 0;
108
27.4k
}
109
110
void ccv_nnc_tensor_free(ccv_nnc_tensor_t* const tensor)
111
18.7k
{
112
18.7k
#ifdef HAVE_CUDA
113
18.7k
  if (CCV_TENSOR_GET_MEMORY(tensor->info.type) == CCV_TENSOR_GPU_MEMORY &&
114
18.7k
    
!(tensor->type & CCV_NO_DATA_ALLOC)3.78k
) // If this is GPU memory and it is allocated, free.
115
3.57k
    cufree(CCV_TENSOR_GET_DEVICE_ID(tensor->info.type), tensor->data.u8);
116
18.7k
  if (tensor->type & CCV_PINNED_MEM)
117
86
    cuunregister(tensor->data.u8);
118
18.7k
#endif
119
18.7k
  ccfree(tensor);
120
18.7k
}
121
122
static inline void _ccv_nnc_tensor_view_set(ccv_nnc_tensor_view_t* const tv, const ccv_nnc_tensor_t* const tensor, const int dim[CCV_NNC_MAX_DIM_ALLOC], const int ofs[CCV_NNC_MAX_DIM_ALLOC], const int inc[CCV_NNC_MAX_DIM_ALLOC])
123
69
{
124
69
  memcpy(tv->inc, inc, sizeof(int) * CCV_NNC_MAX_DIM_ALLOC);
125
69
  memcpy(tv->info.dim, dim, sizeof(int) * CCV_NNC_MAX_DIM_ALLOC);
126
69
  uint8_t* const p = tensor->data.u8;
127
69
  const off_t off = tv->off = ccv_nnc_tensor_view_offset(tv, ofs);
128
69
  tv->data.u8 = p + off;
129
69
}
130
131
ccv_nnc_tensor_view_t* ccv_nnc_tensor_view_new(const ccv_nnc_tensor_t* const tensor, const int dim[CCV_NNC_MAX_DIM_ALLOC], const int ofs[CCV_NNC_MAX_DIM_ALLOC], const int inc[CCV_NNC_MAX_DIM_ALLOC])
132
6
{
133
6
  ccv_nnc_tensor_view_t* tv = (ccv_nnc_tensor_view_t*)ccmalloc(sizeof(ccv_nnc_tensor_view_t));
134
6
  tv->type = (tensor->type & ~0xfff) | CCV_TENSOR_VIEW;
135
6
  tv->alias_ref = (uintptr_t)tensor;
136
6
  tv->refcount = 1;
137
6
  tv->sig = 0;
138
6
  tv->info = tensor->info;
139
6
  _ccv_nnc_tensor_view_set(tv, tensor, dim, ofs, inc);
140
6
  return tv;
141
6
}
142
143
ccv_nnc_tensor_view_t ccv_nnc_tensor_view(const ccv_nnc_tensor_t* const tensor, const int dim[CCV_NNC_MAX_DIM_ALLOC], const int ofs[CCV_NNC_MAX_DIM_ALLOC], const int inc[CCV_NNC_MAX_DIM_ALLOC])
144
63
{
145
63
  assert(!CCV_IS_TENSOR_VIEW(tensor));
146
63
  ccv_nnc_tensor_view_t tv = {
147
63
    .alias_ref = (uintptr_t)tensor,
148
63
    .type = (tensor->type & ~0xfff) | CCV_TENSOR_VIEW, // clean up the channel bits, and then add CCV_TENSOR_VIEW identifier
149
63
    .refcount = 1,
150
63
    .sig = 0,
151
63
    .info = tensor->info,
152
63
  };
153
63
  _ccv_nnc_tensor_view_set(&tv, tensor, dim, ofs, inc);
154
63
  return tv;
155
63
}
156
157
void ccv_nnc_tensor_view_free(ccv_nnc_tensor_view_t* const tensor_view)
158
6
{
159
6
  ccfree(tensor_view);
160
6
}
161
162
void ccv_nnc_tensor_zero(void* const tensor)
163
4.13k
{
164
4.13k
  ccv_nnc_tensor_view_t* tv = (ccv_nnc_tensor_view_t*)tensor;
165
4.13k
  const size_t data_size = CCV_GET_DATA_TYPE_SIZE(tv->info.datatype);
166
4.13k
  if (!CCV_IS_TENSOR_VIEW(tv))
167
4.13k
  {
168
4.13k
    memset(tv->data.u8, 0, data_size * ccv_nnc_tensor_count(tv->info));
169
4.13k
    return;
170
4.13k
  }
171
1
  const int nd = ccv_nnc_tensor_nd(tv->info.dim);
172
1
  const int* tvinc = tv->inc;
173
1
  // reset it to 0.
174
1
  int c, x, y;
175
1
  int count = 1;
176
1
  int mod[CCV_NNC_MAX_DIM_ALLOC - 3];
177
1
  size_t mod_inc[CCV_NNC_MAX_DIM_ALLOC - 2];
178
1
  mod_inc[nd - 3] = data_size * tvinc[nd - 3] * tvinc[nd - 2] * tvinc[nd - 1];
179
4
  for (c = nd - 4; c >= 0; 
c--3
)
180
3
  {
181
3
    // Compute the mod.
182
3
    mod[c] = c == nd - 4 ? 
tv->info.dim[c]1
:
mod[c + 1] * tv->info.dim[c]2
;
183
3
    mod_inc[c] = mod_inc[c + 1] * tvinc[c];
184
3
    count *= tv->info.dim[c];
185
3
  }
186
4
  for (c = 0; c < nd - 3; 
c++3
)
187
3
    mod_inc[c] = mod_inc[c + 1] * (tvinc[c] - tv->info.dim[c]);
188
1
  uint8_t* tvd = tv->data.u8;
189
1
  const size_t tvinc_21 = data_size * tvinc[nd - 2] * tvinc[nd - 1];
190
1
  const size_t tvinc_1 = data_size * tvinc[nd - 1];
191
1
  const size_t tvdim_1 = data_size * tv->info.dim[nd - 1];
192
1.44k
  for (c = 0; c < count; 
c++1.44k
)
193
1.44k
  {
194
4.32k
    for (y = 0; y < ccv_max(1, tv->info.dim[nd - 3]); 
y++2.88k
)
195
2.88k
    {
196
2.88k
      uint8_t* tvp = tvd + y * tvinc_21;
197
11.5k
      for (x = 0; x < ccv_max(1, tv->info.dim[nd - 2]); 
x++8.64k
)
198
8.64k
      {
199
8.64k
        memset(tvp, 0, tvdim_1);
200
8.64k
        tvp += tvinc_1;
201
8.64k
      }
202
2.88k
    }
203
1.44k
    tvd += mod_inc[nd - 3];
204
1.54k
    for (y = nd - 4; y >= 0; 
y--105
)
205
1.54k
      if ((c + 1) % mod[y] != 0)
206
1.43k
        break; // cannot be mod, break out.
207
105
      else
208
105
        tvd += mod_inc[y];
209
1.44k
  }
210
1
}
211
212
int ccv_nnc_tensor_eq(const ccv_nnc_tensor_t* const a, const ccv_nnc_tensor_t* const b)
213
137
{
214
137
  assert(!CCV_IS_TENSOR_VIEW(a));
215
137
  assert(!CCV_IS_TENSOR_VIEW(b));
216
137
  // If a is a dense matrix, just use ccv_matrix_eq
217
137
  if (CCV_TENSOR_IS_DENSE_MATRIX(a->type))
218
137
    
return ccv_matrix_eq((ccv_matrix_t*)a, (ccv_matrix_t*)b)16
;
219
121
  // Otherwise, do our own thing.
220
121
  if (CCV_GET_DATA_TYPE(a->type) != CCV_GET_DATA_TYPE(b->type))
221
121
    
return -10
;
222
121
  // Only support 32F at this point.
223
121
  assert(CCV_GET_DATA_TYPE(a->type) == CCV_32F);
224
121
  int i, c = 1;
225
327
  for (i = 0; i < CCV_NNC_MAX_DIM_ALLOC; 
i++206
)
226
327
  {
227
327
    if (!a->info.dim[i] && 
!b->info.dim[i]121
)
228
121
      break;
229
206
    if (a->info.dim[i] != b->info.dim[i])
230
0
      return -1;
231
206
    c *= a->info.dim[i];
232
206
  }
233
121
  // Read: http://www.cygnus-software.com/papers/comparingfloats/comparingfloats.htm
234
121
  // http://floating-point-gui.de/errors/comparison/
235
121
  static const float epsi = FLT_EPSILON;
236
121
  static const int32_t ulps = 128; // so that for 1 and 1.000015 will be treated as the same.
237
763k
  for (i = 0; i < c; 
i++763k
)
238
763k
  {
239
763k
    // Although this is float point, I use integer as a way to compare.
240
763k
    int32_t i32a = a->data.i32[i];
241
763k
    if (i32a < 0)
242
2.07k
      i32a = 0x80000000 - i32a;
243
763k
    int32_t i32b = b->data.i32[i];
244
763k
    if (i32b < 0)
245
2.07k
      i32b = 0x80000000 - i32b;
246
763k
    if (abs(i32a - i32b) > ulps && 
fabsf(a->data.f32[i] - b->data.f32[i]) > epsi3
)
247
0
      return -1;
248
763k
  }
249
121
  return 0;
250
121
}