Coverage Report

Created: 2021-04-14 19:30

/home/liu/buildslave/linux-x64-runtests/build/lib/cuda/cwc_convnet_ext.c
Line
Count
Source (jump to first uncovered line)
1
#include "cwc.h"
2
#include "../ccv_internal.h"
3
#ifdef HAVE_GSL
4
#include <gsl/gsl_rng.h>
5
#include <gsl/gsl_randist.h>
6
#endif
7
#include "cwc_ext.h"
8
#ifdef USE_OPENMP
9
#include <omp.h>
10
#endif
11
#ifdef USE_DISPATCH
12
#include <dispatch/dispatch.h>
13
#endif
14
15
#ifdef HAVE_GSL
16
static void _cwc_convnet_random_image_manipulation(gsl_rng* rng, ccv_dense_matrix_t* image, float image_manipulation)
17
{
18
  assert(rng && CCV_GET_CHANNEL(image->type) == CCV_C3 && image_manipulation > 0 && image_manipulation <= 1);
19
  int ord[3] = {0, 1, 2};
20
  gsl_ran_shuffle(rng, ord, 3, sizeof(int));
21
  int i;
22
  for (i = 0; i < 3; i++)
23
    // change the applying order
24
    switch (ord[i])
25
    {
26
      case 0:
27
        // introduce some brightness changes to the original image
28
        ccv_scale(image, (ccv_matrix_t**)&image, 0, gsl_rng_uniform_pos(rng) * image_manipulation * 2 + (1 - image_manipulation));
29
        break;
30
      case 1:
31
        // introduce some saturation changes to the original image
32
        ccv_saturation(image, &image, 0, gsl_rng_uniform_pos(rng) * image_manipulation * 2 + (1 - image_manipulation));
33
        break;
34
      case 2:
35
        // introduce some contrast changes to the original image
36
        ccv_contrast(image, &image, 0, gsl_rng_uniform_pos(rng) * image_manipulation * 2 + (1 - image_manipulation));
37
        break;
38
    }
39
}
40
41
void cwc_convnet_batch_formation(gsl_rng* rng, ccv_array_t* categorizeds, ccv_dense_matrix_t* mean_activity, ccv_dense_matrix_t* eigenvectors, ccv_dense_matrix_t* eigenvalues, float image_manipulation, float color_gain, int* idx, ccv_size_t dim, int min_dim, int max_dim, int rows, int cols, int channels, int category_count, int symmetric, int batch, int offset, int size, float* b, int* c)
42
0
{
43
0
  assert(size > 0 && size <= batch);
44
0
  assert(min_dim >= rows && min_dim >= cols);
45
0
  assert(max_dim >= min_dim);
46
0
  float* channel_gains = (float*)alloca(sizeof(float) * channels * size);
47
0
  memset(channel_gains, 0, sizeof(float) * channels * size);
48
0
  int i;
49
0
  gsl_rng** rngs = (gsl_rng**)alloca(sizeof(gsl_rng*) * size);
50
0
  memset(rngs, 0, sizeof(gsl_rng*) * size);
51
0
  if (rng)
52
0
    for (i = 0; i < size; i++)
53
0
    {
54
0
      rngs[i] = gsl_rng_alloc(gsl_rng_default);
55
0
      gsl_rng_set(rngs[i], gsl_rng_get(rng));
56
0
    }
57
0
  parallel_for(i, size) {
58
0
    int j, k;
59
0
    assert(offset + i < categorizeds->rnum);
60
0
    ccv_categorized_t* categorized = (ccv_categorized_t*)ccv_array_get(categorizeds, idx ? idx[offset + i] : offset + i);
61
0
    assert(categorized->c < category_count && categorized->c >= 0); // now only accept classes listed
62
0
    if (c)
63
0
      c[i] = categorized->c;
64
0
    ccv_dense_matrix_t* image = 0;
65
0
    switch (categorized->type)
66
0
    {
67
0
      case CCV_CATEGORIZED_DENSE_MATRIX:
68
0
        image = categorized->matrix;
69
0
        break;
70
0
      case CCV_CATEGORIZED_FILE:
71
0
        image = 0;
72
0
        ccv_read(categorized->file.filename, &image, CCV_IO_ANY_FILE | CCV_IO_RGB_COLOR);
73
0
        break;
74
0
    }
75
0
    if (image)
76
0
    {
77
0
      // first resize to between min_dim and max_dim
78
0
      ccv_dense_matrix_t* input = 0;
79
0
      ccv_size_t resize = dim;
80
0
      if (rngs[i]) // randomize the resized dimensions
81
0
      {
82
0
        int d = gsl_rng_uniform_int(rngs[i], max_dim - min_dim + 1) + min_dim;
83
0
        resize = ccv_size(d, d);
84
0
      }
85
0
      // if neither side is the same as expected, we have to resize first
86
0
      if (resize.height != image->rows && resize.width != image->cols)
87
0
        ccv_convnet_input_formation(resize, image, &input);
88
0
      else
89
0
        input = image;
90
0
      if (rngs[i] && image_manipulation > 0)
91
0
        _cwc_convnet_random_image_manipulation(rngs[i], input, image_manipulation);
92
0
      ccv_dense_matrix_t* patch = 0;
93
0
      if (input->cols != cols || input->rows != rows)
94
0
      {
95
0
        int x = rngs[i] ? gsl_rng_uniform_int(rngs[i], input->cols - cols + 1) : (input->cols - cols + 1) / 2;
96
0
        int y = rngs[i] ? gsl_rng_uniform_int(rngs[i], input->rows - rows + 1) : (input->rows - rows + 1) / 2;
97
0
        ccv_slice(input, (ccv_matrix_t**)&patch, CCV_32F, y, x, rows, cols);
98
0
      } else
99
0
        ccv_shift(input, (ccv_matrix_t**)&patch, CCV_32F, 0, 0); // converting to 32f
100
0
      if (input != image) // only unload if we created new input
101
0
        ccv_matrix_free(input);
102
0
      // we loaded image in, deallocate it now
103
0
      if (categorized->type != CCV_CATEGORIZED_DENSE_MATRIX)
104
0
        ccv_matrix_free(image);
105
0
      // random horizontal reflection
106
0
      if (symmetric && rngs[i] && gsl_rng_uniform_int(rngs[i], 2) == 0)
107
0
        ccv_flip(patch, &patch, 0, CCV_FLIP_X);
108
0
      int x = rngs[i] ? gsl_rng_uniform_int(rngs[i], mean_activity->cols - cols + 1) : (mean_activity->cols - cols + 1) / 2;
109
0
      int y = rngs[i] ? gsl_rng_uniform_int(rngs[i], mean_activity->rows - rows + 1) : (mean_activity->rows - rows + 1) / 2;
110
0
      ccv_dense_matrix_t mean_patch = ccv_reshape(mean_activity, y, x, rows, cols);
111
0
      ccv_subtract(patch, &mean_patch, (ccv_matrix_t**)&patch, 0);
112
0
      assert(channels == CCV_GET_CHANNEL(patch->type));
113
0
      if (color_gain > 0 && rngs[i] && eigenvectors && eigenvalues)
114
0
      {
115
0
        assert(channels == 3); // only support RGB color gain
116
0
        memset(channel_gains + channels * i, 0, sizeof(float) * channels);
117
0
        for (j = 0; j < channels; j++)
118
0
        {
119
0
          float alpha = gsl_ran_gaussian(rngs[i], color_gain) * eigenvalues->data.f64[j];
120
0
          for (k = 0; k < channels; k++)
121
0
            channel_gains[k + i * channels] += eigenvectors->data.f64[j * channels + k] * alpha;
122
0
        }
123
0
      }
124
0
      for (j = 0; j < channels; j++)
125
0
        for (k = 0; k < rows * cols; k++)
126
0
          b[(j * rows * cols + k) * batch + i] = patch->data.f32[k * channels + j] + channel_gains[j + i * channels];
127
0
      ccv_matrix_free(patch);
128
0
    } else
129
0
      PRINT(CCV_CLI_ERROR, "cannot load %s.\n", categorized->file.filename);
130
0
  } parallel_endfor
131
0
  if (rng)
132
0
    for (i = 0; i < size; i++)
133
0
      gsl_rng_free(rngs[i]);
134
0
}
135
#endif
136
137
void cwc_convnet_mean_formation(ccv_array_t* categorizeds, ccv_size_t dim, int channels, int symmetric, ccv_dense_matrix_t** b)
138
0
{
139
0
  int i, count = 0;
140
0
  ccv_dense_matrix_t* c = ccv_dense_matrix_new(dim.height, dim.width, channels | CCV_64F, 0, 0);
141
0
  ccv_zero(c);
142
0
  ccv_dense_matrix_t* db = *b = ccv_dense_matrix_renew(*b, dim.height, dim.width, channels | CCV_32F, channels | CCV_32F, 0);
143
0
  for (i = 0; i < categorizeds->rnum; i++)
144
0
  {
145
0
    if (i % 23 == 0 || i == categorizeds->rnum - 1)
146
0
      FLUSH(CCV_CLI_INFO, " - compute mean activity %d / %d", i + 1, categorizeds->rnum);
147
0
    ccv_categorized_t* categorized = (ccv_categorized_t*)ccv_array_get(categorizeds, i);
148
0
    ccv_dense_matrix_t* image = 0;
149
0
    switch (categorized->type)
150
0
    {
151
0
      case CCV_CATEGORIZED_DENSE_MATRIX:
152
0
        image = categorized->matrix;
153
0
        break;
154
0
      case CCV_CATEGORIZED_FILE:
155
0
        ccv_read(categorized->file.filename, &image, CCV_IO_ANY_FILE | CCV_IO_RGB_COLOR);
156
0
        break;
157
0
    }
158
0
    if (!image)
159
0
    {
160
0
      PRINT(CCV_CLI_ERROR, "cannot load %s.\n", categorized->file.filename);
161
0
      continue;
162
0
    }
163
0
    ccv_dense_matrix_t* patch = 0;
164
0
    if (image->cols != dim.width || image->rows != dim.height)
165
0
    {
166
0
      int x = (image->cols - dim.width + 1) / 2;
167
0
      int y = (image->rows - dim.height + 1) / 2;
168
0
      assert(x == 0 || y == 0);
169
0
      ccv_slice(image, (ccv_matrix_t**)&patch, CCV_32F, y, x, dim.height, dim.width);
170
0
    } else
171
0
      ccv_shift(image, (ccv_matrix_t**)&patch, CCV_32F, 0, 0); // converting to 32f
172
0
    if (categorized->type != CCV_CATEGORIZED_DENSE_MATRIX)
173
0
      ccv_matrix_free(image);
174
0
    ccv_add(patch, c, (ccv_matrix_t**)&c, CCV_64F);
175
0
    ++count;
176
0
    ccv_matrix_free(patch);
177
0
  }
178
0
  if (symmetric)
179
0
  {
180
0
    int j, k;
181
0
    double p = 0.5 / count;
182
0
    double* cptr = c->data.f64;
183
0
    float* dbptr = db->data.f32;
184
0
    for (i = 0; i < db->rows; i++)
185
0
    {
186
0
      for (j = 0; j < db->cols; j++)
187
0
        for (k = 0; k < channels; k++)
188
0
          dbptr[j * channels + k] = p * (cptr[j * channels + k] + cptr[(c->cols - j - 1) * channels + k]);
189
0
      dbptr += db->cols * channels;
190
0
      cptr += c->cols * channels;
191
0
    }
192
0
  } else {
193
0
    double p = 1.0 / count;
194
0
    for (i = 0; i < dim.height * dim.width * channels; i++)
195
0
      db->data.f32[i] = p * c->data.f64[i];
196
0
  }
197
0
  ccv_matrix_free(c);
198
0
  PRINT(CCV_CLI_INFO, "\n");
199
0
}
200
201
void cwc_convnet_channel_eigen(ccv_array_t* categorizeds, ccv_dense_matrix_t* mean_activity, ccv_size_t dim, int channels, ccv_dense_matrix_t** eigenvectors, ccv_dense_matrix_t** eigenvalues)
202
0
{
203
0
  assert(channels == 3); // this function cannot handle anything other than 3x3 covariance matrix
204
0
  double* mean_value = (double*)alloca(sizeof(double) * channels);
205
0
  memset(mean_value, 0, sizeof(double) * channels);
206
0
  assert(CCV_GET_CHANNEL(mean_activity->type) == channels);
207
0
  assert(mean_activity->rows == dim.height);
208
0
  assert(mean_activity->cols == dim.width);
209
0
  int i, j, k, c, count = 0;
210
0
  for (i = 0; i < dim.height * dim.width; i++)
211
0
    for (k = 0; k < channels; k++)
212
0
      mean_value[k] += mean_activity->data.f32[i * channels + k];
213
0
  for (i = 0; i < channels; i++)
214
0
    mean_value[i] = mean_value[i] / (dim.height * dim.width);
215
0
  double* covariance = (double*)alloca(sizeof(double) * channels * channels);
216
0
  memset(covariance, 0, sizeof(double) * channels * channels);
217
0
  for (c = 0; c < categorizeds->rnum; c++)
218
0
  {
219
0
    if (c % 23 == 0 || c == categorizeds->rnum - 1)
220
0
      FLUSH(CCV_CLI_INFO, " - compute covariance matrix for data augmentation (color gain) %d / %d", c + 1, categorizeds->rnum);
221
0
    ccv_categorized_t* categorized = (ccv_categorized_t*)ccv_array_get(categorizeds, c);
222
0
    ccv_dense_matrix_t* image = 0;
223
0
    switch (categorized->type)
224
0
    {
225
0
      case CCV_CATEGORIZED_DENSE_MATRIX:
226
0
        image = categorized->matrix;
227
0
        break;
228
0
      case CCV_CATEGORIZED_FILE:
229
0
        ccv_read(categorized->file.filename, &image, CCV_IO_ANY_FILE | CCV_IO_RGB_COLOR);
230
0
        break;
231
0
    }
232
0
    if (!image)
233
0
    {
234
0
      PRINT(CCV_CLI_ERROR, "cannot load %s.\n", categorized->file.filename);
235
0
      continue;
236
0
    }
237
0
    ccv_dense_matrix_t* patch = 0;
238
0
    if (image->cols != dim.width || image->rows != dim.height)
239
0
    {
240
0
      int x = (image->cols - dim.width + 1) / 2;
241
0
      int y = (image->rows - dim.height + 1) / 2;
242
0
      assert(x == 0 || y == 0);
243
0
      ccv_slice(image, (ccv_matrix_t**)&patch, CCV_32F, y, x, dim.height, dim.width);
244
0
    } else
245
0
      ccv_shift(image, (ccv_matrix_t**)&patch, CCV_32F, 0, 0); // converting to 32f
246
0
    if (categorized->type != CCV_CATEGORIZED_DENSE_MATRIX)
247
0
      ccv_matrix_free(image);
248
0
    for (i = 0; i < dim.width * dim.height; i++)
249
0
      for (j = 0; j < channels; j++)
250
0
        for (k = j; k < channels; k++)
251
0
          covariance[j * channels + k] += (patch->data.f32[i * channels + j] - mean_value[j]) * (patch->data.f32[i * channels + k] - mean_value[k]);
252
0
    ++count;
253
0
    ccv_matrix_free(patch);
254
0
  }
255
0
  for (i = 0; i < channels; i++)
256
0
    for (j = 0; j < i; j++)
257
0
      covariance[i * channels + j] = covariance[j * channels + i];
258
0
  double p = 1.0 / ((double)count * dim.height * dim.width);
259
0
  for (i = 0; i < channels; i++)
260
0
    for (j = 0; j < channels; j++)
261
0
      covariance[i * channels + j] *= p; // scale down
262
0
  ccv_dense_matrix_t covm = ccv_dense_matrix(3, 3, CCV_64F | CCV_C1, covariance, 0);
263
0
  ccv_eigen(&covm, eigenvectors, eigenvalues, CCV_64F, 1e-8);
264
0
  PRINT(CCV_CLI_INFO, "\n");
265
0
}