Coverage Report

Created: 2019-07-03 22:50

/home/liu/buildslave/linux-x64-runtests/build/test/unit/nnc/dataframe.tests.c
Line
Count
Source
1
#include "case.h"
2
#include "ccv_case.h"
3
#include "ccv_nnc_case.h"
4
#include <ccv.h>
5
#include <nnc/ccv_nnc.h>
6
#include <nnc/ccv_nnc_easy.h>
7
#include "3rdparty/dsfmt/dSFMT.h"
8
9
TEST_SETUP()
10
{
11
  ccv_nnc_init();
12
}
13
14
static int _ccv_iter_accessed = 0;
15
16
static void _ccv_iter_int(const int column_idx, const int* row_idxs, const int row_size, void** const data, void* const context, ccv_nnc_stream_context_t* const stream_context)
17
117
{
18
117
  int* const array = (int*)context;
19
117
  int i;
20
268
  for (i = 0; i < row_size; 
i++151
)
21
151
    data[i] = (void*)(intptr_t)array[row_idxs[i]];
22
117
  ++_ccv_iter_accessed;
23
117
}
24
25
TEST_CASE("iterate through a simple dataframe")
26
1
{
27
1
  int int_array[8] = {
28
1
    0, 3, 4, 5, 6, 7, 8, 9
29
1
  };
30
1
  ccv_cnnp_column_data_t columns[] = {
31
1
    {
32
1
      .data_enum = _ccv_iter_int,
33
1
      .context = int_array,
34
1
    }
35
1
  };
36
1
  ccv_cnnp_dataframe_t* const dataframe = ccv_cnnp_dataframe_new(columns, sizeof(columns) / sizeof(columns[0]), 8);
37
1
  ccv_cnnp_dataframe_iter_t* const iter1 = ccv_cnnp_dataframe_iter_new(dataframe, COLUMN_ID_LIST(0));
38
1
  int result[8];
39
1
  int i = 0;
40
1
  void* data;
41
9
  while (0 == ccv_cnnp_dataframe_iter_next(iter1, &data, 1, 0))
42
8
    result[i++] = (int)(intptr_t)data;
43
1
  ccv_cnnp_dataframe_iter_free(iter1);
44
1
  REQUIRE_ARRAY_EQ(int, int_array, result, 8, "iterated result and actual result should be the same");
45
1
  // iter2 test some prefetch capacities.
46
1
  ccv_cnnp_dataframe_iter_t* const iter2 = ccv_cnnp_dataframe_iter_new(dataframe, COLUMN_ID_LIST(0));
47
4
  for (i = 0; i < 3; 
i++3
)
48
3
    ccv_cnnp_dataframe_iter_prefetch(iter2, 1, 0);
49
1
  _ccv_iter_accessed = 0;
50
4
  for (i = 0; i < 3; 
i++3
)
51
3
  {
52
3
    ccv_cnnp_dataframe_iter_next(iter2, &data, 1, 0);
53
3
    result[i] = (int)(intptr_t)data;
54
3
  }
55
1
  REQUIRE_EQ(_ccv_iter_accessed, 0, "the iterator is not accessed at all, because prefetching");
56
1
  ccv_cnnp_dataframe_iter_next(iter2, &data, 1, 0);
57
1
  result[3] = (int)(intptr_t)data;
58
1
  REQUIRE_EQ(_ccv_iter_accessed, 1, "the iterator is accessed, because no prefetching");
59
1
  ccv_cnnp_dataframe_iter_prefetch(iter2, 1, 0);
60
1
  REQUIRE_EQ(_ccv_iter_accessed, 2, "the iterator is accessed again, for prefetching");
61
1
  _ccv_iter_accessed = 0;
62
5
  for (i = 4; i < 8; 
i++4
)
63
4
  {
64
4
    ccv_cnnp_dataframe_iter_next(iter2, &data, 1, 0);
65
4
    result[i] = (int)(intptr_t)data;
66
4
  }
67
1
  REQUIRE_EQ(_ccv_iter_accessed, 3, "the iterator accessed 3 times, the first is prefetching");
68
1
  ccv_cnnp_dataframe_iter_free(iter2);
69
1
  REQUIRE_ARRAY_EQ(int, int_array, result, 8, "iterated result and actual result should be the same");
70
1
  // iter3 test more prefetch behavior.
71
1
  ccv_cnnp_dataframe_iter_t* const iter3 = ccv_cnnp_dataframe_iter_new(dataframe, COLUMN_ID_LIST(0));
72
4
  for (i = 0; i < 3; 
i++3
)
73
3
    ccv_cnnp_dataframe_iter_prefetch(iter3, 1, 0);
74
3
  for (i = 0; i < 2; 
i++2
)
75
2
  {
76
2
    _ccv_iter_accessed = 0;
77
2
    ccv_cnnp_dataframe_iter_next(iter3, &data, 1, 0);
78
2
    REQUIRE_EQ(_ccv_iter_accessed, 0, "no iterator accessed");
79
2
    result[i] = (int)(intptr_t)data;
80
2
    ccv_cnnp_dataframe_iter_prefetch(iter3, 1, 0);
81
2
    REQUIRE_EQ(_ccv_iter_accessed, 1, "iterator accessed for prefetching");
82
2
  }
83
3
  
for (i = 2; 1
i < 4;
i++2
)
84
2
    ccv_cnnp_dataframe_iter_prefetch(iter3, 1, 0);
85
1
  _ccv_iter_accessed = 0;
86
6
  for (i = 2; i < 7; 
i++5
)
87
5
  {
88
5
    ccv_cnnp_dataframe_iter_next(iter3, &data, 1, 0);
89
5
    result[i] = (int)(intptr_t)data;
90
5
  }
91
1
  REQUIRE_EQ(_ccv_iter_accessed, 0, "no iterator accessed");
92
1
  ccv_cnnp_dataframe_iter_next(iter3, &data, 1, 0);
93
1
  result[7] = (int)(intptr_t)data;
94
1
  REQUIRE_EQ(_ccv_iter_accessed, 1, "iterator accessed twice");
95
1
  const int fail1 = ccv_cnnp_dataframe_iter_prefetch(iter3, 1, 0);
96
1
  REQUIRE_EQ(fail1, -1, "cannot advance no more");
97
1
  ccv_cnnp_dataframe_iter_free(iter3);
98
1
  REQUIRE_ARRAY_EQ(int, int_array, result, 8, "iterated result and actual result should be the same");
99
1
  ccv_cnnp_dataframe_free(dataframe);
100
1
}
101
102
static void _ccv_int_plus_1(void*** const column_data, const int column_size, const int batch_size, void** const data, void* const context, ccv_nnc_stream_context_t* const stream_context)
103
77
{
104
77
  int i;
105
170
  for (i = 0; i < batch_size; 
i++93
)
106
93
  {
107
93
    int k = (int)(intptr_t)column_data[0][i];
108
93
    data[i] = (void*)(intptr_t)(k + 1);
109
93
  }
110
77
}
111
112
TEST_CASE("iterate through derived column")
113
1
{
114
1
  int int_array[8] = {
115
1
    2, 3, 4, 5, 6, 7, 8, 9
116
1
  };
117
1
  ccv_cnnp_column_data_t columns[] = {
118
1
    {
119
1
      .data_enum = _ccv_iter_int,
120
1
      .context = int_array,
121
1
    }
122
1
  };
123
1
  ccv_cnnp_dataframe_t* const dataframe = ccv_cnnp_dataframe_new(columns, sizeof(columns) / sizeof(columns[0]), 8);
124
1
  const int derived = ccv_cnnp_dataframe_map(dataframe, _ccv_int_plus_1, 0, 0, COLUMN_ID_LIST(0), 0, 0);
125
1
  assert(derived > 0);
126
1
  ccv_cnnp_dataframe_iter_t* const iter = ccv_cnnp_dataframe_iter_new(dataframe, COLUMN_ID_LIST(derived));
127
1
  int result[8];
128
1
  int i = 0;
129
1
  void* data;
130
9
  while (0 == ccv_cnnp_dataframe_iter_next(iter, &data, 1, 0))
131
8
    result[i++] = (int)(intptr_t)data;
132
1
  ccv_cnnp_dataframe_iter_free(iter);
133
9
  for (i = 0; i < 8; 
i++8
)
134
8
    ++int_array[i];
135
1
  REQUIRE_ARRAY_EQ(int, int_array, result, 8, "iterated result and actual result should be the same");
136
1
  // iter2 test some prefetch capacities.
137
1
  ccv_cnnp_dataframe_iter_t* const iter2 = ccv_cnnp_dataframe_iter_new(dataframe, COLUMN_ID_LIST(derived));
138
4
  for (i = 0; i < 3; 
i++3
)
139
3
    ccv_cnnp_dataframe_iter_prefetch(iter2, 1, 0);
140
1
  _ccv_iter_accessed = 0;
141
4
  for (i = 0; i < 3; 
i++3
)
142
3
  {
143
3
    ccv_cnnp_dataframe_iter_next(iter2, &data, 1, 0);
144
3
    result[i] = (int)(intptr_t)data;
145
3
  }
146
1
  REQUIRE_EQ(_ccv_iter_accessed, 0, "the iterator is not accessed at all, because prefetching");
147
1
  ccv_cnnp_dataframe_iter_next(iter2, &data, 1, 0);
148
1
  result[3] = (int)(intptr_t)data;
149
1
  REQUIRE_EQ(_ccv_iter_accessed, 1, "the iterator is accessed, because no prefetching");
150
1
  ccv_cnnp_dataframe_iter_prefetch(iter2, 1, 0);
151
1
  REQUIRE_EQ(_ccv_iter_accessed, 2, "the iterator is accessed again, for prefetching");
152
1
  _ccv_iter_accessed = 0;
153
3
  for (i = 4; i < 6; 
i++2
)
154
2
  {
155
2
    ccv_cnnp_dataframe_iter_next(iter2, &data, 1, 0);
156
2
    result[i] = (int)(intptr_t)data;
157
2
  }
158
1
  REQUIRE_EQ(_ccv_iter_accessed, 1, "the iterator accessed 3 times, the first is prefetching");
159
1
  const int success0 = ccv_cnnp_dataframe_iter_prefetch(iter2, 1, 0);
160
1
  REQUIRE_EQ(success0, 0, "success");
161
1
  const int success1 = ccv_cnnp_dataframe_iter_prefetch(iter2, 1, 0);
162
1
  REQUIRE_EQ(success1, 0, "success");
163
1
  const int fail0 = ccv_cnnp_dataframe_iter_prefetch(iter2, 1, 0);
164
1
  REQUIRE_EQ(fail0, -1, "should fail");
165
7
  for (i = 0; i < 6; 
i++6
)
166
6
    ++int_array[i];
167
1
  ccv_cnnp_dataframe_iter_free(iter2);
168
1
  REQUIRE_ARRAY_EQ(int, int_array, result, 6, "iterated result and actual result should be the same up to 6");
169
1
  // iter3 test more prefetch behavior.
170
1
  ccv_cnnp_dataframe_iter_t* const iter3 = ccv_cnnp_dataframe_iter_new(dataframe, COLUMN_ID_LIST(derived));
171
4
  for (i = 0; i < 3; 
i++3
)
172
3
    ccv_cnnp_dataframe_iter_prefetch(iter3, 1, 0);
173
3
  for (i = 0; i < 2; 
i++2
)
174
2
  {
175
2
    _ccv_iter_accessed = 0;
176
2
    ccv_cnnp_dataframe_iter_next(iter3, &data, 1, 0);
177
2
    REQUIRE_EQ(_ccv_iter_accessed, 0, "no iterator accessed");
178
2
    result[i] = (int)(intptr_t)data;
179
2
    ccv_cnnp_dataframe_iter_prefetch(iter3, 1, 0);
180
2
    REQUIRE_EQ(_ccv_iter_accessed, 1, "iterator accessed for prefetching");
181
2
  }
182
3
  
for (i = 2; 1
i < 4;
i++2
)
183
2
    ccv_cnnp_dataframe_iter_prefetch(iter3, 1, 0);
184
1
  _ccv_iter_accessed = 0;
185
6
  for (i = 2; i < 7; 
i++5
)
186
5
  {
187
5
    ccv_cnnp_dataframe_iter_next(iter3, &data, 1, 0);
188
5
    result[i] = (int)(intptr_t)data;
189
5
  }
190
1
  REQUIRE_EQ(_ccv_iter_accessed, 0, "no iterator accessed");
191
1
  ccv_cnnp_dataframe_iter_next(iter3, &data, 1, 0);
192
1
  result[7] = (int)(intptr_t)data;
193
1
  REQUIRE_EQ(_ccv_iter_accessed, 1, "iterator accessed twice");
194
1
  const int fail1 = ccv_cnnp_dataframe_iter_prefetch(iter3, 1, 0);
195
1
  REQUIRE_EQ(fail1, -1, "cannot advance no more");
196
9
  for (i = 0; i < 8; 
i++8
)
197
8
    ++int_array[i];
198
1
  ccv_cnnp_dataframe_iter_free(iter3);
199
1
  REQUIRE_ARRAY_EQ(int, int_array, result, 8, "iterated result and actual result should be the same");
200
1
  ccv_cnnp_dataframe_free(dataframe);
201
1
}
202
203
TEST_CASE("iterate through derived column with cursor reset")
204
1
{
205
1
  int int_array[8] = {
206
1
    2, 3, 4, 5, 6, 7, 8, 9
207
1
  };
208
1
  ccv_cnnp_column_data_t columns[] = {
209
1
    {
210
1
      .data_enum = _ccv_iter_int,
211
1
      .context = int_array,
212
1
    }
213
1
  };
214
1
  ccv_cnnp_dataframe_t* const dataframe = ccv_cnnp_dataframe_new(columns, sizeof(columns) / sizeof(columns[0]), 8);
215
1
  const int derived = ccv_cnnp_dataframe_map(dataframe, _ccv_int_plus_1, 0, 0, COLUMN_ID_LIST(0), 0, 0);
216
1
  assert(derived > 0);
217
1
  int i;
218
1
  void* data;
219
1
  int result[8];
220
1
  ccv_cnnp_dataframe_iter_t* const iter = ccv_cnnp_dataframe_iter_new(dataframe, COLUMN_ID_LIST(derived));
221
4
  for (i = 0; i < 3; 
i++3
)
222
3
    ccv_cnnp_dataframe_iter_prefetch(iter, 1, 0);
223
3
  for (i = 0; i < 2; 
i++2
)
224
2
  {
225
2
    _ccv_iter_accessed = 0;
226
2
    ccv_cnnp_dataframe_iter_next(iter, &data, 1, 0);
227
2
    REQUIRE_EQ(_ccv_iter_accessed, 0, "no iterator accessed");
228
2
    result[i] = (int)(intptr_t)data;
229
2
    ccv_cnnp_dataframe_iter_prefetch(iter, 1, 0);
230
2
    REQUIRE_EQ(_ccv_iter_accessed, 1, "iterator accessed for prefetching");
231
2
  }
232
3
  
for (i = 2; 1
i < 4;
i++2
)
233
2
    ccv_cnnp_dataframe_iter_prefetch(iter, 1, 0);
234
1
  ccv_cnnp_dataframe_iter_set_cursor(iter, 0);
235
1
  ccv_cnnp_dataframe_iter_prefetch(iter, 1, 0);
236
1
  ccv_cnnp_dataframe_iter_prefetch(iter, 1, 0);
237
1
  _ccv_iter_accessed = 0;
238
8
  for (i = 0; i < 7; 
i++7
)
239
7
  {
240
7
    ccv_cnnp_dataframe_iter_next(iter, &data, 1, 0);
241
7
    result[i] = (int)(intptr_t)data;
242
7
  }
243
1
  REQUIRE_EQ(_ccv_iter_accessed, 5, "5 iterator accessed");
244
1
  _ccv_iter_accessed = 0;
245
1
  ccv_cnnp_dataframe_iter_next(iter, &data, 1, 0);
246
1
  result[7] = (int)(intptr_t)data;
247
1
  REQUIRE_EQ(_ccv_iter_accessed, 1, "iterator accessed");
248
1
  const int fail1 = ccv_cnnp_dataframe_iter_prefetch(iter, 1, 0);
249
1
  REQUIRE_EQ(fail1, -1, "cannot advance no more");
250
9
  for (i = 0; i < 8; 
i++8
)
251
8
    ++int_array[i];
252
1
  ccv_cnnp_dataframe_iter_free(iter);
253
1
  REQUIRE_ARRAY_EQ(int, int_array, result, 8, "iterated result and actual result should be the same");
254
1
  ccv_cnnp_dataframe_free(dataframe);
255
1
}
256
257
TEST_CASE("iterate with prefetch more than 1 item, variant 1")
258
1
{
259
1
  int int_array[8] = {
260
1
    2, 3, 4, 5, 6, 7, 8, 9
261
1
  };
262
1
  ccv_cnnp_column_data_t columns[] = {
263
1
    {
264
1
      .data_enum = _ccv_iter_int,
265
1
      .context = int_array,
266
1
    }
267
1
  };
268
1
  ccv_cnnp_dataframe_t* const dataframe = ccv_cnnp_dataframe_new(columns, sizeof(columns) / sizeof(columns[0]), 8);
269
1
  const int derived = ccv_cnnp_dataframe_map(dataframe, _ccv_int_plus_1, 0, 0, COLUMN_ID_LIST(0), 0, 0);
270
1
  assert(derived > 0);
271
1
  int i;
272
1
  void* data;
273
1
  int result[8];
274
1
  ccv_cnnp_dataframe_iter_t* const iter = ccv_cnnp_dataframe_iter_new(dataframe, COLUMN_ID_LIST(derived));
275
1
  _ccv_iter_accessed = 0;
276
1
  ccv_cnnp_dataframe_iter_prefetch(iter, 3, 0);
277
1
  REQUIRE_EQ(_ccv_iter_accessed, 1, "iter accessed once for prefetching");
278
3
  for (i = 0; i < 2; 
i++2
)
279
2
  {
280
2
    _ccv_iter_accessed = 0;
281
2
    ccv_cnnp_dataframe_iter_next(iter, &data, 1, 0);
282
2
    REQUIRE_EQ(_ccv_iter_accessed, 0, "no iterator accessed");
283
2
    result[i] = (int)(intptr_t)data;
284
2
    ccv_cnnp_dataframe_iter_prefetch(iter, 1, 0);
285
2
    REQUIRE_EQ(_ccv_iter_accessed, 1, "iterator accessed for prefetching");
286
2
  }
287
1
  ccv_cnnp_dataframe_iter_prefetch(iter, 4, 0);
288
1
  _ccv_iter_accessed = 0;
289
7
  for (i = 2; i < 8; 
i++6
)
290
6
  {
291
6
    ccv_cnnp_dataframe_iter_next(iter, &data, 1, 0);
292
6
    result[i] = (int)(intptr_t)data;
293
6
  }
294
1
  REQUIRE_EQ(_ccv_iter_accessed, 0, "no iterator accessed");
295
9
  for (i = 0; i < 8; 
i++8
)
296
8
    ++int_array[i];
297
1
  ccv_cnnp_dataframe_iter_free(iter);
298
1
  REQUIRE_ARRAY_EQ(int, int_array, result, 8, "iterated result and actual result should be the same");
299
1
  ccv_cnnp_dataframe_free(dataframe);
300
1
}
301
302
TEST_CASE("iterate with prefetch more than 1 item, variant 2")
303
1
{
304
1
  int int_array[8] = {
305
1
    2, 3, 4, 5, 6, 7, 8, 9
306
1
  };
307
1
  ccv_cnnp_column_data_t columns[] = {
308
1
    {
309
1
      .data_enum = _ccv_iter_int,
310
1
      .context = int_array,
311
1
    }
312
1
  };
313
1
  ccv_cnnp_dataframe_t* const dataframe = ccv_cnnp_dataframe_new(columns, sizeof(columns) / sizeof(columns[0]), 8);
314
1
  const int derived = ccv_cnnp_dataframe_map(dataframe, _ccv_int_plus_1, 0, 0, COLUMN_ID_LIST(0), 0, 0);
315
1
  assert(derived > 0);
316
1
  int i;
317
1
  void* data;
318
1
  int result[8];
319
1
  ccv_cnnp_dataframe_iter_t* const iter = ccv_cnnp_dataframe_iter_new(dataframe, COLUMN_ID_LIST(derived));
320
1
  _ccv_iter_accessed = 0;
321
1
  ccv_cnnp_dataframe_iter_prefetch(iter, 3, 0);
322
1
  REQUIRE_EQ(_ccv_iter_accessed, 1, "iter accessed once for prefetching");
323
4
  for (i = 0; i < 3; 
i++3
)
324
3
  {
325
3
    _ccv_iter_accessed = 0;
326
3
    ccv_cnnp_dataframe_iter_next(iter, &data, 1, 0);
327
3
    REQUIRE_EQ(_ccv_iter_accessed, 0, "no iterator accessed");
328
3
    result[i] = (int)(intptr_t)data;
329
3
  }
330
1
  ccv_cnnp_dataframe_iter_prefetch(iter, 2, 0);
331
1
  ccv_cnnp_dataframe_iter_next(iter, &data, 1, 0);
332
1
  result[3] = (int)(intptr_t)data;
333
1
  // Now, I have 1 prefetched, in the middle, and can prefetch 2 more without reallocating.
334
1
  ccv_cnnp_dataframe_iter_prefetch(iter, 3, 0);
335
1
  _ccv_iter_accessed = 0;
336
5
  for (i = 4; i < 8; 
i++4
)
337
4
  {
338
4
    ccv_cnnp_dataframe_iter_next(iter, &data, 1, 0);
339
4
    result[i] = (int)(intptr_t)data;
340
4
  }
341
1
  REQUIRE_EQ(_ccv_iter_accessed, 0, "no iterator accessed");
342
9
  for (i = 0; i < 8; 
i++8
)
343
8
    ++int_array[i];
344
1
  ccv_cnnp_dataframe_iter_free(iter);
345
1
  REQUIRE_ARRAY_EQ(int, int_array, result, 8, "iterated result and actual result should be the same");
346
1
  ccv_cnnp_dataframe_free(dataframe);
347
1
}
348
349
TEST_CASE("data is shuffled")
350
1
{
351
1
  int int_array[8] = {
352
1
    2, 3, 4, 5, 6, 7, 8, 9
353
1
  };
354
1
  ccv_cnnp_column_data_t columns[] = {
355
1
    {
356
1
      .data_enum = _ccv_iter_int,
357
1
      .context = int_array,
358
1
    }
359
1
  };
360
1
  ccv_cnnp_dataframe_t* const dataframe = ccv_cnnp_dataframe_new(columns, sizeof(columns) / sizeof(columns[0]), 8);
361
1
  const int derived = ccv_cnnp_dataframe_map(dataframe, _ccv_int_plus_1, 0, 0, COLUMN_ID_LIST(0), 0, 0);
362
1
  assert(derived > 0);
363
1
  ccv_cnnp_dataframe_shuffle(dataframe);
364
1
  int i;
365
1
  void* data;
366
1
  int result[8];
367
1
  ccv_cnnp_dataframe_iter_t* const iter = ccv_cnnp_dataframe_iter_new(dataframe, COLUMN_ID_LIST(derived));
368
1
  _ccv_iter_accessed = 0;
369
1
  ccv_cnnp_dataframe_iter_prefetch(iter, 3, 0);
370
1
  REQUIRE_EQ(_ccv_iter_accessed, 1, "iter accessed once for prefetching");
371
4
  for (i = 0; i < 3; 
i++3
)
372
3
  {
373
3
    _ccv_iter_accessed = 0;
374
3
    ccv_cnnp_dataframe_iter_next(iter, &data, 1, 0);
375
3
    REQUIRE_EQ(_ccv_iter_accessed, 0, "no iterator accessed");
376
3
    result[i] = (int)(intptr_t)data;
377
3
  }
378
1
  ccv_cnnp_dataframe_iter_prefetch(iter, 2, 0);
379
1
  ccv_cnnp_dataframe_iter_next(iter, &data, 1, 0);
380
1
  result[3] = (int)(intptr_t)data;
381
1
  ccv_cnnp_dataframe_iter_prefetch(iter, 2, 0);
382
1
  _ccv_iter_accessed = 0;
383
5
  for (i = 4; i < 8; 
i++4
)
384
4
  {
385
4
    ccv_cnnp_dataframe_iter_next(iter, &data, 1, 0);
386
4
    result[i] = (int)(intptr_t)data;
387
4
  }
388
1
  REQUIRE_EQ(_ccv_iter_accessed, 1, "no iterator accessed");
389
9
  for (i = 0; i < 8; 
i++8
)
390
8
    ++int_array[i];
391
1
  ccv_cnnp_dataframe_iter_free(iter);
392
1
  REQUIRE_ARRAY_NOT_EQ(int, int_array, result, 8, "iterated result and actual result should not be the same");
393
1
  int covered[8] = {};
394
9
  for (i = 0; i < 8; 
i++8
)
395
8
    covered[result[i] - 3] = 1;
396
1
  int is_covered[8] = {
397
1
    1, 1, 1, 1, 1, 1, 1, 1
398
1
  };
399
1
  REQUIRE_ARRAY_EQ(int, covered, is_covered, 8, "data should covered all");
400
1
  ccv_cnnp_dataframe_free(dataframe);
401
1
}
402
403
static void _ccv_iter_reduce_int(void** const input_data, const int batch_size, void** const data, void* const context, ccv_nnc_stream_context_t* const stream_context)
404
12
{
405
12
  int i;
406
12
  int total = 0;
407
44
  for (i = 0; i < batch_size; 
i++32
)
408
32
    total += (int)(intptr_t)input_data[i];
409
12
  data[0] = (void*)(intptr_t)total;
410
12
}
411
412
TEST_CASE("dataframe reduce")
413
1
{
414
1
  int int_array[8] = {
415
1
    2, 3, 4, 5, 6, 7, 8, 9
416
1
  };
417
1
  ccv_cnnp_column_data_t columns[] = {
418
1
    {
419
1
      .data_enum = _ccv_iter_int,
420
1
      .context = int_array,
421
1
    }
422
1
  };
423
1
  ccv_cnnp_dataframe_t* const dataframe = ccv_cnnp_dataframe_new(columns, sizeof(columns) / sizeof(columns[0]), 8);
424
1
  ccv_cnnp_dataframe_t* const reduce = ccv_cnnp_dataframe_reduce_new(dataframe, _ccv_iter_reduce_int, 0, 0, 3, 0, 0);
425
1
  ccv_cnnp_dataframe_iter_t* const iter = ccv_cnnp_dataframe_iter_new(reduce, COLUMN_ID_LIST(0));
426
1
  int i;
427
1
  void* data;
428
1
  int result[3];
429
1
  _ccv_iter_accessed = 0;
430
3
  for (i = 0; ; i++)
431
4
  {
432
4
    if (0 != ccv_cnnp_dataframe_iter_next(iter, &data, 1, 0))
433
1
      break;
434
3
    result[i] = (int)(intptr_t)data;
435
3
  }
436
1
  REQUIRE_EQ(_ccv_iter_accessed, 3, "should only accessed iterator 3 times");
437
1
  int should_result[3] = {
438
1
    9, 18, 17
439
1
  };
440
1
  REQUIRE_ARRAY_EQ(int, should_result, result, 3, "iterated result and actual result should not be the same");
441
1
  ccv_cnnp_dataframe_iter_free(iter);
442
1
  ccv_cnnp_dataframe_free(reduce);
443
1
  ccv_cnnp_dataframe_free(dataframe);
444
1
}
445
446
TEST_CASE("dataframe map before reduce")
447
1
{
448
1
  int int_array[8] = {
449
1
    2, 3, 4, 5, 6, 7, 8, 9
450
1
  };
451
1
  ccv_cnnp_column_data_t columns[] = {
452
1
    {
453
1
      .data_enum = _ccv_iter_int,
454
1
      .context = int_array,
455
1
    }
456
1
  };
457
1
  ccv_cnnp_dataframe_t* const dataframe = ccv_cnnp_dataframe_new(columns, sizeof(columns) / sizeof(columns[0]), 8);
458
1
  const int derived = ccv_cnnp_dataframe_map(dataframe, _ccv_int_plus_1, 0, 0, COLUMN_ID_LIST(0), 0, 0);
459
1
  assert(derived > 0);
460
1
  ccv_cnnp_dataframe_t* const reduce = ccv_cnnp_dataframe_reduce_new(dataframe, _ccv_iter_reduce_int, 0, derived, 3, 0, 0);
461
1
  ccv_cnnp_dataframe_iter_t* const iter = ccv_cnnp_dataframe_iter_new(reduce, COLUMN_ID_LIST(0));
462
1
  int i;
463
1
  void* data;
464
1
  int result[3];
465
3
  for (i = 0; ; i++)
466
4
  {
467
4
    if (0 != ccv_cnnp_dataframe_iter_next(iter, &data, 1, 0))
468
1
      break;
469
3
    result[i] = (int)(intptr_t)data;
470
3
  }
471
1
  int should_result[3] = {
472
1
    12, 21, 19
473
1
  };
474
1
  REQUIRE_ARRAY_EQ(int, should_result, result, 3, "iterated result and actual result should not be the same");
475
1
  ccv_cnnp_dataframe_iter_free(iter);
476
1
  ccv_cnnp_dataframe_free(reduce);
477
1
  ccv_cnnp_dataframe_free(dataframe);
478
1
}
479
480
TEST_CASE("dataframe map after reduce")
481
1
{
482
1
  int int_array[8] = {
483
1
    2, 3, 4, 5, 6, 7, 8, 9
484
1
  };
485
1
  ccv_cnnp_column_data_t columns[] = {
486
1
    {
487
1
      .data_enum = _ccv_iter_int,
488
1
      .context = int_array,
489
1
    }
490
1
  };
491
1
  ccv_cnnp_dataframe_t* const dataframe = ccv_cnnp_dataframe_new(columns, sizeof(columns) / sizeof(columns[0]), 8);
492
1
  ccv_cnnp_dataframe_t* const reduce = ccv_cnnp_dataframe_reduce_new(dataframe, _ccv_iter_reduce_int, 0, 0, 3, 0, 0);
493
1
  const int derived = ccv_cnnp_dataframe_map(reduce, _ccv_int_plus_1, 0, 0, COLUMN_ID_LIST(0), 0, 0);
494
1
  assert(derived > 0);
495
1
  ccv_cnnp_dataframe_iter_t* const iter = ccv_cnnp_dataframe_iter_new(reduce, COLUMN_ID_LIST(derived));
496
1
  int i;
497
1
  void* data;
498
1
  int result[3];
499
3
  for (i = 0; ; i++)
500
4
  {
501
4
    if (0 != ccv_cnnp_dataframe_iter_next(iter, &data, 1, 0))
502
1
      break;
503
3
    result[i] = (int)(intptr_t)data;
504
3
  }
505
1
  int should_result[3] = {
506
1
    10, 19, 18
507
1
  };
508
1
  REQUIRE_ARRAY_EQ(int, should_result, result, 3, "iterated result and actual result should not be the same");
509
1
  ccv_cnnp_dataframe_iter_free(iter);
510
1
  ccv_cnnp_dataframe_free(reduce);
511
1
  ccv_cnnp_dataframe_free(dataframe);
512
1
}
513
514
TEST_CASE("dataframe map after reduce shuffled")
515
1
{
516
1
  int int_array[8] = {
517
1
    2, 3, 4, 5, 6, 7, 8, 9
518
1
  };
519
1
  ccv_cnnp_column_data_t columns[] = {
520
1
    {
521
1
      .data_enum = _ccv_iter_int,
522
1
      .context = int_array,
523
1
    }
524
1
  };
525
1
  ccv_cnnp_dataframe_t* const dataframe = ccv_cnnp_dataframe_new(columns, sizeof(columns) / sizeof(columns[0]), 8);
526
1
  ccv_cnnp_dataframe_t* const reduce = ccv_cnnp_dataframe_reduce_new(dataframe, _ccv_iter_reduce_int, 0, 0, 3, 0, 0);
527
1
  const int derived = ccv_cnnp_dataframe_map(reduce, _ccv_int_plus_1, 0, 0, COLUMN_ID_LIST(0), 0, 0);
528
1
  ccv_cnnp_dataframe_shuffle(reduce);
529
1
  assert(derived > 0);
530
1
  ccv_cnnp_dataframe_iter_t* const iter = ccv_cnnp_dataframe_iter_new(reduce, COLUMN_ID_LIST(derived));
531
1
  int i;
532
1
  void* data;
533
1
  int result[3];
534
3
  for (i = 0; ; i++)
535
4
  {
536
4
    if (0 != ccv_cnnp_dataframe_iter_next(iter, &data, 1, 0))
537
1
      break;
538
3
    result[i] = (int)(intptr_t)data;
539
3
  }
540
1
  int covered[3] = {};
541
4
  for (i = 0; i < 3; 
i++3
)
542
3
    if (result[i] == 10)
543
1
      covered[0] = 1;
544
2
    else if (result[i] == 19)
545
1
      covered[1] = 1;
546
1
    else if (result[i] == 18)
547
1
      covered[2] = 1;
548
1
  int is_covered[3] = {
549
1
    1, 1, 1
550
1
  };
551
1
  REQUIRE_ARRAY_EQ(int, covered, is_covered, 3, "data should covered all");
552
1
  ccv_cnnp_dataframe_iter_free(iter);
553
1
  ccv_cnnp_dataframe_free(reduce);
554
1
  ccv_cnnp_dataframe_free(dataframe);
555
1
}
556
557
TEST_CASE("iterate through newly added column")
558
1
{
559
1
  int int_array[8] = {
560
1
    2, 3, 4, 5, 6, 7, 8, 9
561
1
  };
562
1
  ccv_cnnp_dataframe_t* const dataframe = ccv_cnnp_dataframe_new(0, 0, 8);
563
1
  const int derived = ccv_cnnp_dataframe_add(dataframe, _ccv_iter_int, 0, 0, int_array, 0);
564
1
  assert(derived >= 0);
565
1
  ccv_cnnp_dataframe_iter_t* const iter = ccv_cnnp_dataframe_iter_new(dataframe, COLUMN_ID_LIST(derived));
566
1
  int result[8];
567
1
  int i = 0;
568
1
  void* data;
569
9
  while (0 == ccv_cnnp_dataframe_iter_next(iter, &data, 1, 0))
570
8
    result[i++] = (int)(intptr_t)data;
571
1
  ccv_cnnp_dataframe_iter_free(iter);
572
1
  REQUIRE_ARRAY_EQ(int, int_array, result, 8, "iterated result and actual result should be the same");
573
1
  // iter2 test some prefetch capacities.
574
1
  ccv_cnnp_dataframe_iter_t* const iter2 = ccv_cnnp_dataframe_iter_new(dataframe, COLUMN_ID_LIST(derived));
575
4
  for (i = 0; i < 3; 
i++3
)
576
3
    ccv_cnnp_dataframe_iter_prefetch(iter2, 1, 0);
577
1
  _ccv_iter_accessed = 0;
578
4
  for (i = 0; i < 3; 
i++3
)
579
3
  {
580
3
    ccv_cnnp_dataframe_iter_next(iter2, &data, 1, 0);
581
3
    result[i] = (int)(intptr_t)data;
582
3
  }
583
1
  REQUIRE_EQ(_ccv_iter_accessed, 0, "the iterator is not accessed at all, because prefetching");
584
1
  ccv_cnnp_dataframe_iter_next(iter2, &data, 1, 0);
585
1
  result[3] = (int)(intptr_t)data;
586
1
  REQUIRE_EQ(_ccv_iter_accessed, 1, "the iterator is accessed, because no prefetching");
587
1
  ccv_cnnp_dataframe_iter_prefetch(iter2, 4, 0);
588
1
  REQUIRE_EQ(_ccv_iter_accessed, 2, "the iterator is accessed again, for prefetching");
589
1
  _ccv_iter_accessed = 0;
590
3
  for (i = 4; i < 6; 
i++2
)
591
2
  {
592
2
    ccv_cnnp_dataframe_iter_next(iter2, &data, 1, 0);
593
2
    result[i] = (int)(intptr_t)data;
594
2
  }
595
1
  REQUIRE_EQ(_ccv_iter_accessed, 0, "the iterator accessed 0 times");
596
1
  const int fail0 = ccv_cnnp_dataframe_iter_prefetch(iter2, 1, 0);
597
1
  REQUIRE_EQ(fail0, -1, "should fail");
598
1
  ccv_cnnp_dataframe_iter_free(iter2);
599
1
  REQUIRE_ARRAY_EQ(int, int_array, result, 6, "iterated result and actual result should be the same up to 6");
600
1
  ccv_cnnp_dataframe_free(dataframe);
601
1
}
602
603
TEST_CASE("make a tuple out of column and derived column")
604
1
{
605
1
  int int_array[8] = {
606
1
    2, 3, 4, 5, 6, 7, 8, 9
607
1
  };
608
1
  ccv_cnnp_column_data_t columns[] = {
609
1
    {
610
1
      .data_enum = _ccv_iter_int,
611
1
      .context = int_array,
612
1
    }
613
1
  };
614
1
  ccv_cnnp_dataframe_t* const dataframe = ccv_cnnp_dataframe_new(columns, sizeof(columns) / sizeof(columns[0]), 8);
615
1
  const int derived = ccv_cnnp_dataframe_map(dataframe, _ccv_int_plus_1, 0, 0, COLUMN_ID_LIST(0), 0, 0);
616
1
  assert(derived > 0);
617
1
  const int tuple = ccv_cnnp_dataframe_make_tuple(dataframe, COLUMN_ID_LIST(derived, 0));
618
1
  ccv_cnnp_dataframe_iter_t* const iter = ccv_cnnp_dataframe_iter_new(dataframe, COLUMN_ID_LIST(tuple));
619
1
  int result[2][8];
620
1
  int i = 0;
621
1
  void* data;
622
9
  while (0 == ccv_cnnp_dataframe_iter_next(iter, &data, 1, 0))
623
8
  {
624
8
    void** int_data = (void**)data;
625
8
    result[0][i] = (int)(intptr_t)int_data[0];
626
8
    result[1][i] = (int)(intptr_t)int_data[1];
627
8
    ++i;
628
8
  }
629
1
  ccv_cnnp_dataframe_iter_free(iter);
630
1
  REQUIRE_EQ(ccv_cnnp_dataframe_tuple_size(dataframe, tuple), 2, "It should contain two columns");
631
1
  ccv_cnnp_dataframe_free(dataframe);
632
1
  REQUIRE_ARRAY_EQ(int, int_array, result[1], 8, "iterated tuple should be the same");
633
9
  for (i = 0; i < 8; 
i++8
)
634
8
    ++int_array[i];
635
1
  REQUIRE_ARRAY_EQ(int, int_array, result[0], 8, "iterated tuple should be the same");
636
1
}
637
638
TEST_CASE("extract value out of a tuple")
639
1
{
640
1
  int int_array[8] = {
641
1
    2, 3, 4, 5, 6, 7, 8, 9
642
1
  };
643
1
  ccv_cnnp_column_data_t columns[] = {
644
1
    {
645
1
      .data_enum = _ccv_iter_int,
646
1
      .context = int_array,
647
1
    }
648
1
  };
649
1
  ccv_cnnp_dataframe_t* const dataframe = ccv_cnnp_dataframe_new(columns, sizeof(columns) / sizeof(columns[0]), 8);
650
1
  const int derived = ccv_cnnp_dataframe_map(dataframe, _ccv_int_plus_1, 0, 0, COLUMN_ID_LIST(0), 0, 0);
651
1
  assert(derived > 0);
652
1
  const int tuple = ccv_cnnp_dataframe_make_tuple(dataframe, COLUMN_ID_LIST(0, derived));
653
1
  const int extract = ccv_cnnp_dataframe_extract_tuple(dataframe, tuple, 1);
654
1
  ccv_cnnp_dataframe_iter_t* const iter = ccv_cnnp_dataframe_iter_new(dataframe, COLUMN_ID_LIST(extract));
655
1
  int result[8];
656
1
  int i = 0;
657
1
  void* data;
658
9
  while (0 == ccv_cnnp_dataframe_iter_next(iter, &data, 1, 0))
659
8
    result[i++] = (int)(intptr_t)data;
660
1
  ccv_cnnp_dataframe_iter_free(iter);
661
1
  REQUIRE_EQ(ccv_cnnp_dataframe_tuple_size(dataframe, tuple), 2, "It should contain two columns");
662
1
  ccv_cnnp_dataframe_free(dataframe);
663
9
  for (i = 0; i < 8; 
i++8
)
664
8
    ++int_array[i];
665
1
  REQUIRE_ARRAY_EQ(int, int_array, result, 8, "iterated tuple should be the same");
666
1
}
667
668
#include "case_main.h"