Coverage Report

Created: 2021-04-05 03:19

/home/liu/buildslave/linux-x64-runtests/build/test/unit/nnc/dataframe.tests.c
Line
Count
Source
1
#include "case.h"
2
#include "ccv_case.h"
3
#include "ccv_nnc_case.h"
4
#include <ccv.h>
5
#include <nnc/ccv_nnc.h>
6
#include <nnc/ccv_nnc_easy.h>
7
#include "3rdparty/dsfmt/dSFMT.h"
8
9
TEST_SETUP()
10
{
11
  ccv_nnc_init();
12
}
13
14
static int _ccv_iter_accessed = 0;
15
16
static void _ccv_iter_int(const int column_idx, const int* row_idxs, const int row_size, void** const data, void* const context, ccv_nnc_stream_context_t* const stream_context)
17
125
{
18
125
  int* const array = (int*)context;
19
125
  int i;
20
284
  for (i = 0; i < row_size; 
i++159
)
21
159
    data[i] = (void*)(intptr_t)array[row_idxs[i]];
22
125
  ++_ccv_iter_accessed;
23
125
}
24
25
TEST_CASE("iterate through a simple dataframe")
26
1
{
27
1
  int int_array[8] = {
28
1
    0, 3, 4, 5, 6, 7, 8, 9
29
1
  };
30
1
  ccv_cnnp_column_data_t columns[] = {
31
1
    {
32
1
      .data_enum = _ccv_iter_int,
33
1
      .context = int_array,
34
1
    }
35
1
  };
36
1
  ccv_cnnp_dataframe_t* const dataframe = ccv_cnnp_dataframe_new(columns, sizeof(columns) / sizeof(columns[0]), 8);
37
1
  ccv_cnnp_dataframe_iter_t* const iter1 = ccv_cnnp_dataframe_iter_new(dataframe, COLUMN_ID_LIST(0));
38
1
  int result[8];
39
1
  int i = 0;
40
1
  void* data;
41
9
  while (0 == ccv_cnnp_dataframe_iter_next(iter1, &data, 1, 0))
42
8
    result[i++] = (int)(intptr_t)data;
43
1
  ccv_cnnp_dataframe_iter_free(iter1);
44
1
  REQUIRE_ARRAY_EQ(int, int_array, result, 8, "iterated result and actual result should be the same");
45
1
  // iter2 test some prefetch capacities.
46
1
  ccv_cnnp_dataframe_iter_t* const iter2 = ccv_cnnp_dataframe_iter_new(dataframe, COLUMN_ID_LIST(0));
47
4
  for (i = 0; i < 3; 
i++3
)
48
3
    ccv_cnnp_dataframe_iter_prefetch(iter2, 1, 0);
49
1
  _ccv_iter_accessed = 0;
50
4
  for (i = 0; i < 3; 
i++3
)
51
3
  {
52
3
    ccv_cnnp_dataframe_iter_next(iter2, &data, 1, 0);
53
3
    result[i] = (int)(intptr_t)data;
54
3
  }
55
1
  REQUIRE_EQ(_ccv_iter_accessed, 0, "the iterator is not accessed at all, because prefetching");
56
1
  ccv_cnnp_dataframe_iter_next(iter2, &data, 1, 0);
57
1
  result[3] = (int)(intptr_t)data;
58
1
  REQUIRE_EQ(_ccv_iter_accessed, 1, "the iterator is accessed, because no prefetching");
59
1
  ccv_cnnp_dataframe_iter_prefetch(iter2, 1, 0);
60
1
  REQUIRE_EQ(_ccv_iter_accessed, 2, "the iterator is accessed again, for prefetching");
61
1
  _ccv_iter_accessed = 0;
62
5
  for (i = 4; i < 8; 
i++4
)
63
4
  {
64
4
    ccv_cnnp_dataframe_iter_next(iter2, &data, 1, 0);
65
4
    result[i] = (int)(intptr_t)data;
66
4
  }
67
1
  REQUIRE_EQ(_ccv_iter_accessed, 3, "the iterator accessed 3 times, the first is prefetching");
68
1
  ccv_cnnp_dataframe_iter_free(iter2);
69
1
  REQUIRE_ARRAY_EQ(int, int_array, result, 8, "iterated result and actual result should be the same");
70
1
  // iter3 test more prefetch behavior.
71
1
  ccv_cnnp_dataframe_iter_t* const iter3 = ccv_cnnp_dataframe_iter_new(dataframe, COLUMN_ID_LIST(0));
72
4
  for (i = 0; i < 3; 
i++3
)
73
3
    ccv_cnnp_dataframe_iter_prefetch(iter3, 1, 0);
74
3
  for (i = 0; i < 2; 
i++2
)
75
2
  {
76
2
    _ccv_iter_accessed = 0;
77
2
    ccv_cnnp_dataframe_iter_next(iter3, &data, 1, 0);
78
2
    REQUIRE_EQ(_ccv_iter_accessed, 0, "no iterator accessed");
79
2
    result[i] = (int)(intptr_t)data;
80
2
    ccv_cnnp_dataframe_iter_prefetch(iter3, 1, 0);
81
2
    REQUIRE_EQ(_ccv_iter_accessed, 1, "iterator accessed for prefetching");
82
2
  }
83
3
  
for (i = 2; 1
i < 4;
i++2
)
84
2
    ccv_cnnp_dataframe_iter_prefetch(iter3, 1, 0);
85
1
  _ccv_iter_accessed = 0;
86
6
  for (i = 2; i < 7; 
i++5
)
87
5
  {
88
5
    ccv_cnnp_dataframe_iter_next(iter3, &data, 1, 0);
89
5
    result[i] = (int)(intptr_t)data;
90
5
  }
91
1
  REQUIRE_EQ(_ccv_iter_accessed, 0, "no iterator accessed");
92
1
  ccv_cnnp_dataframe_iter_next(iter3, &data, 1, 0);
93
1
  result[7] = (int)(intptr_t)data;
94
1
  REQUIRE_EQ(_ccv_iter_accessed, 1, "iterator accessed twice");
95
1
  const int fail1 = ccv_cnnp_dataframe_iter_prefetch(iter3, 1, 0);
96
1
  REQUIRE_EQ(fail1, -1, "cannot advance no more");
97
1
  ccv_cnnp_dataframe_iter_free(iter3);
98
1
  REQUIRE_ARRAY_EQ(int, int_array, result, 8, "iterated result and actual result should be the same");
99
1
  ccv_cnnp_dataframe_free(dataframe);
100
1
}
101
102
static void _ccv_int_plus_1(void* const* const* const column_data, const int column_size, const int batch_size, void** const data, void* const context, ccv_nnc_stream_context_t* const stream_context)
103
85
{
104
85
  int i;
105
186
  for (i = 0; i < batch_size; 
i++101
)
106
101
  {
107
101
    int k = (int)(intptr_t)column_data[0][i];
108
101
    data[i] = (void*)(intptr_t)(k + 1);
109
101
  }
110
85
}
111
112
TEST_CASE("iterate through derived column")
113
1
{
114
1
  int int_array[8] = {
115
1
    2, 3, 4, 5, 6, 7, 8, 9
116
1
  };
117
1
  ccv_cnnp_column_data_t columns[] = {
118
1
    {
119
1
      .data_enum = _ccv_iter_int,
120
1
      .context = int_array,
121
1
    }
122
1
  };
123
1
  ccv_cnnp_dataframe_t* const dataframe = ccv_cnnp_dataframe_new(columns, sizeof(columns) / sizeof(columns[0]), 8);
124
1
  const int derived = ccv_cnnp_dataframe_map(dataframe, _ccv_int_plus_1, 0, 0, COLUMN_ID_LIST(0), 0, 0, "map");
125
1
  REQUIRE(strcmp(ccv_cnnp_dataframe_column_name(dataframe, derived), "map") == 0, "should match the map name");
126
1
  assert(derived > 0);
127
1
  ccv_cnnp_dataframe_iter_t* const iter = ccv_cnnp_dataframe_iter_new(dataframe, COLUMN_ID_LIST(derived));
128
1
  int result[8];
129
1
  int i = 0;
130
1
  void* data;
131
9
  while (0 == ccv_cnnp_dataframe_iter_next(iter, &data, 1, 0))
132
8
    result[i++] = (int)(intptr_t)data;
133
1
  ccv_cnnp_dataframe_iter_free(iter);
134
9
  for (i = 0; i < 8; 
i++8
)
135
8
    ++int_array[i];
136
1
  REQUIRE_ARRAY_EQ(int, int_array, result, 8, "iterated result and actual result should be the same");
137
1
  // iter2 test some prefetch capacities.
138
1
  ccv_cnnp_dataframe_iter_t* const iter2 = ccv_cnnp_dataframe_iter_new(dataframe, COLUMN_ID_LIST(derived));
139
4
  for (i = 0; i < 3; 
i++3
)
140
3
    ccv_cnnp_dataframe_iter_prefetch(iter2, 1, 0);
141
1
  _ccv_iter_accessed = 0;
142
4
  for (i = 0; i < 3; 
i++3
)
143
3
  {
144
3
    ccv_cnnp_dataframe_iter_next(iter2, &data, 1, 0);
145
3
    result[i] = (int)(intptr_t)data;
146
3
  }
147
1
  REQUIRE_EQ(_ccv_iter_accessed, 0, "the iterator is not accessed at all, because prefetching");
148
1
  ccv_cnnp_dataframe_iter_next(iter2, &data, 1, 0);
149
1
  result[3] = (int)(intptr_t)data;
150
1
  REQUIRE_EQ(_ccv_iter_accessed, 1, "the iterator is accessed, because no prefetching");
151
1
  ccv_cnnp_dataframe_iter_prefetch(iter2, 1, 0);
152
1
  REQUIRE_EQ(_ccv_iter_accessed, 2, "the iterator is accessed again, for prefetching");
153
1
  _ccv_iter_accessed = 0;
154
3
  for (i = 4; i < 6; 
i++2
)
155
2
  {
156
2
    ccv_cnnp_dataframe_iter_next(iter2, &data, 1, 0);
157
2
    result[i] = (int)(intptr_t)data;
158
2
  }
159
1
  REQUIRE_EQ(_ccv_iter_accessed, 1, "the iterator accessed 3 times, the first is prefetching");
160
1
  const int success0 = ccv_cnnp_dataframe_iter_prefetch(iter2, 1, 0);
161
1
  REQUIRE_EQ(success0, 0, "success");
162
1
  const int success1 = ccv_cnnp_dataframe_iter_prefetch(iter2, 1, 0);
163
1
  REQUIRE_EQ(success1, 0, "success");
164
1
  const int fail0 = ccv_cnnp_dataframe_iter_prefetch(iter2, 1, 0);
165
1
  REQUIRE_EQ(fail0, -1, "should fail");
166
7
  for (i = 0; i < 6; 
i++6
)
167
6
    ++int_array[i];
168
1
  ccv_cnnp_dataframe_iter_free(iter2);
169
1
  REQUIRE_ARRAY_EQ(int, int_array, result, 6, "iterated result and actual result should be the same up to 6");
170
1
  // iter3 test more prefetch behavior.
171
1
  ccv_cnnp_dataframe_iter_t* const iter3 = ccv_cnnp_dataframe_iter_new(dataframe, COLUMN_ID_LIST(derived));
172
4
  for (i = 0; i < 3; 
i++3
)
173
3
    ccv_cnnp_dataframe_iter_prefetch(iter3, 1, 0);
174
3
  for (i = 0; i < 2; 
i++2
)
175
2
  {
176
2
    _ccv_iter_accessed = 0;
177
2
    ccv_cnnp_dataframe_iter_next(iter3, &data, 1, 0);
178
2
    REQUIRE_EQ(_ccv_iter_accessed, 0, "no iterator accessed");
179
2
    result[i] = (int)(intptr_t)data;
180
2
    ccv_cnnp_dataframe_iter_prefetch(iter3, 1, 0);
181
2
    REQUIRE_EQ(_ccv_iter_accessed, 1, "iterator accessed for prefetching");
182
2
  }
183
3
  
for (i = 2; 1
i < 4;
i++2
)
184
2
    ccv_cnnp_dataframe_iter_prefetch(iter3, 1, 0);
185
1
  _ccv_iter_accessed = 0;
186
6
  for (i = 2; i < 7; 
i++5
)
187
5
  {
188
5
    ccv_cnnp_dataframe_iter_next(iter3, &data, 1, 0);
189
5
    result[i] = (int)(intptr_t)data;
190
5
  }
191
1
  REQUIRE_EQ(_ccv_iter_accessed, 0, "no iterator accessed");
192
1
  ccv_cnnp_dataframe_iter_next(iter3, &data, 1, 0);
193
1
  result[7] = (int)(intptr_t)data;
194
1
  REQUIRE_EQ(_ccv_iter_accessed, 1, "iterator accessed twice");
195
1
  const int fail1 = ccv_cnnp_dataframe_iter_prefetch(iter3, 1, 0);
196
1
  REQUIRE_EQ(fail1, -1, "cannot advance no more");
197
9
  for (i = 0; i < 8; 
i++8
)
198
8
    ++int_array[i];
199
1
  ccv_cnnp_dataframe_iter_free(iter3);
200
1
  REQUIRE_ARRAY_EQ(int, int_array, result, 8, "iterated result and actual result should be the same");
201
1
  ccv_cnnp_dataframe_free(dataframe);
202
1
}
203
204
TEST_CASE("iterate through derived column with cursor reset")
205
1
{
206
1
  int int_array[8] = {
207
1
    2, 3, 4, 5, 6, 7, 8, 9
208
1
  };
209
1
  ccv_cnnp_column_data_t columns[] = {
210
1
    {
211
1
      .data_enum = _ccv_iter_int,
212
1
      .context = int_array,
213
1
    }
214
1
  };
215
1
  ccv_cnnp_dataframe_t* const dataframe = ccv_cnnp_dataframe_new(columns, sizeof(columns) / sizeof(columns[0]), 8);
216
1
  const int derived = ccv_cnnp_dataframe_map(dataframe, _ccv_int_plus_1, 0, 0, COLUMN_ID_LIST(0), 0, 0, 0);
217
1
  assert(derived > 0);
218
1
  int i;
219
1
  void* data;
220
1
  int result[8];
221
1
  ccv_cnnp_dataframe_iter_t* const iter = ccv_cnnp_dataframe_iter_new(dataframe, COLUMN_ID_LIST(derived));
222
4
  for (i = 0; i < 3; 
i++3
)
223
3
    ccv_cnnp_dataframe_iter_prefetch(iter, 1, 0);
224
3
  for (i = 0; i < 2; 
i++2
)
225
2
  {
226
2
    _ccv_iter_accessed = 0;
227
2
    ccv_cnnp_dataframe_iter_next(iter, &data, 1, 0);
228
2
    REQUIRE_EQ(_ccv_iter_accessed, 0, "no iterator accessed");
229
2
    result[i] = (int)(intptr_t)data;
230
2
    ccv_cnnp_dataframe_iter_prefetch(iter, 1, 0);
231
2
    REQUIRE_EQ(_ccv_iter_accessed, 1, "iterator accessed for prefetching");
232
2
  }
233
3
  
for (i = 2; 1
i < 4;
i++2
)
234
2
    ccv_cnnp_dataframe_iter_prefetch(iter, 1, 0);
235
1
  ccv_cnnp_dataframe_iter_set_cursor(iter, 0);
236
1
  ccv_cnnp_dataframe_iter_prefetch(iter, 1, 0);
237
1
  ccv_cnnp_dataframe_iter_prefetch(iter, 1, 0);
238
1
  _ccv_iter_accessed = 0;
239
8
  for (i = 0; i < 7; 
i++7
)
240
7
  {
241
7
    ccv_cnnp_dataframe_iter_next(iter, &data, 1, 0);
242
7
    result[i] = (int)(intptr_t)data;
243
7
  }
244
1
  REQUIRE_EQ(_ccv_iter_accessed, 5, "5 iterator accessed");
245
1
  _ccv_iter_accessed = 0;
246
1
  ccv_cnnp_dataframe_iter_next(iter, &data, 1, 0);
247
1
  result[7] = (int)(intptr_t)data;
248
1
  REQUIRE_EQ(_ccv_iter_accessed, 1, "iterator accessed");
249
1
  const int fail1 = ccv_cnnp_dataframe_iter_prefetch(iter, 1, 0);
250
1
  REQUIRE_EQ(fail1, -1, "cannot advance no more");
251
9
  for (i = 0; i < 8; 
i++8
)
252
8
    ++int_array[i];
253
1
  ccv_cnnp_dataframe_iter_free(iter);
254
1
  REQUIRE_ARRAY_EQ(int, int_array, result, 8, "iterated result and actual result should be the same");
255
1
  ccv_cnnp_dataframe_free(dataframe);
256
1
}
257
258
TEST_CASE("iterate with prefetch more than 1 item, variant 1")
259
1
{
260
1
  int int_array[8] = {
261
1
    2, 3, 4, 5, 6, 7, 8, 9
262
1
  };
263
1
  ccv_cnnp_column_data_t columns[] = {
264
1
    {
265
1
      .data_enum = _ccv_iter_int,
266
1
      .context = int_array,
267
1
    }
268
1
  };
269
1
  ccv_cnnp_dataframe_t* const dataframe = ccv_cnnp_dataframe_new(columns, sizeof(columns) / sizeof(columns[0]), 8);
270
1
  const int derived = ccv_cnnp_dataframe_map(dataframe, _ccv_int_plus_1, 0, 0, COLUMN_ID_LIST(0), 0, 0, 0);
271
1
  assert(derived > 0);
272
1
  int i;
273
1
  void* data;
274
1
  int result[8];
275
1
  ccv_cnnp_dataframe_iter_t* const iter = ccv_cnnp_dataframe_iter_new(dataframe, COLUMN_ID_LIST(derived));
276
1
  _ccv_iter_accessed = 0;
277
1
  ccv_cnnp_dataframe_iter_prefetch(iter, 3, 0);
278
1
  REQUIRE_EQ(_ccv_iter_accessed, 1, "iter accessed once for prefetching");
279
3
  for (i = 0; i < 2; 
i++2
)
280
2
  {
281
2
    _ccv_iter_accessed = 0;
282
2
    ccv_cnnp_dataframe_iter_next(iter, &data, 1, 0);
283
2
    REQUIRE_EQ(_ccv_iter_accessed, 0, "no iterator accessed");
284
2
    result[i] = (int)(intptr_t)data;
285
2
    ccv_cnnp_dataframe_iter_prefetch(iter, 1, 0);
286
2
    REQUIRE_EQ(_ccv_iter_accessed, 1, "iterator accessed for prefetching");
287
2
  }
288
1
  ccv_cnnp_dataframe_iter_prefetch(iter, 4, 0);
289
1
  _ccv_iter_accessed = 0;
290
7
  for (i = 2; i < 8; 
i++6
)
291
6
  {
292
6
    ccv_cnnp_dataframe_iter_next(iter, &data, 1, 0);
293
6
    result[i] = (int)(intptr_t)data;
294
6
  }
295
1
  REQUIRE_EQ(_ccv_iter_accessed, 0, "no iterator accessed");
296
9
  for (i = 0; i < 8; 
i++8
)
297
8
    ++int_array[i];
298
1
  ccv_cnnp_dataframe_iter_free(iter);
299
1
  REQUIRE_ARRAY_EQ(int, int_array, result, 8, "iterated result and actual result should be the same");
300
1
  ccv_cnnp_dataframe_free(dataframe);
301
1
}
302
303
TEST_CASE("iterate with prefetch more than 1 item, variant 2")
304
1
{
305
1
  int int_array[8] = {
306
1
    2, 3, 4, 5, 6, 7, 8, 9
307
1
  };
308
1
  ccv_cnnp_column_data_t columns[] = {
309
1
    {
310
1
      .data_enum = _ccv_iter_int,
311
1
      .context = int_array,
312
1
    }
313
1
  };
314
1
  ccv_cnnp_dataframe_t* const dataframe = ccv_cnnp_dataframe_new(columns, sizeof(columns) / sizeof(columns[0]), 8);
315
1
  const int derived = ccv_cnnp_dataframe_map(dataframe, _ccv_int_plus_1, 0, 0, COLUMN_ID_LIST(0), 0, 0, 0);
316
1
  assert(derived > 0);
317
1
  int i;
318
1
  void* data;
319
1
  int result[8];
320
1
  ccv_cnnp_dataframe_iter_t* const iter = ccv_cnnp_dataframe_iter_new(dataframe, COLUMN_ID_LIST(derived));
321
1
  _ccv_iter_accessed = 0;
322
1
  ccv_cnnp_dataframe_iter_prefetch(iter, 3, 0);
323
1
  REQUIRE_EQ(_ccv_iter_accessed, 1, "iter accessed once for prefetching");
324
4
  for (i = 0; i < 3; 
i++3
)
325
3
  {
326
3
    _ccv_iter_accessed = 0;
327
3
    ccv_cnnp_dataframe_iter_next(iter, &data, 1, 0);
328
3
    REQUIRE_EQ(_ccv_iter_accessed, 0, "no iterator accessed");
329
3
    result[i] = (int)(intptr_t)data;
330
3
  }
331
1
  ccv_cnnp_dataframe_iter_prefetch(iter, 2, 0);
332
1
  ccv_cnnp_dataframe_iter_next(iter, &data, 1, 0);
333
1
  result[3] = (int)(intptr_t)data;
334
1
  // Now, I have 1 prefetched, in the middle, and can prefetch 2 more without reallocating.
335
1
  ccv_cnnp_dataframe_iter_prefetch(iter, 3, 0);
336
1
  _ccv_iter_accessed = 0;
337
5
  for (i = 4; i < 8; 
i++4
)
338
4
  {
339
4
    ccv_cnnp_dataframe_iter_next(iter, &data, 1, 0);
340
4
    result[i] = (int)(intptr_t)data;
341
4
  }
342
1
  REQUIRE_EQ(_ccv_iter_accessed, 0, "no iterator accessed");
343
9
  for (i = 0; i < 8; 
i++8
)
344
8
    ++int_array[i];
345
1
  ccv_cnnp_dataframe_iter_free(iter);
346
1
  REQUIRE_ARRAY_EQ(int, int_array, result, 8, "iterated result and actual result should be the same");
347
1
  ccv_cnnp_dataframe_free(dataframe);
348
1
}
349
350
TEST_CASE("data is shuffled")
351
1
{
352
1
  int int_array[8] = {
353
1
    2, 3, 4, 5, 6, 7, 8, 9
354
1
  };
355
1
  ccv_cnnp_column_data_t columns[] = {
356
1
    {
357
1
      .data_enum = _ccv_iter_int,
358
1
      .context = int_array,
359
1
    }
360
1
  };
361
1
  ccv_cnnp_dataframe_t* const dataframe = ccv_cnnp_dataframe_new(columns, sizeof(columns) / sizeof(columns[0]), 8);
362
1
  const int derived = ccv_cnnp_dataframe_map(dataframe, _ccv_int_plus_1, 0, 0, COLUMN_ID_LIST(0), 0, 0, 0);
363
1
  assert(derived > 0);
364
1
  ccv_cnnp_dataframe_shuffle(dataframe);
365
1
  int i;
366
1
  void* data;
367
1
  int result[8];
368
1
  ccv_cnnp_dataframe_iter_t* const iter = ccv_cnnp_dataframe_iter_new(dataframe, COLUMN_ID_LIST(derived));
369
1
  _ccv_iter_accessed = 0;
370
1
  ccv_cnnp_dataframe_iter_prefetch(iter, 3, 0);
371
1
  REQUIRE_EQ(_ccv_iter_accessed, 1, "iter accessed once for prefetching");
372
4
  for (i = 0; i < 3; 
i++3
)
373
3
  {
374
3
    _ccv_iter_accessed = 0;
375
3
    ccv_cnnp_dataframe_iter_next(iter, &data, 1, 0);
376
3
    REQUIRE_EQ(_ccv_iter_accessed, 0, "no iterator accessed");
377
3
    result[i] = (int)(intptr_t)data;
378
3
  }
379
1
  ccv_cnnp_dataframe_iter_prefetch(iter, 2, 0);
380
1
  ccv_cnnp_dataframe_iter_next(iter, &data, 1, 0);
381
1
  result[3] = (int)(intptr_t)data;
382
1
  ccv_cnnp_dataframe_iter_prefetch(iter, 2, 0);
383
1
  _ccv_iter_accessed = 0;
384
5
  for (i = 4; i < 8; 
i++4
)
385
4
  {
386
4
    ccv_cnnp_dataframe_iter_next(iter, &data, 1, 0);
387
4
    result[i] = (int)(intptr_t)data;
388
4
  }
389
1
  REQUIRE_EQ(_ccv_iter_accessed, 1, "no iterator accessed");
390
9
  for (i = 0; i < 8; 
i++8
)
391
8
    ++int_array[i];
392
1
  ccv_cnnp_dataframe_iter_free(iter);
393
1
  REQUIRE_ARRAY_NOT_EQ(int, int_array, result, 8, "iterated result and actual result should not be the same");
394
1
  int covered[8] = {};
395
9
  for (i = 0; i < 8; 
i++8
)
396
8
    covered[result[i] - 3] = 1;
397
1
  int is_covered[8] = {
398
1
    1, 1, 1, 1, 1, 1, 1, 1
399
1
  };
400
1
  REQUIRE_ARRAY_EQ(int, covered, is_covered, 8, "data should covered all");
401
1
  ccv_cnnp_dataframe_free(dataframe);
402
1
}
403
404
static void _ccv_iter_sample_int(void* const* const input_data, const int batch_size, void** const data, void* const context, ccv_nnc_stream_context_t* const stream_context)
405
12
{
406
12
  int i;
407
12
  int total = 0;
408
44
  for (i = 0; i < batch_size; 
i++32
)
409
32
    total += (int)(intptr_t)input_data[i];
410
12
  data[0] = (void*)(intptr_t)total;
411
12
}
412
413
TEST_CASE("dataframe sample")
414
1
{
415
1
  int int_array[8] = {
416
1
    2, 3, 4, 5, 6, 7, 8, 9
417
1
  };
418
1
  ccv_cnnp_column_data_t columns[] = {
419
1
    {
420
1
      .data_enum = _ccv_iter_int,
421
1
      .context = int_array,
422
1
    }
423
1
  };
424
1
  ccv_cnnp_dataframe_t* const dataframe = ccv_cnnp_dataframe_new(columns, sizeof(columns) / sizeof(columns[0]), 8);
425
1
  ccv_cnnp_dataframe_t* const sample = ccv_cnnp_dataframe_sample_new(dataframe, _ccv_iter_sample_int, 0, 0, 3, 0, 0);
426
1
  ccv_cnnp_dataframe_iter_t* const iter = ccv_cnnp_dataframe_iter_new(sample, COLUMN_ID_LIST(0));
427
1
  int i;
428
1
  void* data;
429
1
  int result[3];
430
1
  _ccv_iter_accessed = 0;
431
3
  for (i = 0; ; i++)
432
4
  {
433
4
    if (0 != ccv_cnnp_dataframe_iter_next(iter, &data, 1, 0))
434
1
      break;
435
3
    result[i] = (int)(intptr_t)data;
436
3
  }
437
1
  REQUIRE_EQ(_ccv_iter_accessed, 3, "should only accessed iterator 3 times");
438
1
  int should_result[3] = {
439
1
    9, 18, 17
440
1
  };
441
1
  REQUIRE_ARRAY_EQ(int, should_result, result, 3, "iterated result and actual result should not be the same");
442
1
  ccv_cnnp_dataframe_iter_free(iter);
443
1
  ccv_cnnp_dataframe_free(sample);
444
1
  ccv_cnnp_dataframe_free(dataframe);
445
1
}
446
447
TEST_CASE("dataframe map before sample")
448
1
{
449
1
  int int_array[8] = {
450
1
    2, 3, 4, 5, 6, 7, 8, 9
451
1
  };
452
1
  ccv_cnnp_column_data_t columns[] = {
453
1
    {
454
1
      .data_enum = _ccv_iter_int,
455
1
      .context = int_array,
456
1
    }
457
1
  };
458
1
  ccv_cnnp_dataframe_t* const dataframe = ccv_cnnp_dataframe_new(columns, sizeof(columns) / sizeof(columns[0]), 8);
459
1
  const int derived = ccv_cnnp_dataframe_map(dataframe, _ccv_int_plus_1, 0, 0, COLUMN_ID_LIST(0), 0, 0, 0);
460
1
  assert(derived > 0);
461
1
  ccv_cnnp_dataframe_t* const sample = ccv_cnnp_dataframe_sample_new(dataframe, _ccv_iter_sample_int, 0, derived, 3, 0, 0);
462
1
  ccv_cnnp_dataframe_iter_t* const iter = ccv_cnnp_dataframe_iter_new(sample, COLUMN_ID_LIST(0));
463
1
  int i;
464
1
  void* data;
465
1
  int result[3];
466
3
  for (i = 0; ; i++)
467
4
  {
468
4
    if (0 != ccv_cnnp_dataframe_iter_next(iter, &data, 1, 0))
469
1
      break;
470
3
    result[i] = (int)(intptr_t)data;
471
3
  }
472
1
  int should_result[3] = {
473
1
    12, 21, 19
474
1
  };
475
1
  REQUIRE_ARRAY_EQ(int, should_result, result, 3, "iterated result and actual result should not be the same");
476
1
  ccv_cnnp_dataframe_iter_free(iter);
477
1
  ccv_cnnp_dataframe_free(sample);
478
1
  ccv_cnnp_dataframe_free(dataframe);
479
1
}
480
481
TEST_CASE("dataframe map after sample")
482
1
{
483
1
  int int_array[8] = {
484
1
    2, 3, 4, 5, 6, 7, 8, 9
485
1
  };
486
1
  ccv_cnnp_column_data_t columns[] = {
487
1
    {
488
1
      .data_enum = _ccv_iter_int,
489
1
      .context = int_array,
490
1
    }
491
1
  };
492
1
  ccv_cnnp_dataframe_t* const dataframe = ccv_cnnp_dataframe_new(columns, sizeof(columns) / sizeof(columns[0]), 8);
493
1
  ccv_cnnp_dataframe_t* const sample = ccv_cnnp_dataframe_sample_new(dataframe, _ccv_iter_sample_int, 0, 0, 3, 0, 0);
494
1
  const int derived = ccv_cnnp_dataframe_map(sample, _ccv_int_plus_1, 0, 0, COLUMN_ID_LIST(0), 0, 0, 0);
495
1
  assert(derived > 0);
496
1
  ccv_cnnp_dataframe_iter_t* const iter = ccv_cnnp_dataframe_iter_new(sample, COLUMN_ID_LIST(derived));
497
1
  int i;
498
1
  void* data;
499
1
  int result[3];
500
3
  for (i = 0; ; i++)
501
4
  {
502
4
    if (0 != ccv_cnnp_dataframe_iter_next(iter, &data, 1, 0))
503
1
      break;
504
3
    result[i] = (int)(intptr_t)data;
505
3
  }
506
1
  int should_result[3] = {
507
1
    10, 19, 18
508
1
  };
509
1
  REQUIRE_ARRAY_EQ(int, should_result, result, 3, "iterated result and actual result should not be the same");
510
1
  ccv_cnnp_dataframe_iter_free(iter);
511
1
  ccv_cnnp_dataframe_free(sample);
512
1
  ccv_cnnp_dataframe_free(dataframe);
513
1
}
514
515
TEST_CASE("dataframe map after sample shuffled")
516
1
{
517
1
  int int_array[8] = {
518
1
    2, 3, 4, 5, 6, 7, 8, 9
519
1
  };
520
1
  ccv_cnnp_column_data_t columns[] = {
521
1
    {
522
1
      .data_enum = _ccv_iter_int,
523
1
      .context = int_array,
524
1
    }
525
1
  };
526
1
  ccv_cnnp_dataframe_t* const dataframe = ccv_cnnp_dataframe_new(columns, sizeof(columns) / sizeof(columns[0]), 8);
527
1
  ccv_cnnp_dataframe_t* const sample = ccv_cnnp_dataframe_sample_new(dataframe, _ccv_iter_sample_int, 0, 0, 3, 0, 0);
528
1
  const int derived = ccv_cnnp_dataframe_map(sample, _ccv_int_plus_1, 0, 0, COLUMN_ID_LIST(0), 0, 0, 0);
529
1
  ccv_cnnp_dataframe_shuffle(sample);
530
1
  assert(derived > 0);
531
1
  ccv_cnnp_dataframe_iter_t* const iter = ccv_cnnp_dataframe_iter_new(sample, COLUMN_ID_LIST(derived));
532
1
  int i;
533
1
  void* data;
534
1
  int result[3];
535
3
  for (i = 0; ; i++)
536
4
  {
537
4
    if (0 != ccv_cnnp_dataframe_iter_next(iter, &data, 1, 0))
538
1
      break;
539
3
    result[i] = (int)(intptr_t)data;
540
3
  }
541
1
  int covered[3] = {};
542
4
  for (i = 0; i < 3; 
i++3
)
543
3
    if (result[i] == 10)
544
1
      covered[0] = 1;
545
2
    else if (result[i] == 19)
546
1
      covered[1] = 1;
547
1
    else if (result[i] == 18)
548
1
      covered[2] = 1;
549
1
  int is_covered[3] = {
550
1
    1, 1, 1
551
1
  };
552
1
  REQUIRE_ARRAY_EQ(int, covered, is_covered, 3, "data should covered all");
553
1
  ccv_cnnp_dataframe_iter_free(iter);
554
1
  ccv_cnnp_dataframe_free(sample);
555
1
  ccv_cnnp_dataframe_free(dataframe);
556
1
}
557
558
TEST_CASE("iterate through newly added column")
559
1
{
560
1
  int int_array[8] = {
561
1
    2, 3, 4, 5, 6, 7, 8, 9
562
1
  };
563
1
  ccv_cnnp_dataframe_t* const dataframe = ccv_cnnp_dataframe_new(0, 0, 8);
564
1
  const int derived = ccv_cnnp_dataframe_add(dataframe, _ccv_iter_int, 0, 0, int_array, 0, 0);
565
1
  assert(derived >= 0);
566
1
  ccv_cnnp_dataframe_iter_t* const iter = ccv_cnnp_dataframe_iter_new(dataframe, COLUMN_ID_LIST(derived));
567
1
  int result[8];
568
1
  int i = 0;
569
1
  void* data;
570
9
  while (0 == ccv_cnnp_dataframe_iter_next(iter, &data, 1, 0))
571
8
    result[i++] = (int)(intptr_t)data;
572
1
  ccv_cnnp_dataframe_iter_free(iter);
573
1
  REQUIRE_ARRAY_EQ(int, int_array, result, 8, "iterated result and actual result should be the same");
574
1
  // iter2 test some prefetch capacities.
575
1
  ccv_cnnp_dataframe_iter_t* const iter2 = ccv_cnnp_dataframe_iter_new(dataframe, COLUMN_ID_LIST(derived));
576
4
  for (i = 0; i < 3; 
i++3
)
577
3
    ccv_cnnp_dataframe_iter_prefetch(iter2, 1, 0);
578
1
  _ccv_iter_accessed = 0;
579
4
  for (i = 0; i < 3; 
i++3
)
580
3
  {
581
3
    ccv_cnnp_dataframe_iter_next(iter2, &data, 1, 0);
582
3
    result[i] = (int)(intptr_t)data;
583
3
  }
584
1
  REQUIRE_EQ(_ccv_iter_accessed, 0, "the iterator is not accessed at all, because prefetching");
585
1
  ccv_cnnp_dataframe_iter_next(iter2, &data, 1, 0);
586
1
  result[3] = (int)(intptr_t)data;
587
1
  REQUIRE_EQ(_ccv_iter_accessed, 1, "the iterator is accessed, because no prefetching");
588
1
  ccv_cnnp_dataframe_iter_prefetch(iter2, 4, 0);
589
1
  REQUIRE_EQ(_ccv_iter_accessed, 2, "the iterator is accessed again, for prefetching");
590
1
  _ccv_iter_accessed = 0;
591
3
  for (i = 4; i < 6; 
i++2
)
592
2
  {
593
2
    ccv_cnnp_dataframe_iter_next(iter2, &data, 1, 0);
594
2
    result[i] = (int)(intptr_t)data;
595
2
  }
596
1
  REQUIRE_EQ(_ccv_iter_accessed, 0, "the iterator accessed 0 times");
597
1
  const int fail0 = ccv_cnnp_dataframe_iter_prefetch(iter2, 1, 0);
598
1
  REQUIRE_EQ(fail0, -1, "should fail");
599
1
  ccv_cnnp_dataframe_iter_free(iter2);
600
1
  REQUIRE_ARRAY_EQ(int, int_array, result, 6, "iterated result and actual result should be the same up to 6");
601
1
  ccv_cnnp_dataframe_free(dataframe);
602
1
}
603
604
TEST_CASE("make a tuple out of column and derived column")
605
1
{
606
1
  int int_array[8] = {
607
1
    2, 3, 4, 5, 6, 7, 8, 9
608
1
  };
609
1
  ccv_cnnp_column_data_t columns[] = {
610
1
    {
611
1
      .data_enum = _ccv_iter_int,
612
1
      .context = int_array,
613
1
    }
614
1
  };
615
1
  ccv_cnnp_dataframe_t* const dataframe = ccv_cnnp_dataframe_new(columns, sizeof(columns) / sizeof(columns[0]), 8);
616
1
  const int derived = ccv_cnnp_dataframe_map(dataframe, _ccv_int_plus_1, 0, 0, COLUMN_ID_LIST(0), 0, 0, 0);
617
1
  assert(derived > 0);
618
1
  const int tuple = ccv_cnnp_dataframe_make_tuple(dataframe, COLUMN_ID_LIST(derived, 0), 0);
619
1
  ccv_cnnp_dataframe_iter_t* const iter = ccv_cnnp_dataframe_iter_new(dataframe, COLUMN_ID_LIST(tuple));
620
1
  int result[2][8];
621
1
  int i = 0;
622
1
  void* data;
623
9
  while (0 == ccv_cnnp_dataframe_iter_next(iter, &data, 1, 0))
624
8
  {
625
8
    void** int_data = (void**)data;
626
8
    result[0][i] = (int)(intptr_t)int_data[0];
627
8
    result[1][i] = (int)(intptr_t)int_data[1];
628
8
    ++i;
629
8
  }
630
1
  ccv_cnnp_dataframe_iter_free(iter);
631
1
  REQUIRE_EQ(ccv_cnnp_dataframe_tuple_size(dataframe, tuple), 2, "It should contain two columns");
632
1
  ccv_cnnp_dataframe_free(dataframe);
633
1
  REQUIRE_ARRAY_EQ(int, int_array, result[1], 8, "iterated tuple should be the same");
634
9
  for (i = 0; i < 8; 
i++8
)
635
8
    ++int_array[i];
636
1
  REQUIRE_ARRAY_EQ(int, int_array, result[0], 8, "iterated tuple should be the same");
637
1
}
638
639
TEST_CASE("extract value out of a tuple")
640
1
{
641
1
  int int_array[8] = {
642
1
    2, 3, 4, 5, 6, 7, 8, 9
643
1
  };
644
1
  ccv_cnnp_column_data_t columns[] = {
645
1
    {
646
1
      .data_enum = _ccv_iter_int,
647
1
      .context = int_array,
648
1
    }
649
1
  };
650
1
  ccv_cnnp_dataframe_t* const dataframe = ccv_cnnp_dataframe_new(columns, sizeof(columns) / sizeof(columns[0]), 8);
651
1
  const int derived = ccv_cnnp_dataframe_map(dataframe, _ccv_int_plus_1, 0, 0, COLUMN_ID_LIST(0), 0, 0, 0);
652
1
  assert(derived > 0);
653
1
  const int tuple = ccv_cnnp_dataframe_make_tuple(dataframe, COLUMN_ID_LIST(0, derived), 0);
654
1
  const int extract = ccv_cnnp_dataframe_extract_tuple(dataframe, tuple, 1, 0);
655
1
  ccv_cnnp_dataframe_iter_t* const iter = ccv_cnnp_dataframe_iter_new(dataframe, COLUMN_ID_LIST(extract));
656
1
  int result[8];
657
1
  int i = 0;
658
1
  void* data;
659
9
  while (0 == ccv_cnnp_dataframe_iter_next(iter, &data, 1, 0))
660
8
    result[i++] = (int)(intptr_t)data;
661
1
  ccv_cnnp_dataframe_iter_free(iter);
662
1
  REQUIRE_EQ(ccv_cnnp_dataframe_tuple_size(dataframe, tuple), 2, "It should contain two columns");
663
1
  ccv_cnnp_dataframe_free(dataframe);
664
9
  for (i = 0; i < 8; 
i++8
)
665
8
    ++int_array[i];
666
1
  REQUIRE_ARRAY_EQ(int, int_array, result, 8, "iterated tuple should be the same");
667
1
}
668
669
TEST_CASE("iterate through derived column with peek")
670
1
{
671
1
  int int_array[8] = {
672
1
    2, 3, 4, 5, 6, 7, 8, 9
673
1
  };
674
1
  ccv_cnnp_column_data_t columns[] = {
675
1
    {
676
1
      .data_enum = _ccv_iter_int,
677
1
      .context = int_array,
678
1
    }
679
1
  };
680
1
  ccv_cnnp_dataframe_t* const dataframe = ccv_cnnp_dataframe_new(columns, sizeof(columns) / sizeof(columns[0]), 8);
681
1
  const int derived = ccv_cnnp_dataframe_map(dataframe, _ccv_int_plus_1, 0, 0, COLUMN_ID_LIST(0), 0, 0, 0);
682
1
  assert(derived > 0);
683
1
  ccv_cnnp_dataframe_iter_t* const iter = ccv_cnnp_dataframe_iter_new(dataframe, COLUMN_ID_LIST(0, derived));
684
1
  int result0[8];
685
1
  int result1[8];
686
1
  int i = 0;
687
1
  void* data;
688
9
  while (0 == ccv_cnnp_dataframe_iter_next(iter, 0, 0, 0))
689
8
  {
690
8
    ccv_cnnp_dataframe_iter_peek(iter, &data, 0, 1, 0);
691
8
    result0[i] = (int)(intptr_t)data;
692
8
    ccv_cnnp_dataframe_iter_peek(iter, &data, 1, 1, 0);
693
8
    result1[i] = (int)(intptr_t)data;
694
8
    ++i;
695
8
  }
696
1
  ccv_cnnp_dataframe_iter_free(iter);
697
1
  REQUIRE_ARRAY_EQ(int, int_array, result0, 8, "iterated result and actual result should be the same");
698
9
  for (i = 0; i < 8; 
i++8
)
699
8
    ++int_array[i];
700
1
  REQUIRE_ARRAY_EQ(int, int_array, result1, 8, "iterated result and actual result should be the same");
701
1
  ccv_cnnp_dataframe_free(dataframe);
702
1
}
703
704
#include "case_main.h"