Coverage Report

Created: 2021-04-07 03:47

/home/liu/buildslave/linux-x64-runtests/build/lib/ccv_cache.c
Line
Count
Source (jump to first uncovered line)
1
#include "ccv.h"
2
#include "ccv_internal.h"
3
4
5.06M
#define CCV_GET_CACHE_TYPE(x) ((x) >> 60)
5
86.6M
#define CCV_GET_TERMINAL_AGE(x) (((x) >> 32) & 0x0FFFFFFF)
6
4.53M
#define CCV_GET_TERMINAL_SIZE(x) ((x) & 0xFFFFFFFF)
7
5.06M
#define CCV_SET_TERMINAL_TYPE(x, y, z) (((uint64_t)(x) << 60) | ((uint64_t)(y) << 32) | (z))
8
9
void ccv_cache_init(ccv_cache_t* cache, size_t up, int cache_types, ccv_cache_index_free_f ffree, ...)
10
4
{
11
4
  cache->rnum = 0;
12
4
  cache->age = 0;
13
4
  cache->up = up;
14
4
  cache->size = 0;
15
4
  assert(cache_types > 0 && cache_types <= 16);
16
4
  va_list arguments;
17
4
  va_start(arguments, ffree);
18
4
  int i;
19
4
  cache->ffree[0] = ffree;
20
7
  for (i = 1; i < cache_types; 
i++3
)
21
3
    cache->ffree[i] = va_arg(arguments, ccv_cache_index_free_f);
22
4
  va_end(arguments);
23
4
  memset(&cache->origin, 0, sizeof(ccv_cache_index_t));
24
4
}
25
26
static int bits_in_16bits[0x1u << 16];
27
static int bits_in_16bits_init = 0;
28
29
65.5k
static int sparse_bitcount(unsigned int n) {
30
65.5k
  int count = 0;
31
589k
  while (n) {
32
524k
    count++;
33
524k
    n &= (n - 1);
34
524k
  }
35
65.5k
  return count;
36
65.5k
}
37
38
1
static void precomputed_16bits() {
39
1
  int i;
40
65.5k
  for (i = 0; i < (0x1u << 16); 
i++65.5k
)
41
65.5k
    bits_in_16bits[i] = sparse_bitcount(i);
42
1
  bits_in_16bits_init = 1;
43
1
}
44
45
62.7M
static uint32_t compute_bits(uint64_t m) {
46
62.7M
  return (bits_in_16bits[m & 0xffff] + bits_in_16bits[(m >> 16) & 0xffff] +
47
62.7M
      bits_in_16bits[(m >> 32) & 0xffff] + bits_in_16bits[(m >> 48) & 0xffff]);
48
62.7M
}
49
50
/* update age along a path in the radix tree */
51
static void _ccv_cache_aging(ccv_cache_index_t* branch, uint64_t sign)
52
4.53M
{
53
4.53M
  if (!bits_in_16bits_init)
54
0
    precomputed_16bits();
55
4.53M
  int i;
56
4.53M
  uint64_t j = 63;
57
4.53M
  ccv_cache_index_t* breadcrumb[10];
58
17.4M
  for (i = 0; i < 10; 
i++12.9M
)
59
17.4M
  {
60
17.4M
    breadcrumb[i] = branch;
61
17.4M
    int leaf = branch->terminal.off & 0x1;
62
17.4M
    int full = branch->terminal.off & 0x2;
63
17.4M
    if (leaf)
64
1.47M
    {
65
1.47M
      break;
66
15.9M
    } else {
67
15.9M
      ccv_cache_index_t* set = (ccv_cache_index_t*)(branch->branch.set - (branch->branch.set & 0x3));
68
15.9M
      int dice = (sign & j) >> (i * 6);
69
15.9M
      if (full)
70
9.38M
      {
71
9.38M
        branch = set + dice;
72
9.38M
      } else {
73
6.60M
        uint64_t k = 1;
74
6.60M
        k = k << dice;
75
6.60M
        if (k & branch->branch.bitmap) {
76
3.54M
          uint64_t m = (k - 1) & branch->branch.bitmap;
77
3.54M
          branch = set + compute_bits(m);
78
3.54M
        } else {
79
3.06M
          break;
80
3.06M
        }
81
12.9M
      }
82
12.9M
      j <<= 6;
83
12.9M
    }
84
17.4M
  }
85
4.53M
  assert(i < 10);
86
21.9M
  
for (; 4.53M
i >= 0;
i--17.4M
)
87
17.4M
  {
88
17.4M
    branch = breadcrumb[i];
89
17.4M
    int leaf = branch->terminal.off & 0x1;
90
17.4M
    if (!leaf)
91
15.9M
    {
92
15.9M
      ccv_cache_index_t* set = (ccv_cache_index_t*)(branch->branch.set - (branch->branch.set & 0x3));
93
15.9M
      uint32_t total = compute_bits(branch->branch.bitmap);
94
15.9M
      uint32_t min_age = (set[0].terminal.off & 0x1) ? 
CCV_GET_TERMINAL_AGE4.15M
(set[0].terminal.type) :
set[0].branch.age11.8M
;
95
804M
      for (j = 1; j < total; 
j++788M
)
96
788M
      {
97
788M
        uint32_t age = (set[j].terminal.off & 0x1) ? 
CCV_GET_TERMINAL_AGE74.3M
(set[j].terminal.type) :
set[j].branch.age714M
;
98
788M
        if (age < min_age)
99
49.2M
          min_age = age;
100
788M
      }
101
15.9M
      branch->branch.age = min_age;
102
15.9M
    }
103
17.4M
  }
104
4.53M
}
105
106
static ccv_cache_index_t* _ccv_cache_seek(ccv_cache_index_t* branch, uint64_t sign, int* depth)
107
5.86M
{
108
5.86M
  if (!bits_in_16bits_init)
109
1
    precomputed_16bits();
110
5.86M
  int i;
111
5.86M
  uint64_t j = 63;
112
23.0M
  for (i = 0; i < 10; 
i++17.1M
)
113
23.0M
  {
114
23.0M
    int leaf = branch->terminal.off & 0x1;
115
23.0M
    int full = branch->terminal.off & 0x2;
116
23.0M
    if (leaf)
117
2.25M
    {
118
2.25M
      if (depth)
119
1.64M
        *depth = i;
120
2.25M
      return branch;
121
20.7M
    } else {
122
20.7M
      ccv_cache_index_t* set = (ccv_cache_index_t*)(branch->branch.set - (branch->branch.set & 0x3));
123
20.7M
      int dice = (sign & j) >> (i * 6);
124
20.7M
      if (full)
125
12.0M
      {
126
12.0M
        branch = set + dice;
127
12.0M
      } else {
128
8.74M
        uint64_t k = 1;
129
8.74M
        k = k << dice;
130
8.74M
        if (k & branch->branch.bitmap) {
131
5.12M
          uint64_t m = (k - 1) & branch->branch.bitmap;
132
5.12M
          branch = set + compute_bits(m);
133
5.12M
        } else {
134
3.61M
          if (depth)
135
3.42M
            *depth = i;
136
3.61M
          return branch;
137
3.61M
        }
138
17.1M
      }
139
17.1M
      j <<= 6;
140
17.1M
    }
141
23.0M
  }
142
5.86M
  
return 00
;
143
5.86M
}
144
145
void* ccv_cache_get(ccv_cache_t* cache, uint64_t sign, uint8_t* type)
146
800k
{
147
800k
  if (cache->rnum == 0)
148
0
    return 0;
149
800k
  ccv_cache_index_t* branch = _ccv_cache_seek(&cache->origin, sign, 0);
150
800k
  if (!branch)
151
0
    return 0;
152
800k
  int leaf = branch->terminal.off & 0x1;
153
800k
  if (!leaf)
154
188k
    return 0;
155
611k
  if (branch->terminal.sign != sign)
156
77.4k
    return 0;
157
533k
  if (type)
158
0
    *type = CCV_GET_CACHE_TYPE(branch->terminal.type);
159
533k
  return (void*)(branch->terminal.off - (branch->terminal.off & 0x3));
160
533k
}
161
162
// only call this function when the cache space is delpeted
163
static void _ccv_cache_lru(ccv_cache_t* cache)
164
679k
{
165
679k
  ccv_cache_index_t* branch = &cache->origin;
166
679k
  int leaf = branch->terminal.off & 0x1;
167
679k
  if (leaf)
168
0
  {
169
0
    void* result = (void*)(branch->terminal.off - (branch->terminal.off & 0x3));
170
0
    uint8_t type = CCV_GET_CACHE_TYPE(branch->terminal.type);
171
0
    if (result != 0)
172
0
    {
173
0
      assert(type >= 0 && type < 16);
174
0
      cache->ffree[type](result);
175
0
    }
176
0
    cache->rnum = 0;
177
0
    cache->size = 0;
178
0
    return;
179
679k
  }
180
679k
  uint32_t min_age = branch->branch.age;
181
679k
  int i, j;
182
3.30M
  for (i = 0; i < 10; 
i++2.62M
)
183
3.30M
  {
184
3.30M
    ccv_cache_index_t* old_branch = branch;
185
3.30M
    int leaf = branch->terminal.off & 0x1;
186
3.30M
    if (leaf)
187
679k
    {
188
679k
      ccv_cache_delete(cache, branch->terminal.sign);
189
679k
      break;
190
2.62M
    } else {
191
2.62M
      ccv_cache_index_t* set = (ccv_cache_index_t*)(branch->branch.set - (branch->branch.set & 0x3));
192
2.62M
      uint32_t total = compute_bits(branch->branch.bitmap);
193
63.9M
      for (j = 0; j < total; 
j++61.3M
)
194
63.9M
      {
195
63.9M
        uint32_t age = (set[j].terminal.off & 0x1) ? 
CCV_GET_TERMINAL_AGE6.86M
(set[j].terminal.type) :
set[j].branch.age57.1M
;
196
63.9M
        assert(age >= min_age);
197
63.9M
        if (age == min_age)
198
2.62M
        {
199
2.62M
          branch = set + j;
200
2.62M
          break;
201
2.62M
        }
202
63.9M
      }
203
2.62M
      assert(old_branch != branch);
204
2.62M
    }
205
3.30M
  }
206
679k
  assert(i < 10);
207
679k
}
208
209
static void _ccv_cache_depleted(ccv_cache_t* cache, size_t size)
210
679k
{
211
1.35M
  while (cache->size > size)
212
679k
    _ccv_cache_lru(cache);
213
679k
}
214
215
int ccv_cache_put(ccv_cache_t* cache, uint64_t sign, void* x, uint32_t size, uint8_t type)
216
5.06M
{
217
5.06M
  assert(((uint64_t)x & 0x3) == 0);
218
5.06M
  if (size > cache->up)
219
0
    return -1;
220
5.06M
  if (size + cache->size > cache->up)
221
679k
    _ccv_cache_depleted(cache, cache->up - size);
222
5.06M
  if (cache->rnum == 0)
223
5
  {
224
5
    cache->age = 1;
225
5
    cache->origin.terminal.off = (uint64_t)x | 0x1;
226
5
    cache->origin.terminal.sign = sign;
227
5
    cache->origin.terminal.type = CCV_SET_TERMINAL_TYPE(type, cache->age, size);
228
5
    cache->size = size;
229
5
    cache->rnum = 1;
230
5
    return 0;
231
5
  }
232
5.06M
  ++cache->age;
233
5.06M
  int i, depth = -1;
234
5.06M
  ccv_cache_index_t* branch = _ccv_cache_seek(&cache->origin, sign, &depth);
235
5.06M
  if (!branch)
236
0
    return -1;
237
5.06M
  int leaf = branch->terminal.off & 0x1;
238
5.06M
  uint64_t on = 1;
239
5.06M
  assert(depth >= 0);
240
5.06M
  if (leaf)
241
1.64M
  {
242
1.64M
    if (sign == branch->terminal.sign)
243
356k
    {
244
356k
      cache->ffree[CCV_GET_CACHE_TYPE(branch->terminal.type)]((void*)(branch->terminal.off - (branch->terminal.off & 0x3)));
245
356k
      branch->terminal.off = (uint64_t)x | 0x1;
246
356k
      uint32_t old_size = CCV_GET_TERMINAL_SIZE(branch->terminal.type);
247
356k
      cache->size = cache->size + size - old_size;
248
356k
      branch->terminal.type = CCV_SET_TERMINAL_TYPE(type, cache->age, size);
249
356k
      _ccv_cache_aging(&cache->origin, sign);
250
356k
      return 1;
251
1.28M
    } else {
252
1.28M
      ccv_cache_index_t t = *branch;
253
1.28M
      uint32_t age = CCV_GET_TERMINAL_AGE(branch->terminal.type);
254
1.28M
      uint64_t j = 63;
255
1.28M
      j = j << (depth * 6);
256
1.28M
      int dice, udice;
257
1.28M
      assert(depth < 10);
258
1.30M
      
for (i = depth; 1.28M
i < 10;
i++20.5k
)
259
1.30M
      {
260
1.30M
        dice = (t.terminal.sign & j) >> (i * 6);
261
1.30M
        udice = (sign & j) >> (i * 6);
262
1.30M
        if (dice == udice)
263
20.5k
        {
264
20.5k
          branch->branch.bitmap = on << dice;
265
20.5k
          ccv_cache_index_t* set = (ccv_cache_index_t*)ccmalloc(sizeof(ccv_cache_index_t));
266
20.5k
          assert(((uint64_t)set & 0x3) == 0);
267
20.5k
          branch->branch.set = (uint64_t)set;
268
20.5k
          branch->branch.age = age;
269
20.5k
          branch = set;
270
1.28M
        } else {
271
1.28M
          break;
272
1.28M
        }
273
20.5k
        j <<= 6;
274
20.5k
      }
275
1.28M
      branch->branch.bitmap = (on << dice) | (on << udice);
276
1.28M
      ccv_cache_index_t* set = (ccv_cache_index_t*)ccmalloc(sizeof(ccv_cache_index_t) * 2);
277
1.28M
      assert(((uint64_t)set & 0x3) == 0);
278
1.28M
      branch->branch.set = (uint64_t)set;
279
1.28M
      branch->branch.age = age;
280
1.28M
      int u = dice < udice;
281
1.28M
      set[u].terminal.sign = sign;
282
1.28M
      set[u].terminal.off = (uint64_t)x | 0x1;
283
1.28M
      set[u].terminal.type = CCV_SET_TERMINAL_TYPE(type, cache->age, size);
284
1.28M
      set[1 - u] = t;
285
1.28M
    }
286
3.42M
  } else {
287
3.42M
    uint64_t k = 1, j = 63;
288
3.42M
    k = k << ((sign & (j << (depth * 6))) >> (depth * 6));
289
3.42M
    uint64_t m = (k - 1) & branch->branch.bitmap;
290
3.42M
    uint32_t start = compute_bits(m);
291
3.42M
    uint32_t total = compute_bits(branch->branch.bitmap);
292
3.42M
    ccv_cache_index_t* set = (ccv_cache_index_t*)(branch->branch.set - (branch->branch.set & 0x3));
293
3.42M
    set = (ccv_cache_index_t*)ccrealloc(set, sizeof(ccv_cache_index_t) * (total + 1));
294
3.42M
    assert(((uint64_t)set & 0x3) == 0);
295
27.6M
    
for (i = total; 3.42M
i > start;
i--24.2M
)
296
24.2M
      set[i] = set[i - 1];
297
3.42M
    set[start].terminal.off = (uint64_t)x | 0x1;
298
3.42M
    set[start].terminal.sign = sign;
299
3.42M
    set[start].terminal.type = CCV_SET_TERMINAL_TYPE(type, cache->age, size);
300
3.42M
    branch->branch.set = (uint64_t)set;
301
3.42M
    branch->branch.bitmap |= k;
302
3.42M
    if (total == 63)
303
4.28k
      branch->branch.set |= 0x2;
304
3.42M
  }
305
5.06M
  cache->rnum++;
306
4.71M
  cache->size += size;
307
4.71M
  return 0;
308
5.06M
}
309
310
static void _ccv_cache_cleanup(ccv_cache_index_t* branch)
311
1.13M
{
312
1.13M
  int leaf = branch->terminal.off & 0x1;
313
1.13M
  if (!leaf)
314
1.13M
  {
315
1.13M
    int i;
316
1.13M
    uint64_t total = compute_bits(branch->branch.bitmap);
317
1.13M
    ccv_cache_index_t* set = (ccv_cache_index_t*)(branch->branch.set - (branch->branch.set & 0x3));
318
3.38M
    for (i = 0; i < total; 
i++2.25M
)
319
2.25M
    {
320
2.25M
      if (!(set[i].terminal.off & 0x1))
321
17.8k
        _ccv_cache_cleanup(set + i);
322
2.25M
    }
323
1.13M
    ccfree(set);
324
1.13M
  }
325
1.13M
}
326
327
static void _ccv_cache_cleanup_and_free(ccv_cache_index_t* branch, ccv_cache_index_free_f ffree[])
328
704k
{
329
704k
  int leaf = branch->terminal.off & 0x1;
330
704k
  if (!leaf)
331
171k
  {
332
171k
    int i;
333
171k
    uint64_t total = compute_bits(branch->branch.bitmap);
334
171k
    ccv_cache_index_t* set = (ccv_cache_index_t*)(branch->branch.set - (branch->branch.set & 0x3));
335
875k
    for (i = 0; i < total; 
i++704k
)
336
704k
      _ccv_cache_cleanup_and_free(set + i, ffree);
337
171k
    ccfree(set);
338
533k
  } else {
339
533k
    assert(CCV_GET_CACHE_TYPE(branch->terminal.type) >= 0 && CCV_GET_CACHE_TYPE(branch->terminal.type) < 16);
340
533k
    ffree[CCV_GET_CACHE_TYPE(branch->terminal.type)]((void*)(branch->terminal.off - (branch->terminal.off & 0x3)));
341
533k
  }
342
704k
}
343
344
void* ccv_cache_out(ccv_cache_t* cache, uint64_t sign, uint8_t* type)
345
4.75M
{
346
4.75M
  if (!bits_in_16bits_init)
347
0
    precomputed_16bits();
348
4.75M
  if (cache->rnum == 0)
349
487k
    return 0;
350
4.26M
  int i, found = 0, depth = -1;
351
4.26M
  ccv_cache_index_t* parent = 0;
352
4.26M
  ccv_cache_index_t* uncle = &cache->origin;
353
4.26M
  ccv_cache_index_t* branch = &cache->origin;
354
4.26M
  uint64_t j = 63;
355
20.2M
  for (i = 0; i < 10; 
i++15.9M
)
356
20.2M
  {
357
20.2M
    int leaf = branch->terminal.off & 0x1;
358
20.2M
    int full = branch->terminal.off & 0x2;
359
20.2M
    if (leaf)
360
4.20M
    {
361
4.20M
      found = 1;
362
4.20M
      break;
363
4.20M
    }
364
16.0M
    if (parent != 0 && 
compute_bits(parent->branch.bitmap) > 111.7M
)
365
11.7M
      uncle = branch;
366
16.0M
    parent = branch;
367
16.0M
    depth = i;
368
16.0M
    ccv_cache_index_t* set = (ccv_cache_index_t*)(branch->branch.set - (branch->branch.set & 0x3));
369
16.0M
    int dice = (sign & j) >> (i * 6);
370
16.0M
    if (full)
371
8.84M
    {
372
8.84M
      branch = set + dice;
373
8.84M
    } else {
374
7.20M
      uint64_t k = 1;
375
7.20M
      k = k << dice;
376
7.20M
      if (k & branch->branch.bitmap)
377
7.14M
      {
378
7.14M
        uint64_t m = (k - 1) & branch->branch.bitmap;
379
7.14M
        branch = set + compute_bits(m);
380
7.14M
      } else {
381
62.9k
        return 0;
382
62.9k
      }
383
15.9M
    }
384
15.9M
    j <<= 6;
385
15.9M
  }
386
4.26M
  
if (4.20M
!found4.20M
)
387
0
    return 0;
388
4.20M
  int leaf = branch->terminal.off & 0x1;
389
4.20M
  if (!leaf)
390
0
    return 0;
391
4.20M
  if (branch->terminal.sign != sign)
392
25.6k
    return 0;
393
4.17M
  void* result = (void*)(branch->terminal.off - (branch->terminal.off & 0x3));
394
4.17M
  if (type)
395
4.17M
    *type = CCV_GET_CACHE_TYPE(branch->terminal.type);
396
4.17M
  uint32_t size = CCV_GET_TERMINAL_SIZE(branch->terminal.type);
397
4.17M
  if (branch != &cache->origin)
398
4.17M
  {
399
4.17M
    uint64_t k = 1, j = 63;
400
4.17M
    int dice = (sign & (j << (depth * 6))) >> (depth * 6);
401
4.17M
    k = k << dice;
402
4.17M
    uint64_t m = (k - 1) & parent->branch.bitmap;
403
4.17M
    uint32_t start = compute_bits(m);
404
4.17M
    uint32_t total = compute_bits(parent->branch.bitmap);
405
4.17M
    assert(total > 1);
406
4.17M
    ccv_cache_index_t* set = (ccv_cache_index_t*)(parent->branch.set - (parent->branch.set & 0x3));
407
4.17M
    if (total > 2 || 
(1.12M
total == 21.12M
&&
!(set[1 - start].terminal.off & 0x1)1.12M
))
408
3.06M
    {
409
3.06M
      parent->branch.bitmap &= ~k;
410
23.9M
      for (i = start + 1; i < total; 
i++20.8M
)
411
20.8M
        set[i - 1] = set[i];
412
3.06M
      set = (ccv_cache_index_t*)ccrealloc(set, sizeof(ccv_cache_index_t) * (total - 1));
413
3.06M
      parent->branch.set = (uint64_t)set;
414
3.06M
    } else {
415
1.11M
      ccv_cache_index_t t = set[1 - start];
416
1.11M
      _ccv_cache_cleanup(uncle);
417
1.11M
      *uncle = t;
418
1.11M
    }
419
4.17M
    _ccv_cache_aging(&cache->origin, sign);
420
4.17M
  } else {
421
4
    // if I only have one item, reset age to 1
422
4
    cache->age = 1;
423
4
  }
424
4.17M
  cache->rnum--;
425
4.17M
  cache->size -= size;
426
4.17M
  return result;
427
4.17M
}
428
429
int ccv_cache_delete(ccv_cache_t* cache, uint64_t sign)
430
1.74M
{
431
1.74M
  uint8_t type = 0;
432
1.74M
  void* result = ccv_cache_out(cache, sign, &type);
433
1.74M
  if (result != 0)
434
1.65M
  {
435
1.65M
    assert(type >= 0 && type < 16);
436
1.65M
    cache->ffree[type](result);
437
1.65M
    return 0;
438
88.6k
  }
439
88.6k
  return -1;
440
88.6k
}
441
442
void ccv_cache_cleanup(ccv_cache_t* cache)
443
4
{
444
4
  if (cache->rnum > 0)
445
1
  {
446
1
    _ccv_cache_cleanup_and_free(&cache->origin, cache->ffree);
447
1
    cache->size = 0;
448
1
    cache->age = 0;
449
1
    cache->rnum = 0;
450
1
    memset(&cache->origin, 0, sizeof(ccv_cache_index_t));
451
1
  }
452
4
}
453
454
void ccv_cache_close(ccv_cache_t* cache)
455
4
{
456
4
  // for radix-tree based cache, close/cleanup are the same (it is not the same for cuckoo based one,
457
4
  // because for cuckoo based one, it will free up space in close whereas only cleanup space in cleanup
458
4
  ccv_cache_cleanup(cache);
459
4
}