Coverage Report

Created: 2019-07-03 22:50

/home/liu/buildslave/linux-x64-runtests/build/lib/nnc/ccv_cnnp_model_io.c
Line
Count
Source (jump to first uncovered line)
1
#include "ccv_nnc.h"
2
#include "ccv_nnc_easy.h"
3
#include "ccv_nnc_internal.h"
4
#include "ccv_internal.h"
5
#include "_ccv_cnnp_model.h"
6
#include "3rdparty/sqlite3/sqlite3.h"
7
#ifdef HAVE_CUDA
8
#include "gpu/ccv_nnc_compat.h"
9
#endif
10
11
#ifdef NDEBUG
12
#define SQLITE_ENFORCE(stmt) (void)(stmt)
13
#else
14
4
#define SQLITE_ENFORCE assert
15
#endif
16
17
void ccv_cnnp_model_checkpoint(ccv_cnnp_model_t* const model, const char* const fn, const int flags)
18
2
{
19
2
  ccv_cnnp_compiled_data_t* const compiled_data = model->compiled_data;
20
2
  assert(compiled_data); // The model has to be compiled.
21
2
  sqlite3* conn = 0;
22
2
  if (SQLITE_OK != sqlite3_open(fn, &conn))
23
0
    return;
24
2
  const int tensors_init = !!compiled_data->tensors.trainables;
25
2
  int i;
26
2
  const int parallel_count = ccv_max(compiled_data->parallel_count, 1);
27
2
  const int trainable_size = compiled_data->trainables->rnum;
28
2
  const int retainable_size = compiled_data->retainables->rnum * parallel_count;
29
2
  if (!tensors_init || 
flags == CCV_CNNP_MODEL_CHECKPOINT_READ_ONLY1
)
30
1
  {
31
1
    const char tensor_checkpoint_select_qs[] =
32
1
      "SELECT id, data FROM tensor_checkpoint ORDER BY id";
33
1
    sqlite3_stmt* tensor_checkpoint_select_stmt = 0;
34
1
    if (SQLITE_OK != sqlite3_prepare_v2(conn, tensor_checkpoint_select_qs, sizeof(tensor_checkpoint_select_qs), &tensor_checkpoint_select_stmt, 0))
35
0
    {
36
0
      sqlite3_close(conn);
37
0
      return;
38
0
    }
39
1
    if (!tensors_init)
40
1
      ccv_cnnp_model_tensors_init(model->graph, compiled_data);
41
11
    for (i = 0; i < trainable_size && 
SQLITE_ROW10
== sqlite3_step(tensor_checkpoint_select_stmt)10
;
i++10
)
42
10
    {
43
10
      const void* const data = sqlite3_column_blob(tensor_checkpoint_select_stmt, 1);
44
10
      ccv_nnc_tensor_t* const tensor = compiled_data->tensors.trainables[i];
45
10
      size_t data_size = ccv_nnc_tensor_data_size(tensor->info);
46
10
#ifdef HAVE_CUDA
47
10
      if (CCV_TENSOR_GET_MEMORY(tensor->info.type) == CCV_TENSOR_GPU_MEMORY)
48
0
        cumemcpy(tensor->data.u8, tensor->info.type, data, CCV_TENSOR_CPU_MEMORY, ccv_min(data_size, sqlite3_column_bytes(tensor_checkpoint_select_stmt, 1)));
49
10
      else
50
10
        memcpy(tensor->data.u8, data, ccv_min(data_size, sqlite3_column_bytes(tensor_checkpoint_select_stmt, 1)));
51
#else
52
      memcpy(tensor->data.u8, data, ccv_min(data_size, sqlite3_column_bytes(tensor_checkpoint_select_stmt, 1)));
53
#endif
54
    }
55
1
    for (i = 0; i < retainable_size && 
SQLITE_ROW0
== sqlite3_step(tensor_checkpoint_select_stmt)0
;
i++0
)
56
0
    {
57
0
      const void* const data = sqlite3_column_blob(tensor_checkpoint_select_stmt, 1);
58
0
      if (!data)
59
0
        continue;
60
0
      ccv_nnc_tensor_t* const tensor = compiled_data->tensors.retainables[i];
61
0
      size_t data_size = ccv_nnc_tensor_data_size(tensor->info);
62
0
#ifdef HAVE_CUDA
63
0
      if (CCV_TENSOR_GET_MEMORY(tensor->info.type) == CCV_TENSOR_GPU_MEMORY)
64
0
        cumemcpy(tensor->data.u8, tensor->info.type, data, CCV_TENSOR_CPU_MEMORY, ccv_min(data_size, sqlite3_column_bytes(tensor_checkpoint_select_stmt, 1)));
65
0
      else
66
0
        memcpy(tensor->data.u8, data, ccv_min(data_size, sqlite3_column_bytes(tensor_checkpoint_select_stmt, 1)));
67
#else
68
      memcpy(tensor->data.u8, data, ccv_min(data_size, sqlite3_column_bytes(tensor_checkpoint_select_stmt, 1)));
69
#endif
70
    }
71
1
    sqlite3_finalize(tensor_checkpoint_select_stmt);
72
1
    sqlite3_close(conn);
73
1
    return;
74
1
  }
75
1
  const char tensor_checkpoint_create_table_qs[] = "CREATE TABLE IF NOT EXISTS tensor_checkpoint "
76
1
    "(id INTEGER, type INTEGER, format INTEGER, datatype INTEGER, "
77
1
    "dim BLOB, data BLOB, PRIMARY KEY (id))";
78
1
  SQLITE_ENFORCE(SQLITE_OK == sqlite3_exec(conn, tensor_checkpoint_create_table_qs, 0, 0, 0));
79
1
  const char tensor_checkpoint_insert_qs[] =
80
1
    "REPLACE INTO tensor_checkpoint "
81
1
    "(id, type, format, datatype, dim, data) VALUES ("
82
1
    "$id, $type, $format, $datatype, $dim, $data)";
83
1
  sqlite3_stmt* tensor_checkpoint_insert_stmt = 0;
84
1
  SQLITE_ENFORCE(SQLITE_OK == sqlite3_prepare_v2(conn, tensor_checkpoint_insert_qs, sizeof(tensor_checkpoint_insert_qs), &tensor_checkpoint_insert_stmt, 0));
85
1
  SQLITE_ENFORCE(SQLITE_OK == sqlite3_exec(conn, "BEGIN", 0, 0, 0));
86
1
#ifdef HAVE_CUDA
87
1
  size_t workspace_size = 0;
88
1
  void* workspace = 0;
89
1
#endif
90
11
  for (i = 0; i < trainable_size; 
i++10
)
91
10
  {
92
10
    const ccv_nnc_tensor_t* const tensor = compiled_data->tensors.trainables[i];
93
10
    assert(!CCV_IS_TENSOR_VIEW(tensor));
94
10
    sqlite3_bind_int(tensor_checkpoint_insert_stmt, 1, i);
95
10
    sqlite3_bind_int(tensor_checkpoint_insert_stmt, 2, tensor->info.type);
96
10
    sqlite3_bind_int(tensor_checkpoint_insert_stmt, 3, tensor->info.format);
97
10
    sqlite3_bind_int(tensor_checkpoint_insert_stmt, 4, tensor->info.datatype);
98
10
    sqlite3_bind_blob(tensor_checkpoint_insert_stmt, 5, tensor->info.dim, sizeof(tensor->info.dim), 0);
99
10
    const size_t data_size = ccv_nnc_tensor_data_size(tensor->info);
100
10
#ifdef HAVE_CUDA
101
10
    if (CCV_TENSOR_GET_MEMORY(tensor->info.type) == CCV_TENSOR_GPU_MEMORY)
102
0
    {
103
0
      if (!workspace)
104
0
      {
105
0
        workspace = ccmalloc(data_size);
106
0
        workspace_size = data_size;
107
0
      } else if (data_size > workspace_size) {
108
0
        workspace = ccrealloc(workspace, data_size);
109
0
        workspace_size = data_size;
110
0
      }
111
0
      cumemcpy(workspace, CCV_TENSOR_CPU_MEMORY, tensor->data.u8, tensor->info.type, data_size);
112
0
      sqlite3_bind_blob(tensor_checkpoint_insert_stmt, 6, workspace, data_size, 0);
113
0
    } else
114
10
      sqlite3_bind_blob(tensor_checkpoint_insert_stmt, 6, tensor->data.u8, data_size, 0);
115
#else
116
    sqlite3_bind_blob(tensor_checkpoint_insert_stmt, 6, tensor->data.u8, data_size, 0);
117
#endif
118
    sqlite3_step(tensor_checkpoint_insert_stmt);
119
10
    sqlite3_reset(tensor_checkpoint_insert_stmt);
120
10
    sqlite3_clear_bindings(tensor_checkpoint_insert_stmt);
121
10
  }
122
1
  for (i = 0; i < retainable_size; 
i++0
)
123
0
  {
124
0
    const ccv_nnc_tensor_t* const tensor = compiled_data->tensors.retainables[i];
125
0
    if (!tensor)
126
0
    {
127
0
      // Inject empty one.
128
0
      sqlite3_bind_int(tensor_checkpoint_insert_stmt, 1, i + trainable_size);
129
0
      sqlite3_step(tensor_checkpoint_insert_stmt);
130
0
      sqlite3_reset(tensor_checkpoint_insert_stmt);
131
0
      sqlite3_clear_bindings(tensor_checkpoint_insert_stmt);
132
0
      continue;
133
0
    }
134
0
    assert(!CCV_IS_TENSOR_VIEW(tensor));
135
0
    sqlite3_bind_int(tensor_checkpoint_insert_stmt, 1, i + trainable_size);
136
0
    sqlite3_bind_int(tensor_checkpoint_insert_stmt, 2, tensor->info.type);
137
0
    sqlite3_bind_int(tensor_checkpoint_insert_stmt, 3, tensor->info.format);
138
0
    sqlite3_bind_int(tensor_checkpoint_insert_stmt, 4, tensor->info.datatype);
139
0
    sqlite3_bind_blob(tensor_checkpoint_insert_stmt, 5, tensor->info.dim, sizeof(tensor->info.dim), 0);
140
0
    const size_t data_size = ccv_nnc_tensor_data_size(tensor->info);
141
0
#ifdef HAVE_CUDA
142
0
    if (CCV_TENSOR_GET_MEMORY(tensor->info.type) == CCV_TENSOR_GPU_MEMORY)
143
0
    {
144
0
      if (!workspace)
145
0
      {
146
0
        workspace = ccmalloc(data_size);
147
0
        workspace_size = data_size;
148
0
      } else if (data_size > workspace_size) {
149
0
        workspace = ccrealloc(workspace, data_size);
150
0
        workspace_size = data_size;
151
0
      }
152
0
      cumemcpy(workspace, CCV_TENSOR_CPU_MEMORY, tensor->data.u8, tensor->info.type, data_size);
153
0
      sqlite3_bind_blob(tensor_checkpoint_insert_stmt, 6, workspace, data_size, 0);
154
0
    } else
155
0
      sqlite3_bind_blob(tensor_checkpoint_insert_stmt, 6, tensor->data.u8, data_size, 0);
156
#else
157
    sqlite3_bind_blob(tensor_checkpoint_insert_stmt, 6, tensor->data.u8, data_size, 0);
158
#endif
159
    sqlite3_step(tensor_checkpoint_insert_stmt);
160
0
    sqlite3_reset(tensor_checkpoint_insert_stmt);
161
0
    sqlite3_clear_bindings(tensor_checkpoint_insert_stmt);
162
0
  }
163
1
  sqlite3_finalize(tensor_checkpoint_insert_stmt);
164
1
#ifdef HAVE_CUDA
165
1
  if (workspace)
166
1
    
ccfree0
(workspace)0
;
167
1
#endif
168
1
  SQLITE_ENFORCE(SQLITE_OK == sqlite3_exec(conn, "COMMIT", 0, 0, 0));
169
1
  sqlite3_close(conn);
170
1
}