## Coverage Report

#### Created: 2026-05-04 15:30

next uncovered line (L), next uncovered region (R), next uncovered branch (B)

    /home/liu/actions-runner/_work/ccv/ccv/lib/nnc/ccv_nnc_tensor_io.c

    Line
    Count
    Source
    1
    #include "ccv_nnc.h"
    2
    #include "ccv_nnc_easy.h"
    3
    #include "ccv_nnc_internal.h"
    4
    #include "ccv_internal.h"
    5
    #include "_ccv_nnc_symbolic_graph.h"
    6
    #include "3rdparty/sqlite3/sqlite3.h"
    7
    #include <limits.h>
    8
    #include <stdint.h>
    9
    #ifdef HAVE_CUDA
    10
    #include "gpu/ccv_nnc_compat.h"
    11
    #elif HAVE_MPS
    12
    #include "mps/ccv_nnc_mps.h"
    13
    #endif
    14
    15
    #ifdef NDEBUG
    16
    #define SQLITE_ENFORCE(stmt) (void)(stmt)
    17
    #else
    18
    68
    #define SQLITE_ENFORCE assert
    19
    #endif
    20
    21
    // SQLite INTEGER is 64-bit. Tensor IO already packs side metadata into the
    22
    // high 32 bits of type / datatype. Tensor formats are low 32-bit int enum
    23
    // values (NCHW/NHWC/CHWN), so bit 32 can mark that tensors.data is the head
    24
    // chunk. Always mask this out before assigning to ccv_nnc_tensor_param_t.format.
    25
    42
    #define CCV_NNC_TENSOR_IO_SPLIT_FORMAT (((sqlite_int64)1) << 32)
    26
    14
    #define CCV_NNC_TENSOR_IO_FORMAT_MASK 0xffffffffll
    27
    4
    #define CCV_NNC_TENSOR_IO_SPLIT_HEADROOM 1024
    28
    29
    static int _ccv_nnc_tensor_io_blob_chunk_size(sqlite3* const conn)
    30
    2
    {
    31
    2
      const int limit = sqlite3_limit(conn, SQLITE_LIMIT_LENGTH, -1);
    32
    2
      if (limit <= 0)
    33
    0
        return INT_MAX;
    34
    2
      if (limit > CCV_NNC_TENSOR_IO_SPLIT_HEADROOM * 2)
    35
    2
        return limit - CCV_NNC_TENSOR_IO_SPLIT_HEADROOM;
    36
    0
      return limit > 1 ? limit / 2 : 1;
    37
    2
    }
    38
    39
    static int _ccv_nnc_tensor_io_delete_splits(sqlite3* const conn, const char* const name)
    40
    2
    {
    41
    2
      const char tensor_split_delete_qs[] = "DELETE FROM tensor_splits WHERE name=$name";
    42
    2
      sqlite3_stmt* tensor_split_delete_stmt = 0;
    43
    2
      int rc = sqlite3_prepare_v2(conn, tensor_split_delete_qs, sizeof(tensor_split_delete_qs), &tensor_split_delete_stmt, 0);
    44
    2
      if (rc != SQLITE_OK)
    45
    0
        return rc;
    46
    2
      sqlite3_bind_text(tensor_split_delete_stmt, 1, name, -1, SQLITE_STATIC);
    47
    2
      rc = sqlite3_step(tensor_split_delete_stmt);
    48
    2
      sqlite3_finalize(tensor_split_delete_stmt);
    49
    2
      return rc == SQLITE_DONE ? SQLITE_OK : rc0;
    50
    2
    }
    51
    52
    static int _ccv_nnc_tensor_io_write_splits(sqlite3* const conn, const char* const name, const unsigned char* const data, const size_t data_size, const size_t offset, const int chunk_size)
    53
    2
    {
    54
    2
      const char tensor_split_insert_qs[] =
    55
    2
        "REPLACE INTO tensor_splits "
    56
    2
        "(name, part, data) VALUES ($name, $part, $data)";
    57
    2
      sqlite3_stmt* tensor_split_insert_stmt = 0;
    58
    2
      int rc = sqlite3_prepare_v2(conn, tensor_split_insert_qs, sizeof(tensor_split_insert_qs), &tensor_split_insert_stmt, 0);
    59
    2
      if (rc != SQLITE_OK)
    60
    0
        return rc;
    61
    2
      size_t pos = offset;
    62
    2
      int part = 0;
    63
    6
      while (pos < data_size)
    64
    4
      {
    65
    4
        const size_t tail_size = data_size - pos;
    66
    4
        const int write_size = (int)ccv_min(tail_size, (size_t)chunk_size);
    67
    4
        sqlite3_bind_text(tensor_split_insert_stmt, 1, name, -1, SQLITE_STATIC);
    68
    4
        sqlite3_bind_int(tensor_split_insert_stmt, 2, part++);
    69
    4
        rc = sqlite3_bind_blob(tensor_split_insert_stmt, 3, data + pos, write_size, SQLITE_STATIC);
    70
    4
        if (rc != SQLITE_OK)
    71
    0
          break;
    72
    4
        rc = sqlite3_step(tensor_split_insert_stmt);
    73
    4
        if (rc != SQLITE_DONE)
    74
    0
          break;
    75
    4
        sqlite3_reset(tensor_split_insert_stmt);
    76
    4
        sqlite3_clear_bindings(tensor_split_insert_stmt);
    77
    4
        pos += write_size;
    78
    4
      }
    79
    2
      sqlite3_finalize(tensor_split_insert_stmt);
    80
    2
      return rc == SQLITE_DONE ? SQLITE_OK : rc0;
    81
    2
    }
    82
    83
    static int _ccv_nnc_tensor_io_read_split_data(sqlite3* const conn, const char* const name, const void* const first_data, const size_t first_size, const void** const data_out, size_t* const data_size_out, unsigned char** const workspace_out)
    84
    2
    {
    85
    2
      const char tensor_split_size_qs[] = "SELECT COUNT(*), SUM(length(data)) FROM tensor_splits WHERE name=$name";
    86
    2
      sqlite3_stmt* tensor_split_size_stmt = 0;
    87
    2
      int rc = sqlite3_prepare_v2(conn, tensor_split_size_qs, sizeof(tensor_split_size_qs), &tensor_split_size_stmt, 0);
    88
    2
      if (rc != SQLITE_OK)
    89
    0
        return rc;
    90
    2
      sqlite3_bind_text(tensor_split_size_stmt, 1, name, -1, SQLITE_STATIC);
    91
    2
      rc = sqlite3_step(tensor_split_size_stmt);
    92
    2
      if (rc != SQLITE_ROW)
    93
    0
      {
    94
    0
        sqlite3_finalize(tensor_split_size_stmt);
    95
    0
        return rc;
    96
    0
      }
    97
    2
      const sqlite_int64 split_count = sqlite3_column_int64(tensor_split_size_stmt, 0);
    98
    2
      const sqlite_int64 tail_size = sqlite3_column_int64(tensor_split_size_stmt, 1);
    99
    2
      sqlite3_finalize(tensor_split_size_stmt);
    100
    2
      if (split_count <= 0 || tail_size < 0 || (size_t)tail_size > SIZE_MAX - first_size)
    101
    0
        return SQLITE_CORRUPT;
    102
    2
      const size_t total_size = first_size + (size_t)tail_size;
    103
    2
      unsigned char* const workspace = (unsigned char*)ccmalloc(total_size);
    104
    2
      if (first_size > 0)
    105
    2
        memcpy(workspace, first_data, first_size);
    106
    2
      const char tensor_split_select_qs[] = "SELECT part, data FROM tensor_splits WHERE name=$name ORDER BY part";
    107
    2
      sqlite3_stmt* tensor_split_select_stmt = 0;
    108
    2
      rc = sqlite3_prepare_v2(conn, tensor_split_select_qs, sizeof(tensor_split_select_qs), &tensor_split_select_stmt, 0);
    109
    2
      if (rc != SQLITE_OK)
    110
    0
      {
    111
    0
        ccfree(workspace);
    112
    0
        return rc;
    113
    0
      }
    114
    2
      sqlite3_bind_text(tensor_split_select_stmt, 1, name, -1, SQLITE_STATIC);
    115
    2
      size_t offset = first_size;
    116
    2
      int expected_part = 0;
    117
    6
      while ((rc = sqlite3_step(tensor_split_select_stmt)) == SQLITE_ROW)
    118
    4
      {
    119
    4
        if (sqlite3_column_int(tensor_split_select_stmt, 0) != expected_part++)
    120
    0
        {
    121
    0
          sqlite3_finalize(tensor_split_select_stmt);
    122
    0
          ccfree(workspace);
    123
    0
          return SQLITE_CORRUPT;
    124
    0
        }
    125
    4
        const int split_size = sqlite3_column_bytes(tensor_split_select_stmt, 1);
    126
    4
        if (split_size < 0 || (size_t)split_size > total_size - offset)
    127
    0
        {
    128
    0
          sqlite3_finalize(tensor_split_select_stmt);
    129
    0
          ccfree(workspace);
    130
    0
          return SQLITE_CORRUPT;
    131
    0
        }
    132
    4
        const void* const split_data = sqlite3_column_blob(tensor_split_select_stmt, 1);
    133
    4
        if (split_size > 0)
    134
    4
          memcpy(workspace + offset, split_data, split_size);
    135
    4
        offset += split_size;
    136
    4
      }
    137
    2
      sqlite3_finalize(tensor_split_select_stmt);
    138
    2
      if (rc != SQLITE_DONE || offset != total_size)
    139
    0
      {
    140
    0
        ccfree(workspace);
    141
    0
        return rc == SQLITE_DONE ? SQLITE_CORRUPT : rc;
    142
    0
      }
    143
    2
      *data_out = workspace;
    144
    2
      *data_size_out = total_size;
    145
    2
      *workspace_out = workspace;
    146
    2
      return SQLITE_OK;
    147
    2
    }
    148
    149
    // MARK - Level-1 API
    150
    151
    int ccv_nnc_tensor_write(const ccv_nnc_tensor_t* const tensor, void* const handle, const char* const name, const ccv_nnc_tensor_io_option_t* const options)
    152
    34
    {
    153
    34
      assert(CCV_IS_TENSOR_CONTIGUOUS(tensor));
    154
    34
      assert(name);
    155
    34
      sqlite3* conn = (sqlite3*)handle;
    156
    34
      if (!conn)
    157
    0
        return CCV_IO_ERROR;
    158
    34
      const char tensor_create_table_qs[] = "CREATE TABLE IF NOT EXISTS tensors "
    159
    34
        "(name TEXT, type INTEGER, format INTEGER, datatype INTEGER, "
    160
    34
        "dim BLOB, data BLOB, PRIMARY KEY (name))";
    161
    34
      SQLITE_ENFORCE(SQLITE_OK == sqlite3_exec(conn, tensor_create_table_qs, 0, 0, 0));
    162
    34
      const char tensor_insert_qs[] =
    163
    34
        "REPLACE INTO tensors "
    164
    34
        "(name, type, format, datatype, dim, data) VALUES ("
    165
    34
        "$name, $type, $format, $datatype, $dim, $data)";
    166
    34
      sqlite3_stmt* tensor_insert_stmt = 0;
    167
    34
      SQLITE_ENFORCE(SQLITE_OK == sqlite3_prepare_v2(conn, tensor_insert_qs, sizeof(tensor_insert_qs), &tensor_insert_stmt, 0));
    168
    34
      sqlite3_bind_text(tensor_insert_stmt, 1, name, -1, 0);
    169
    34
      ccv_nnc_tensor_param_t params = tensor->info;
    170
    34
      const size_t data_size = ccv_nnc_tensor_data_size_without_padding(tensor->info);
    171
    34
      unsigned char* workspace = 0;
    172
    34
      const void* payload = tensor->data.u8;
    173
    34
      size_t payload_size = data_size;
    174
    34
      unsigned int identifier = 0;
    175
    34
    #ifdef HAVE_CUDA
    176
    34
      if (CCV_TENSOR_GET_MEMORY(tensor->info.type) == CCV_TENSOR_GPU_MEMORY)
    177
    9
      {
    178
    9
        if (!options || !options->encode6)
    179
    3
        {
    180
    3
          workspace = ccmalloc(data_size);
    181
    3
          cumemcpy(workspace, CCV_TENSOR_CPU_MEMORY, tensor->data.u8, tensor->info.type, data_size);
    182
    3
          payload = workspace;
    183
    6
        } else {
    184
    6
          workspace = ccmalloc(data_size * 2 + 4);
    185
    6
          cumemcpy(workspace, CCV_TENSOR_CPU_MEMORY, tensor->data.u8, tensor->info.type, data_size);
    186
    6
          size_t encoded_size = data_size + 4;
    187
    6
          if (options->encode(workspace, data_size, tensor->info.datatype, tensor->info.dim, ccv_nnc_tensor_nd(tensor->info.dim), options->context, workspace + data_size, &encoded_size, &params, &identifier))
    188
    3
          {
    189
    3
            payload = workspace + data_size;
    190
    3
            payload_size = encoded_size;
    191
    3
          }
    192
    3
          else
    193
    3
            payload = workspace;
    194
    6
        }
    195
    25
      } else {
    196
    25
        if (!options || !options->encode7)
    197
    18
          payload = tensor->data.u8;
    198
    7
        else {
    199
    7
          workspace = ccmalloc(data_size + 4);
    200
    7
          size_t encoded_size = data_size + 4;
    201
    7
          if (options->encode(tensor->data.u8, data_size, tensor->info.datatype, tensor->info.dim, ccv_nnc_tensor_nd(tensor->info.dim), options->context, workspace, &encoded_size, &params, &identifier))
    202
    4
          {
    203
    4
            payload = workspace;
    204
    4
            payload_size = encoded_size;
    205
    4
          }
    206
    3
          else
    207
    3
            payload = tensor->data.u8;
    208
    7
        }
    209
    25
      }
    210
    #elif defined(HAVE_MPS)
    211
      if (CCV_TENSOR_GET_MEMORY(tensor->info.type) == CCV_TENSOR_GPU_MEMORY)
    212
      {
    213
        if (!options || !options->encode)
    214
        {
    215
          workspace = ccmalloc(data_size);
    216
          mpmemcpy(workspace, 0, CCV_TENSOR_CPU_MEMORY, tensor->data.u8, tensor->dataof, tensor->info.type, data_size);
    217
          payload = workspace;
    218
        } else {
    219
          workspace = ccmalloc(data_size * 2 + 4);
    220
          mpmemcpy(workspace, 0, CCV_TENSOR_CPU_MEMORY, tensor->data.u8, tensor->dataof, tensor->info.type, data_size);
    221
          size_t encoded_size = data_size + 4;
    222
          if (options->encode(workspace, data_size, tensor->info.datatype, tensor->info.dim, ccv_nnc_tensor_nd(tensor->info.dim), options->context, workspace + data_size, &encoded_size, &params, &identifier))
    223
          {
    224
            payload = workspace + data_size;
    225
            payload_size = encoded_size;
    226
          }
    227
          else
    228
            payload = workspace;
    229
        }
    230
      } else {
    231
        if (!options || !options->encode)
    232
          payload = tensor->data.u8;
    233
        else {
    234
          workspace = ccmalloc(data_size + 4); // Allocate extra 4 bytes in case we need to copy the QX tensor out.
    235
          size_t encoded_size = data_size + 4;
    236
          if (options->encode(tensor->data.u8, data_size, tensor->info.datatype, tensor->info.dim, ccv_nnc_tensor_nd(tensor->info.dim), options->context, workspace, &encoded_size, &params, &identifier))
    237
          {
    238
            payload = workspace;
    239
            payload_size = encoded_size;
    240
          }
    241
          else
    242
            payload = tensor->data.u8;
    243
        }
    244
      }
    245
    #else
    246
      if (!options || !options->encode)
    247
        payload = tensor->data.u8;
    248
      else {
    249
        workspace = ccmalloc(data_size + 4);
    250
        size_t encoded_size = data_size + 4;
    251
        if (options->encode(tensor->data.u8, data_size, tensor->info.datatype, tensor->info.dim, ccv_nnc_tensor_nd(tensor->info.dim), options->context, workspace, &encoded_size, &params, &identifier))
    252
        {
    253
          payload = workspace;
    254
          payload_size = encoded_size;
    255
        }
    256
        else
    257
          payload = tensor->data.u8;
    258
      }
    259
    #endif
    260
    34
      int result = SQLITE_TOOBIG;
    261
    34
      int bind_result = payload_size <= INT_MAX ? sqlite3_bind_blob(tensor_insert_stmt, 6, payload, (int)payload_size, 0) : SQLITE_TOOBIG0;
    262
    34
      if (bind_result == SQLITE_OK)
    263
    32
      {
    264
    32
        sqlite3_bind_int64(tensor_insert_stmt, 2, ((sqlite_int64)identifier << 32) | params.type);
    265
    32
        sqlite3_bind_int(tensor_insert_stmt, 3, params.format);
    266
    32
        sqlite3_bind_int64(tensor_insert_stmt, 4, ((sqlite_int64)params.reserved << 32) | params.datatype);
    267
    32
        sqlite3_bind_blob(tensor_insert_stmt, 5, params.dim, sizeof(params.dim), 0);
    268
    32
        result = sqlite3_step(tensor_insert_stmt);
    269
    32
        if (result == SQLITE_DONE)
    270
    32
        {
    271
    32
          sqlite3_reset(tensor_insert_stmt);
    272
    32
          sqlite3_clear_bindings(tensor_insert_stmt);
    273
    32
          sqlite3_finalize(tensor_insert_stmt);
    274
    32
          if (workspace)
    275
    15
            free(workspace);
    276
    32
          return CCV_IO_FINAL;
    277
    32
        }
    278
    0
        sqlite3_reset(tensor_insert_stmt);
    279
    0
        sqlite3_clear_bindings(tensor_insert_stmt);
    280
    0
        if (result != SQLITE_TOOBIG)
    281
    0
        {
    282
    0
          sqlite3_finalize(tensor_insert_stmt);
    283
    0
          if (workspace)
    284
    0
            free(workspace);
    285
    0
          return CCV_IO_ERROR;
    286
    0
        }
    287
    2
      } else {
    288
    2
        sqlite3_reset(tensor_insert_stmt);
    289
    2
        sqlite3_clear_bindings(tensor_insert_stmt);
    290
    2
        if (bind_result != SQLITE_TOOBIG)
    291
    0
        {
    292
    0
          sqlite3_finalize(tensor_insert_stmt);
    293
    0
          if (workspace)
    294
    0
            free(workspace);
    295
    0
          return CCV_IO_ERROR;
    296
    0
        }
    297
    2
      }
    298
    2
      if (sqlite3_exec(conn, "SAVEPOINT ccv_nnc_tensor_write", 0, 0, 0) != SQLITE_OK)
    299
    0
      {
    300
    0
        sqlite3_finalize(tensor_insert_stmt);
    301
    0
        if (workspace)
    302
    0
          free(workspace);
    303
    0
        return CCV_IO_ERROR;
    304
    0
      }
    305
    2
      const char tensor_split_create_table_qs[] = "CREATE TABLE IF NOT EXISTS tensor_splits "
    306
    2
        "(name TEXT, part INTEGER, data BLOB, PRIMARY KEY (name, part))";
    307
    2
      if (sqlite3_exec(conn, tensor_split_create_table_qs, 0, 0, 0) != SQLITE_OK)
    308
    0
      {
    309
    0
        sqlite3_finalize(tensor_insert_stmt);
    310
    0
        sqlite3_exec(conn, "ROLLBACK TO ccv_nnc_tensor_write", 0, 0, 0);
    311
    0
        sqlite3_exec(conn, "RELEASE ccv_nnc_tensor_write", 0, 0, 0);
    312
    0
        if (workspace)
    313
    0
          free(workspace);
    314
    0
        return CCV_IO_ERROR;
    315
    0
      }
    316
    2
      sqlite3_bind_text(tensor_insert_stmt, 1, name, -1, 0);
    317
    2
      const int chunk_size = _ccv_nnc_tensor_io_blob_chunk_size(conn);
    318
    2
      assert(chunk_size > 0);
    319
    2
      const size_t first_size = ccv_min(payload_size, (size_t)chunk_size);
    320
    2
      bind_result = first_size < payload_size ? sqlite3_bind_blob(tensor_insert_stmt, 6, payload, (int)first_size, SQLITE_STATIC) : SQLITE_TOOBIG0;
    321
    2
      if (bind_result == SQLITE_OK)
    322
    2
        bind_result = _ccv_nnc_tensor_io_delete_splits(conn, name);
    323
    2
      if (bind_result == SQLITE_OK)
    324
    2
        bind_result = _ccv_nnc_tensor_io_write_splits(conn, name, (const unsigned char*)payload, payload_size, first_size, chunk_size);
    325
    2
      if (bind_result != SQLITE_OK)
    326
    0
      {
    327
    0
        sqlite3_finalize(tensor_insert_stmt);
    328
    0
        sqlite3_exec(conn, "ROLLBACK TO ccv_nnc_tensor_write", 0, 0, 0);
    329
    0
        sqlite3_exec(conn, "RELEASE ccv_nnc_tensor_write", 0, 0, 0);
    330
    0
        if (workspace)
    331
    0
          free(workspace);
    332
    0
        return CCV_IO_ERROR;
    333
    0
      }
    334
    2
      sqlite3_bind_int64(tensor_insert_stmt, 2, ((sqlite_int64)identifier << 32) | params.type);
    335
    2
      sqlite3_bind_int64(tensor_insert_stmt, 3, ((sqlite_int64)params.format & CCV_NNC_TENSOR_IO_FORMAT_MASK) | CCV_NNC_TENSOR_IO_SPLIT_FORMAT);
    336
    2
      sqlite3_bind_int64(tensor_insert_stmt, 4, ((sqlite_int64)params.reserved << 32) | params.datatype);
    337
    2
      sqlite3_bind_blob(tensor_insert_stmt, 5, params.dim, sizeof(params.dim), 0);
    338
    2
      result = sqlite3_step(tensor_insert_stmt);
    339
    2
      sqlite3_reset(tensor_insert_stmt);
    340
    2
      sqlite3_clear_bindings(tensor_insert_stmt);
    341
    2
      sqlite3_finalize(tensor_insert_stmt);
    342
    2
      if (result == SQLITE_DONE)
    343
    2
      {
    344
    2
        if (sqlite3_exec(conn, "RELEASE ccv_nnc_tensor_write", 0, 0, 0) != SQLITE_OK)
    345
    0
        {
    346
    0
          if (workspace)
    347
    0
            free(workspace);
    348
    0
          return CCV_IO_ERROR;
    349
    0
        }
    350
    2
      } else {
    351
    0
        sqlite3_exec(conn, "ROLLBACK TO ccv_nnc_tensor_write", 0, 0, 0);
    352
    0
        sqlite3_exec(conn, "RELEASE ccv_nnc_tensor_write", 0, 0, 0);
    353
    0
      }
    354
    2
      if (workspace)
    355
    1
        free(workspace);
    356
    2
      return result == SQLITE_DONE ? CCV_IO_FINAL : CCV_IO_ERROR0;
    357
    2
    }
    358
    359
    int ccv_nnc_tensor_read(void* const handle, const char* const name, const ccv_nnc_tensor_io_option_t* const options, const int flags, const ccv_nnc_tensor_param_t* const tensor_params_optional, ccv_nnc_tensor_t** const tensor_out)
    360
    40
    {
    361
    40
      assert(name);
    362
    40
      sqlite3* conn = (sqlite3*)handle;
    363
    40
      if (!conn)
    364
    0
        return CCV_IO_ERROR;
    365
    40
      const char tensor_select_qs[] =
    366
    40
        "SELECT data, type, format, datatype, dim FROM tensors WHERE name=$name";
    367
    40
      sqlite3_stmt* tensor_select_stmt = 0;
    368
    40
      if (SQLITE_OK != sqlite3_prepare_v2(conn, tensor_select_qs, sizeof(tensor_select_qs), &tensor_select_stmt, 0))
    369
    0
        return CCV_IO_ERROR;
    370
    40
      sqlite3_bind_text(tensor_select_stmt, 1, name, -1, 0);
    371
    40
      if (SQLITE_ROW != sqlite3_step(tensor_select_stmt))
    372
    0
      {
    373
    0
        sqlite3_finalize(tensor_select_stmt);
    374
    0
        return CCV_IO_ERROR;
    375
    0
      }
    376
    40
      ccv_nnc_tensor_t* tensor = *tensor_out;
    377
    40
      ccv_nnc_tensor_param_t tensor_params;
    378
    40
      int datatype = 0;
    379
    40
      unsigned int identifier = 0;
    380
    40
      const sqlite_int64 format_mix = sqlite3_column_int64(tensor_select_stmt, 2);
    381
    40
      const int has_splits = !!(format_mix & CCV_NNC_TENSOR_IO_SPLIT_FORMAT);
    382
    40
      if (!tensor) // If the tensor is not provided, we need to create one.
    383
    22
      {
    384
    22
        if (tensor_params_optional)
    385
    10
        {
    386
    10
          identifier = (sqlite3_column_int64(tensor_select_stmt, 1) >> 32) & 0xffffffff;
    387
    10
          datatype = sqlite3_column_int64(tensor_select_stmt, 3) & 0xffffffff;
    388
    10
          tensor_params = *tensor_params_optional;
    389
    10
          assert(!(flags & CCV_NNC_TENSOR_READ_METADATA_ONLY));
    390
    12
        } else {
    391
    12
          const sqlite_int64 type = sqlite3_column_int64(tensor_select_stmt, 1);
    392
    12
          identifier = (type >> 32) & 0xffffffff;
    393
    12
          tensor_params.type = (type & 0xffffffff);
    394
    12
          tensor_params.format = (int)(format_mix & CCV_NNC_TENSOR_IO_FORMAT_MASK);
    395
    12
          const sqlite_int64 datatype_mix = sqlite3_column_int64(tensor_select_stmt, 3);
    396
    12
          datatype = tensor_params.datatype = (datatype_mix & 0xffffffff);
    397
    12
          tensor_params.reserved = (datatype_mix >> 32) & 0xffffffff;
    398
    12
          const void* const dim = sqlite3_column_blob(tensor_select_stmt, 4);
    399
    12
          memcpy(tensor_params.dim, dim, ccv_min(sizeof(tensor_params.dim), sqlite3_column_bytes(tensor_select_stmt, 4)));
    400
    12
        }
    401
    22
        if (flags & CCV_NNC_TENSOR_READ_CPU_MEMORY) // Reset type to CPU memory.
    402
    0
          tensor_params.type = (tensor_params.type & 0xfff00000) | CCV_TENSOR_CPU_MEMORY;
    403
    22
        if (!options || !options->decode5)
    404
    17
        {
    405
    17
          if (flags & CCV_NNC_TENSOR_READ_METADATA_ONLY)
    406
    2
          {
    407
    2
            *tensor_out = tensor = ccv_nnc_tensor_new(0, tensor_params, CCV_NO_DATA_ALLOC); // Set the data point to 1 so it is allocated without data.
    408
    2
            assert(tensor->data.u8 == 0); // Set it back to 0.
    409
            // Already done loading metadata, return.
    410
    2
            sqlite3_reset(tensor_select_stmt);
    411
    2
            sqlite3_clear_bindings(tensor_select_stmt);
    412
    2
            sqlite3_finalize(tensor_select_stmt);
    413
    2
            return CCV_IO_FINAL;
    414
    2
          } else
    415
    15
            *tensor_out = tensor = ccv_nnc_tensor_new(0, tensor_params, 0);
    416
    17
        } else {
    417
    5
          assert(!(flags & CCV_NNC_TENSOR_READ_METADATA_ONLY));
    418
    5
        }
    419
    22
      } else {
    420
    18
        identifier = (sqlite3_column_int64(tensor_select_stmt, 1) >> 32) & 0xffffffff;
    421
    18
        datatype = sqlite3_column_int64(tensor_select_stmt, 3) & 0xffffffff;
    422
    18
        tensor_params = tensor->info;
    423
    18
        assert(!(flags & CCV_NNC_TENSOR_READ_METADATA_ONLY));
    424
    18
      }
    425
    38
      const void* data = sqlite3_column_blob(tensor_select_stmt, 0);
    426
    38
      size_t data_bytes = sqlite3_column_bytes(tensor_select_stmt, 0);
    427
    38
      unsigned char* split_workspace = 0;
    428
    38
      if (has_splits)
    429
    2
      {
    430
    2
        const int split_result = _ccv_nnc_tensor_io_read_split_data(conn, name, data, data_bytes, &data, &data_bytes, &split_workspace);
    431
    2
        if (split_result != SQLITE_OK)
    432
    0
        {
    433
    0
          sqlite3_reset(tensor_select_stmt);
    434
    0
          sqlite3_clear_bindings(tensor_select_stmt);
    435
    0
          sqlite3_finalize(tensor_select_stmt);
    436
    0
          return CCV_IO_ERROR;
    437
    0
        }
    438
    2
      }
    439
    38
      int dim[CCV_NNC_MAX_DIM_ALLOC];
    440
    38
      memcpy(dim, sqlite3_column_blob(tensor_select_stmt, 4), ccv_min(sizeof(dim), sqlite3_column_bytes(tensor_select_stmt, 4)));
    441
    38
      const int nd = ccv_nnc_tensor_nd(dim);
    442
    38
      if (datatype != tensor_params.datatype && CCV_GET_DATA_TYPE12(tensor_params.datatype) != CCV_QX12)
    443
    12
      {
    444
        // Only ever works for 16F to 32F or 32F to 16F transparently.
    445
    12
        assert((datatype == CCV_16F && tensor_params.datatype == CCV_32F) || (datatype == CCV_32F && tensor_params.datatype == CCV_16F));
    446
    12
        const size_t tensor_count = ccv_nnc_tensor_count(tensor_params);
    447
    12
        ccv_nnc_tensor_param_t params = tensor_params;
    448
    12
        params.datatype = datatype;
    449
    12
        const size_t source_data_size = ccv_nnc_tensor_data_size(params);
    450
    12
    #ifdef HAVE_CUDA
    451
    12
        if (CCV_TENSOR_GET_MEMORY(tensor_params.type) == CCV_TENSOR_GPU_MEMORY)
    452
    6
        {
    453
    6
          const size_t data_size = ccv_nnc_tensor_data_size(tensor_params);
    454
    6
          unsigned char* workspace;
    455
    6
          unsigned char* copying;
    456
    6
          size_t decoded_size = data_size;
    457
    6
          if (!options || !options->decode4)
    458
    2
          {
    459
    2
            copying = workspace = ccmalloc(data_size);
    460
    2
            if (datatype == CCV_16F && tensor_params.datatype == CCV_32F1)
    461
    1
              ccv_half_precision_to_float((uint16_t*)data, (float*)workspace, ccv_min(tensor_count, data_bytes / sizeof(uint16_t)));
    462
    1
            else if (datatype == CCV_32F && tensor_params.datatype == CCV_16F)
    463
    1
              ccv_float_to_half_precision((float*)data, (uint16_t*)workspace, ccv_min(tensor_count, data_bytes / sizeof(float)));
    464
    0
            else
    465
    0
              { assert(0); }
    466
    4
          } else {
    467
    4
            copying = workspace = ccmalloc(data_size + source_data_size);
    468
    4
            if (datatype == CCV_16F && tensor_params.datatype == CCV_32F2)
    469
    2
            {
    470
    2
              decoded_size = source_data_size;
    471
    2
              if (options->decode(data, data_bytes, datatype, dim, nd, identifier, options->context, tensor_params, tensor_out, workspace + data_size, &decoded_size))
    472
    1
              {
    473
                // If we loaded quantized tensor, don't do the conversion.
    474
    1
                if (CCV_GET_DATA_TYPE(tensor_out[0]->info.datatype) == CCV_QX)
    475
    0
                  copying = workspace + data_size;
    476
    1
                else {
    477
    1
                  ccv_half_precision_to_float((uint16_t*)(workspace + data_size), (float*)workspace, ccv_min(tensor_count, ccv_min(source_data_size, decoded_size) / sizeof(uint16_t)));
    478
    1
                  decoded_size = data_size;
    479
    1
                }
    480
    1
              } else {
    481
    1
                if (!tensor)
    482
    0
                  *tensor_out = tensor = ccv_nnc_tensor_new(0, tensor_params, 0);
    483
    1
                ccv_half_precision_to_float((uint16_t*)data, (float*)workspace, ccv_min(tensor_count, data_bytes / sizeof(uint16_t)));
    484
    1
                decoded_size = data_size;
    485
    1
              }
    486
    2
            } else if (datatype == CCV_32F && tensor_params.datatype == CCV_16F) {
    487
    2
              decoded_size = source_data_size;
    488
    2
              if (options->decode(data, data_bytes, datatype, dim, nd, identifier, options->context, tensor_params, tensor_out, workspace + data_size, &decoded_size))
    489
    1
              {
    490
    1
                if (CCV_GET_DATA_TYPE(tensor_out[0]->info.datatype) == CCV_QX)
    491
    0
                  copying = workspace + data_size;
    492
    1
                else {
    493
    1
                  ccv_float_to_half_precision((float*)(workspace + data_size), (uint16_t*)workspace, ccv_min(tensor_count, ccv_min(source_data_size, decoded_size) / sizeof(float)));
    494
    1
                  decoded_size = data_size;
    495
    1
                }
    496
    1
              } else {
    497
    1
                if (!tensor)
    498
    0
                  *tensor_out = tensor = ccv_nnc_tensor_new(0, tensor_params, 0);
    499
    1
                ccv_float_to_half_precision((float*)data, (uint16_t*)workspace, ccv_min(tensor_count, data_bytes / sizeof(float)));
    500
    1
                decoded_size = data_size;
    501
    1
              }
    502
    2
            } else
    503
    0
              { assert(0); }
    504
    4
          }
    505
    6
          cumemcpy(tensor_out[0]->data.u8, tensor_out[0]->info.type, copying, CCV_TENSOR_CPU_MEMORY, decoded_size);
    506
    6
          ccfree(workspace);
    507
    6
        } else {
    508
    6
          if (!options || !options->decode4)
    509
    2
          {
    510
    2
            if (datatype == CCV_16F && tensor_params.datatype == CCV_32F1)
    511
    1
              ccv_half_precision_to_float((uint16_t*)data, tensor->data.f32, ccv_min(tensor_count, data_bytes / sizeof(uint16_t)));
    512
    1
            else if (datatype == CCV_32F && tensor_params.datatype == CCV_16F)
    513
    1
              ccv_float_to_half_precision((float*)data, (uint16_t*)tensor->data.f16, ccv_min(tensor_count, data_bytes / sizeof(float)));
    514
    0
            else
    515
    0
              { assert(0); }
    516
    4
          } else {
    517
    4
            void* const workspace = ccmalloc(source_data_size);
    518
    4
            if (datatype == CCV_16F && tensor_params.datatype == CCV_32F2)
    519
    2
            {
    520
    2
              size_t decoded_size = source_data_size;
    521
    2
              if (options->decode(data, data_bytes, datatype, dim, nd, identifier, options->context, tensor_params, tensor_out, workspace, &decoded_size))
    522
    1
              {
    523
    1
                if (CCV_GET_DATA_TYPE(tensor_out[0]->info.datatype) == CCV_QX)
    524
    0
                {
    525
    0
                  if (decoded_size > 0)
    526
    0
                    memcpy(tensor_out[0]->data.f32, workspace, ccv_min(source_data_size, decoded_size));
    527
    0
                } else
    528
    1
                  ccv_half_precision_to_float((uint16_t*)workspace, tensor_out[0]->data.f32, ccv_min(tensor_count, ccv_min(source_data_size, decoded_size) / sizeof(uint16_t)));
    529
    1
              } else {
    530
    1
                if (!tensor)
    531
    0
                  *tensor_out = tensor = ccv_nnc_tensor_new(0, tensor_params, 0);
    532
    1
                ccv_half_precision_to_float((uint16_t*)data, tensor->data.f32, ccv_min(tensor_count, data_bytes / sizeof(uint16_t)));
    533
    1
              }
    534
    2
            } else if (datatype == CCV_32F && tensor_params.datatype == CCV_16F) {
    535
    2
              size_t decoded_size = source_data_size;
    536
    2
              if (options->decode(data, data_bytes, datatype, dim, nd, identifier, options->context, tensor_params, tensor_out, workspace, &decoded_size))
    537
    1
              {
    538
    1
                if (CCV_GET_DATA_TYPE(tensor_out[0]->info.datatype) == CCV_QX)
    539
    0
                {
    540
    0
                  if (decoded_size > 0)
    541
    0
                    memcpy(tensor_out[0]->data.f16, workspace, ccv_min(source_data_size, decoded_size));
    542
    0
                } else
    543
    1
                  ccv_float_to_half_precision((float*)workspace, (uint16_t*)tensor_out[0]->data.f16, ccv_min(tensor_count, ccv_min(source_data_size, decoded_size) / sizeof(float)));
    544
    1
              } else {
    545
    1
                if (!tensor)
    546
    0
                  *tensor_out = tensor = ccv_nnc_tensor_new(0, tensor_params, 0);
    547
    1
                ccv_float_to_half_precision((float*)data, (uint16_t*)tensor->data.f16, ccv_min(tensor_count, data_bytes / sizeof(float)));
    548
    1
              }
    549
    2
            } else
    550
    0
              { assert(0); }
    551
    4
            ccfree(workspace);
    552
    4
          }
    553
    6
        }
    554
    #elif defined(HAVE_MPS)
    555
        if (CCV_TENSOR_GET_MEMORY(tensor_params.type) == CCV_TENSOR_GPU_MEMORY)
    556
        {
    557
          const size_t data_size = ccv_nnc_tensor_data_size(tensor_params);
    558
          unsigned char* workspace;
    559
          unsigned char* copying;
    560
          size_t decoded_size = data_size;
    561
          if (!options || !options->decode)
    562
          {
    563
            copying = workspace = ccmalloc(data_size);
    564
            if (datatype == CCV_16F && tensor_params.datatype == CCV_32F)
    565
              ccv_half_precision_to_float((uint16_t*)data, (float*)workspace, ccv_min(tensor_count, data_bytes / sizeof(uint16_t)));
    566
            else if (datatype == CCV_32F && tensor_params.datatype == CCV_16F)
    567
              ccv_float_to_half_precision((float*)data, (uint16_t*)workspace, ccv_min(tensor_count, data_bytes / sizeof(float)));
    568
            else
    569
              { assert(0); }
    570
          } else {
    571
            copying = workspace = ccmalloc(data_size + source_data_size);
    572
            if (datatype == CCV_16F && tensor_params.datatype == CCV_32F)
    573
            {
    574
              decoded_size = source_data_size;
    575
              if (options->decode(data, data_bytes, datatype, dim, nd, identifier, options->context, tensor_params, tensor_out, workspace + data_size, &decoded_size))
    576
              {
    577
                if (CCV_GET_DATA_TYPE(tensor_out[0]->info.datatype) == CCV_QX)
    578
                  copying = workspace + data_size;
    579
                else {
    580
                  ccv_half_precision_to_float((uint16_t*)(workspace + data_size), (float*)workspace, ccv_min(tensor_count, ccv_min(source_data_size, decoded_size) / sizeof(uint16_t)));
    581
                  decoded_size = data_size;
    582
                }
    583
              } else {
    584
                if (!tensor)
    585
                  *tensor_out = tensor = ccv_nnc_tensor_new(0, tensor_params, 0);
    586
                ccv_half_precision_to_float((uint16_t*)data, (float*)workspace, ccv_min(tensor_count, data_bytes / sizeof(uint16_t)));
    587
                decoded_size = data_size;
    588
              }
    589
            } else if (datatype == CCV_32F && tensor_params.datatype == CCV_16F) {
    590
              decoded_size = source_data_size;
    591
              if (options->decode(data, data_bytes, datatype, dim, nd, identifier, options->context, tensor_params, tensor_out, workspace + data_size, &decoded_size))
    592
              {
    593
                if (CCV_GET_DATA_TYPE(tensor_out[0]->info.datatype) == CCV_QX)
    594
                  copying = workspace + data_size;
    595
                else {
    596
                  ccv_float_to_half_precision((float*)(workspace + data_size), (uint16_t*)workspace, ccv_min(tensor_count, ccv_min(source_data_size, decoded_size) / sizeof(float)));
    597
                  decoded_size = data_size;
    598
                }
    599
              } else {
    600
                if (!tensor)
    601
                  *tensor_out = tensor = ccv_nnc_tensor_new(0, tensor_params, 0);
    602
                ccv_float_to_half_precision((float*)data, (uint16_t*)workspace, ccv_min(tensor_count, data_bytes / sizeof(float)));
    603
                decoded_size = data_size;
    604
              }
    605
            } else
    606
              { assert(0); }
    607
          }
    608
          assert(tensor_out[0]->dataof == 0);
    609
          mpmemcpy(tensor_out[0]->data.u8, tensor_out[0]->dataof, tensor_out[0]->info.type, copying, 0, CCV_TENSOR_CPU_MEMORY, decoded_size);
    610
          ccfree(workspace);
    611
        } else {
    612
          if (!options || !options->decode)
    613
          {
    614
            if (datatype == CCV_16F && tensor_params.datatype == CCV_32F)
    615
              ccv_half_precision_to_float((uint16_t*)data, tensor->data.f32, ccv_min(tensor_count, data_bytes / sizeof(uint16_t)));
    616
            else if (datatype == CCV_32F && tensor_params.datatype == CCV_16F)
    617
              ccv_float_to_half_precision((float*)data, (uint16_t*)tensor->data.f16, ccv_min(tensor_count, data_bytes / sizeof(float)));
    618
            else
    619
              { assert(0); }
    620
          } else {
    621
            void* const workspace = ccmalloc(source_data_size);
    622
            if (datatype == CCV_16F && tensor_params.datatype == CCV_32F)
    623
            {
    624
              size_t decoded_size = source_data_size;
    625
              if (options->decode(data, data_bytes, datatype, dim, nd, identifier, options->context, tensor_params, tensor_out, workspace, &decoded_size))
    626
              {
    627
                if (CCV_GET_DATA_TYPE(tensor_out[0]->info.datatype) == CCV_QX)
    628
                {
    629
                  if (decoded_size > 0)
    630
                    memcpy(tensor_out[0]->data.f32, workspace, ccv_min(source_data_size, decoded_size));
    631
                } else
    632
                  ccv_half_precision_to_float((uint16_t*)workspace, tensor_out[0]->data.f32, ccv_min(tensor_count, ccv_min(source_data_size, decoded_size) / sizeof(uint16_t)));
    633
              } else {
    634
                if (!tensor)
    635
                  *tensor_out = tensor = ccv_nnc_tensor_new(0, tensor_params, 0);
    636
                ccv_half_precision_to_float((uint16_t*)data, tensor->data.f32, ccv_min(tensor_count, data_bytes / sizeof(uint16_t)));
    637
              }
    638
            } else if (datatype == CCV_32F && tensor_params.datatype == CCV_16F) {
    639
              size_t decoded_size = source_data_size;
    640
              if (options->decode(data, data_bytes, datatype, dim, nd, identifier, options->context, tensor_params, tensor_out, workspace, &decoded_size))
    641
              {
    642
                if (CCV_GET_DATA_TYPE(tensor_out[0]->info.datatype) == CCV_QX)
    643
                {
    644
                  if (decoded_size > 0)
    645
                    memcpy(tensor_out[0]->data.f16, workspace, ccv_min(source_data_size, decoded_size));
    646
                } else
    647
                  ccv_float_to_half_precision((float*)workspace, (uint16_t*)tensor_out[0]->data.f16, ccv_min(tensor_count, ccv_min(source_data_size, decoded_size) / sizeof(float)));
    648
              } else {
    649
                if (!tensor)
    650
                  *tensor_out = tensor = ccv_nnc_tensor_new(0, tensor_params, 0);
    651
                ccv_float_to_half_precision((float*)data, (uint16_t*)tensor->data.f16, ccv_min(tensor_count, data_bytes / sizeof(float)));
    652
              }
    653
            } else
    654
              { assert(0); }
    655
            ccfree(workspace);
    656
          }
    657
        }
    658
    #else
    659
        if (!options || !options->decode)
    660
        {
    661
          if (datatype == CCV_16F && tensor_params.datatype == CCV_32F)
    662
            ccv_half_precision_to_float((uint16_t*)data, tensor->data.f32, ccv_min(tensor_count, data_bytes / sizeof(uint16_t)));
    663
          else if (datatype == CCV_32F && tensor_params.datatype == CCV_16F)
    664
            ccv_float_to_half_precision((float*)data, (uint16_t*)tensor->data.f16, ccv_min(tensor_count, data_bytes / sizeof(float)));
    665
          else
    666
            { assert(0); }
    667
        } else {
    668
          void* const workspace = ccmalloc(source_data_size);
    669
          if (datatype == CCV_16F && tensor_params.datatype == CCV_32F)
    670
          {
    671
            size_t decoded_size = source_data_size;
    672
            if (options->decode(data, data_bytes, datatype, dim, nd, identifier, options->context, tensor_params, tensor_out, workspace, &decoded_size))
    673
            {
    674
              if (CCV_GET_DATA_TYPE(tensor_out[0]->info.datatype) == CCV_QX)
    675
              {
    676
                if (decoded_size > 0)
    677
                  memcpy(tensor_out[0]->data.f32, workspace, ccv_min(source_data_size, decoded_size));
    678
              } else
    679
                ccv_half_precision_to_float((uint16_t*)workspace, tensor_out[0]->data.f32, ccv_min(tensor_count, ccv_min(source_data_size, decoded_size) / sizeof(uint16_t)));
    680
            } else {
    681
              if (!tensor)
    682
                *tensor_out = tensor = ccv_nnc_tensor_new(0, tensor_params, 0);
    683
              ccv_half_precision_to_float((uint16_t*)data, tensor->data.f32, ccv_min(tensor_count, data_bytes / sizeof(uint16_t)));
    684
            }
    685
          } else if (datatype == CCV_32F && tensor_params.datatype == CCV_16F) {
    686
            size_t decoded_size = source_data_size;
    687
            if (options->decode(data, data_bytes, datatype, dim, nd, identifier, options->context, tensor_params, tensor_out, workspace, &decoded_size))
    688
            {
    689
              if (CCV_GET_DATA_TYPE(tensor_out[0]->info.datatype) == CCV_QX)
    690
              {
    691
                if (decoded_size > 0)
    692
                  memcpy(tensor_out[0]->data.f16, workspace, ccv_min(source_data_size, decoded_size));
    693
              } else
    694
                ccv_float_to_half_precision((float*)workspace, (uint16_t*)tensor_out[0]->data.f16, ccv_min(tensor_count, ccv_min(source_data_size, decoded_size) / sizeof(float)));
    695
            } else {
    696
              if (!tensor)
    697
                *tensor_out = tensor = ccv_nnc_tensor_new(0, tensor_params, 0);
    698
              ccv_float_to_half_precision((float*)data, (uint16_t*)tensor->data.f16, ccv_min(tensor_count, data_bytes / sizeof(float)));
    699
            }
    700
          } else
    701
            { assert(0); }
    702
          ccfree(workspace);
    703
        }
    704
    #endif
    705
    26
      } else {
    706
        // If it is QX, we need to have a custom decoder to decode properly.
    707
    26
        if (datatype != tensor_params.datatype)
    708
    0
          { assert(options && options->decode); }
    709
    26
        size_t data_size = ccv_nnc_tensor_data_size(tensor_params);
    710
    26
    #ifdef HAVE_CUDA
    711
    26
        if (!options || !options->decode9)
    712
    17
        {
    713
    17
          if (CCV_TENSOR_GET_MEMORY(tensor_params.type) == CCV_TENSOR_GPU_MEMORY)
    714
    1
            cumemcpy(tensor->data.u8, tensor->info.type, data, CCV_TENSOR_CPU_MEMORY, ccv_min(data_size, data_bytes));
    715
    16
          else
    716
    16
            memcpy(tensor->data.u8, data, ccv_min(data_size, data_bytes));
    717
    17
        } else {
    718
    9
          if (CCV_TENSOR_GET_MEMORY(tensor_params.type) == CCV_TENSOR_GPU_MEMORY)
    719
    2
          {
    720
    2
            void* const workspace = ccmalloc(data_size);
    721
    2
            size_t decoded_size = data_size;
    722
    2
            if (options->decode(data, data_bytes, datatype, dim, nd, identifier, options->context, tensor_params, tensor_out, workspace, &decoded_size))
    723
    1
              cumemcpy(tensor_out[0]->data.u8, tensor_out[0]->info.type, workspace, CCV_TENSOR_CPU_MEMORY, ccv_min(data_size, decoded_size));
    724
    1
            else {
    725
    1
              if (!tensor)
    726
    1
                *tensor_out = tensor = ccv_nnc_tensor_new(0, tensor_params, 0);
    727
    1
              cumemcpy(tensor->data.u8, tensor->info.type, data, CCV_TENSOR_CPU_MEMORY, ccv_min(data_size, data_bytes));
    728
    1
            }
    729
    2
            ccfree(workspace);
    730
    7
          } else {
    731
    7
            size_t decoded_size = data_size;
    732
    7
            if (!options->decode(data, data_bytes, datatype, dim, nd, identifier, options->context, tensor_params, tensor_out, tensor ? tensor->data.u84 : 03, &decoded_size))
    733
    3
            {
    734
    3
              if (!tensor)
    735
    1
                *tensor_out = tensor = ccv_nnc_tensor_new(0, tensor_params, 0);
    736
    3
              memcpy(tensor->data.u8, data, ccv_min(data_size, data_bytes));
    737
    3
            }
    738
    7
          }
    739
    9
        }
    740
    #elif defined(HAVE_MPS)
    741
        if (!options || !options->decode)
    742
        {
    743
          if (CCV_TENSOR_GET_MEMORY(tensor_params.type) == CCV_TENSOR_GPU_MEMORY)
    744
          {
    745
            assert(tensor->dataof == 0);
    746
            mpmemcpy(tensor->data.u8, tensor->dataof, tensor->info.type, data, 0, CCV_TENSOR_CPU_MEMORY, ccv_min(data_size, data_bytes));
    747
          } else
    748
            memcpy(tensor->data.u8, data, ccv_min(data_size, data_bytes));
    749
        } else {
    750
          if (CCV_TENSOR_GET_MEMORY(tensor_params.type) == CCV_TENSOR_GPU_MEMORY)
    751
          {
    752
            if (tensor)
    753
              { assert(tensor->dataof == 0); }
    754
            void* const workspace = ccmalloc(data_size);
    755
            size_t decoded_size = data_size;
    756
            if (options->decode(data, data_bytes, datatype, dim, nd, identifier, options->context, tensor_params, tensor_out, workspace, &decoded_size)) {
    757
              mpmemcpy(tensor_out[0]->data.u8, tensor_out[0]->dataof, tensor_out[0]->info.type, workspace, 0, CCV_TENSOR_CPU_MEMORY, ccv_min(data_size, decoded_size));
    758
            } else {
    759
              if (!tensor)
    760
                *tensor_out = tensor = ccv_nnc_tensor_new(0, tensor_params, 0);
    761
              mpmemcpy(tensor->data.u8, tensor->dataof, tensor->info.type, data, 0, CCV_TENSOR_CPU_MEMORY, ccv_min(data_size, data_bytes));
    762
            }
    763
            ccfree(workspace);
    764
          } else {
    765
            size_t decoded_size = data_size;
    766
            if (!options->decode(data, data_bytes, datatype, dim, nd, identifier, options->context, tensor_params, tensor_out, tensor ? tensor->data.u8 : 0, &decoded_size))
    767
            {
    768
              if (!tensor)
    769
                *tensor_out = tensor = ccv_nnc_tensor_new(0, tensor_params, 0);
    770
              memcpy(tensor->data.u8, data, ccv_min(data_size, data_bytes));
    771
            }
    772
          }
    773
        }
    774
    #else
    775
        if (!options || !options->decode)
    776
          memcpy(tensor->data.u8, data, ccv_min(data_size, data_bytes));
    777
        else {
    778
          size_t decoded_size = data_size;
    779
          if (!options->decode(data, data_bytes, datatype, dim, nd, identifier, options->context, tensor_params, tensor_out, tensor ? tensor->data.u8 : 0, &decoded_size))
    780
          {
    781
            if (!tensor)
    782
              *tensor_out = tensor = ccv_nnc_tensor_new(0, tensor_params, 0);
    783
            memcpy(tensor->data.u8, data, ccv_min(data_size, data_bytes));
    784
          }
    785
        }
    786
    #endif
    787
    26
      }
    788
    38
      tensor_out[0]->type &= ~CCV_GARBAGE; // If it is marked as garbage, remove that mark now.
    789
    38
      sqlite3_reset(tensor_select_stmt);
    790
    38
      sqlite3_clear_bindings(tensor_select_stmt);
    791
    38
      sqlite3_finalize(tensor_select_stmt);
    792
    38
      if (split_workspace)
    793
    2
        ccfree(split_workspace);
    794
    38
      return CCV_IO_FINAL;
    795
    38
    }