## Coverage Report

#### Created: 2026-05-04 15:30

next uncovered line (L), next uncovered region (R), next uncovered branch (B)

    /home/liu/actions-runner/_work/ccv/ccv/lib/nnc/ccv_cnnp_model.c

    Line
    Count
    Source
    1
    #include "ccv_nnc.h"
    2
    #include "ccv_nnc_easy.h"
    3
    #include "ccv_nnc_internal.h"
    4
    #include "ccv_internal.h"
    5
    #include "_ccv_cnnp_model.h"
    6
    #include "_ccv_nnc_graph.h"
    7
    #include "_ccv_nnc_symbolic_graph.h"
    8
    #ifdef HAVE_CUDA
    9
    #include "gpu/ccv_nnc_compat.h"
    10
    #endif
    11
    12
    // MARK - Level-5 API
    13
    14
    ccv_cnnp_model_io_t ccv_cnnp_model_apply(ccv_cnnp_model_t* const model, const ccv_cnnp_model_io_t* const inputs, const int input_size)
    15
    600
    {
    16
    600
      if (!model->io)
    17
    591
        model->io = ccv_array_new(sizeof(ccv_cnnp_model_io_t), 1, 0);
    18
    600
      ccv_cnnp_model_io_t model_io = ccmalloc(sizeof(struct ccv_cnnp_model_io_s) + sizeof(ccv_nnc_tensor_symbol_t) * model->output_size);
    19
    600
      model_io->param_ref = 0;
    20
    600
      model_io->param_sel = 0;
    21
    600
      model_io->visit = 0;
    22
    600
      model_io->model = model;
    23
    600
      model_io->dependencies = 0;
    24
    600
      model_io->dependents = 0;
    25
    600
      model_io->outgoings = 0;
    26
    600
      model_io->outputs = (ccv_nnc_tensor_symbol_t*)(model_io + 1);
    27
    600
      ccv_array_push(model->io, &model_io);
    28
    600
      if (input_size > 0)
    29
    597
      {
    30
    597
        model_io->incomings = ccv_array_new(sizeof(ccv_cnnp_model_io_t), input_size, 0);
    31
    597
        ccv_array_resize(model_io->incomings, input_size);
    32
    597
        int i;
    33
    597
        memcpy(ccv_array_get(model_io->incomings, 0), inputs, sizeof(ccv_cnnp_model_io_t) * input_size);
    34
    1.35k
        for (i = 0; i < input_size; i++755)
    35
    755
        {
    36
    755
          if (!inputs[i]->outgoings)
    37
    651
            inputs[i]->outgoings = ccv_array_new(sizeof(ccv_cnnp_model_io_t), 1, 0);
    38
    755
          ccv_array_push(inputs[i]->outgoings, &model_io);
    39
    755
        }
    40
    597
      } else {
    41
    3
        model_io->incomings = 0;
    42
    3
      }
    43
    600
      return model_io;
    44
    600
    }
    45
    46
    void ccv_cnnp_model_add_dependencies(ccv_cnnp_model_io_t model_io, const ccv_cnnp_model_io_t* const dependencies, const int dependency_size)
    47
    2
    {
    48
    2
      assert(dependency_size > 0);
    49
    2
      if (!model_io->dependencies)
    50
    2
        model_io->dependencies = ccv_array_new(sizeof(ccv_cnnp_model_io_t), dependency_size, 0);
    51
    2
      int i, j;
    52
    5
      for (i = 0; i < dependency_size; i++3)
    53
    3
      {
    54
    3
        int flag = 0;
    55
        // Check if it is already exist or not.
    56
    4
        for (j = 0; !flag && j < model_io->dependencies->rnum; j++1)
    57
    1
          if (*(ccv_cnnp_model_io_t*)ccv_array_get(model_io->dependencies, j) == dependencies[i])
    58
    0
            flag = 1;
    59
    3
        if (flag)
    60
    0
          continue;
    61
    3
        ccv_array_push(model_io->dependencies, dependencies + i);
    62
    3
        ++dependencies[i]->dependents;
    63
    3
      }
    64
    2
    }
    65
    66
    int ccv_cnnp_model_output_size(const ccv_cnnp_model_t* const model)
    67
    0
    {
    68
    0
      return model->output_size;
    69
    0
    }
    70
    71
    int ccv_cnnp_model_is_trainable(const ccv_cnnp_model_t* const model)
    72
    16
    {
    73
      // If the model is compiled, it is default to 1 unless it is not.
    74
    16
      if (model->compiled_data)
    75
    4
        return model->is_trainable >= 0 ? model->is_trainable : 10;
    76
    12
      return model->is_trainable;
    77
    16
    }
    78
    79
    ccv_cnnp_model_io_t ccv_cnnp_model_parameters(ccv_cnnp_model_t* const model, const int selector, const int index)
    80
    397
    {
    81
    397
      if (!model->io)
    82
    41
        model->io = ccv_array_new(sizeof(ccv_cnnp_model_io_t), 1, 0);
    83
    397
      ccv_cnnp_model_io_t model_io = ccmalloc(sizeof(struct ccv_cnnp_model_io_s));
    84
    397
      model_io->param_ref = index >= 0 ? index + 144 : ALL_PARAMETERS353;
    85
    397
      model_io->param_sel = selector >= 0 ? selector + 1308 : ALL_PARAMETERS89;
    86
    397
      model_io->visit = 0;
    87
    397
      model_io->model = model;
    88
    397
      model_io->outputs = 0;
    89
    397
      model_io->dependencies = 0;
    90
    397
      model_io->dependents = 0;
    91
    397
      model_io->incomings = 0;
    92
    397
      model_io->outgoings = 0;
    93
    397
      ccv_array_push(model->io, &model_io);
    94
    397
      return model_io;
    95
    397
    }
    96
    97
    void ccv_cnnp_model_notify_hook(ccv_cnnp_model_t* const model, ccv_cnnp_model_notify_f func, void* const context)
    98
    3
    {
    99
    3
      model->notify_hook.func = func;
    100
    3
      model->notify_hook.context = context;
    101
    3
    }
    102
    103
    void ccv_cnnp_model_notify(const ccv_cnnp_model_t* const model, const int tag, void* const payload)
    104
    14
    {
    105
    14
      if (model->notify_hook.func)
    106
    3
        model->notify_hook.func(model, tag, payload, model->notify_hook.context);
    107
    14
      if (model->isa->notify)
    108
    1
        model->isa->notify(model, tag, payload);
    109
    14
    }
    110
    111
    static int _ccv_nnc_array_dedup_graph_exec_symbols(ccv_nnc_graph_exec_symbol_t* const graph_exec_symbols, int graph_exec_symbol_size)
    112
    2.24k
    {
    113
    2.24k
      int i, j;
    114
    4.86k
      for (i = 0; i < graph_exec_symbol_size; i++2.61k)
    115
    2.61k
      {
    116
    2.61k
        ccv_nnc_graph_exec_symbol_t* const graph_exec_symbol = graph_exec_symbols + i;
    117
        // Check whether this tensor symbol has any duplicate.
    118
    23.2k
        for (j = i + 1; j < graph_exec_symbol_size;)
    119
    20.6k
        {
    120
    20.6k
          ccv_nnc_graph_exec_symbol_t* const other_symbol = graph_exec_symbols + j;
    121
          // If there is a same tensor symbol, remove it.
    122
    20.6k
          if (other_symbol->d == graph_exec_symbol->d && other_symbol->graph == graph_exec_symbol->graph2.71k)
    123
    2.71k
          {
    124
    2.71k
            if (j + 1 < graph_exec_symbol_size)
    125
    439
              *other_symbol = graph_exec_symbols[graph_exec_symbol_size - 1];
    126
    2.71k
            --graph_exec_symbol_size;
    127
    2.71k
            continue;
    128
    2.71k
          }
    129
    17.9k
          ++j;
    130
    17.9k
        }
    131
    2.61k
      }
    132
    2.24k
      return graph_exec_symbol_size;
    133
    2.24k
    }
    134
    135
    void ccv_cnnp_model_add_to_array(void* const context, const ccv_nnc_tensor_symbol_t symbol, const int is_trainable)
    136
    3.17k
    {
    137
    3.17k
      ccv_cnnp_model_add_to_array_context_t* const add_to_array_context = (ccv_cnnp_model_add_to_array_context_t*)context;
    138
    3.17k
      ccv_cnnp_model_t* const model = add_to_array_context->sequence->model;
    139
    3.17k
      int i;
    140
    3.17k
      if (add_to_array_context->add_parameter_indices && !model->parameter_indices2.98k)
    141
    2.53k
        model->parameter_indices = ccv_array_new(sizeof(int), 0, 0);
    142
    37.1k
      for (i = 0; i < add_to_array_context->symbols->rnum; i++33.9k)
    143
    34.0k
      {
    144
    34.0k
        const ccv_nnc_tensor_symbol_t other_symbol = *(ccv_nnc_tensor_symbol_t*)ccv_array_get(add_to_array_context->symbols, i);
    145
    34.0k
        if (other_symbol.d == symbol.d && other_symbol.graph == symbol.graph40)
    146
    40
        {
    147
          // Only add to parameter_indices if it is trainable.
    148
    40
          if (add_to_array_context->add_parameter_indices)
    149
    27
            ccv_array_add_unique_int(model->parameter_indices, i);
    150
          // Found it, return, don't add it.
    151
    40
          return;
    152
    40
        }
    153
    34.0k
      }
    154
      // Only add to parameter_indices if it is trainable.
    155
    3.13k
      if (add_to_array_context->add_parameter_indices)
    156
    2.96k
        ccv_array_push(model->parameter_indices, &add_to_array_context->symbols->rnum);
    157
      // This is a new one, no need to add_unique_int, it is unique.
    158
    3.13k
      ccv_array_push(add_to_array_context->symbols, &symbol);
    159
    3.13k
      if (add_to_array_context->trainables)
    160
    2.97k
        ccv_array_push(add_to_array_context->trainables, &is_trainable);
    161
    3.13k
      char id[2048];
    162
    3.13k
      id[0] = add_to_array_context->prefix;
    163
    3.13k
      id[1] = '-';
    164
    3.13k
      int total_len = 2;
    165
    6.51k
      for (i = 0; i < add_to_array_context->sequence->sequences->rnum; i++3.37k)
    166
    3.37k
      {
    167
    3.37k
        const ccv_cnnp_model_name_t* const name = (ccv_cnnp_model_name_t*)ccv_array_get(add_to_array_context->sequence->sequences, i);
    168
    3.37k
        int len;
    169
    3.37k
        if (name->name && name->name[0] != '\0'376)
    170
    376
          len = snprintf(id + total_len, 2048 - total_len, "%s-%d-", name->name, name->sequence);
    171
    3.00k
        else
    172
    3.00k
          len = snprintf(id + total_len, 2048 - total_len, "%d-", name->sequence);
    173
    3.37k
        total_len += len;
    174
    3.37k
        if (total_len >= 2047)
    175
    0
          break;
    176
    3.37k
      }
    177
    3.13k
      if (total_len < 2047)
    178
    3.13k
        total_len += snprintf(id + total_len, 2048 - total_len, "%d", add_to_array_context->sequence->it);
    179
    3.13k
      assert(total_len < 2048);
    180
    3.13k
      char *heap_id = (char*)ccmalloc(total_len + 1);
    181
    3.13k
      memcpy(heap_id, id, total_len + 1);
    182
    3.13k
      ccv_array_push(add_to_array_context->ids, &heap_id);
    183
    3.13k
      ++add_to_array_context->sequence->it;
    184
    3.13k
    }
    185
    186
    static void _ccv_cnnp_compiled_data_init(ccv_cnnp_compiled_data_t* const compiled_data, const int output_size, ccv_array_t* const gradient_checkpoints)
    187
    2.31k
    {
    188
    2.31k
      compiled_data->f = compiled_data->fits + output_size;
    189
    2.31k
      compiled_data->xpu_alloc.mp_hdr = -1;
    190
    2.31k
      compiled_data->xpu_alloc.freed = kh_init(dy_str);
    191
    2.31k
      compiled_data->xpu_alloc.allocd = kh_init(dy_alloc);
    192
    2.31k
      compiled_data->gradient_checkpoints = gradient_checkpoints;
    193
    2.31k
    }
    194
    195
    static int _ccv_cnnp_model_root_parallel_count(const ccv_cnnp_model_t* const model)
    196
    7.41k
    {
    197
    7.41k
      return ccv_max(model->parallel_count, 1);
    198
    7.41k
    }
    199
    200
    static int _ccv_cnnp_model_effective_parallel_count(const ccv_cnnp_model_t* const model)
    201
    2.65k
    {
    202
    2.65k
      int parallel_count = _ccv_cnnp_model_root_parallel_count(model);
    203
    2.65k
      if (model->graph && model->graph->data_parallel.count > parallel_count)
    204
    3
        parallel_count = model->graph->data_parallel.count;
    205
    2.65k
      return parallel_count;
    206
    2.65k
    }
    207
    208
    static int _ccv_cnnp_compiled_data_parallel_count(const ccv_cnnp_model_t* const model, const ccv_cnnp_compiled_data_t* const compiled_data)
    209
    232
    {
    210
    232
      return compiled_data->parallel_count > 0 ? compiled_data->parallel_count : _ccv_cnnp_model_effective_parallel_count(model)0;
    211
    232
    }
    212
    213
    ccv_nnc_tensor_symbol_t ccv_cnnp_model_get_symbol(ccv_cnnp_model_t* const self, const ccv_nnc_tensor_symbol_t symbol)
    214
    2.98k
    {
    215
    2.98k
      assert(self->data);
    216
    2.98k
      ccv_cnnp_model_build_data_t* const build_data = (ccv_cnnp_model_build_data_t*)self->data;
    217
    2.98k
      if (build_data->parallel_count <= 1 || build_data->parallel_rank == 016)
    218
    2.97k
        return symbol;
    219
    12
      const int rank = build_data->parallel_rank;
    220
    12
      assert(rank > 0);
    221
    12
      assert(rank < build_data->parallel_count);
    222
    12
      ccv_nnc_symbolic_graph_t* const graph = (ccv_nnc_symbolic_graph_t*)symbol.graph;
    223
    12
      ccv_nnc_tensor_symbol_t copy = ccv_nnc_tensor_symbol_copy(graph, symbol, rank);
    224
    12
      if (copy.d != CCV_NNC_NO_TENSOR_SYMBOL)
    225
    0
        return copy;
    226
    12
      ccv_nnc_tensor_param_t params = ccv_nnc_tensor_symbol_params(graph, symbol);
    227
    12
      if (CCV_TENSOR_GET_MEMORY(params.type) == CCV_TENSOR_GPU_MEMORY)
    228
    12
        CCV_TENSOR_SET_DEVICE_ID(params.type, rank);
    229
    12
      copy = ccv_nnc_tensor_symbol_new(graph, params, 0);
    230
    12
      ccv_nnc_tensor_symbol_set_copy(graph, symbol, rank, copy);
    231
    12
      return copy;
    232
    12
    }
    233
    234
    typedef struct {
    235
      void* old_graph_exec_symbol_new_hook_context;
    236
      ccv_nnc_graph_exec_symbol_new_hook_f old_graph_exec_symbol_new_hook;
    237
      ccv_nnc_symbolic_graph_t* graph;
    238
      ccv_cnnp_model_build_data_t* build_data;
    239
    } ccv_cnnp_model_set_exec_flags_context_t;
    240
    241
    static void _ccv_cnnp_model_set_exec_flags(void* context, const ccv_nnc_graph_exec_symbol_t symbol, const ccv_nnc_cmd_t cmd, const ccv_nnc_tensor_symbol_t* const inputs, const int input_size, const ccv_nnc_tensor_symbol_t* const outputs, const int output_size, const char* const name)
    242
    2.96k
    {
    243
    2.96k
      ccv_cnnp_model_set_exec_flags_context_t* flags_context = (ccv_cnnp_model_set_exec_flags_context_t*)context;
    244
    2.96k
      if (flags_context->build_data->exec_flags)
    245
    0
        ccv_nnc_graph_exec_symbol_set_flags(flags_context->graph, symbol, flags_context->build_data->exec_flags);
    246
    2.96k
      if (flags_context->old_graph_exec_symbol_new_hook)
    247
    2.20k
        flags_context->old_graph_exec_symbol_new_hook(flags_context->old_graph_exec_symbol_new_hook_context, symbol, cmd, inputs, input_size, outputs, output_size, name);
    248
    2.96k
    }
    249
    250
    static void _ccv_cnnp_model_compile(ccv_cnnp_model_t* const model, const ccv_nnc_tensor_param_t* const inputs, const int input_size, const ccv_nnc_cmd_t loss)
    251
    2.31k
    {
    252
    2.31k
      assert(model->graph);
    253
    2.31k
      model->inputs = ccmalloc(sizeof(ccv_nnc_tensor_symbol_t) * input_size);
    254
    2.31k
      int i;
    255
    4.70k
      for (i = 0; i < input_size; i++2.39k)
    256
    2.39k
        model->inputs[i] = ccv_nnc_tensor_symbol_new(model->graph, inputs[i], 0);
    257
    2.31k
      ccv_array_t* const parameters = ccv_array_new(sizeof(ccv_nnc_tensor_symbol_t), 0, 0);
    258
    2.31k
      ccv_array_t* const parameter_ids = ccv_array_new(sizeof(char*), 0, 0);
    259
    2.31k
      ccv_array_t* const parameter_trainables = ccv_array_new(sizeof(int), 0, 0);
    260
    2.31k
      ccv_cnnp_model_sequence_t model_sequence = {
    261
    2.31k
        .bank = kh_init(ccv_cnnp_model_name_bank)
    262
    2.31k
      };
    263
    2.31k
      ccv_cnnp_model_add_to_array_context_t add_to_parameter_context = {
    264
    2.31k
        .add_parameter_indices = 1,
    265
    2.31k
        .prefix = 't',
    266
    2.31k
        .sequence = &model_sequence,
    267
    2.31k
        .symbols = parameters,
    268
    2.31k
        .ids = parameter_ids,
    269
    2.31k
        .trainables = parameter_trainables,
    270
    2.31k
      };
    271
    2.31k
      ccv_array_t* const internals = ccv_array_new(sizeof(ccv_nnc_tensor_symbol_t), 0, 0);
    272
    2.31k
      ccv_array_t* const internal_ids = ccv_array_new(sizeof(char*), 0, 0);
    273
    2.31k
      ccv_cnnp_model_add_to_array_context_t add_to_output_context = {
    274
    2.31k
        .add_parameter_indices = 0,
    275
    2.31k
        .prefix = 'r',
    276
    2.31k
        .sequence = &model_sequence,
    277
    2.31k
        .symbols = internals,
    278
    2.31k
        .ids = internal_ids,
    279
    2.31k
        .trainables = 0,
    280
    2.31k
      };
    281
    2.31k
      ccv_cnnp_model_build_data_t build_data = {
    282
    2.31k
        .exec_flags = 0,
    283
    2.31k
        .is_trainable = model->is_trainable >= 0 ? model->is_trainable2.31k : 14,
    284
    2.31k
        .parallel_count = 1,
    285
    2.31k
        .parallel_rank = 0,
    286
    2.31k
        .model_sequence = &model_sequence,
    287
    2.31k
        .add_to_array = ccv_cnnp_model_add_to_array,
    288
    2.31k
        .parameters = parameters,
    289
    2.31k
        .context = {
    290
    2.31k
          .add_to_parameter = &add_to_parameter_context,
    291
    2.31k
          .add_to_output = &add_to_output_context,
    292
    2.31k
        },
    293
    2.31k
        .gradient_checkpoints = 0,
    294
    2.31k
      };
    295
    2.31k
      model->data = &build_data;
    296
    2.31k
      ccv_cnnp_model_set_exec_flags_context_t flags_context = {
    297
    2.31k
        .graph = model->graph,
    298
    2.31k
        .build_data = &build_data,
    299
    2.31k
        .old_graph_exec_symbol_new_hook = 0,
    300
    2.31k
        .old_graph_exec_symbol_new_hook_context = 0
    301
    2.31k
      };
    302
    2.31k
      flags_context.old_graph_exec_symbol_new_hook_context = ccv_nnc_graph_exec_symbol_new_hook(model->graph, _ccv_cnnp_model_set_exec_flags, &flags_context, &flags_context.old_graph_exec_symbol_new_hook);
    303
    2.31k
      ccv_cnnp_model_build(model, model->graph, model->inputs, input_size, 0, 0);
    304
      // Reset back to previous hook.
    305
    2.31k
      ccv_nnc_graph_exec_symbol_new_hook(model->graph, flags_context.old_graph_exec_symbol_new_hook, flags_context.old_graph_exec_symbol_new_hook_context, 0);
    306
    4.65k
      for (i = 0; i < model->output_size; i++2.34k)
    307
    2.34k
      {
    308
    2.34k
        const ccv_nnc_tensor_symbol_t output = model->outputs[i];
    309
    2.34k
        const ccv_nnc_tensor_symbol_t alias_to = ccv_nnc_tensor_symbol_alias_to(model->graph, output);
    310
    2.34k
        if (alias_to.d == CCV_NNC_NO_TENSOR_SYMBOL)
    311
    1.33k
          continue;
    312
        // If output is an alias, insert data transform regardless for result correctness (we cannot bind an alias). You can check ccv_nnc_tensor_bind_symbol method
    313
        // to see that we can correctly bind a tensor which from it, has aliases, but we cannot bind an alias tensor correctly (this is expected, sort of, to be
    314
        // honest, because we cannot handle cases of alias is part of the original tensor but bind differently).
    315
    1.00k
        const ccv_nnc_tensor_param_t output_params = ccv_nnc_tensor_symbol_params(model->graph, output);
    316
    1.00k
        model->outputs[i] = ccv_nnc_tensor_symbol_new(model->graph, output_params, 0);
    317
    1.00k
        ccv_nnc_graph_exec_symbol_t make_contiguous = ccv_nnc_graph_exec_symbol_new(model->graph, CMD_FORMAT_TRANSFORM_FORWARD(), &output, 1, model->outputs + i, 1, "contiguous");
    318
    1.00k
        ccv_nnc_graph_exec_symbol_set_flags(model->graph, make_contiguous, CCV_NNC_GRAPH_EXEC_DISABLE_OPT);
    319
    1.00k
      }
    320
    2.31k
      model->data = 0;
    321
    2.31k
      kh_destroy(ccv_cnnp_model_name_bank, model_sequence.bank);
    322
    2.31k
      if (model_sequence.sequences)
    323
    2.29k
        ccv_array_free(model_sequence.sequences);
    324
      // Check if there are parameters that are not trainables. If there are, we will allocate uint64 bitmap to record that.
    325
    2.31k
      int not_trainables = 0;
    326
      // Assert no parameter is alias.
    327
    5.27k
      for (i = 0; i < parameters->rnum; i++2.96k)
    328
    2.96k
      {
    329
    2.96k
        const ccv_nnc_tensor_symbol_t parameter = *(ccv_nnc_tensor_symbol_t*)ccv_array_get(parameters, i);
    330
    2.96k
        const ccv_nnc_tensor_symbol_t alias_to = ccv_nnc_tensor_symbol_alias_to(parameter.graph, parameter);
    331
    2.96k
        assert(alias_to.graph == 0); // Cannot find the one alias to.
    332
    2.96k
        if (*(int*)ccv_array_get(parameter_trainables, i) == 0)
    333
    14
          not_trainables = 1;
    334
    2.96k
      }
    335
    2.31k
      assert(parameters->rnum == parameter_trainables->rnum);
    336
    2.31k
      uint64_t* parameter_flags = 0;
    337
    2.31k
      if (not_trainables)
    338
    10
      {
    339
    10
        parameter_flags = (uint64_t*)cccalloc(((parameters->rnum + 63) >> 6), sizeof(uint64_t));
    340
    44
        for (i = 0; i < parameter_trainables->rnum; i++34)
    341
    34
          if (*(int*)ccv_array_get(parameter_trainables, i))
    342
    20
            parameter_flags[i >> 6] |= ((uint64_t)1 << (i & 63));
    343
    10
      }
    344
    2.31k
      ccv_array_free(parameter_trainables);
    345
      // Assert no internal is alias.
    346
    2.48k
      for (i = 0; i < internals->rnum; i++165)
    347
    165
      {
    348
    165
        const ccv_nnc_tensor_symbol_t internal = *(ccv_nnc_tensor_symbol_t*)ccv_array_get(internals, i);
    349
    165
        const ccv_nnc_tensor_symbol_t alias_to = ccv_nnc_tensor_symbol_alias_to(internal.graph, internal);
    350
    165
        assert(alias_to.graph == 0); // Cannot find the one alias to.
    351
    165
      }
    352
    2.31k
      const int output_size = model->output_size;
    353
    2.31k
      ccv_nnc_graph_exec_symbol_autogen(model->graph, 0, 0, CCV_NNC_AUTOGEN_ALL_EXECS | CCV_NNC_AUTOGEN_SOURCES_AND_DESTINATIONS);
    354
    2.31k
      const int parameters_rnum = parameters->rnum;
    355
    2.31k
      if (input_size > 0)
    356
    2.31k
      {
    357
    2.31k
        ccv_array_resize(parameters, parameters_rnum + input_size);
    358
    2.31k
        memcpy(ccv_array_get(parameters, parameters_rnum), model->inputs, input_size * sizeof(ccv_nnc_tensor_symbol_t));
    359
    2.31k
      }
    360
    2.31k
      ccv_nnc_symbolic_graph_simplify(model->graph,
    361
    2.31k
        SYMBOLIC_GRAPH_PASSES(CCV_NNC_SIMPLIFY_COMMON_SUBEXPRESSION_ELIMINATION,
    362
    2.31k
          CCV_NNC_SIMPLIFY_DATA_TRANSFER_OPT,
    363
    2.31k
          CCV_NNC_SIMPLIFY_OPS_FUSION,
    364
    2.31k
          CCV_NNC_SIMPLIFY_GRAPH_PRUNING),
    365
    2.31k
        ccv_array_get(parameters, 0), parameters_rnum + input_size,
    366
    2.31k
        model->outputs, output_size,
    367
    2.31k
        SYMBOLIC_GRAPH_SOURCES(model->graph), SYMBOLIC_GRAPH_DESTINATIONS(model->graph));
    368
    2.31k
      ccv_nnc_graph_exec_symbol_autogen(model->graph, 0, 0, CCV_NNC_AUTOGEN_SOURCES_AND_DESTINATIONS);
    369
      // Size it down.
    370
    2.31k
      parameters->rnum = parameters_rnum;
    371
    2.31k
      ccv_cnnp_compiled_data_t* compiled_data = model->compiled_data = cccalloc(1, sizeof(ccv_cnnp_compiled_data_t) + sizeof(ccv_nnc_tensor_symbol_t) * (output_size * 2 - 1));
    372
    2.31k
      _ccv_cnnp_compiled_data_init(compiled_data, output_size, build_data.gradient_checkpoints);
    373
    2.31k
      const int evaluate_to_size = compiled_data->evaluate.to_size = ccv_nnc_symbolic_graph_destination_size(model->graph);
    374
    2.31k
      assert(evaluate_to_size > 0);
    375
    2.31k
      compiled_data->evaluate.tos = ccmalloc(sizeof(ccv_nnc_graph_exec_symbol_t) * evaluate_to_size);
    376
    2.31k
      memcpy(compiled_data->evaluate.tos, ccv_nnc_symbolic_graph_destinations(model->graph), sizeof(ccv_nnc_graph_exec_symbol_t) * evaluate_to_size);
    377
    2.31k
      compiled_data->loss = loss;
    378
    2.31k
      if (loss.cmd == CCV_NNC_NOOP)
    379
    2.30k
      {
    380
        // If no loss function provided, there is no fits.
    381
    4.63k
        for (i = 0; i < output_size; i++2.33k)
    382
    2.33k
        {
    383
    2.33k
          compiled_data->fits[i] = NO_TENSOR_SYMBOL;
    384
    2.33k
          const ccv_nnc_tensor_symbol_t alias_to = ccv_nnc_tensor_symbol_alias_to(model->graph, model->outputs[i]);
    385
    2.33k
          if (alias_to.d < 0)
    386
    2.33k
            compiled_data->f[i] = model->outputs[i];
    387
    0
          else { // We cannot differentiate against an alias, therefore, we have to verify this output is full, and we can diff against the original.
    388
    0
            int ofs[CCV_NNC_MAX_DIM_ALLOC];
    389
    0
            int inc[CCV_NNC_MAX_DIM_ALLOC];
    390
    0
            ccv_nnc_tensor_symbol_alias_params(model->graph, model->outputs[i], ofs, inc);
    391
    0
            int j;
    392
    0
            for (j = 0; j < CCV_NNC_MAX_DIM_ALLOC; j++)
    393
    0
              { assert(ofs[j] == 0); } // There is no ofs.
    394
    0
            compiled_data->f[i] = alias_to; // Unfortunately, I cannot assert the size yet.
    395
    0
          }
    396
    2.33k
        }
    397
    2.30k
      } else {
    398
    20
        for (i = 0; i < output_size; i++10)
    399
    10
        {
    400
    10
          const ccv_nnc_tensor_param_t info = ccv_nnc_tensor_symbol_params(model->graph, model->outputs[i]);
    401
    10
          const ccv_nnc_tensor_symbol_t fit = compiled_data->fits[i] = ccv_nnc_tensor_symbol_new(model->graph, info, 0);
    402
    10
          compiled_data->f[i] = ccv_nnc_tensor_symbol_new(model->graph, ccv_nnc_tensor_auto, 0);
    403
    10
          ccv_nnc_graph_exec_symbol_new(model->graph, loss, TENSOR_SYMBOL_LIST(model->outputs[i], fit), TENSOR_SYMBOL_LIST(compiled_data->f[i]), 0);
    404
    10
        }
    405
    10
      }
    406
    2.31k
      ccv_nnc_graph_exec_symbol_autogen(model->graph, 0, 0, CCV_NNC_AUTOGEN_ALL_EXECS | CCV_NNC_AUTOGEN_SOURCES_AND_DESTINATIONS);
    407
    2.31k
      ccv_nnc_symbolic_graph_simplify(model->graph,
    408
    2.31k
        SYMBOLIC_GRAPH_PASSES(CCV_NNC_SIMPLIFY_OPS_FUSION), // Only do Ops fusion, in this way, we can fuse the loss function.
    409
    2.31k
        0, 0, // No need to provide binds at this point.
    410
    2.31k
        compiled_data->f, model->output_size,
    411
    2.31k
        SYMBOLIC_GRAPH_SOURCES(model->graph), SYMBOLIC_GRAPH_DESTINATIONS(model->graph));
    412
    2.31k
      ccv_nnc_graph_exec_symbol_autogen(model->graph, 0, 0, CCV_NNC_AUTOGEN_SOURCES_AND_DESTINATIONS);
    413
      // If inputs are from GPU, stream type is GPU.
    414
    2.31k
      compiled_data->parameters = parameters;
    415
    2.31k
      compiled_data->parameter_flags = parameter_flags;
    416
    2.31k
      compiled_data->internals = internals;
    417
    2.31k
      compiled_data->ids.parameters = parameter_ids;
    418
    2.31k
      compiled_data->ids.internals = internal_ids;
    419
    2.31k
      ccv_cnnp_model_gradient_checkpoints_cleanup_after_build(compiled_data, model->graph);
    420
    2.31k
    }
    421
    422
    static void _ccv_cnnp_graph_push_graph_exec_symbol(void* context, const ccv_nnc_graph_exec_symbol_t symbol, const ccv_nnc_cmd_t cmd, const ccv_nnc_tensor_symbol_t* const inputs, const int input_size, const ccv_nnc_tensor_symbol_t* const outputs, const int output_size, const char* const name)
    423
    8.82k
    {
    424
    8.82k
      ccv_array_t* const stack = (ccv_array_t*)context;
    425
    8.82k
      ccv_array_push(stack, &symbol.d);
    426
    8.82k
    }
    427
    428
    static void _ccv_nnc_tensor_symbol_reinit(const ccv_nnc_symbolic_graph_t* const src_graph, ccv_nnc_symbolic_graph_t* const dest_graph, const int src_index, const int dest_index)
    429
    38.5k
    {
    430
    38.5k
      const ccv_nnc_tensor_symbol_t src_symbol = {
    431
    38.5k
        .d = src_index,
    432
    38.5k
        .graph = src_graph
    433
    38.5k
      };
    434
    38.5k
      const ccv_nnc_tensor_symbol_t dest_symbol = {
    435
    38.5k
        .d = dest_index,
    436
    38.5k
        .graph = dest_graph
    437
    38.5k
      };
    438
    38.5k
      const ccv_nnc_tensor_param_t params = ccv_nnc_tensor_symbol_params(src_graph, src_symbol);
    439
    38.5k
      ccv_nnc_tensor_symbol_set(dest_graph, dest_symbol, params);
    440
    38.5k
      int ofs[CCV_NNC_MAX_DIM_ALLOC];
    441
    38.5k
      int inc[CCV_NNC_MAX_DIM_ALLOC];
    442
    38.5k
      if (0 == ccv_nnc_tensor_symbol_alias_params(src_graph, src_symbol, ofs, inc))
    443
    2.00k
        ccv_nnc_tensor_symbol_alias_set(dest_graph, dest_symbol, ofs, inc);
    444
    38.5k
    }
    445
    446
    static int _ccv_nnc_tensor_symbol_check_dim(const ccv_nnc_symbolic_graph_t* const src_graph, ccv_nnc_symbolic_graph_t* const dest_graph, const int src_index, const int dest_index)
    447
    2.41k
    {
    448
    2.41k
      const ccv_nnc_tensor_symbol_t src_symbol = {
    449
    2.41k
        .d = src_index,
    450
    2.41k
        .graph = src_graph
    451
    2.41k
      };
    452
    2.41k
      const ccv_nnc_tensor_param_t src_params = ccv_nnc_tensor_symbol_params(src_graph, src_symbol);
    453
    2.41k
      const ccv_nnc_tensor_symbol_t dest_symbol = {
    454
    2.41k
        .d = dest_index,
    455
    2.41k
        .graph = dest_graph
    456
    2.41k
      };
    457
    2.41k
      const ccv_nnc_tensor_param_t dest_params = ccv_nnc_tensor_symbol_params(dest_graph, dest_symbol);
    458
    2.41k
      return memcmp(src_params.dim, dest_params.dim, sizeof(src_params.dim)) == 0;
    459
    2.41k
    }
    460
    461
    static void _ccv_cnnp_model_gradient_init(ccv_cnnp_model_t* const model, const int gradient_mode, const uint64_t disable_outgrad, ccv_nnc_tensor_t* const* const fits, const int fit_size);
    462
    static void _ccv_cnnp_compiled_data_graph_free(ccv_cnnp_compiled_data_t* const compiled_data);
    463
    464
    typedef struct {
    465
      int parallel_count;
    466
      ccv_nnc_symbolic_graph_t* graph;
    467
      ccv_nnc_graph_exec_arena_t* graph_exec_arena;
    468
    } ccv_nnc_graph_exec_update_t;
    469
    470
    static void _ccv_cnnp_cmd_update_for_execs(void* const context, const ccv_nnc_graph_exec_symbol_t symbol, const ccv_nnc_cmd_t cmd, const ccv_nnc_hint_t hint)
    471
    58
    {
    472
    58
      ccv_nnc_graph_exec_update_t* const graph_exec_update = (ccv_nnc_graph_exec_update_t*)context;
    473
    58
      ccv_nnc_graph_exec_arena_t* const graph_exec_arena = graph_exec_update->graph_exec_arena;
    474
    58
      ccv_nnc_graph_exec_t graph_exec = ccv_nnc_graph_exec_from_symbol(graph_exec_arena, symbol);
    475
    58
      ccv_nnc_graph_exec_set(graph_exec.graph, graph_exec, cmd);
    476
    58
      ccv_nnc_graph_exec_set_hint(graph_exec.graph, graph_exec, hint);
    477
    58
      const ccv_nnc_symbolic_graph_t* const graph = graph_exec_update->graph;
    478
    58
      const int parallel_count = graph_exec_update->parallel_count;
    479
    58
      int i;
    480
    178
      for (i = 1; i < parallel_count; i++120)
    481
    120
      {
    482
    120
        const ccv_nnc_graph_exec_t copy = ccv_nnc_graph_exec_from_symbol(graph_exec_arena, ccv_nnc_graph_exec_symbol_copy(graph, symbol, i));
    483
    120
        if (!CCV_NO_GRAPH_EXEC(copy))
    484
    120
        {
    485
    120
          ccv_nnc_graph_exec_set(copy.graph, copy, cmd);
    486
    120
          ccv_nnc_graph_exec_set_hint(copy.graph, copy, hint);
    487
    120
        }
    488
    120
      }
    489
    58
    }
    490
    491
    void ccv_cnnp_model_absorb(ccv_cnnp_model_t* const model, ccv_cnnp_model_t* const init, const ccv_nnc_tensor_param_t* const inputs, const int input_size)
    492
    2.20k
    {
    493
    2.20k
      assert(model->graph);
    494
    2.20k
      assert(model->compiled_data);
    495
    2.20k
      assert(!init->graph);
    496
    2.20k
      ccv_cnnp_compiled_data_t* const compiled_data = model->compiled_data;
    497
    2.20k
      init->graph = ccv_nnc_symbolic_graph_new();
    498
    2.20k
      ccv_array_t* const stack = ccv_array_new(sizeof(int), 0, 0);
    499
    2.20k
      ccv_nnc_graph_exec_symbol_new_hook(init->graph, _ccv_cnnp_graph_push_graph_exec_symbol, stack, 0);
    500
    2.20k
      _ccv_cnnp_model_compile(init, inputs, input_size, compiled_data->loss);
    501
    2.20k
      init->parallel_count = model->parallel_count;
    502
    2.20k
      init->memory_compression = model->memory_compression;
    503
    2.20k
      init->memory_reduction = model->memory_reduction;
    504
    2.20k
      init->gradient_checkpointing = model->gradient_checkpointing;
    505
    2.20k
      init->compiled_data->stream_type = model->compiled_data->stream_type;
    506
    2.20k
      init->compiled_data->minimize.minimizer = model->compiled_data->minimize.minimizer;
    507
    2.20k
      init->compiled_data->minimize.max_saved_aux_size = model->compiled_data->minimize.max_saved_aux_size;
    508
    2.20k
      if (model->compiled_data->gradient_mode != CCV_CNNP_COMPILED_DATA_GRADIENT_NONE)
    509
    2.20k
        _ccv_cnnp_model_gradient_init(init, model->compiled_data->gradient_mode, model->compiled_data->disable_outgrad, 0, 0);
    510
    2.20k
      ccv_nnc_graph_exec_symbol_new_hook(init->graph, 0, 0, 0);
    511
    2.20k
      ccv_nnc_symbolic_graph_tensor_auto(init->graph, TRAVERSE_FULL);
    512
    2.20k
      int i, j;
    513
      // Verify parameters, internals and saved_aux in both graph has the same dimensionality.
    514
    4.61k
      for (i = 0; i < compiled_data->parameters->rnum; i++2.41k)
    515
    2.41k
      {
    516
    2.41k
        const int d = ((ccv_nnc_tensor_symbol_t*)ccv_array_get(compiled_data->parameters, i))->d;
    517
    2.41k
        assert(_ccv_nnc_tensor_symbol_check_dim(model->graph, init->graph, d, d));
    518
    2.41k
      }
    519
    2.20k
      for (i = 0; i < compiled_data->internals->rnum; i++0)
    520
    0
      {
    521
    0
        const int d = ((ccv_nnc_tensor_symbol_t*)ccv_array_get(compiled_data->internals, i))->d;
    522
    0
        assert(_ccv_nnc_tensor_symbol_check_dim(model->graph, init->graph, d, d));
    523
    0
      }
    524
      // Update inputs.
    525
    2.20k
      assert(model->input_size == init->input_size);
    526
    4.40k
      for (i = 0; 2.20ki < model->input_size; i++2.20k)
    527
    2.20k
        if (model->inputs[i].d >= 0)
    528
    2.20k
        {
    529
    2.20k
          assert(init->inputs[i].d >= 0);
    530
    2.20k
          _ccv_nnc_tensor_symbol_reinit(init->graph, model->graph, init->inputs[i].d, model->inputs[i].d);
    531
    2.20k
        }
    532
      // Update outputs.
    533
    2.20k
      assert(model->output_size == init->output_size);
    534
    4.40k
      for (i = 0; 2.20ki < model->output_size; i++2.20k)
    535
    2.20k
      {
    536
    2.20k
        if (model->outputs[i].d >= 0)
    537
    2.20k
        {
    538
    2.20k
          assert(init->outputs[i].d >= 0);
    539
    2.20k
          _ccv_nnc_tensor_symbol_reinit(init->graph, model->graph, init->outputs[i].d, model->outputs[i].d);
    540
    2.20k
        }
    541
    2.20k
        if (model->outputs[i].d != model->compiled_data->f[i].d)
    542
    0
        {
    543
    0
          assert(init->outputs[i].d != init->compiled_data->f[i].d);
    544
    0
          if (model->compiled_data->f[i].d >= 0)
    545
    0
          {
    546
    0
            assert(init->compiled_data->f[i].d >= 0);
    547
    0
            _ccv_nnc_tensor_symbol_reinit(init->graph, model->graph, init->compiled_data->f[i].d, model->compiled_data->f[i].d);
    548
    0
          }
    549
    0
        }
    550
    2.20k
      }
    551
      // Go through the graph to set tensor on matching symbols
    552
    11.0k
      for (i = 0; 2.20ki < stack->rnum; i++8.82k)
    553
    8.82k
      {
    554
    8.82k
        const int d = *(int*)ccv_array_get(stack, i);
    555
        // If exceed range, skip.
    556
    8.82k
        if (d >= ccv_nnc_graph_exec_symbol_count(init->graph) ||
    557
    8.82k
          d >= ccv_nnc_graph_exec_symbol_count(model->graph))
    558
    0
          continue;
    559
    8.82k
        const ccv_nnc_graph_exec_symbol_t src_symbol = {
    560
    8.82k
          .d = d,
    561
    8.82k
          .graph = init->graph
    562
    8.82k
        };
    563
    8.82k
        const ccv_nnc_graph_exec_symbol_t dest_symbol = {
    564
    8.82k
          .d = d,
    565
    8.82k
          .graph = model->graph
    566
    8.82k
        };
    567
    8.82k
        const ccv_nnc_cmd_t src_cmd = ccv_nnc_graph_exec_symbol_cmd(init->graph, src_symbol);
    568
    8.82k
        const ccv_nnc_cmd_t dest_cmd = ccv_nnc_graph_exec_symbol_cmd(model->graph, dest_symbol);
    569
        // If the name doesn't match, skip.
    570
    8.82k
        if (dest_cmd.cmd != src_cmd.cmd && src_cmd.cmd != CCV_NNC_NOOP0)
    571
    0
          continue;
    572
        // Now get all the inputs and outputs, if matches, set them.
    573
    8.82k
        const int* src_inputs;
    574
    8.82k
        int src_input_size;
    575
    8.82k
        const int* src_outputs;
    576
    8.82k
        int src_output_size;
    577
    8.82k
        ccv_nnc_graph_exec_symbol_io(init->graph, src_symbol, &src_inputs, &src_input_size, &src_outputs, &src_output_size);
    578
    8.82k
        const int* dest_inputs;
    579
    8.82k
        int dest_input_size;
    580
    8.82k
        const int* dest_outputs;
    581
    8.82k
        int dest_output_size;
    582
    8.82k
        ccv_nnc_graph_exec_symbol_io(model->graph, dest_symbol, &dest_inputs, &dest_input_size, &dest_outputs, &dest_output_size);
    583
        // We may have unmatched input / output size because this is the minimizer and it has
    584
        // different saved_aux (for example, when we shrunk with CMD_NOOP).
    585
    8.82k
        if (src_input_size != dest_input_size)
    586
    0
          continue;
    587
    8.82k
        if (src_output_size != dest_output_size)
    588
    0
          continue;
    589
    8.82k
        ccv_nnc_graph_exec_symbol_set(model->graph, dest_symbol, src_cmd);
    590
        // There may be mismatches of the source tensor symbols and destination tensor symbols. The reason is because
    591
        // we may later passed-in the minimizer, therefore, we may allocate tensors for minimizer later in the original
    592
        // graph whereas in the newly created graph, it is streamlined (the minimizer exists from the beginning). That
    593
        // will make the order of tensor symbols creation different, therefore, exact which tensor is which wrong as
    594
        // well. However, set a new minimizer won't change the exec symbol ordering, because we never create new exec
    595
        // symbols after gradient init step. Changing a new minimizer just updated that exec symbols setting, it is not
    596
        // a new exec symbol.
    597
    33.7k
        for (j = 0; j < src_input_size; j++24.8k)
    598
    24.8k
          if (src_inputs[j] >= 0)
    599
    20.4k
            _ccv_nnc_tensor_symbol_reinit(init->graph, model->graph, src_inputs[j], dest_inputs[j]);
    600
    22.4k
        for (j = 0; j < src_output_size; j++13.6k)
    601
    13.6k
          if (src_outputs[j] >= 0)
    602
    13.6k
            _ccv_nnc_tensor_symbol_reinit(init->graph, model->graph, src_outputs[j], dest_outputs[j]);
    603
    8.82k
      }
    604
    2.20k
      ccv_array_free(stack);
    605
      // After this, we get all tensors in the model graph resolved through tensor_auto.
    606
    2.20k
      ccv_nnc_symbolic_graph_tensor_auto(model->graph, TRAVERSE_FULL);
    607
      // Verify symbols we get matches.
    608
    2.20k
      const int parameter_size = compiled_data->parameters->rnum;
    609
    4.61k
      for (i = 0; i < parameter_size; i++2.41k)
    610
    2.41k
        { assert(((ccv_nnc_tensor_symbol_t*)ccv_array_get(compiled_data->parameters, i))->d == ((ccv_nnc_tensor_symbol_t*)ccv_array_get(init->compiled_data->parameters, i))->d); }
    611
    2.20k
      const int internal_size = compiled_data->internals->rnum;
    612
    2.20k
      for (i = 0; i < internal_size; i++0)
    613
    0
        { assert(((ccv_nnc_tensor_symbol_t*)ccv_array_get(compiled_data->internals, i))->d == ((ccv_nnc_tensor_symbol_t*)ccv_array_get(init->compiled_data->internals, i))->d); }
    614
      // Go through compiled data.
    615
    2.20k
      if (compiled_data->tensor_arena)
    616
    2.20k
      {
    617
    2.20k
        const int flag = ccv_nnc_tensor_arena_reinit(compiled_data->tensor_arena, model->graph);
    618
    2.20k
        if (flag == 0 && compiled_data->graph_exec_arena)
    619
    2.20k
        {
    620
    2.20k
          ccv_nnc_graph_exec_reinit(compiled_data->graph_exec_arena, compiled_data->graph, model->graph);
    621
          // Since we will reinit, if we previously set is_test, we need to set it again.
    622
    2.20k
          if (compiled_data->is_test)
    623
    1
          {
    624
    1
            const int parallel_count = ccv_max(model->parallel_count, 1);
    625
    1
            ccv_nnc_graph_exec_update_t update = {
    626
    1
              .parallel_count = parallel_count,
    627
    1
              .graph = model->graph,
    628
    1
              .graph_exec_arena = compiled_data->graph_exec_arena,
    629
    1
            };
    630
    1
            ccv_cnnp_model_set_is_test(model, 1, _ccv_cnnp_cmd_update_for_execs, &update);
    631
    1
          }
    632
    2.20k
        } else
    633
          // Free-up tensor arena & graph exec arena.
    634
    0
          _ccv_cnnp_compiled_data_graph_free(compiled_data);
    635
    2.20k
      }
    636
      // There are other compiled graphs, for accum and apply gradients.
    637
      // However, the main conclusion is, these absorb operations shouldn't impact parameters.
    638
      // Thus, it won't impact the shape of gradients (only outgrad). Since for outgrad, we
    639
      // don't allocate ourselves, it is not a concern. For normal gradients, the shape cannot
    640
      // be changed otherwise parameters' shape will be meaningless. The same goes to internals.
    641
      // That is why we don't update these compiled graphs at all this point.
    642
      // Free the model, we've already "absorbed" it.
    643
    2.20k
      ccv_cnnp_model_free(init);
    644
    2.20k
    }
    645
    646
    void ccv_cnnp_model_compile(ccv_cnnp_model_t* const model, const ccv_nnc_tensor_param_t* const inputs, const int input_size, const ccv_nnc_cmd_t minimizer, const ccv_nnc_cmd_t loss)
    647
    2.31k
    {
    648
    2.31k
      assert(input_size == model->input_size || model->input_size == 0);
    649
    2.31k
      if (model->input_size == 0)
    650
    10
        model->input_size = input_size;
    651
    2.31k
      if (!model->graph) // The graph is not compiled yet.
    652
    109
      {
    653
    109
        model->graph = ccv_nnc_symbolic_graph_new();
    654
    109
        _ccv_cnnp_model_compile(model, inputs, input_size, loss);
    655
    109
        assert(model->compiled_data);
    656
    109
        int i, flag = 0;
    657
    274
        for (i = 0; !flag && i < input_size250; i++165)
    658
    165
          flag = (CCV_TENSOR_GET_MEMORY(inputs[i].type) == CCV_TENSOR_GPU_MEMORY);
    659
        // If inputs are from GPU, stream type is GPU.
    660
    109
        model->compiled_data->stream_type = flag ? CCV_STREAM_CONTEXT_GPU24 : CCV_STREAM_CONTEXT_CPU85;
    661
    109
        model->compiled_data->minimize.minimizer = minimizer;
    662
    109
        model->compiled_data->minimize.max_saved_aux_size = ccv_nnc_minimizer_saved_aux_size(minimizer);
    663
    2.20k
      } else {
    664
        // Now, finally fill in this part. If the graph is already compiled, we make a copy of the model.
    665
        // And then absorb the "new model" to the old one.
    666
    2.20k
        ccv_cnnp_model_t* const init = ccv_cnnp_model_copy(model, model->is_trainable);
    667
    2.20k
        ccv_cnnp_model_absorb(model, init, inputs, input_size);
    668
        // Reset minimizer.
    669
    2.20k
        ccv_cnnp_model_set_minimizer(model, minimizer, 1, 0, 0);
    670
    2.20k
      }
    671
    2.31k
    }
    672
    673
    ccv_cnnp_model_t* ccv_cnnp_model_copy(const ccv_cnnp_model_t* const model, const int is_trainable)
    674
    2.20k
    {
    675
    2.20k
      ccv_cnnp_model_t* const new_model = _ccv_cnnp_model_copy(model, 0);
    676
    2.20k
      new_model->is_trainable = is_trainable;
    677
    2.20k
      return new_model;
    678
    2.20k
    }
    679
    680
    void ccv_cnnp_model_tensor_auto(ccv_cnnp_model_t* const model, ccv_nnc_tensor_param_t* const outputs, const int output_size)
    681
    4.45k
    {
    682
    4.45k
      assert(model->graph);
    683
    4.45k
      assert(output_size == model->output_size);
    684
    4.45k
      ccv_nnc_symbolic_graph_t* const graph = model->graph;
    685
    4.45k
      ccv_nnc_symbolic_graph_tensor_auto(graph, TRAVERSE_FULL);
    686
    4.45k
      int i;
    687
    8.92k
      for (i = 0; i < output_size; i++4.47k)
    688
    4.47k
      {
    689
    4.47k
        assert(model->outputs[i].d != CCV_NNC_NO_TENSOR_SYMBOL);
    690
    4.47k
        outputs[i] = ccv_nnc_tensor_symbol_params(graph, model->outputs[i]);
    691
    4.47k
      }
    692
    4.45k
    }
    693
    694
    void ccv_cnnp_model_set_workspace_size(ccv_cnnp_model_t* const model, size_t workspace_size)
    695
    3
    {
    696
    3
      if (workspace_size == model->workspace_size)
    697
    0
        return;
    698
    3
      model->workspace_size = workspace_size;
    699
    3
      ccv_cnnp_compiled_data_t* const compiled_data = model->compiled_data;
    700
    3
      if (compiled_data && compiled_data->graph)
    701
    0
        ccv_nnc_graph_autotune(compiled_data->graph, workspace_size, 0, TRAVERSE_FULL);
    702
    3
    }
    703
    704
    size_t ccv_cnnp_model_workspace_size(ccv_cnnp_model_t* const model)
    705
    0
    {
    706
    0
      return model->workspace_size;
    707
    0
    }
    708
    709
    void ccv_cnnp_model_set_data_parallel(ccv_cnnp_model_t* const model, const int parallel)
    710
    15
    {
    711
    15
      if (parallel == 0)
    712
    0
        model->parallel_count = ccv_nnc_device_count(CCV_STREAM_CONTEXT_GPU);
    713
    15
      else
    714
    15
        model->parallel_count = parallel;
    715
    15
      ccv_cnnp_compiled_data_t* const compiled_data = model->compiled_data;
    716
    15
      if (compiled_data)
    717
    11
        { assert(!compiled_data->graph); }
    718
    15
    }
    719
    720
    void ccv_cnnp_model_set_max_concurrency(ccv_cnnp_model_t* const model, const int max_stream_count)
    721
    0
    {
    722
    0
      model->max_stream_count = max_stream_count;
    723
    0
      ccv_cnnp_compiled_data_t* const compiled_data = model->compiled_data;
    724
    0
      if (compiled_data)
    725
    0
        { assert(!compiled_data->graph); }
    726
    0
    }
    727
    728
    void ccv_cnnp_model_set_memory_compression(ccv_cnnp_model_t* const model, const int memory_compression)
    729
    0
    {
    730
    0
      model->memory_compression = memory_compression;
    731
    0
      ccv_cnnp_compiled_data_t* const compiled_data = model->compiled_data;
    732
    0
      if (compiled_data)
    733
    0
        { assert(!compiled_data->graph); }
    734
    0
    }
    735
    736
    void ccv_cnnp_model_set_memory_reduction(ccv_cnnp_model_t* const model, const int memory_reduction)
    737
    0
    {
    738
    0
      model->memory_reduction = memory_reduction;
    739
    0
      ccv_cnnp_compiled_data_t* const compiled_data = model->compiled_data;
    740
    0
      if (compiled_data)
    741
    0
        { assert(!compiled_data->graph); }
    742
    0
    }
    743
    744
    void ccv_cnnp_model_set_gradient_checkpointing(ccv_cnnp_model_t* const model, const int gradient_checkpointing)
    745
    2
    {
    746
    2
      model->gradient_checkpointing = gradient_checkpointing;
    747
    2
    }
    748
    749
    int ccv_cnnp_model_gradient_checkpointing(ccv_cnnp_model_t* const model)
    750
    0
    {
    751
    0
      return model->gradient_checkpointing;
    752
    0
    }
    753
    754
    typedef struct {
    755
      int parallel_count;
    756
      ccv_nnc_symbolic_graph_t* graph;
    757
      ccv_cnnp_compiled_data_t* compiled_data;
    758
      ccv_nnc_tensor_arena_t* tensor_arena;
    759
    } ccv_nnc_tensor_init_states_t;
    760
    761
    static int _ccv_cnnp_any_to_init(const ccv_cnnp_compiled_data_t* const compiled_data)
    762
    111
    {
    763
    111
      int i;
    764
    111
      const uint32_t* const init_v = CCV_NNC_INIT_V(compiled_data->tensors_init.v);
    765
    195
      for (i = 0; i < compiled_data->parameters->rnum; i++84)
    766
    123
      {
    767
    123
        const int d = ((ccv_nnc_tensor_symbol_t*)ccv_array_get(compiled_data->parameters, i))->d;
    768
    123
        if (!(init_v[d >> 5] & (1u << (d & 0x1f))))
    769
    39
          return 1;
    770
    123
      }
    771
    72
      for (i = 0; i < compiled_data->internals->rnum; i++0)
    772
    6
      {
    773
    6
        const int d = ((ccv_nnc_tensor_symbol_t*)ccv_array_get(compiled_data->internals, i))->d;
    774
    6
        if (!(init_v[d >> 5] & (1u << (d & 0x1f))))
    775
    6
          return 1;
    776
    6
      }
    777
    66
      return 0;
    778
    72
    }
    779
    780
    static void _ccv_cnnp_init_states_for_tensors(void* const context, const ccv_nnc_cmd_t cmd, const ccv_nnc_hint_t hint, const int flags, ccv_nnc_tensor_t* const input, const ccv_nnc_tensor_symbol_t output_symbol)
    781
    341
    {
    782
    341
      ccv_nnc_tensor_init_states_t* const tensor_init_states = (ccv_nnc_tensor_init_states_t*)context;
    783
    341
      ccv_nnc_tensor_arena_t* const tensor_arena = tensor_init_states->tensor_arena;
    784
    341
      ccv_nnc_tensor_t* const output_tensor = ccv_nnc_tensor_from_symbol(tensor_arena, output_symbol);
    785
    341
      if (!output_tensor)
    786
    0
        return;
    787
    341
      const int d = output_symbol.d;
    788
    341
      assert(d < tensor_init_states->compiled_data->tensors_init.size);
    789
    341
      uint32_t* const init_v = CCV_NNC_INIT_V(tensor_init_states->compiled_data->tensors_init.v);
    790
    341
      if (init_v[d >> 5] & (1u << (d & 0x1f)))
    791
    34
        return;
    792
    307
      init_v[d >> 5] |= (1u << (d & 0x1f));
    793
    307
      ccv_nnc_cmd_exec(cmd, hint, flags, &input, input ? 116 : 0291, &output_tensor, 1, 0);
    794
    307
      const ccv_nnc_symbolic_graph_t* const graph = tensor_init_states->graph;
    795
    307
      const int parallel_count = tensor_init_states->parallel_count;
    796
    307
      int i;
    797
    787
      for (i = 1; i < parallel_count; i++480)
    798
    480
      {
    799
    480
        ccv_nnc_tensor_t* const copy = ccv_nnc_tensor_from_symbol(tensor_arena, ccv_nnc_tensor_symbol_copy(graph, output_symbol, i));
    800
    480
        if (copy)
    801
    480
          ccv_nnc_cmd_exec(CMD_DATA_TRANSFER_FORWARD(), ccv_nnc_no_hint, 0, &output_tensor, 1, &copy, 1, 0);
    802
    480
      }
    803
    307
    }
    804
    805
    // This method can only handle cases we added new tensors and exec, never delete. This invariant is true because
    806
    // we setup everything (including calling simplify method) in ccv_cnnp_model_compile method, before this rewind setup.
    807
    static void _ccv_cnnp_model_rewind_graph(ccv_cnnp_model_t* const model)
    808
    2
    {
    809
    2
      assert(model->graph);
    810
    2
      assert(model->compiled_data);
    811
    2
      ccv_cnnp_compiled_data_t* const compiled_data = model->compiled_data;
    812
    2
      assert(compiled_data->rewindables);
    813
    2
      int i;
    814
    51
      for (i = 0; i < compiled_data->rewindables->rnum; i++49)
    815
    49
      {
    816
    49
        const ccv_cnnp_rewind_symbol_t* const rewind_symbol = (ccv_cnnp_rewind_symbol_t*)ccv_array_get(compiled_data->rewindables, i);
    817
    49
        if (rewind_symbol->type == CCV_CNNP_REWIND_GRAPH_EXEC)
    818
    16
          ccv_nnc_graph_exec_symbol_free(model->graph, rewind_symbol->graph_exec);
    819
    33
        else if (rewind_symbol->type == CCV_CNNP_REWIND_TENSOR)
    820
    33
          ccv_nnc_tensor_symbol_free(model->graph, rewind_symbol->tensor);
    821
    49
      }
    822
    2
      ccv_array_clear(compiled_data->rewindables);
    823
    2
      ccv_nnc_graph_exec_symbol_autogen(model->graph, 0, 0, CCV_NNC_AUTOGEN_SOURCES_AND_DESTINATIONS);
    824
    2
    }
    825
    826
    static void _ccv_cnnp_model_tensor_symbol_new_hook(void* context, const ccv_nnc_tensor_symbol_t symbol, const ccv_nnc_tensor_param_t info, const char* const name)
    827
    6.14k
    {
    828
    6.14k
      const ccv_cnnp_rewind_symbol_t rewind_symbol = {
    829
    6.14k
        .type = CCV_CNNP_REWIND_TENSOR,
    830
    6.14k
        .tensor = symbol
    831
    6.14k
      };
    832
    6.14k
      ccv_array_t* const rewind_symbols = (ccv_array_t*)context;
    833
    6.14k
      ccv_array_push(rewind_symbols, &rewind_symbol);
    834
    6.14k
    }
    835
    836
    static void _ccv_cnnp_model_tensor_symbol_alias_new_hook(void* context, const ccv_nnc_tensor_symbol_t symbol, const ccv_nnc_tensor_symbol_t from_symbol, const int ofs[CCV_NNC_MAX_DIM_ALLOC], const int inc[CCV_NNC_MAX_DIM_ALLOC], const ccv_nnc_tensor_param_t info, const char* const name)
    837
    476
    {
    838
    476
      const ccv_cnnp_rewind_symbol_t rewind_symbol = {
    839
    476
        .type = CCV_CNNP_REWIND_TENSOR,
    840
    476
        .tensor = symbol
    841
    476
      };
    842
    476
      ccv_array_t* const rewind_symbols = (ccv_array_t*)context;
    843
    476
      ccv_array_push(rewind_symbols, &rewind_symbol);
    844
    476
    }
    845
    846
    static void _ccv_cnnp_model_graph_exec_symbol_new_hook(void* context, const ccv_nnc_graph_exec_symbol_t symbol, const ccv_nnc_cmd_t cmd, const ccv_nnc_tensor_symbol_t* const inputs, const int input_size, const ccv_nnc_tensor_symbol_t* const outputs, const int output_size, const char* const name)
    847
    2.34k
    {
    848
    2.34k
      const ccv_cnnp_rewind_symbol_t rewind_symbol = {
    849
    2.34k
        .type = CCV_CNNP_REWIND_GRAPH_EXEC,
    850
    2.34k
        .graph_exec = symbol
    851
    2.34k
      };
    852
    2.34k
      ccv_array_t* const rewind_symbols = (ccv_array_t*)context;
    853
    2.34k
      ccv_array_push(rewind_symbols, &rewind_symbol);
    854
    2.34k
    }
    855
    856
    static void _ccv_cnnp_model_graph_symbol_exec_set_for_graph_exec_arena(const ccv_nnc_graph_exec_arena_t* const graph_exec_arena, const int parallel_count, const ccv_nnc_graph_exec_symbol_t exec_symbol, const ccv_nnc_cmd_t cmd, ccv_nnc_symbolic_graph_t* const symbolic_graph)
    857
    35.0k
    {
    858
    35.0k
      ccv_nnc_graph_exec_t const update_exec = ccv_nnc_graph_exec_from_symbol(graph_exec_arena, exec_symbol);
    859
    35.0k
      if (!CCV_NO_GRAPH_EXEC(update_exec))
    860
    19.9k
        ccv_nnc_graph_exec_set(update_exec.graph, update_exec, cmd);
    861
    35.0k
      int i;
    862
    49.9k
      for (i = 1; i < parallel_count; i++14.8k)
    863
    14.8k
      {
    864
    14.8k
        ccv_nnc_graph_exec_symbol_t copy_symbol = ccv_nnc_graph_exec_symbol_copy(symbolic_graph, exec_symbol, i);
    865
    14.8k
        const ccv_nnc_graph_exec_t copy = ccv_nnc_graph_exec_from_symbol(graph_exec_arena, copy_symbol);
    866
    14.8k
        if (!CCV_NO_GRAPH_EXEC(copy))
    867
    14.6k
          ccv_nnc_graph_exec_set(copy.graph, copy, cmd);
    868
    14.8k
      }
    869
    35.0k
    }
    870
    871
    static void _ccv_cnnp_model_graph_exec_symbol_set(ccv_nnc_symbolic_graph_t* const symbolic_graph, ccv_cnnp_compiled_data_t* const compiled_data, const int parallel_count, const ccv_nnc_graph_exec_symbol_t exec_symbol, const ccv_nnc_cmd_t cmd)
    872
    20.0k
    {
    873
    20.0k
      assert(compiled_data);
    874
    20.0k
      assert(symbolic_graph);
    875
    20.0k
      ccv_nnc_graph_exec_symbol_set(symbolic_graph, exec_symbol, cmd);
    876
    20.0k
      int i;
    877
    35.0k
      for (i = 1; i < parallel_count; i++14.9k)
    878
    14.9k
      {
    879
    14.9k
        ccv_nnc_graph_exec_symbol_t copy_symbol = ccv_nnc_graph_exec_symbol_copy(symbolic_graph, exec_symbol, i);
    880
    14.9k
        if (copy_symbol.graph)
    881
    14.8k
          ccv_nnc_graph_exec_symbol_set(symbolic_graph, copy_symbol, cmd);
    882
    14.9k
      }
    883
    20.0k
      ccv_nnc_graph_exec_arena_t* const graph_exec_arena = compiled_data->graph_exec_arena;
    884
    20.0k
      if (graph_exec_arena)
    885
    20.0k
        _ccv_cnnp_model_graph_symbol_exec_set_for_graph_exec_arena(graph_exec_arena, parallel_count, exec_symbol, cmd, symbolic_graph);
    886
      // Skip backward graph exec arena because it is for a specific accum symbolic graph, not the main graph (model->graph)
    887
    20.0k
      ccv_nnc_graph_exec_arena_t* const gradient_graph_exec_arena = compiled_data->apply_gradients.graph_exec_arena;
    888
    20.0k
      if (gradient_graph_exec_arena)
    889
    15.0k
        _ccv_cnnp_model_graph_symbol_exec_set_for_graph_exec_arena(gradient_graph_exec_arena, parallel_count, exec_symbol, cmd, symbolic_graph);
    890
    20.0k
    }
    891
    892
    static int _ccv_cnnp_set_minimizer_for_parameter(ccv_nnc_symbolic_graph_t* const graph, ccv_cnnp_compiled_data_t* const compiled_data, ccv_nnc_graph_exec_symbol_t* const update_nodes, ccv_nnc_tensor_symbol_t* const updated_parameters, ccv_nnc_tensor_symbol_map_t* const saved_aux, const int parallel_count, const ccv_nnc_cmd_t minimizer, const int saved_aux_size, const int max_saved_aux_size, const int parameter_indice)
    893
    20.0k
    {
    894
    20.0k
      int this_parameter_flag = 0;
    895
    20.0k
      if (update_nodes[parameter_indice].d == CCV_NNC_NO_TENSOR_SYMBOL)
    896
    0
        return this_parameter_flag;
    897
    20.0k
      const ccv_nnc_cmd_t old_minimizer = ccv_nnc_graph_exec_symbol_cmd(graph, update_nodes[parameter_indice]);
    898
    20.0k
      int j, k;
    899
      // For no-op, we can preserve previous saved_aux_size.
    900
    20.0k
      if (old_minimizer.cmd != minimizer.cmd && minimizer.cmd != CCV_NNC_NOOP71)
    901
    67
      {
    902
        // If the old minimizer is a noop, then the old_saved_aux_size should be whatever its previous
    903
        // saved_aux_size is, otherwise we will reinit the saved_aux repeatedly if you switch between
    904
        // noop and a minimizer. We don't want that because we do that in high-level frameworks to
    905
        // make sure some model parameters don't update if we don't want them to.
    906
    67
        int old_saved_aux_size;
    907
    67
        if (old_minimizer.cmd == CCV_NNC_NOOP)
    908
    67
        {
    909
    67
          int input_size;
    910
    67
          ccv_nnc_graph_exec_symbol_io(graph, update_nodes[parameter_indice], 0, &input_size, 0, 0);
    911
    67
          if (input_size < 2) // This is not legit.
    912
    0
            old_saved_aux_size = ccv_nnc_minimizer_saved_aux_size(old_minimizer);
    913
    67
          else // See ccv_nnc_minimizer_saved_aux_size, the saved_aux is inputs excluding gradients and parameters.
    914
    67
            old_saved_aux_size = input_size - 2;
    915
    67
        } else
    916
    0
          old_saved_aux_size = ccv_nnc_minimizer_saved_aux_size(old_minimizer);
    917
    67
        if (old_saved_aux_size != saved_aux_size)
    918
    65
        {
    919
    65
          this_parameter_flag = 1;
    920
    65
          if (saved_aux_size > old_saved_aux_size)
    921
    65
          {
    922
            // Allocate new tensor symbols.
    923
    65
            const ccv_nnc_tensor_param_t info = ccv_nnc_tensor_symbol_params(graph, updated_parameters[parameter_indice]);
    924
    189
            for (j = old_saved_aux_size; j < saved_aux_size; j++124)
    925
    124
            {
    926
    124
              saved_aux[parameter_indice * max_saved_aux_size + j].source = ccv_nnc_tensor_symbol_new(graph, info, 0);
    927
    124
              saved_aux[parameter_indice * max_saved_aux_size + j].destination = ccv_nnc_tensor_symbol_new(graph, info, 0);
    928
    124
              const int device_id = CCV_TENSOR_GET_DEVICE_ID(info.type);
    929
    460
              for (k = 1; k < parallel_count; k++336)
    930
    336
              {
    931
    336
                ccv_nnc_tensor_param_t dev_info = info;
    932
    336
                if (k != device_id)
    933
    336
                  CCV_TENSOR_SET_DEVICE_ID(dev_info.type, k);
    934
    0
                else
    935
    0
                  CCV_TENSOR_SET_DEVICE_ID(dev_info.type, 0);
    936
    336
                const ccv_nnc_tensor_symbol_t src_copy = ccv_nnc_tensor_symbol_new(graph, dev_info, 0);
    937
    336
                const ccv_nnc_tensor_symbol_t dest_copy = ccv_nnc_tensor_symbol_new(graph, dev_info, 0);
    938
    336
                ccv_nnc_tensor_symbol_set_copy(graph, saved_aux[parameter_indice * max_saved_aux_size + j].source, k, src_copy);
    939
    336
                ccv_nnc_tensor_symbol_set_copy(graph, saved_aux[parameter_indice * max_saved_aux_size + j].destination, k, dest_copy);
    940
    336
              }
    941
    124
            }
    942
    65
          } else {
    943
    0
            for (j = saved_aux_size; j < old_saved_aux_size; j++)
    944
    0
            {
    945
    0
              for (k = 1; k < parallel_count; k++)
    946
    0
              {
    947
    0
                const ccv_nnc_tensor_symbol_t src_copy = ccv_nnc_tensor_symbol_copy(graph, saved_aux[parameter_indice * max_saved_aux_size + j].source, k);
    948
    0
                if (src_copy.d >= 0)
    949
    0
                {
    950
    0
                  ccv_nnc_tensor_symbol_free(graph, src_copy);
    951
    0
                  ccv_nnc_tensor_symbol_set_copy(graph, saved_aux[parameter_indice * max_saved_aux_size + j].source, k, NO_TENSOR_SYMBOL);
    952
    0
                }
    953
    0
                const ccv_nnc_tensor_symbol_t dest_copy = ccv_nnc_tensor_symbol_copy(graph, saved_aux[parameter_indice * max_saved_aux_size + j].destination, k);
    954
    0
                if (dest_copy.d >= 0)
    955
    0
                {
    956
    0
                  ccv_nnc_tensor_symbol_free(graph, dest_copy);
    957
    0
                  ccv_nnc_tensor_symbol_set_copy(graph, saved_aux[parameter_indice * max_saved_aux_size + j].destination, k, NO_TENSOR_SYMBOL);
    958
    0
                }
    959
    0
              }
    960
    0
              ccv_nnc_tensor_symbol_free(graph, saved_aux[parameter_indice * max_saved_aux_size + j].source);
    961
    0
              ccv_nnc_tensor_symbol_free(graph, saved_aux[parameter_indice * max_saved_aux_size + j].destination);
    962
    0
              saved_aux[parameter_indice * max_saved_aux_size + j].source = saved_aux[parameter_indice * max_saved_aux_size + j].destination = NO_TENSOR_SYMBOL;
    963
    0
            }
    964
    0
          }
    965
    65
        }
    966
    67
      }
    967
    20.0k
      _ccv_cnnp_model_graph_exec_symbol_set(graph, compiled_data, parallel_count, update_nodes[parameter_indice], minimizer);
    968
    20.0k
      if (this_parameter_flag)
    969
    65
      {
    970
    65
        ccv_nnc_tensor_symbol_t update_inputs[saved_aux_size + 2];
    971
    65
        ccv_nnc_tensor_symbol_t update_outputs[saved_aux_size + 1];
    972
    65
        const int* inputs = 0;
    973
    65
        int input_size = 0;
    974
    65
        ccv_nnc_graph_exec_symbol_io(graph, update_nodes[parameter_indice], &inputs, &input_size, 0, 0);
    975
    65
        assert(input_size >= 1);
    976
    65
        update_inputs[0].d = inputs[0];
    977
    65
        update_inputs[0].graph = graph;
    978
    65
        update_inputs[1].d = inputs[1];
    979
    65
        update_inputs[1].graph = graph;
    980
    65
        update_outputs[0] = updated_parameters[parameter_indice];
    981
    189
        for (j = 0; j < saved_aux_size; j++124)
    982
    124
        {
    983
    124
          update_inputs[j + 2] = saved_aux[parameter_indice * max_saved_aux_size + j].source;
    984
    124
          update_outputs[j + 1] = saved_aux[parameter_indice * max_saved_aux_size + j].destination;
    985
    124
        }
    986
    65
        ccv_nnc_graph_exec_symbol_set_io(graph, update_nodes[parameter_indice], update_inputs, saved_aux_size + 2, update_outputs, saved_aux_size + 1);
    987
    233
        for (k = 1; k < parallel_count; k++168)
    988
    168
        {
    989
    168
          const ccv_nnc_graph_exec_symbol_t copy = ccv_nnc_graph_exec_symbol_copy(graph, update_nodes[parameter_indice], k);
    990
    168
          assert(copy.d >= 0);
    991
    168
          ccv_nnc_graph_exec_symbol_io(graph, copy, &inputs, &input_size, 0, 0);
    992
    168
          assert(input_size >= 1);
    993
    168
          update_inputs[0].d = inputs[0];
    994
    168
          update_inputs[0].graph = graph;
    995
    168
          update_inputs[1].d = inputs[1];
    996
    168
          update_inputs[1].graph = graph;
    997
    168
          update_outputs[0] = ccv_nnc_tensor_symbol_copy(graph, updated_parameters[parameter_indice], k);
    998
    504
          for (j = 0; j < saved_aux_size; j++336)
    999
    336
          {
    1000
    336
            update_inputs[j + 2] = ccv_nnc_tensor_symbol_copy(graph, saved_aux[parameter_indice * max_saved_aux_size + j].source, k);
    1001
    336
            update_outputs[j + 1] = ccv_nnc_tensor_symbol_copy(graph, saved_aux[parameter_indice * max_saved_aux_size + j].destination, k);
    1002
    336
          }
    1003
    168
          ccv_nnc_graph_exec_symbol_set_io(graph, copy, update_inputs, saved_aux_size + 2, update_outputs, saved_aux_size + 1);
    1004
    168
        }
    1005
    65
      }
    1006
    20.0k
      return this_parameter_flag;
    1007
    20.0k
    }
    1008
    1009
    typedef struct {
    1010
      int parameter_size;
    1011
      ccv_nnc_cmd_t minimizer;
    1012
      ccv_cnnp_model_io_t parameters[1];
    1013
    } ccv_cnnp_set_minimizer_for_parameter_t;
    1014
    1015
    static int _ccv_cnnp_apply_parameters_with_minimizer(ccv_cnnp_model_t* const model)
    1016
    296
    {
    1017
    296
      ccv_cnnp_compiled_data_t* const compiled_data = model->compiled_data;
    1018
    296
      assert(compiled_data);
    1019
    296
      const int max_saved_aux_size = compiled_data->minimize.max_saved_aux_size;
    1020
      // We update all parameters, at this point, we have one minimizer.
    1021
    296
      const int parameter_size = compiled_data->parameters->rnum;
    1022
    296
      ccv_nnc_graph_exec_symbol_t* const update_nodes = compiled_data->update_nodes;
    1023
    296
      ccv_nnc_symbolic_graph_t* const symbolic_graph = model->graph;
    1024
    296
      assert(symbolic_graph);
    1025
    296
      const int parallel_count = _ccv_cnnp_model_root_parallel_count(model);
    1026
    296
      assert(_ccv_cnnp_model_effective_parallel_count(model) == parallel_count && "local replicated stateful models only support forward / no-grad evaluation for now");
    1027
    296
      ccv_array_t* const parameters = compiled_data->minimize.parameters;
    1028
    296
      ccv_array_t* const parameter_indices = ccv_array_new(sizeof(int), 0, 0);
    1029
    296
      int i, j, flag = 0;
    1030
    301
      for (i = 0; i < parameters->rnum; i++5)
    1031
    5
      {
    1032
    5
        ccv_cnnp_set_minimizer_for_parameter_t* const set_minimizer_for_parameter = *(ccv_cnnp_set_minimizer_for_parameter_t**)ccv_array_get(parameters, i);
    1033
    10
        for (j = 0; j < set_minimizer_for_parameter->parameter_size; j++5)
    1034
    5
        {
    1035
    5
          const int param_sel = set_minimizer_for_parameter->parameters[j]->param_sel > 0 ? set_minimizer_for_parameter->parameters[j]->param_sel - 13 : set_minimizer_for_parameter->parameters[j]->param_sel2;
    1036
    5
          assert(set_minimizer_for_parameter->parameters[j]->param_sel != 0);
    1037
    5
          const int old_rnum = parameter_indices->rnum;
    1038
    5
          ccv_cnnp_model_add_to_parameter_indices(set_minimizer_for_parameter->parameters[j]->model, param_sel, parameter_indices);
    1039
    5
          const int param_ref = set_minimizer_for_parameter->parameters[j]->param_ref > 0 ? set_minimizer_for_parameter->parameters[j]->param_ref - 10 : set_minimizer_for_parameter->parameters[j]->param_ref;
    1040
    5
          assert(set_minimizer_for_parameter->parameters[j]->param_ref != 0);
    1041
    5
          if (param_ref >= 0)
    1042
    0
          {
    1043
    0
            assert(param_ref + old_rnum < parameter_indices->rnum);
    1044
    0
            *(int*)ccv_array_get(parameter_indices, old_rnum) = *(int*)ccv_array_get(parameter_indices, param_ref + old_rnum);
    1045
    0
            parameter_indices->rnum = old_rnum + 1;
    1046
    0
          }
    1047
    5
        }
    1048
    5
        const int saved_aux_size = ccv_nnc_minimizer_saved_aux_size(set_minimizer_for_parameter->minimizer);
    1049
        // We may have duplicated indices, but that is OK, we will set it twice.
    1050
    58
        for (j = 0; j < parameter_indices->rnum; j++53)
    1051
    53
        {
    1052
    53
          const int d = *(int*)ccv_array_get(parameter_indices, j);
    1053
    53
          assert(d <= parameter_size);
    1054
    53
          if (_ccv_cnnp_set_minimizer_for_parameter(symbolic_graph, compiled_data, update_nodes, compiled_data->updated_parameters, compiled_data->saved_aux, parallel_count, set_minimizer_for_parameter->minimizer, saved_aux_size, max_saved_aux_size, d))
    1055
    0
            flag = 1;
    1056
    53
        }
    1057
    5
        ccv_array_clear(parameter_indices);
    1058
    5
      }
    1059
    296
      ccv_array_free(parameter_indices);
    1060
    296
      return flag;
    1061
    296
    }
    1062
    1063
    static void _ccv_cnnp_scatter_saved_aux(ccv_nnc_tensor_symbol_map_t* const saved_aux, const int parameter_size, const int old_saved_aux_size, const int new_saved_aux_size)
    1064
    2.25k
    {
    1065
    2.25k
      if (new_saved_aux_size == old_saved_aux_size)
    1066
    2.24k
        return;
    1067
    2.25k
      assert(new_saved_aux_size > old_saved_aux_size)7;
    1068
    7
      int i, j;
    1069
    72
      for (i = parameter_size - 1; i >= 0; i--65)
    1070
    65
      {
    1071
    189
        for (j = new_saved_aux_size - 1; j >= old_saved_aux_size; j--124)
    1072
    124
          saved_aux[i * new_saved_aux_size + j].source = saved_aux[i * new_saved_aux_size + j].destination = NO_TENSOR_SYMBOL;
    1073
    65
        for (j = old_saved_aux_size - 1; j >= 0; j--0)
    1074
    0
          saved_aux[i * new_saved_aux_size + j] = saved_aux[i * old_saved_aux_size + j];
    1075
    65
      }
    1076
    7
    }
    1077
    1078
    static void _ccv_cnnp_model_set_rewindables(ccv_cnnp_model_t* const model)
    1079
    49
    {
    1080
    49
      ccv_cnnp_compiled_data_t* const compiled_data = model->compiled_data;
    1081
    49
      assert(compiled_data);
    1082
    49
      if (!compiled_data->rewindables)
    1083
    49
        compiled_data->rewindables = ccv_array_new(sizeof(ccv_cnnp_rewind_symbol_t), 0, 0);
    1084
    49
      ccv_nnc_tensor_symbol_new_hook(model->graph, _ccv_cnnp_model_tensor_symbol_new_hook, compiled_data->rewindables, 0);
    1085
    49
      ccv_nnc_tensor_symbol_alias_new_hook(model->graph, _ccv_cnnp_model_tensor_symbol_alias_new_hook, compiled_data->rewindables, 0);
    1086
    49
      ccv_nnc_graph_exec_symbol_new_hook(model->graph, _ccv_cnnp_model_graph_exec_symbol_new_hook, compiled_data->rewindables, 0);
    1087
    49
    }
    1088
    1089
    static void _ccv_cnnp_model_gradient_init(ccv_cnnp_model_t* const model, const int gradient_mode, const uint64_t disable_outgrad, ccv_nnc_tensor_t* const* const fits, const int fit_size)
    1090
    2.24k
    {
    1091
    2.24k
      ccv_cnnp_compiled_data_t* const compiled_data = model->compiled_data;
    1092
    2.24k
      assert(compiled_data->gradient_mode == CCV_CNNP_COMPILED_DATA_GRADIENT_NONE);
    1093
    2.24k
      assert(gradient_mode != CCV_CNNP_COMPILED_DATA_GRADIENT_NONE);
    1094
    2.24k
      const int evaluate_to_size = compiled_data->evaluate.to_size;
    1095
    2.24k
      assert(evaluate_to_size > 0);
    1096
    2.24k
      const int parallel_count = _ccv_cnnp_model_root_parallel_count(model);
    1097
    2.24k
      assert(_ccv_cnnp_model_effective_parallel_count(model) == parallel_count && "local replicated stateful models only support forward / no-grad evaluation for now");
    1098
    2.24k
      compiled_data->evaluate.tos = ccrealloc(compiled_data->evaluate.tos, sizeof(ccv_nnc_graph_exec_symbol_t) * evaluate_to_size * parallel_count + sizeof(ccv_nnc_graph_exec_t) * evaluate_to_size * parallel_count);
    1099
    2.24k
      compiled_data->evaluate.to_ops = (ccv_nnc_graph_exec_t*)(compiled_data->evaluate.tos + evaluate_to_size * parallel_count);
    1100
    2.24k
      int i, j;
    1101
    2.24k
      const int output_size = model->output_size;
    1102
    2.24k
      assert(!fits || fit_size == output_size * parallel_count);
    1103
    2.24k
      if (fits)
    1104
    12
        for (i = 0; 6i < output_size; i++6)
    1105
    6
          ccv_nnc_tensor_symbol_set(model->graph, compiled_data->fits[i], fits[i]->info);
    1106
    2.24k
      const int max_saved_aux_size = compiled_data->minimize.max_saved_aux_size;
    1107
    2.24k
      const int parameter_size = compiled_data->parameters->rnum;
    1108
    2.24k
      compiled_data->updated_parameters = (ccv_nnc_tensor_symbol_t*)ccmalloc(sizeof(ccv_nnc_tensor_symbol_t) * parameter_size + sizeof(ccv_nnc_graph_exec_symbol_t) * parameter_size + sizeof(ccv_nnc_tensor_symbol_map_t) * max_saved_aux_size * parameter_size);
    1109
    2.24k
      compiled_data->update_nodes = (ccv_nnc_graph_exec_symbol_t*)(compiled_data->updated_parameters + parameter_size);
    1110
    2.24k
      compiled_data->saved_aux = (ccv_nnc_tensor_symbol_map_t*)(compiled_data->update_nodes + parameter_size);
    1111
    2.24k
      int parameter_size_maybe_more = parameter_size;
    1112
    2.24k
      compiled_data->disable_outgrad = disable_outgrad;
    1113
    2.24k
      int outgrad_size;
    1114
    2.24k
      if (gradient_mode == CCV_CNNP_COMPILED_DATA_GRADIENT_TRAINABLES || model->input_size == 02.23k)
    1115
    9
        outgrad_size = 0;
    1116
    2.23k
      else if (disable_outgrad == CCV_CNNP_DISABLE_OUTGRAD_NONE) // Compute minimize with gradients including inputs.
    1117
    2.23k
        outgrad_size = model->input_size;
    1118
    3
      else {
    1119
    3
        assert(disable_outgrad != CCV_CNNP_DISABLE_OUTGRAD_ALL); // If it is disable all, gradient mode won't be this.
    1120
    3
        outgrad_size = 0;
    1121
    10
        for (i = 0; i < model->input_size; i++7)
    1122
    7
          if (!(disable_outgrad & ((uint64_t)1 << i)))
    1123
    3
            ++outgrad_size;
    1124
    3
      }
    1125
    2.24k
      compiled_data->outgrad_size = outgrad_size;
    1126
    2.24k
      parameter_size_maybe_more += outgrad_size;
    1127
    2.24k
      compiled_data->gradients = (ccv_nnc_tensor_symbol_t*)ccmalloc(sizeof(ccv_nnc_tensor_symbol_t) * parameter_size_maybe_more + sizeof(ccv_nnc_graph_exec_symbol_t) * parameter_size_maybe_more * parallel_count);
    1128
    2.24k
      compiled_data->outgrads = parameter_size_maybe_more > parameter_size ? compiled_data->gradients + parameter_size2.23k : 09;
    1129
    2.24k
      compiled_data->backward.tos = (ccv_nnc_graph_exec_symbol_t*)(compiled_data->gradients + parameter_size_maybe_more);
    1130
    2.24k
      compiled_data->backward.to_size = parameter_size_maybe_more;
    1131
    2.24k
      ccv_nnc_tensor_symbol_t* parameters = (ccv_nnc_tensor_symbol_t*)ccv_array_get(compiled_data->parameters, 0);
    1132
    2.24k
      if (compiled_data->parameter_flags)
    1133
    4
      {
    1134
    4
        parameters = (ccv_nnc_tensor_symbol_t*)ccmalloc(sizeof(ccv_nnc_tensor_symbol_t) * parameter_size);
    1135
    25
        for (i = 0; i < parameter_size; i++21)
    1136
    21
          if (compiled_data->parameter_flags[i >> 6] & ((uint64_t)1 << (i & 63)))
    1137
    14
            parameters[i] = *(ccv_nnc_tensor_symbol_t*)ccv_array_get(compiled_data->parameters, i);
    1138
    7
          else
    1139
    7
            parameters[i] = NO_TENSOR_SYMBOL;
    1140
    4
      }
    1141
    2.24k
      if (gradient_mode == CCV_CNNP_COMPILED_DATA_GRADIENT_TRAINABLES || model->input_size == 02.23k)
    1142
    9
        ccv_nnc_symbolic_graph_minimize(model->graph, compiled_data->minimize.minimizer, compiled_data->f, output_size, parameters, parameter_size, 0, 0, SYMBOLIC_GRAPH_SOURCES(model->graph), SYMBOLIC_GRAPH_DESTINATIONS(model->graph), compiled_data->gradients, compiled_data->updated_parameters, compiled_data->saved_aux, compiled_data->update_nodes);
    1143
    2.23k
      else if (disable_outgrad == CCV_CNNP_DISABLE_OUTGRAD_NONE) // Compute minimize with gradients including inputs.
    1144
    2.23k
        ccv_nnc_symbolic_graph_minimize(model->graph, compiled_data->minimize.minimizer, compiled_data->f, output_size, parameters, parameter_size, model->inputs, model->input_size, SYMBOLIC_GRAPH_SOURCES(model->graph), SYMBOLIC_GRAPH_DESTINATIONS(model->graph), compiled_data->gradients, compiled_data->updated_parameters, compiled_data->saved_aux, compiled_data->update_nodes);
    1145
    3
      else { // Compute minimize with gradients including selected inputs.
    1146
    3
        assert(model->input_size > 0);
    1147
    3
        assert(disable_outgrad != CCV_CNNP_DISABLE_OUTGRAD_ALL); // If it is disable all, gradient mode won't be this.
    1148
    3
        assert(outgrad_size > 0);
    1149
    3
        ccv_nnc_tensor_symbol_t outgrads[outgrad_size];
    1150
    3
        j = 0;
    1151
    10
        for (i = 0; i < model->input_size; i++7)
    1152
    7
          if (!(disable_outgrad & ((uint64_t)1 << i)))
    1153
    3
            outgrads[j++] = model->inputs[i];
    1154
    3
        ccv_nnc_symbolic_graph_minimize(model->graph, compiled_data->minimize.minimizer, compiled_data->f, output_size, parameters, parameter_size, outgrads, outgrad_size, SYMBOLIC_GRAPH_SOURCES(model->graph), SYMBOLIC_GRAPH_DESTINATIONS(model->graph), compiled_data->gradients, compiled_data->updated_parameters, compiled_data->saved_aux, compiled_data->update_nodes);
    1155
    3
      }
    1156
    2.24k
      if (compiled_data->parameter_flags)
    1157
    4
        ccfree(parameters);
    1158
    2.24k
      _ccv_cnnp_scatter_saved_aux(compiled_data->saved_aux, parameter_size, ccv_nnc_minimizer_saved_aux_size(compiled_data->minimize.minimizer), compiled_data->minimize.max_saved_aux_size);
    1159
    2.24k
      if (compiled_data->minimize.parameters)
    1160
    5
        _ccv_cnnp_apply_parameters_with_minimizer(model);
    1161
      // Go through gradient checkpoints to generate tensor inputs for backward pass just before executing the backward pass.
    1162
    2.24k
      ccv_cnnp_model_apply_gradient_checkpoints(compiled_data, model->graph);
    1163
    4.49k
      for (i = 0; i < output_size; i++2.24k)
    1164
    2.24k
      {
    1165
    2.24k
        const ccv_nnc_tensor_symbol_t df = ccv_nnc_tensor_symbol_for_backward(model->graph, compiled_data->f[i]);
    1166
        // Init this to 1 so we can backprop.
    1167
    2.24k
        ccv_nnc_tensor_symbol_set_flags(model->graph, df, CCV_NNC_TENSOR_SYMBOL_INIT_ONES);
    1168
    2.24k
      }
    1169
    2.24k
      compiled_data->backward.to_size = 0;
    1170
    7.16k
      for (i = 0; i < parameter_size_maybe_more; i++4.92k)
    1171
    4.92k
        if (compiled_data->gradients[i].d != CCV_NNC_NO_TENSOR_SYMBOL)
    1172
    4.91k
          compiled_data->backward.tos[compiled_data->backward.to_size++] = ccv_nnc_graph_exec_symbol_for_backward(model->graph, compiled_data->gradients[i]);
    1173
    2.24k
      ccv_nnc_graph_exec_symbol_autogen(model->graph, 0, 0, CCV_NNC_AUTOGEN_ALL_EXECS);
    1174
    2.24k
      ccv_nnc_symbolic_graph_set_destinations(model->graph, compiled_data->update_nodes, parameter_size);
    1175
    4.50k
      for (i = 0; i < parameter_size_maybe_more - parameter_size; i++2.25k)
    1176
    2.25k
      {
    1177
    2.25k
        if (compiled_data->outgrads[i].d < 0) // When we go through input, we might find zero-length inputs, and for these, we cannot have any outgrads.
    1178
    0
          continue;
    1179
    2.25k
        const ccv_nnc_graph_exec_symbol_t outgrad = ccv_nnc_graph_exec_symbol_for_backward(model->graph, compiled_data->outgrads[i]);
    1180
    2.25k
        const int* tos;
    1181
    2.25k
        int to_size;
    1182
    2.25k
        ccv_nnc_graph_exec_symbol_to(model->graph, outgrad, &tos, &to_size);
    1183
    2.25k
        if (to_size == 0) // If this is the end (no minimizers afterwards). We need to attach this as a destination. Otherwise this is covered in update_nodes.
    1184
    14
        {
    1185
    14
          const ccv_nnc_graph_exec_symbol_t* destinations = ccv_nnc_symbolic_graph_destinations(model->graph);
    1186
    14
          const int destination_count = ccv_nnc_symbolic_graph_destination_size(model->graph);
    1187
    14
          int flag = 0;
    1188
    14
          const int outgrad_destination_start = ccv_max(0, destination_count - i);
    1189
    16
          for (j = i - 1; !flag && j >= 014; j--2)
    1190
    2
            if (j + outgrad_destination_start < destination_count)
    1191
    2
              flag = (destinations[j + outgrad_destination_start].d == outgrad.d);
    1192
    14
          if (!flag) // Only if we cannot find it, we add it.
    1193
    12
            ccv_nnc_symbolic_graph_add_destination(model->graph, outgrad);
    1194
    14
        }
    1195
    2.25k
      }
    1196
    2.24k
      if (parallel_count > 1)
    1197
    8
      {
    1198
    8
        ccv_nnc_symbolic_graph_data_parallel(model->graph, parallel_count,
    1199
    8
          0, 0,
    1200
    8
          compiled_data->gradients, parameter_size /* No need to deal with outgrads, we don't allreduce outgrads */,
    1201
    8
          compiled_data->gradients /* We only care about gradients before allreduce, thus, update our current pointers */,
    1202
    8
          0, 0, 0,
    1203
    8
          CCV_NNC_PARALLEL_REDUCE_OP_SUM,
    1204
    8
          SYMBOLIC_GRAPH_SOURCES(model->graph), SYMBOLIC_GRAPH_DESTINATIONS(model->graph));
    1205
    8
        ccv_nnc_graph_exec_symbol_autogen(model->graph, 0, 0, CCV_NNC_AUTOGEN_SOURCES_AND_DESTINATIONS);
    1206
    16
        for (i = 0; i < evaluate_to_size; i++8)
    1207
    32
          for (j = 1; 8j < parallel_count; j++24)
    1208
    24
          {
    1209
    24
            const ccv_nnc_graph_exec_symbol_t copy = ccv_nnc_graph_exec_symbol_copy(model->graph, compiled_data->evaluate.tos[i], j);
    1210
    24
            if (copy.d != CCV_NNC_NO_GRAPH_EXEC_SYMBOL)
    1211
    24
              compiled_data->evaluate.tos[compiled_data->evaluate.to_size++] = copy;
    1212
    24
          }
    1213
    8
        const int backward_to_size = compiled_data->backward.to_size;
    1214
    146
        for (i = 0; i < backward_to_size; i++138)
    1215
    552
          for (j = 1; 138j < parallel_count; j++414)
    1216
    414
          {
    1217
    414
            const ccv_nnc_graph_exec_symbol_t copy = ccv_nnc_graph_exec_symbol_copy(model->graph, compiled_data->backward.tos[i], j);
    1218
    414
            if (copy.d != CCV_NNC_NO_GRAPH_EXEC_SYMBOL)
    1219
    414
              compiled_data->backward.tos[compiled_data->backward.to_size++] = copy;
    1220
    414
          }
    1221
    8
      }
    1222
      // Only use memory compression if we are in gradient parameter mode.
    1223
    2.24k
      if (gradient_mode == CCV_CNNP_COMPILED_DATA_GRADIENT_TRAINABLES || gradient_mode == CCV_CNNP_COMPILED_DATA_GRADIENT_TRAINABLES_AND_INPUTS2.23k)
    1224
    2.24k
      {
    1225
    2.24k
        if (model->memory_compression)
    1226
    0
          ccv_nnc_symbolic_graph_memory_compression(model->graph, SYMBOLIC_GRAPH_SOURCES(model->graph), SYMBOLIC_GRAPH_DESTINATIONS(model->graph));
    1227
    2.24k
        if (model->memory_reduction)
    1228
    0
          ccv_nnc_symbolic_graph_memory_reduction(model->graph, SYMBOLIC_GRAPH_SOURCES(model->graph), SYMBOLIC_GRAPH_DESTINATIONS(model->graph));
    1229
    2.24k
      }
    1230
    2.24k
      compiled_data->backward.to_size = _ccv_nnc_array_dedup_graph_exec_symbols(compiled_data->backward.tos, compiled_data->backward.to_size);
    1231
    2.24k
      compiled_data->gradient_mode = gradient_mode;
    1232
    2.24k
    }
    1233
    1234
    void ccv_cnnp_model_tensors_init_0(const ccv_cnnp_model_t* const model, ccv_cnnp_compiled_data_t* const compiled_data)
    1235
    106
    {
    1236
    106
      assert(!compiled_data->tensors.parameters);
    1237
    106
      const int parameter_size = compiled_data->parameters->rnum;
    1238
    106
      const int parallel_count = _ccv_cnnp_model_effective_parallel_count(model);
    1239
    106
      compiled_data->parallel_count = parallel_count;
    1240
    106
      const int internal_size = compiled_data->internals->rnum;
    1241
    106
      compiled_data->tensors_init.size = ccv_nnc_tensor_symbol_count(model->graph);
    1242
    106
      compiled_data->tensors_init.v = cccalloc(((compiled_data->tensors_init.size + 31) >> 5), sizeof(uint32_t));
    1243
    106
      compiled_data->tensors.parameters = (ccv_nnc_tensor_t**)cccalloc((parameter_size + internal_size) * parallel_count, sizeof(ccv_nnc_tensor_t*));
    1244
    106
      compiled_data->tensors.internals = compiled_data->tensors.parameters + parameter_size * parallel_count;
    1245
    106
    }
    1246
    1247
    int ccv_cnnp_model_tensors_any_to_alloc(const ccv_cnnp_model_t* const model, ccv_cnnp_compiled_data_t* const compiled_data)
    1248
    41
    {
    1249
    41
      int i, j;
    1250
    41
      const int parameter_size = compiled_data->parameters->rnum;
    1251
    41
      const int parallel_count = _ccv_cnnp_compiled_data_parallel_count(model, compiled_data);
    1252
    41
      const int internal_size = compiled_data->internals->rnum;
    1253
    91
      for (i = 0; i < parameter_size; i++50)
    1254
    72
      {
    1255
        // parameters has to be allocated all together.
    1256
    72
        if (compiled_data->tensors.parameters[i])
    1257
    50
        {
    1258
    65
          for (j = 1; j < parallel_count; j++15)
    1259
    15
            { assert(compiled_data->tensors.parameters[i + j * parameter_size]); }
    1260
    50
          continue;
    1261
    50
        }
    1262
    22
        return 1;
    1263
    72
      }
    1264
    19
      for (i = 0; i < internal_size; i++0)
    1265
    0
      {
    1266
    0
        if (!compiled_data->tensors.internals[i])
    1267
    0
          return 1;
    1268
    0
        for (j = 1; j < parallel_count; j++)
    1269
    0
          if (!compiled_data->tensors.internals[i + j * internal_size])
    1270
    0
            return 1;
    1271
    0
      }
    1272
    19
      return 0;
    1273
    19
    }
    1274
    1275
    void ccv_cnnp_model_tensors_init_1(const ccv_cnnp_model_t* const model, ccv_cnnp_compiled_data_t* const compiled_data)
    1276
    86
    {
    1277
    86
      int i, j;
    1278
    86
      const int parameter_size = compiled_data->parameters->rnum;
    1279
    86
      const int parallel_count = _ccv_cnnp_compiled_data_parallel_count(model, compiled_data);
    1280
    86
      compiled_data->parallel_count = parallel_count;
    1281
    86
      const int internal_size = compiled_data->internals->rnum;
    1282
    352
      for (i = 0; i < parameter_size; i++266)
    1283
    266
      {
    1284
        // parameters has to be allocated all together.
    1285
    266
        if (compiled_data->tensors.parameters[i])
    1286
    13
        {
    1287
    13
          for (j = 1; j < parallel_count; j++0)
    1288
    0
            { assert(compiled_data->tensors.parameters[i + j * parameter_size]); }
    1289
    13
          continue;
    1290
    13
        }
    1291
    253
        const ccv_nnc_tensor_symbol_t parameter = *(ccv_nnc_tensor_symbol_t*)ccv_array_get(compiled_data->parameters, i);
    1292
    253
        ccv_nnc_tensor_param_t info = ccv_nnc_tensor_symbol_params(parameter.graph, parameter);
    1293
    253
        if (CCV_TENSOR_GET_DEVICE(info.type) == CCV_COMPUTE_DEVICE_ANY)
    1294
    68
          CCV_TENSOR_SET_DEVICE_ID(info.type, 0);
    1295
    253
        const int device_id = CCV_TENSOR_GET_DEVICE_ID(info.type);
    1296
    253
        compiled_data->tensors.parameters[i] = ccv_nnc_tensor_new(0, info, 0);
    1297
    655
        for (j = 1; j < parallel_count; j++402)
    1298
    402
        {
    1299
    402
          if (j != device_id)
    1300
    402
            CCV_TENSOR_SET_DEVICE_ID(info.type, j);
    1301
    0
          else
    1302
    0
            CCV_TENSOR_SET_DEVICE_ID(info.type, 0);
    1303
    402
          compiled_data->tensors.parameters[i + j * parameter_size] = ccv_nnc_tensor_new(0, info, 0);
    1304
    402
        }
    1305
    253
      }
    1306
    86
      const uint32_t* const init_v = CCV_NNC_INIT_V(compiled_data->tensors_init.v);
    1307
    148
      for (i = 0; i < internal_size; i++62)
    1308
    62
      {
    1309
    62
        const ccv_nnc_tensor_symbol_t retained = *(ccv_nnc_tensor_symbol_t*)ccv_array_get(compiled_data->internals, i);
    1310
    62
        const int d = retained.d;
    1311
    62
        if (init_v[d >> 5] & (1u << (d & 0x1f)))
    1312
    0
          continue;
    1313
    62
        ccv_nnc_tensor_param_t info = ccv_nnc_tensor_symbol_params(retained.graph, retained);
    1314
    62
        if (CCV_TENSOR_GET_DEVICE(info.type) == CCV_COMPUTE_DEVICE_ANY)
    1315
    7
          CCV_TENSOR_SET_DEVICE_ID(info.type, 0);
    1316
    62
        const int device_id = CCV_TENSOR_GET_DEVICE_ID(info.type);
    1317
    62
        if (!compiled_data->tensors.internals[i])
    1318
    62
          compiled_data->tensors.internals[i] = ccv_nnc_tensor_new(0, info, 0);
    1319
    158
        for (j = 1; j < parallel_count; j++96)
    1320
    96
        {
    1321
    96
          if (j != device_id)
    1322
    96
            CCV_TENSOR_SET_DEVICE_ID(info.type, j);
    1323
    0
          else
    1324
    0
            CCV_TENSOR_SET_DEVICE_ID(info.type, 0);
    1325
    96
          if (!compiled_data->tensors.internals[i + j * internal_size])
    1326
    96
            compiled_data->tensors.internals[i + j * internal_size] = ccv_nnc_tensor_new(0, info, 0);
    1327
    96
        }
    1328
    62
      }
    1329
    86
      compiled_data->tensors_init.v = CCV_NNC_INIT_V(compiled_data->tensors_init.v); // Remove 1 if any.
    1330
    86
    }
    1331
    1332
    static void _ccv_cnnp_model_tensors_init(const ccv_cnnp_model_t* const model, ccv_cnnp_compiled_data_t* const compiled_data)
    1333
    81
    {
    1334
    81
      ccv_cnnp_model_tensors_init_0(model, compiled_data);
    1335
    81
      ccv_cnnp_model_tensors_init_1(model, compiled_data);
    1336
    81
    }
    1337
    1338
    static void _ccv_cnnp_model_copy_tensors(const uint32_t* const tensors_init, const ccv_nnc_tensor_symbol_t* const tensor_symbols, ccv_nnc_tensor_t* const* const tensors, const int tensor_size, const int parallel_count)
    1339
    9
    {
    1340
    9
      assert(parallel_count > 0);
    1341
    9
      int i, j;
    1342
    19
      for (i = 0; i < tensor_size; i++10)
    1343
    10
      {
    1344
    10
        if (!tensors[i])
    1345
    0
          continue;
    1346
    10
        const int d = tensor_symbols[i].d;
    1347
    10
        if (!(tensors_init[d >> 5] & (1u << (d & 0x1f))))
    1348
    0
          continue;
    1349
    40
        for (j = 1; 10j < parallel_count; j++30)
    1350
    30
          if (tensors[i + j * tensor_size])
    1351
    30
          {
    1352
    30
            ccv_nnc_tensor_t* const input = CCV_NNC_TENSOR(tensors[i]);
    1353
    30
            ccv_nnc_tensor_t* const output = CCV_NNC_TENSOR(tensors[i + j * tensor_size]);
    1354
    30
            ccv_nnc_cmd_exec(CMD_DATA_TRANSFER_FORWARD(), ccv_nnc_no_hint, 0, &input, 1, &output, 1, 0);
    1355
    30
          }
    1356
    10
      }
    1357
    9
    }
    1358
    1359
    static void _ccv_cnnp_model_remove_nocopies(const ccv_nnc_symbolic_graph_t* const graph, const ccv_nnc_tensor_symbol_t* const tensor_symbols, ccv_nnc_tensor_t** const tensors, const int tensor_size, const int parallel_count)
    1360
    111
    {
    1361
    111
      assert(parallel_count > 0);
    1362
    111
      int i, j;
    1363
    174
      for (i = 0; i < tensor_size; i++63)
    1364
    63
      {
    1365
    63
        const ccv_nnc_tensor_symbol_t tensor_symbol = tensor_symbols[i];
    1366
    159
        for (j = 1; j < parallel_count; j++96)
    1367
    96
        {
    1368
    96
          const ccv_nnc_tensor_symbol_t copy = ccv_nnc_tensor_symbol_copy(graph, tensor_symbol, j);
    1369
    96
          ccv_nnc_tensor_t* copy_tensor = tensors[i + j * tensor_size];
    1370
    96
          if (copy_tensor && copy.d == CCV_NNC_NO_TENSOR_SYMBOL)
    1371
    0
          { // We shouldn't allocate this, free it up.
    1372
    0
            ccv_nnc_tensor_free(tensors[i + j * tensor_size]);
    1373
    0
            tensors[i + j * tensor_size] = 0;
    1374
    0
          }
    1375
    96
        }
    1376
    63
      }
    1377
    111
    }
    1378
    1379
    static void _ccv_cnnp_model_bind_tensors(const ccv_nnc_symbolic_graph_t* const graph, const ccv_nnc_tensor_symbol_t* const tensor_symbols, ccv_nnc_tensor_t* const* const tensors, const int tensor_size, const int parallel_count, ccv_array_t* const tensor_binds)
    1380
    584
    {
    1381
    584
      assert(parallel_count > 0);
    1382
    584
      int i, j;
    1383
    2.01k
      for (i = 0; i < tensor_size; i++1.43k)
    1384
    1.43k
      {
    1385
    1.43k
        ccv_nnc_tensor_symbol_t tensor_symbol = tensor_symbols[i];
    1386
    1.43k
        if (tensor_symbol.d == CCV_NNC_NO_TENSOR_SYMBOL)
    1387
    7
          continue;
    1388
    1.42k
        if (graph)
    1389
    1.42k
        {
    1390
    1.42k
          const ccv_nnc_tensor_symbol_t alias_to = ccv_nnc_tensor_symbol_alias_to(graph, tensor_symbol);
    1391
    1.42k
          if (alias_to.d != CCV_NNC_NO_TENSOR_SYMBOL)
    1392
    0
            tensor_symbol = alias_to;
    1393
    1.42k
        }
    1394
    1.42k
        ccv_nnc_tensor_t* const tensor = CCV_NNC_TENSOR(tensors[i]);
    1395
    1.42k
        if (tensor && tensor_symbol.d != CCV_NNC_NO_TENSOR_SYMBOL1.42k)
    1396
    1.42k
        {
    1397
    1.42k
          const ccv_nnc_tensor_bind_t retained_bind = {
    1398
    1.42k
            .symbol = tensor_symbol,
    1399
    1.42k
            .tensor = tensor
    1400
    1.42k
          };
    1401
    1.42k
          ccv_array_push(tensor_binds, &retained_bind);
    1402
    1.42k
        }
    1403
    2.98k
        for (j = 1; j < parallel_count; j++1.56k)
    1404
    1.56k
        {
    1405
    1.56k
          const ccv_nnc_tensor_symbol_t copy = ccv_nnc_tensor_symbol_copy(graph, tensor_symbol, j);
    1406
    1.56k
          ccv_nnc_tensor_t* copy_tensor = tensors[i + j * tensor_size];
    1407
    1.56k
          if (copy_tensor && copy.d != CCV_NNC_NO_TENSOR_SYMBOL)
    1408
    1.56k
          {
    1409
    1.56k
            const ccv_nnc_tensor_bind_t bind = {
    1410
    1.56k
              .symbol = copy,
    1411
    1.56k
              .tensor = tensors[i + j * tensor_size]
    1412
    1.56k
            };
    1413
    1.56k
            ccv_array_push(tensor_binds, &bind);
    1414
    1.56k
          }
    1415
    1.56k
        }
    1416
    1.42k
      }
    1417
    584
    }
    1418
    1419
    static void _ccv_cnnp_compiled_data_graph_free(ccv_cnnp_compiled_data_t* const compiled_data)
    1420
    2.42k
    {
    1421
    2.42k
      if (compiled_data->graph)
    1422
    111
        ccv_nnc_graph_free(compiled_data->graph);
    1423
    2.42k
      compiled_data->graph = 0;
    1424
    2.42k
      compiled_data->is_test = 0;
    1425
    2.42k
      if (compiled_data->tensor_arena)
    1426
    111
        ccv_nnc_tensor_arena_free(compiled_data->tensor_arena);
    1427
    2.42k
      compiled_data->tensor_arena = 0;
    1428
    2.42k
      if (compiled_data->graph_exec_arena)
    1429
    111
        ccv_nnc_graph_exec_arena_free(compiled_data->graph_exec_arena);
    1430
    2.42k
      compiled_data->graph_exec_arena = 0;
    1431
    2.42k
      if (compiled_data->backward.from_ops)
    1432
    37
        ccfree(compiled_data->backward.from_ops);
    1433
    2.42k
      compiled_data->backward.from_ops = 0;
    1434
    2.42k
      if (compiled_data->evaluate.schedule)
    1435
    42
        ccv_nnc_graph_static_schedule_free(compiled_data->evaluate.schedule);
    1436
    2.42k
      compiled_data->evaluate.schedule = 0;
    1437
    2.42k
      if (compiled_data->backward.schedule)
    1438
    31
        ccv_nnc_graph_static_schedule_free(compiled_data->backward.schedule);
    1439
    2.42k
      compiled_data->backward.schedule = 0;
    1440
    2.42k
    }
    1441
    1442
    static void _ccv_cnnp_compiled_data_gradient_free(ccv_cnnp_compiled_data_t* const compiled_data)
    1443
    2.31k
    {
    1444
    2.31k
      if (compiled_data->gradients)
    1445
    2.24k
        ccfree(compiled_data->gradients);
    1446
    2.31k
      compiled_data->gradients = 0;
    1447
    2.31k
      if (compiled_data->updated_parameters)
    1448
    2.24k
        ccfree(compiled_data->updated_parameters);
    1449
    2.31k
      compiled_data->updated_parameters = 0;
    1450
    2.31k
      compiled_data->update_nodes = 0;
    1451
    2.31k
      compiled_data->saved_aux = 0;
    1452
    2.31k
    }
    1453
    1454
    static void _ccv_cnnp_compiled_data_backward_free(ccv_cnnp_compiled_data_t* const compiled_data)
    1455
    2.36k
    {
    1456
    2.36k
      if (compiled_data->backward.gradients)
    1457
    5
        ccfree(compiled_data->backward.gradients);
    1458
    2.36k
      compiled_data->backward.gradients = 0;
    1459
    2.36k
      if (compiled_data->backward.accum)
    1460
    5
        ccv_nnc_graph_free(compiled_data->backward.accum);
    1461
    2.36k
      compiled_data->backward.accum = 0;
    1462
    2.36k
      if (compiled_data->backward.tensor_arena)
    1463
    5
        ccv_nnc_tensor_arena_free(compiled_data->backward.tensor_arena);
    1464
    2.36k
      compiled_data->backward.tensor_arena = 0;
    1465
    2.36k
      if (compiled_data->backward.graph_exec_arena)
    1466
    5
        ccv_nnc_graph_exec_arena_free(compiled_data->backward.graph_exec_arena);
    1467
    2.36k
      compiled_data->backward.graph_exec_arena = 0;
    1468
    2.36k
    }
    1469
    1470
    static void _ccv_cnnp_compiled_data_apply_gradients_free(ccv_cnnp_compiled_data_t* const compiled_data)
    1471
    2.33k
    {
    1472
    2.33k
      if (compiled_data->apply_gradients.graph)
    1473
    24
        ccv_nnc_graph_free(compiled_data->apply_gradients.graph);
    1474
    2.33k
      compiled_data->apply_gradients.graph = 0;
    1475
    2.33k
      if (compiled_data->apply_gradients.tensor_arena)
    1476
    24
        ccv_nnc_tensor_arena_free(compiled_data->apply_gradients.tensor_arena);
    1477
    2.33k
      compiled_data->apply_gradients.tensor_arena = 0;
    1478
    2.33k
      if (compiled_data->apply_gradients.graph_exec_arena)
    1479
    24
        ccv_nnc_graph_exec_arena_free(compiled_data->apply_gradients.graph_exec_arena);
    1480
    2.33k
      compiled_data->apply_gradients.graph_exec_arena = 0;
    1481
    2.33k
    }
    1482
    1483
    // Compile the graph to run ccv_cnnp_model_fit
    1484
    static void _ccv_cnnp_model_fit_jit(ccv_cnnp_model_t* const model, ccv_nnc_tensor_t* const* const inputs, const int input_size, ccv_nnc_tensor_t* const* const fits, const int fit_size, ccv_nnc_tensor_t* const* const outputs, const int output_size)
    1485
    8
    {
    1486
    8
      int i, j;
    1487
    8
      ccv_cnnp_compiled_data_t* const compiled_data = model->compiled_data;
    1488
    8
      assert(!compiled_data->graph || compiled_data->graph_mode != CCV_CNNP_MODEL_GRAPH_FIT_MODE);
    1489
    8
      compiled_data->graph_mode = CCV_CNNP_MODEL_GRAPH_FIT_MODE;
    1490
    8
      const int parallel_count = _ccv_cnnp_model_root_parallel_count(model);
    1491
    8
      assert(output_size == model->output_size * parallel_count);
    1492
    8
      assert(!fits || output_size == fit_size);
    1493
    8
      assert(output_size > 0);
    1494
    8
      if (compiled_data->gradient_mode == CCV_CNNP_COMPILED_DATA_GRADIENT_NONE)
    1495
    8
      {
    1496
    8
        _ccv_cnnp_model_set_rewindables(model);
    1497
    8
        _ccv_cnnp_model_gradient_init(model, CCV_CNNP_COMPILED_DATA_GRADIENT_TRAINABLES, CCV_CNNP_DISABLE_OUTGRAD_ALL, fits, fit_size);
    1498
    8
      } else if (0compiled_data->gradient_mode != CCV_CNNP_COMPILED_DATA_GRADIENT_TRAINABLES0) {
    1499
    0
        _ccv_cnnp_model_rewind_graph(model);
    1500
    0
        _ccv_cnnp_compiled_data_gradient_free(compiled_data);
    1501
    0
        compiled_data->gradient_mode = CCV_CNNP_COMPILED_DATA_GRADIENT_NONE;
    1502
    0
        _ccv_cnnp_model_gradient_init(model, CCV_CNNP_COMPILED_DATA_GRADIENT_TRAINABLES, CCV_CNNP_DISABLE_OUTGRAD_ALL, fits, fit_size);
    1503
    0
      }
    1504
    8
      const int tensors_init = !!compiled_data->tensors_init.v;
    1505
    8
      if (!tensors_init)
    1506
    4
        _ccv_cnnp_model_tensors_init(model, compiled_data);
    1507
    4
      else if ((uintptr_t)compiled_data->tensors_init.v & (uintptr_t)1)
    1508
      // Check if it is not fully allocated, if it is not, init_1.
    1509
    3
        ccv_cnnp_model_tensors_init_1(model, compiled_data);
    1510
    8
      ccv_array_t* const tensor_binds = ccv_array_new(sizeof(ccv_nnc_tensor_bind_t), 0, 0);
    1511
    8
      assert((input_size % parallel_count) == 0);
    1512
    8
      assert((output_size % parallel_count) == 0);
    1513
    8
      assert((fit_size % parallel_count) == 0);
    1514
    8
      const int input_size_per_p = input_size / parallel_count;
    1515
    8
      _ccv_cnnp_model_bind_tensors(model->graph, model->inputs, inputs, input_size_per_p, parallel_count, tensor_binds);
    1516
    8
      const int output_size_per_p = output_size / parallel_count;
    1517
    8
      _ccv_cnnp_model_bind_tensors(model->graph, model->outputs, outputs, output_size_per_p, parallel_count, tensor_binds);
    1518
    8
      const int fit_size_per_p = fit_size / parallel_count;
    1519
    8
      _ccv_cnnp_model_bind_tensors(model->graph, compiled_data->fits, fits, fit_size_per_p, parallel_count, tensor_binds);
    1520
    8
      const int parameter_size = compiled_data->parameters->rnum;
    1521
    8
      _ccv_cnnp_model_bind_tensors(model->graph, (ccv_nnc_tensor_symbol_t*)ccv_array_get(compiled_data->parameters, 0), compiled_data->tensors.parameters, parameter_size, parallel_count, tensor_binds);
    1522
    8
      _ccv_cnnp_model_bind_tensors(model->graph, compiled_data->updated_parameters, compiled_data->tensors.parameters, parameter_size, parallel_count, tensor_binds);
    1523
    8
      const int internal_size = compiled_data->internals->rnum;
    1524
    8
      _ccv_cnnp_model_remove_nocopies(model->graph, (ccv_nnc_tensor_symbol_t*)ccv_array_get(compiled_data->internals, 0), compiled_data->tensors.internals, internal_size, parallel_count);
    1525
    8
      _ccv_cnnp_model_bind_tensors(model->graph, (ccv_nnc_tensor_symbol_t*)ccv_array_get(compiled_data->internals, 0), compiled_data->tensors.internals, internal_size, parallel_count, tensor_binds);
    1526
    8
      ccv_nnc_symbolic_graph_compile(model->graph, compiled_data->compile_params, (ccv_nnc_tensor_bind_t*)ccv_array_get(tensor_binds, 0), tensor_binds->rnum, 0, 0, SYMBOLIC_GRAPH_SOURCES(model->graph), SYMBOLIC_GRAPH_DESTINATIONS(model->graph), &compiled_data->graph, &compiled_data->tensor_arena, &compiled_data->graph_exec_arena);
    1527
    8
      ccv_array_free(tensor_binds);
    1528
    8
      const uint32_t* const init_v = CCV_NNC_INIT_V(compiled_data->tensors_init.v);
    1529
    8
      if (tensors_init && parallel_count > 14)
    1530
    0
        _ccv_cnnp_model_copy_tensors(init_v, (ccv_nnc_tensor_symbol_t*)ccv_array_get(compiled_data->parameters, 0), compiled_data->tensors.parameters, compiled_data->parameters->rnum, parallel_count);
    1531
      // If tensor is not init'ed, we need to init states first.
    1532
    8
      if (_ccv_cnnp_any_to_init(compiled_data))
    1533
    7
      {
    1534
    7
        ccv_nnc_tensor_init_states_t tensor_init_states = {
    1535
    7
          .parallel_count = parallel_count,
    1536
    7
          .graph = model->graph,
    1537
    7
          .compiled_data = compiled_data,
    1538
    7
          .tensor_arena = compiled_data->tensor_arena
    1539
    7
        };
    1540
    7
        ccv_cnnp_model_init_states(model, model->graph, _ccv_cnnp_init_states_for_tensors, &tensor_init_states);
    1541
    7
      }
    1542
    8
      compiled_data->is_test = 0;
    1543
    8
      const int saved_aux_size = ccv_nnc_minimizer_saved_aux_size(compiled_data->minimize.minimizer);
    1544
      // No need to set because it is default to training mode.
    1545
      // ccv_cnnp_model_set_is_test(model, 0, _ccv_cnnp_cmd_update_for_execs, &update);
    1546
    105
      for (i = 0; i < saved_aux_size * parameter_size; i++97)
    1547
    97
      {
    1548
    97
        if (compiled_data->saved_aux[i].source.d == CCV_NNC_NO_TENSOR_SYMBOL)
    1549
    5
          continue;
    1550
    92
        ccv_nnc_tensor_t* const tensor = ccv_nnc_tensor_from_symbol(compiled_data->tensor_arena, compiled_data->saved_aux[i].source);
    1551
    92
        ccv_nnc_cmd_exec(CMD_SET_FORWARD(0), ccv_nnc_no_hint, 0, 0, 0, &tensor, 1, 0);
    1552
    296
        for (j = 1; j < parallel_count; j++204)
    1553
    204
        {
    1554
    204
          ccv_nnc_tensor_t* const copy = ccv_nnc_tensor_from_symbol(compiled_data->tensor_arena, ccv_nnc_tensor_symbol_copy(model->graph, compiled_data->saved_aux[i].source, j));
    1555
    204
          if (copy)
    1556
    204
            ccv_nnc_cmd_exec(CMD_SET_FORWARD(0), ccv_nnc_no_hint, 0, 0, 0, &copy, 1, 0);
    1557
    204
        }
    1558
    92
      }
    1559
    8
      const int evaluate_to_size = compiled_data->evaluate.to_size;
    1560
    8
      compiled_data->evaluate.to_op_size = 0;
    1561
    22
      for (i = 0; i < evaluate_to_size; i++14)
    1562
    14
      {
    1563
    14
        ccv_nnc_graph_exec_t const to = ccv_nnc_graph_exec_from_symbol(compiled_data->graph_exec_arena, compiled_data->evaluate.tos[i]);
    1564
    14
        if (to.graph)
    1565
    14
          compiled_data->evaluate.to_ops[compiled_data->evaluate.to_op_size++] = to;
    1566
    14
      }
    1567
    8
      ccv_nnc_graph_set_default_static_schedule(compiled_data->graph, compiled_data->stream_type, model->max_stream_count);
    1568
    8
      ccv_nnc_graph_autotune(compiled_data->graph, model->workspace_size, 0, TRAVERSE_FULL);
    1569
    8
    }
    1570
    1571
    ccv_nnc_stream_context_t* ccv_cnnp_model_default_stream(const ccv_cnnp_model_t* const model)
    1572
    0
    {
    1573
    0
      const ccv_cnnp_compiled_data_t* const compiled_data = model->compiled_data;
    1574
    0
      if (!compiled_data || !compiled_data->graph)
    1575
    0
        return 0;
    1576
    0
      return ccv_nnc_graph_default_stream(compiled_data->graph);
    1577
    0
    }
    1578
    1579
    uint64_t ccv_cnnp_model_memory_size(const ccv_cnnp_model_t* const model)
    1580
    0
    {
    1581
    0
      const ccv_cnnp_compiled_data_t* const compiled_data = model->compiled_data;
    1582
    0
      if (!compiled_data || !compiled_data->tensor_arena)
    1583
    0
        return 0;
    1584
    0
      return ccv_nnc_tensor_arena_size(compiled_data->tensor_arena);
    1585
    0
    }
    1586
    1587
    static void _ccv_cnnp_bind_tensors_to_arena(ccv_nnc_tensor_arena_t* const tensor_arena, const ccv_nnc_symbolic_graph_t* const graph, const ccv_nnc_tensor_symbol_t* const tensor_symbols, ccv_nnc_tensor_t* const* const tensors, const int tensor_size, const int parallel_count)
    1588
    38.9k
    {
    1589
    38.9k
      int i, j;
    1590
    114k
      for (i = 0; i < tensor_size; i++75.6k)
    1591
    75.6k
      {
    1592
    75.6k
        ccv_nnc_tensor_symbol_t tensor_symbol = tensor_symbols[i];
    1593
    75.6k
        if (tensor_symbol.d == CCV_NNC_NO_TENSOR_SYMBOL)
    1594
    0
          continue;
    1595
    75.6k
        if (graph)
    1596
    72.7k
        {
    1597
    72.7k
          const ccv_nnc_tensor_symbol_t alias_to = ccv_nnc_tensor_symbol_alias_to(graph, tensor_symbol);
    1598
    72.7k
          if (alias_to.d != CCV_NNC_NO_TENSOR_SYMBOL)
    1599
    0
            tensor_symbol = alias_to;
    1600
    72.7k
        }
    1601
    75.6k
        ccv_nnc_tensor_bind_symbol(tensor_arena, tensor_symbol, tensors[i]);
    1602
    77.4k
        for (j = 1; j < parallel_count; j++1.77k)
    1603
    1.77k
        {
    1604
    1.77k
          const ccv_nnc_tensor_symbol_t copy = ccv_nnc_tensor_symbol_copy(graph, tensor_symbol, j);
    1605
    1.77k
          if (copy.d != CCV_NNC_NO_TENSOR_SYMBOL)
    1606
    1.77k
            ccv_nnc_tensor_bind_symbol(tensor_arena, copy, tensors[i + tensor_size * j]);
    1607
    1.77k
        }
    1608
    75.6k
      }
    1609
    38.9k
    }
    1610
    1611
    void ccv_cnnp_model_fit(ccv_cnnp_model_t* const model, ccv_nnc_tensor_t* const* const inputs, const int input_size, ccv_nnc_tensor_t* const* const fits, const int fit_size, ccv_nnc_tensor_t* const* const outputs, const int output_size, ccv_nnc_tensor_tape_t* const tensor_tape, ccv_nnc_stream_context_t* const stream_context)
    1612
    2.54k
    {
    1613
    2.54k
      ccv_cnnp_compiled_data_t* const compiled_data = model->compiled_data;
    1614
    2.54k
      assert(compiled_data);
    1615
    2.54k
      const int parallel_count = ccv_max(model->parallel_count, 1);
    1616
    2.54k
      assert(output_size == model->output_size * parallel_count);
    1617
    2.54k
      assert(input_size == model->input_size * parallel_count);
    1618
    2.54k
      assert(!fits || fit_size == output_size);
    1619
    2.54k
      assert(model->graph);
    1620
    2.54k
      if (!compiled_data->graph || compiled_data->graph_mode != CCV_CNNP_MODEL_GRAPH_FIT_MODE2.53k)
    1621
    8
      {
    1622
    8
        _ccv_cnnp_compiled_data_graph_free(compiled_data);
    1623
    8
        _ccv_cnnp_compiled_data_backward_free(compiled_data);
    1624
    8
        _ccv_cnnp_compiled_data_apply_gradients_free(compiled_data);
    1625
        // Compile the symbolic graph down only when needed.
    1626
    8
        _ccv_cnnp_model_fit_jit(model, inputs, input_size, fits, fit_size, outputs, output_size);
    1627
    2.53k
      } else {
    1628
    2.53k
        assert((input_size % parallel_count) == 0);
    1629
    2.53k
        assert((output_size % parallel_count) == 0);
    1630
    2.53k
        assert((fit_size % parallel_count) == 0);
    1631
    2.53k
        const int input_size_per_p = input_size / parallel_count;
    1632
    2.53k
        _ccv_cnnp_bind_tensors_to_arena(compiled_data->tensor_arena, model->graph, model->inputs, inputs, input_size_per_p, parallel_count);
    1633
    2.53k
        const int output_size_per_p = output_size / parallel_count;
    1634
    2.53k
        _ccv_cnnp_bind_tensors_to_arena(compiled_data->tensor_arena, model->graph, model->outputs, outputs, output_size_per_p, parallel_count);
    1635
    2.53k
        const int fit_size_per_p = fit_size / parallel_count;
    1636
    2.53k
        _ccv_cnnp_bind_tensors_to_arena(compiled_data->tensor_arena, model->graph, compiled_data->fits, fits, fit_size_per_p, parallel_count);
    1637
    2.53k
      }
    1638
    2.54k
      if (compiled_data->is_test)
    1639
    0
      {
    1640
    0
        compiled_data->is_test = 0;
    1641
    0
        ccv_nnc_graph_exec_update_t update = {
    1642
    0
          .parallel_count = parallel_count,
    1643
    0
          .graph = model->graph,
    1644
    0
          .graph_exec_arena = compiled_data->graph_exec_arena,
    1645
    0
        };
    1646
    0
        ccv_cnnp_model_set_is_test(model, 0, _ccv_cnnp_cmd_update_for_execs, &update);
    1647
    0
      }
    1648
    2.54k
      ccv_nnc_graph_run_with_schedule(compiled_data->graph, 0, 0, tensor_tape, stream_context);
    1649
    2.54k
    }
    1650
    1651
    // Compile the graph to run ccv_cnnp_model_evaluate with require_grad = false (MULTISTAGE_MODE_NO_GRAD).
    1652
    static void _ccv_cnnp_model_multistage_no_grad_jit(ccv_cnnp_model_t* const model, ccv_nnc_tensor_t* const* const inputs, const int input_size, ccv_nnc_tensor_t* const* const outputs, const int output_size)
    1653
    66
    {
    1654
    66
      ccv_cnnp_compiled_data_t* const compiled_data = model->compiled_data;
    1655
    66
      compiled_data->graph_mode = CCV_CNNP_MODEL_GRAPH_MULTISTAGE_MODE_NO_GRAD;
    1656
    66
      const int parallel_count = ccv_max(model->parallel_count, 1);
    1657
    66
      assert(output_size == model->output_size * parallel_count);
    1658
    66
      assert(output_size > 0);
    1659
      // If the gradient is not initialized, continue to setup parallel process. We don't init gradient here, but rather,
    1660
      // we setup proper rewindables so the graph can be rewinded to previous state before we run data parallel.
    1661
    66
      if (parallel_count > 1 && compiled_data->gradient_mode == CCV_CNNP_COMPILED_DATA_GRADIENT_NONE6)
    1662
    6
      {
    1663
    6
        const int evaluate_to_size = compiled_data->evaluate.to_size;
    1664
    6
        compiled_data->evaluate.tos = ccrealloc(compiled_data->evaluate.tos, sizeof(ccv_nnc_graph_exec_symbol_t) * evaluate_to_size * parallel_count + sizeof(ccv_nnc_graph_exec_t) * evaluate_to_size * parallel_count);
    1665
    6
        _ccv_cnnp_model_set_rewindables(model);
    1666
    6
        ccv_nnc_symbolic_graph_data_parallel(model->graph, parallel_count,
    1667
    6
          0, 0,
    1668
    6
          0, 0, 0,
    1669
    6
          0, 0, 0,
    1670
    6
          CCV_NNC_PARALLEL_REDUCE_OP_SUM,
    1671
    6
          SYMBOLIC_GRAPH_SOURCES(model->graph), SYMBOLIC_GRAPH_DESTINATIONS(model->graph));
    1672
    6
        ccv_nnc_graph_exec_symbol_autogen(model->graph, 0, 0, CCV_NNC_AUTOGEN_SOURCES_AND_DESTINATIONS);
    1673
    6
        int i, j;
    1674
    12
        for (i = 0; i < evaluate_to_size; i++6)
    1675
    24
          for (j = 1; 6j < parallel_count; j++18)
    1676
    18
          {
    1677
    18
            const ccv_nnc_graph_exec_symbol_t copy = ccv_nnc_graph_exec_symbol_copy(model->graph, compiled_data->evaluate.tos[i], j);
    1678
    18
            if (copy.d != CCV_NNC_NO_GRAPH_EXEC_SYMBOL)
    1679
    18
              compiled_data->evaluate.tos[compiled_data->evaluate.to_size++] = copy;
    1680
    18
          }
    1681
    6
      }
    1682
    66
      const int tensors_init = !!compiled_data->tensors_init.v;
    1683
    66
      if (!tensors_init)
    1684
    39
        _ccv_cnnp_model_tensors_init(model, compiled_data);
    1685
    27
      else if ((uintptr_t)compiled_data->tensors_init.v & (uintptr_t)1)
    1686
      // Check if it is not fully allocated, if it is not, init_1.
    1687
    1
        ccv_cnnp_model_tensors_init_1(model, compiled_data);
    1688
    66
      const int tensor_parallel_count = _ccv_cnnp_compiled_data_parallel_count(model, compiled_data);
    1689
    66
      ccv_array_t* const tensor_binds = ccv_array_new(sizeof(ccv_nnc_tensor_bind_t), 0, 0);
    1690
    66
      assert((input_size % parallel_count) == 0);
    1691
    66
      assert((output_size % parallel_count) == 0);
    1692
    66
      const int input_size_per_p = input_size / parallel_count;
    1693
    66
      _ccv_cnnp_model_bind_tensors(model->graph, model->inputs, inputs, input_size_per_p, parallel_count, tensor_binds);
    1694
    66
      const int output_size_per_p = output_size / parallel_count;
    1695
    66
      _ccv_cnnp_model_bind_tensors(model->graph, model->outputs, outputs, output_size_per_p, parallel_count, tensor_binds);
    1696
    66
      const int parameter_size = compiled_data->parameters->rnum;
    1697
    66
      _ccv_cnnp_model_bind_tensors(model->graph, (ccv_nnc_tensor_symbol_t*)ccv_array_get(compiled_data->parameters, 0), compiled_data->tensors.parameters, parameter_size, tensor_parallel_count, tensor_binds);
    1698
    66
      const int internal_size = compiled_data->internals->rnum;
    1699
    66
      _ccv_cnnp_model_remove_nocopies(model->graph, (ccv_nnc_tensor_symbol_t*)ccv_array_get(compiled_data->internals, 0), compiled_data->tensors.internals, internal_size, tensor_parallel_count);
    1700
    66
      _ccv_cnnp_model_bind_tensors(model->graph, (ccv_nnc_tensor_symbol_t*)ccv_array_get(compiled_data->internals, 0), compiled_data->tensors.internals, internal_size, tensor_parallel_count, tensor_binds);
    1701
      // If we generated gradient for the graph, only compile part of the graph because the rest is irrelevant for evaluation.
    1702
    66
      ccv_nnc_symbolic_graph_compile(model->graph, compiled_data->compile_params, (ccv_nnc_tensor_bind_t*)ccv_array_get(tensor_binds, 0), tensor_binds->rnum, 0, 0, SYMBOLIC_GRAPH_SOURCES(model->graph), compiled_data->evaluate.tos, compiled_data->evaluate.to_size, &compiled_data->graph, &compiled_data->tensor_arena, &compiled_data->graph_exec_arena);
    1703
    66
      ccv_array_free(tensor_binds);
    1704
    66
      const uint32_t* const init_v = CCV_NNC_INIT_V(compiled_data->tensors_init.v);
    1705
      // If tensor is not init'ed, we need to init states first.
    1706
    66
      if (tensors_init && tensor_parallel_count > 127)
    1707
    9
        _ccv_cnnp_model_copy_tensors(init_v, (ccv_nnc_tensor_symbol_t*)ccv_array_get(compiled_data->parameters, 0), compiled_data->tensors.parameters, compiled_data->parameters->rnum, tensor_parallel_count);
    1708
    66
      if (_ccv_cnnp_any_to_init(compiled_data))
    1709
    17
      {
    1710
    17
        ccv_nnc_tensor_init_states_t tensor_init_states = {
    1711
    17
          .parallel_count = tensor_parallel_count,
    1712
    17
          .graph = model->graph,
    1713
    17
          .compiled_data = compiled_data,
    1714
    17
          .tensor_arena = compiled_data->tensor_arena
    1715
    17
        };
    1716
    17
        ccv_cnnp_model_init_states(model, model->graph, _ccv_cnnp_init_states_for_tensors, &tensor_init_states);
    1717
    17
      }
    1718
    66
      compiled_data->is_test = 1;
    1719
    66
      ccv_nnc_graph_exec_update_t update = {
    1720
    66
        .parallel_count = parallel_count,
    1721
    66
        .graph = model->graph,
    1722
    66
        .graph_exec_arena = compiled_data->graph_exec_arena,
    1723
    66
      };
    1724
    66
      ccv_cnnp_model_set_is_test(model, 1, _ccv_cnnp_cmd_update_for_execs, &update);
    1725
    66
      ccv_nnc_graph_set_default_static_schedule(compiled_data->graph, compiled_data->stream_type, model->max_stream_count);
    1726
    66
      ccv_nnc_graph_autotune(compiled_data->graph, model->workspace_size, 0, TRAVERSE_FULL);
    1727
    66
    }
    1728
    1729
    static void _ccv_cnnp_model_gradient_tensors_init(const ccv_cnnp_model_t* const model, ccv_cnnp_compiled_data_t* const compiled_data)
    1730
    36
    {
    1731
    36
      assert(!compiled_data->tensors.gradients);
    1732
    36
      const int parameter_size = compiled_data->parameters->rnum;
    1733
    36
      const int parallel_count = ccv_max(model->parallel_count, 1);
    1734
    36
      compiled_data->tensors.gradients = (ccv_nnc_tensor_t**)ccmalloc(sizeof(ccv_nnc_tensor_t*) * parameter_size * 2 * parallel_count);
    1735
    36
      compiled_data->tensors.accum_gradients = compiled_data->tensors.gradients + parameter_size * parallel_count;
    1736
    36
      int i, j;
    1737
    186
      for (i = 0; i < parameter_size; i++150)
    1738
    150
      {
    1739
    150
        if (compiled_data->parameter_flags && !(compiled_data->parameter_flags[i >> 6] & ((uint64_t)1 << (i & 63)))6)
    1740
    2
        {
    1741
    2
          compiled_data->tensors.gradients[i] = 0;
    1742
    2
          compiled_data->tensors.accum_gradients[i] = 0;
    1743
    2
          for (j = 1; j < parallel_count; j++0)
    1744
    0
          {
    1745
    0
            compiled_data->tensors.gradients[i + j * parameter_size] = 0;
    1746
    0
            compiled_data->tensors.accum_gradients[i + j * parameter_size] = 0;
    1747
    0
          }
    1748
    2
          continue;
    1749
    2
        }
    1750
    148
        const ccv_nnc_tensor_symbol_t parameter = *(ccv_nnc_tensor_symbol_t*)ccv_array_get(compiled_data->parameters, i);
    1751
    148
        ccv_nnc_tensor_param_t info = ccv_nnc_tensor_symbol_params(parameter.graph, parameter);
    1752
    148
        if (CCV_TENSOR_GET_DEVICE(info.type) == CCV_COMPUTE_DEVICE_ANY)
    1753
    38
          CCV_TENSOR_SET_DEVICE_ID(info.type, 0);
    1754
    148
        const int device_id = CCV_TENSOR_GET_DEVICE_ID(info.type);
    1755
    148
        compiled_data->tensors.gradients[i] = ccv_nnc_tensor_new(0, info, 0);
    1756
    148
        compiled_data->tensors.accum_gradients[i] = 0; // delay the accumulated gradient allocation until when we need it.
    1757
    328
        for (j = 1; j < parallel_count; j++180)
    1758
    180
        {
    1759
    180
          if (j != device_id)
    1760
    180
            CCV_TENSOR_SET_DEVICE_ID(info.type, j);
    1761
    0
          else
    1762
    0
            CCV_TENSOR_SET_DEVICE_ID(info.type, 0);
    1763
    180
          compiled_data->tensors.gradients[i + j * parameter_size] = ccv_nnc_tensor_new(0, info, 0);
    1764
    180
          compiled_data->tensors.accum_gradients[i + j * parameter_size] = 0;
    1765
    180
        }
    1766
    148
      }
    1767
    36
    }
    1768
    1769
    static int _ccv_cnnp_is_disable_outgrad_all(const uint64_t disable_outgrad, const int input_size)
    1770
    8.05k
    {
    1771
    8.05k
      if (disable_outgrad == CCV_CNNP_DISABLE_OUTGRAD_ALL)
    1772
    15
        return 1;
    1773
    8.03k
      if (disable_outgrad == CCV_CNNP_DISABLE_OUTGRAD_NONE)
    1774
    8.03k
        return 0;
    1775
    7
      int i;
    1776
    7
      for (i = 0; i < input_size; i++0)
    1777
    7
        if (!(disable_outgrad & ((uint64_t)1 << i)))
    1778
    7
          return 0;
    1779
    0
      return 1;
    1780
    7
    }
    1781
    1782
    // Compile the graph to run ccv_cnnp_model_evaluate with requires_grad = true (MULTISTAGE_MODE).
    1783
    // Particularly, this method compiles the evaluation and backprop graph (the main graph).
    1784
    static void _ccv_cnnp_model_multistage_jit_0(ccv_cnnp_model_t* const model, const uint64_t disable_outgrad, const int is_test, ccv_nnc_tensor_t* const* const inputs, const int input_size, ccv_nnc_tensor_t* const* const outputs, const int output_size)
    1785
    37
    {
    1786
    37
      int i, j;
    1787
    37
      ccv_cnnp_compiled_data_t* const compiled_data = model->compiled_data;
    1788
    37
      const int target_gradient_mode = _ccv_cnnp_is_disable_outgrad_all(disable_outgrad, model->input_size) ? CCV_CNNP_COMPILED_DATA_GRADIENT_TRAINABLES1 : CCV_CNNP_COMPILED_DATA_GRADIENT_TRAINABLES_AND_INPUTS36;
    1789
    37
      assert(!compiled_data->graph || compiled_data->graph_mode != CCV_CNNP_MODEL_GRAPH_MULTISTAGE_MODE || compiled_data->gradient_mode != target_gradient_mode);
    1790
    37
      compiled_data->graph_mode = CCV_CNNP_MODEL_GRAPH_MULTISTAGE_MODE;
    1791
    37
      const int parallel_count = ccv_max(model->parallel_count, 1);
    1792
    37
      assert(output_size == model->output_size * parallel_count);
    1793
    37
      assert(output_size > 0);
    1794
      // There shouldn't be a loss function if we evaluate with multistage jit.
    1795
    37
      assert(compiled_data->loss.cmd == CCV_NNC_NOOP);
    1796
    37
      if (compiled_data->gradient_mode == CCV_CNNP_COMPILED_DATA_GRADIENT_NONE)
    1797
    35
      {
    1798
    35
        _ccv_cnnp_model_set_rewindables(model);
    1799
    35
        _ccv_cnnp_model_gradient_init(model, target_gradient_mode, disable_outgrad, 0, 0); // The type of outputs and fits should be the same. We only use type here.
    1800
    35
      } else if (2compiled_data->gradient_mode != target_gradient_mode2) {
    1801
    2
        _ccv_cnnp_model_rewind_graph(model);
    1802
    2
        _ccv_cnnp_compiled_data_gradient_free(compiled_data);
    1803
    2
        compiled_data->gradient_mode = CCV_CNNP_COMPILED_DATA_GRADIENT_NONE;
    1804
    2
        _ccv_cnnp_model_gradient_init(model, target_gradient_mode, disable_outgrad, 0, 0); // The type of outputs and fits should be the same. We only use type here.
    1805
    2
      }
    1806
    37
      const int tensors_init = !!compiled_data->tensors_init.v;
    1807
    37
      if (!tensors_init)
    1808
    29
        _ccv_cnnp_model_tensors_init(model, compiled_data);
    1809
    8
      else if ((uintptr_t)compiled_data->tensors_init.v & (uintptr_t)1)
    1810
      // Check if it is not fully allocated, if it is not, init_1.
    1811
    1
        ccv_cnnp_model_tensors_init_1(model, compiled_data);
    1812
    37
      ccv_array_t* const tensor_binds = ccv_array_new(sizeof(ccv_nnc_tensor_bind_t), 0, 0);
    1813
    37
      assert((input_size % parallel_count) == 0);
    1814
    37
      assert((output_size % parallel_count) == 0);
    1815
    37
      const int input_size_per_p = input_size / parallel_count;
    1816
    37
      _ccv_cnnp_model_bind_tensors(model->graph, model->inputs, inputs, input_size_per_p, parallel_count, tensor_binds);
    1817
    37
      const int output_size_per_p = output_size / parallel_count;
    1818
    37
      _ccv_cnnp_model_bind_tensors(model->graph, model->outputs, outputs, output_size_per_p, parallel_count, tensor_binds);
    1819
    37
      const int parameter_size = compiled_data->parameters->rnum;
    1820
    37
      _ccv_cnnp_model_bind_tensors(model->graph, (ccv_nnc_tensor_symbol_t*)ccv_array_get(compiled_data->parameters, 0), compiled_data->tensors.parameters, parameter_size, parallel_count, tensor_binds);
    1821
    37
      const int internal_size = compiled_data->internals->rnum;
    1822
    37
      _ccv_cnnp_model_remove_nocopies(model->graph, (ccv_nnc_tensor_symbol_t*)ccv_array_get(compiled_data->internals, 0), compiled_data->tensors.internals, internal_size, parallel_count);
    1823
    37
      _ccv_cnnp_model_bind_tensors(model->graph, (ccv_nnc_tensor_symbol_t*)ccv_array_get(compiled_data->internals, 0), compiled_data->tensors.internals, internal_size, parallel_count, tensor_binds);
    1824
    37
      if (!compiled_data->tensors.gradients)
    1825
    36
        _ccv_cnnp_model_gradient_tensors_init(model, compiled_data);
    1826
    37
      _ccv_cnnp_model_bind_tensors(model->graph, compiled_data->gradients, compiled_data->tensors.gradients, parameter_size, parallel_count, tensor_binds);
    1827
    37
      if (compiled_data->backward.to_size > 0)
    1828
    37
        ccv_nnc_symbolic_graph_compile(model->graph, compiled_data->compile_params, (ccv_nnc_tensor_bind_t*)ccv_array_get(tensor_binds, 0), tensor_binds->rnum, 0, 0, SYMBOLIC_GRAPH_SOURCES(model->graph), compiled_data->backward.tos, compiled_data->backward.to_size, &compiled_data->graph, &compiled_data->tensor_arena, &compiled_data->graph_exec_arena);
    1829
    0
      else
    1830
    0
        ccv_nnc_symbolic_graph_compile(model->graph, compiled_data->compile_params, (ccv_nnc_tensor_bind_t*)ccv_array_get(tensor_binds, 0), tensor_binds->rnum, 0, 0, SYMBOLIC_GRAPH_SOURCES(model->graph), compiled_data->evaluate.tos, compiled_data->evaluate.to_size, &compiled_data->graph, &compiled_data->tensor_arena, &compiled_data->graph_exec_arena);
    1831
    37
      ccv_array_free(tensor_binds);
    1832
    37
      const uint32_t* const init_v = CCV_NNC_INIT_V(compiled_data->tensors_init.v);
    1833
    37
      if (tensors_init && parallel_count > 18)
    1834
    0
        _ccv_cnnp_model_copy_tensors(init_v, (ccv_nnc_tensor_symbol_t*)ccv_array_get(compiled_data->parameters, 0), compiled_data->tensors.parameters, compiled_data->parameters->rnum, parallel_count);
    1835
      // If tensor is not init'ed, we need to init states first.
    1836
    37
      if (_ccv_cnnp_any_to_init(compiled_data))
    1837
    21
      {
    1838
    21
        ccv_nnc_tensor_init_states_t tensor_init_states = {
    1839
    21
          .parallel_count = parallel_count,
    1840
    21
          .graph = model->graph,
    1841
    21
          .compiled_data = compiled_data,
    1842
    21
          .tensor_arena = compiled_data->tensor_arena
    1843
    21
        };
    1844
    21
        ccv_cnnp_model_init_states(model, model->graph, _ccv_cnnp_init_states_for_tensors, &tensor_init_states);
    1845
    21
      }
    1846
    37
      compiled_data->is_test = is_test;
    1847
    37
      ccv_nnc_graph_exec_update_t update = {
    1848
    37
        .parallel_count = parallel_count,
    1849
    37
        .graph = model->graph,
    1850
    37
        .graph_exec_arena = compiled_data->graph_exec_arena,
    1851
    37
      };
    1852
    37
      ccv_cnnp_model_set_is_test(model, is_test, _ccv_cnnp_cmd_update_for_execs, &update);
    1853
    37
      const int evaluate_to_size = compiled_data->evaluate.to_size;
    1854
    37
      compiled_data->evaluate.to_op_size = 0;
    1855
    37
      ccv_array_t* const backward_from = ccv_array_new(sizeof(int), 0, 0);
    1856
    92
      for (i = 0; i < evaluate_to_size; i++55)
    1857
    55
      {
    1858
    55
        ccv_nnc_graph_exec_t const to_op = ccv_nnc_graph_exec_from_symbol(compiled_data->graph_exec_arena, compiled_data->evaluate.tos[i]);
    1859
    55
        if (to_op.graph)
    1860
    55
          compiled_data->evaluate.to_ops[compiled_data->evaluate.to_op_size++] = to_op;
    1861
    55
        const int* tos;
    1862
    55
        int to_size;
    1863
    55
        ccv_nnc_graph_exec_symbol_to(model->graph, compiled_data->evaluate.tos[i], &tos, &to_size);
    1864
    110
        for (j = 0; j < to_size; j++55)
    1865
    55
        {
    1866
    55
          ccv_nnc_graph_exec_t const to_op = ccv_nnc_graph_exec_from_symbol(compiled_data->graph_exec_arena, (ccv_nnc_graph_exec_symbol_t){
    1867
    55
            .d = tos[j],
    1868
    55
            .graph = model->graph
    1869
    55
          });
    1870
    55
          if (to_op.graph)
    1871
    55
            ccv_array_add_unique_int(backward_from, to_op.d);
    1872
    55
        }
    1873
    55
      }
    1874
    37
      assert(backward_from->rnum > 0);
    1875
    37
      compiled_data->backward.from_op_size = backward_from->rnum;
    1876
    37
      compiled_data->backward.from_ops = (ccv_nnc_graph_exec_t*)ccmalloc(sizeof(ccv_nnc_graph_exec_t) * backward_from->rnum);
    1877
    92
      for (i = 0; i < backward_from->rnum; i++55)
    1878
    55
        compiled_data->backward.from_ops[i] = (ccv_nnc_graph_exec_t){
    1879
    55
          .d = *(int*)ccv_array_get(backward_from, i),
    1880
    55
          .graph = compiled_data->graph,
    1881
    55
        };
    1882
      // If there are any set node (to set some tensors to 0) inserted through backward pass, these won't be executed if we just do sources -> evaluate.to_ops, backward.from_ops -> destinations. We need this logic to find out these nodes and explicitly adding them to backward.from_ops.
    1883
    37
      ccv_nnc_graph_exec_info_t* const exec_info = (ccv_nnc_graph_exec_info_t*)ccv_array_get(compiled_data->graph->exec_info, 0);
    1884
    37
      const int exec_info_size = compiled_data->graph->exec_info->rnum;
    1885
    37
      uint32_t* const visited = cccalloc((exec_info_size + 31) >> 5, sizeof(uint32_t));
    1886
    37
      const ccv_nnc_graph_exec_t* const sources = (ccv_nnc_graph_exec_t*)ccv_array_get(compiled_data->graph->sources, 0);
    1887
    37
      const int source_size = compiled_data->graph->sources->rnum;
    1888
    74
      ccv_nnc_graph_visit_t* visit = ccv_nnc_graph_visit_new37(compiled_data->graph, exec_info, exec_info_size, sources, source_size, compiled_data->evaluate.to_ops, compiled_data->evaluate.to_op_size, 0);
    1889
    628
      ccv_nnc_graph_visit_for(visit, exec_info, node, idx) {
    1890
    628
        visited[(idx >> 5)] |= (1u << (idx & 31));
    1891
    628
      } ccv_nnc_graph_visit_endfor
    1892
    74
      ccv_nnc_graph_visit_free(visit);
    1893
    74
      const ccv_nnc_graph_exec_t* const destinations = (ccv_nnc_graph_exec_t*)ccv_array_get37(compiled_data->graph->destinations, 0);
    1894
    74
      const int destination_size = compiled_data->graph->destinations->rnum;
    1895
    74
      visit = ccv_nnc_graph_visit_new37(compiled_data->graph, exec_info, exec_info_size, compiled_data->backward.from_ops, compiled_data->backward.from_op_size, destinations, destination_size, 0);
    1896
    680
      ccv_nnc_graph_visit_for(visit, exec_info, node, idx) {
    1897
    680
        visited[(idx >> 5)] |= (1u << (idx & 31));
    1898
    680
      } ccv_nnc_graph_visit_endfor
    1899
    74
      ccv_nnc_graph_visit_free(visit);
    1900
    74
      visit = ccv_nnc_graph_visit_new37(compiled_data->graph, exec_info, exec_info_size, sources, source_size, destinations, destination_size, 0);
    1901
      // Find any missing nodes to be added as source. Right now, these are only set nodes.
    1902
    1.36k
      ccv_nnc_graph_visit_for(visit, exec_info, node, idx) {
    1903
    1.36k
        if (!(visited[(idx >> 5)] & (1u << (idx & 31))))
    1904
    55
        {
    1905
    55
          assert(exec_info[idx].cmd.cmd == CCV_NNC_SET_FORWARD);
    1906
    55
          if (exec_info[idx].cmd.info.blas.a[0] == 0) // Special-casing for empty out the tensor set function, not for the set grad to 1 one.
    1907
    0
            ccv_array_add_unique_int(backward_from, idx);
    1908
    55
        }
    1909
    1.36k
      } ccv_nnc_graph_visit_endfor
    1910
    37
      ccv_nnc_graph_visit_free(visit);
    1911
    37
      ccfree(visited);
    1912
    37
      if (backward_from->rnum != compiled_data->backward.from_op_size) // If it doesn't match, need to redo this.
    1913
    0
      {
    1914
    0
        compiled_data->backward.from_op_size = backward_from->rnum;
    1915
    0
        compiled_data->backward.from_ops = (ccv_nnc_graph_exec_t*)ccrealloc(compiled_data->backward.from_ops, sizeof(ccv_nnc_graph_exec_t) * backward_from->rnum);
    1916
    0
        for (i = 0; i < backward_from->rnum; i++)
    1917
    0
          compiled_data->backward.from_ops[i] = (ccv_nnc_graph_exec_t){
    1918
    0
            .d = *(int*)ccv_array_get(backward_from, i),
    1919
    0
            .graph = compiled_data->graph,
    1920
    0
          };
    1921
    0
      }
    1922
    37
      ccv_array_free(backward_from);
    1923
    37
      ccv_nnc_graph_set_default_static_schedule(compiled_data->graph, compiled_data->stream_type, model->max_stream_count);
    1924
    37
      ccv_nnc_graph_autotune(compiled_data->graph, model->workspace_size, 0, TRAVERSE_FULL);
    1925
    37
    }
    1926
    1927
    void ccv_cnnp_model_dry_run(ccv_cnnp_model_t* const model, const ccv_cnnp_evaluate_param_t params, ccv_nnc_tensor_t* const* const inputs, const int input_size, ccv_nnc_tensor_t* const* const outputs, const int output_size)
    1928
    8.01k
    {
    1929
    8.01k
      ccv_cnnp_compiled_data_t* const compiled_data = model->compiled_data;
    1930
    8.01k
      assert(compiled_data);
    1931
    8.01k
      const int parallel_count = ccv_max(model->parallel_count, 1);
    1932
    8.01k
      assert(output_size == model->output_size * parallel_count);
    1933
    8.01k
      assert(input_size == model->input_size * parallel_count);
    1934
    8.01k
      assert(model->graph);
    1935
    8.01k
      const int target_gradient_mode = _ccv_cnnp_is_disable_outgrad_all(params.disable_outgrad, model->input_size) ? CCV_CNNP_COMPILED_DATA_GRADIENT_TRAINABLES14 : CCV_CNNP_COMPILED_DATA_GRADIENT_TRAINABLES_AND_INPUTS8.00k;
    1936
    8.01k
      const int mode_mismatch = (params.requires_grad && (7.86kcompiled_data->graph_mode != CCV_CNNP_MODEL_GRAPH_MULTISTAGE_MODE7.86k || compiled_data->gradient_mode != target_gradient_mode7.82k || compiled_data->disable_outgrad != params.disable_outgrad7.82k));
    1937
    8.01k
      if (!compiled_data->graph || mode_mismatch7.91k)
    1938
    103
      {
    1939
    103
        _ccv_cnnp_compiled_data_graph_free(compiled_data);
    1940
    103
        if (mode_mismatch) // If mode mismatch, we need to redo the backward as well (no need to redo apply_gradients, it doesn't require target_gradient_mode or disable_outgrad.
    1941
    37
          _ccv_cnnp_compiled_data_backward_free(compiled_data);
    1942
    103
        if (params.requires_grad)
    1943
    37
          _ccv_cnnp_model_multistage_jit_0(model, params.disable_outgrad, params.is_test, inputs, input_size, outputs, output_size);
    1944
    66
        else
    1945
    66
          _ccv_cnnp_model_multistage_no_grad_jit(model, inputs, input_size, outputs, output_size);
    1946
    7.91k
      } else {
    1947
    7.91k
        ccv_nnc_tensor_arena_clear_bindings(compiled_data->tensor_arena);
    1948
    7.91k
        assert((input_size % parallel_count) == 0);
    1949
    7.91k
        const int input_size_per_p = input_size / parallel_count;
    1950
    7.91k
        _ccv_cnnp_bind_tensors_to_arena(compiled_data->tensor_arena, model->graph, model->inputs, inputs, input_size_per_p, parallel_count);
    1951
    7.91k
        assert((output_size % parallel_count) == 0);
    1952
    7.91k
        const int output_size_per_p = output_size / parallel_count;
    1953
    7.91k
        _ccv_cnnp_bind_tensors_to_arena(compiled_data->tensor_arena, model->graph, model->outputs, outputs, output_size_per_p, parallel_count);
    1954
    7.91k
      }
    1955
    8.01k
      if (compiled_data->is_test != params.is_test)
    1956
    71
      {
    1957
    71
        compiled_data->is_test = params.is_test;
    1958
    71
        ccv_nnc_graph_exec_update_t update = {
    1959
    71
          .parallel_count = parallel_count,
    1960
    71
          .graph = model->graph,
    1961
    71
          .graph_exec_arena = compiled_data->graph_exec_arena,
    1962
    71
        };
    1963
    71
        ccv_cnnp_model_set_is_test(model, params.is_test, _ccv_cnnp_cmd_update_for_execs, &update);
    1964
    71
      }
    1965
    8.01k
    }
    1966
    1967
    void ccv_cnnp_model_evaluate(ccv_cnnp_model_t* const model, const ccv_cnnp_evaluate_param_t params, ccv_nnc_tensor_t* const* const inputs, const int input_size, ccv_nnc_tensor_t* const* const outputs, const int output_size, ccv_nnc_tensor_tape_t* const tensor_tape, ccv_nnc_stream_context_t* const stream_context)
    1968
    8.01k
    {
    1969
    8.01k
      ccv_cnnp_compiled_data_t* const compiled_data = model->compiled_data;
    1970
    8.01k
      assert(compiled_data);
    1971
    8.01k
      ccv_cnnp_model_dry_run(model, params, inputs, input_size, outputs, output_size);
    1972
    8.01k
      if (compiled_data->graph_mode == CCV_CNNP_MODEL_GRAPH_MULTISTAGE_MODE_NO_GRAD)
    1973
    80
        ccv_nnc_graph_run_with_schedule(compiled_data->graph, 0, 0, tensor_tape, stream_context);
    1974
    7.93k
      else {
    1975
    7.93k
        if (!compiled_data->evaluate.schedule)
    1976
    42
          compiled_data->evaluate.schedule = ccv_nnc_graph_static_schedule_new(compiled_data->graph, compiled_data->stream_type, model->max_stream_count, 0, 0, compiled_data->evaluate.to_ops, compiled_data->evaluate.to_op_size);
    1977
    7.93k
        ccv_nnc_graph_run_with_schedule(compiled_data->graph, 0, compiled_data->evaluate.schedule, tensor_tape, stream_context);
    1978
    7.93k
      }
    1979
    8.01k
    }
    1980
    1981
    // Compile the graph to run ccv_cnnp_model_backward after ccv_cnnp_model_evaluate with requires_grad = true (MULTISTAGE_MODE).
    1982
    // Particularly, this method compiles the accumulator graph.
    1983
    static void _ccv_cnnp_model_multistage_jit_1(ccv_cnnp_model_t* const model)
    1984
    5
    {
    1985
    5
      ccv_cnnp_compiled_data_t* const compiled_data = model->compiled_data;
    1986
    5
      assert(compiled_data);
    1987
    5
      assert(compiled_data->graph_mode == CCV_CNNP_MODEL_GRAPH_MULTISTAGE_MODE);
    1988
    5
      ccv_nnc_symbolic_graph_t* accum = ccv_nnc_symbolic_graph_new();
    1989
    5
      const int parallel_count = ccv_max(model->parallel_count, 1);
    1990
    5
      const int parameter_size = compiled_data->parameters->rnum;
    1991
    5
      int i, j;
    1992
    5
      compiled_data->backward.gradients = (ccv_nnc_tensor_symbol_t*)ccmalloc(sizeof(ccv_nnc_tensor_symbol_t) * parameter_size * parallel_count * 3);
    1993
    5
      compiled_data->backward.accum_gradients = compiled_data->backward.gradients + parameter_size * parallel_count;
    1994
    5
      compiled_data->backward.updated_accum_gradients = compiled_data->backward.accum_gradients + parameter_size * parallel_count;
    1995
    20
      for (i = 0; i < parameter_size; i++15)
    1996
    30
        for (j = 0; 15j < parallel_count; j++15)
    1997
    15
          if (compiled_data->tensors.gradients[i + j * parameter_size])
    1998
    15
          {
    1999
    15
            const ccv_nnc_tensor_param_t info = compiled_data->tensors.gradients[i + j * parameter_size]->info;
    2000
            // Now, the old gradient is the accumulated gradient, getting new gradient tensor setup so we can collect them.
    2001
    15
            compiled_data->tensors.accum_gradients[i + j * parameter_size] = compiled_data->tensors.gradients[i + j * parameter_size];
    2002
    15
            compiled_data->tensors.gradients[i + j * parameter_size] = ccv_nnc_tensor_new(0, info, 0);
    2003
    15
            ccv_nnc_tensor_symbol_t inputs[2];
    2004
    15
            inputs[0] = compiled_data->backward.accum_gradients[i + j * parameter_size] = ccv_nnc_tensor_symbol_new(accum, info, 0);
    2005
    15
            inputs[1] = compiled_data->backward.gradients[i + j * parameter_size] = ccv_nnc_tensor_symbol_new(accum, info, 0);
    2006
    15
            ccv_nnc_tensor_symbol_t output = compiled_data->backward.updated_accum_gradients[i + j * parameter_size] = ccv_nnc_tensor_symbol_new(accum, info, 0);
    2007
    15
            ccv_nnc_graph_exec_symbol_new(accum, CMD_EWSUM_FORWARD(), inputs, 2, &output, 1, 0);
    2008
    15
          } else {
    2009
    0
            compiled_data->backward.accum_gradients[i + j * parameter_size] = NO_TENSOR_SYMBOL;
    2010
    0
            compiled_data->backward.gradients[i + j * parameter_size] = NO_TENSOR_SYMBOL;
    2011
    0
            compiled_data->backward.updated_accum_gradients[i + j * parameter_size] = NO_TENSOR_SYMBOL;
    2012
    0
          }
    2013
    5
      ccv_nnc_graph_exec_symbol_autogen(accum, 0, 0, CCV_NNC_AUTOGEN_ALL_EXECS | CCV_NNC_AUTOGEN_SOURCES_AND_DESTINATIONS);
    2014
    5
      if (ccv_nnc_symbolic_graph_source_size(accum) == 0)
    2015
    0
      {
    2016
    0
        ccv_nnc_symbolic_graph_free(accum);
    2017
        // Create empty graph.
    2018
    0
        compiled_data->backward.accum = ccv_nnc_graph_new();
    2019
    0
        ccv_nnc_graph_topsort(compiled_data->backward.accum, 0, 0);
    2020
    0
        return;
    2021
    0
      }
    2022
    5
      ccv_array_t* const tensor_binds = ccv_array_new(sizeof(ccv_nnc_tensor_bind_t), 0, 0);
    2023
    5
      _ccv_cnnp_model_bind_tensors(accum, compiled_data->backward.accum_gradients, compiled_data->tensors.accum_gradients, parameter_size * parallel_count, 1, tensor_binds);
    2024
    5
      _ccv_cnnp_model_bind_tensors(accum, compiled_data->backward.gradients, compiled_data->tensors.gradients, parameter_size * parallel_count, 1, tensor_binds);
    2025
    5
      _ccv_cnnp_model_bind_tensors(accum, compiled_data->backward.updated_accum_gradients, compiled_data->tensors.accum_gradients, parameter_size * parallel_count, 1, tensor_binds);
    2026
    5
      ccv_nnc_symbolic_graph_compile(accum, compiled_data->compile_params, (ccv_nnc_tensor_bind_t*)ccv_array_get(tensor_binds, 0), tensor_binds->rnum, 0, 0, SYMBOLIC_GRAPH_SOURCES(accum), SYMBOLIC_GRAPH_DESTINATIONS(accum), &compiled_data->backward.accum, &compiled_data->backward.tensor_arena, &compiled_data->backward.graph_exec_arena);
    2027
    5
      ccv_nnc_symbolic_graph_free(accum);
    2028
    5
      ccv_array_free(tensor_binds);
    2029
    5
      ccv_nnc_graph_set_default_static_schedule(compiled_data->backward.accum, compiled_data->stream_type, model->max_stream_count);
    2030
    5
    }
    2031
    2032
    void ccv_cnnp_model_backward(ccv_cnnp_model_t* const model, ccv_nnc_tensor_t* const* const ingrads, const int ingrad_size, ccv_nnc_tensor_t* const* const outgrads, const int outgrad_size, ccv_nnc_tensor_tape_t* const tensor_tape, ccv_nnc_stream_context_t* const stream_context)
    2033
    7.91k
    {
    2034
    7.91k
      ccv_cnnp_compiled_data_t* const compiled_data = model->compiled_data;
    2035
    7.91k
      assert(compiled_data);
    2036
    7.91k
      assert(compiled_data->graph_mode == CCV_CNNP_MODEL_GRAPH_MULTISTAGE_MODE);
    2037
    7.91k
      const int parallel_count = ccv_max(model->parallel_count, 1);
    2038
    7.91k
      assert(ingrad_size == 0 || ingrad_size == model->output_size * parallel_count);
    2039
    7.91k
      if (outgrad_size > 0)
    2040
    2.51k
        { assert(outgrad_size == compiled_data->outgrad_size * parallel_count); }
    2041
    7.91k
      assert(model->graph);
    2042
    7.91k
      assert(compiled_data->graph);
    2043
    7.91k
      const int parameter_size = compiled_data->parameters->rnum;
    2044
      // If we need to accumulate the gradients now, do jit on accumulator.
    2045
    7.91k
      if (compiled_data->backward.count > 0)
    2046
    1.71k
      {
    2047
    1.71k
        if (!compiled_data->backward.accum)
    2048
    5
          _ccv_cnnp_model_multistage_jit_1(model);
    2049
    1.71k
        else if (compiled_data->backward.count == 1) {
    2050
          //  On this round, we need to switch accumulated gradients with gradients (so we can do accumulation properly).
    2051
    496
          int i;
    2052
    1.48k
          for (i = 0; i < parameter_size * parallel_count; i++986)
    2053
    986
          {
    2054
    986
            ccv_nnc_tensor_t* tensor;
    2055
    986
            CCV_SWAP(compiled_data->tensors.accum_gradients[i], compiled_data->tensors.gradients[i], tensor);
    2056
    986
          }
    2057
    496
          if (compiled_data->backward.tensor_arena)
    2058
    496
          {
    2059
    496
            ccv_nnc_tensor_arena_clear_bindings(compiled_data->backward.tensor_arena);
    2060
            // Do rebind in case we messed up the binding (we switch accum_gradients and gradients).
    2061
    496
            _ccv_cnnp_bind_tensors_to_arena(compiled_data->backward.tensor_arena, 0, compiled_data->backward.gradients, compiled_data->tensors.gradients, parameter_size * parallel_count, 1);
    2062
    496
            _ccv_cnnp_bind_tensors_to_arena(compiled_data->backward.tensor_arena, 0, compiled_data->backward.accum_gradients, compiled_data->tensors.accum_gradients, parameter_size * parallel_count, 1);
    2063
    496
            _ccv_cnnp_bind_tensors_to_arena(compiled_data->backward.tensor_arena, 0, compiled_data->backward.updated_accum_gradients, compiled_data->tensors.accum_gradients, parameter_size * parallel_count, 1);
    2064
    496
          }
    2065
    496
        }
    2066
    1.71k
      }
    2067
    7.91k
      const int ingrad_size_per_p = model->output_size;
    2068
    7.91k
      const int outgrad_size_per_p = compiled_data->outgrad_size;
    2069
    7.91k
      int i, j;
    2070
    15.8k
      for (i = 0; i < ingrad_size_per_p; i++7.91k)
    2071
    7.91k
      {
    2072
    7.91k
        const ccv_nnc_tensor_symbol_t ingrad = ccv_nnc_tensor_symbol_for_backward(model->graph, compiled_data->f[i]);
    2073
    7.91k
        if (!ingrad_size || !ingrads3.79k || ingrads[i] == 03.79k)
    2074
    4.22k
        {
    2075
          // Set it to 1 if it is not specified.
    2076
    4.22k
          ccv_nnc_tensor_t* const ingrad_tensor = ccv_nnc_tensor_from_symbol(compiled_data->tensor_arena, ingrad);
    2077
    4.22k
          if (ingrad_tensor)
    2078
    4.22k
            ccv_nnc_cmd_exec(CMD_SET_FORWARD(1), ccv_nnc_no_hint, 0, 0, 0, TENSOR_LIST(ingrad_tensor), stream_context);
    2079
    4.34k
          for (j = 1; j < parallel_count; j++120)
    2080
    120
          {
    2081
    120
            ccv_nnc_tensor_t* const ingrad_tensor = ccv_nnc_tensor_from_symbol(compiled_data->tensor_arena, ccv_nnc_tensor_symbol_copy(model->graph, ingrad, j));
    2082
    120
            if (ingrad_tensor)
    2083
    120
              ccv_nnc_cmd_exec(CMD_SET_FORWARD(1), ccv_nnc_no_hint, 0, 0, 0, TENSOR_LIST(ingrad_tensor), stream_context);
    2084
    120
          }
    2085
    4.22k
        } else {
    2086
          // Make sure the length matches, in case it is an alias.
    2087
    3.69k
          assert(ccv_nnc_tensor_count(ingrads[i]->info) == ccv_nnc_tensor_count(ccv_nnc_tensor_symbol_params(model->graph, ingrad)));
    2088
    3.69k
          ccv_nnc_tensor_bind_symbol(compiled_data->tensor_arena, ingrad, ingrads[i]);
    2089
    3.70k
          for (j = 1; j < parallel_count; j++6)
    2090
    6
            ccv_nnc_tensor_bind_symbol(compiled_data->tensor_arena, ccv_nnc_tensor_symbol_copy(model->graph, ingrad, j), ingrads[i + ingrad_size_per_p * j]);
    2091
    3.69k
        }
    2092
    7.91k
      }
    2093
    7.91k
      if (outgrad_size > 0)
    2094
    2.51k
      {
    2095
    2.51k
        assert(compiled_data->gradient_mode == CCV_CNNP_COMPILED_DATA_GRADIENT_TRAINABLES_AND_INPUTS && "shouldn't pass disable_outgrad to ccv_cnnp_model_evaluate before if you plan to compute outgrad");
    2096
    5.14k
        for (i = 0; 2.51ki < outgrad_size_per_p; i++2.63k)
    2097
    2.63k
          if (outgrads[i])
    2098
    2.43k
          {
    2099
    2.43k
            const ccv_nnc_tensor_symbol_t outgrad = compiled_data->outgrads[i];
    2100
    2.43k
            ccv_nnc_tensor_bind_symbol(compiled_data->tensor_arena, outgrad, outgrads[i]);
    2101
    2.44k
            for (j = 1; j < parallel_count; j++6)
    2102
    6
              ccv_nnc_tensor_bind_symbol(compiled_data->tensor_arena, ccv_nnc_tensor_symbol_copy(model->graph, outgrad, j), outgrads[i + outgrad_size_per_p * j]);
    2103
    2.43k
          }
    2104
    5.40k
      } else {
    2105
    5.40k
        assert(compiled_data->gradient_mode == CCV_CNNP_COMPILED_DATA_GRADIENT_TRAINABLES ||
    2106
    5.40k
          compiled_data->gradient_mode == CCV_CNNP_COMPILED_DATA_GRADIENT_TRAINABLES_AND_INPUTS);
    2107
    5.40k
      }
    2108
      // We need to rebind here because in ccv_cnnp_evaluate, we clear bindings, that will reset all bindings for the gradients.
    2109
      // For parameters and internals these are fine because when we clear bindings, it restores to original bindings, which are these
    2110
      // parameters and internals. The same cannot be said for gradients due to the accum_gradients switching.
    2111
    7.91k
      _ccv_cnnp_bind_tensors_to_arena(compiled_data->tensor_arena, model->graph, compiled_data->gradients, compiled_data->tensors.gradients, parameter_size, parallel_count);
    2112
    7.91k
      if (!compiled_data->backward.schedule)
    2113
    31
        compiled_data->backward.schedule = ccv_nnc_graph_static_schedule_new(compiled_data->graph, compiled_data->stream_type, model->max_stream_count, compiled_data->backward.from_ops, compiled_data->backward.from_op_size, 0, 0);
    2114
      // Run the backward pass.
    2115
    7.91k
      ccv_nnc_graph_run_with_schedule(compiled_data->graph, 0, compiled_data->backward.schedule, tensor_tape, stream_context);
    2116
      // If we need to run accumulation round, do that now.
    2117
    7.91k
      if (compiled_data->backward.count > 0)
    2118
    1.71k
        ccv_nnc_graph_run_with_schedule(compiled_data->backward.accum, 0, 0, 0, stream_context);
    2119
      // Update the count, this determines whether we need to accumulate or not.
    2120
    7.91k
      ++compiled_data->backward.count;
    2121
    7.91k
    }
    2122
    2123
    // Compile the graph to run ccv_cnnp_model_apply_gradients after ccv_cnnp_model_backward (MULTISTAGE_MODE).
    2124
    // Particularly, this method compiles the parameter update graph.
    2125
    static void _ccv_cnnp_model_multistage_jit_2(ccv_cnnp_model_t* const model)
    2126
    24
    {
    2127
    24
      ccv_cnnp_compiled_data_t* const compiled_data = model->compiled_data;
    2128
    24
      assert(compiled_data->graph_mode == CCV_CNNP_MODEL_GRAPH_MULTISTAGE_MODE);
    2129
    24
      const int parallel_count = ccv_max(model->parallel_count, 1);
    2130
    24
      const int parameter_size = compiled_data->parameters->rnum;
    2131
    24
      ccv_array_t* const tensor_binds = ccv_array_new(sizeof(ccv_nnc_tensor_bind_t), 0, 0);
    2132
    24
      _ccv_cnnp_model_bind_tensors(model->graph, (ccv_nnc_tensor_symbol_t*)ccv_array_get(compiled_data->parameters, 0), compiled_data->tensors.parameters, parameter_size, parallel_count, tensor_binds);
    2133
    24
      _ccv_cnnp_model_bind_tensors(model->graph, compiled_data->updated_parameters, compiled_data->tensors.parameters, parameter_size, parallel_count, tensor_binds);
    2134
      // Bind accumulated gradients.
    2135
    24
      if (compiled_data->backward.count > 1)
    2136
    4
        _ccv_cnnp_model_bind_tensors(model->graph, compiled_data->gradients, compiled_data->tensors.accum_gradients, parameter_size, parallel_count, tensor_binds);
    2137
    20
      else
    2138
    20
        _ccv_cnnp_model_bind_tensors(model->graph, compiled_data->gradients, compiled_data->tensors.gradients, parameter_size, parallel_count, tensor_binds);
    2139
    24
      ccv_array_t* const apply_gradients_from = ccv_array_new(sizeof(int), 0, 0);
    2140
    24
      int i, j;
    2141
    256
      for (i = 0; i < compiled_data->backward.to_size; i++232)
    2142
    232
      {
    2143
    232
        const int* tos;
    2144
    232
        int to_size;
    2145
    232
        ccv_nnc_graph_exec_symbol_to(model->graph, compiled_data->backward.tos[i], &tos, &to_size);
    2146
    738
        for (j = 0; j < to_size; j++506)
    2147
    506
        {
    2148
          // Check if this is already show up in the backward graph, if that is the case, it won't be in the apply
    2149
          // gradients graph.
    2150
    506
          const ccv_nnc_graph_exec_t exec = ccv_nnc_graph_exec_from_symbol(compiled_data->graph_exec_arena, (ccv_nnc_graph_exec_symbol_t){
    2151
    506
            .d = tos[j],
    2152
    506
            .graph = model->graph,
    2153
    506
          });
    2154
    506
          if (!exec.graph)
    2155
    316
            ccv_array_add_unique_int(apply_gradients_from, tos[j]);
    2156
    506
        }
    2157
    232
      }
    2158
    24
      const int from_size = apply_gradients_from->rnum;
    2159
    24
      if (from_size == 0)
    2160
    0
      {
    2161
    0
        ccv_array_free(apply_gradients_from);
    2162
    0
        ccv_array_free(tensor_binds);
    2163
    0
        return;
    2164
    0
      }
    2165
    24
      ccv_nnc_graph_exec_symbol_t* const froms = (ccv_nnc_graph_exec_symbol_t*)ccmalloc(sizeof(ccv_nnc_graph_exec_symbol_t) * from_size);
    2166
    160
      for (i = 0; i < from_size; i++136)
    2167
    136
        froms[i] = (ccv_nnc_graph_exec_symbol_t){
    2168
    136
          .d = *(int*)ccv_array_get(apply_gradients_from, i),
    2169
    136
          .graph = model->graph
    2170
    136
        };
    2171
    24
      ccv_array_free(apply_gradients_from);
    2172
      // It can only ends with updates on the parameters.
    2173
    24
      ccv_array_t* const tos = ccv_array_new(sizeof(ccv_nnc_graph_exec_symbol_t), parameter_size * parallel_count, 0);
    2174
    160
      for (i = 0;  i < parameter_size; i++136)
    2175
    136
      {
    2176
    136
        if (compiled_data->update_nodes[i].d == CCV_NNC_NO_TENSOR_SYMBOL)
    2177
    0
          continue;
    2178
    136
        ccv_array_push(tos, &compiled_data->update_nodes[i]);
    2179
    316
        for (j = 1; j < parallel_count; j++180)
    2180
    180
        {
    2181
    180
          const ccv_nnc_graph_exec_symbol_t copy = ccv_nnc_graph_exec_symbol_copy(model->graph, compiled_data->update_nodes[i], j);
    2182
    180
          ccv_array_push(tos, &copy);
    2183
    180
        }
    2184
    136
      }
    2185
    24
      ccv_nnc_symbolic_graph_compile(model->graph, compiled_data->compile_params, (ccv_nnc_tensor_bind_t*)ccv_array_get(tensor_binds, 0), tensor_binds->rnum, 0, 0, froms, from_size, (ccv_nnc_graph_exec_symbol_t*)ccv_array_get(tos, 0), tos->rnum, &compiled_data->apply_gradients.graph, &compiled_data->apply_gradients.tensor_arena, &compiled_data->apply_gradients.graph_exec_arena);
    2186
    24
      ccv_array_free(tos);
    2187
    24
      ccv_array_free(tensor_binds);
    2188
    24
      ccfree(froms);
    2189
    24
      const int max_saved_aux_size = compiled_data->minimize.max_saved_aux_size;
    2190
    219
      for (i = 0; i < max_saved_aux_size * parameter_size; i++195)
    2191
    195
      {
    2192
        // Skip on no tensor.
    2193
    195
        if (compiled_data->saved_aux[i].source.d == CCV_NNC_NO_TENSOR_SYMBOL)
    2194
    0
          continue;
    2195
    195
        ccv_nnc_tensor_t* const tensor = ccv_nnc_tensor_from_symbol(compiled_data->apply_gradients.tensor_arena, compiled_data->saved_aux[i].source);
    2196
    195
        ccv_nnc_cmd_exec(CMD_SET_FORWARD(0), ccv_nnc_no_hint, 0, 0, 0, &tensor, 1, 0);
    2197
    543
        for (j = 1; j < parallel_count; j++348)
    2198
    348
        {
    2199
    348
          ccv_nnc_tensor_t* const copy = ccv_nnc_tensor_from_symbol(compiled_data->apply_gradients.tensor_arena, ccv_nnc_tensor_symbol_copy(model->graph, compiled_data->saved_aux[i].source, j));
    2200
    348
          if (copy)
    2201
    348
            ccv_nnc_cmd_exec(CMD_SET_FORWARD(0), ccv_nnc_no_hint, 0, 0, 0, &copy, 1, 0);
    2202
    348
        }
    2203
    195
      }
    2204
    24
      ccv_nnc_graph_set_default_static_schedule(compiled_data->apply_gradients.graph, compiled_data->stream_type, model->max_stream_count);
    2205
    24
    }
    2206
    2207
    void ccv_cnnp_model_apply_gradients(ccv_cnnp_model_t* const model, ccv_nnc_stream_context_t* const stream_context)
    2208
    7.84k
    {
    2209
    7.84k
      ccv_cnnp_compiled_data_t* const compiled_data = model->compiled_data;
    2210
    7.84k
      assert(compiled_data);
    2211
    7.84k
      assert(compiled_data->graph_mode == CCV_CNNP_MODEL_GRAPH_MULTISTAGE_MODE);
    2212
    7.84k
      const int parallel_count = ccv_max(model->parallel_count, 1);
    2213
    7.84k
      assert(model->graph);
    2214
    7.84k
      assert(compiled_data->graph);
    2215
      // Skip if there is no backward pass.
    2216
    7.84k
      if (compiled_data->backward.count <= 0)
    2217
    1.65k
        return;
    2218
      // Skip if there is no parameters.
    2219
    6.19k
      if (compiled_data->parameters->rnum == 0)
    2220
    6
      {
    2221
    6
        compiled_data->backward.count = 0;
    2222
    6
        return;
    2223
    6
      }
    2224
    6.19k
      if (!compiled_data->apply_gradients.graph)
    2225
    24
        _ccv_cnnp_model_multistage_jit_2(model);
    2226
    6.16k
      else {
    2227
    6.16k
        const int parameter_size = compiled_data->parameters->rnum;
    2228
    6.16k
        ccv_nnc_tensor_arena_clear_bindings(compiled_data->apply_gradients.tensor_arena);
    2229
        // Change to bind accum_gradients if we do gradient accumulation (run backward more than once).
    2230
    6.16k
        if (compiled_data->backward.count > 1)
    2231
    497
          _ccv_cnnp_bind_tensors_to_arena(compiled_data->apply_gradients.tensor_arena, model->graph, compiled_data->gradients, compiled_data->tensors.accum_gradients, parameter_size, parallel_count);
    2232
    5.67k
        else
    2233
    5.67k
          _ccv_cnnp_bind_tensors_to_arena(compiled_data->apply_gradients.tensor_arena, model->graph, compiled_data->gradients, compiled_data->tensors.gradients, parameter_size, parallel_count);
    2234
    6.16k
      }
    2235
    6.19k
      if (compiled_data->apply_gradients.graph)
    2236
    6.19k
        ccv_nnc_graph_run_with_schedule(compiled_data->apply_gradients.graph, 0, 0, 0, stream_context);
    2237
      // Reset backward count to 0.
    2238
    6.19k
      compiled_data->backward.count = 0;
    2239
    6.19k
    }
    2240
    2241
    void ccv_cnnp_model_set_parameter(ccv_cnnp_model_t* const model, const ccv_cnnp_model_io_t parameter, const ccv_nnc_tensor_t* const tensor)
    2242
    39
    {
    2243
    39
      ccv_cnnp_compiled_data_t* const compiled_data = model->compiled_data;
    2244
    39
      const int param_sel = parameter->param_sel > 0 ? parameter->param_sel - 18 : parameter->param_sel31;
    2245
    39
      assert(parameter->param_sel != 0);
    2246
    39
      const int tensors_init = !!compiled_data->tensors_init.v;
    2247
    39
      int this_tensor_init = tensors_init;
    2248
    39
      if (!tensors_init)
    2249
    22
        ccv_cnnp_model_tensors_init_0(model, compiled_data);
    2250
    17
      else if ((uintptr_t)compiled_data->tensors_init.v & (uintptr_t)1)
    2251
      // Check if it is not fully allocated, if it is not, init_1.
    2252
    16
        this_tensor_init = 0;
    2253
    39
      ccv_array_t* const parameter_indices = ccv_array_new(sizeof(int), 0, 0);
    2254
    39
      ccv_cnnp_model_add_to_parameter_indices(parameter->model, param_sel, parameter_indices);
    2255
    39
      const int param_ref = parameter->param_ref > 0 ? parameter->param_ref - 138 : parameter->param_ref1;
    2256
    39
      if (param_ref < 0)
    2257
    1
        { assert(parameter_indices->rnum == 1); }
    2258
    38
      else
    2259
    38
        { assert(param_ref < parameter_indices->rnum); }
    2260
    39
      const int d = *(int*)ccv_array_get(parameter_indices, param_ref >= 0 ? param_ref : 0);
    2261
    39
      ccv_array_free(parameter_indices);
    2262
    39
      const int parameter_size = compiled_data->parameters->rnum;
    2263
    39
      assert(d >= 0);
    2264
    39
      assert(d < parameter_size);
    2265
    39
      const int parallel_count = _ccv_cnnp_compiled_data_parallel_count(model, compiled_data);
    2266
    39
      int i;
    2267
    39
      if (!this_tensor_init)
    2268
    38
      {
    2269
    38
        if (compiled_data->tensors.parameters[d])
    2270
    0
        {
    2271
    0
          for (i = 1; i < parallel_count; i++)
    2272
    0
            { assert(compiled_data->tensors.parameters[d + i * parameter_size]); }
    2273
    0
          this_tensor_init = 1;
    2274
    38
        } else {
    2275
    38
          const ccv_nnc_tensor_symbol_t parameter = *(ccv_nnc_tensor_symbol_t*)ccv_array_get(compiled_data->parameters, d);
    2276
    38
          ccv_nnc_tensor_param_t info = ccv_nnc_tensor_symbol_params(parameter.graph, parameter);
    2277
    38
          if (CCV_TENSOR_GET_DEVICE(info.type) == CCV_COMPUTE_DEVICE_ANY)
    2278
    34
            CCV_TENSOR_SET_DEVICE_ID(info.type, 0);
    2279
    38
          const int device_id = CCV_TENSOR_GET_DEVICE_ID(info.type);
    2280
    38
          compiled_data->tensors.parameters[d] = ccv_nnc_tensor_new(0, info, 0);
    2281
    50
          for (i = 1; i < parallel_count; i++12)
    2282
    12
          {
    2283
    12
            if (i != device_id)
    2284
    12
              CCV_TENSOR_SET_DEVICE_ID(info.type, i);
    2285
    0
            else
    2286
    0
              CCV_TENSOR_SET_DEVICE_ID(info.type, 0);
    2287
    12
            compiled_data->tensors.parameters[d + i * parameter_size] = ccv_nnc_tensor_new(0, info, 0);
    2288
    12
          }
    2289
    38
        }
    2290
    38
      }
    2291
    39
      ccv_nnc_tensor_t* const dest = CCV_NNC_TENSOR(compiled_data->tensors.parameters[d]);
    2292
    39
      assert(dest);
    2293
    39
      ccv_nnc_cmd_exec(CMD_DATA_TRANSFER_FORWARD(), ccv_nnc_no_hint, 0, TENSOR_LIST((ccv_nnc_tensor_t*)tensor), TENSOR_LIST(dest), 0);
    2294
    51
      for (i = 1; i < parallel_count; i++12)
    2295
    12
      {
    2296
    12
        ccv_nnc_tensor_t* const copy_tensor = CCV_NNC_TENSOR(compiled_data->tensors.parameters[d + i * parameter_size]);
    2297
    12
        if (copy_tensor)
    2298
    12
          ccv_nnc_cmd_exec(CMD_DATA_TRANSFER_FORWARD(), ccv_nnc_no_hint, 0, TENSOR_LIST(dest), TENSOR_LIST(copy_tensor), 0);
    2299
    12
      }
    2300
      // Mark this symbol as init'ed.
    2301
    39
      const int s = ((ccv_nnc_tensor_symbol_t*)ccv_array_get(compiled_data->parameters, d))->d;
    2302
    39
      uint32_t* const init_v = CCV_NNC_INIT_V(compiled_data->tensors_init.v);
    2303
    39
      init_v[s >> 5] |= (1u << (s & 0x1f));
    2304
      // If we just allocated this tensor, now it is time to check if we need to mark it as fully allocated.
    2305
    39
      if (!this_tensor_init)
    2306
    38
      {
    2307
    38
        if (ccv_cnnp_model_tensors_any_to_alloc(model, compiled_data))
    2308
    21
          compiled_data->tensors_init.v = (uint32_t*)((uintptr_t)compiled_data->tensors_init.v | (uintptr_t)1);
    2309
    17
        else // Remove the flag.
    2310
    17
          compiled_data->tensors_init.v = CCV_NNC_INIT_V(compiled_data->tensors_init.v);
    2311
    38
      }
    2312
    39
    }
    2313
    2314
    void ccv_cnnp_model_parameter_copy(ccv_cnnp_model_t* const model, const ccv_cnnp_model_io_t parameter, ccv_nnc_tensor_t* const tensor)
    2315
    6
    {
    2316
    6
      ccv_cnnp_compiled_data_t* const compiled_data = model->compiled_data;
    2317
    6
      const int param_sel = parameter->param_sel > 0 ? parameter->param_sel - 13 : parameter->param_sel3;
    2318
    6
      assert(parameter->param_sel != 0);
    2319
    6
      assert(compiled_data->tensors.parameters);
    2320
    6
      ccv_array_t* const parameter_indices = ccv_array_new(sizeof(int), 0, 0);
    2321
    6
      ccv_cnnp_model_add_to_parameter_indices(parameter->model, param_sel, parameter_indices);
    2322
    6
      const int param_ref = parameter->param_ref > 0 ? parameter->param_ref - 13 : parameter->param_ref3;
    2323
    6
      if (param_ref < 0)
    2324
    3
        { assert(parameter_indices->rnum == 1); }
    2325
    3
      else
    2326
    3
        { assert(param_ref < parameter_indices->rnum); }
    2327
    6
      const int d = *(int*)ccv_array_get(parameter_indices, param_ref >= 0 ? param_ref : 0);
    2328
    6
      ccv_array_free(parameter_indices);
    2329
    6
      const int parameter_size = compiled_data->parameters->rnum;
    2330
    6
      assert(d >= 0);
    2331
    6
      assert(d < parameter_size);
    2332
      // We don't need to consider parallel_count, every parameter on each device is identical.
    2333
    6
      ccv_nnc_tensor_t* const src = CCV_NNC_TENSOR(compiled_data->tensors.parameters[d]);
    2334
    6
      assert(src);
    2335
    6
      ccv_nnc_cmd_exec(CMD_DATA_TRANSFER_FORWARD(), ccv_nnc_no_hint, 0, TENSOR_LIST(src), TENSOR_LIST(tensor), 0);
    2336
    6
    }
    2337
    2338
    ccv_nnc_tensor_param_t ccv_cnnp_model_parameter_tensor_params(ccv_cnnp_model_t* const model, const ccv_cnnp_model_io_t parameter)
    2339
    1
    {
    2340
    1
      ccv_cnnp_compiled_data_t* const compiled_data = model->compiled_data;
    2341
    1
      const int param_sel = parameter->param_sel > 0 ? parameter->param_sel - 10 : parameter->param_sel;
    2342
    1
      assert(parameter->param_sel != 0);
    2343
    1
      assert(compiled_data->tensors.parameters);
    2344
    1
      ccv_array_t* const parameter_indices = ccv_array_new(sizeof(int), 0, 0);
    2345
    1
      ccv_cnnp_model_add_to_parameter_indices(parameter->model, param_sel, parameter_indices);
    2346
    1
      const int param_ref = parameter->param_ref > 0 ? parameter->param_ref - 10 : parameter->param_ref;
    2347
    1
      if (param_ref < 0)
    2348
    1
        { assert(parameter_indices->rnum == 1); }
    2349
    0
      else
    2350
    0
        { assert(param_ref < parameter_indices->rnum); }
    2351
    1
      const int d = *(int*)ccv_array_get(parameter_indices, param_ref >= 0 ? param_ref : 0);
    2352
    1
      ccv_array_free(parameter_indices);
    2353
    1
      const int parameter_size = compiled_data->parameters->rnum;
    2354
    1
      assert(d >= 0);
    2355
    1
      assert(d < parameter_size);
    2356
      // We don't need to consider parallel_count, every parameter on each device is identical.
    2357
    1
      ccv_nnc_tensor_t* const tensor = CCV_NNC_TENSOR(compiled_data->tensors.parameters[d]);
    2358
    1
      assert(tensor);
    2359
    1
      return tensor->info;
    2360
    1
    }
    2361
    2362
    const char* ccv_cnnp_model_parameter_name(ccv_cnnp_model_t* const model, const ccv_cnnp_model_io_t parameter)
    2363
    2
    {
    2364
    2
      ccv_cnnp_compiled_data_t* const compiled_data = model->compiled_data;
    2365
    2
      const int param_sel = parameter->param_sel > 0 ? parameter->param_sel - 1 : parameter->param_sel0;
    2366
    2
      assert(parameter->param_sel != 0);
    2367
    2
      ccv_array_t* const parameter_indices = ccv_array_new(sizeof(int), 0, 0);
    2368
    2
      ccv_cnnp_model_add_to_parameter_indices(parameter->model, param_sel, parameter_indices);
    2369
    2
      const int param_ref = parameter->param_ref > 0 ? parameter->param_ref - 1 : parameter->param_ref0;
    2370
    2
      if (param_ref < 0)
    2371
    0
        { assert(parameter_indices->rnum == 1); }
    2372
    2
      else
    2373
    2
        { assert(param_ref < parameter_indices->rnum); }
    2374
    2
      const int d = *(int*)ccv_array_get(parameter_indices, param_ref >= 0 ? param_ref : 0);
    2375
    2
      ccv_array_free(parameter_indices);
    2376
    2
      const int parameter_size = compiled_data->parameters->rnum;
    2377
    2
      assert(d >= 0);
    2378
    2
      assert(d < parameter_size);
    2379
    2
      return *(char**)ccv_array_get(compiled_data->ids.parameters, d);
    2380
    2
    }
    2381
    2382
    int ccv_cnnp_model_parameter_count(ccv_cnnp_model_t* const model)
    2383
    0
    {
    2384
    0
      assert(model->compiled_data);
    2385
    0
      ccv_cnnp_compiled_data_t* const compiled_data = model->compiled_data;
    2386
    0
      return compiled_data->parameters->rnum;
    2387
    0
    }
    2388
    2389
    uint64_t ccv_cnnp_model_parameters_size(ccv_cnnp_model_t* const model)
    2390
    0
    {
    2391
    0
      assert(model->compiled_data);
    2392
    0
      ccv_cnnp_compiled_data_t* const compiled_data = model->compiled_data;
    2393
    0
      const int parameter_size = compiled_data->parameters->rnum;
    2394
    0
      int i;
    2395
    0
      const ccv_nnc_symbolic_graph_t* const graph = model->graph;
    2396
    0
      uint64_t size = 0;
    2397
    0
      const int tensors_init = !!compiled_data->tensors_init.v;
    2398
    0
      uint32_t* const init_v = tensors_init ? CCV_NNC_INIT_V(compiled_data->tensors_init.v) : 0;
    2399
    0
      for (i = 0; i < parameter_size; i++)
    2400
    0
      {
    2401
    0
        const int d = ((ccv_nnc_tensor_symbol_t*)ccv_array_get(compiled_data->parameters, i))->d;
    2402
    0
        if (tensors_init && compiled_data->tensors.parameters && (init_v[d >> 5] | (1u << (d & 0x1f))) && compiled_data->tensors.parameters[i])
    2403
    0
        {
    2404
    0
          ccv_nnc_tensor_param_t params = compiled_data->tensors.parameters[i]->info;
    2405
    0
          size += ccv_nnc_tensor_data_size(params);
    2406
    0
          continue;
    2407
    0
        }
    2408
    0
        ccv_nnc_tensor_param_t params = ccv_nnc_tensor_symbol_params(graph, (ccv_nnc_tensor_symbol_t){
    2409
    0
          .graph = graph,
    2410
    0
          .d = d
    2411
    0
        });
    2412
    0
        size += ccv_nnc_tensor_data_size(params);
    2413
    0
      }
    2414
    0
      return size;
    2415
    0
    }
    2416
    2417
    int ccv_cnnp_model_parameters_move(ccv_cnnp_model_t* const model, char** const names, ccv_nnc_tensor_t** const tensors, const int count, int type)
    2418
    3
    {
    2419
    3
      assert(model->compiled_data);
    2420
    3
      ccv_cnnp_compiled_data_t* const compiled_data = model->compiled_data;
    2421
    3
      if (count != compiled_data->parameters->rnum)
    2422
    0
        return 0;
    2423
    3
      if (CCV_TENSOR_GET_DEVICE(type) == CCV_COMPUTE_DEVICE_ANY)
    2424
    0
        CCV_TENSOR_SET_DEVICE_ID(type, 0);
    2425
    3
      int i;
    2426
      // We don't need to consider parallel_count, every parameter on each device is identical.
    2427
    6
      for (i = 0; i < count; i++3)
    2428
    3
      {
    2429
    3
        ccv_nnc_tensor_t* tensor = compiled_data->tensors.parameters[i];
    2430
    3
        if ((uintptr_t)tensor & (uintptr_t)1) // If it is not owned. We don't do anything.
    2431
    0
        {
    2432
    0
          tensors[i] = 0;
    2433
    0
          continue;
    2434
    0
        }
    2435
    3
        tensor = CCV_NNC_TENSOR(tensor);
    2436
    3
        if (tensor->info.type == type)
    2437
    3
          tensors[i] = tensor;
    2438
    0
        else {
    2439
    0
          ccv_nnc_tensor_param_t info = tensor->info;
    2440
    0
          info.type = type;
    2441
    0
          tensors[i] = ccv_nnc_tensor_new(0, info, 0); // Create this tensor, don't initiate copy yet.
    2442
    0
        }
    2443
    3
      }
    2444
    6
      for (i = 0; i < count; i++3)
    2445
    3
      {
    2446
    3
        ccv_nnc_tensor_t* tensor = compiled_data->tensors.parameters[i];
    2447
    3
        if ((uintptr_t)tensor & (uintptr_t)1) // If it is not owned. We don't do anything.
    2448
    0
          continue;
    2449
    3
        tensor = CCV_NNC_TENSOR(tensor);
    2450
        // Now initiate transfer. We should do this one on a stream.
    2451
    3
        if (tensor->info.type != type)
    2452
    0
          ccv_nnc_cmd_exec(CMD_DATA_TRANSFER_FORWARD(), ccv_nnc_no_hint, 0, TENSOR_LIST(tensor), TENSOR_LIST(tensors[i]), 0);
    2453
    3
      }
    2454
      // Copy names and remove parameters.
    2455
    6
      for (i = 0; i < count; i++3)
    2456
    3
      {
    2457
    3
        ccv_nnc_tensor_t* const tensor = compiled_data->tensors.parameters[i];
    2458
    3
        if ((uintptr_t)tensor & (uintptr_t)1) // If it is not owned. We don't do anything.
    2459
    0
        {
    2460
    0
          names[i] = 0;
    2461
    0
          continue;
    2462
    0
        }
    2463
    3
        const char* const name = *(char**)ccv_array_get(compiled_data->ids.parameters, i);
    2464
    3
        const size_t name_len = ccv_min(strnlen(name, 1023), 1023);
    2465
    3
        names[i] = ccmalloc(name_len + 1);
    2466
    3
        names[i][name_len] = 0;
    2467
    3
        memcpy(names[i], name, name_len);
    2468
    3
        if (tensor->info.type == type)
    2469
    3
          compiled_data->tensors.parameters[i] = 0; // Only move when it is moved.
    2470
    3
      }
    2471
    3
      return 1;
    2472
    3
    }
    2473
    2474
    KHASH_MAP_INIT_STR(ccv_cnnp_parameter_id, int)
    2475
    2476
    void ccv_cnnp_model_set_parameters_from_key_values(ccv_cnnp_model_t* const model, char* const* const names, ccv_nnc_tensor_t** const tensors, const int count, const int invalidates)
    2477
    2
    {
    2478
    2
      assert(model->compiled_data);
    2479
    2
      ccv_cnnp_compiled_data_t* const compiled_data = model->compiled_data;
    2480
    2
      int i;
    2481
    2
      khash_t(ccv_cnnp_parameter_id)* id_map = 0;
    2482
    2
      if (count != compiled_data->parameters->rnum)
    2483
    0
      {
    2484
    0
        id_map = kh_init(ccv_cnnp_parameter_id);
    2485
        // Build the map between name and the index.
    2486
    0
        for (i = 0; i < count; i++)
    2487
    0
        {
    2488
    0
          int ret;
    2489
    0
          const khiter_t k = kh_put(ccv_cnnp_parameter_id, id_map, names[i], &ret);
    2490
    0
          assert(ret != 0);
    2491
    0
          kh_val(id_map, k) = i;
    2492
    0
        }
    2493
    0
      }
    2494
    2
      const int parameter_size = compiled_data->parameters->rnum;
    2495
    2
      int* copy_back = 0;
    2496
    2
      const int tensors_init = !!compiled_data->tensors_init.v;
    2497
    2
      if (!tensors_init)
    2498
    1
        ccv_cnnp_model_tensors_init_0(model, compiled_data);
    2499
    2
      const int parallel_count = ccv_max(model->parallel_count, 1);
    2500
    2
      uint32_t* const init_v = CCV_NNC_INIT_V(compiled_data->tensors_init.v);
    2501
    4
      for (i = 0; i < parameter_size; i++2)
    2502
    2
      {
    2503
    2
        int j = i;
    2504
    2
        const char* const name = *(char**)ccv_array_get(compiled_data->ids.parameters, i);
    2505
    2
        if (i >= 0 || strncmp(name, names[i], 1023) != 00)
    2506
    2
        {
    2507
          // Build the map.
    2508
    2
          if (id_map == 0)
    2509
    2
          {
    2510
    2
            id_map = kh_init(ccv_cnnp_parameter_id);
    2511
    4
            for (j = 0; j < count; j++2)
    2512
    2
            {
    2513
    2
              int ret;
    2514
    2
              const khiter_t k = kh_put(ccv_cnnp_parameter_id, id_map, names[j], &ret);
    2515
    2
              assert(ret != 0);
    2516
    2
              kh_val(id_map, k) = j;
    2517
    2
            }
    2518
    2
          }
    2519
    2
          const khiter_t k = kh_get(ccv_cnnp_parameter_id, id_map, name);
    2520
    2
          if (k == kh_end(id_map)) // Cannot find the name, skip.
    2521
    0
            continue;
    2522
    2
          j = kh_val(id_map, k);
    2523
    2
        }
    2524
    2
        if (compiled_data->tensors.parameters[i]) // Cannot be a shared parameter to read.
    2525
    0
          { assert(!((uintptr_t)compiled_data->tensors.parameters[i] & (uintptr_t)1)); }
    2526
    2
        const ccv_nnc_tensor_symbol_t parameter = *(ccv_nnc_tensor_symbol_t*)ccv_array_get(compiled_data->parameters, i);
    2527
    2
        ccv_nnc_tensor_param_t info = ccv_nnc_tensor_symbol_params(parameter.graph, parameter);
    2528
    2
        if (CCV_TENSOR_GET_DEVICE(info.type) == CCV_COMPUTE_DEVICE_ANY)
    2529
    1
          CCV_TENSOR_SET_DEVICE_ID(info.type, 0);
    2530
    2
        const int d = parameter.d;
    2531
    2
        if (info.type == tensors[j]->info.type && invalidates) // Can move.
    2532
    1
        {
    2533
          // Deallocate it if needed.
    2534
    1
          if (!((uintptr_t)compiled_data->tensors.parameters[i] & (uintptr_t)1))
    2535
    1
            if (compiled_data->tensors.parameters[i])
    2536
    0
              ccv_nnc_tensor_free(compiled_data->tensors.parameters[i]);
    2537
    1
          compiled_data->tensors.parameters[i] = tensors[j];
    2538
    1
          tensors[j] = 0;
    2539
    1
        } else {
    2540
    1
          if (!compiled_data->tensors.parameters[i])
    2541
    1
          { // Not allocated, to allocate first.
    2542
            // Create new one, make sure we create this by having the right parameters.
    2543
    1
            const int type = info.type;
    2544
    1
            info = tensors[j]->info;
    2545
    1
            info.type = type; // Revert back the type.
    2546
    1
            compiled_data->tensors.parameters[i] = ccv_nnc_tensor_new(0, info, 0);
    2547
    1
          }
    2548
    1
          if (!copy_back)
    2549
    1
            copy_back = (int*)cccalloc(parameter_size, sizeof(int));
    2550
    1
          copy_back[i] = j + 1;
    2551
    1
        }
    2552
    2
        init_v[d >> 5] |= (1u << (d & 0x1f));
    2553
        // Create this tensor for other data parallel allocations.
    2554
    2
        info = compiled_data->tensors.parameters[i]->info; // In case we loaded a different info.
    2555
    2
        const int device_id = CCV_TENSOR_GET_DEVICE_ID(info.type);
    2556
    2
        for (j = 1; j < parallel_count; j++0)
    2557
    0
          if (!compiled_data->tensors.parameters[i + j * parameter_size])
    2558
    0
          {
    2559
    0
            if (j != device_id)
    2560
    0
              CCV_TENSOR_SET_DEVICE_ID(info.type, j);
    2561
    0
            else
    2562
    0
              CCV_TENSOR_SET_DEVICE_ID(info.type, 0);
    2563
    0
            compiled_data->tensors.parameters[i + j * parameter_size] = ccv_nnc_tensor_new(0, info, 0);
    2564
    0
          }
    2565
          // No need to copy over, this is done in ccv_cnnp_model.c's copy_tensors method.
    2566
    2
      }
    2567
    2
      if (id_map)
    2568
    2
        kh_destroy(ccv_cnnp_parameter_id, id_map);
    2569
      // Now do the transfer.
    2570
    2
      if (copy_back)
    2571
    1
      {
    2572
    2
        for (i = 0; i < parameter_size; i++1)
    2573
    1
        {
    2574
    1
          ccv_nnc_tensor_t* const tensor = CCV_NNC_TENSOR(compiled_data->tensors.parameters[i]);
    2575
    1
          if (copy_back[i] == 0)
    2576
    0
            continue;
    2577
    1
          const int j = copy_back[i] - 1;
    2578
    1
          ccv_nnc_cmd_exec(CMD_DATA_TRANSFER_FORWARD(), ccv_nnc_no_hint, 0, TENSOR_LIST(tensors[j]), TENSOR_LIST(tensor), 0);
    2579
    1
        }
    2580
    1
        ccfree(copy_back);
    2581
    1
      }
    2582
    2
    }
    2583
    2584
    ccv_cnnp_model_io_t ccv_cnnp_model_parameter_first(ccv_cnnp_model_t* const model, ccv_cnnp_model_parameters_filter_f first, void* const context)
    2585
    0
    {
    2586
    0
      ccv_cnnp_compiled_data_t* const compiled_data = model->compiled_data;
    2587
    0
      assert(compiled_data);
    2588
    0
      const int parameter_size = compiled_data->parameters->rnum;
    2589
    0
      int i;
    2590
    0
      for (i = 0; i < parameter_size; i++)
    2591
    0
      {
    2592
    0
        const char* const name = *(char**)ccv_array_get(compiled_data->ids.parameters, i);
    2593
    0
        if (first(model, name, context))
    2594
    0
          return ccv_cnnp_model_parameters(model, -1, i);
    2595
    0
      }
    2596
    0
      return 0;
    2597
    0
    }
    2598
    2599
    ccv_array_t* ccv_cnnp_model_parameters_filter(ccv_cnnp_model_t* const model, ccv_cnnp_model_parameters_filter_f filter, void* const context)
    2600
    0
    {
    2601
    0
      ccv_cnnp_compiled_data_t* const compiled_data = model->compiled_data;
    2602
    0
      assert(compiled_data);
    2603
    0
      ccv_array_t* const parameters = ccv_array_new(sizeof(ccv_cnnp_model_io_t), 0, 0);
    2604
    0
      const int parameter_size = compiled_data->parameters->rnum;
    2605
    0
      int i;
    2606
    0
      for (i = 0; i < parameter_size; i++)
    2607
    0
      {
    2608
    0
        const char* const name = *(char**)ccv_array_get(compiled_data->ids.parameters, i);
    2609
    0
        if (filter(model, name, context))
    2610
    0
        {
    2611
    0
          ccv_cnnp_model_io_t parameter = ccv_cnnp_model_parameters(model, -1, i);
    2612
    0
          ccv_array_push(parameters, &parameter);
    2613
    0
        }
    2614
    0
      }
    2615
    0
      return parameters;
    2616
    2617
    0
    }
    2618
    2619
    CCV_WARN_UNUSED(ccv_cnnp_model_io_t) ccv_cnnp_model_parameter_first_uninit(ccv_cnnp_model_t* const model)
    2620
    0
    {
    2621
    0
      ccv_cnnp_compiled_data_t* const compiled_data = model->compiled_data;
    2622
    0
      assert(compiled_data);
    2623
    0
      const int tensors_init = !!compiled_data->tensors_init.v;
    2624
    0
      if (!tensors_init) // If nothing initialized, we return parameter 0.
    2625
    0
        return ccv_cnnp_model_parameters(model, -1, 0);
    2626
    0
      const int parameter_size = compiled_data->parameters->rnum;
    2627
    0
      int i;
    2628
    0
      const uint32_t* const init_v = CCV_NNC_INIT_V(compiled_data->tensors_init.v);
    2629
    0
      for (i = 0; i < parameter_size; i++)
    2630
    0
      {
    2631
    0
        const int d = ((ccv_nnc_tensor_symbol_t*)ccv_array_get(compiled_data->parameters, i))->d;
    2632
    0
        if (!(init_v[d >> 5] & (1u << (d & 0x1f))))
    2633
    0
          return ccv_cnnp_model_parameters(model, -1, i);
    2634
    0
      }
    2635
    0
      return 0;
    2636
    0
    }
    2637
    2638
    static ccv_array_t* _ccv_cnnp_model_parameter_indices(const ccv_cnnp_model_t* const model, const ccv_cnnp_model_io_t parameters, int* const param_ref)
    2639
    49
    {
    2640
    49
      const int to_param_sel = parameters->param_sel > 0 ? parameters->param_sel - 10 : parameters->param_sel;
    2641
    49
      assert(parameters->param_sel != 0);
    2642
    49
      ccv_array_t* const to_parameter_indices = ccv_array_new(sizeof(int), 0, 0);
    2643
    49
      ccv_cnnp_model_add_to_parameter_indices(parameters->model, to_param_sel, to_parameter_indices);
    2644
    49
      *param_ref = parameters->param_ref > 0 ? parameters->param_ref - 10 : parameters->param_ref;
    2645
    49
      return to_parameter_indices;
    2646
    49
    }
    2647
    2648
    static void _ccv_cnnp_model_to_parameter_indices_and_from_parameter_indices(ccv_cnnp_model_t* const model, const ccv_cnnp_model_io_t parameters, const ccv_cnnp_model_t* const from_model, const ccv_cnnp_model_io_t from_parameters, ccv_array_t** const parameter_indices, int* const param_ref, ccv_array_t** const from_parameter_indices, int* const from_param_ref, const int only_init_0)
    2649
    14
    {
    2650
      // If the model is not compiled yet. Compile them now.
    2651
    14
      if (!model->graph)
    2652
    3
      {
    2653
    3
        model->graph = ccv_nnc_symbolic_graph_new();
    2654
    3
        assert(from_model->compiled_data);
    2655
    3
        const int input_size = from_model->input_size;
    2656
    3
        ccv_nnc_tensor_param_t input_params[input_size];
    2657
    3
        int i;
    2658
    9
        for (i = 0; i < input_size; i++6)
    2659
    6
          input_params[i] = ccv_nnc_tensor_symbol_params(from_model->graph, from_model->inputs[i]);
    2660
    3
        _ccv_cnnp_model_compile(model, input_params, input_size, from_model->compiled_data->loss);
    2661
    3
        model->parallel_count = from_model->parallel_count;
    2662
    3
        model->memory_compression = from_model->memory_compression;
    2663
    3
        model->memory_reduction = from_model->memory_reduction;
    2664
    3
        model->gradient_checkpointing = from_model->gradient_checkpointing;
    2665
    3
        model->compiled_data->stream_type = from_model->compiled_data->stream_type;
    2666
    3
        model->compiled_data->minimize.minimizer = from_model->compiled_data->minimize.minimizer;
    2667
    3
        model->compiled_data->minimize.max_saved_aux_size = from_model->compiled_data->minimize.max_saved_aux_size;
    2668
    3
      }
    2669
    14
      ccv_cnnp_compiled_data_t* const to_compiled_data = model->compiled_data;
    2670
    14
      assert(to_compiled_data);
    2671
    14
      const int to_tensors_init = !!to_compiled_data->tensors_init.v;
    2672
    14
      if (!to_tensors_init)
    2673
    10
      {
    2674
    10
        if (only_init_0)
    2675
    1
          ccv_cnnp_model_tensors_init_0(model, to_compiled_data);
    2676
    9
        else
    2677
    9
          _ccv_cnnp_model_tensors_init(model, to_compiled_data);
    2678
    10
      } else if (4!only_init_04 && (uintptr_t)to_compiled_data->tensors_init.v & (uintptr_t)13)
    2679
        // Check if it is not fully allocated, if it is not, init_1.
    2680
    0
          ccv_cnnp_model_tensors_init_1(model, to_compiled_data);
    2681
    14
      assert(to_compiled_data->tensors.parameters);
    2682
    14
      *parameter_indices = _ccv_cnnp_model_parameter_indices(model, parameters, param_ref);
    2683
    14
      *from_parameter_indices = _ccv_cnnp_model_parameter_indices(from_model, from_parameters, from_param_ref);
    2684
    14
      if (*from_param_ref < 0 && *param_ref >= 0)
    2685
    0
        { assert((*from_parameter_indices)->rnum == 1); }
    2686
    14
      else if (*from_param_ref >= 0)
    2687
    0
        { assert(*from_param_ref < (*from_parameter_indices)->rnum); }
    2688
    14
      if (*param_ref < 0 && *from_param_ref >= 0)
    2689
    0
        { assert((*parameter_indices)->rnum == 1); }
    2690
    14
      else if (*param_ref >= 0)
    2691
    0
        { assert(*param_ref < (*parameter_indices)->rnum); }
    2692
    14
    }
    2693
    2694
    void ccv_cnnp_model_set_parameters(ccv_cnnp_model_t* const model, const ccv_cnnp_model_io_t parameters, const ccv_cnnp_model_t* const from_model, const ccv_cnnp_model_io_t from_parameters)
    2695
    9
    {
    2696
    9
      ccv_array_t* to_parameter_indices;
    2697
    9
      int to_param_ref;
    2698
    9
      ccv_array_t* from_parameter_indices;
    2699
    9
      int from_param_ref;
    2700
    9
      _ccv_cnnp_model_to_parameter_indices_and_from_parameter_indices(model, parameters, from_model, from_parameters, &to_parameter_indices, &to_param_ref, &from_parameter_indices, &from_param_ref, 0);
    2701
      // Should be exactly the same tensor.
    2702
    9
      if (to_param_ref < 0 && from_param_ref < 0)
    2703
    9
        { assert(from_parameter_indices->rnum == to_parameter_indices->rnum); }
    2704
      // To models.
    2705
    9
      ccv_cnnp_compiled_data_t* const to_compiled_data = model->compiled_data;
    2706
    9
      assert(to_compiled_data);
    2707
      // From models.
    2708
    9
      const ccv_cnnp_compiled_data_t* const from_compiled_data = from_model->compiled_data;
    2709
    9
      const int parallel_count = ccv_max(model->parallel_count, 1);
    2710
    9
      const int to_parameter_size = to_compiled_data->parameters->rnum;
    2711
    9
      const int rnum = (to_param_ref < 0 && from_param_ref < 0) ? from_parameter_indices->rnum : 10;
    2712
    9
      int i, j;
    2713
    9
      const uint32_t* const from_init_v = CCV_NNC_INIT_V(from_compiled_data->tensors_init.v);
    2714
    9
      uint32_t* const to_init_v = CCV_NNC_INIT_V(to_compiled_data->tensors_init.v);
    2715
    18
      for (i = 0; i < rnum; i++9)
    2716
    9
      {
    2717
    9
        const int src_d = *(int*)ccv_array_get(from_parameter_indices,from_param_ref >= 0 ? from_param_ref : i);
    2718
    9
        assert(src_d >= 0);
    2719
    9
        assert(src_d < from_compiled_data->parameters->rnum);
    2720
    9
        const int s = ((ccv_nnc_tensor_symbol_t*)ccv_array_get(from_compiled_data->parameters, src_d))->d;
    2721
        // If the original is not init'ed. We cannot copy from.
    2722
    9
        if (!(from_init_v[s >> 5] & (1u << (s & 0x1f))))
    2723
    0
          continue;
    2724
    9
        const int dest_d = *(int*)ccv_array_get(to_parameter_indices, to_param_ref >= 0 ? to_param_ref : i);
    2725
    9
        assert(dest_d >= 0);
    2726
    9
        assert(dest_d < to_compiled_data->parameters->rnum);
    2727
    9
        ccv_nnc_tensor_t* const src = CCV_NNC_TENSOR(from_compiled_data->tensors.parameters[src_d]);
    2728
    9
        assert(src);
    2729
    9
        ccv_nnc_tensor_t* const dest = CCV_NNC_TENSOR(to_compiled_data->tensors.parameters[dest_d]);
    2730
    9
        assert(dest);
    2731
    9
        ccv_nnc_cmd_exec(CMD_DATA_TRANSFER_FORWARD(), ccv_nnc_no_hint, 0, TENSOR_LIST(src), TENSOR_LIST(dest), 0);
    2732
    27
        for (j = 1; j < parallel_count; j++18)
    2733
    18
        {
    2734
    18
          ccv_nnc_tensor_t* const copy_tensor = CCV_NNC_TENSOR(to_compiled_data->tensors.parameters[dest_d + j * to_parameter_size]);
    2735
    18
          if (copy_tensor)
    2736
    18
            ccv_nnc_cmd_exec(CMD_DATA_TRANSFER_FORWARD(), ccv_nnc_no_hint, 0, TENSOR_LIST(dest), TENSOR_LIST(copy_tensor), 0);
    2737
    18
        }
    2738
        // Mark this symbol as init'ed.
    2739
    9
        const int d = ((ccv_nnc_tensor_symbol_t*)ccv_array_get(to_compiled_data->parameters, dest_d))->d;
    2740
    9
        to_init_v[d >> 5] |= (1u << (d & 0x1f));
    2741
    9
      }
    2742
    9
      ccv_array_free(to_parameter_indices);
    2743
    9
      ccv_array_free(from_parameter_indices);
    2744
    9
    }
    2745
    2746
    void ccv_cnnp_model_share_parameters(ccv_cnnp_model_t* const model, const ccv_cnnp_model_io_t parameters, const ccv_cnnp_model_t* const from_model, const ccv_cnnp_model_io_t from_parameters, ccv_cnnp_model_parameters_renamer_f renamer, void* const context)
    2747
    2
    {
    2748
    2
      ccv_array_t* to_parameter_indices;
    2749
    2
      int to_param_ref;
    2750
    2
      ccv_array_t* from_parameter_indices;
    2751
    2
      int from_param_ref;
    2752
    2
      _ccv_cnnp_model_to_parameter_indices_and_from_parameter_indices(model, parameters, from_model, from_parameters, &to_parameter_indices, &to_param_ref, &from_parameter_indices, &from_param_ref, 1);
    2753
      // Should be exactly the same tensor.
    2754
    2
      if (renamer == 0 && to_param_ref < 01 && from_param_ref < 01)
    2755
    1
        { assert(from_parameter_indices->rnum == to_parameter_indices->rnum); }
    2756
      // To models.
    2757
    2
      ccv_cnnp_compiled_data_t* const to_compiled_data = model->compiled_data;
    2758
    2
      assert(to_compiled_data);
    2759
      // From models.
    2760
    2
      const ccv_cnnp_compiled_data_t* const from_compiled_data = from_model->compiled_data;
    2761
    2
      const int parallel_count = ccv_max(model->parallel_count, 1);
    2762
    2
      assert(parallel_count == ccv_max(from_model->parallel_count, 1)); // Should have the same parallel count can share parameters.
    2763
    2
      const int from_parameter_size = from_compiled_data->parameters->rnum;
    2764
    2
      const int to_parameter_size = to_compiled_data->parameters->rnum;
    2765
    2
      const int rnum = (to_param_ref < 0 && from_param_ref < 0) ? to_parameter_indices->rnum : 10;
    2766
    2
      int i, j;
    2767
    2
      khash_t(ccv_cnnp_parameter_id)* id_map = 0;
    2768
    2
      char* updated_name = 0;
    2769
    2
      const uint32_t* const from_init_v = CCV_NNC_INIT_V(from_compiled_data->tensors_init.v);
    2770
    2
      uint32_t* const to_init_v = CCV_NNC_INIT_V(to_compiled_data->tensors_init.v);
    2771
    8
      for (i = 0; i < rnum; i++6)
    2772
    6
      {
    2773
    6
        int src_d = (from_param_ref >= 0 ? from_param_ref0 : i) < from_parameter_indices->rnum ? *(int*)4ccv_array_get4(from_parameter_indices,from_param_ref >= 0 ? from_param_ref : i) : from_parameter_size2;
    2774
        // Need to figure out how to use the renamer here.
    2775
    6
        const int dest_d = *(int*)ccv_array_get(to_parameter_indices, to_param_ref >= 0 ? to_param_ref : i);
    2776
    6
        assert(dest_d >= 0);
    2777
    6
        assert(dest_d < to_parameter_size);
    2778
    6
        if (renamer)
    2779
    3
        {
    2780
    3
          const char* const src_name = (src_d < from_parameter_size && src_d >= 01) ? *(char**)1ccv_array_get1(from_compiled_data->ids.parameters, src_d) : 02;
    2781
    3
          const char* const dest_name = *(char**)ccv_array_get(to_compiled_data->ids.parameters, dest_d);
    2782
    3
          if (!updated_name)
    2783
    1
            updated_name = (char*)ccmalloc(1024);
    2784
    3
          const size_t src_name_len = src_name == 0 ? 02 : ccv_min1(strnlen(src_name, 1023), 1023);
    2785
    3
          if (src_name_len > 0)
    2786
    1
            memcpy(updated_name, src_name, src_name_len);
    2787
    3
          updated_name[src_name_len] = 0;
    2788
    3
          if (renamer(context, dest_name, updated_name, 1024) != 0)
    2789
    0
            continue; // Skip this.
    2790
    3
          if (src_name != 0 && memcmp(updated_name, src_name, src_name_len) == 01 && strnlen(updated_name, 1023) == src_name_len0)
    2791
    0
          {
    2792
            // Nothing changed.
    2793
    3
          } else {
    2794
    3
            if (!id_map)
    2795
    1
            {
    2796
    1
              id_map = kh_init(ccv_cnnp_parameter_id);
    2797
    2
              for (j = 0; j < from_parameter_size; j++1)
    2798
    1
              {
    2799
    1
                int ret;
    2800
    1
                const khiter_t k = kh_put(ccv_cnnp_parameter_id, id_map, *(char**)ccv_array_get(from_compiled_data->ids.parameters, j), &ret);
    2801
    1
                assert(ret != 0);
    2802
    1
                kh_val(id_map, k) = j;
    2803
    1
              }
    2804
    1
            }
    2805
    3
            const khiter_t k = kh_get(ccv_cnnp_parameter_id, id_map, updated_name);
    2806
    3
            if (k == kh_end(id_map)) // Cannot find the name, skip.
    2807
    2
              continue;
    2808
    1
            src_d = kh_val(id_map, k);
    2809
    1
            assert(src_d >= 0);
    2810
    1
            assert(src_d < from_parameter_size);
    2811
    1
          }
    2812
    3
        }
    2813
    6
        assert(src_d >= 0)4;
    2814
    4
        assert(src_d < from_parameter_size);
    2815
    4
        const int s = ((ccv_nnc_tensor_symbol_t*)ccv_array_get(from_compiled_data->parameters, src_d))->d;
    2816
        // If the original is not init'ed. We cannot share from.
    2817
    4
        if (!(from_init_v[s >> 5] & (1u << (s & 0x1f))))
    2818
    0
          continue;
    2819
    8
        for (j = 0; 4j < parallel_count; j++4)
    2820
    4
        {
    2821
    4
          ccv_nnc_tensor_t* const src = CCV_NNC_TENSOR(from_compiled_data->tensors.parameters[src_d + j * from_parameter_size]);
    2822
    4
          assert(src);
    2823
    4
          ccv_nnc_tensor_t* const dest = to_compiled_data->tensors.parameters[dest_d + j * to_parameter_size];
    2824
    4
          if (dest && !((uintptr_t)dest & (uintptr_t)1)0)
    2825
    0
            ccv_nnc_tensor_free(dest);
    2826
    4
          to_compiled_data->tensors.parameters[dest_d + j * to_parameter_size] = (ccv_nnc_tensor_t*)((uintptr_t)src | (uintptr_t)1);
    2827
    4
        }
    2828
        // Mark this symbol as init'ed.
    2829
    4
        const int d = ((ccv_nnc_tensor_symbol_t*)ccv_array_get(to_compiled_data->parameters, dest_d))->d;
    2830
    4
        to_init_v[d >> 5] |= (1u << (d & 0x1f));
    2831
    4
      }
    2832
    2
      ccv_array_free(to_parameter_indices);
    2833
    2
      ccv_array_free(from_parameter_indices);
    2834
    2
      if (id_map)
    2835
    1
        kh_destroy(ccv_cnnp_parameter_id, id_map);
    2836
    2
      if (updated_name)
    2837
    1
        ccfree(updated_name);
    2838
      // Mark it as incomplete so we will call init_1.
    2839
    2
      if (ccv_cnnp_model_tensors_any_to_alloc(model, to_compiled_data))
    2840
    1
        to_compiled_data->tensors_init.v = (uint32_t*)((uintptr_t)to_compiled_data->tensors_init.v | (uintptr_t)1);
    2841
    1
      else // Remove the flag.
    2842
    1
        to_compiled_data->tensors_init.v = CCV_NNC_INIT_V(to_compiled_data->tensors_init.v);
    2843
    2
    }
    2844
    2845
    ccv_nnc_stream_context_t* ccv_cnnp_compiled_data_get_stream(ccv_cnnp_compiled_data_t* const compiled_data, const int type)
    2846
    24
    {
    2847
    24
      if (!compiled_data->stream_map)
    2848
    4
        compiled_data->stream_map = kh_init(stream_map);
    2849
    24
      int ret = 0;
    2850
    24
      khiter_t k = kh_put(stream_map, compiled_data->stream_map, type, &ret);
    2851
    24
      assert(ret >= 0);
    2852
    24
      ccv_nnc_stream_context_t* stream = kh_val(compiled_data->stream_map, k);
    2853
      // If ret == 0, the key already exist, we can return directly, otherwise, create and return.
    2854
    24
      if (ret != 0)
    2855
    16
      {
    2856
    16
        stream = ccv_nnc_stream_context_new(type);
    2857
    16
        kh_val(compiled_data->stream_map, k) = stream;
    2858
    16
      }
    2859
    24
      return stream;
    2860
    24
    }
    2861
    2862
    void ccv_cnnp_model_parameters_zip_map(ccv_cnnp_model_t* const model, const ccv_cnnp_model_io_t parameters, const ccv_nnc_cmd_t cmd, const ccv_nnc_hint_t hint, const int flags, ccv_nnc_tensor_t* const* const aux_ins, const int aux_in_size, ccv_nnc_tensor_t* const* const aux_outs, const int aux_out_size, ccv_nnc_stream_context_t* const stream_context, const ccv_cnnp_model_t* const from_model, const ccv_cnnp_model_io_t from_parameters)
    2863
    3
    {
    2864
    3
      ccv_array_t* to_parameter_indices;
    2865
    3
      int to_param_ref;
    2866
    3
      ccv_array_t* from_parameter_indices;
    2867
    3
      int from_param_ref;
    2868
    3
      _ccv_cnnp_model_to_parameter_indices_and_from_parameter_indices(model, parameters, from_model, from_parameters, &to_parameter_indices, &to_param_ref, &from_parameter_indices, &from_param_ref, 0);
    2869
      // Should be exactly the same tensor.
    2870
    3
      if (to_param_ref < 0 && from_param_ref < 0)
    2871
    3
        { assert(from_parameter_indices->rnum == to_parameter_indices->rnum); }
    2872
      // To models.
    2873
    3
      ccv_cnnp_compiled_data_t* const to_compiled_data = model->compiled_data;
    2874
    3
      assert(to_compiled_data);
    2875
      // From models.
    2876
    3
      const ccv_cnnp_compiled_data_t* const from_compiled_data = from_model->compiled_data;
    2877
    3
      const int parallel_count = ccv_max(model->parallel_count, 1);
    2878
    3
      const int to_parameter_size = to_compiled_data->parameters->rnum;
    2879
    3
      const int rnum = (to_param_ref < 0 && from_param_ref < 0) ? from_parameter_indices->rnum : 10;
    2880
    3
      assert(aux_in_size >= 0);
    2881
    3
      assert(aux_out_size >= 0);
    2882
    3
      int i, j;
    2883
    3
      ccv_nnc_tensor_t* inputs[aux_in_size + 2];
    2884
    3
      ccv_nnc_tensor_t* outputs[aux_out_size + 1];
    2885
    3
      for (i = 0; i < aux_in_size; i++0)
    2886
    0
        inputs[i + 2] = aux_ins[i];
    2887
    3
      for (i = 0; i < aux_out_size; i++0)
    2888
    0
        outputs[i + 1] = aux_outs[i];
    2889
    3
      const uint32_t* const from_init_v = CCV_NNC_INIT_V(from_compiled_data->tensors_init.v);
    2890
    3
      uint32_t* const to_init_v = CCV_NNC_INIT_V(to_compiled_data->tensors_init.v);
    2891
    6
      for (i = 0; i < rnum; i++3)
    2892
    3
      {
    2893
    3
        const int src_d = *(int*)ccv_array_get(from_parameter_indices,from_param_ref >= 0 ? from_param_ref : i);
    2894
    3
        assert(src_d >= 0);
    2895
    3
        assert(src_d < from_compiled_data->parameters->rnum);
    2896
    3
        const int s = ((ccv_nnc_tensor_symbol_t*)ccv_array_get(from_compiled_data->parameters, src_d))->d;
    2897
        // If the original is not init'ed. We cannot copy from.
    2898
    3
        if (!(from_init_v[s >> 5] & (1u << (s & 0x1f))))
    2899
    0
          continue;
    2900
    3
        const int dest_d = *(int*)ccv_array_get(to_parameter_indices, to_param_ref >= 0 ? to_param_ref : i);
    2901
    3
        assert(dest_d >= 0);
    2902
    3
        assert(dest_d < to_compiled_data->parameters->rnum);
    2903
    3
        if (parallel_count > 1)
    2904
    2
        {
    2905
    2
          ccv_nnc_stream_context_t* streams[parallel_count];
    2906
    2
          ccv_nnc_stream_signal_t* signal;
    2907
    2
          if (stream_context)
    2908
    1
            signal = ccv_nnc_stream_context_emit_signal_new(stream_context);
    2909
    10
          for (j = 0; j < parallel_count; j++8)
    2910
    8
          {
    2911
    8
            ccv_nnc_tensor_t* const src = CCV_NNC_TENSOR(from_compiled_data->tensors.parameters[src_d + j * to_parameter_size]);
    2912
    8
            ccv_nnc_tensor_t* const dest = CCV_NNC_TENSOR(to_compiled_data->tensors.parameters[dest_d + j * to_parameter_size]);
    2913
    8
            if (!dest || !src)
    2914
    0
            {
    2915
    0
              streams[j] = 0;
    2916
    0
              continue;
    2917
    0
            }
    2918
            // At the moment, can only handle them on the same device.
    2919
    8
            assert(CCV_TENSOR_GET_MEMORY(src->info.type) == CCV_TENSOR_GET_MEMORY(dest->info.type));
    2920
    8
            assert(CCV_TENSOR_GET_DEVICE_ID(src->info.type) == CCV_TENSOR_GET_DEVICE_ID(dest->info.type));
    2921
    8
            const int stream_type = CCV_TENSOR_GET_MEMORY(src->info.type) == CCV_TENSOR_GPU_MEMORY ? CCV_STREAM_CONTEXT_GPU : CCV_STREAM_CONTEXT_CPU0;
    2922
    8
            const int device_id = CCV_TENSOR_GET_DEVICE_ID(src->info.type);
    2923
    8
            int type = stream_type;
    2924
    8
            CCV_STREAM_SET_DEVICE_ID(type, device_id);
    2925
    8
            ccv_nnc_stream_context_t* const stream_0 = ccv_cnnp_compiled_data_get_stream(to_compiled_data, type);
    2926
            // Wait signal to finish.
    2927
    8
            if (stream_context)
    2928
    4
              ccv_nnc_stream_context_wait_signal(stream_0, signal);
    2929
    8
            inputs[0] = outputs[0] = dest;
    2930
    8
            inputs[1] = src;
    2931
    8
            ccv_nnc_cmd_exec(cmd, hint, flags, inputs, aux_in_size + 2, outputs, aux_out_size + 1, stream_0);
    2932
    8
            if (stream_context)
    2933
    4
            {
    2934
    4
              ccv_nnc_stream_signal_t* const signal = ccv_nnc_stream_context_emit_signal_new(stream_0);
    2935
    4
              ccv_nnc_stream_context_wait_signal(stream_context, signal);
    2936
    4
            }
    2937
    8
            streams[j] = stream_0;
    2938
    8
          }
    2939
          // If this should be blocking, blocking it.
    2940
    2
          if (!stream_context)
    2941
    5
            for (j = 0; 1j < parallel_count; j++4)
    2942
    4
              if (streams[j])
    2943
    4
                ccv_nnc_stream_context_wait(streams[j]);
    2944
    2
        } else {
    2945
    1
          ccv_nnc_tensor_t* const src = CCV_NNC_TENSOR(from_compiled_data->tensors.parameters[src_d]);
    2946
    1
          assert(src);
    2947
    1
          ccv_nnc_tensor_t* const dest = CCV_NNC_TENSOR(to_compiled_data->tensors.parameters[dest_d]);
    2948
    1
          assert(dest);
    2949
    1
          inputs[0] = outputs[0] = dest;
    2950
    1
          inputs[1] = src;
    2951
    1
          ccv_nnc_cmd_exec(cmd, hint, flags, inputs, aux_in_size + 2, outputs, aux_out_size + 1, stream_context);
    2952
    1
        }
    2953
        // Mark this symbol as init'ed.
    2954
    3
        const int d = ((ccv_nnc_tensor_symbol_t*)ccv_array_get(to_compiled_data->parameters, dest_d))->d;
    2955
    3
        to_init_v[d >> 5] |= (1u << (d & 0x1f));
    2956
    3
      }
    2957
    3
      ccv_array_free(to_parameter_indices);
    2958
    3
      ccv_array_free(from_parameter_indices);
    2959
    3
    }
    2960
    2961
    void ccv_cnnp_model_parameters_map(ccv_cnnp_model_t* const model, const ccv_cnnp_model_io_t parameters, const ccv_nnc_cmd_t cmd, const ccv_nnc_hint_t hint, const int flags, ccv_nnc_tensor_t* const* const aux_ins, const int aux_in_size, ccv_nnc_tensor_t* const* const aux_outs, const int aux_out_size, ccv_nnc_stream_context_t* const stream_context)
    2962
    15
    {
    2963
    15
      int to_param_ref;
    2964
    15
      ccv_array_t* const to_parameter_indices = _ccv_cnnp_model_parameter_indices(model, parameters, &to_param_ref);
    2965
      // To models.
    2966
    15
      ccv_cnnp_compiled_data_t* const to_compiled_data = model->compiled_data;
    2967
    15
      assert(to_compiled_data);
    2968
      // Tensor has to be inited already.
    2969
    15
      assert(!!to_compiled_data->tensors_init.v);
    2970
    15
      assert(to_compiled_data->tensors.parameters);
    2971
      // From models.
    2972
    15
      const int parallel_count = ccv_max(model->parallel_count, 1);
    2973
    15
      const int to_parameter_size = to_compiled_data->parameters->rnum;
    2974
    15
      const int rnum = (to_param_ref < 0) ? to_parameter_indices->rnum : 10;
    2975
    15
      assert(aux_in_size >= 0);
    2976
    15
      assert(aux_out_size >= 0);
    2977
    15
      int i, j;
    2978
    15
      ccv_nnc_tensor_t* inputs[aux_in_size + 1];
    2979
    15
      ccv_nnc_tensor_t* outputs[aux_out_size + 1];
    2980
    15
      for (i = 0; i < aux_in_size; i++0)
    2981
    0
        inputs[i + 1] = aux_ins[i];
    2982
    15
      for (i = 0; i < aux_out_size; i++0)
    2983
    0
        outputs[i + 1] = aux_outs[i];
    2984
    30
      for (i = 0; i < rnum; i++15)
    2985
    15
      {
    2986
    15
        const int dest_d = *(int*)ccv_array_get(to_parameter_indices, to_param_ref >= 0 ? to_param_ref : i);
    2987
    15
        assert(dest_d >= 0);
    2988
    15
        assert(dest_d < to_compiled_data->parameters->rnum);
    2989
    15
        if (parallel_count > 1)
    2990
    4
        {
    2991
    4
          ccv_nnc_stream_context_t* streams[parallel_count];
    2992
    4
          ccv_nnc_stream_signal_t* signal;
    2993
    4
          if (stream_context)
    2994
    1
            signal = ccv_nnc_stream_context_emit_signal_new(stream_context);
    2995
    20
          for (j = 0; j < parallel_count; j++16)
    2996
    16
          {
    2997
    16
            ccv_nnc_tensor_t* const dest = CCV_NNC_TENSOR(to_compiled_data->tensors.parameters[dest_d + j * to_parameter_size]);
    2998
    16
            if (!dest)
    2999
    0
            {
    3000
    0
              streams[j] = 0;
    3001
    0
              continue;
    3002
    0
            }
    3003
    16
            const int stream_type = CCV_TENSOR_GET_MEMORY(dest->info.type) == CCV_TENSOR_GPU_MEMORY ? CCV_STREAM_CONTEXT_GPU : CCV_STREAM_CONTEXT_CPU0;
    3004
    16
            const int device_id = CCV_TENSOR_GET_DEVICE_ID(dest->info.type);
    3005
    16
            int type = stream_type;
    3006
    16
            CCV_STREAM_SET_DEVICE_ID(type, device_id);
    3007
    16
            ccv_nnc_stream_context_t* const stream_0 = ccv_cnnp_compiled_data_get_stream(to_compiled_data, type);
    3008
            // Wait signal to finish.
    3009
    16
            if (stream_context)
    3010
    4
              ccv_nnc_stream_context_wait_signal(stream_0, signal);
    3011
    16
            inputs[0] = outputs[0] = dest;
    3012
    16
            ccv_nnc_cmd_exec(cmd, hint, flags, inputs, aux_in_size + 1, outputs, aux_out_size + 1, stream_0);
    3013
    16
            if (stream_context)
    3014
    4
            {
    3015
    4
              ccv_nnc_stream_signal_t* const signal = ccv_nnc_stream_context_emit_signal_new(stream_0);
    3016
    4
              ccv_nnc_stream_context_wait_signal(stream_context, signal);
    3017
    4
            }
    3018
    16
            streams[j] = stream_0;
    3019
    16
          }
    3020
          // If this should be blocking, blocking it.
    3021
    4
          if (!stream_context)
    3022
    15
            for (j = 0; 3j < parallel_count; j++12)
    3023
    12
              if (streams[j])
    3024
    12
                ccv_nnc_stream_context_wait(streams[j]);
    3025
    11
        } else {
    3026
    11
          ccv_nnc_tensor_t* const dest = CCV_NNC_TENSOR(to_compiled_data->tensors.parameters[dest_d]);
    3027
    11
          assert(dest);
    3028
    11
          inputs[0] = outputs[0] = dest;
    3029
    11
          ccv_nnc_cmd_exec(cmd, hint, flags, inputs, aux_in_size + 1, outputs, aux_out_size + 1, stream_context);
    3030
    11
        }
    3031
        // No need to mark this symbol as init'ed, it is already.
    3032
    15
      }
    3033
    15
      ccv_array_free(to_parameter_indices);
    3034
    15
    }
    3035
    3036
    void ccv_cnnp_model_parameter_gradients_map(ccv_cnnp_model_t* const model, const ccv_cnnp_model_io_t parameters, const ccv_nnc_cmd_t cmd, const ccv_nnc_hint_t hint, const int flags, ccv_nnc_tensor_t* const* const aux_ins, const int aux_in_size, ccv_nnc_tensor_t* const* const aux_outs, const int aux_out_size, ccv_nnc_stream_context_t* const stream_context)
    3037
    6
    {
    3038
    6
      int to_param_ref;
    3039
    6
      ccv_array_t* const to_parameter_indices = _ccv_cnnp_model_parameter_indices(model, parameters, &to_param_ref);
    3040
      // To models.
    3041
    6
      ccv_cnnp_compiled_data_t* const to_compiled_data = model->compiled_data;
    3042
    6
      assert(to_compiled_data);
    3043
      // Tensor has to be inited already.
    3044
    6
      assert(!!to_compiled_data->tensors_init.v);
    3045
    6
      ccv_nnc_tensor_t** tensor_gradients;
    3046
    6
      if (to_compiled_data->backward.count > 1)
    3047
    3
        tensor_gradients = to_compiled_data->tensors.accum_gradients;
    3048
    3
      else
    3049
    3
        tensor_gradients = to_compiled_data->tensors.gradients;
    3050
    6
      assert(tensor_gradients);
    3051
      // From models.
    3052
    6
      const int parallel_count = ccv_max(model->parallel_count, 1);
    3053
    6
      const int to_parameter_size = to_compiled_data->parameters->rnum;
    3054
    6
      const int rnum = (to_param_ref < 0) ? to_parameter_indices->rnum : 10;
    3055
    6
      assert(aux_in_size >= 0);
    3056
    6
      assert(aux_out_size >= 0);
    3057
    6
      int i, j;
    3058
    6
      ccv_nnc_tensor_t* inputs[aux_in_size + 1];
    3059
    6
      ccv_nnc_tensor_t* outputs[aux_out_size + 1];
    3060
    10
      for (i = 0; i < aux_in_size; i++4)
    3061
    4
        inputs[i + 1] = aux_ins[i];
    3062
    14
      for (i = 0; i < aux_out_size; i++8)
    3063
    8
        outputs[i + 1] = aux_outs[i];
    3064
    12
      for (i = 0; i < rnum; i++6)
    3065
    6
      {
    3066
    6
        const int dest_d = *(int*)ccv_array_get(to_parameter_indices, to_param_ref >= 0 ? to_param_ref : i);
    3067
    6
        assert(dest_d >= 0);
    3068
    6
        assert(dest_d < to_compiled_data->parameters->rnum);
    3069
    6
        if (parallel_count > 1)
    3070
    0
        {
    3071
    0
          ccv_nnc_stream_context_t* streams[parallel_count];
    3072
    0
          ccv_nnc_stream_signal_t* signal;
    3073
    0
          if (stream_context)
    3074
    0
            signal = ccv_nnc_stream_context_emit_signal_new(stream_context);
    3075
    0
          for (j = 0; j < parallel_count; j++)
    3076
    0
          {
    3077
    0
            ccv_nnc_tensor_t* const dest = tensor_gradients[dest_d + j * to_parameter_size];
    3078
    0
            if (!dest)
    3079
    0
            {
    3080
    0
              streams[j] = 0;
    3081
    0
              continue;
    3082
    0
            }
    3083
    0
            const int stream_type = CCV_TENSOR_GET_MEMORY(dest->info.type) == CCV_TENSOR_GPU_MEMORY ? CCV_STREAM_CONTEXT_GPU : CCV_STREAM_CONTEXT_CPU;
    3084
    0
            const int device_id = CCV_TENSOR_GET_DEVICE_ID(dest->info.type);
    3085
    0
            int type = stream_type;
    3086
    0
            CCV_STREAM_SET_DEVICE_ID(type, device_id);
    3087
    0
            ccv_nnc_stream_context_t* const stream_0 = ccv_cnnp_compiled_data_get_stream(to_compiled_data, type);
    3088
            // Wait signal to finish.
    3089
    0
            if (stream_context)
    3090
    0
              ccv_nnc_stream_context_wait_signal(stream_0, signal);
    3091
    0
            inputs[0] = outputs[0] = dest;
    3092
    0
            ccv_nnc_cmd_exec(cmd, hint, flags, inputs, aux_in_size + 1, outputs, aux_out_size + 1, stream_0);
    3093
    0
            if (stream_context)
    3094
    0
            {
    3095
    0
              ccv_nnc_stream_signal_t* const signal = ccv_nnc_stream_context_emit_signal_new(stream_0);
    3096
    0
              ccv_nnc_stream_context_wait_signal(stream_context, signal);
    3097
    0
            }
    3098
    0
            streams[j] = stream_0;
    3099
    0
          }
    3100
          // If this should be blocking, blocking it.
    3101
    0
          if (!stream_context)
    3102
    0
            for (j = 0; j < parallel_count; j++)
    3103
    0
              if (streams[j])
    3104
    0
                ccv_nnc_stream_context_wait(streams[j]);
    3105
    6
        } else {
    3106
    6
          ccv_nnc_tensor_t* const dest = tensor_gradients[dest_d];
    3107
    6
          if (!dest)
    3108
    0
            continue;
    3109
    6
          assert(dest);
    3110
    6
          inputs[0] = outputs[0] = dest;
    3111
    6
          ccv_nnc_cmd_exec(cmd, hint, flags, inputs, aux_in_size + 1, outputs, aux_out_size + 1, stream_context);
    3112
    6
        }
    3113
        // No need to mark this symbol as init'ed, it is already.
    3114
    6
      }
    3115
    6
      ccv_array_free(to_parameter_indices);
    3116
    6
    }
    3117
    3118
    void ccv_cnnp_model_parameters_to_unified_memory(ccv_cnnp_model_t* const model, const ccv_cnnp_model_io_t parameters, ccv_nnc_stream_context_t* const stream_context)
    3119
    0
    {
    3120
      // Only CUDA backend has this feature.
    3121
    0
    #ifdef HAVE_CUDA
    3122
    0
      int to_param_ref;
    3123
    0
      ccv_array_t* const to_parameter_indices = _ccv_cnnp_model_parameter_indices(model, parameters, &to_param_ref);
    3124
      // To models.
    3125
    0
      ccv_cnnp_compiled_data_t* const compiled_data = model->compiled_data;
    3126
    0
      assert(compiled_data);
    3127
      // Tensor has to be inited already.
    3128
    0
      assert(!!compiled_data->tensors_init.v);
    3129
    0
      assert(compiled_data->tensors.parameters);
    3130
      // From models.
    3131
    0
      const int parallel_count = ccv_max(model->parallel_count, 1);
    3132
    0
      const int rnum = (to_param_ref < 0) ? to_parameter_indices->rnum : 1;
    3133
    0
      int i;
    3134
    0
      for (i = 0; i < rnum; i++)
    3135
    0
      {
    3136
    0
        const int dest_d = *(int*)ccv_array_get(to_parameter_indices, to_param_ref >= 0 ? to_param_ref : i);
    3137
    0
        assert(dest_d >= 0);
    3138
    0
        assert(dest_d < compiled_data->parameters->rnum);
    3139
    0
        if (parallel_count > 1)
    3140
    0
        {
    3141
    0
          assert(0 && "Cannot support this when data parallel is in effect.");
    3142
    0
        } else {
    3143
    0
          ccv_nnc_tensor_t* const src = CCV_NNC_TENSOR(compiled_data->tensors.parameters[dest_d]);
    3144
    0
          assert(src);
    3145
    0
          ccv_nnc_tensor_param_t params = src->info;
    3146
    0
          if (CCV_TENSOR_GET_MEMORY(params.type) != CCV_TENSOR_GPU_MEMORY)
    3147
    0
            continue;
    3148
    0
          const size_t size = ccv_nnc_tensor_data_size(params);
    3149
    0
          if (size <= 0)
    3150
    0
            continue;
    3151
    0
          const int should_free = !((uintptr_t)compiled_data->tensors.parameters[dest_d] & (uintptr_t)1);
    3152
    0
          const int tfb = (CCV_TENSOR_GET_MEMORY(params.type) == CCV_TENSOR_CPU_MEMORY && params.format == CCV_TENSOR_FORMAT_NHWC && params.dim[2] > 0 && params.dim[2] <= CCV_MAX_CHANNEL && params.dim[0] > 0 && params.dim[1] > 0 && params.dim[3] == 0);
    3153
    0
          ccv_nnc_tensor_t* const tensor = (ccv_nnc_tensor_t*)ccmalloc(sizeof(ccv_nnc_tensor_t));
    3154
    0
          tensor->dataof = 0;
    3155
    0
          tensor->alias_ref = 0;
    3156
    0
          tensor->sig = 0;
    3157
    0
          tensor->refcount = 1;
    3158
    0
          tensor->info = params;
    3159
    0
          if (tfb)
    3160
    0
          {
    3161
    0
            tensor->type = CCV_NO_DATA_ALLOC | CCV_MATRIX_DENSE | CCV_GET_DATA_TYPE(params.datatype) | params.dim[2];
    3162
            // This corresponding to mat->step
    3163
    0
            tensor->info.dim[4] = CCV_GET_STEP(params.dim[1], (CCV_GET_DATA_TYPE(params.datatype) | params.dim[2]));
    3164
    0
          } else // This won't be recognized by ccv_dense_matrix_t
    3165
    0
            tensor->type = CCV_NO_DATA_ALLOC | CCV_MATRIX_DENSE | CCV_GET_DATA_TYPE(params.datatype);
    3166
          // Remove this flag so it can be deallocated as usual.
    3167
    0
          tensor->type &= ~CCV_NO_DATA_ALLOC;
    3168
    0
          assert(CCV_TENSOR_GET_DEVICE(params.type) != CCV_COMPUTE_DEVICE_ANY);
    3169
    0
          void* ptr = cumallocmanaged(CCV_TENSOR_GET_DEVICE_ID(params.type), size);
    3170
    0
          if (ptr) // If allocated successfully. Otherwise we go through the fallback path.
    3171
    0
          {
    3172
    0
            tensor->data.u8 = (uint8_t*)ptr;
    3173
    0
            tensor->type |= CCV_MAPPED_MEM; // This denotes the tensor is mapped to CPU, and would prefer a explicit prefetch call.
    3174
    0
          } else {
    3175
            // Allocation failed.
    3176
    0
            ccfree(tensor);
    3177
    0
            continue;
    3178
    0
          }
    3179
          // TODO: Cannot run this on the stream context yet, due to allocation and deallocations.
    3180
    0
          ccv_nnc_cmd_exec(CMD_DATA_TRANSFER_FORWARD(), ccv_nnc_no_hint, 0, &src, 1, &tensor, 1, 0);
    3181
    0
          cumemadvisereadmostly(CCV_TENSOR_GET_DEVICE_ID(params.type), tensor->data.u8, size);
    3182
    0
          compiled_data->tensors.parameters[dest_d] = tensor;
    3183
          // Can free out the old one.
    3184
    0
          if (should_free)
    3185
    0
            ccv_nnc_tensor_free(src);
    3186
    0
        }
    3187
        // No need to mark this symbol as init'ed, it is already.
    3188
    0
      }
    3189
    0
      ccv_array_free(to_parameter_indices);
    3190
    0
    #endif
    3191
    0
    }
    3192
    3193
    ccv_nnc_cmd_t ccv_cnnp_model_minimizer(ccv_cnnp_model_t* const model)
    3194
    2.20k
    {
    3195
    2.20k
      ccv_cnnp_compiled_data_t* const compiled_data = model->compiled_data;
    3196
    2.20k
      assert(compiled_data);
    3197
    2.20k
      return compiled_data->minimize.minimizer;
    3198
    2.20k
    }
    3199
    3200
    void ccv_cnnp_model_set_minimizer(ccv_cnnp_model_t* const model, const ccv_nnc_cmd_t minimizer, const int reset, const ccv_cnnp_model_io_t* const set_parameters, const int set_parameter_size)
    3201
    4.36k
    {
    3202
    4.36k
      ccv_cnnp_compiled_data_t* const compiled_data = model->compiled_data;
    3203
    4.36k
      assert(compiled_data);
    3204
    4.36k
      const int parameter_size = compiled_data->parameters->rnum;
    3205
    4.36k
      if (parameter_size == 0)
    3206
    6
        return;
    3207
    4.35k
      if (reset)
    3208
    2.49k
        { assert(set_parameters == 0 && set_parameter_size == 0); }
    3209
    4.35k
      const int old_max_saved_aux_size = compiled_data->minimize.max_saved_aux_size;
    3210
    4.35k
      const int saved_aux_size = ccv_nnc_minimizer_saved_aux_size(minimizer);
    3211
    4.35k
      if (saved_aux_size > compiled_data->minimize.max_saved_aux_size)
    3212
    7
        compiled_data->minimize.max_saved_aux_size = saved_aux_size;
    3213
    4.35k
      const int max_saved_aux_size = compiled_data->minimize.max_saved_aux_size;
    3214
      // We update all parameters, at this point, we have one minimizer.
    3215
    4.35k
      if (set_parameters == 0 || set_parameter_size == 0301)
    3216
    4.05k
        compiled_data->minimize.minimizer = minimizer;
    3217
    4.35k
      int i;
    3218
    4.35k
      if (set_parameters && set_parameter_size301)
    3219
    301
      {
    3220
        // I need to save what's the minimizer along with this.
    3221
    301
        if (!compiled_data->minimize.parameters)
    3222
    5
          compiled_data->minimize.parameters = ccv_array_new(sizeof(ccv_cnnp_set_minimizer_for_parameter_t*), 1, 0);
    3223
    301
        ccv_cnnp_set_minimizer_for_parameter_t* const set_minimizer_for_parameter = ccmalloc(sizeof(ccv_cnnp_set_minimizer_for_parameter_t) + (set_parameter_size - 1) * sizeof(ccv_cnnp_model_io_t));
    3224
    301
        set_minimizer_for_parameter->minimizer = minimizer;
    3225
    301
        set_minimizer_for_parameter->parameter_size = set_parameter_size;
    3226
    301
        memcpy(set_minimizer_for_parameter->parameters, set_parameters, sizeof(ccv_cnnp_model_io_t) * set_parameter_size);
    3227
    301
        ccv_array_push(compiled_data->minimize.parameters, &set_minimizer_for_parameter);
    3228
    301
      }
    3229
      // If reset is true, clear the parameters array.
    3230
    4.35k
      if (reset && compiled_data->minimize.parameters2.49k)
    3231
    291
      {
    3232
    582
        for (i = 0; i < compiled_data->minimize.parameters->rnum; i++291)
    3233
    291
          ccfree(*(ccv_cnnp_set_minimizer_for_parameter_t**)ccv_array_get(compiled_data->minimize.parameters, i));
    3234
    291
        ccv_array_clear(compiled_data->minimize.parameters);
    3235
    291
      }
    3236
    4.35k
      if (!compiled_data->update_nodes)
    3237
    9
        return;
    3238
    4.34k
      ccv_nnc_symbolic_graph_t* const symbolic_graph = model->graph;
    3239
    4.34k
      assert(symbolic_graph);
    3240
    4.34k
      if (saved_aux_size > old_max_saved_aux_size)
    3241
    7
      {
    3242
    7
        assert(compiled_data->updated_parameters);
    3243
        // Reallocate first, move them around later.
    3244
    7
        compiled_data->updated_parameters = (ccv_nnc_tensor_symbol_t*)ccrealloc(compiled_data->updated_parameters, sizeof(ccv_nnc_tensor_symbol_t) * parameter_size + sizeof(ccv_nnc_graph_exec_symbol_t) * parameter_size + sizeof(ccv_nnc_tensor_symbol_map_t) * saved_aux_size * parameter_size);
    3245
    7
        compiled_data->update_nodes = (ccv_nnc_graph_exec_symbol_t*)(compiled_data->updated_parameters + parameter_size);
    3246
    7
        compiled_data->saved_aux = (ccv_nnc_tensor_symbol_map_t*)(compiled_data->update_nodes + parameter_size);
    3247
        // We need to do this from back to front because saved_aux_size > old_saved_aux_size, it could overlap.
    3248
    7
        _ccv_cnnp_scatter_saved_aux(compiled_data->saved_aux, parameter_size, old_max_saved_aux_size, saved_aux_size);
    3249
    7
      }
    3250
    4.34k
      int flag = 0;
    3251
    4.34k
      const int parallel_count = ccv_max(model->parallel_count, 1);
    3252
    4.34k
      if (set_parameters && set_parameter_size296)
    3253
    296
      {
    3254
    296
        ccv_array_t* const parameter_indices = ccv_array_new(sizeof(int), 0, 0);
    3255
    592
        for (i = 0; i < set_parameter_size; i++296)
    3256
    296
        {
    3257
    296
          const int param_sel = set_parameters[i]->param_sel > 0 ? set_parameters[i]->param_sel - 1291 : set_parameters[i]->param_sel5;
    3258
    296
          assert(set_parameters[i]->param_sel != 0);
    3259
    296
          const int old_rnum = parameter_indices->rnum;
    3260
    296
          ccv_cnnp_model_add_to_parameter_indices(set_parameters[i]->model, param_sel, parameter_indices);
    3261
    296
          const int param_ref = set_parameters[i]->param_ref > 0 ? set_parameters[i]->param_ref - 10 : set_parameters[i]->param_ref;
    3262
    296
          assert(set_parameters[i]->param_ref != 0);
    3263
    296
          if (param_ref >= 0)
    3264
    0
          {
    3265
    0
            assert(param_ref + old_rnum < parameter_indices->rnum);
    3266
    0
            *(int*)ccv_array_get(parameter_indices, old_rnum) = *(int*)ccv_array_get(parameter_indices, param_ref + old_rnum);
    3267
    0
            parameter_indices->rnum = old_rnum + 1;
    3268
    0
          }
    3269
    296
        }
    3270
        // We may have duplicated indices, but that is OK, we will set it twice.
    3271
    5.24k
        for (i = 0; 296i < parameter_indices->rnum; i++4.95k)
    3272
    4.95k
        {
    3273
    4.95k
          const int d = *(int*)ccv_array_get(parameter_indices, i);
    3274
    4.95k
          if (_ccv_cnnp_set_minimizer_for_parameter(symbolic_graph, compiled_data, compiled_data->update_nodes, compiled_data->updated_parameters, compiled_data->saved_aux, parallel_count, minimizer, saved_aux_size, max_saved_aux_size, d))
    3275
    0
            flag = 1;
    3276
    4.95k
        }
    3277
    296
        ccv_array_free(parameter_indices);
    3278
    4.05k
      } else {
    3279
    19.1k
        for (i = 0; i < parameter_size; i++15.0k)
    3280
    15.0k
          if (_ccv_cnnp_set_minimizer_for_parameter(symbolic_graph, compiled_data, compiled_data->update_nodes, compiled_data->updated_parameters, compiled_data->saved_aux, parallel_count, minimizer, saved_aux_size, max_saved_aux_size, i))
    3281
    65
            flag = 1;
    3282
    4.05k
        if (compiled_data->minimize.parameters)
    3283
    291
          if (_ccv_cnnp_apply_parameters_with_minimizer(model))
    3284
    0
            flag = 1;
    3285
    4.05k
      }
    3286
    4.34k
      if (flag)
    3287
    7
      {
    3288
        // If saved_aux_size doesn't match, we need to remove / add new saved_aux to the graph. But first, free up apply gradients graph.
    3289
    7
        if (compiled_data->graph_mode == CCV_CNNP_MODEL_GRAPH_FIT_MODE)
    3290
    0
          _ccv_cnnp_compiled_data_graph_free(compiled_data);
    3291
    7
        _ccv_cnnp_compiled_data_apply_gradients_free(compiled_data);
    3292
    7
      }
    3293
    4.34k
    }
    3294
    3295
    void ccv_cnnp_model_set_compile_params(ccv_cnnp_model_t* const model, const ccv_nnc_symbolic_graph_compile_param_t compile_params)
    3296
    0
    {
    3297
    0
      ccv_cnnp_compiled_data_t* const compiled_data = model->compiled_data;
    3298
    0
      assert(compiled_data);
    3299
    0
      compiled_data->compile_params = compile_params;
    3300
    0
    }
    3301
    3302
    void ccv_cnnp_model_dot(const ccv_cnnp_model_t* const model, const int flags, FILE** const outs, const int out_size)
    3303
    48
    {
    3304
    48
      if (model->graph && out_size > 047)
    3305
    47
        ccv_nnc_symbolic_graph_dot(model->graph, flags, outs[0]);
    3306
    48
      if (model->compiled_data && model->compiled_data->graph47 && out_size > 116)
    3307
    0
        ccv_nnc_graph_dot(model->compiled_data->graph, flags, outs[1]);
    3308
    48
      if (model->compiled_data && model->compiled_data->backward.accum47 && out_size > 20)
    3309
    0
        ccv_nnc_graph_dot(model->compiled_data->backward.accum, flags, outs[2]);
    3310
    48
      if (model->compiled_data && model->compiled_data->apply_gradients.graph47 && out_size > 33)
    3311
    0
        ccv_nnc_graph_dot(model->compiled_data->apply_gradients.graph, flags, outs[3]);
    3312
    48
    }
    3313
    3314
    void ccv_cnnp_model_format(const ccv_cnnp_model_t* const model, const ccv_nnc_symbolic_graph_format_f format_fn, void* const context)
    3315
    0
    {
    3316
    0
      if (model->graph)
    3317
    0
        ccv_nnc_symbolic_graph_format(model->graph, 0, 0, 0, 0, format_fn, context);
    3318
    0
    }
    3319
    3320
    static void _ccv_cnnp_compiled_data_free(const ccv_cnnp_model_t* const model, ccv_cnnp_compiled_data_t* const compiled_data)
    3321
    2.31k
    {
    3322
    2.31k
      int i;
    3323
    2.31k
      const int parameter_size = compiled_data->parameters->rnum;
    3324
    2.31k
      ccv_array_free(compiled_data->parameters);
    3325
    2.31k
      if (compiled_data->parameter_flags)
    3326
    10
        ccfree(compiled_data->parameter_flags);
    3327
    2.31k
      const int internal_size = compiled_data->internals->rnum;
    3328
    2.31k
      ccv_array_free(compiled_data->internals);
    3329
    2.31k
      assert(compiled_data->ids.parameters->rnum == parameter_size);
    3330
    2.31k
      assert(compiled_data->ids.internals->rnum == internal_size);
    3331
    5.27k
      for (i = 0; 2.31ki < parameter_size; i++2.96k)
    3332
    2.96k
        ccfree(*(char**)ccv_array_get(compiled_data->ids.parameters, i));
    3333
    2.31k
      ccv_array_free(compiled_data->ids.parameters);
    3334
    2.48k
      for (i = 0; i < internal_size; i++165)
    3335
    165
        ccfree(*(char**)ccv_array_get(compiled_data->ids.internals, i));
    3336
    2.31k
      ccv_array_free(compiled_data->ids.internals);
    3337
    2.31k
      const int parallel_count = compiled_data->parallel_count > 0 ? compiled_data->parallel_count106 : _ccv_cnnp_model_root_parallel_count(model)2.20k;
    3338
    2.31k
      if (compiled_data->tensors.parameters)
    3339
    106
      {
    3340
    826
        for (i = 0; i < parameter_size * parallel_count; i++720)
    3341
          // If it is not marked as not belonging, we can free it.
    3342
    720
          if (!((uintptr_t)compiled_data->tensors.parameters[i] & (uintptr_t)1))
    3343
    716
            if (compiled_data->tensors.parameters[i])
    3344
    714
              ccv_nnc_tensor_free(compiled_data->tensors.parameters[i]);
    3345
    265
        for (i = 0; i < internal_size * parallel_count; i++159)
    3346
    159
          if (compiled_data->tensors.internals[i])
    3347
    158
            ccv_nnc_tensor_free(compiled_data->tensors.internals[i]);
    3348
    106
        ccfree(compiled_data->tensors.parameters);
    3349
    106
      }
    3350
    2.31k
      if (compiled_data->tensors.gradients)
    3351
    36
      {
    3352
    366
        for (i = 0; i < parameter_size * parallel_count; i++330)
    3353
    330
        {
    3354
    330
          if (compiled_data->tensors.gradients[i])
    3355
    328
            ccv_nnc_tensor_free(compiled_data->tensors.gradients[i]);
    3356
    330
          if (compiled_data->tensors.accum_gradients[i])
    3357
    15
            ccv_nnc_tensor_free(compiled_data->tensors.accum_gradients[i]);
    3358
    330
        }
    3359
    36
        ccfree(compiled_data->tensors.gradients);
    3360
    36
      }
    3361
    2.31k
      if (compiled_data->minimize.parameters)
    3362
    5
      {
    3363
    15
        for (i = 0; i < compiled_data->minimize.parameters->rnum; i++10)
    3364
    10
          ccfree(*(ccv_cnnp_set_minimizer_for_parameter_t**)ccv_array_get(compiled_data->minimize.parameters, i));
    3365
    5
        ccv_array_free(compiled_data->minimize.parameters);
    3366
    5
      }
    3367
    2.31k
      if (compiled_data->rewindables)
    3368
    49
        ccv_array_free(compiled_data->rewindables);
    3369
    2.31k
      if (compiled_data->tensors_init.v)
    3370
    106
        ccfree(CCV_NNC_INIT_V(compiled_data->tensors_init.v));
    3371
    2.31k
      if (compiled_data->evaluate.tos)
    3372
    2.31k
        ccfree(compiled_data->evaluate.tos);
    3373
    2.31k
      compiled_data->evaluate.tos = 0;
    3374
    2.31k
      if (compiled_data->stream_map)
    3375
    4
      {
    3376
    4
        khiter_t k;
    3377
    36
        for (k = kh_begin4(compiled_data->stream_map); k != kh_end(compiled_data->stream_map); ++k32)
    3378
    32
        {
    3379
    32
          if (!kh_exist(compiled_data->stream_map, k))
    3380
    16
            continue;
    3381
    16
          ccv_nnc_stream_context_t* const stream = kh_val(compiled_data->stream_map, k);
    3382
    16
          ccv_nnc_stream_context_free(stream);
    3383
    16
        }
    3384
    4
        kh_destroy(stream_map, compiled_data->stream_map);
    3385
    4
      }
    3386
    2.31k
      _ccv_cnnp_compiled_data_graph_free(compiled_data);
    3387
    2.31k
      _ccv_cnnp_compiled_data_gradient_free(compiled_data);
    3388
    2.31k
      _ccv_cnnp_compiled_data_backward_free(compiled_data);
    3389
    2.31k
      _ccv_cnnp_compiled_data_apply_gradients_free(compiled_data);
    3390
    2.31k
      if (compiled_data->gradient_checkpoints)
    3391
    2
      {
    3392
    4
        for (i = 0; i < compiled_data->gradient_checkpoints->rnum; i++2)
    3393
    2
        {
    3394
    2
          ccv_cnnp_model_gradient_checkpoint_t* const checkpoint = (ccv_cnnp_model_gradient_checkpoint_t*)ccv_array_get(compiled_data->gradient_checkpoints, i);
    3395
    2
          assert(checkpoint->inputs);
    3396
    2
          ccfree(checkpoint->inputs);
    3397
    2
          ccv_array_free(checkpoint->tensor_symbols);
    3398
    2
        }
    3399
    2
        ccv_array_free(compiled_data->gradient_checkpoints);
    3400
    2
      }
    3401
    2.31k
      ccv_nnc_xpu_alloc_destroy(&compiled_data->xpu_alloc);
    3402
    2.31k
      ccfree(compiled_data);
    3403
    2.31k
    }
    3404
    3405
    void ccv_cnnp_model_free(ccv_cnnp_model_t* const model)
    3406
    5.50k
    {
    3407
    5.50k
      ccv_cnnp_model_deinit(model);
    3408
    5.50k
      if (model->isa->dealloc)
    3409
    1.23k
        model->isa->dealloc(model);
    3410
    5.50k
      if (model->io)
    3411
    846
      {
    3412
    846
        int i;
    3413
    2.05k
        for (i = 0; i < model->io->rnum; i++1.21k)
    3414
    1.21k
        {
    3415
    1.21k
          ccv_cnnp_model_io_t model_io = *(ccv_cnnp_model_io_t*)ccv_array_get(model->io, i);
    3416
    1.21k
          if (model_io->outgoings)
    3417
    693
            ccv_array_free(model_io->outgoings);
    3418
    1.21k
          if (model_io->incomings)
    3419
    634
            ccv_array_free(model_io->incomings);
    3420
    1.21k
          if (model_io->dependencies)
    3421
    2
            ccv_array_free(model_io->dependencies);
    3422
    1.21k
          ccfree(model_io);
    3423
    1.21k
        }
    3424
    846
        ccv_array_free(model->io);
    3425
    846
      }
    3426
    5.50k
      if (model->parameter_indices)
    3427
    2.53k
        ccv_array_free(model->parameter_indices);
    3428
    5.50k
      if (model->inputs)
    3429
    2.31k
        ccfree(model->inputs);
    3430
    5.50k
      if (model->graph)
    3431
    2.31k
        ccv_nnc_symbolic_graph_free(model->graph);
    3432
    5.50k
      if (model->compiled_data)
    3433
    2.31k
        _ccv_cnnp_compiled_data_free(model, model->compiled_data);
    3434
    5.50k
      if (model->name)
    3435
    241
        ccfree(model->name);
    3436
    5.50k
      ccfree(model);
    3437
    5.50k
    }
    3438
    3439
    void ccv_cnnp_model_cancel(ccv_cnnp_model_t* const model)
    3440
    0
    {
    3441
    0
      ccv_cnnp_compiled_data_t* const compiled_data = model->compiled_data;
    3442
    0
      if (!compiled_data)
    3443
    0
        return;
    3444
    0
      if (compiled_data->graph)
    3445
    0
        ccv_nnc_graph_cancel(compiled_data->graph);
    3446
    0
      if (compiled_data->apply_gradients.graph)
    3447
    0
        ccv_nnc_graph_cancel(compiled_data->apply_gradients.graph);
    3448
    0
    }
    3449
    3450
    void ccv_cnnp_model_set_flags(ccv_cnnp_model_t* const model, const int flags)
    3451
    0
    {
    3452
    0
      model->exec_flags = flags;
    3453
    0
    }
    3454
    3455
    int ccv_cnnp_model_flags(ccv_cnnp_model_t* const model)
    3456
    0
    {
    3457
    0
      return model->exec_flags;
    3458
    0
    }