## Coverage Report

#### Created: 2026-05-04 15:30

next uncovered line (L), next uncovered region (R), next uncovered branch (B)

    /home/liu/actions-runner/_work/ccv/ccv/test/int/nnc/nccl.tests.c

    Line
    Count
    Source
    1
    #include "case.h"
    2
    #include "ccv_case.h"
    3
    #include "ccv_nnc_case.h"
    4
    #include <ccv.h>
    5
    #include <nnc/ccv_nnc.h>
    6
    #include <nnc/ccv_nnc_easy.h>
    7
    #include <3rdparty/dsfmt/dSFMT.h>
    8
    9
    TEST_SETUP()
    10
    {
    11
      ccv_nnc_init();
    12
    }
    13
    14
    TEST_CASE("nccl with allreduce in blocking mode")
    15
    1
    {
    16
    1
      const int device_count = ccv_nnc_device_count(CCV_STREAM_CONTEXT_GPU);
    17
    1
      GUARD_ELSE_RETURN(ccv_nnc_cmd_ok(CCV_NNC_COMM_ALLREDUCE_FORWARD, CCV_NNC_BACKEND_GPU_NCCL) && device_count > 1);
    18
    1
      ccv_nnc_tensor_t* tensors[device_count];
    19
    1
      int i;
    20
    5
      for (i = 0; i < device_count; i++4)
    21
    4
      {
    22
    4
        ccv_nnc_tensor_param_t info = GPU_TENSOR_NHWC(000, 32F, 100);
    23
    4
        CCV_TENSOR_SET_DEVICE_ID(info.type, i);
    24
    4
        tensors[i] = ccv_nnc_tensor_new(0, info, 0);
    25
    4
        ccv_nnc_cmd_exec(CMD_SET_FORWARD(i), ccv_nnc_no_hint, 0, 0, 0, &tensors[i], 1, 0);
    26
    4
      }
    27
    1
      ccv_nnc_cmd_exec(CMD_COMM_ALLREDUCE_FORWARD(), ccv_nnc_no_hint, 0, tensors, device_count, tensors, device_count, 0);
    28
    1
      ccv_nnc_tensor_t* cpu_tensors[device_count];
    29
    5
      for (i = 0; i < device_count; i++4)
    30
    4
      {
    31
    4
        cpu_tensors[i] = ccv_nnc_tensor_new(0, CPU_TENSOR_NHWC(32F, 100), 0);
    32
    4
        ccv_nnc_cmd_exec(CMD_DATA_TRANSFER_FORWARD(), ccv_nnc_no_hint, 0, &tensors[i], 1, &cpu_tensors[i], 1, 0);
    33
    4
      }
    34
    1
      ccv_nnc_tensor_t* demo_tensor = ccv_nnc_tensor_new(0, CPU_TENSOR_NHWC(32F, 100), 0);
    35
    1
      ccv_nnc_cmd_exec(CMD_SET_FORWARD((device_count - 1) * device_count / 2), ccv_nnc_no_hint, 0, 0, 0, &demo_tensor, 1, 0);
    36
    5
      for (i = 0; i < device_count; i++4)
    37
    4
        REQUIRE_TENSOR_EQ(demo_tensor, cpu_tensors[i], "all values should be summed");
    38
    1
      ccv_nnc_tensor_free(demo_tensor);
    39
    5
      for (i = 0; i < device_count; i++4)
    40
    4
      {
    41
    4
        ccv_nnc_tensor_free(tensors[i]);
    42
    4
        ccv_nnc_tensor_free(cpu_tensors[i]);
    43
    4
      }
    44
    1
    }
    45
    46
    TEST_CASE("nccl with broadcast in blocking mode")
    47
    1
    {
    48
    1
      const int device_count = ccv_nnc_device_count(CCV_STREAM_CONTEXT_GPU);
    49
    1
      GUARD_ELSE_RETURN(ccv_nnc_cmd_ok(CCV_NNC_COMM_BROADCAST_FORWARD, CCV_NNC_BACKEND_GPU_NCCL) && device_count > 1);
    50
    1
      ccv_nnc_tensor_t* tensors[device_count];
    51
    1
      int i;
    52
    5
      for (i = 0; i < device_count; i++4)
    53
    4
      {
    54
    4
        ccv_nnc_tensor_param_t info = GPU_TENSOR_NHWC(000, 32F, 100);
    55
    4
        CCV_TENSOR_SET_DEVICE_ID(info.type, i);
    56
    4
        tensors[i] = ccv_nnc_tensor_new(0, info, 0);
    57
    4
        ccv_nnc_cmd_exec(CMD_SET_FORWARD(i + 1), ccv_nnc_no_hint, 0, 0, 0, &tensors[i], 1, 0);
    58
    4
      }
    59
    1
      ccv_nnc_cmd_exec(CMD_COMM_BROADCAST_FORWARD(), ccv_nnc_no_hint, 0, tensors, 1, tensors, device_count, 0);
    60
    1
      ccv_nnc_tensor_t* cpu_tensors[device_count];
    61
    5
      for (i = 0; i < device_count; i++4)
    62
    4
      {
    63
    4
        cpu_tensors[i] = ccv_nnc_tensor_new(0, CPU_TENSOR_NHWC(32F, 100), 0);
    64
    4
        ccv_nnc_cmd_exec(CMD_DATA_TRANSFER_FORWARD(), ccv_nnc_no_hint, 0, &tensors[i], 1, &cpu_tensors[i], 1, 0);
    65
    4
      }
    66
    1
      ccv_nnc_tensor_t* demo_tensor = ccv_nnc_tensor_new(0, CPU_TENSOR_NHWC(32F, 100), 0);
    67
    1
      ccv_nnc_cmd_exec(CMD_SET_FORWARD(1), ccv_nnc_no_hint, 0, 0, 0, &demo_tensor, 1, 0);
    68
    5
      for (i = 0; i < device_count; i++4)
    69
    4
        REQUIRE_TENSOR_EQ(demo_tensor, cpu_tensors[i], "all values should be summed");
    70
    1
      ccv_nnc_tensor_free(demo_tensor);
    71
    5
      for (i = 0; i < device_count; i++4)
    72
    4
      {
    73
    4
        ccv_nnc_tensor_free(tensors[i]);
    74
    4
        ccv_nnc_tensor_free(cpu_tensors[i]);
    75
    4
      }
    76
    1
    }
    77
    78
    TEST_CASE("nccl with reduce in blocking mode")
    79
    1
    {
    80
    1
      const int device_count = ccv_nnc_device_count(CCV_STREAM_CONTEXT_GPU);
    81
    1
      GUARD_ELSE_RETURN(ccv_nnc_cmd_ok(CCV_NNC_COMM_REDUCE_FORWARD, CCV_NNC_BACKEND_GPU_NCCL) && device_count > 1);
    82
    1
      ccv_nnc_tensor_t* tensors[device_count];
    83
    1
      int i;
    84
    5
      for (i = 0; i < device_count; i++4)
    85
    4
      {
    86
    4
        ccv_nnc_tensor_param_t info = GPU_TENSOR_NHWC(000, 32F, 100);
    87
    4
        CCV_TENSOR_SET_DEVICE_ID(info.type, i);
    88
    4
        tensors[i] = ccv_nnc_tensor_new(0, info, 0);
    89
    4
        ccv_nnc_cmd_exec(CMD_SET_FORWARD(i + 1), ccv_nnc_no_hint, 0, 0, 0, &tensors[i], 1, 0);
    90
    4
      }
    91
    1
      ccv_nnc_cmd_exec(CMD_COMM_REDUCE_FORWARD(), ccv_nnc_no_hint, 0, tensors, device_count, tensors, 1, 0);
    92
    1
      ccv_nnc_tensor_t* cpu_tensor;
    93
    1
      cpu_tensor = ccv_nnc_tensor_new(0, CPU_TENSOR_NHWC(32F, 100), 0);
    94
    1
      ccv_nnc_cmd_exec(CMD_DATA_TRANSFER_FORWARD(), ccv_nnc_no_hint, 0, &tensors[0], 1, &cpu_tensor, 1, 0);
    95
    1
      ccv_nnc_tensor_t* demo_tensor = ccv_nnc_tensor_new(0, CPU_TENSOR_NHWC(32F, 100), 0);
    96
    1
      ccv_nnc_cmd_exec(CMD_SET_FORWARD((device_count + 1) * device_count / 2), ccv_nnc_no_hint, 0, 0, 0, &demo_tensor, 1, 0);
    97
    1
      REQUIRE_TENSOR_EQ(demo_tensor, cpu_tensor, "all values should be summed");
    98
    1
      ccv_nnc_tensor_free(demo_tensor);
    99
    1
      ccv_nnc_tensor_free(cpu_tensor);
    100
    5
      for (i = 0; i < device_count; i++4)
    101
    4
        ccv_nnc_tensor_free(tensors[i]);
    102
    1
    }
    103
    104
    TEST_CASE("nccl with all-to-all in blocking mode against cpu reference")
    105
    1
    {
    106
    1
      const int device_count = ccv_nnc_device_count(CCV_STREAM_CONTEXT_GPU);
    107
    1
      GUARD_ELSE_RETURN(ccv_nnc_cmd_ok(CCV_NNC_COMM_ALL_TO_ALL_FORWARD, CCV_NNC_BACKEND_GPU_NCCL) && ccv_nnc_cmd_ok(CCV_NNC_COMM_ALL_TO_ALL_FORWARD, CCV_NNC_BACKEND_CPU_REF) && device_count > 1);
    108
    1
      const int chunk_count = 17;
    109
    1
      const int tensor_count = device_count * chunk_count;
    110
    1
      ccv_nnc_tensor_t* cpu_inputs[device_count];
    111
    1
      ccv_nnc_tensor_t* cpu_expected[device_count];
    112
    1
      ccv_nnc_tensor_t* cpu_outputs[device_count];
    113
    1
      ccv_nnc_tensor_t* gpu_inputs[device_count];
    114
    1
      ccv_nnc_tensor_t* gpu_outputs[device_count];
    115
    1
      int i, j;
    116
    5
      for (i = 0; i < device_count; i++4)
    117
    4
      {
    118
    4
        cpu_inputs[i] = ccv_nnc_tensor_new(0, CPU_TENSOR_NHWC(32F, tensor_count), 0);
    119
    4
        cpu_expected[i] = ccv_nnc_tensor_new(0, CPU_TENSOR_NHWC(32F, tensor_count), 0);
    120
    4
        cpu_outputs[i] = ccv_nnc_tensor_new(0, CPU_TENSOR_NHWC(32F, tensor_count), 0);
    121
    276
        for (j = 0; j < tensor_count; j++272)
    122
    272
          cpu_inputs[i]->data.f32[j] = (float)(i * tensor_count + j);
    123
    4
        ccv_nnc_tensor_param_t info = GPU_TENSOR_NHWC(000, 32F, tensor_count);
    124
    4
        CCV_TENSOR_SET_DEVICE_ID(info.type, i);
    125
    4
        gpu_inputs[i] = ccv_nnc_tensor_new(0, info, 0);
    126
    4
        gpu_outputs[i] = ccv_nnc_tensor_new(0, info, 0);
    127
    4
        ccv_nnc_cmd_exec(CMD_DATA_TRANSFER_FORWARD(), ccv_nnc_no_hint, 0, &cpu_inputs[i], 1, &gpu_inputs[i], 1, 0);
    128
    4
      }
    129
    1
      ccv_nnc_cmd_exec(CMD_COMM_ALL_TO_ALL_FORWARD(0), ccv_nnc_no_hint, 0, cpu_inputs, device_count, cpu_expected, device_count, 0);
    130
    1
      ccv_nnc_cmd_exec(CMD_COMM_ALL_TO_ALL_FORWARD(0), ccv_nnc_no_hint, 0, gpu_inputs, device_count, gpu_outputs, device_count, 0);
    131
    5
      for (i = 0; i < device_count; i++4)
    132
    4
        ccv_nnc_cmd_exec(CMD_DATA_TRANSFER_FORWARD(), ccv_nnc_no_hint, 0, &gpu_outputs[i], 1, &cpu_outputs[i], 1, 0);
    133
    5
      for (i = 0; i < device_count; i++4)
    134
    4
        REQUIRE_TENSOR_EQ(cpu_expected[i], cpu_outputs[i], "all-to-all should match cpu reference");
    135
    5
      for (i = 0; i < device_count; i++4)
    136
    4
      {
    137
    4
        ccv_nnc_tensor_free(cpu_inputs[i]);
    138
    4
        ccv_nnc_tensor_free(cpu_expected[i]);
    139
    4
        ccv_nnc_tensor_free(cpu_outputs[i]);
    140
    4
        ccv_nnc_tensor_free(gpu_inputs[i]);
    141
    4
        ccv_nnc_tensor_free(gpu_outputs[i]);
    142
    4
      }
    143
    1
    }
    144
    145
    static ccv_nnc_stream_context_t* _neighbor_discovery(const int device_id, void* const contexts)
    146
    16
    {
    147
    16
      ccv_nnc_stream_context_t** stream_contexts = (ccv_nnc_stream_context_t**)contexts;
    148
    16
      return stream_contexts[device_id];
    149
    16
    }
    150
    151
    TEST_CASE("nccl with allreduce in non-blocking mode")
    152
    1
    {
    153
    1
      const int device_count = ccv_nnc_device_count(CCV_STREAM_CONTEXT_GPU);
    154
    1
      GUARD_ELSE_RETURN(ccv_nnc_cmd_ok(CCV_NNC_COMM_ALLREDUCE_FORWARD, CCV_NNC_BACKEND_GPU_NCCL) && device_count > 1);
    155
    1
      ccv_nnc_tensor_t* tensors[device_count];
    156
    1
      ccv_nnc_stream_context_t* contexts[device_count];
    157
    1
      int i;
    158
    5
      for (i = 0; i < device_count; i++4)
    159
    4
      {
    160
    4
        ccv_nnc_tensor_param_t info = GPU_TENSOR_NHWC(000, 32F, 100);
    161
    4
        CCV_TENSOR_SET_DEVICE_ID(info.type, i);
    162
    4
        tensors[i] = ccv_nnc_tensor_new(0, info, 0);
    163
    4
        ccv_nnc_cmd_exec(CMD_SET_FORWARD(i + 0.5), ccv_nnc_no_hint, 0, 0, 0, &tensors[i], 1, 0);
    164
    4
        int stream_type = CCV_STREAM_CONTEXT_GPU;
    165
    4
        CCV_STREAM_SET_DEVICE_ID(stream_type, i);
    166
    4
        contexts[i] = ccv_nnc_stream_context_new(stream_type);
    167
    4
      }
    168
    1
      ccv_nnc_stream_context_set_neighbor_discovery(contexts[0], _neighbor_discovery, contexts);
    169
    1
      ccv_nnc_cmd_exec(CMD_COMM_ALLREDUCE_FORWARD(), ccv_nnc_no_hint, 0, tensors, device_count, tensors, device_count, contexts[0]);
    170
    1
      ccv_nnc_tensor_t* cpu_tensors[device_count];
    171
    5
      for (i = 0; i < device_count; i++4)
    172
    4
      {
    173
    4
        cpu_tensors[i] = ccv_nnc_tensor_new(0, CPU_TENSOR_NHWC(32F, 100), 0);
    174
    4
        ccv_nnc_cmd_exec(CMD_DATA_TRANSFER_FORWARD(), ccv_nnc_no_hint, 0, &tensors[i], 1, &cpu_tensors[i], 1, contexts[i]);
    175
    4
      }
    176
    1
      ccv_nnc_tensor_t* demo_tensor = ccv_nnc_tensor_new(0, CPU_TENSOR_NHWC(32F, 100), 0);
    177
    1
      ccv_nnc_cmd_exec(CMD_SET_FORWARD((device_count - 1) * device_count / 2 + 0.5 * device_count), ccv_nnc_no_hint, 0, 0, 0, &demo_tensor, 1, 0);
    178
    5
      for (i = 0; i < device_count; i++4)
    179
    4
        ccv_nnc_stream_context_wait(contexts[i]);
    180
    5
      for (i = 0; i < device_count; i++4)
    181
    4
        REQUIRE_TENSOR_EQ(demo_tensor, cpu_tensors[i], "all values should be summed");
    182
    1
      ccv_nnc_tensor_free(demo_tensor);
    183
    5
      for (i = 0; i < device_count; i++4)
    184
    4
      {
    185
    4
        ccv_nnc_tensor_free(tensors[i]);
    186
    4
        ccv_nnc_tensor_free(cpu_tensors[i]);
    187
    4
        ccv_nnc_stream_context_free(contexts[i]);
    188
    4
      }
    189
    1
    }
    190
    191
    TEST_CASE("nccl with broadcast in non-blocking mode")
    192
    1
    {
    193
    1
      const int device_count = ccv_nnc_device_count(CCV_STREAM_CONTEXT_GPU);
    194
    1
      GUARD_ELSE_RETURN(ccv_nnc_cmd_ok(CCV_NNC_COMM_BROADCAST_FORWARD, CCV_NNC_BACKEND_GPU_NCCL) && device_count > 1);
    195
    1
      ccv_nnc_tensor_t* tensors[device_count];
    196
    1
      ccv_nnc_stream_context_t* contexts[device_count];
    197
    1
      int i;
    198
    5
      for (i = 0; i < device_count; i++4)
    199
    4
      {
    200
    4
        ccv_nnc_tensor_param_t info = GPU_TENSOR_NHWC(000, 32F, 100);
    201
    4
        CCV_TENSOR_SET_DEVICE_ID(info.type, i);
    202
    4
        tensors[i] = ccv_nnc_tensor_new(0, info, 0);
    203
    4
        ccv_nnc_cmd_exec(CMD_SET_FORWARD(i + 1), ccv_nnc_no_hint, 0, 0, 0, &tensors[i], 1, 0);
    204
    4
        int stream_type = CCV_STREAM_CONTEXT_GPU;
    205
    4
        CCV_STREAM_SET_DEVICE_ID(stream_type, i);
    206
    4
        contexts[i] = ccv_nnc_stream_context_new(stream_type);
    207
    4
      }
    208
    1
      ccv_nnc_stream_context_set_neighbor_discovery(contexts[0], _neighbor_discovery, contexts);
    209
    1
      ccv_nnc_cmd_exec(CMD_COMM_BROADCAST_FORWARD(), ccv_nnc_no_hint, 0, tensors, 1, tensors, device_count, contexts[0]);
    210
    1
      ccv_nnc_tensor_t* cpu_tensors[device_count];
    211
    5
      for (i = 0; i < device_count; i++4)
    212
    4
      {
    213
    4
        cpu_tensors[i] = ccv_nnc_tensor_new(0, CPU_TENSOR_NHWC(32F, 100), 0);
    214
    4
        ccv_nnc_cmd_exec(CMD_DATA_TRANSFER_FORWARD(), ccv_nnc_no_hint, 0, &tensors[i], 1, &cpu_tensors[i], 1, contexts[i]);
    215
    4
      }
    216
    1
      ccv_nnc_tensor_t* demo_tensor = ccv_nnc_tensor_new(0, CPU_TENSOR_NHWC(32F, 100), 0);
    217
    1
      ccv_nnc_cmd_exec(CMD_SET_FORWARD(1), ccv_nnc_no_hint, 0, 0, 0, &demo_tensor, 1, 0);
    218
    5
      for (i = 0; i < device_count; i++4)
    219
    4
        ccv_nnc_stream_context_wait(contexts[i]);
    220
    5
      for (i = 0; i < device_count; i++4)
    221
    4
        REQUIRE_TENSOR_EQ(demo_tensor, cpu_tensors[i], "all values should be summed");
    222
    1
      ccv_nnc_tensor_free(demo_tensor);
    223
    5
      for (i = 0; i < device_count; i++4)
    224
    4
      {
    225
    4
        ccv_nnc_tensor_free(tensors[i]);
    226
    4
        ccv_nnc_tensor_free(cpu_tensors[i]);
    227
    4
        ccv_nnc_stream_context_free(contexts[i]);
    228
    4
      }
    229
    1
    }
    230
    231
    TEST_CASE("nccl with reduce in non-blocking mode")
    232
    1
    {
    233
    1
      const int device_count = ccv_nnc_device_count(CCV_STREAM_CONTEXT_GPU);
    234
    1
      GUARD_ELSE_RETURN(ccv_nnc_cmd_ok(CCV_NNC_COMM_REDUCE_FORWARD, CCV_NNC_BACKEND_GPU_NCCL) && device_count > 1);
    235
    1
      ccv_nnc_tensor_t* tensors[device_count];
    236
    1
      ccv_nnc_stream_context_t* contexts[device_count];
    237
    1
      int i;
    238
    5
      for (i = 0; i < device_count; i++4)
    239
    4
      {
    240
    4
        ccv_nnc_tensor_param_t info = GPU_TENSOR_NHWC(000, 32F, 100);
    241
    4
        CCV_TENSOR_SET_DEVICE_ID(info.type, i);
    242
    4
        tensors[i] = ccv_nnc_tensor_new(0, info, 0);
    243
    4
        ccv_nnc_cmd_exec(CMD_SET_FORWARD(i + 1), ccv_nnc_no_hint, 0, 0, 0, &tensors[i], 1, 0);
    244
    4
        int stream_type = CCV_STREAM_CONTEXT_GPU;
    245
    4
        CCV_STREAM_SET_DEVICE_ID(stream_type, i);
    246
    4
        contexts[i] = ccv_nnc_stream_context_new(stream_type);
    247
    4
      }
    248
    1
      ccv_nnc_stream_context_set_neighbor_discovery(contexts[0], _neighbor_discovery, contexts);
    249
    1
      ccv_nnc_cmd_exec(CMD_COMM_REDUCE_FORWARD(), ccv_nnc_no_hint, 0, tensors, device_count, tensors, 1, contexts[0]);
    250
    1
      ccv_nnc_tensor_t* cpu_tensor;
    251
    1
      cpu_tensor = ccv_nnc_tensor_new(0, CPU_TENSOR_NHWC(32F, 100), 0);
    252
    1
      ccv_nnc_cmd_exec(CMD_DATA_TRANSFER_FORWARD(), ccv_nnc_no_hint, 0, &tensors[0], 1, &cpu_tensor, 1, contexts[0]);
    253
    1
      ccv_nnc_tensor_t* demo_tensor = ccv_nnc_tensor_new(0, CPU_TENSOR_NHWC(32F, 100), 0);
    254
    1
      ccv_nnc_cmd_exec(CMD_SET_FORWARD((device_count + 1) * device_count / 2), ccv_nnc_no_hint, 0, 0, 0, &demo_tensor, 1, 0);
    255
    1
      ccv_nnc_stream_context_wait(contexts[0]);
    256
    1
      REQUIRE_TENSOR_EQ(demo_tensor, cpu_tensor, "all values should be summed");
    257
    1
      ccv_nnc_tensor_free(demo_tensor);
    258
    1
      ccv_nnc_tensor_free(cpu_tensor);
    259
    5
      for (i = 0; i < device_count; i++4)
    260
    4
      {
    261
    4
        ccv_nnc_tensor_free(tensors[i]);
    262
    4
        ccv_nnc_stream_context_wait(contexts[i]);
    263
    4
        ccv_nnc_stream_context_free(contexts[i]);
    264
    4
      }
    265
    1
    }
    266
    267
    TEST_CASE("nccl with all-to-all in non-blocking mode against cpu reference")
    268
    1
    {
    269
    1
      const int device_count = ccv_nnc_device_count(CCV_STREAM_CONTEXT_GPU);
    270
    1
      GUARD_ELSE_RETURN(ccv_nnc_cmd_ok(CCV_NNC_COMM_ALL_TO_ALL_FORWARD, CCV_NNC_BACKEND_GPU_NCCL) && ccv_nnc_cmd_ok(CCV_NNC_COMM_ALL_TO_ALL_FORWARD, CCV_NNC_BACKEND_CPU_REF) && device_count > 1);
    271
    1
      const int axis = 1;
    272
    1
      const int outer_count = 3;
    273
    1
      const int chunk_count = 23;
    274
    1
      const int axis_count = device_count * chunk_count;
    275
    1
      const int inner_count = 5;
    276
    1
      const int tensor_count = outer_count * axis_count * inner_count;
    277
    1
      ccv_nnc_tensor_t* cpu_inputs[device_count];
    278
    1
      ccv_nnc_tensor_t* cpu_expected[device_count];
    279
    1
      ccv_nnc_tensor_t* cpu_outputs[device_count];
    280
    1
      ccv_nnc_tensor_t* gpu_inputs[device_count];
    281
    1
      ccv_nnc_tensor_t* gpu_outputs[device_count];
    282
    1
      ccv_nnc_stream_context_t* contexts[device_count];
    283
    1
      int i, j;
    284
    5
      for (i = 0; i < device_count; i++4)
    285
    4
      {
    286
    4
        cpu_inputs[i] = ccv_nnc_tensor_new(0, CPU_TENSOR_NHWC(32F, outer_count, axis_count, inner_count), 0);
    287
    4
        cpu_expected[i] = ccv_nnc_tensor_new(0, CPU_TENSOR_NHWC(32F, outer_count, axis_count, inner_count), 0);
    288
    4
        cpu_outputs[i] = ccv_nnc_tensor_new(0, CPU_TENSOR_NHWC(32F, outer_count, axis_count, inner_count), 0);
    289
    5.52k
        for (j = 0; j < tensor_count; j++5.52k)
    290
    5.52k
          cpu_inputs[i]->data.f32[j] = (float)(i * tensor_count + j + 0.25);
    291
    4
        ccv_nnc_tensor_param_t info = GPU_TENSOR_NHWC(000, 32F, outer_count, axis_count, inner_count);
    292
    4
        CCV_TENSOR_SET_DEVICE_ID(info.type, i);
    293
    4
        gpu_inputs[i] = ccv_nnc_tensor_new(0, info, 0);
    294
    4
        gpu_outputs[i] = ccv_nnc_tensor_new(0, info, 0);
    295
    4
        int stream_type = CCV_STREAM_CONTEXT_GPU;
    296
    4
        CCV_STREAM_SET_DEVICE_ID(stream_type, i);
    297
    4
        contexts[i] = ccv_nnc_stream_context_new(stream_type);
    298
    4
        ccv_nnc_cmd_exec(CMD_DATA_TRANSFER_FORWARD(), ccv_nnc_no_hint, 0, &cpu_inputs[i], 1, &gpu_inputs[i], 1, contexts[i]);
    299
    4
      }
    300
    1
      ccv_nnc_stream_context_set_neighbor_discovery(contexts[0], _neighbor_discovery, contexts);
    301
    1
      ccv_nnc_cmd_exec(CMD_COMM_ALL_TO_ALL_FORWARD(axis), ccv_nnc_no_hint, 0, cpu_inputs, device_count, cpu_expected, device_count, 0);
    302
    1
      ccv_nnc_cmd_exec(CMD_COMM_ALL_TO_ALL_FORWARD(axis), ccv_nnc_no_hint, 0, gpu_inputs, device_count, gpu_outputs, device_count, contexts[0]);
    303
    5
      for (i = 0; i < device_count; i++4)
    304
    4
        ccv_nnc_cmd_exec(CMD_DATA_TRANSFER_FORWARD(), ccv_nnc_no_hint, 0, &gpu_outputs[i], 1, &cpu_outputs[i], 1, contexts[i]);
    305
    5
      for (i = 0; i < device_count; i++4)
    306
    4
        ccv_nnc_stream_context_wait(contexts[i]);
    307
    5
      for (i = 0; i < device_count; i++4)
    308
    4
        REQUIRE_TENSOR_EQ(cpu_expected[i], cpu_outputs[i], "all-to-all should match cpu reference");
    309
    5
      for (i = 0; i < device_count; i++4)
    310
    4
      {
    311
    4
        ccv_nnc_tensor_free(cpu_inputs[i]);
    312
    4
        ccv_nnc_tensor_free(cpu_expected[i]);
    313
    4
        ccv_nnc_tensor_free(cpu_outputs[i]);
    314
    4
        ccv_nnc_tensor_free(gpu_inputs[i]);
    315
    4
        ccv_nnc_tensor_free(gpu_outputs[i]);
    316
    4
        ccv_nnc_stream_context_free(contexts[i]);
    317
    4
      }
    318
    1
    }
    319
    320
    #include "case_main.h"