Skip to content

Commit 43b8872

Browse files
fix: track registered CSR clients
In L0 its not possible to track objects relations. For example CmdList may be removed before Event. In such case, Event needs to safely skip unregister call, without accessing CmdList/CmdQueue object. Related-To: NEO-8884 Signed-off-by: Dunajski, Bartosz <[email protected]> Source: 53f635e
1 parent 12d60ca commit 43b8872

File tree

18 files changed

+127
-77
lines changed

18 files changed

+127
-77
lines changed

level_zero/core/source/cmdlist/cmdlist_hw_immediate.inl

Lines changed: 1 addition & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -339,9 +339,7 @@ inline ze_result_t CommandListCoreFamilyImmediate<gfxCoreFamily>::executeCommand
339339
svmAllocMgr->prefetchSVMAllocs(*this->device->getNEODevice(), *csr);
340340
}
341341

342-
if (cmdQ->getClientId() == CommandQueue::clientNotRegistered) {
343-
cmdQ->setClientId(csr->registerClient());
344-
}
342+
cmdQ->registerCsrClient();
345343

346344
std::unique_lock<std::mutex> lockForIndirect;
347345
if (this->hasIndirectAllocationsAllowed()) {

level_zero/core/source/cmdqueue/cmdqueue.cpp

Lines changed: 5 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -253,10 +253,11 @@ CommandQueue *CommandQueue::create(uint32_t productFamily, Device *device, NEO::
253253
}
254254

255255
void CommandQueueImp::unregisterCsrClient() {
256-
if (getClientId() != CommandQueue::clientNotRegistered) {
257-
this->csr->unregisterClient();
258-
setClientId(CommandQueue::clientNotRegistered);
259-
}
256+
this->csr->unregisterClient(this);
257+
}
258+
259+
void CommandQueueImp::registerCsrClient() {
260+
this->csr->registerClient(this);
260261
}
261262

262263
ze_command_queue_mode_t CommandQueueImp::getSynchronousMode() const {

level_zero/core/source/cmdqueue/cmdqueue.h

Lines changed: 1 addition & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -62,19 +62,15 @@ struct CommandQueue : _ze_command_queue_handle_t {
6262

6363
bool peekIsCopyOnlyCommandQueue() const { return this->isCopyOnlyCommandQueue; }
6464

65-
uint32_t getClientId() const { return this->clientId; }
66-
void setClientId(uint32_t value) { this->clientId = value; }
6765
virtual void unregisterCsrClient() = 0;
66+
virtual void registerCsrClient() = 0;
6867

6968
TaskCountType getTaskCount() const { return taskCount; }
7069
void setTaskCount(TaskCountType newTaskCount) { taskCount = newTaskCount; }
7170

72-
static constexpr uint32_t clientNotRegistered = std::numeric_limits<uint32_t>::max();
73-
7471
protected:
7572
bool frontEndTrackingEnabled() const;
7673

77-
uint32_t clientId = clientNotRegistered;
7874
uint32_t partitionCount = 1;
7975
uint32_t activeSubDevices = 1;
8076
std::atomic<TaskCountType> taskCount = 0;

level_zero/core/source/cmdqueue/cmdqueue_hw.inl

Lines changed: 1 addition & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -73,9 +73,7 @@ ze_result_t CommandQueueHw<gfxCoreFamily>::executeCommandLists(
7373
svmAllocMgr->prefetchSVMAllocs(*device->getNEODevice(), *csr);
7474
}
7575

76-
if (this->clientId == CommandQueue::clientNotRegistered) {
77-
this->clientId = this->csr->registerClient();
78-
}
76+
registerCsrClient();
7977

8078
auto neoDevice = device->getNEODevice();
8179
auto ctx = CommandListExecutionContext{phCommandLists,

level_zero/core/source/cmdqueue/cmdqueue_imp.h

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -90,6 +90,7 @@ struct CommandQueueImp : public CommandQueue {
9090
void printKernelsPrintfOutput(bool hangDetected);
9191
void checkAssert();
9292
void unregisterCsrClient() override;
93+
void registerCsrClient() override;
9394

9495
protected:
9596
MOCKABLE_VIRTUAL NEO::SubmissionStatus submitBatchBuffer(size_t offset, NEO::ResidencyContainer &residencyContainer, void *endingCmdPtr,

level_zero/core/source/event/event.cpp

Lines changed: 3 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -402,9 +402,10 @@ void Event::setLatestUsedCmdQueue(CommandQueue *newCmdQ) {
402402
}
403403

404404
void Event::unsetCmdQueue() {
405-
if (latestUsedCmdQueue) {
406-
latestUsedCmdQueue->unregisterCsrClient();
405+
for (auto &csr : csrs) {
406+
csr->unregisterClient(latestUsedCmdQueue);
407407
}
408+
408409
latestUsedCmdQueue = nullptr;
409410
}
410411

level_zero/core/test/unit_tests/sources/cmdlist/test_cmdlist_1.cpp

Lines changed: 15 additions & 11 deletions
Original file line numberDiff line numberDiff line change
@@ -1200,7 +1200,6 @@ HWTEST2_F(CommandListCreate, givenDirectSubmissionAndImmCmdListWhenDispatchingDi
12001200

12011201
auto ultCsr = static_cast<NEO::UltCommandStreamReceiver<FamilyType> *>(whiteBoxCmdList->csr);
12021202
ultCsr->recordFlusheBatchBuffer = true;
1203-
ultCsr->unregisterClient();
12041203

12051204
EXPECT_FALSE(NEO::RelaxedOrderingHelper::isRelaxedOrderingDispatchAllowed(*ultCsr, 1));
12061205

@@ -1299,8 +1298,9 @@ HWTEST2_F(CommandListCreate, whenDispatchingThenPassNumCsrClients, IsAtLeastXeHp
12991298
auto ultCsr = static_cast<NEO::UltCommandStreamReceiver<FamilyType> *>(whiteBoxCmdList->csr);
13001299
ultCsr->recordFlusheBatchBuffer = true;
13011300

1302-
ultCsr->registerClient();
1303-
ultCsr->registerClient();
1301+
int client1, client2;
1302+
ultCsr->registerClient(&client1);
1303+
ultCsr->registerClient(&client2);
13041304

13051305
auto result = commandList->appendLaunchKernel(kernel.toHandle(), &groupCount, nullptr, 0, nullptr, launchParams, false);
13061306

@@ -1458,8 +1458,9 @@ HWTEST2_F(CommandListCreate, givenDirectSubmissionAndImmCmdListWhenDispatchingTh
14581458

14591459
auto directSubmission = new MockDirectSubmissionHw<FamilyType, RenderDispatcher<FamilyType>>(*ultCsr);
14601460
ultCsr->directSubmission.reset(directSubmission);
1461-
ultCsr->registerClient();
1462-
ultCsr->registerClient();
1461+
int client1, client2;
1462+
ultCsr->registerClient(&client1);
1463+
ultCsr->registerClient(&client2);
14631464

14641465
auto verifyFlags = [&ultCsr, useImmediateFlushTask](ze_result_t result, bool dispatchFlag, bool bbFlag) {
14651466
EXPECT_EQ(ZE_RESULT_SUCCESS, result);
@@ -1580,8 +1581,9 @@ HWTEST2_F(CommandListCreate, givenInOrderExecutionWhenDispatchingRelaxedOrdering
15801581

15811582
auto directSubmission = new MockDirectSubmissionHw<FamilyType, RenderDispatcher<FamilyType>>(*ultCsr);
15821583
ultCsr->directSubmission.reset(directSubmission);
1583-
ultCsr->registerClient();
1584-
ultCsr->registerClient();
1584+
int client1, client2;
1585+
ultCsr->registerClient(&client1);
1586+
ultCsr->registerClient(&client2);
15851587

15861588
commandList->appendLaunchKernel(kernel.toHandle(), &groupCount, event, 0, nullptr, launchParams, false);
15871589

@@ -1628,8 +1630,9 @@ HWTEST2_F(CommandListCreate, givenInOrderExecutionWhenDispatchingBarrierThenAllo
16281630

16291631
auto directSubmission = new MockDirectSubmissionHw<FamilyType, RenderDispatcher<FamilyType>>(*ultCsr);
16301632
ultCsr->directSubmission.reset(directSubmission);
1631-
ultCsr->registerClient();
1632-
ultCsr->registerClient();
1633+
int client1, client2;
1634+
ultCsr->registerClient(&client1);
1635+
ultCsr->registerClient(&client2);
16331636

16341637
// Initialize NP state
16351638
commandList->appendBarrier(nullptr, 1, &event, false);
@@ -1687,8 +1690,9 @@ HWTEST2_F(CommandListCreate, givenInOrderExecutionWhenDispatchingRelaxedOrdering
16871690

16881691
auto directSubmission = new MockDirectSubmissionHw<FamilyType, RenderDispatcher<FamilyType>>(*ultCsr);
16891692
ultCsr->directSubmission.reset(directSubmission);
1690-
ultCsr->registerClient();
1691-
ultCsr->registerClient();
1693+
int client1, client2;
1694+
ultCsr->registerClient(&client1);
1695+
ultCsr->registerClient(&client2);
16921696

16931697
auto cmdStream = cmdList->getCmdContainer().getCommandStream();
16941698

level_zero/core/test/unit_tests/sources/cmdlist/test_cmdlist_7.cpp

Lines changed: 4 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -2964,7 +2964,8 @@ HWTEST2_F(ImmediateCommandListHostSynchronize, givenCsrClientCountWhenCallingSyn
29642964

29652965
auto cmdList = createCmdList<gfxCoreFamily>(csr);
29662966

2967-
cmdList->cmdQImmediate->setClientId(csr->registerClient());
2967+
cmdList->cmdQImmediate->registerCsrClient();
2968+
29682969
auto clientCount = csr->getNumClients();
29692970

29702971
EXPECT_EQ(cmdList->hostSynchronize(0), ZE_RESULT_SUCCESS);
@@ -2975,7 +2976,8 @@ HWTEST2_F(ImmediateCommandListHostSynchronize, givenCsrClientCountWhenCallingSyn
29752976

29762977
EXPECT_EQ(clientCount - 1, csr->getNumClients());
29772978

2978-
cmdList->cmdQImmediate->setClientId(csr->registerClient());
2979+
cmdList->cmdQImmediate->registerCsrClient();
2980+
29792981
clientCount = csr->getNumClients();
29802982

29812983
csr->callBaseWaitForCompletionWithTimeout = false;

level_zero/core/test/unit_tests/xe_hpc_core/test_cmdqueue_xe_hpc_core.cpp

Lines changed: 5 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -715,7 +715,8 @@ HWTEST2_F(CommandQueueCommandsXeHpc, givenFlushTaskSubmissionEnabledAndSplitBcsC
715715
context->allocHostMem(&hostDesc, size, alignment, &dstPtr);
716716
auto ultCsr = static_cast<NEO::UltCommandStreamReceiver<FamilyType> *>(whiteBoxCmdList->csr);
717717
ultCsr->recordFlusheBatchBuffer = true;
718-
ultCsr->registerClient();
718+
int client;
719+
ultCsr->registerClient(&client);
719720

720721
auto result = commandList0->appendMemoryCopy(dstPtr, srcPtr, size, nullptr, 0, nullptr, false, false);
721722
ASSERT_EQ(ZE_RESULT_SUCCESS, result);
@@ -778,8 +779,9 @@ HWTEST2_F(CommandQueueCommandsXeHpc, givenFlushTaskSubmissionEnabledAndSplitBcsC
778779
context->allocHostMem(&hostDesc, size, alignment, &dstPtr);
779780
auto ultCsr = static_cast<NEO::UltCommandStreamReceiver<FamilyType> *>(whiteBoxCmdList->csr);
780781
ultCsr->recordFlusheBatchBuffer = true;
781-
ultCsr->registerClient();
782-
ultCsr->registerClient();
782+
int client1, client2;
783+
ultCsr->registerClient(&client1);
784+
ultCsr->registerClient(&client2);
783785

784786
auto directSubmission = new MockDirectSubmissionHw<FamilyType, RenderDispatcher<FamilyType>>(*ultCsr);
785787
ultCsr->directSubmission.reset(directSubmission);

opencl/source/command_queue/command_queue.cpp

Lines changed: 4 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -1406,7 +1406,7 @@ void CommandQueue::registerGpgpuCsrClient() {
14061406
if (!gpgpuCsrClientRegistered) {
14071407
gpgpuCsrClientRegistered = true;
14081408

1409-
getGpgpuCommandStreamReceiver().registerClient();
1409+
getGpgpuCommandStreamReceiver().registerClient(this);
14101410
}
14111411
}
14121412

@@ -1417,13 +1417,13 @@ void CommandQueue::registerBcsCsrClient(CommandStreamReceiver &bcsCsr) {
14171417

14181418
if (!bcsState.csrClientRegistered) {
14191419
bcsState.csrClientRegistered = true;
1420-
bcsCsr.registerClient();
1420+
bcsCsr.registerClient(this);
14211421
}
14221422
}
14231423

14241424
void CommandQueue::unregisterGpgpuCsrClient() {
14251425
if (gpgpuCsrClientRegistered) {
1426-
gpgpuEngine->commandStreamReceiver->unregisterClient();
1426+
gpgpuEngine->commandStreamReceiver->unregisterClient(this);
14271427
gpgpuCsrClientRegistered = false;
14281428
}
14291429
}
@@ -1434,7 +1434,7 @@ void CommandQueue::unregisterBcsCsrClient(CommandStreamReceiver &bcsCsr) {
14341434
auto &bcsState = bcsStates[EngineHelpers::getBcsIndex(engineType)];
14351435

14361436
if (bcsState.isValid() && bcsState.csrClientRegistered) {
1437-
bcsCsr.unregisterClient();
1437+
bcsCsr.unregisterClient(this);
14381438
bcsState.csrClientRegistered = false;
14391439
}
14401440
}

opencl/test/unit_test/command_queue/command_queue_hw_1_tests.cpp

Lines changed: 3 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -1254,11 +1254,12 @@ HWTEST_F(CommandQueueHwTest, givenRelaxedOrderingEnabledWhenCheckingIfAllowedByC
12541254
EXPECT_FALSE(mockCmdQueueHw.relaxedOrderingForGpgpuAllowed(0));
12551255
EXPECT_FALSE(mockCmdQueueHw.relaxedOrderingForGpgpuAllowed(1));
12561256

1257-
ultCsr.registerClient();
1257+
int client1, client2;
1258+
ultCsr.registerClient(&client1);
12581259
EXPECT_FALSE(mockCmdQueueHw.relaxedOrderingForGpgpuAllowed(0));
12591260
EXPECT_FALSE(mockCmdQueueHw.relaxedOrderingForGpgpuAllowed(1));
12601261

1261-
ultCsr.registerClient();
1262+
ultCsr.registerClient(&client2);
12621263

12631264
EXPECT_FALSE(mockCmdQueueHw.relaxedOrderingForGpgpuAllowed(0));
12641265
EXPECT_TRUE(mockCmdQueueHw.relaxedOrderingForGpgpuAllowed(1));

opencl/test/unit_test/command_queue/enqueue_kernel_2_tests.cpp

Lines changed: 12 additions & 8 deletions
Original file line numberDiff line numberDiff line change
@@ -1165,8 +1165,9 @@ HWTEST2_F(RelaxedOrderingEnqueueKernelTests, givenEnqueueKernelWhenProgrammingDe
11651165
auto &ultCsr = pDevice->getUltCommandStreamReceiver<FamilyType>();
11661166
auto directSubmission = new MockDirectSubmissionHw<FamilyType, RenderDispatcher<FamilyType>>(ultCsr);
11671167
ultCsr.directSubmission.reset(directSubmission);
1168-
ultCsr.registerClient();
1169-
ultCsr.registerClient();
1168+
int client1, client2;
1169+
ultCsr.registerClient(&client1);
1170+
ultCsr.registerClient(&client2);
11701171

11711172
MockCommandQueueHw<FamilyType> mockCmdQueueHw{context, pClDevice, nullptr};
11721173

@@ -1264,8 +1265,9 @@ HWTEST2_F(RelaxedOrderingEnqueueKernelTests, givenBarrierWithDependenciesWhenFlu
12641265
auto &ultCsr = pDevice->getUltCommandStreamReceiver<FamilyType>();
12651266
auto directSubmission = new MockDirectSubmissionHw<FamilyType, RenderDispatcher<FamilyType>>(ultCsr);
12661267
ultCsr.directSubmission.reset(directSubmission);
1267-
ultCsr.registerClient();
1268-
ultCsr.registerClient();
1268+
int client1, client2;
1269+
ultCsr.registerClient(&client1);
1270+
ultCsr.registerClient(&client2);
12691271

12701272
MockCommandQueueHw<FamilyType> mockCmdQueueHw{context, pClDevice, nullptr};
12711273

@@ -1311,8 +1313,9 @@ HWTEST2_F(RelaxedOrderingEnqueueKernelTests, givenPipeControlForIoqDependencyRes
13111313
auto &ultCsr = pDevice->getUltCommandStreamReceiver<FamilyType>();
13121314
auto directSubmission = new MockDirectSubmissionHw<FamilyType, RenderDispatcher<FamilyType>>(ultCsr);
13131315
ultCsr.directSubmission.reset(directSubmission);
1314-
ultCsr.registerClient();
1315-
ultCsr.registerClient();
1316+
int client1, client2;
1317+
ultCsr.registerClient(&client1);
1318+
ultCsr.registerClient(&client2);
13161319

13171320
MockKernelWithInternals mockKernel(*pClDevice);
13181321

@@ -1331,8 +1334,9 @@ HWTEST2_F(RelaxedOrderingEnqueueKernelTests, givenEnqueueWithPipeControlWhenSend
13311334
auto &ultCsr = pDevice->getUltCommandStreamReceiver<FamilyType>();
13321335
auto directSubmission = new MockDirectSubmissionHw<FamilyType, RenderDispatcher<FamilyType>>(ultCsr);
13331336
ultCsr.directSubmission.reset(directSubmission);
1334-
ultCsr.registerClient();
1335-
ultCsr.registerClient();
1337+
int client1, client2;
1338+
ultCsr.registerClient(&client1);
1339+
ultCsr.registerClient(&client2);
13361340

13371341
ultCsr.recordFlusheBatchBuffer = true;
13381342

opencl/test/unit_test/command_stream/command_stream_receiver_hw_1_tests.cpp

Lines changed: 13 additions & 10 deletions
Original file line numberDiff line numberDiff line change
@@ -1091,8 +1091,9 @@ HWTEST2_F(RelaxedOrderingBcsTests, givenDependenciesWhenFlushingThenProgramCorre
10911091
ASSERT_EQ(CL_SUCCESS, retVal);
10921092

10931093
auto &csr = pDevice->getUltCommandStreamReceiver<FamilyType>();
1094-
csr.registerClient();
1095-
csr.registerClient();
1094+
int client1, client2;
1095+
csr.registerClient(&client1);
1096+
csr.registerClient(&client2);
10961097
csr.recordFlusheBatchBuffer = true;
10971098

10981099
csr.blitterDirectSubmission = std::make_unique<MockDirectSubmissionHw<FamilyType, BlitterDispatcher<FamilyType>>>(csr);
@@ -1133,8 +1134,9 @@ HWTEST2_F(RelaxedOrderingBcsTests, givenDependenciesWhenFlushingThenProgramProgr
11331134
ASSERT_EQ(CL_SUCCESS, retVal);
11341135

11351136
auto &csr = pDevice->getUltCommandStreamReceiver<FamilyType>();
1136-
csr.registerClient();
1137-
csr.registerClient();
1137+
int client1, client2;
1138+
csr.registerClient(&client1);
1139+
csr.registerClient(&client1);
11381140
csr.recordFlusheBatchBuffer = true;
11391141

11401142
csr.blitterDirectSubmission = std::make_unique<MockDirectSubmissionHw<FamilyType, BlitterDispatcher<FamilyType>>>(csr);
@@ -1158,12 +1160,12 @@ HWTEST2_F(RelaxedOrderingBcsTests, givenDependenciesWhenFlushingThenProgramProgr
11581160
EXPECT_FALSE(csr.bcsRelaxedOrderingAllowed(blitPropertiesContainer, false));
11591161

11601162
blitPropertiesContainer.push_back(blitProperties);
1161-
csr.unregisterClient();
1162-
csr.unregisterClient();
1163+
csr.unregisterClient(&client1);
1164+
csr.unregisterClient(&client2);
11631165
EXPECT_FALSE(csr.bcsRelaxedOrderingAllowed(blitPropertiesContainer, false));
11641166

1165-
csr.registerClient();
1166-
csr.registerClient();
1167+
csr.registerClient(&client1);
1168+
csr.registerClient(&client2);
11671169
csr.blitterDirectSubmission.reset();
11681170
EXPECT_FALSE(csr.bcsRelaxedOrderingAllowed(blitPropertiesContainer, false));
11691171
}
@@ -1174,8 +1176,9 @@ HWTEST2_F(RelaxedOrderingBcsTests, givenTagUpdateWhenFlushingThenDisableRelaxedO
11741176
ASSERT_EQ(CL_SUCCESS, retVal);
11751177

11761178
auto &csr = pDevice->getUltCommandStreamReceiver<FamilyType>();
1177-
csr.registerClient();
1178-
csr.registerClient();
1179+
int client1, client2;
1180+
csr.registerClient(&client1);
1181+
csr.registerClient(&client2);
11791182
csr.recordFlusheBatchBuffer = true;
11801183

11811184
csr.blitterDirectSubmission = std::make_unique<MockDirectSubmissionHw<FamilyType, BlitterDispatcher<FamilyType>>>(csr);

opencl/test/unit_test/command_stream/command_stream_receiver_hw_2_tests.cpp

Lines changed: 5 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -709,8 +709,11 @@ HWTEST_F(BcsTests, givenTaskStreamWhenFlushingThenPassNumClients) {
709709
graphicsAllocation->getGpuAddress(), 0,
710710
0, 0, {1, 1, 1}, 0, 0, 0, 0);
711711

712-
csr.registerClient();
713-
csr.registerClient();
712+
int client1, client2;
713+
csr.registerClient(&client1);
714+
csr.registerClient(&client2);
715+
716+
EXPECT_EQ(2u, csr.getNumClients());
714717

715718
flushBcsTask(&csr, blitProperties, true, *pDevice);
716719

shared/source/command_stream/command_stream_receiver.cpp

Lines changed: 21 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -95,6 +95,8 @@ CommandStreamReceiver::CommandStreamReceiver(ExecutionEnvironment &executionEnvi
9595
productHelper.fillStateBaseAddressPropertiesSupportStructure(sbaSupportFlags);
9696
this->doubleSbaWa = productHelper.isAdditionalStateBaseAddressWARequired(hwInfo);
9797
this->l1CachePolicyData.init(productHelper);
98+
99+
registeredClients.reserve(16);
98100
}
99101

100102
CommandStreamReceiver::~CommandStreamReceiver() {
@@ -1080,5 +1082,24 @@ bool CommandStreamReceiver::isRayTracingStateProgramingNeeded(Device &device) co
10801082
return device.getRTMemoryBackedBuffer() && getBtdCommandDirty();
10811083
}
10821084

1085+
void CommandStreamReceiver::registerClient(void *client) {
1086+
std::unique_lock<MutexType> lock(registeredClientsMutex);
1087+
auto element = std::find(registeredClients.begin(), registeredClients.end(), client);
1088+
if (element == registeredClients.end()) {
1089+
registeredClients.push_back(client);
1090+
numClients++;
1091+
}
1092+
}
1093+
1094+
void CommandStreamReceiver::unregisterClient(void *client) {
1095+
std::unique_lock<MutexType> lock(registeredClientsMutex);
1096+
1097+
auto element = std::find(registeredClients.begin(), registeredClients.end(), client);
1098+
if (element != registeredClients.end()) {
1099+
registeredClients.erase(element);
1100+
numClients--;
1101+
}
1102+
}
1103+
10831104
std::function<void()> CommandStreamReceiver::debugConfirmationFunction = []() { std::cin.get(); };
10841105
} // namespace NEO

0 commit comments

Comments
 (0)