diff --git a/CHANGE_NOTES b/CHANGE_NOTES index 834620a7..2f448346 100644 --- a/CHANGE_NOTES +++ b/CHANGE_NOTES @@ -52,6 +52,18 @@ Examples: - See em-odp/README for usage and compilation instructions. - See em-odp/include/event_machine/README_API for API changes +-------------------------------------------------------------------------------- +Event Machine (EM) on ODP v3.8.0-1 +-------------------------------------------------------------------------------- +- Fixes: + - event: fix em_event_clone_part() + In em_event_clone_part(), when cloning an event allocated not from an + EM-pool, but directly from an odp-pool, the event header and the EM user area + was not copied correctly. + When using odp_packet_copy_part() to clone the event, the ODP user area is + not copied resulting in an invalid EM event header and EM user area. + This is now fixed. + -------------------------------------------------------------------------------- Event Machine (EM) on ODP v3.8.0 -------------------------------------------------------------------------------- diff --git a/configure.ac b/configure.ac index 19824955..0afb3ab6 100755 --- a/configure.ac +++ b/configure.ac @@ -5,7 +5,7 @@ AC_PREREQ([2.69]) m4_define([em_version_api_major], [3]) m4_define([em_version_api_minor], [8]) m4_define([em_version_implementation], [0]) -m4_define([em_version_fix], [0]) +m4_define([em_version_fix], [1]) m4_define([em_version_api], [em_version_api_major.em_version_api_minor]) m4_define([em_version], [m4_if(m4_eval(em_version_fix), [0], diff --git a/src/em_event.c b/src/em_event.c old mode 100644 new mode 100755 index bf361ba8..63473103 --- a/src/em_event.c +++ b/src/em_event.c @@ -1,304 +1,328 @@ -/* - * Copyright (c) 2015, Nokia Solutions and Networks - * All rights reserved. - * - * Redistribution and use in source and binary forms, with or without - * modification, are permitted provided that the following conditions - * are met: - * - * * Redistributions of source code must retain the above copyright - * notice, this list of conditions and the following disclaimer. - * * Redistributions in binary form must reproduce the above copyright - * notice, this list of conditions and the following disclaimer in the - * documentation and/or other materials provided with the distribution. - * * Neither the name of the copyright holder nor the names of its - * contributors may be used to endorse or promote products derived - * from this software without specific prior written permission. - * - * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS - * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT - * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR - * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT - * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, - * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT - * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, - * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY - * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT - * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE - * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. - */ - -#include "em_include.h" - -/* - * Sanity check that no extra padding is added to the event_hdr_t by - * alignment directives etc. - */ -typedef event_hdr_t _ev_hdr__size_check__arr_t[3]; -COMPILE_TIME_ASSERT(sizeof(_ev_hdr__size_check__arr_t) == - 3 * sizeof(event_hdr_t), EVENT_HDR_SIZE_ERROR2); - -/* - * Verify the value set for EM_CHECK_LEVEL - this define is set either from - * the include/event_machine/platform/event_machine_config.h file or by the - * configure.ac option --enable-check-level=N. - */ -COMPILE_TIME_ASSERT(EM_CHECK_LEVEL >= 0 && EM_CHECK_LEVEL <= 3, - EM_CHECK_LEVEL__BAD_VALUE); - -em_status_t event_init(void) -{ - return EM_OK; -} - -void print_event_info(void) -{ - EM_PRINT("\n" - "EM Events\n" - "---------\n" - "event-hdr size: %zu B\n", - sizeof(event_hdr_t)); - - EM_DBG("\t\toffset\tsize\n" - "\t\t------\t----\n" - "esv.state_cnt:\t%3zu B\t%2zu B\n" - "esv.state:\t%3zu B\t%2zu B\n" - "flags:\t\t%3zu B\t%2zu B\n" - "align_offset:\t%3zu B\t%2zu B\n" - "event_type:\t%3zu B\t%2zu B\n" - "event:\t\t%3zu B\t%2zu B\n" - "event_size:\t%3zu B\t%2zu B\n" - "egrp_gen:\t%3zu B\t%2zu B\n" - "egrp:\t\t%3zu B\t%2zu B\n" - "tmo:\t\t%3zu B\t%2zu B\n" - "user_area info:\t%3zu B\t%2zu B\n" - "end_hdr_data:\t%3zu B\t%2zu B\n" - " \t\t%3zu B\n" - "end:\t\t%3zu B\t%2zu B\n", - offsetof(event_hdr_t, state_cnt), sizeof_field(event_hdr_t, state_cnt), - offsetof(event_hdr_t, state), sizeof_field(event_hdr_t, state), - offsetof(event_hdr_t, flags), sizeof_field(event_hdr_t, flags), - offsetof(event_hdr_t, align_offset), sizeof_field(event_hdr_t, align_offset), - offsetof(event_hdr_t, event_type), sizeof_field(event_hdr_t, event_type), - offsetof(event_hdr_t, event), sizeof_field(event_hdr_t, event), - offsetof(event_hdr_t, event_size), sizeof_field(event_hdr_t, event_size), - offsetof(event_hdr_t, egrp_gen), sizeof_field(event_hdr_t, egrp_gen), - offsetof(event_hdr_t, egrp), sizeof_field(event_hdr_t, egrp), - offsetof(event_hdr_t, tmo), sizeof_field(event_hdr_t, tmo), - offsetof(event_hdr_t, user_area), sizeof_field(event_hdr_t, user_area), - offsetof(event_hdr_t, end_hdr_data), sizeof_field(event_hdr_t, end_hdr_data), - offsetof(event_hdr_t, end) - offsetof(event_hdr_t, end_hdr_data), - offsetof(event_hdr_t, end), sizeof_field(event_hdr_t, end)); - - EM_PRINT("\n"); -} - -/** - * Helper for em_event_clone(). - * - * Clone an event originating from an external odp pkt-pool. - * Initialize the new cloned event as an EM event and return it. - * - * Alloc and copy content via ODP. - * Also the ev_hdr in the odp-pkt user_area is copied. - */ -em_event_t pkt_clone_odp(odp_packet_t pkt, odp_pool_t pkt_pool, - uint32_t offset, uint32_t size, bool is_clone_part) -{ - odp_packet_t clone_pkt; - - if (is_clone_part) - clone_pkt = odp_packet_copy_part(pkt, offset, size, pkt_pool); - else - clone_pkt = odp_packet_copy(pkt, pkt_pool); - - if (unlikely(clone_pkt == ODP_PACKET_INVALID)) - return EM_EVENT_UNDEF; - - odp_packet_user_flag_set(clone_pkt, USER_FLAG_SET); - - odp_event_t odp_clone_event = odp_packet_to_event(clone_pkt); - event_hdr_t *clone_hdr = odp_packet_user_area(clone_pkt); - em_event_t clone_event = event_odp2em(odp_clone_event); - - /* - * Init hdr of event, also ESV init if needed. - * The clone_hdr is a copy of parent's, update only relevant fields. - */ - if (esv_enabled()) - clone_event = evstate_init(clone_event, clone_hdr, false); - else - clone_hdr->event = clone_event; - - /* clone_hdr->event_type = use parent's type as is */ - clone_hdr->egrp = EM_EVENT_GROUP_UNDEF; - clone_hdr->flags.all = 0; - - return clone_event; -} - -void -output_queue_track(queue_elem_t *const output_q_elem) -{ - output_queue_track_t *const track = - &em_locm.output_queue_track; - const uint32_t qidx = output_q_elem->output.idx; - - if (track->used_queues[qidx] == NULL) { - track->used_queues[qidx] = output_q_elem; - track->idx[track->idx_cnt++] = qidx; - } -} - -void -output_queue_drain(const queue_elem_t *output_q_elem) -{ - const em_queue_t output_queue = (em_queue_t)(uintptr_t)output_q_elem->queue; - const em_output_func_t output_fn = - output_q_elem->output.output_conf.output_fn; - void *const output_fn_args = - output_q_elem->output.output_conf.output_fn_args; - - const int deq_max = 32; - - em_event_t output_ev_tbl[deq_max]; - /* use same event-tbl, dequeue odp events into the EM event-tbl */ - odp_event_t *const odp_deq_events = (odp_event_t *)output_ev_tbl; - - const odp_queue_t odp_queue = output_q_elem->odp_queue; - unsigned int output_num; - int deq; - int ret; - - const bool esv_ena = esv_enabled(); - - do { - deq = odp_queue_deq_multi(odp_queue, - odp_deq_events/*out=output_ev_tbl[]*/, - deq_max); - if (unlikely(deq <= 0)) - return; - - output_num = (unsigned int)deq; - /* odp_deq_events[] == output_ev_tbl[] */ - if (esv_ena) { - event_hdr_t *ev_hdrs[output_num]; - - /* Restore hdls from ev_hdrs, odp-ev conv lost evgen */ - event_to_hdr_multi(output_ev_tbl, ev_hdrs, output_num); - for (unsigned int i = 0; i < output_num; i++) - output_ev_tbl[i] = ev_hdrs[i]->event; - } - - ret = output_fn(output_ev_tbl, output_num, - output_queue, output_fn_args); - - if (unlikely((unsigned int)ret != output_num)) - em_free_multi(&output_ev_tbl[ret], output_num - ret); - } while (deq > 0); -} - -void -output_queue_buffering_drain(void) -{ - output_queue_track_t *const track = &em_locm.output_queue_track; - - for (unsigned int i = 0; i < track->idx_cnt; i++) { - int qidx = track->idx[i]; - queue_elem_t *output_q_elem = track->used_queues[qidx]; - env_spinlock_t *lock = &output_q_elem->output.lock; - - /* - * drain if lock available, otherwise another core is already - * draining so no need to do anything. - */ - if (env_spinlock_trylock(lock)) { - output_queue_drain(output_q_elem); - env_spinlock_unlock(lock); - } - - track->idx[i] = 0; - track->used_queues[qidx] = NULL; - } - track->idx_cnt = 0; -} - -uint32_t event_vector_tbl(em_event_t vector_event, - em_event_t **event_tbl /*out*/) -{ - odp_event_t odp_event = event_em2odp(vector_event); - odp_packet_vector_t pkt_vec = odp_packet_vector_from_event(odp_event); - odp_packet_t *pkt_tbl = NULL; - const int pkts = odp_packet_vector_tbl(pkt_vec, &pkt_tbl/*out*/); - - *event_tbl = (em_event_t *)pkt_tbl; /* Careful! Points to same table */ - - if (!pkts) - return 0; - - event_hdr_t *ev_hdr_tbl[pkts]; - - /* - * Init the event-table as needed, might contain EM events or - * ODP packets depending on source. - */ - if (esv_enabled()) { - odp_packet_t odp_pkttbl[pkts]; - - /* - * Drop ESV generation from event handles by converting to - * odp-packets, then init as needed as EM events. - */ - events_em2pkt(*event_tbl/*in*/, odp_pkttbl/*out*/, pkts); - - event_init_pkt_multi(odp_pkttbl /*in*/, *event_tbl /*in,out*/, - ev_hdr_tbl /*out*/, pkts, false); - } else { - event_init_pkt_multi(pkt_tbl /*in*/, *event_tbl /*in,out*/, - ev_hdr_tbl /*out*/, pkts, false); - } - - return pkts; -} - -em_status_t event_vector_max_size(em_event_t vector_event, uint32_t *max_size /*out*/, - em_escope_t escope) -{ - odp_event_t odp_event = event_em2odp(vector_event); - odp_packet_vector_t pktvec = odp_packet_vector_from_event(odp_event); - odp_pool_t odp_pool = odp_packet_vector_pool(pktvec); - pool_subpool_t pool_subpool = pool_subpool_odp2em(odp_pool); - em_pool_t pool = (em_pool_t)(uintptr_t)pool_subpool.pool; - int subpool = pool_subpool.subpool; - - if (unlikely(pool == EM_POOL_UNDEF)) { - /* - * Don't report an error if 'pool == EM_POOL_UNDEF' since that - * might happen if the vector is input from pktio that is using - * external (to EM) odp vector pools. - */ - *max_size = 0; - return EM_OK; /* EM does not have the max_size info */ - } - - const mpool_elem_t *pool_elem = pool_elem_get(pool); - - if (unlikely(!pool_elem || - (EM_CHECK_LEVEL > 2 && !pool_allocated(pool_elem)))) { - *max_size = 0; - return INTERNAL_ERROR(EM_ERR_BAD_STATE, escope, - "Invalid pool:%" PRI_POOL "", pool); - } - - if (unlikely(subpool >= pool_elem->num_subpools)) { - /* not found */ - *max_size = 0; - return INTERNAL_ERROR(EM_ERR_NOT_FOUND, escope, - "Subpool not found, pool:%" PRI_POOL "", pool); - } - - /* subpool index found, store corresponding size */ - *max_size = pool_elem->size[subpool]; - - return EM_OK; -} +/* + * Copyright (c) 2015, Nokia Solutions and Networks + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions + * are met: + * + * * Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * * Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in the + * documentation and/or other materials provided with the distribution. + * * Neither the name of the copyright holder nor the names of its + * contributors may be used to endorse or promote products derived + * from this software without specific prior written permission. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS + * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT + * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR + * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT + * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, + * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT + * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, + * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY + * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT + * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE + * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + */ + +#include "em_include.h" + +/* + * Sanity check that no extra padding is added to the event_hdr_t by + * alignment directives etc. + */ +typedef event_hdr_t _ev_hdr__size_check__arr_t[3]; +COMPILE_TIME_ASSERT(sizeof(_ev_hdr__size_check__arr_t) == + 3 * sizeof(event_hdr_t), EVENT_HDR_SIZE_ERROR2); + +/* + * Verify the value set for EM_CHECK_LEVEL - this define is set either from + * the include/event_machine/platform/event_machine_config.h file or by the + * configure.ac option --enable-check-level=N. + */ +COMPILE_TIME_ASSERT(EM_CHECK_LEVEL >= 0 && EM_CHECK_LEVEL <= 3, + EM_CHECK_LEVEL__BAD_VALUE); + +em_status_t event_init(void) +{ + return EM_OK; +} + +void print_event_info(void) +{ + EM_PRINT("\n" + "EM Events\n" + "---------\n" + "event-hdr size: %zu B\n", + sizeof(event_hdr_t)); + + EM_DBG("\t\toffset\tsize\n" + "\t\t------\t----\n" + "esv.state_cnt:\t%3zu B\t%2zu B\n" + "esv.state:\t%3zu B\t%2zu B\n" + "flags:\t\t%3zu B\t%2zu B\n" + "align_offset:\t%3zu B\t%2zu B\n" + "event_type:\t%3zu B\t%2zu B\n" + "event:\t\t%3zu B\t%2zu B\n" + "event_size:\t%3zu B\t%2zu B\n" + "egrp_gen:\t%3zu B\t%2zu B\n" + "egrp:\t\t%3zu B\t%2zu B\n" + "tmo:\t\t%3zu B\t%2zu B\n" + "user_area info:\t%3zu B\t%2zu B\n" + "end_hdr_data:\t%3zu B\t%2zu B\n" + " \t\t%3zu B\n" + "end:\t\t%3zu B\t%2zu B\n", + offsetof(event_hdr_t, state_cnt), sizeof_field(event_hdr_t, state_cnt), + offsetof(event_hdr_t, state), sizeof_field(event_hdr_t, state), + offsetof(event_hdr_t, flags), sizeof_field(event_hdr_t, flags), + offsetof(event_hdr_t, align_offset), sizeof_field(event_hdr_t, align_offset), + offsetof(event_hdr_t, event_type), sizeof_field(event_hdr_t, event_type), + offsetof(event_hdr_t, event), sizeof_field(event_hdr_t, event), + offsetof(event_hdr_t, event_size), sizeof_field(event_hdr_t, event_size), + offsetof(event_hdr_t, egrp_gen), sizeof_field(event_hdr_t, egrp_gen), + offsetof(event_hdr_t, egrp), sizeof_field(event_hdr_t, egrp), + offsetof(event_hdr_t, tmo), sizeof_field(event_hdr_t, tmo), + offsetof(event_hdr_t, user_area), sizeof_field(event_hdr_t, user_area), + offsetof(event_hdr_t, end_hdr_data), sizeof_field(event_hdr_t, end_hdr_data), + offsetof(event_hdr_t, end) - offsetof(event_hdr_t, end_hdr_data), + offsetof(event_hdr_t, end), sizeof_field(event_hdr_t, end)); + + EM_PRINT("\n"); +} + +/** + * Helper for em_event_clone(). + * + * Clone an event originating from an external odp pkt-pool. + * Initialize the new cloned event as an EM event and return it. + * + * Alloc and copy content via ODP. + * Also the ev_hdr in the odp-pkt user_area is copied. + */ +em_event_t pkt_clone_odp(odp_packet_t pkt, odp_pool_t pkt_pool, + uint32_t offset, uint32_t size, + bool clone_uarea, bool is_clone_part) +{ + odp_packet_t clone_pkt; + + if (is_clone_part) { + /* only data is copied, ODP-uarea isn't */ + clone_pkt = odp_packet_copy_part(pkt, offset, size, pkt_pool); + if (unlikely(clone_pkt == ODP_PACKET_INVALID)) + return EM_EVENT_UNDEF; + + const void *src_odp_uarea = odp_packet_user_area(pkt); + void *dst_odp_uarea = odp_packet_user_area(clone_pkt); + size_t cpy_size = sizeof(event_hdr_t); + + if (clone_uarea) { + /* copy ODP-uarea (EM-hdr + EM-uarea) */ + uint32_t src_uarea_size = odp_packet_user_area_size(pkt); + uint32_t dst_uarea_size = odp_packet_user_area_size(clone_pkt); + + if (unlikely(dst_uarea_size < src_uarea_size)) { + odp_packet_free(clone_pkt); + return EM_EVENT_UNDEF; + } + /* update 'cpy_size' to include the whole ODP-uarea (EM-hdr + EM-uarea) */ + cpy_size = src_uarea_size; + } + /* copy the EM-hdr and possibly also the EM-uarea if requested */ + memcpy(dst_odp_uarea, src_odp_uarea, cpy_size); + } else { + /* identical clone, also ODP-uarea (EM-hdr + EM-uarea) is copied */ + clone_pkt = odp_packet_copy(pkt, pkt_pool); + if (unlikely(clone_pkt == ODP_PACKET_INVALID)) + return EM_EVENT_UNDEF; + } + + odp_packet_user_flag_set(clone_pkt, USER_FLAG_SET); + + odp_event_t odp_clone_event = odp_packet_to_event(clone_pkt); + event_hdr_t *clone_hdr = odp_packet_user_area(clone_pkt); + em_event_t clone_event = event_odp2em(odp_clone_event); + + /* + * Init hdr of event, also ESV init if needed. + * The clone_hdr is a copy of parent's, update only relevant fields. + */ + if (esv_enabled()) + clone_event = evstate_init(clone_event, clone_hdr, false); + else + clone_hdr->event = clone_event; + + clone_hdr->flags.all = 0; + clone_hdr->egrp = EM_EVENT_GROUP_UNDEF; + /* other fields: use parent's values as is */ + + return clone_event; +} + +void +output_queue_track(queue_elem_t *const output_q_elem) +{ + output_queue_track_t *const track = + &em_locm.output_queue_track; + const uint32_t qidx = output_q_elem->output.idx; + + if (track->used_queues[qidx] == NULL) { + track->used_queues[qidx] = output_q_elem; + track->idx[track->idx_cnt++] = qidx; + } +} + +void +output_queue_drain(const queue_elem_t *output_q_elem) +{ + const em_queue_t output_queue = (em_queue_t)(uintptr_t)output_q_elem->queue; + const em_output_func_t output_fn = + output_q_elem->output.output_conf.output_fn; + void *const output_fn_args = + output_q_elem->output.output_conf.output_fn_args; + + const int deq_max = 32; + + em_event_t output_ev_tbl[deq_max]; + /* use same event-tbl, dequeue odp events into the EM event-tbl */ + odp_event_t *const odp_deq_events = (odp_event_t *)output_ev_tbl; + + const odp_queue_t odp_queue = output_q_elem->odp_queue; + unsigned int output_num; + int deq; + int ret; + + const bool esv_ena = esv_enabled(); + + do { + deq = odp_queue_deq_multi(odp_queue, + odp_deq_events/*out=output_ev_tbl[]*/, + deq_max); + if (unlikely(deq <= 0)) + return; + + output_num = (unsigned int)deq; + /* odp_deq_events[] == output_ev_tbl[] */ + if (esv_ena) { + event_hdr_t *ev_hdrs[output_num]; + + /* Restore hdls from ev_hdrs, odp-ev conv lost evgen */ + event_to_hdr_multi(output_ev_tbl, ev_hdrs, output_num); + for (unsigned int i = 0; i < output_num; i++) + output_ev_tbl[i] = ev_hdrs[i]->event; + } + + ret = output_fn(output_ev_tbl, output_num, + output_queue, output_fn_args); + + if (unlikely((unsigned int)ret != output_num)) + em_free_multi(&output_ev_tbl[ret], output_num - ret); + } while (deq > 0); +} + +void +output_queue_buffering_drain(void) +{ + output_queue_track_t *const track = &em_locm.output_queue_track; + + for (unsigned int i = 0; i < track->idx_cnt; i++) { + int qidx = track->idx[i]; + queue_elem_t *output_q_elem = track->used_queues[qidx]; + env_spinlock_t *lock = &output_q_elem->output.lock; + + /* + * drain if lock available, otherwise another core is already + * draining so no need to do anything. + */ + if (env_spinlock_trylock(lock)) { + output_queue_drain(output_q_elem); + env_spinlock_unlock(lock); + } + + track->idx[i] = 0; + track->used_queues[qidx] = NULL; + } + track->idx_cnt = 0; +} + +uint32_t event_vector_tbl(em_event_t vector_event, + em_event_t **event_tbl /*out*/) +{ + odp_event_t odp_event = event_em2odp(vector_event); + odp_packet_vector_t pkt_vec = odp_packet_vector_from_event(odp_event); + odp_packet_t *pkt_tbl = NULL; + const int pkts = odp_packet_vector_tbl(pkt_vec, &pkt_tbl/*out*/); + + *event_tbl = (em_event_t *)pkt_tbl; /* Careful! Points to same table */ + + if (!pkts) + return 0; + + event_hdr_t *ev_hdr_tbl[pkts]; + + /* + * Init the event-table as needed, might contain EM events or + * ODP packets depending on source. + */ + if (esv_enabled()) { + odp_packet_t odp_pkttbl[pkts]; + + /* + * Drop ESV generation from event handles by converting to + * odp-packets, then init as needed as EM events. + */ + events_em2pkt(*event_tbl/*in*/, odp_pkttbl/*out*/, pkts); + + event_init_pkt_multi(odp_pkttbl /*in*/, *event_tbl /*in,out*/, + ev_hdr_tbl /*out*/, pkts, false); + } else { + event_init_pkt_multi(pkt_tbl /*in*/, *event_tbl /*in,out*/, + ev_hdr_tbl /*out*/, pkts, false); + } + + return pkts; +} + +em_status_t event_vector_max_size(em_event_t vector_event, uint32_t *max_size /*out*/, + em_escope_t escope) +{ + odp_event_t odp_event = event_em2odp(vector_event); + odp_packet_vector_t pktvec = odp_packet_vector_from_event(odp_event); + odp_pool_t odp_pool = odp_packet_vector_pool(pktvec); + pool_subpool_t pool_subpool = pool_subpool_odp2em(odp_pool); + em_pool_t pool = (em_pool_t)(uintptr_t)pool_subpool.pool; + int subpool = pool_subpool.subpool; + + if (unlikely(pool == EM_POOL_UNDEF)) { + /* + * Don't report an error if 'pool == EM_POOL_UNDEF' since that + * might happen if the vector is input from pktio that is using + * external (to EM) odp vector pools. + */ + *max_size = 0; + return EM_OK; /* EM does not have the max_size info */ + } + + const mpool_elem_t *pool_elem = pool_elem_get(pool); + + if (unlikely(!pool_elem || + (EM_CHECK_LEVEL > 2 && !pool_allocated(pool_elem)))) { + *max_size = 0; + return INTERNAL_ERROR(EM_ERR_BAD_STATE, escope, + "Invalid pool:%" PRI_POOL "", pool); + } + + if (unlikely(subpool >= pool_elem->num_subpools)) { + /* not found */ + *max_size = 0; + return INTERNAL_ERROR(EM_ERR_NOT_FOUND, escope, + "Subpool not found, pool:%" PRI_POOL "", pool); + } + + /* subpool index found, store corresponding size */ + *max_size = pool_elem->size[subpool]; + + return EM_OK; +} diff --git a/src/em_event.h b/src/em_event.h old mode 100644 new mode 100755 index 581ac6f1..9c10e3c2 --- a/src/em_event.h +++ b/src/em_event.h @@ -1,1537 +1,1538 @@ -/* - * Copyright (c) 2015-2023, Nokia Solutions and Networks - * All rights reserved. - * - * Redistribution and use in source and binary forms, with or without - * modification, are permitted provided that the following conditions - * are met: - * - * * Redistributions of source code must retain the above copyright - * notice, this list of conditions and the following disclaimer. - * * Redistributions in binary form must reproduce the above copyright - * notice, this list of conditions and the following disclaimer in the - * documentation and/or other materials provided with the distribution. - * * Neither the name of the copyright holder nor the names of its - * contributors may be used to endorse or promote products derived - * from this software without specific prior written permission. - * - * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS - * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT - * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR - * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT - * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, - * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT - * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, - * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY - * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT - * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE - * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. - */ - - /** - * @file - * EM internal event functions - * - */ - -#ifndef EM_EVENT_H_ -#define EM_EVENT_H_ - -#ifdef __cplusplus -extern "C" { -#endif - -#ifndef __clang__ -COMPILE_TIME_ASSERT((uintptr_t)EM_EVENT_UNDEF == (uintptr_t)ODP_EVENT_INVALID, - EM_EVENT_NOT_EQUAL_TO_ODP_EVENT); -COMPILE_TIME_ASSERT(EM_TMO_TYPE_NONE == 0, - "EM_TMO_TYPE_NONE must be 0"); -#endif - -em_status_t event_init(void); -void print_event_info(void); -em_event_t pkt_clone_odp(odp_packet_t pkt, odp_pool_t pkt_pool, - uint32_t offset, uint32_t size, bool is_clone_part); -void output_queue_track(queue_elem_t *const output_q_elem); -void output_queue_drain(const queue_elem_t *output_q_elem); -void output_queue_buffering_drain(void); - -uint32_t event_vector_tbl(em_event_t vector_event, em_event_t **event_tbl/*out*/); -em_status_t event_vector_max_size(em_event_t vector_event, uint32_t *max_size /*out*/, - em_escope_t escope); - -/** - * Initialize the event header of a packet allocated outside of EM. - */ -static inline em_event_t -evhdr_init_pkt(event_hdr_t *ev_hdr, em_event_t event, - odp_packet_t odp_pkt, bool is_extev) -{ - const int user_flag_set = odp_packet_user_flag(odp_pkt); - const bool esv_ena = esv_enabled(); - - if (user_flag_set) { - /* Event already initialized by EM */ - if (esv_ena) { - event = ev_hdr->event; - if (is_extev) - event = evstate_em2usr(event, ev_hdr, EVSTATE__DISPATCH); - } - - return event; - } - - /* - * ODP pkt from outside of EM - not allocated by EM & needs init - */ - odp_packet_user_flag_set(odp_pkt, USER_FLAG_SET); - ev_hdr->event_type = EM_EVENT_TYPE_PACKET; - ev_hdr->egrp = EM_EVENT_GROUP_UNDEF; - ev_hdr->user_area.all = 0; /* uarea fields init when used */ - - if (!esv_ena) { - ev_hdr->flags.all = 0; - ev_hdr->event = event; - return event; - } - - /* - * ESV enabled: - */ - if (!em_shm->opt.esv.prealloc_pools || ev_hdr->flags.refs_used) { - /* No prealloc OR pkt was a ref before being freed into the pool */ - event = evstate_init(event, ev_hdr, is_extev); - } else { - /* esv.prealloc_pools == true: */ - odp_pool_t odp_pool = odp_packet_pool(odp_pkt); - em_pool_t pool = pool_odp2em(odp_pool); - - if (pool == EM_POOL_UNDEF) { - /* External odp pkt originates from an ODP-pool */ - event = evstate_init(event, ev_hdr, is_extev); - } else { - /* External odp pkt originates from an EM-pool */ - event = evstate_update(event, ev_hdr, is_extev); - } - } - ev_hdr->flags.all = 0; /* clear only after evstate_...() */ - - return event; -} - -/** - * Initialize the event headers of packets allocated outside of EM. - */ -static inline void -evhdr_init_pkt_multi(event_hdr_t *const ev_hdrs[], - em_event_t events[/*in,out*/], - const odp_packet_t odp_pkts[/*in*/], - const int num, bool is_extev) -{ - const bool esv_ena = esv_enabled(); - int user_flag_set; - - int needs_init_idx[num]; - int needs_init_num = 0; - int idx; - - for (int i = 0; i < num; i++) { - user_flag_set = odp_packet_user_flag(odp_pkts[i]); - if (user_flag_set) { - /* Event already initialized by EM */ - if (esv_ena) { - events[i] = ev_hdrs[i]->event; - if (is_extev) - events[i] = evstate_em2usr(events[i], ev_hdrs[i], - EVSTATE__DISPATCH_MULTI); - } - /* else events[i] = events[i] */ - } else { - odp_packet_user_flag_set(odp_pkts[i], USER_FLAG_SET); - needs_init_idx[needs_init_num] = i; - needs_init_num++; - } - } - - if (needs_init_num == 0) - return; - - /* - * ODP pkt from outside of EM - not allocated by EM & needs init - */ - - if (!esv_ena) { - for (int i = 0; i < needs_init_num; i++) { - idx = needs_init_idx[i]; - ev_hdrs[idx]->flags.all = 0; - ev_hdrs[idx]->event_type = EM_EVENT_TYPE_PACKET; - ev_hdrs[idx]->event = events[idx]; - ev_hdrs[idx]->egrp = EM_EVENT_GROUP_UNDEF; - ev_hdrs[idx]->user_area.all = 0; /* uarea fields init when used */ - } - - return; - } - - /* - * ESV enabled: - */ - if (!em_shm->opt.esv.prealloc_pools) { - for (int i = 0; i < needs_init_num; i++) { - idx = needs_init_idx[i]; - events[idx] = evstate_init(events[idx], ev_hdrs[idx], is_extev); - } - } else { - /* em_shm->opt.esv.prealloc_pools == true */ - for (int i = 0; i < needs_init_num; i++) { - idx = needs_init_idx[i]; - - odp_pool_t odp_pool = odp_packet_pool(odp_pkts[idx]); - em_pool_t pool = pool_odp2em(odp_pool); - - if (pool == EM_POOL_UNDEF || ev_hdrs[idx]->flags.refs_used) { - /* - * External odp pkt originates from an ODP-pool, - * or pkt was a ref before being freed into the pool. - */ - events[idx] = evstate_init(events[idx], ev_hdrs[idx], is_extev); - } else { - /* External odp pkt originates from an EM-pool */ - events[idx] = evstate_update(events[idx], ev_hdrs[idx], is_extev); - } - } - } - - for (int i = 0; i < needs_init_num; i++) { - idx = needs_init_idx[i]; - ev_hdrs[idx]->flags.all = 0; /* clear only after evstate_...() */ - ev_hdrs[idx]->event_type = EM_EVENT_TYPE_PACKET; - ev_hdrs[idx]->egrp = EM_EVENT_GROUP_UNDEF; - ev_hdrs[idx]->user_area.all = 0; /* uarea fields init when used */ - } -} - -/** - * Initialize the event header of a packet vector allocated outside of EM. - */ -static inline em_event_t -evhdr_init_pktvec(event_hdr_t *ev_hdr, em_event_t event, - odp_packet_vector_t odp_pktvec, bool is_extev) -{ - const int user_flag = odp_packet_vector_user_flag(odp_pktvec); - const bool esv_ena = esv_enabled(); - - if (user_flag == USER_FLAG_SET) { - /* Event already initialized by EM */ - if (esv_ena) { - event = ev_hdr->event; - if (is_extev) - event = evstate_em2usr(event, ev_hdr, EVSTATE__DISPATCH); - } - - return event; - } - - /* - * ODP pkt from outside of EM - not allocated by EM & needs init - */ - odp_packet_vector_user_flag_set(odp_pktvec, USER_FLAG_SET); - /* can clear flags before evstate_...(), flags.refs_used not set for vecs */ - ev_hdr->flags.all = 0; - ev_hdr->event_type = EM_EVENT_TYPE_VECTOR; - ev_hdr->egrp = EM_EVENT_GROUP_UNDEF; - ev_hdr->user_area.all = 0; /* uarea fields init when used */ - - if (!esv_ena) { - ev_hdr->event = event; - return event; - } - - /* - * ESV enabled: - */ - if (!em_shm->opt.esv.prealloc_pools) { - event = evstate_init(event, ev_hdr, is_extev); - } else { - /* esv.prealloc_pools == true: */ - odp_pool_t odp_pool = odp_packet_vector_pool(odp_pktvec); - em_pool_t pool = pool_odp2em(odp_pool); - - if (pool == EM_POOL_UNDEF) { - /* External odp pkt originates from an ODP-pool */ - event = evstate_init(event, ev_hdr, is_extev); - } else { - /* External odp pkt originates from an EM-pool */ - event = evstate_update(event, ev_hdr, is_extev); - if (is_extev) - return event; - } - } - - return event; -} - -/** - * Initialize the event headers of packet vectors allocated outside of EM. - */ -static inline void -evhdr_init_pktvec_multi(event_hdr_t *ev_hdrs[/*out*/], - em_event_t events[/*in,out*/], - const odp_packet_vector_t odp_pktvecs[/*in*/], - const int num, bool is_extev) -{ - const bool esv_ena = esv_enabled(); - - int needs_init_idx[num]; - int needs_init_num = 0; - int idx; - - for (int i = 0; i < num; i++) { - int user_flag = odp_packet_vector_user_flag(odp_pktvecs[i]); - - if (user_flag == USER_FLAG_SET) { - /* Event already initialized by EM */ - if (esv_ena) { - events[i] = ev_hdrs[i]->event; - if (is_extev) - events[i] = evstate_em2usr(events[i], ev_hdrs[i], - EVSTATE__DISPATCH_MULTI); - } - /* else events[i] = events[i] */ - } else { - odp_packet_vector_user_flag_set(odp_pktvecs[i], USER_FLAG_SET); - needs_init_idx[needs_init_num] = i; - needs_init_num++; - } - } - - if (needs_init_num == 0) - return; - - /* - * ODP pkt vector from outside of EM - not allocated by EM & needs init - */ - - if (!esv_ena) { - for (int i = 0; i < needs_init_num; i++) { - idx = needs_init_idx[i]; - ev_hdrs[idx]->flags.all = 0; - ev_hdrs[idx]->event_type = EM_EVENT_TYPE_VECTOR; - ev_hdrs[idx]->event = events[idx]; - ev_hdrs[idx]->egrp = EM_EVENT_GROUP_UNDEF; - ev_hdrs[idx]->user_area.all = 0; /* uarea fields init when used */ - } - - return; - } - - /* - * ESV enabled: - */ - for (int i = 0; i < needs_init_num; i++) { - idx = needs_init_idx[i]; - /* can clear flags before evstate_...(), flags.refs_used not set for vecs */ - ev_hdrs[idx]->flags.all = 0; - ev_hdrs[idx]->event_type = EM_EVENT_TYPE_VECTOR; - ev_hdrs[idx]->egrp = EM_EVENT_GROUP_UNDEF; - ev_hdrs[idx]->user_area.all = 0; /* uarea fields init when used */ - } - - if (!em_shm->opt.esv.prealloc_pools) { - for (int i = 0; i < needs_init_num; i++) { - idx = needs_init_idx[i]; - events[idx] = evstate_init(events[idx], ev_hdrs[idx], is_extev); - } - - return; - } - - /* - * em_shm->opt.esv.prealloc_pools == true - */ - for (int i = 0; i < needs_init_num; i++) { - idx = needs_init_idx[i]; - - odp_pool_t odp_pool = odp_packet_vector_pool(odp_pktvecs[idx]); - em_pool_t pool = pool_odp2em(odp_pool); - - if (pool == EM_POOL_UNDEF) { - /* External odp pkt originates from an ODP-pool */ - events[idx] = evstate_init(events[idx], ev_hdrs[idx], is_extev); - } else { - /* External odp pkt originates from an EM-pool */ - events[idx] = evstate_update(events[idx], ev_hdrs[idx], is_extev); - } - } -} - -/** - * Initialize an external ODP event that have been input into EM. - * - * Initialize the event header if needed, i.e. if event originated from outside - * of EM from pktio or other input and was not allocated by EM via em_alloc(). - * The odp pkt-user-ptr is used to determine whether the header has been - * initialized or not. - */ -static inline em_event_t -event_init_odp(odp_event_t odp_event, bool is_extev, event_hdr_t **ev_hdr__out) -{ - const odp_event_type_t odp_type = odp_event_type(odp_event); - em_event_t event = event_odp2em(odp_event); /* return value, updated by ESV */ - - switch (odp_type) { - case ODP_EVENT_PACKET: { - odp_packet_t odp_pkt = odp_packet_from_event(odp_event); - event_hdr_t *ev_hdr = odp_packet_user_area(odp_pkt); - - /* init event-hdr if needed (also ESV-state if used) */ - event = evhdr_init_pkt(ev_hdr, event, odp_pkt, is_extev); - if (ev_hdr__out) - *ev_hdr__out = ev_hdr; - return event; - } - case ODP_EVENT_BUFFER: { - const bool esv_ena = esv_enabled(); - - if (!ev_hdr__out && !esv_ena) - return event; - - odp_buffer_t odp_buf = odp_buffer_from_event(odp_event); - event_hdr_t *ev_hdr = odp_buffer_user_area(odp_buf); - - if (esv_ena) { /* update event handle (ESV) */ - event = ev_hdr->event; - if (is_extev) - event = evstate_em2usr(event, ev_hdr, EVSTATE__DISPATCH); - } - if (ev_hdr__out) - *ev_hdr__out = ev_hdr; - return event; - } - case ODP_EVENT_PACKET_VECTOR: { - odp_packet_vector_t odp_pktvec = odp_packet_vector_from_event(odp_event); - event_hdr_t *ev_hdr = odp_packet_vector_user_area(odp_pktvec); - - /* init event-hdr if needed (also ESV-state if used) */ - event = evhdr_init_pktvec(ev_hdr, event, odp_pktvec, is_extev); - if (ev_hdr__out) - *ev_hdr__out = ev_hdr; - return event; - } - case ODP_EVENT_TIMEOUT: { - odp_timeout_t odp_tmo = odp_timeout_from_event(odp_event); - event_hdr_t *ev_hdr = odp_timeout_user_area(odp_tmo); - const bool esv_ena = esv_enabled(); - - if (esv_ena) { - /* - * Update event handle, no other ESV checks done. - * Some timers might send a copy of the original event - * in tear-down, thus keep ptr but update evgen. - */ - evhdl_t evhdl = {.event = event}; /* .evptr from here */ - evhdl_t evhdr_hdl = {.event = ev_hdr->event}; /* .evgen from here */ - - evhdl.evgen = evhdr_hdl.evgen; /* update .evgen */ - ev_hdr->event = evhdl.event; /* store updated hdl in hdr */ - event = evhdl.event; /* return updated event */ - } - - if (ev_hdr__out) - *ev_hdr__out = ev_hdr; - return event; - } - default: - INTERNAL_ERROR(EM_FATAL(EM_ERR_NOT_IMPLEMENTED), - EM_ESCOPE_EVENT_INIT_ODP, - "Unexpected odp event type:%u", odp_type); - __builtin_unreachable(); - /* never reached */ - return EM_EVENT_UNDEF; - } -} - -/* Helper to event_init_odp_multi() */ -static inline void -event_init_pkt_multi(const odp_packet_t odp_pkts[/*in*/], - em_event_t events[/*in,out*/], event_hdr_t *ev_hdrs[/*out*/], - const int num, bool is_extev) -{ - for (int i = 0; i < num; i++) - ev_hdrs[i] = odp_packet_user_area(odp_pkts[i]); - - evhdr_init_pkt_multi(ev_hdrs, events, odp_pkts, num, is_extev); -} - -/* Helper to event_init_odp_multi() */ -static inline void -event_init_buf_multi(const odp_buffer_t odp_bufs[/*in*/], - em_event_t events[/*in,out*/], event_hdr_t *ev_hdrs[/*out*/], - const int num, bool is_extev) -{ - for (int i = 0; i < num; i++) - ev_hdrs[i] = odp_buffer_user_area(odp_bufs[i]); - - if (esv_enabled()) { - /* update event handle (ESV) */ - for (int i = 0; i < num; i++) - events[i] = ev_hdrs[i]->event; - - if (is_extev) - evstate_em2usr_multi(events, ev_hdrs, num, - EVSTATE__DISPATCH_MULTI); - } -} - -/* Helper to event_init_odp_multi() */ -static inline void -event_init_tmo_multi(const odp_timeout_t odp_tmos[/*in*/], - em_event_t events[/*in,out*/], event_hdr_t *ev_hdrs[/*out*/], - const int num) -{ - for (int i = 0; i < num; i++) - ev_hdrs[i] = odp_timeout_user_area(odp_tmos[i]); - - /* ignore ESV */ - (void)events; -} - -/* Helper to event_init_odp_multi() */ -static inline void -event_init_pktvec_multi(const odp_packet_vector_t odp_pktvecs[/*in*/], - em_event_t events[/*in,out*/], event_hdr_t *ev_hdrs[/*out*/], - const int num, bool is_extev) -{ - for (int i = 0; i < num; i++) - ev_hdrs[i] = odp_packet_vector_user_area(odp_pktvecs[i]); - - evhdr_init_pktvec_multi(ev_hdrs, events, odp_pktvecs, num, is_extev); -} - -/** - * Convert from EM events to event headers and initialize the headers as needed. - * - * Initialize the event header if needed, i.e. if event originated from outside - * of EM from pktio or other input and was not allocated by EM via em_alloc(). - * The odp pkt-user-ptr is used to determine whether the header has been - * initialized or not. - */ -static inline void -event_init_odp_multi(const odp_event_t odp_events[/*in*/], - em_event_t events[/*out*/], event_hdr_t *ev_hdrs[/*out*/], - const int num, bool is_extev) -{ - for (int i = 0; i < num; i++) - events[i] = event_init_odp(odp_events[i], is_extev, &ev_hdrs[i]); -} - -/** - * Allocate an event based on an odp-buf. - */ -static inline event_hdr_t * -event_alloc_buf(const mpool_elem_t *const pool_elem, uint32_t size) -{ - odp_buffer_t odp_buf = ODP_BUFFER_INVALID; - int subpool; - - /* - * Allocate from the 'best fit' subpool, or if that is full, from the - * next subpool that has buffers available of a bigger size. - */ - subpool = pool_find_subpool(pool_elem, size); - if (unlikely(subpool < 0)) - return NULL; - - for (; subpool < pool_elem->num_subpools; subpool++) { - odp_pool_t odp_pool = pool_elem->odp_pool[subpool]; - - if (EM_CHECK_LEVEL >= 3 && - unlikely(odp_pool == ODP_POOL_INVALID)) - return NULL; - - odp_buf = odp_buffer_alloc(odp_pool); - if (likely(odp_buf != ODP_BUFFER_INVALID)) - break; - } - - if (unlikely(odp_buf == ODP_BUFFER_INVALID)) - return NULL; - - /* - * odp buffer now allocated - init the EM event header - * in the odp user area. - */ - event_hdr_t *const ev_hdr = odp_buffer_user_area(odp_buf); - odp_event_t odp_event = odp_buffer_to_event(odp_buf); - em_event_t event = event_odp2em(odp_event); - - ev_hdr->event = event; /* store this event handle */ - ev_hdr->align_offset = pool_elem->align_offset; - - /* init common ev_hdr fields in the caller */ - - return ev_hdr; -} - -/** - * Allocate & initialize multiple events based on odp-bufs. - */ -static inline int -event_alloc_buf_multi(em_event_t events[/*out*/], const int num, - const mpool_elem_t *pool_elem, uint32_t size, - em_event_type_t type) -{ - odp_buffer_t odp_bufs[num]; - odp_event_t odp_event; - event_hdr_t *ev_hdrs[num]; - int subpool; - const bool esv_ena = esv_enabled(); - - /* - * Allocate from the 'best fit' subpool, or if that is full, from the - * next subpool that has buffers available of a bigger size. - */ - subpool = pool_find_subpool(pool_elem, size); - if (unlikely(subpool < 0)) - return 0; - - int num_req = num; - int num_bufs = 0; - int i; - - for (; subpool < pool_elem->num_subpools; subpool++) { - odp_pool_t odp_pool = pool_elem->odp_pool[subpool]; - - if (EM_CHECK_LEVEL >= 3 && - unlikely(odp_pool == ODP_POOL_INVALID)) - return 0; - - int ret = odp_buffer_alloc_multi(odp_pool, &odp_bufs[num_bufs], - num_req); - if (unlikely(ret <= 0)) - continue; /* try next subpool */ - - /* store the allocated events[] */ - for (i = num_bufs; i < num_bufs + ret; i++) { - odp_event = odp_buffer_to_event(odp_bufs[i]); - events[i] = event_odp2em(odp_event); - } - - /* Init 'ret' ev-hdrs from this 'subpool'=='odp-pool' */ - for (i = num_bufs; i < num_bufs + ret; i++) - ev_hdrs[i] = odp_buffer_user_area(odp_bufs[i]); - - if (esv_ena) { - /* reads ev_hdrs[i]->flags if prealloc_pools used */ - evstate_alloc_multi(&events[num_bufs] /*in/out*/, - &ev_hdrs[num_bufs], ret); - } - - for (i = num_bufs; i < num_bufs + ret; i++) { - ev_hdrs[i]->flags.all = 0; - ev_hdrs[i]->event_type = type; - if (!esv_ena) - ev_hdrs[i]->event = events[i]; - ev_hdrs[i]->event_size = size; - ev_hdrs[i]->egrp = EM_EVENT_GROUP_UNDEF; - - ev_hdrs[i]->user_area.all = 0; - ev_hdrs[i]->user_area.size = pool_elem->user_area.size; - ev_hdrs[i]->user_area.isinit = 1; - - ev_hdrs[i]->align_offset = pool_elem->align_offset; - } - - num_bufs += ret; - if (likely(num_bufs == num)) - break; /* all allocated */ - num_req -= ret; - } - - return num_bufs; /* number of allocated bufs (0 ... num) */ -} - -/** - * Allocate & initialize an event based on an odp-pkt. - */ -static inline event_hdr_t * -event_alloc_pkt(const mpool_elem_t *pool_elem, uint32_t size) -{ - const uint32_t push_len = pool_elem->align_offset; - uint32_t pull_len; - uint32_t alloc_size; - odp_packet_t odp_pkt = ODP_PACKET_INVALID; - int subpool; - - if (size > push_len) { - alloc_size = size - push_len; - pull_len = 0; - } else { - alloc_size = 1; /* min allowed */ - pull_len = push_len + 1 - size; - } - - /* - * Allocate from the 'best fit' subpool, or if that is full, from the - * next subpool that has pkts available of a bigger size. - */ - subpool = pool_find_subpool(pool_elem, size); - if (unlikely(subpool < 0)) - return NULL; - - for (; subpool < pool_elem->num_subpools; subpool++) { - odp_pool_t odp_pool = pool_elem->odp_pool[subpool]; - - if (EM_CHECK_LEVEL >= 3 && - unlikely(odp_pool == ODP_POOL_INVALID)) - return NULL; - - odp_pkt = odp_packet_alloc(odp_pool, alloc_size); - if (likely(odp_pkt != ODP_PACKET_INVALID)) - break; - } - - if (unlikely(odp_pkt == ODP_PACKET_INVALID)) - return NULL; - - /* - * odp packet now allocated - adjust the payload start address and - * init the EM event header in the odp-pkt user-area - */ - - /* Adjust event payload start-address based on alignment config */ - const void *ptr; - - if (push_len) { - ptr = odp_packet_push_head(odp_pkt, push_len); - if (EM_CHECK_LEVEL >= 3 && unlikely(!ptr)) - goto err_pktalloc; - } - if (pull_len) { - ptr = odp_packet_pull_tail(odp_pkt, pull_len); - if (EM_CHECK_LEVEL >= 3 && unlikely(!ptr)) - goto err_pktalloc; - } - - /* - * Set the pkt user ptr to be able to recognize pkt-events that - * EM has created vs pkts from pkt-input that needs their - * ev-hdrs to be initialized. - */ - odp_packet_user_flag_set(odp_pkt, USER_FLAG_SET); - - event_hdr_t *const ev_hdr = odp_packet_user_area(odp_pkt); - odp_event_t odp_event = odp_packet_to_event(odp_pkt); - em_event_t event = event_odp2em(odp_event); - - if (EM_CHECK_LEVEL >= 3 && unlikely(ev_hdr == NULL)) - goto err_pktalloc; - - /* store this event handle */ - ev_hdr->event = event; - - /* init common ev_hdr fields in the caller */ - - return ev_hdr; - -err_pktalloc: - odp_packet_free(odp_pkt); - return NULL; -} - -/* - * Helper for event_alloc_pkt_multi() - */ -static inline int -pktalloc_multi(odp_packet_t odp_pkts[/*out*/], int num, - odp_pool_t odp_pool, uint32_t size, - uint32_t push_len, uint32_t pull_len) -{ - int ret = odp_packet_alloc_multi(odp_pool, size, odp_pkts, num); - - if (unlikely(ret <= 0)) - return 0; - - const int num_pkts = ret; /* return value > 0 */ - const void *ptr = NULL; - int i; - - /* Adjust payload start-address based on alignment config */ - if (push_len) { - for (i = 0; i < num_pkts; i++) { - ptr = odp_packet_push_head(odp_pkts[i], push_len); - if (EM_CHECK_LEVEL >= 3 && unlikely(!ptr)) - goto err_pktalloc_multi; - } - } - if (pull_len) { - for (i = 0; i < num_pkts; i++) { - ptr = odp_packet_pull_tail(odp_pkts[i], pull_len); - if (EM_CHECK_LEVEL >= 3 && unlikely(!ptr)) - goto err_pktalloc_multi; /* only before esv */ - } - } - - /* - * Set the pkt user ptr to be able to recognize pkt-events that - * EM has created vs pkts from pkt-input that needs their - * ev-hdrs to be initialized. - */ - for (i = 0; i < num_pkts; i++) - odp_packet_user_flag_set(odp_pkts[i], USER_FLAG_SET); - - return num_pkts; - -err_pktalloc_multi: - odp_packet_free_multi(odp_pkts, num_pkts); - return 0; -} - -/** - * Allocate & initialize multiple events based on odp-pkts. - */ -static inline int -event_alloc_pkt_multi(em_event_t events[/*out*/], const int num, - const mpool_elem_t *pool_elem, uint32_t size, - em_event_type_t type) -{ - const uint32_t push_len = pool_elem->align_offset; - uint32_t pull_len; - odp_packet_t odp_pkts[num]; - /* use same output-array: odp_events[] = events[] */ - odp_event_t *const odp_events = (odp_event_t *)events; - event_hdr_t *ev_hdrs[num]; - uint32_t alloc_size; - int subpool; - const bool esv_ena = esv_enabled(); - - if (size > push_len) { - alloc_size = size - push_len; - pull_len = 0; - } else { - alloc_size = 1; /* min allowed */ - pull_len = push_len + 1 - size; - } - - /* - * Allocate from the 'best fit' subpool, or if that is full, from the - * next subpool that has pkts available of a bigger size. - */ - subpool = pool_find_subpool(pool_elem, size); - if (unlikely(subpool < 0)) - return 0; - - int num_req = num; - int num_pkts = 0; - int i; - - for (; subpool < pool_elem->num_subpools; subpool++) { - odp_pool_t odp_pool = pool_elem->odp_pool[subpool]; - - if (EM_CHECK_LEVEL >= 3 && - unlikely(odp_pool == ODP_POOL_INVALID)) - return 0; - - int ret = pktalloc_multi(&odp_pkts[num_pkts], num_req, - odp_pool, alloc_size, - push_len, pull_len); - if (unlikely(ret <= 0)) - continue; /* try next subpool */ - - /* - * Init 'ret' ev-hdrs from this 'subpool'=='odp-pool'. - * Note: odp_events[] points&writes into events[out] - */ - odp_packet_to_event_multi(&odp_pkts[num_pkts], - &odp_events[num_pkts], ret); - - for (i = num_pkts; i < num_pkts + ret; i++) - ev_hdrs[i] = odp_packet_user_area(odp_pkts[i]); - - /* - * Note: events[] == odp_events[] before ESV init. - * Don't touch odp_events[] during this loop-round anymore. - */ - if (esv_ena) { - /* reads ev_hdrs[i]->flags if prealloc_pools used */ - evstate_alloc_multi(&events[num_pkts] /*in/out*/, - &ev_hdrs[num_pkts], ret); - } - - for (i = num_pkts; i < num_pkts + ret; i++) { - ev_hdrs[i]->flags.all = 0; - ev_hdrs[i]->event_type = type; - if (!esv_ena) - ev_hdrs[i]->event = events[i]; - ev_hdrs[i]->event_size = size; /* original size */ - ev_hdrs[i]->egrp = EM_EVENT_GROUP_UNDEF; - - ev_hdrs[i]->user_area.all = 0; - ev_hdrs[i]->user_area.size = pool_elem->user_area.size; - ev_hdrs[i]->user_area.isinit = 1; - /*ev_hdrs[i]->align_offset = needed by odp bufs only*/ - } - - num_pkts += ret; - if (likely(num_pkts == num)) - break; /* all allocated */ - num_req -= ret; - } - - return num_pkts; /* number of allocated pkts */ -} - -static inline event_hdr_t * -event_alloc_vector(const mpool_elem_t *pool_elem, uint32_t size) -{ - odp_packet_vector_t odp_pktvec = ODP_PACKET_VECTOR_INVALID; - int subpool; - - /* - * Allocate from the 'best fit' subpool, or if that is full, from the - * next subpool that has pkts available of a bigger size. - */ - subpool = pool_find_subpool(pool_elem, size); - if (unlikely(subpool < 0)) - return NULL; - - for (; subpool < pool_elem->num_subpools; subpool++) { - odp_pool_t odp_pool = pool_elem->odp_pool[subpool]; - - if (EM_CHECK_LEVEL >= 3 && - unlikely(odp_pool == ODP_POOL_INVALID)) - return NULL; - - odp_pktvec = odp_packet_vector_alloc(odp_pool); - if (likely(odp_pktvec != ODP_PACKET_VECTOR_INVALID)) - break; - } - - if (unlikely(odp_pktvec == ODP_PACKET_VECTOR_INVALID)) - return NULL; - - /* - * Packet vector now allocated: - * Init the EM event header in the odp-pkt-vector user-area. - */ - - /* - * Set the pktvec user flag to be able to recognize vectors that - * EM has created vs. vectors from pkt-input that needs their - * ev-hdrs to be initialized. - */ - odp_packet_vector_user_flag_set(odp_pktvec, USER_FLAG_SET); - - event_hdr_t *const ev_hdr = odp_packet_vector_user_area(odp_pktvec); - odp_event_t odp_event = odp_packet_vector_to_event(odp_pktvec); - em_event_t event = event_odp2em(odp_event); - - if (EM_CHECK_LEVEL >= 3 && unlikely(ev_hdr == NULL)) - goto err_vecalloc; - - ev_hdr->event = event; /* store this event handle */ - - /* init common ev_hdr fields in the caller */ - - return ev_hdr; - -err_vecalloc: - odp_packet_vector_free(odp_pktvec); - return NULL; -} - -/* - * Helper for event_alloc_vec_multi() - */ -static inline int -vecalloc_multi(odp_packet_vector_t odp_pktvecs[/*out*/], int num, - odp_pool_t odp_pool) -{ - int i; - - for (i = 0; i < num; i++) { - odp_pktvecs[i] = odp_packet_vector_alloc(odp_pool); - if (unlikely(odp_pktvecs[i] == ODP_PACKET_VECTOR_INVALID)) - break; - } - - const int num_vecs = i; - - if (unlikely(num_vecs == 0)) - return 0; - - /* - * Set the pkt vector user ptr to be able to recognize vector-events - * that EM has created vs vectors from pkt-input that needs their - * ev-hdrs to be initialized. - */ - for (i = 0; i < num_vecs; i++) - odp_packet_vector_user_flag_set(odp_pktvecs[i], USER_FLAG_SET); - - return num_vecs; -} - -/** - * Allocate & initialize multiple events based on odp-pkt-vectors. - */ -static inline int -event_alloc_vector_multi(em_event_t events[/*out*/], const int num, - const mpool_elem_t *pool_elem, uint32_t size, - em_event_type_t type) -{ - odp_packet_vector_t odp_pktvecs[num]; - /* use same output-array: odp_events[] = events[] */ - odp_event_t *const odp_events = (odp_event_t *)events; - event_hdr_t *ev_hdrs[num]; - int subpool; - const bool esv_ena = esv_enabled(); - - /* - * Allocate from the 'best fit' subpool, or if that is full, from the - * next subpool that has pkts available of a bigger size. - */ - subpool = pool_find_subpool(pool_elem, size); - if (unlikely(subpool < 0)) - return 0; - - int num_req = num; - int num_vecs = 0; - int i; - - for (; subpool < pool_elem->num_subpools; subpool++) { - odp_pool_t odp_pool = pool_elem->odp_pool[subpool]; - - if (EM_CHECK_LEVEL >= 3 && - unlikely(odp_pool == ODP_POOL_INVALID)) - return 0; - - int ret = vecalloc_multi(&odp_pktvecs[num_vecs], num_req, - odp_pool); - if (unlikely(ret <= 0)) - continue; /* try next subpool */ - - /* - * Init 'ret' ev-hdrs from this 'subpool'=='odp-pool'. - * Note: odp_events[] points&writes into events[out] - */ - for (i = num_vecs; i < num_vecs + ret; i++) { - odp_events[i] = odp_packet_vector_to_event(odp_pktvecs[i]); - ev_hdrs[i] = odp_packet_vector_user_area(odp_pktvecs[i]); - } - - /* - * Note: events[] == odp_events[] before ESV init. - * Don't touch odp_events[] during this loop-round anymore. - */ - if (esv_ena) { - /* reads ev_hdrs[i]->flags if prealloc_pools used */ - evstate_alloc_multi(&events[num_vecs] /*in/out*/, - &ev_hdrs[num_vecs], ret); - } - - for (i = num_vecs; i < num_vecs + ret; i++) { - ev_hdrs[i]->flags.all = 0; - ev_hdrs[i]->event_type = type; - if (!esv_ena) - ev_hdrs[i]->event = events[i]; - ev_hdrs[i]->event_size = size; /* original vec size */ - ev_hdrs[i]->egrp = EM_EVENT_GROUP_UNDEF; - - ev_hdrs[i]->user_area.all = 0; - ev_hdrs[i]->user_area.size = pool_elem->user_area.size; - ev_hdrs[i]->user_area.isinit = 1; - /*ev_hdrs[i]->align_offset = needed by odp bufs only*/ - } - - num_vecs += ret; - if (likely(num_vecs == num)) - break; /* all allocated */ - num_req -= ret; - } - - return num_vecs; /* number of allocated pkts */ -} - -/** - * Helper for em_alloc() and em_event_clone() - */ -static inline em_event_t -event_alloc(const mpool_elem_t *pool_elem, uint32_t size, em_event_type_t type, - const uint16_t api_op) -{ - /* - * EM event pools created with type=PKT can support: - * - SW events (bufs) - * - pkt events. - * - * EM event pools created with type=SW can support: - * - SW events (bufs) only - */ - event_hdr_t *ev_hdr = NULL; - - if (pool_elem->event_type == EM_EVENT_TYPE_PACKET) - ev_hdr = event_alloc_pkt(pool_elem, size); - else if (pool_elem->event_type == EM_EVENT_TYPE_SW) - ev_hdr = event_alloc_buf(pool_elem, size); - else if (pool_elem->event_type == EM_EVENT_TYPE_VECTOR) - ev_hdr = event_alloc_vector(pool_elem, size); - - if (unlikely(!ev_hdr)) - return EM_EVENT_UNDEF; - - /* - * event now allocated: - * ev_hdr->event = stored by event_alloc_pkt/buf/vector() - */ - /* Update event ESV state for alloc */ - if (esv_enabled()) - (void)evstate_alloc(ev_hdr->event, ev_hdr, api_op); - - ev_hdr->flags.all = 0; /* clear only after evstate_alloc() */ - ev_hdr->event_type = type; /* store the event type */ - ev_hdr->event_size = size; /* store requested size */ - ev_hdr->egrp = EM_EVENT_GROUP_UNDEF; - - ev_hdr->user_area.all = 0; - ev_hdr->user_area.size = pool_elem->user_area.size; - ev_hdr->user_area.isinit = 1; - /* ev_hdr->align_offset = init by event_alloc_buf() when needed */ - - return ev_hdr->event; -} - -/** - * Start-up helper for pool preallocation - */ -static inline event_prealloc_hdr_t * -event_prealloc(const mpool_elem_t *pool_elem, uint32_t size) -{ - /* - * EM event pools created with type=PKT can support: - * - SW events (bufs) - * - pkt events. - * - * EM event pools created with type=SW can support: - * - SW events (bufs) only - */ - event_hdr_t *ev_hdr = NULL; - - if (pool_elem->event_type == EM_EVENT_TYPE_PACKET) - ev_hdr = event_alloc_pkt(pool_elem, size); - else if (pool_elem->event_type == EM_EVENT_TYPE_SW) - ev_hdr = event_alloc_buf(pool_elem, size); - else if (pool_elem->event_type == EM_EVENT_TYPE_VECTOR) - ev_hdr = event_alloc_vector(pool_elem, size); - - if (unlikely(ev_hdr == NULL)) - return NULL; - - /* event now allocated */ - - if (esv_enabled()) { - em_event_t event = ev_hdr->event; - - (void)evstate_prealloc(event, ev_hdr); - } - ev_hdr->flags.all = 0; /* clear only after evstate_alloc() */ - ev_hdr->user_area.all = 0; - - event_prealloc_hdr_t *prealloc_hdr = (event_prealloc_hdr_t *)ev_hdr; - - return prealloc_hdr; -} - -static inline event_prealloc_hdr_t * -list_node_to_prealloc_hdr(list_node_t *const list_node) -{ - event_prealloc_hdr_t *const ev_hdr = (event_prealloc_hdr_t *)(uintptr_t) - ((uint8_t *)list_node - offsetof(event_prealloc_hdr_t, list_node)); - - return likely(list_node != NULL) ? ev_hdr : NULL; -} - -/** - * @brief Convert event vector table content to odp packets in-place. - * - * Convert an EM event vector table, containing em_event_t:s with - * esv-info (evgen), to a table of odp packets (remove handles' evgen in-place). - */ -static inline void -vector_tbl2odp(odp_event_t odp_event_pktvec) -{ - odp_packet_vector_t pkt_vec = odp_packet_vector_from_event(odp_event_pktvec); - odp_packet_t *pkt_tbl = NULL; - const int pkts = odp_packet_vector_tbl(pkt_vec, &pkt_tbl/*out*/); - - if (likely(pkts > 0)) { - /* Careful! Points to same table */ - em_event_t *event_tbl = (em_event_t *)pkt_tbl; - - /* Drop ESV event generation (evgen) from event handle */ - (void)events_em2pkt_inplace(event_tbl, pkts); - } -} - -/** - * @brief Convert ODP packet vector table content to EM events. - * - * Convert an ODP packet vector table to a table of EM events. - * The content must be known to be raw odp packets. - * - * For recovery purposes only. - */ -static inline void -vector_tbl2em(odp_event_t odp_event_pktvec) -{ - odp_packet_vector_t pkt_vec = odp_packet_vector_from_event(odp_event_pktvec); - odp_packet_t *pkt_tbl = NULL; - const int pkts = odp_packet_vector_tbl(pkt_vec, &pkt_tbl/*out*/); - - if (likely(pkts > 0)) { - em_event_t *const ev_tbl = (em_event_t *const)pkt_tbl; - odp_packet_t odp_pkttbl[pkts]; - event_hdr_t *ev_hdr_tbl[pkts]; - - /* - * Copy pkts from vector's pkt-table using events_em2pkt() that - * also drops any evgen-info from the handles if present. - */ - events_em2pkt(ev_tbl/*in*/, odp_pkttbl/*out*/, pkts); - - event_init_pkt_multi(odp_pkttbl /*in*/, ev_tbl /*in,out*/, - ev_hdr_tbl /*out*/, pkts, false); - } -} - -static inline em_status_t -send_event(em_event_t event, const queue_elem_t *q_elem) -{ - const bool esv_ena = esv_enabled(); - odp_event_t odp_event = event_em2odp(event); - odp_queue_t odp_queue = q_elem->odp_queue; - int ret; - - if (unlikely(EM_CHECK_LEVEL > 1 && - (odp_event == ODP_EVENT_INVALID || - odp_queue == ODP_QUEUE_INVALID))) - return EM_ERR_NOT_FOUND; - - if (unlikely(EM_CHECK_LEVEL > 0 && - q_elem->state != EM_QUEUE_STATE_READY)) { - return EM_ERR_BAD_STATE; - } - - /* - * Vector: convert the event vector table to a table of odp packets - * (in-place) before passing the vector and contents to the scheduler. - */ - if (esv_ena && odp_event_type(odp_event) == ODP_EVENT_PACKET_VECTOR) - vector_tbl2odp(odp_event); - - /* Enqueue event for scheduling */ - ret = odp_queue_enq(odp_queue, odp_event); - - if (unlikely(EM_CHECK_LEVEL > 0 && ret != 0)) { - /* Restore EM vector event-table before returning vector to user */ - if (esv_ena && odp_event_type(odp_event) == ODP_EVENT_PACKET_VECTOR) - vector_tbl2em(odp_event); - - return EM_ERR_LIB_FAILED; - } - - return EM_OK; -} - -static inline int -send_event_multi(const em_event_t events[], const int num, - const queue_elem_t *q_elem) -{ - const bool esv_ena = esv_enabled(); - odp_event_t odp_events[num]; - odp_queue_t odp_queue = q_elem->odp_queue; - - if (unlikely(EM_CHECK_LEVEL > 1 && odp_queue == ODP_QUEUE_INVALID)) - return 0; - - if (unlikely(EM_CHECK_LEVEL > 0 && - q_elem->state != EM_QUEUE_STATE_READY)) { - return 0; - } - - events_em2odp(events, odp_events/*out*/, num); - - /* - * Vector: convert the event vector table to a table of odp packets - * (in-place) before passing the vector and contents to the scheduler. - */ - if (esv_ena) { - for (int i = 0; i < num; i++) { - if (odp_event_type(odp_events[i]) == ODP_EVENT_PACKET_VECTOR) - vector_tbl2odp(odp_events[i]); - } - } - - /* Enqueue events for scheduling */ - int ret = odp_queue_enq_multi(odp_queue, odp_events, num); - - if (likely(ret == num)) - return num; /* Success! */ - - /* - * Fail: could not enqueue all events (ret != num) - */ - int enq = ret < 0 ? 0 : ret; - - /* Restore EM vector event-table before returning vector to user */ - if (esv_ena) { - for (int i = enq; i < num; i++) { - if (odp_event_type(odp_events[i]) == ODP_EVENT_PACKET_VECTOR) - vector_tbl2em(odp_events[i]); - } - } - - return enq; /* enq < num */ -} - -static inline em_status_t -send_local(em_event_t event, const queue_elem_t *q_elem) -{ - em_locm_t *const locm = &em_locm; - const em_queue_prio_t prio = q_elem->priority; - evhdl_t evhdl = {.event = event}; - int ret; - - if (unlikely(EM_CHECK_LEVEL > 0 && - q_elem->state != EM_QUEUE_STATE_READY)) - return EM_ERR_BAD_STATE; - - em_queue_t queue = (em_queue_t)(uintptr_t)q_elem->queue; - stash_entry_t entry = {.qidx = queue_hdl2idx(queue), - .evptr = evhdl.evptr}; - - ret = odp_stash_put_u64(locm->local_queues.prio[prio].stash, - &entry.u64, 1); - if (likely(ret == 1)) { - locm->local_queues.empty = 0; - locm->local_queues.prio[prio].empty_prio = 0; - return EM_OK; - } - - return EM_ERR_LIB_FAILED; -} - -static inline int -send_local_multi(const em_event_t events[], const int num, - const queue_elem_t *q_elem) -{ - em_locm_t *const locm = &em_locm; - const em_queue_prio_t prio = q_elem->priority; - const evhdl_t *const evhdl_tbl = (const evhdl_t *const)events; - - if (unlikely(EM_CHECK_LEVEL > 0 && - q_elem->state != EM_QUEUE_STATE_READY)) - return 0; - - stash_entry_t entry_tbl[num]; - em_queue_t queue = (em_queue_t)(uintptr_t)q_elem->queue; - const uint16_t qidx = (uint16_t)queue_hdl2idx(queue); - - for (int i = 0; i < num; i++) { - entry_tbl[i].qidx = qidx; - entry_tbl[i].evptr = evhdl_tbl[i].evptr; - } - - int ret = odp_stash_put_u64(locm->local_queues.prio[prio].stash, - &entry_tbl[0].u64, num); - if (likely(ret > 0)) { - locm->local_queues.empty = 0; - locm->local_queues.prio[prio].empty_prio = 0; - return ret; - } - - return 0; -} - -/** - * Send one event to a queue of type EM_QUEUE_TYPE_OUTPUT - */ -static inline em_status_t -send_output(em_event_t event, queue_elem_t *const output_q_elem) -{ - const em_sched_context_type_t sched_ctx_type = - em_locm.current.sched_context_type; - - if (unlikely(EM_CHECK_LEVEL > 0 && - output_q_elem->state != EM_QUEUE_STATE_UNSCHEDULED)) - return EM_ERR_BAD_STATE; - - /* - * An event sent to an output queue from an ordered context needs to - * be 're-ordered' before calling the user provided output-function. - * Order is maintained by enqueuing and dequeuing into an odp-queue - * that takes care of order. - */ - if (sched_ctx_type == EM_SCHED_CONTEXT_TYPE_ORDERED) { - const odp_queue_t odp_queue = output_q_elem->odp_queue; - odp_event_t odp_event = event_em2odp(event); - int ret; - - if (unlikely(EM_CHECK_LEVEL > 1 && - (odp_event == ODP_EVENT_INVALID || - odp_queue == ODP_QUEUE_INVALID))) - return EM_ERR_NOT_FOUND; - - if (!EM_OUTPUT_QUEUE_IMMEDIATE) - output_queue_track(output_q_elem); - - /* enqueue to enforce odp to handle ordering */ - ret = odp_queue_enq(odp_queue, odp_event); - if (unlikely(ret != 0)) - return EM_ERR_LIB_FAILED; - - /* return value must be EM_OK after this since event enqueued */ - - if (EM_OUTPUT_QUEUE_IMMEDIATE) { - env_spinlock_t *const lock = - &output_q_elem->output.lock; - - if (!env_spinlock_trylock(lock)) - return EM_OK; - output_queue_drain(output_q_elem); - env_spinlock_unlock(lock); - } - - return EM_OK; - } - - /* - * No ordered context - call output_fn() directly - */ - const em_queue_t output_queue = (em_queue_t)(uintptr_t)output_q_elem->queue; - const em_output_func_t output_fn = - output_q_elem->output.output_conf.output_fn; - void *const output_fn_args = - output_q_elem->output.output_conf.output_fn_args; - int sent; - - sent = output_fn(&event, 1, output_queue, output_fn_args); - if (unlikely(sent != 1)) - return EM_ERR_OPERATION_FAILED; - - return EM_OK; -} - -/** - * Send events to a queue of type EM_QUEUE_TYPE_OUTPUT - */ -static inline int -send_output_multi(const em_event_t events[], const unsigned int num, - queue_elem_t *const output_q_elem) -{ - const em_sched_context_type_t sched_ctx_type = - em_locm.current.sched_context_type; - int sent; - - if (unlikely(EM_CHECK_LEVEL > 0 && - output_q_elem->state != EM_QUEUE_STATE_UNSCHEDULED)) - return 0; - - /* - * Event sent to an output queue from an ordered context needs to - * be 're-ordered' before calling the user provided output-function. - * Order is maintained by enqueuing and dequeuing into an odp-queue - * that takes care of order. - */ - if (sched_ctx_type == EM_SCHED_CONTEXT_TYPE_ORDERED) { - const odp_queue_t odp_queue = output_q_elem->odp_queue; - odp_event_t odp_events[num]; - - if (unlikely(EM_CHECK_LEVEL > 1 && - odp_queue == ODP_QUEUE_INVALID)) - return 0; - - if (!EM_OUTPUT_QUEUE_IMMEDIATE) - output_queue_track(output_q_elem); - - events_em2odp(events, odp_events/*out*/, num); - - /* enqueue to enforce odp to handle ordering */ - sent = odp_queue_enq_multi(odp_queue, odp_events, num); - if (unlikely(sent <= 0)) - return 0; - - /* the return value must be the number of enqueued events */ - - if (EM_OUTPUT_QUEUE_IMMEDIATE) { - env_spinlock_t *const lock = - &output_q_elem->output.lock; - - if (!env_spinlock_trylock(lock)) - return sent; - output_queue_drain(output_q_elem); - env_spinlock_unlock(lock); - } - - return sent; - } - - /* - * No ordered context - call output_fn() directly - */ - const em_queue_t output_queue = (em_queue_t)(uintptr_t)output_q_elem->queue; - const em_output_func_t output_fn = output_q_elem->output.output_conf.output_fn; - void *const output_fn_args = output_q_elem->output.output_conf.output_fn_args; - - sent = output_fn(events, num, output_queue, output_fn_args); - - return sent; -} - -/** - * Return a pointer to the EM event user payload. - * Helper to e.g. EM API em_event_pointer() - */ -static inline void * -event_pointer(em_event_t event) -{ - const odp_event_t odp_event = event_em2odp(event); - const odp_event_type_t odp_etype = odp_event_type(odp_event); - void *ev_ptr = NULL; /* return value */ - - if (odp_etype == ODP_EVENT_PACKET) { - const odp_packet_t odp_pkt = odp_packet_from_event(odp_event); - - ev_ptr = odp_packet_data(odp_pkt); - } else if (odp_etype == ODP_EVENT_BUFFER) { - const odp_buffer_t odp_buf = odp_buffer_from_event(odp_event); - const event_hdr_t *ev_hdr = odp_buffer_user_area(odp_buf); - const uint32_t align_offset = ev_hdr->align_offset; - - ev_ptr = odp_buffer_addr(odp_buf); - - if (align_offset) - ev_ptr = (void *)((uintptr_t)ev_ptr + 32 - align_offset); - } - - return ev_ptr; /* NULL for unrecognized odp_etype, also for vectors */ -} - -static inline bool -event_has_ref(em_event_t event) -{ - odp_event_t odp_event = event_em2odp(event); - odp_event_type_t odp_etype = odp_event_type(odp_event); - - if (odp_etype != ODP_EVENT_PACKET) - return false; - - odp_packet_t odp_pkt = odp_packet_from_event(odp_event); - - return odp_packet_has_ref(odp_pkt) ? true : false; -} - -#ifdef __cplusplus -} -#endif - -#endif /* EM_EVENT_H_ */ +/* + * Copyright (c) 2015-2023, Nokia Solutions and Networks + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions + * are met: + * + * * Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * * Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in the + * documentation and/or other materials provided with the distribution. + * * Neither the name of the copyright holder nor the names of its + * contributors may be used to endorse or promote products derived + * from this software without specific prior written permission. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS + * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT + * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR + * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT + * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, + * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT + * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, + * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY + * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT + * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE + * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + */ + + /** + * @file + * EM internal event functions + * + */ + +#ifndef EM_EVENT_H_ +#define EM_EVENT_H_ + +#ifdef __cplusplus +extern "C" { +#endif + +#ifndef __clang__ +COMPILE_TIME_ASSERT((uintptr_t)EM_EVENT_UNDEF == (uintptr_t)ODP_EVENT_INVALID, + EM_EVENT_NOT_EQUAL_TO_ODP_EVENT); +COMPILE_TIME_ASSERT(EM_TMO_TYPE_NONE == 0, + "EM_TMO_TYPE_NONE must be 0"); +#endif + +em_status_t event_init(void); +void print_event_info(void); +em_event_t pkt_clone_odp(odp_packet_t pkt, odp_pool_t pkt_pool, + uint32_t offset, uint32_t size, + bool clone_uarea, bool is_clone_part); +void output_queue_track(queue_elem_t *const output_q_elem); +void output_queue_drain(const queue_elem_t *output_q_elem); +void output_queue_buffering_drain(void); + +uint32_t event_vector_tbl(em_event_t vector_event, em_event_t **event_tbl/*out*/); +em_status_t event_vector_max_size(em_event_t vector_event, uint32_t *max_size /*out*/, + em_escope_t escope); + +/** + * Initialize the event header of a packet allocated outside of EM. + */ +static inline em_event_t +evhdr_init_pkt(event_hdr_t *ev_hdr, em_event_t event, + odp_packet_t odp_pkt, bool is_extev) +{ + const int user_flag_set = odp_packet_user_flag(odp_pkt); + const bool esv_ena = esv_enabled(); + + if (user_flag_set) { + /* Event already initialized by EM */ + if (esv_ena) { + event = ev_hdr->event; + if (is_extev) + event = evstate_em2usr(event, ev_hdr, EVSTATE__DISPATCH); + } + + return event; + } + + /* + * ODP pkt from outside of EM - not allocated by EM & needs init + */ + odp_packet_user_flag_set(odp_pkt, USER_FLAG_SET); + ev_hdr->event_type = EM_EVENT_TYPE_PACKET; + ev_hdr->egrp = EM_EVENT_GROUP_UNDEF; + ev_hdr->user_area.all = 0; /* uarea fields init when used */ + + if (!esv_ena) { + ev_hdr->flags.all = 0; + ev_hdr->event = event; + return event; + } + + /* + * ESV enabled: + */ + if (!em_shm->opt.esv.prealloc_pools || ev_hdr->flags.refs_used) { + /* No prealloc OR pkt was a ref before being freed into the pool */ + event = evstate_init(event, ev_hdr, is_extev); + } else { + /* esv.prealloc_pools == true: */ + odp_pool_t odp_pool = odp_packet_pool(odp_pkt); + em_pool_t pool = pool_odp2em(odp_pool); + + if (pool == EM_POOL_UNDEF) { + /* External odp pkt originates from an ODP-pool */ + event = evstate_init(event, ev_hdr, is_extev); + } else { + /* External odp pkt originates from an EM-pool */ + event = evstate_update(event, ev_hdr, is_extev); + } + } + ev_hdr->flags.all = 0; /* clear only after evstate_...() */ + + return event; +} + +/** + * Initialize the event headers of packets allocated outside of EM. + */ +static inline void +evhdr_init_pkt_multi(event_hdr_t *const ev_hdrs[], + em_event_t events[/*in,out*/], + const odp_packet_t odp_pkts[/*in*/], + const int num, bool is_extev) +{ + const bool esv_ena = esv_enabled(); + int user_flag_set; + + int needs_init_idx[num]; + int needs_init_num = 0; + int idx; + + for (int i = 0; i < num; i++) { + user_flag_set = odp_packet_user_flag(odp_pkts[i]); + if (user_flag_set) { + /* Event already initialized by EM */ + if (esv_ena) { + events[i] = ev_hdrs[i]->event; + if (is_extev) + events[i] = evstate_em2usr(events[i], ev_hdrs[i], + EVSTATE__DISPATCH_MULTI); + } + /* else events[i] = events[i] */ + } else { + odp_packet_user_flag_set(odp_pkts[i], USER_FLAG_SET); + needs_init_idx[needs_init_num] = i; + needs_init_num++; + } + } + + if (needs_init_num == 0) + return; + + /* + * ODP pkt from outside of EM - not allocated by EM & needs init + */ + + if (!esv_ena) { + for (int i = 0; i < needs_init_num; i++) { + idx = needs_init_idx[i]; + ev_hdrs[idx]->flags.all = 0; + ev_hdrs[idx]->event_type = EM_EVENT_TYPE_PACKET; + ev_hdrs[idx]->event = events[idx]; + ev_hdrs[idx]->egrp = EM_EVENT_GROUP_UNDEF; + ev_hdrs[idx]->user_area.all = 0; /* uarea fields init when used */ + } + + return; + } + + /* + * ESV enabled: + */ + if (!em_shm->opt.esv.prealloc_pools) { + for (int i = 0; i < needs_init_num; i++) { + idx = needs_init_idx[i]; + events[idx] = evstate_init(events[idx], ev_hdrs[idx], is_extev); + } + } else { + /* em_shm->opt.esv.prealloc_pools == true */ + for (int i = 0; i < needs_init_num; i++) { + idx = needs_init_idx[i]; + + odp_pool_t odp_pool = odp_packet_pool(odp_pkts[idx]); + em_pool_t pool = pool_odp2em(odp_pool); + + if (pool == EM_POOL_UNDEF || ev_hdrs[idx]->flags.refs_used) { + /* + * External odp pkt originates from an ODP-pool, + * or pkt was a ref before being freed into the pool. + */ + events[idx] = evstate_init(events[idx], ev_hdrs[idx], is_extev); + } else { + /* External odp pkt originates from an EM-pool */ + events[idx] = evstate_update(events[idx], ev_hdrs[idx], is_extev); + } + } + } + + for (int i = 0; i < needs_init_num; i++) { + idx = needs_init_idx[i]; + ev_hdrs[idx]->flags.all = 0; /* clear only after evstate_...() */ + ev_hdrs[idx]->event_type = EM_EVENT_TYPE_PACKET; + ev_hdrs[idx]->egrp = EM_EVENT_GROUP_UNDEF; + ev_hdrs[idx]->user_area.all = 0; /* uarea fields init when used */ + } +} + +/** + * Initialize the event header of a packet vector allocated outside of EM. + */ +static inline em_event_t +evhdr_init_pktvec(event_hdr_t *ev_hdr, em_event_t event, + odp_packet_vector_t odp_pktvec, bool is_extev) +{ + const int user_flag = odp_packet_vector_user_flag(odp_pktvec); + const bool esv_ena = esv_enabled(); + + if (user_flag == USER_FLAG_SET) { + /* Event already initialized by EM */ + if (esv_ena) { + event = ev_hdr->event; + if (is_extev) + event = evstate_em2usr(event, ev_hdr, EVSTATE__DISPATCH); + } + + return event; + } + + /* + * ODP pkt from outside of EM - not allocated by EM & needs init + */ + odp_packet_vector_user_flag_set(odp_pktvec, USER_FLAG_SET); + /* can clear flags before evstate_...(), flags.refs_used not set for vecs */ + ev_hdr->flags.all = 0; + ev_hdr->event_type = EM_EVENT_TYPE_VECTOR; + ev_hdr->egrp = EM_EVENT_GROUP_UNDEF; + ev_hdr->user_area.all = 0; /* uarea fields init when used */ + + if (!esv_ena) { + ev_hdr->event = event; + return event; + } + + /* + * ESV enabled: + */ + if (!em_shm->opt.esv.prealloc_pools) { + event = evstate_init(event, ev_hdr, is_extev); + } else { + /* esv.prealloc_pools == true: */ + odp_pool_t odp_pool = odp_packet_vector_pool(odp_pktvec); + em_pool_t pool = pool_odp2em(odp_pool); + + if (pool == EM_POOL_UNDEF) { + /* External odp pkt originates from an ODP-pool */ + event = evstate_init(event, ev_hdr, is_extev); + } else { + /* External odp pkt originates from an EM-pool */ + event = evstate_update(event, ev_hdr, is_extev); + if (is_extev) + return event; + } + } + + return event; +} + +/** + * Initialize the event headers of packet vectors allocated outside of EM. + */ +static inline void +evhdr_init_pktvec_multi(event_hdr_t *ev_hdrs[/*out*/], + em_event_t events[/*in,out*/], + const odp_packet_vector_t odp_pktvecs[/*in*/], + const int num, bool is_extev) +{ + const bool esv_ena = esv_enabled(); + + int needs_init_idx[num]; + int needs_init_num = 0; + int idx; + + for (int i = 0; i < num; i++) { + int user_flag = odp_packet_vector_user_flag(odp_pktvecs[i]); + + if (user_flag == USER_FLAG_SET) { + /* Event already initialized by EM */ + if (esv_ena) { + events[i] = ev_hdrs[i]->event; + if (is_extev) + events[i] = evstate_em2usr(events[i], ev_hdrs[i], + EVSTATE__DISPATCH_MULTI); + } + /* else events[i] = events[i] */ + } else { + odp_packet_vector_user_flag_set(odp_pktvecs[i], USER_FLAG_SET); + needs_init_idx[needs_init_num] = i; + needs_init_num++; + } + } + + if (needs_init_num == 0) + return; + + /* + * ODP pkt vector from outside of EM - not allocated by EM & needs init + */ + + if (!esv_ena) { + for (int i = 0; i < needs_init_num; i++) { + idx = needs_init_idx[i]; + ev_hdrs[idx]->flags.all = 0; + ev_hdrs[idx]->event_type = EM_EVENT_TYPE_VECTOR; + ev_hdrs[idx]->event = events[idx]; + ev_hdrs[idx]->egrp = EM_EVENT_GROUP_UNDEF; + ev_hdrs[idx]->user_area.all = 0; /* uarea fields init when used */ + } + + return; + } + + /* + * ESV enabled: + */ + for (int i = 0; i < needs_init_num; i++) { + idx = needs_init_idx[i]; + /* can clear flags before evstate_...(), flags.refs_used not set for vecs */ + ev_hdrs[idx]->flags.all = 0; + ev_hdrs[idx]->event_type = EM_EVENT_TYPE_VECTOR; + ev_hdrs[idx]->egrp = EM_EVENT_GROUP_UNDEF; + ev_hdrs[idx]->user_area.all = 0; /* uarea fields init when used */ + } + + if (!em_shm->opt.esv.prealloc_pools) { + for (int i = 0; i < needs_init_num; i++) { + idx = needs_init_idx[i]; + events[idx] = evstate_init(events[idx], ev_hdrs[idx], is_extev); + } + + return; + } + + /* + * em_shm->opt.esv.prealloc_pools == true + */ + for (int i = 0; i < needs_init_num; i++) { + idx = needs_init_idx[i]; + + odp_pool_t odp_pool = odp_packet_vector_pool(odp_pktvecs[idx]); + em_pool_t pool = pool_odp2em(odp_pool); + + if (pool == EM_POOL_UNDEF) { + /* External odp pkt originates from an ODP-pool */ + events[idx] = evstate_init(events[idx], ev_hdrs[idx], is_extev); + } else { + /* External odp pkt originates from an EM-pool */ + events[idx] = evstate_update(events[idx], ev_hdrs[idx], is_extev); + } + } +} + +/** + * Initialize an external ODP event that have been input into EM. + * + * Initialize the event header if needed, i.e. if event originated from outside + * of EM from pktio or other input and was not allocated by EM via em_alloc(). + * The odp pkt-user-ptr is used to determine whether the header has been + * initialized or not. + */ +static inline em_event_t +event_init_odp(odp_event_t odp_event, bool is_extev, event_hdr_t **ev_hdr__out) +{ + const odp_event_type_t odp_type = odp_event_type(odp_event); + em_event_t event = event_odp2em(odp_event); /* return value, updated by ESV */ + + switch (odp_type) { + case ODP_EVENT_PACKET: { + odp_packet_t odp_pkt = odp_packet_from_event(odp_event); + event_hdr_t *ev_hdr = odp_packet_user_area(odp_pkt); + + /* init event-hdr if needed (also ESV-state if used) */ + event = evhdr_init_pkt(ev_hdr, event, odp_pkt, is_extev); + if (ev_hdr__out) + *ev_hdr__out = ev_hdr; + return event; + } + case ODP_EVENT_BUFFER: { + const bool esv_ena = esv_enabled(); + + if (!ev_hdr__out && !esv_ena) + return event; + + odp_buffer_t odp_buf = odp_buffer_from_event(odp_event); + event_hdr_t *ev_hdr = odp_buffer_user_area(odp_buf); + + if (esv_ena) { /* update event handle (ESV) */ + event = ev_hdr->event; + if (is_extev) + event = evstate_em2usr(event, ev_hdr, EVSTATE__DISPATCH); + } + if (ev_hdr__out) + *ev_hdr__out = ev_hdr; + return event; + } + case ODP_EVENT_PACKET_VECTOR: { + odp_packet_vector_t odp_pktvec = odp_packet_vector_from_event(odp_event); + event_hdr_t *ev_hdr = odp_packet_vector_user_area(odp_pktvec); + + /* init event-hdr if needed (also ESV-state if used) */ + event = evhdr_init_pktvec(ev_hdr, event, odp_pktvec, is_extev); + if (ev_hdr__out) + *ev_hdr__out = ev_hdr; + return event; + } + case ODP_EVENT_TIMEOUT: { + odp_timeout_t odp_tmo = odp_timeout_from_event(odp_event); + event_hdr_t *ev_hdr = odp_timeout_user_area(odp_tmo); + const bool esv_ena = esv_enabled(); + + if (esv_ena) { + /* + * Update event handle, no other ESV checks done. + * Some timers might send a copy of the original event + * in tear-down, thus keep ptr but update evgen. + */ + evhdl_t evhdl = {.event = event}; /* .evptr from here */ + evhdl_t evhdr_hdl = {.event = ev_hdr->event}; /* .evgen from here */ + + evhdl.evgen = evhdr_hdl.evgen; /* update .evgen */ + ev_hdr->event = evhdl.event; /* store updated hdl in hdr */ + event = evhdl.event; /* return updated event */ + } + + if (ev_hdr__out) + *ev_hdr__out = ev_hdr; + return event; + } + default: + INTERNAL_ERROR(EM_FATAL(EM_ERR_NOT_IMPLEMENTED), + EM_ESCOPE_EVENT_INIT_ODP, + "Unexpected odp event type:%u", odp_type); + __builtin_unreachable(); + /* never reached */ + return EM_EVENT_UNDEF; + } +} + +/* Helper to event_init_odp_multi() */ +static inline void +event_init_pkt_multi(const odp_packet_t odp_pkts[/*in*/], + em_event_t events[/*in,out*/], event_hdr_t *ev_hdrs[/*out*/], + const int num, bool is_extev) +{ + for (int i = 0; i < num; i++) + ev_hdrs[i] = odp_packet_user_area(odp_pkts[i]); + + evhdr_init_pkt_multi(ev_hdrs, events, odp_pkts, num, is_extev); +} + +/* Helper to event_init_odp_multi() */ +static inline void +event_init_buf_multi(const odp_buffer_t odp_bufs[/*in*/], + em_event_t events[/*in,out*/], event_hdr_t *ev_hdrs[/*out*/], + const int num, bool is_extev) +{ + for (int i = 0; i < num; i++) + ev_hdrs[i] = odp_buffer_user_area(odp_bufs[i]); + + if (esv_enabled()) { + /* update event handle (ESV) */ + for (int i = 0; i < num; i++) + events[i] = ev_hdrs[i]->event; + + if (is_extev) + evstate_em2usr_multi(events, ev_hdrs, num, + EVSTATE__DISPATCH_MULTI); + } +} + +/* Helper to event_init_odp_multi() */ +static inline void +event_init_tmo_multi(const odp_timeout_t odp_tmos[/*in*/], + em_event_t events[/*in,out*/], event_hdr_t *ev_hdrs[/*out*/], + const int num) +{ + for (int i = 0; i < num; i++) + ev_hdrs[i] = odp_timeout_user_area(odp_tmos[i]); + + /* ignore ESV */ + (void)events; +} + +/* Helper to event_init_odp_multi() */ +static inline void +event_init_pktvec_multi(const odp_packet_vector_t odp_pktvecs[/*in*/], + em_event_t events[/*in,out*/], event_hdr_t *ev_hdrs[/*out*/], + const int num, bool is_extev) +{ + for (int i = 0; i < num; i++) + ev_hdrs[i] = odp_packet_vector_user_area(odp_pktvecs[i]); + + evhdr_init_pktvec_multi(ev_hdrs, events, odp_pktvecs, num, is_extev); +} + +/** + * Convert from EM events to event headers and initialize the headers as needed. + * + * Initialize the event header if needed, i.e. if event originated from outside + * of EM from pktio or other input and was not allocated by EM via em_alloc(). + * The odp pkt-user-ptr is used to determine whether the header has been + * initialized or not. + */ +static inline void +event_init_odp_multi(const odp_event_t odp_events[/*in*/], + em_event_t events[/*out*/], event_hdr_t *ev_hdrs[/*out*/], + const int num, bool is_extev) +{ + for (int i = 0; i < num; i++) + events[i] = event_init_odp(odp_events[i], is_extev, &ev_hdrs[i]); +} + +/** + * Allocate an event based on an odp-buf. + */ +static inline event_hdr_t * +event_alloc_buf(const mpool_elem_t *const pool_elem, uint32_t size) +{ + odp_buffer_t odp_buf = ODP_BUFFER_INVALID; + int subpool; + + /* + * Allocate from the 'best fit' subpool, or if that is full, from the + * next subpool that has buffers available of a bigger size. + */ + subpool = pool_find_subpool(pool_elem, size); + if (unlikely(subpool < 0)) + return NULL; + + for (; subpool < pool_elem->num_subpools; subpool++) { + odp_pool_t odp_pool = pool_elem->odp_pool[subpool]; + + if (EM_CHECK_LEVEL >= 3 && + unlikely(odp_pool == ODP_POOL_INVALID)) + return NULL; + + odp_buf = odp_buffer_alloc(odp_pool); + if (likely(odp_buf != ODP_BUFFER_INVALID)) + break; + } + + if (unlikely(odp_buf == ODP_BUFFER_INVALID)) + return NULL; + + /* + * odp buffer now allocated - init the EM event header + * in the odp user area. + */ + event_hdr_t *const ev_hdr = odp_buffer_user_area(odp_buf); + odp_event_t odp_event = odp_buffer_to_event(odp_buf); + em_event_t event = event_odp2em(odp_event); + + ev_hdr->event = event; /* store this event handle */ + ev_hdr->align_offset = pool_elem->align_offset; + + /* init common ev_hdr fields in the caller */ + + return ev_hdr; +} + +/** + * Allocate & initialize multiple events based on odp-bufs. + */ +static inline int +event_alloc_buf_multi(em_event_t events[/*out*/], const int num, + const mpool_elem_t *pool_elem, uint32_t size, + em_event_type_t type) +{ + odp_buffer_t odp_bufs[num]; + odp_event_t odp_event; + event_hdr_t *ev_hdrs[num]; + int subpool; + const bool esv_ena = esv_enabled(); + + /* + * Allocate from the 'best fit' subpool, or if that is full, from the + * next subpool that has buffers available of a bigger size. + */ + subpool = pool_find_subpool(pool_elem, size); + if (unlikely(subpool < 0)) + return 0; + + int num_req = num; + int num_bufs = 0; + int i; + + for (; subpool < pool_elem->num_subpools; subpool++) { + odp_pool_t odp_pool = pool_elem->odp_pool[subpool]; + + if (EM_CHECK_LEVEL >= 3 && + unlikely(odp_pool == ODP_POOL_INVALID)) + return 0; + + int ret = odp_buffer_alloc_multi(odp_pool, &odp_bufs[num_bufs], + num_req); + if (unlikely(ret <= 0)) + continue; /* try next subpool */ + + /* store the allocated events[] */ + for (i = num_bufs; i < num_bufs + ret; i++) { + odp_event = odp_buffer_to_event(odp_bufs[i]); + events[i] = event_odp2em(odp_event); + } + + /* Init 'ret' ev-hdrs from this 'subpool'=='odp-pool' */ + for (i = num_bufs; i < num_bufs + ret; i++) + ev_hdrs[i] = odp_buffer_user_area(odp_bufs[i]); + + if (esv_ena) { + /* reads ev_hdrs[i]->flags if prealloc_pools used */ + evstate_alloc_multi(&events[num_bufs] /*in/out*/, + &ev_hdrs[num_bufs], ret); + } + + for (i = num_bufs; i < num_bufs + ret; i++) { + ev_hdrs[i]->flags.all = 0; + ev_hdrs[i]->event_type = type; + if (!esv_ena) + ev_hdrs[i]->event = events[i]; + ev_hdrs[i]->event_size = size; + ev_hdrs[i]->egrp = EM_EVENT_GROUP_UNDEF; + + ev_hdrs[i]->user_area.all = 0; + ev_hdrs[i]->user_area.size = pool_elem->user_area.size; + ev_hdrs[i]->user_area.isinit = 1; + + ev_hdrs[i]->align_offset = pool_elem->align_offset; + } + + num_bufs += ret; + if (likely(num_bufs == num)) + break; /* all allocated */ + num_req -= ret; + } + + return num_bufs; /* number of allocated bufs (0 ... num) */ +} + +/** + * Allocate & initialize an event based on an odp-pkt. + */ +static inline event_hdr_t * +event_alloc_pkt(const mpool_elem_t *pool_elem, uint32_t size) +{ + const uint32_t push_len = pool_elem->align_offset; + uint32_t pull_len; + uint32_t alloc_size; + odp_packet_t odp_pkt = ODP_PACKET_INVALID; + int subpool; + + if (size > push_len) { + alloc_size = size - push_len; + pull_len = 0; + } else { + alloc_size = 1; /* min allowed */ + pull_len = push_len + 1 - size; + } + + /* + * Allocate from the 'best fit' subpool, or if that is full, from the + * next subpool that has pkts available of a bigger size. + */ + subpool = pool_find_subpool(pool_elem, size); + if (unlikely(subpool < 0)) + return NULL; + + for (; subpool < pool_elem->num_subpools; subpool++) { + odp_pool_t odp_pool = pool_elem->odp_pool[subpool]; + + if (EM_CHECK_LEVEL >= 3 && + unlikely(odp_pool == ODP_POOL_INVALID)) + return NULL; + + odp_pkt = odp_packet_alloc(odp_pool, alloc_size); + if (likely(odp_pkt != ODP_PACKET_INVALID)) + break; + } + + if (unlikely(odp_pkt == ODP_PACKET_INVALID)) + return NULL; + + /* + * odp packet now allocated - adjust the payload start address and + * init the EM event header in the odp-pkt user-area + */ + + /* Adjust event payload start-address based on alignment config */ + const void *ptr; + + if (push_len) { + ptr = odp_packet_push_head(odp_pkt, push_len); + if (EM_CHECK_LEVEL >= 3 && unlikely(!ptr)) + goto err_pktalloc; + } + if (pull_len) { + ptr = odp_packet_pull_tail(odp_pkt, pull_len); + if (EM_CHECK_LEVEL >= 3 && unlikely(!ptr)) + goto err_pktalloc; + } + + /* + * Set the pkt user ptr to be able to recognize pkt-events that + * EM has created vs pkts from pkt-input that needs their + * ev-hdrs to be initialized. + */ + odp_packet_user_flag_set(odp_pkt, USER_FLAG_SET); + + event_hdr_t *const ev_hdr = odp_packet_user_area(odp_pkt); + odp_event_t odp_event = odp_packet_to_event(odp_pkt); + em_event_t event = event_odp2em(odp_event); + + if (EM_CHECK_LEVEL >= 3 && unlikely(ev_hdr == NULL)) + goto err_pktalloc; + + /* store this event handle */ + ev_hdr->event = event; + + /* init common ev_hdr fields in the caller */ + + return ev_hdr; + +err_pktalloc: + odp_packet_free(odp_pkt); + return NULL; +} + +/* + * Helper for event_alloc_pkt_multi() + */ +static inline int +pktalloc_multi(odp_packet_t odp_pkts[/*out*/], int num, + odp_pool_t odp_pool, uint32_t size, + uint32_t push_len, uint32_t pull_len) +{ + int ret = odp_packet_alloc_multi(odp_pool, size, odp_pkts, num); + + if (unlikely(ret <= 0)) + return 0; + + const int num_pkts = ret; /* return value > 0 */ + const void *ptr = NULL; + int i; + + /* Adjust payload start-address based on alignment config */ + if (push_len) { + for (i = 0; i < num_pkts; i++) { + ptr = odp_packet_push_head(odp_pkts[i], push_len); + if (EM_CHECK_LEVEL >= 3 && unlikely(!ptr)) + goto err_pktalloc_multi; + } + } + if (pull_len) { + for (i = 0; i < num_pkts; i++) { + ptr = odp_packet_pull_tail(odp_pkts[i], pull_len); + if (EM_CHECK_LEVEL >= 3 && unlikely(!ptr)) + goto err_pktalloc_multi; /* only before esv */ + } + } + + /* + * Set the pkt user ptr to be able to recognize pkt-events that + * EM has created vs pkts from pkt-input that needs their + * ev-hdrs to be initialized. + */ + for (i = 0; i < num_pkts; i++) + odp_packet_user_flag_set(odp_pkts[i], USER_FLAG_SET); + + return num_pkts; + +err_pktalloc_multi: + odp_packet_free_multi(odp_pkts, num_pkts); + return 0; +} + +/** + * Allocate & initialize multiple events based on odp-pkts. + */ +static inline int +event_alloc_pkt_multi(em_event_t events[/*out*/], const int num, + const mpool_elem_t *pool_elem, uint32_t size, + em_event_type_t type) +{ + const uint32_t push_len = pool_elem->align_offset; + uint32_t pull_len; + odp_packet_t odp_pkts[num]; + /* use same output-array: odp_events[] = events[] */ + odp_event_t *const odp_events = (odp_event_t *)events; + event_hdr_t *ev_hdrs[num]; + uint32_t alloc_size; + int subpool; + const bool esv_ena = esv_enabled(); + + if (size > push_len) { + alloc_size = size - push_len; + pull_len = 0; + } else { + alloc_size = 1; /* min allowed */ + pull_len = push_len + 1 - size; + } + + /* + * Allocate from the 'best fit' subpool, or if that is full, from the + * next subpool that has pkts available of a bigger size. + */ + subpool = pool_find_subpool(pool_elem, size); + if (unlikely(subpool < 0)) + return 0; + + int num_req = num; + int num_pkts = 0; + int i; + + for (; subpool < pool_elem->num_subpools; subpool++) { + odp_pool_t odp_pool = pool_elem->odp_pool[subpool]; + + if (EM_CHECK_LEVEL >= 3 && + unlikely(odp_pool == ODP_POOL_INVALID)) + return 0; + + int ret = pktalloc_multi(&odp_pkts[num_pkts], num_req, + odp_pool, alloc_size, + push_len, pull_len); + if (unlikely(ret <= 0)) + continue; /* try next subpool */ + + /* + * Init 'ret' ev-hdrs from this 'subpool'=='odp-pool'. + * Note: odp_events[] points&writes into events[out] + */ + odp_packet_to_event_multi(&odp_pkts[num_pkts], + &odp_events[num_pkts], ret); + + for (i = num_pkts; i < num_pkts + ret; i++) + ev_hdrs[i] = odp_packet_user_area(odp_pkts[i]); + + /* + * Note: events[] == odp_events[] before ESV init. + * Don't touch odp_events[] during this loop-round anymore. + */ + if (esv_ena) { + /* reads ev_hdrs[i]->flags if prealloc_pools used */ + evstate_alloc_multi(&events[num_pkts] /*in/out*/, + &ev_hdrs[num_pkts], ret); + } + + for (i = num_pkts; i < num_pkts + ret; i++) { + ev_hdrs[i]->flags.all = 0; + ev_hdrs[i]->event_type = type; + if (!esv_ena) + ev_hdrs[i]->event = events[i]; + ev_hdrs[i]->event_size = size; /* original size */ + ev_hdrs[i]->egrp = EM_EVENT_GROUP_UNDEF; + + ev_hdrs[i]->user_area.all = 0; + ev_hdrs[i]->user_area.size = pool_elem->user_area.size; + ev_hdrs[i]->user_area.isinit = 1; + /*ev_hdrs[i]->align_offset = needed by odp bufs only*/ + } + + num_pkts += ret; + if (likely(num_pkts == num)) + break; /* all allocated */ + num_req -= ret; + } + + return num_pkts; /* number of allocated pkts */ +} + +static inline event_hdr_t * +event_alloc_vector(const mpool_elem_t *pool_elem, uint32_t size) +{ + odp_packet_vector_t odp_pktvec = ODP_PACKET_VECTOR_INVALID; + int subpool; + + /* + * Allocate from the 'best fit' subpool, or if that is full, from the + * next subpool that has pkts available of a bigger size. + */ + subpool = pool_find_subpool(pool_elem, size); + if (unlikely(subpool < 0)) + return NULL; + + for (; subpool < pool_elem->num_subpools; subpool++) { + odp_pool_t odp_pool = pool_elem->odp_pool[subpool]; + + if (EM_CHECK_LEVEL >= 3 && + unlikely(odp_pool == ODP_POOL_INVALID)) + return NULL; + + odp_pktvec = odp_packet_vector_alloc(odp_pool); + if (likely(odp_pktvec != ODP_PACKET_VECTOR_INVALID)) + break; + } + + if (unlikely(odp_pktvec == ODP_PACKET_VECTOR_INVALID)) + return NULL; + + /* + * Packet vector now allocated: + * Init the EM event header in the odp-pkt-vector user-area. + */ + + /* + * Set the pktvec user flag to be able to recognize vectors that + * EM has created vs. vectors from pkt-input that needs their + * ev-hdrs to be initialized. + */ + odp_packet_vector_user_flag_set(odp_pktvec, USER_FLAG_SET); + + event_hdr_t *const ev_hdr = odp_packet_vector_user_area(odp_pktvec); + odp_event_t odp_event = odp_packet_vector_to_event(odp_pktvec); + em_event_t event = event_odp2em(odp_event); + + if (EM_CHECK_LEVEL >= 3 && unlikely(ev_hdr == NULL)) + goto err_vecalloc; + + ev_hdr->event = event; /* store this event handle */ + + /* init common ev_hdr fields in the caller */ + + return ev_hdr; + +err_vecalloc: + odp_packet_vector_free(odp_pktvec); + return NULL; +} + +/* + * Helper for event_alloc_vec_multi() + */ +static inline int +vecalloc_multi(odp_packet_vector_t odp_pktvecs[/*out*/], int num, + odp_pool_t odp_pool) +{ + int i; + + for (i = 0; i < num; i++) { + odp_pktvecs[i] = odp_packet_vector_alloc(odp_pool); + if (unlikely(odp_pktvecs[i] == ODP_PACKET_VECTOR_INVALID)) + break; + } + + const int num_vecs = i; + + if (unlikely(num_vecs == 0)) + return 0; + + /* + * Set the pkt vector user ptr to be able to recognize vector-events + * that EM has created vs vectors from pkt-input that needs their + * ev-hdrs to be initialized. + */ + for (i = 0; i < num_vecs; i++) + odp_packet_vector_user_flag_set(odp_pktvecs[i], USER_FLAG_SET); + + return num_vecs; +} + +/** + * Allocate & initialize multiple events based on odp-pkt-vectors. + */ +static inline int +event_alloc_vector_multi(em_event_t events[/*out*/], const int num, + const mpool_elem_t *pool_elem, uint32_t size, + em_event_type_t type) +{ + odp_packet_vector_t odp_pktvecs[num]; + /* use same output-array: odp_events[] = events[] */ + odp_event_t *const odp_events = (odp_event_t *)events; + event_hdr_t *ev_hdrs[num]; + int subpool; + const bool esv_ena = esv_enabled(); + + /* + * Allocate from the 'best fit' subpool, or if that is full, from the + * next subpool that has pkts available of a bigger size. + */ + subpool = pool_find_subpool(pool_elem, size); + if (unlikely(subpool < 0)) + return 0; + + int num_req = num; + int num_vecs = 0; + int i; + + for (; subpool < pool_elem->num_subpools; subpool++) { + odp_pool_t odp_pool = pool_elem->odp_pool[subpool]; + + if (EM_CHECK_LEVEL >= 3 && + unlikely(odp_pool == ODP_POOL_INVALID)) + return 0; + + int ret = vecalloc_multi(&odp_pktvecs[num_vecs], num_req, + odp_pool); + if (unlikely(ret <= 0)) + continue; /* try next subpool */ + + /* + * Init 'ret' ev-hdrs from this 'subpool'=='odp-pool'. + * Note: odp_events[] points&writes into events[out] + */ + for (i = num_vecs; i < num_vecs + ret; i++) { + odp_events[i] = odp_packet_vector_to_event(odp_pktvecs[i]); + ev_hdrs[i] = odp_packet_vector_user_area(odp_pktvecs[i]); + } + + /* + * Note: events[] == odp_events[] before ESV init. + * Don't touch odp_events[] during this loop-round anymore. + */ + if (esv_ena) { + /* reads ev_hdrs[i]->flags if prealloc_pools used */ + evstate_alloc_multi(&events[num_vecs] /*in/out*/, + &ev_hdrs[num_vecs], ret); + } + + for (i = num_vecs; i < num_vecs + ret; i++) { + ev_hdrs[i]->flags.all = 0; + ev_hdrs[i]->event_type = type; + if (!esv_ena) + ev_hdrs[i]->event = events[i]; + ev_hdrs[i]->event_size = size; /* original vec size */ + ev_hdrs[i]->egrp = EM_EVENT_GROUP_UNDEF; + + ev_hdrs[i]->user_area.all = 0; + ev_hdrs[i]->user_area.size = pool_elem->user_area.size; + ev_hdrs[i]->user_area.isinit = 1; + /*ev_hdrs[i]->align_offset = needed by odp bufs only*/ + } + + num_vecs += ret; + if (likely(num_vecs == num)) + break; /* all allocated */ + num_req -= ret; + } + + return num_vecs; /* number of allocated pkts */ +} + +/** + * Helper for em_alloc() and em_event_clone() + */ +static inline em_event_t +event_alloc(const mpool_elem_t *pool_elem, uint32_t size, em_event_type_t type, + const uint16_t api_op) +{ + /* + * EM event pools created with type=PKT can support: + * - SW events (bufs) + * - pkt events. + * + * EM event pools created with type=SW can support: + * - SW events (bufs) only + */ + event_hdr_t *ev_hdr = NULL; + + if (pool_elem->event_type == EM_EVENT_TYPE_PACKET) + ev_hdr = event_alloc_pkt(pool_elem, size); + else if (pool_elem->event_type == EM_EVENT_TYPE_SW) + ev_hdr = event_alloc_buf(pool_elem, size); + else if (pool_elem->event_type == EM_EVENT_TYPE_VECTOR) + ev_hdr = event_alloc_vector(pool_elem, size); + + if (unlikely(!ev_hdr)) + return EM_EVENT_UNDEF; + + /* + * event now allocated: + * ev_hdr->event = stored by event_alloc_pkt/buf/vector() + */ + /* Update event ESV state for alloc */ + if (esv_enabled()) + (void)evstate_alloc(ev_hdr->event, ev_hdr, api_op); + + ev_hdr->flags.all = 0; /* clear only after evstate_alloc() */ + ev_hdr->event_type = type; /* store the event type */ + ev_hdr->event_size = size; /* store requested size */ + ev_hdr->egrp = EM_EVENT_GROUP_UNDEF; + + ev_hdr->user_area.all = 0; + ev_hdr->user_area.size = pool_elem->user_area.size; + ev_hdr->user_area.isinit = 1; + /* ev_hdr->align_offset = init by event_alloc_buf() when needed */ + + return ev_hdr->event; +} + +/** + * Start-up helper for pool preallocation + */ +static inline event_prealloc_hdr_t * +event_prealloc(const mpool_elem_t *pool_elem, uint32_t size) +{ + /* + * EM event pools created with type=PKT can support: + * - SW events (bufs) + * - pkt events. + * + * EM event pools created with type=SW can support: + * - SW events (bufs) only + */ + event_hdr_t *ev_hdr = NULL; + + if (pool_elem->event_type == EM_EVENT_TYPE_PACKET) + ev_hdr = event_alloc_pkt(pool_elem, size); + else if (pool_elem->event_type == EM_EVENT_TYPE_SW) + ev_hdr = event_alloc_buf(pool_elem, size); + else if (pool_elem->event_type == EM_EVENT_TYPE_VECTOR) + ev_hdr = event_alloc_vector(pool_elem, size); + + if (unlikely(ev_hdr == NULL)) + return NULL; + + /* event now allocated */ + + if (esv_enabled()) { + em_event_t event = ev_hdr->event; + + (void)evstate_prealloc(event, ev_hdr); + } + ev_hdr->flags.all = 0; /* clear only after evstate_alloc() */ + ev_hdr->user_area.all = 0; + + event_prealloc_hdr_t *prealloc_hdr = (event_prealloc_hdr_t *)ev_hdr; + + return prealloc_hdr; +} + +static inline event_prealloc_hdr_t * +list_node_to_prealloc_hdr(list_node_t *const list_node) +{ + event_prealloc_hdr_t *const ev_hdr = (event_prealloc_hdr_t *)(uintptr_t) + ((uint8_t *)list_node - offsetof(event_prealloc_hdr_t, list_node)); + + return likely(list_node != NULL) ? ev_hdr : NULL; +} + +/** + * @brief Convert event vector table content to odp packets in-place. + * + * Convert an EM event vector table, containing em_event_t:s with + * esv-info (evgen), to a table of odp packets (remove handles' evgen in-place). + */ +static inline void +vector_tbl2odp(odp_event_t odp_event_pktvec) +{ + odp_packet_vector_t pkt_vec = odp_packet_vector_from_event(odp_event_pktvec); + odp_packet_t *pkt_tbl = NULL; + const int pkts = odp_packet_vector_tbl(pkt_vec, &pkt_tbl/*out*/); + + if (likely(pkts > 0)) { + /* Careful! Points to same table */ + em_event_t *event_tbl = (em_event_t *)pkt_tbl; + + /* Drop ESV event generation (evgen) from event handle */ + (void)events_em2pkt_inplace(event_tbl, pkts); + } +} + +/** + * @brief Convert ODP packet vector table content to EM events. + * + * Convert an ODP packet vector table to a table of EM events. + * The content must be known to be raw odp packets. + * + * For recovery purposes only. + */ +static inline void +vector_tbl2em(odp_event_t odp_event_pktvec) +{ + odp_packet_vector_t pkt_vec = odp_packet_vector_from_event(odp_event_pktvec); + odp_packet_t *pkt_tbl = NULL; + const int pkts = odp_packet_vector_tbl(pkt_vec, &pkt_tbl/*out*/); + + if (likely(pkts > 0)) { + em_event_t *const ev_tbl = (em_event_t *const)pkt_tbl; + odp_packet_t odp_pkttbl[pkts]; + event_hdr_t *ev_hdr_tbl[pkts]; + + /* + * Copy pkts from vector's pkt-table using events_em2pkt() that + * also drops any evgen-info from the handles if present. + */ + events_em2pkt(ev_tbl/*in*/, odp_pkttbl/*out*/, pkts); + + event_init_pkt_multi(odp_pkttbl /*in*/, ev_tbl /*in,out*/, + ev_hdr_tbl /*out*/, pkts, false); + } +} + +static inline em_status_t +send_event(em_event_t event, const queue_elem_t *q_elem) +{ + const bool esv_ena = esv_enabled(); + odp_event_t odp_event = event_em2odp(event); + odp_queue_t odp_queue = q_elem->odp_queue; + int ret; + + if (unlikely(EM_CHECK_LEVEL > 1 && + (odp_event == ODP_EVENT_INVALID || + odp_queue == ODP_QUEUE_INVALID))) + return EM_ERR_NOT_FOUND; + + if (unlikely(EM_CHECK_LEVEL > 0 && + q_elem->state != EM_QUEUE_STATE_READY)) { + return EM_ERR_BAD_STATE; + } + + /* + * Vector: convert the event vector table to a table of odp packets + * (in-place) before passing the vector and contents to the scheduler. + */ + if (esv_ena && odp_event_type(odp_event) == ODP_EVENT_PACKET_VECTOR) + vector_tbl2odp(odp_event); + + /* Enqueue event for scheduling */ + ret = odp_queue_enq(odp_queue, odp_event); + + if (unlikely(EM_CHECK_LEVEL > 0 && ret != 0)) { + /* Restore EM vector event-table before returning vector to user */ + if (esv_ena && odp_event_type(odp_event) == ODP_EVENT_PACKET_VECTOR) + vector_tbl2em(odp_event); + + return EM_ERR_LIB_FAILED; + } + + return EM_OK; +} + +static inline int +send_event_multi(const em_event_t events[], const int num, + const queue_elem_t *q_elem) +{ + const bool esv_ena = esv_enabled(); + odp_event_t odp_events[num]; + odp_queue_t odp_queue = q_elem->odp_queue; + + if (unlikely(EM_CHECK_LEVEL > 1 && odp_queue == ODP_QUEUE_INVALID)) + return 0; + + if (unlikely(EM_CHECK_LEVEL > 0 && + q_elem->state != EM_QUEUE_STATE_READY)) { + return 0; + } + + events_em2odp(events, odp_events/*out*/, num); + + /* + * Vector: convert the event vector table to a table of odp packets + * (in-place) before passing the vector and contents to the scheduler. + */ + if (esv_ena) { + for (int i = 0; i < num; i++) { + if (odp_event_type(odp_events[i]) == ODP_EVENT_PACKET_VECTOR) + vector_tbl2odp(odp_events[i]); + } + } + + /* Enqueue events for scheduling */ + int ret = odp_queue_enq_multi(odp_queue, odp_events, num); + + if (likely(ret == num)) + return num; /* Success! */ + + /* + * Fail: could not enqueue all events (ret != num) + */ + int enq = ret < 0 ? 0 : ret; + + /* Restore EM vector event-table before returning vector to user */ + if (esv_ena) { + for (int i = enq; i < num; i++) { + if (odp_event_type(odp_events[i]) == ODP_EVENT_PACKET_VECTOR) + vector_tbl2em(odp_events[i]); + } + } + + return enq; /* enq < num */ +} + +static inline em_status_t +send_local(em_event_t event, const queue_elem_t *q_elem) +{ + em_locm_t *const locm = &em_locm; + const em_queue_prio_t prio = q_elem->priority; + evhdl_t evhdl = {.event = event}; + int ret; + + if (unlikely(EM_CHECK_LEVEL > 0 && + q_elem->state != EM_QUEUE_STATE_READY)) + return EM_ERR_BAD_STATE; + + em_queue_t queue = (em_queue_t)(uintptr_t)q_elem->queue; + stash_entry_t entry = {.qidx = queue_hdl2idx(queue), + .evptr = evhdl.evptr}; + + ret = odp_stash_put_u64(locm->local_queues.prio[prio].stash, + &entry.u64, 1); + if (likely(ret == 1)) { + locm->local_queues.empty = 0; + locm->local_queues.prio[prio].empty_prio = 0; + return EM_OK; + } + + return EM_ERR_LIB_FAILED; +} + +static inline int +send_local_multi(const em_event_t events[], const int num, + const queue_elem_t *q_elem) +{ + em_locm_t *const locm = &em_locm; + const em_queue_prio_t prio = q_elem->priority; + const evhdl_t *const evhdl_tbl = (const evhdl_t *const)events; + + if (unlikely(EM_CHECK_LEVEL > 0 && + q_elem->state != EM_QUEUE_STATE_READY)) + return 0; + + stash_entry_t entry_tbl[num]; + em_queue_t queue = (em_queue_t)(uintptr_t)q_elem->queue; + const uint16_t qidx = (uint16_t)queue_hdl2idx(queue); + + for (int i = 0; i < num; i++) { + entry_tbl[i].qidx = qidx; + entry_tbl[i].evptr = evhdl_tbl[i].evptr; + } + + int ret = odp_stash_put_u64(locm->local_queues.prio[prio].stash, + &entry_tbl[0].u64, num); + if (likely(ret > 0)) { + locm->local_queues.empty = 0; + locm->local_queues.prio[prio].empty_prio = 0; + return ret; + } + + return 0; +} + +/** + * Send one event to a queue of type EM_QUEUE_TYPE_OUTPUT + */ +static inline em_status_t +send_output(em_event_t event, queue_elem_t *const output_q_elem) +{ + const em_sched_context_type_t sched_ctx_type = + em_locm.current.sched_context_type; + + if (unlikely(EM_CHECK_LEVEL > 0 && + output_q_elem->state != EM_QUEUE_STATE_UNSCHEDULED)) + return EM_ERR_BAD_STATE; + + /* + * An event sent to an output queue from an ordered context needs to + * be 're-ordered' before calling the user provided output-function. + * Order is maintained by enqueuing and dequeuing into an odp-queue + * that takes care of order. + */ + if (sched_ctx_type == EM_SCHED_CONTEXT_TYPE_ORDERED) { + const odp_queue_t odp_queue = output_q_elem->odp_queue; + odp_event_t odp_event = event_em2odp(event); + int ret; + + if (unlikely(EM_CHECK_LEVEL > 1 && + (odp_event == ODP_EVENT_INVALID || + odp_queue == ODP_QUEUE_INVALID))) + return EM_ERR_NOT_FOUND; + + if (!EM_OUTPUT_QUEUE_IMMEDIATE) + output_queue_track(output_q_elem); + + /* enqueue to enforce odp to handle ordering */ + ret = odp_queue_enq(odp_queue, odp_event); + if (unlikely(ret != 0)) + return EM_ERR_LIB_FAILED; + + /* return value must be EM_OK after this since event enqueued */ + + if (EM_OUTPUT_QUEUE_IMMEDIATE) { + env_spinlock_t *const lock = + &output_q_elem->output.lock; + + if (!env_spinlock_trylock(lock)) + return EM_OK; + output_queue_drain(output_q_elem); + env_spinlock_unlock(lock); + } + + return EM_OK; + } + + /* + * No ordered context - call output_fn() directly + */ + const em_queue_t output_queue = (em_queue_t)(uintptr_t)output_q_elem->queue; + const em_output_func_t output_fn = + output_q_elem->output.output_conf.output_fn; + void *const output_fn_args = + output_q_elem->output.output_conf.output_fn_args; + int sent; + + sent = output_fn(&event, 1, output_queue, output_fn_args); + if (unlikely(sent != 1)) + return EM_ERR_OPERATION_FAILED; + + return EM_OK; +} + +/** + * Send events to a queue of type EM_QUEUE_TYPE_OUTPUT + */ +static inline int +send_output_multi(const em_event_t events[], const unsigned int num, + queue_elem_t *const output_q_elem) +{ + const em_sched_context_type_t sched_ctx_type = + em_locm.current.sched_context_type; + int sent; + + if (unlikely(EM_CHECK_LEVEL > 0 && + output_q_elem->state != EM_QUEUE_STATE_UNSCHEDULED)) + return 0; + + /* + * Event sent to an output queue from an ordered context needs to + * be 're-ordered' before calling the user provided output-function. + * Order is maintained by enqueuing and dequeuing into an odp-queue + * that takes care of order. + */ + if (sched_ctx_type == EM_SCHED_CONTEXT_TYPE_ORDERED) { + const odp_queue_t odp_queue = output_q_elem->odp_queue; + odp_event_t odp_events[num]; + + if (unlikely(EM_CHECK_LEVEL > 1 && + odp_queue == ODP_QUEUE_INVALID)) + return 0; + + if (!EM_OUTPUT_QUEUE_IMMEDIATE) + output_queue_track(output_q_elem); + + events_em2odp(events, odp_events/*out*/, num); + + /* enqueue to enforce odp to handle ordering */ + sent = odp_queue_enq_multi(odp_queue, odp_events, num); + if (unlikely(sent <= 0)) + return 0; + + /* the return value must be the number of enqueued events */ + + if (EM_OUTPUT_QUEUE_IMMEDIATE) { + env_spinlock_t *const lock = + &output_q_elem->output.lock; + + if (!env_spinlock_trylock(lock)) + return sent; + output_queue_drain(output_q_elem); + env_spinlock_unlock(lock); + } + + return sent; + } + + /* + * No ordered context - call output_fn() directly + */ + const em_queue_t output_queue = (em_queue_t)(uintptr_t)output_q_elem->queue; + const em_output_func_t output_fn = output_q_elem->output.output_conf.output_fn; + void *const output_fn_args = output_q_elem->output.output_conf.output_fn_args; + + sent = output_fn(events, num, output_queue, output_fn_args); + + return sent; +} + +/** + * Return a pointer to the EM event user payload. + * Helper to e.g. EM API em_event_pointer() + */ +static inline void * +event_pointer(em_event_t event) +{ + const odp_event_t odp_event = event_em2odp(event); + const odp_event_type_t odp_etype = odp_event_type(odp_event); + void *ev_ptr = NULL; /* return value */ + + if (odp_etype == ODP_EVENT_PACKET) { + const odp_packet_t odp_pkt = odp_packet_from_event(odp_event); + + ev_ptr = odp_packet_data(odp_pkt); + } else if (odp_etype == ODP_EVENT_BUFFER) { + const odp_buffer_t odp_buf = odp_buffer_from_event(odp_event); + const event_hdr_t *ev_hdr = odp_buffer_user_area(odp_buf); + const uint32_t align_offset = ev_hdr->align_offset; + + ev_ptr = odp_buffer_addr(odp_buf); + + if (align_offset) + ev_ptr = (void *)((uintptr_t)ev_ptr + 32 - align_offset); + } + + return ev_ptr; /* NULL for unrecognized odp_etype, also for vectors */ +} + +static inline bool +event_has_ref(em_event_t event) +{ + odp_event_t odp_event = event_em2odp(event); + odp_event_type_t odp_etype = odp_event_type(odp_event); + + if (odp_etype != ODP_EVENT_PACKET) + return false; + + odp_packet_t odp_pkt = odp_packet_from_event(odp_event); + + return odp_packet_has_ref(odp_pkt) ? true : false; +} + +#ifdef __cplusplus +} +#endif + +#endif /* EM_EVENT_H_ */ diff --git a/src/event_machine_event.c b/src/event_machine_event.c index aa4b1b75..b6e44a16 100644 --- a/src/event_machine_event.c +++ b/src/event_machine_event.c @@ -1361,7 +1361,8 @@ static em_event_t event_clone_part(em_event_t event, em_pool_t pool/*or EM_POOL_ * Not an EM-pool, e.g. event from external pktio odp-pool. * Allocate and clone pkt via ODP directly. */ - clone_event = pkt_clone_odp(pkt, odp_pool, offset, size, is_clone_part); + clone_event = pkt_clone_odp(pkt, odp_pool, offset, size, + clone_uarea, is_clone_part); if (unlikely(clone_event == EM_EVENT_UNDEF)) { INTERNAL_ERROR(EM_ERR_OPERATION_FAILED, escope, "Cloning from ext odp-pool:%" PRIu64 " failed",