10#ifndef _RTE_RING_HTS_ELEM_PVT_H_
11#define _RTE_RING_HTS_ELEM_PVT_H_
25__rte_ring_hts_update_tail(
struct rte_ring_hts_headtail *ht, uint32_t old_tail,
26 uint32_t num, uint32_t enqueue)
32 tail = old_tail + num;
33 __atomic_store_n(&ht->ht.pos.tail, tail, __ATOMIC_RELEASE);
42__rte_ring_hts_head_wait(
const struct rte_ring_hts_headtail *ht,
43 union __rte_ring_hts_pos *p)
45 while (p->pos.head != p->pos.tail) {
47 p->raw = __atomic_load_n(&ht->ht.raw, __ATOMIC_ACQUIRE);
55__rte_ring_hts_move_prod_head(
struct rte_ring *r,
unsigned int num,
57 uint32_t *free_entries)
60 union __rte_ring_hts_pos np, op;
62 const uint32_t capacity = r->
capacity;
64 op.raw = __atomic_load_n(&r->hts_prod.ht.raw, __ATOMIC_ACQUIRE);
75 __rte_ring_hts_head_wait(&r->hts_prod, &op);
83 *free_entries = capacity + r->cons.
tail - op.pos.head;
93 np.pos.tail = op.pos.tail;
94 np.pos.head = op.pos.head + n;
101 }
while (__atomic_compare_exchange_n(&r->hts_prod.ht.raw,
103 0, __ATOMIC_ACQUIRE, __ATOMIC_ACQUIRE) == 0);
105 *old_head = op.pos.head;
113__rte_ring_hts_move_cons_head(
struct rte_ring *r,
unsigned int num,
118 union __rte_ring_hts_pos np, op;
120 op.raw = __atomic_load_n(&r->hts_cons.ht.raw, __ATOMIC_ACQUIRE);
132 __rte_ring_hts_head_wait(&r->hts_cons, &op);
139 *entries = r->prod.
tail - op.pos.head;
148 np.pos.tail = op.pos.tail;
149 np.pos.head = op.pos.head + n;
156 }
while (__atomic_compare_exchange_n(&r->hts_cons.ht.raw,
158 0, __ATOMIC_ACQUIRE, __ATOMIC_ACQUIRE) == 0);
160 *old_head = op.pos.head;
187__rte_ring_do_hts_enqueue_elem(
struct rte_ring *r,
const void *obj_table,
189 uint32_t *free_space)
193 n = __rte_ring_hts_move_prod_head(r, n, behavior, &head, &free);
196 __rte_ring_enqueue_elems(r, head, obj_table, esize, n);
197 __rte_ring_hts_update_tail(&r->hts_prod, head, n, 1);
200 if (free_space != NULL)
201 *free_space = free - n;
228__rte_ring_do_hts_dequeue_elem(
struct rte_ring *r,
void *obj_table,
232 uint32_t entries, head;
234 n = __rte_ring_hts_move_cons_head(r, n, behavior, &head, &entries);
237 __rte_ring_dequeue_elems(r, head, obj_table, esize, n);
238 __rte_ring_hts_update_tail(&r->hts_cons, head, n, 0);
241 if (available != NULL)
242 *available = entries - n;
#define __rte_always_inline
static void rte_pause(void)