From 41c07166178416d5578a615702d69009f3df95b4 Mon Sep 17 00:00:00 2001 From: Radha Krishna Simha Jiguru Date: Sat, 4 Apr 2020 11:05:48 +0530 Subject: [PATCH] qcacmn: Add a change to do fast transmit completion Lot of checks in Tx completion path are for special handlings such as when ol stats are enabled some protective debug checks etc.. Add a logic to fast free of buffer at transmit completion. If extended stats not enabled(Typically needed for enterprise cases) do the buffer free faster. This is controlled through a flag. Change-Id: I04873b5e3643d8e93e5b248fcaf23504dcb7624f --- dp/wifi3.0/dp_tx.c | 192 +++++++++++++++++++++++++++------------- dp/wifi3.0/dp_tx.h | 3 +- dp/wifi3.0/dp_tx_desc.h | 18 ++++ dp/wifi3.0/dp_types.h | 10 ++- hal/wifi3.0/hal_tx.h | 36 ++++++++ 5 files changed, 195 insertions(+), 64 deletions(-) diff --git a/dp/wifi3.0/dp_tx.c b/dp/wifi3.0/dp_tx.c index bdda0a3b3408..61ad1218161a 100644 --- a/dp/wifi3.0/dp_tx.c +++ b/dp/wifi3.0/dp_tx.c @@ -262,6 +262,8 @@ dp_tx_desc_release(struct dp_tx_desc_s *tx_desc, uint8_t desc_pool_id) soc = pdev->soc; + dp_tx_outstanding_dec(pdev); + if (tx_desc->frm_type == dp_tx_frm_tso) dp_tx_tso_desc_release(soc, tx_desc); @@ -271,8 +273,6 @@ dp_tx_desc_release(struct dp_tx_desc_s *tx_desc, uint8_t desc_pool_id) if (tx_desc->flags & DP_TX_DESC_FLAG_ME) dp_tx_me_free_buf(tx_desc->pdev, tx_desc->me_buffer); - dp_tx_outstanding_dec(pdev); - if (tx_desc->flags & DP_TX_DESC_FLAG_TO_FW) qdf_atomic_dec(&pdev->num_tx_exception); @@ -1114,10 +1114,8 @@ static QDF_STATUS dp_tx_hw_enqueue(struct dp_soc *soc, struct dp_vdev *vdev, *tx_exc_metadata) { uint8_t type; - uint16_t length; void *hal_tx_desc; uint32_t *hal_tx_desc_cached; - qdf_dma_addr_t dma_addr; /* * Setting it initialization statically here to avoid @@ -1144,19 +1142,20 @@ static QDF_STATUS dp_tx_hw_enqueue(struct dp_soc *soc, struct dp_vdev *vdev, hal_tx_desc_cached = (void *) cached_desc; if (tx_desc->flags & DP_TX_DESC_FLAG_FRAG) { - length = HAL_TX_EXT_DESC_WITH_META_DATA; + tx_desc->length = HAL_TX_EXT_DESC_WITH_META_DATA; type = HAL_TX_BUF_TYPE_EXT_DESC; - dma_addr = tx_desc->msdu_ext_desc->paddr; + tx_desc->dma_addr = tx_desc->msdu_ext_desc->paddr; } else { - length = qdf_nbuf_len(tx_desc->nbuf) - tx_desc->pkt_offset; + tx_desc->length = qdf_nbuf_len(tx_desc->nbuf) - + tx_desc->pkt_offset; type = HAL_TX_BUF_TYPE_BUFFER; - dma_addr = qdf_nbuf_mapped_paddr_get(tx_desc->nbuf); + tx_desc->dma_addr = qdf_nbuf_mapped_paddr_get(tx_desc->nbuf); } - qdf_assert_always(dma_addr); + qdf_assert_always(tx_desc->dma_addr); hal_tx_desc_set_buf_addr(soc->hal_soc, hal_tx_desc_cached, - dma_addr, bm_id, tx_desc->id, + tx_desc->dma_addr, bm_id, tx_desc->id, type); hal_tx_desc_set_lmac_id(soc->hal_soc, hal_tx_desc_cached, vdev->lmac_id); @@ -1173,7 +1172,7 @@ static QDF_STATUS dp_tx_hw_enqueue(struct dp_soc *soc, struct dp_vdev *vdev, (vdev->bss_ast_hash & 0xF)); hal_tx_desc_set_fw_metadata(hal_tx_desc_cached, fw_metadata); - hal_tx_desc_set_buf_length(hal_tx_desc_cached, length); + hal_tx_desc_set_buf_length(hal_tx_desc_cached, tx_desc->length); hal_tx_desc_set_buf_offset(hal_tx_desc_cached, tx_desc->pkt_offset); hal_tx_desc_set_encap_type(hal_tx_desc_cached, tx_desc->tx_encap_type); hal_tx_desc_set_addr_search_flags(hal_tx_desc_cached, @@ -1200,7 +1199,7 @@ static QDF_STATUS dp_tx_hw_enqueue(struct dp_soc *soc, struct dp_vdev *vdev, tx_desc->timestamp = qdf_ktime_to_ms(qdf_ktime_get()); dp_verbose_debug("length:%d , type = %d, dma_addr %llx, offset %d desc id %u", - length, type, (uint64_t)dma_addr, + tx_desc->length, type, (uint64_t)tx_desc->dma_addr, tx_desc->pkt_offset, tx_desc->id); hal_ring_hdl = dp_tx_get_hal_ring_hdl(soc, ring_id); @@ -1227,7 +1226,7 @@ static QDF_STATUS dp_tx_hw_enqueue(struct dp_soc *soc, struct dp_vdev *vdev, tx_desc->flags |= DP_TX_DESC_FLAG_QUEUED_TX; dp_vdev_peer_stats_update_protocol_cnt_tx(vdev, tx_desc->nbuf); hal_tx_desc_sync(hal_tx_desc_cached, hal_tx_desc); - DP_STATS_INC_PKT(vdev, tx_i.processed, 1, length); + DP_STATS_INC_PKT(vdev, tx_i.processed, 1, tx_desc->length); status = QDF_STATUS_SUCCESS; ring_access_fail: @@ -1703,6 +1702,9 @@ dp_tx_send_msdu_single(struct dp_vdev *vdev, qdf_nbuf_t nbuf, if (msdu_info->exception_fw) HTT_TX_TCL_METADATA_VALID_HTT_SET(htt_tcl_metadata, 1); + dp_tx_desc_update_fast_comp_flag(soc, tx_desc, + !pdev->enhanced_stats_en); + if (qdf_unlikely(QDF_STATUS_SUCCESS != dp_tx_msdu_single_map(vdev, tx_desc, nbuf))) { /* Handle failure */ @@ -3458,13 +3460,49 @@ dp_tx_comp_process_desc_list(struct dp_soc *soc, { struct dp_tx_desc_s *desc; struct dp_tx_desc_s *next; - struct hal_tx_completion_status ts = {0}; + struct hal_tx_completion_status ts; struct dp_peer *peer; qdf_nbuf_t netbuf; desc = comp_head; while (desc) { + if (qdf_likely(desc->flags & DP_TX_DESC_FLAG_SIMPLE)) { + struct dp_pdev *pdev = desc->pdev; + + peer = dp_peer_find_by_id(soc, desc->peer_id); + if (qdf_likely(peer)) { + /* + * Increment peer statistics + * Minimal statistics update done here + */ + DP_STATS_INC_PKT(peer, tx.comp_pkt, 1, + desc->length); + + if (desc->tx_status != + HAL_TX_TQM_RR_FRAME_ACKED) + peer->stats.tx.tx_failed++; + + dp_peer_unref_del_find_by_id(peer); + } + + qdf_assert(pdev); + dp_tx_outstanding_dec(pdev); + + /* + * Calling a QDF WRAPPER here is creating signifcant + * performance impact so avoided the wrapper call here + */ + next = desc->next; + qdf_mem_unmap_nbytes_single(soc->osdev, + desc->dma_addr, + QDF_DMA_TO_DEVICE, + desc->length); + qdf_nbuf_free(desc->nbuf); + dp_tx_desc_free(soc, desc, desc->pool_id); + desc = next; + continue; + } hal_tx_comp_get_status(&desc->comp, &ts, soc->hal_soc); peer = dp_peer_find_by_id(soc, ts.peer_id); dp_tx_comp_process_tx_status(desc, &ts, peer, ring_id); @@ -3511,12 +3549,35 @@ void dp_tx_process_htt_completion(struct dp_tx_desc_s *tx_desc, uint8_t *status, struct cdp_tid_tx_stats *tid_stats = NULL; struct htt_soc *htt_handle; - qdf_assert(tx_desc->pdev); + /* + * If the descriptor is already freed in vdev_detach, + * continue to next descriptor + */ + if (!tx_desc->vdev && !tx_desc->flags) { + QDF_TRACE(QDF_MODULE_ID_DP, + QDF_TRACE_LEVEL_INFO, + "Descriptor freed in vdev_detach %d", + tx_desc->id); + return; + } pdev = tx_desc->pdev; - vdev = tx_desc->vdev; soc = pdev->soc; + if (qdf_unlikely(tx_desc->pdev->is_pdev_down)) { + QDF_TRACE(QDF_MODULE_ID_DP, + QDF_TRACE_LEVEL_INFO, + "pdev in down state %d", + tx_desc->id); + dp_tx_comp_free_buf(soc, tx_desc); + dp_tx_desc_release(tx_desc, tx_desc->pool_id); + return; + } + + qdf_assert(tx_desc->pdev); + + vdev = tx_desc->vdev; + if (!vdev) return; tx_status = HTT_TX_WBM_COMPLETION_V2_TX_STATUS_GET(htt_desc[0]); @@ -3648,8 +3709,9 @@ uint32_t dp_tx_comp_handler(struct dp_intr *int_ctx, struct dp_soc *soc, more_data: /* Re-initialize local variables to be re-used */ - head_desc = NULL; - tail_desc = NULL; + head_desc = NULL; + tail_desc = NULL; + if (qdf_unlikely(dp_srng_access_start(int_ctx, soc, hal_ring_hdl))) { dp_err("HAL RING Access Failed -- %pK", hal_ring_hdl); return 0; @@ -3664,6 +3726,7 @@ more_data: /* Find head descriptor from completion ring */ while (qdf_likely(num_avail_for_reap)) { + tx_comp_hal_desc = dp_srng_dst_get_next(soc, hal_ring_hdl); if (qdf_unlikely(!tx_comp_hal_desc)) break; @@ -3672,8 +3735,10 @@ more_data: /* If this buffer was not released by TQM or FW, then it is not * Tx completion indication, assert */ - if ((buffer_src != HAL_TX_COMP_RELEASE_SOURCE_TQM) && - (buffer_src != HAL_TX_COMP_RELEASE_SOURCE_FW)) { + if (qdf_unlikely(buffer_src != + HAL_TX_COMP_RELEASE_SOURCE_TQM) && + (qdf_unlikely(buffer_src != + HAL_TX_COMP_RELEASE_SOURCE_FW))) { uint8_t wbm_internal_error; dp_err_rl( @@ -3725,35 +3790,6 @@ more_data: (tx_desc_id & DP_TX_DESC_ID_OFFSET_MASK) >> DP_TX_DESC_ID_OFFSET_OS); - /* - * If the descriptor is already freed in vdev_detach, - * continue to next descriptor - */ - if (!tx_desc->vdev && !tx_desc->flags) { - QDF_TRACE(QDF_MODULE_ID_DP, - QDF_TRACE_LEVEL_INFO, - "Descriptor freed in vdev_detach %d", - tx_desc_id); - - num_processed += !(count & DP_TX_NAPI_BUDGET_DIV_MASK); - count++; - continue; - } - - if (qdf_unlikely(tx_desc->pdev->is_pdev_down)) { - QDF_TRACE(QDF_MODULE_ID_DP, - QDF_TRACE_LEVEL_INFO, - "pdev in down state %d", - tx_desc_id); - - num_processed += !(count & DP_TX_NAPI_BUDGET_DIV_MASK); - count++; - - dp_tx_comp_free_buf(soc, tx_desc); - dp_tx_desc_release(tx_desc, tx_desc->pool_id); - continue; - } - /* * If the release source is FW, process the HTT status */ @@ -3765,6 +3801,43 @@ more_data: dp_tx_process_htt_completion(tx_desc, htt_tx_status, ring_id); } else { + /* + * If the fast completion mode is enabled extended + * metadata from descriptor is not copied + */ + if (qdf_likely(tx_desc->flags & + DP_TX_DESC_FLAG_SIMPLE)) { + tx_desc->peer_id = + hal_tx_comp_get_peer_id(tx_comp_hal_desc); + tx_desc->tx_status = + hal_tx_comp_get_tx_status(tx_comp_hal_desc); + goto add_to_pool; + } + + /* + * If the descriptor is already freed in vdev_detach, + * continue to next descriptor + */ + if (qdf_unlikely(!tx_desc->vdev) && + qdf_unlikely(!tx_desc->flags)) { + QDF_TRACE(QDF_MODULE_ID_DP, + QDF_TRACE_LEVEL_INFO, + "Descriptor freed in vdev_detach %d", + tx_desc_id); + continue; + } + + if (qdf_unlikely(tx_desc->pdev->is_pdev_down)) { + QDF_TRACE(QDF_MODULE_ID_DP, + QDF_TRACE_LEVEL_INFO, + "pdev in down state %d", + tx_desc_id); + + dp_tx_comp_free_buf(soc, tx_desc); + dp_tx_desc_release(tx_desc, tx_desc->pool_id); + goto next_desc; + } + /* Pool id is not matching. Error */ if (tx_desc->pool_id != pool_id) { QDF_TRACE(QDF_MODULE_ID_DP, @@ -3778,12 +3851,18 @@ more_data: if (!(tx_desc->flags & DP_TX_DESC_FLAG_ALLOCATED) || !(tx_desc->flags & DP_TX_DESC_FLAG_QUEUED_TX)) { QDF_TRACE(QDF_MODULE_ID_DP, - QDF_TRACE_LEVEL_FATAL, - "Txdesc invalid, flgs = %x,id = %d", - tx_desc->flags, tx_desc_id); + QDF_TRACE_LEVEL_FATAL, + "Txdesc invalid, flgs = %x,id = %d", + tx_desc->flags, tx_desc_id); qdf_assert_always(0); } + /* Collect hw completion contents */ + hal_tx_comp_desc_sync(tx_comp_hal_desc, + &tx_desc->comp, 1); +add_to_pool: + DP_HIST_PACKET_COUNT_INC(tx_desc->pdev->pdev_id); + /* First ring descriptor on the cycle */ if (!head_desc) { head_desc = tx_desc; @@ -3793,15 +3872,8 @@ more_data: tail_desc->next = tx_desc; tx_desc->next = NULL; tail_desc = tx_desc; - - DP_HIST_PACKET_COUNT_INC(tx_desc->pdev->pdev_id); - - /* Collect hw completion contents */ - hal_tx_comp_desc_sync(tx_comp_hal_desc, - &tx_desc->comp, 1); - } - +next_desc: num_processed += !(count & DP_TX_NAPI_BUDGET_DIV_MASK); /* diff --git a/dp/wifi3.0/dp_tx.h b/dp/wifi3.0/dp_tx.h index 1850ec624b01..5caaca1f170b 100644 --- a/dp/wifi3.0/dp_tx.h +++ b/dp/wifi3.0/dp_tx.h @@ -25,7 +25,7 @@ #define DP_TX_MAX_NUM_FRAGS 6 -#define DP_TX_DESC_FLAG_ALLOCATED 0x1 +#define DP_TX_DESC_FLAG_SIMPLE 0x1 #define DP_TX_DESC_FLAG_TO_FW 0x2 #define DP_TX_DESC_FLAG_FRAG 0x4 #define DP_TX_DESC_FLAG_RAW 0x8 @@ -34,6 +34,7 @@ #define DP_TX_DESC_FLAG_COMPLETED_TX 0x40 #define DP_TX_DESC_FLAG_ME 0x80 #define DP_TX_DESC_FLAG_TDLS_FRAME 0x100 +#define DP_TX_DESC_FLAG_ALLOCATED 0x200 #define DP_TX_FREE_SINGLE_BUF(soc, buf) \ do { \ diff --git a/dp/wifi3.0/dp_tx_desc.h b/dp/wifi3.0/dp_tx_desc.h index c281f8c96986..7403d9e860d2 100644 --- a/dp/wifi3.0/dp_tx_desc.h +++ b/dp/wifi3.0/dp_tx_desc.h @@ -726,6 +726,24 @@ dp_tx_is_desc_id_valid(struct dp_soc *soc, uint32_t tx_desc_id) } #endif /* QCA_DP_TX_DESC_ID_CHECK */ +#ifdef QCA_DP_TX_DESC_FAST_COMP_ENABLE +static inline void dp_tx_desc_update_fast_comp_flag(struct dp_soc *soc, + struct dp_tx_desc_s *desc, + uint8_t allow_fast_comp) +{ + if (qdf_likely(!(desc->flags & DP_TX_DESC_FLAG_TO_FW)) && + qdf_likely(allow_fast_comp)) { + desc->flags |= DP_TX_DESC_FLAG_SIMPLE; + } +} +#else +static inline void dp_tx_desc_update_fast_comp_flag(struct dp_soc *soc, + struct dp_tx_desc_s *desc, + uint8_t allow_fast_comp) +{ +} +#endif /* QCA_DP_TX_DESC_FAST_COMP_ENABLE */ + /** * dp_tx_desc_find() - find dp tx descriptor from cokie * @soc - handle for the device sending the data diff --git a/dp/wifi3.0/dp_types.h b/dp/wifi3.0/dp_types.h index 55488a435099..e4903f8232cf 100644 --- a/dp/wifi3.0/dp_types.h +++ b/dp/wifi3.0/dp_types.h @@ -384,15 +384,19 @@ struct dp_tx_ext_desc_pool_s { struct dp_tx_desc_s { struct dp_tx_desc_s *next; qdf_nbuf_t nbuf; + uint16_t length; + uint16_t flags; + qdf_dma_addr_t dma_addr; uint32_t id; struct dp_vdev *vdev; struct dp_pdev *pdev; - struct dp_tx_ext_desc_elem_s *msdu_ext_desc; - uint16_t flags; - uint16_t tx_encap_type; + uint8_t tx_encap_type; uint8_t frm_type; uint8_t pkt_offset; uint8_t pool_id; + uint16_t peer_id; + uint16_t tx_status; + struct dp_tx_ext_desc_elem_s *msdu_ext_desc; void *me_buffer; void *tso_desc; void *tso_num_desc; diff --git a/hal/wifi3.0/hal_tx.h b/hal/wifi3.0/hal_tx.h index 4b35d577704e..044fe2fa6350 100644 --- a/hal/wifi3.0/hal_tx.h +++ b/hal/wifi3.0/hal_tx.h @@ -804,6 +804,42 @@ uint8_t hal_tx_comp_get_release_reason(void *hal_desc, return hal_soc->ops->hal_tx_comp_get_release_reason(hal_desc); } +/** + * hal_tx_comp_get_peer_id() - Get peer_id value() + * @hal_desc: completion ring descriptor pointer + * + * This function will get peer_id value from Tx completion descriptor + * + * Return: buffer release source + */ +static inline uint16_t hal_tx_comp_get_peer_id(void *hal_desc) +{ + uint32_t comp_desc = + *(uint32_t *)(((uint8_t *)hal_desc) + + WBM_RELEASE_RING_7_SW_PEER_ID_OFFSET); + + return (comp_desc & WBM_RELEASE_RING_7_SW_PEER_ID_MASK) >> + WBM_RELEASE_RING_7_SW_PEER_ID_LSB; +} + +/** + * hal_tx_comp_get_tx_status() - Get tx transmission status() + * @hal_desc: completion ring descriptor pointer + * + * This function will get transmit status value from Tx completion descriptor + * + * Return: buffer release source + */ +static inline uint8_t hal_tx_comp_get_tx_status(void *hal_desc) +{ + uint32_t comp_desc = + *(uint32_t *)(((uint8_t *)hal_desc) + + WBM_RELEASE_RING_2_TQM_RELEASE_REASON_OFFSET); + + return (comp_desc & WBM_RELEASE_RING_2_TQM_RELEASE_REASON_MASK) >> + WBM_RELEASE_RING_2_TQM_RELEASE_REASON_LSB; +} + /** * hal_tx_comp_desc_sync() - collect hardware descriptor contents * @hal_desc: hardware descriptor pointer