Time_squeeze和Netdev_budget

# sysctl net.core.netdev_budget net.core.netdev_budget = 300

// /home/kangxiaoning/workspace/kernel-4.19.90-2404.2.0/net/core/dev.c static __latent_entropy void net_rx_action(struct softirq_action *h) { struct softnet_data *sd = this_cpu_ptr(&softnet_data); unsigned long time_limit = jiffies + usecs_to_jiffies(READ_ONCE(netdev_budget_usecs)); int budget = READ_ONCE(netdev_budget); LIST_HEAD(list); LIST_HEAD(repoll); local_irq_disable(); list_splice_init(&sd->poll_list, &list); local_irq_enable(); for (;;) { struct napi_struct *n; if (list_empty(&list)) { if (!sd_has_rps_ipi_waiting(sd) && list_empty(&repoll)) goto out; break; } n = list_first_entry(&list, struct napi_struct, poll_list); budget -= napi_poll(n, &repoll); /* If softirq window is exhausted then punt. * Allow this to run for 2 jiffies since which will allow * an average latency of 1.5/HZ. */ if (unlikely(budget <= 0 || time_after_eq(jiffies, time_limit))) { sd->time_squeeze++; break; } } local_irq_disable(); list_splice_tail_init(&sd->poll_list, &list); list_splice_tail(&repoll, &list); list_splice(&list, &sd->poll_list); if (!list_empty(&sd->poll_list)) __raise_softirq_irqoff(NET_RX_SOFTIRQ); net_rps_action_and_irq_enable(sd); out: __kfree_skb_flush(); }

netif_napi_add(netdev, &c->napi, mlx5e_napi_poll, 64);

// /home/kangxiaoning/workspace/kernel-4.19.90-2404.2.0/drivers/net/ethernet/mellanox/mlx5/core/en_txrx.c int mlx5e_napi_poll(struct napi_struct *napi, int budget) { struct mlx5e_channel *c = container_of(napi, struct mlx5e_channel, napi); struct mlx5e_ch_stats *ch_stats = c->stats; bool busy = false; int work_done = 0; int i; ch_stats->poll++; for (i = 0; i < c->num_tc; i++) busy |= mlx5e_poll_tx_cq(&c->sq[i].cq, budget); // 清理发送队列的ring buffer busy |= mlx5e_poll_xdpsq_cq(&c->xdpsq.cq); if (c->xdp) busy |= mlx5e_poll_xdpsq_cq(&c->rq.xdpsq.cq); if (likely(budget)) { /* budget=0 means: don't poll rx rings */ work_done = mlx5e_poll_rx_cq(&c->rq.cq, budget); // 处理接收队列的数据包 busy |= work_done == budget; } busy |= c->rq.post_wqes(&c->rq); if (busy) { if (likely(mlx5e_channel_no_affinity_change(c))) return budget; ch_stats->aff_change++; if (budget && work_done == budget) work_done--; } if (unlikely(!napi_complete_done(napi, work_done))) return work_done; ch_stats->arm++; for (i = 0; i < c->num_tc; i++) { mlx5e_handle_tx_dim(&c->sq[i]); mlx5e_cq_arm(&c->sq[i].cq); } mlx5e_handle_rx_dim(&c->rq); mlx5e_cq_arm(&c->rq.cq); mlx5e_cq_arm(&c->icosq.cq); mlx5e_cq_arm(&c->xdpsq.cq); return work_done; }

#define MLX5E_TX_CQ_POLL_BUDGET 128

// /home/kangxiaoning/workspace/kernel-4.19.90-2404.2.0/drivers/net/ethernet/mellanox/mlx5/core/en_tx.c bool mlx5e_poll_tx_cq(struct mlx5e_cq *cq, int napi_budget) { struct mlx5e_sq_stats *stats; // 发送队列统计信息 struct mlx5e_txqsq *sq; // 发送队列结构体 struct mlx5_cqe64 *cqe; // 完成队列事件 u32 dma_fifo_cc; // DMA FIFO计数器 u32 nbytes; // 处理的总字节数 u16 npkts; // 处理的数据包数量 u16 sqcc; // 发送队列计数器 int i; // 循环计数 // 获取发送队列结构体 sq = container_of(cq, struct mlx5e_txqsq, cq); // 确保发送队列处于启用状态 if (unlikely(!test_bit(MLX5E_SQ_STATE_ENABLED, &sq->state))) return false; // 获取完成队列中的完成事件 cqe = mlx5_cqwq_get_cqe(&cq->wq); if (!cqe) return false; // 初始化统计信息 stats = sq->stats; npkts = 0; nbytes = 0; // 更新发送队列计数器 sqcc = sq->cc; // 避免频繁更新缓存线 dma_fifo_cc = sq->dma_fifo_cc; // 循环处理完成事件 i = 0; do { u16 wqe_counter; // 工作队列元素计数 bool last_wqe; // 是否为最后一个WQE // 移除完成事件 mlx5_cqwq_pop(&cq->wq); // 获取工作队列元素计数 wqe_counter = be16_to_cpu(cqe->wqe_counter); // 处理错误事件 if (unlikely(cqe->op_own >> 4 == MLX5_CQE_REQ_ERR)) { if (!test_and_set_bit(MLX5E_SQ_STATE_RECOVERING, &sq->state)) { // 处理错误事件并安排恢复工作 mlx5e_dump_error_cqe(sq, (struct mlx5_err_cqe *)cqe); queue_work(cq->channel->priv->wq, &sq->recover.recover_work); } stats->cqe_err++; // 统计错误事件数量 } // 遍历WQE中的所有DMA映射 do { struct mlx5e_tx_wqe_info *wi; // WQE信息 struct sk_buff *skb; // 数据包 u16 ci; // WQE索引 int j; // 内部循环计数 // 获取最后一个WQE标志 last_wqe = (sqcc == wqe_counter); // 获取WQE索引 ci = mlx5_wq_cyc_ctr2ix(&sq->wq, sqcc); wi = &sq->db.wqe_info[ci]; skb = wi->skb; // 如果是NOP，则跳过 if (unlikely(!skb)) { sqcc++; continue; } // 如果设置了硬件时间戳，则更新数据包的时间戳 if (unlikely(skb_shinfo(skb)->tx_flags & SKBTX_HW_TSTAMP)) { struct skb_shared_hwtstamps hwts = {}; hwts.hwtstamp = mlx5_timecounter_cyc2time(sq->clock, get_cqe_ts(cqe)); skb_tstamp_tx(skb, &hwts); } // 解映射DMA缓冲区，也就是清理ring buffer for (j = 0; j < wi->num_dma; j++) { struct mlx5e_sq_dma *dma = mlx5e_dma_get(sq, dma_fifo_cc++); mlx5e_tx_dma_unmap(sq->pdev, dma); } // 更新统计信息 npkts++; nbytes += wi->num_bytes; sqcc += wi->num_wqebbs; napi_consume_skb(skb, napi_budget); } while (!last_wqe); // 直到最后一个wqe完成才结束循环 } while ((++i < MLX5E_TX_CQ_POLL_BUDGET) && (cqe = mlx5_cqwq_get_cqe(&cq->wq))); // 超过MLX5E_TX_CQ_POLL_BUDGET预算，或者没有更多的cqe则结束循环 // 更新处理的事件数量 stats->cqes += i; // 更新数据库记录以释放完成队列的空间 mlx5_cqwq_update_db_record(&cq->wq); // 确保完成队列空间被释放 wmb(); // 更新DMA FIFO计数器和发送队列计数器 sq->dma_fifo_cc = dma_fifo_cc; sq->cc = sqcc; // 唤醒发送队列 netdev_tx_completed_queue(sq->txq, npkts, nbytes); // 如果发送队列被停止且有足够的空间，则唤醒队列 if (netif_tx_queue_stopped(sq->txq) && mlx5e_wqc_has_room_for(&sq->wq, sq->cc, sq->pc, MLX5E_SQ_STOP_ROOM) && !test_bit(MLX5E_SQ_STATE_RECOVERING, &sq->state)) { netif_tx_wake_queue(sq->txq); stats->wake++; } // 返回是否达到处理预算 return (i == MLX5E_TX_CQ_POLL_BUDGET); }

// /home/kangxiaoning/workspace/kernel-4.19.90-2404.2.0/drivers/net/ethernet/mellanox/mlx5/core/en_rx.c int mlx5e_poll_rx_cq(struct mlx5e_cq *cq, int budget) { struct mlx5e_rq *rq = container_of(cq, struct mlx5e_rq, cq); // 获取与CQ关联的接收队列 struct mlx5e_xdpsq *xdpsq = &rq->xdpsq; struct mlx5_cqe64 *cqe; // 定义CQE指针 int work_done = 0; // 初始化已处理的工作量 // 如果接收队列未启用，则直接返回0 if (unlikely(!test_bit(MLX5E_RQ_STATE_ENABLED, &rq->state))) return 0; // 如果存在待解压缩的CQ条目 if (cq->decmprs_left) { // 处理解压缩的CQ条目，并更新已处理的工作量 work_done += mlx5e_decompress_cqes_cont(rq, cq, 0, budget); // 如果还有待解压缩的CQ条目或已达到预算限制，则结束处理 if (cq->decmprs_left || work_done >= budget) goto out; } // 获取CQ中的下一个CQE cqe = mlx5_cqwq_get_cqe(&cq->wq); // 如果没有CQE，则根据已处理的工作量情况决定是否返回 if (!cqe) { if (unlikely(work_done)) goto out; return 0; } // 循环处理CQ条目 do { // 如果CQE是压缩格式，则开始解压缩并更新已处理的工作量 if (mlx5_get_cqe_format(cqe) == MLX5_COMPRESSED) { work_done += mlx5e_decompress_cqes_start(rq, cq, budget - work_done); continue; } // 从CQ中移除当前CQE mlx5_cqwq_pop(&cq->wq); // 调用处理函数处理当前CQE，最终调用的是`mlx5e_handle_rx_cqe()` rq->handle_rx_cqe(rq, cqe); } while ((++work_done < budget) && (cqe = mlx5_cqwq_get_cqe(&cq->wq))); // 有预算并且有CQE的情况下循环处理 out: if (xdpsq->doorbell) { mlx5e_xmit_xdp_doorbell(xdpsq); xdpsq->doorbell = false; } if (xdpsq->redirect_flush) { xdp_do_flush_map(); xdpsq->redirect_flush = false; } mlx5_cqwq_update_db_record(&cq->wq); // 确保在释放CQ空间之前更新已完成的工作量 wmb(); // 返回已处理的工作量 return work_done; }

rq->handle_rx_cqe = c->priv->profile->rx_handlers.handle_rx_cqe;

static const struct mlx5e_profile mlx5e_nic_profile = { .init = mlx5e_nic_init, .cleanup = mlx5e_nic_cleanup, .init_rx = mlx5e_init_nic_rx, .cleanup_rx = mlx5e_cleanup_nic_rx, .init_tx = mlx5e_init_nic_tx, .cleanup_tx = mlx5e_cleanup_nic_tx, .enable = mlx5e_nic_enable, .disable = mlx5e_nic_disable, .update_stats = mlx5e_update_ndo_stats, .max_nch = mlx5e_get_max_num_channels, .update_carrier = mlx5e_update_carrier, .rx_handlers.handle_rx_cqe = mlx5e_handle_rx_cqe, .rx_handlers.handle_rx_cqe_mpwqe = mlx5e_handle_rx_cqe_mpwrq, .max_tc = MLX5E_MAX_NUM_TC, };

// /home/kangxiaoning/workspace/kernel-4.19.90-2404.2.0/drivers/net/ethernet/mellanox/mlx5/core/en_rx.c void mlx5e_handle_rx_cqe(struct mlx5e_rq *rq, struct mlx5_cqe64 *cqe) { struct mlx5_wq_cyc *wq = &rq->wqe.wq; // 获取请求队列中的循环工作队列结构 struct mlx5e_wqe_frag_info *wi; // 定义工作队列元素片段信息指针 struct sk_buff *skb; // 定义网络缓冲区结构指针 u32 cqe_bcnt; // 定义CQE字节计数值 u16 ci; // 定义工作队列元素计数器索引 // 从CQE中获取工作队列元素计数器索引 ci = mlx5_wq_cyc_ctr2ix(wq, be16_to_cpu(cqe->wqe_counter)); wi = get_frag(rq, ci); // 根据索引获取对应的工作队列元素片段信息 cqe_bcnt = be32_to_cpu(cqe->byte_cnt); // 从CQE中获取字节计数值 // 从工作队列元素中重构网络缓冲区skb skb = rq->wqe.skb_from_cqe(rq, cqe, wi, cqe_bcnt); if (!skb) { // 如果skb为空 /* 处理可能的XDP情况 */ if (__test_and_clear_bit(MLX5E_RQ_FLAG_XDP_XMIT, rq->flags)) { /* 不释放页面，等待XDP_TX完成后释放 */ goto wq_cyc_pop; } goto free_wqe; // 直接跳转到释放WQE } // 完成接收CQE的处理，并准备接收skb mlx5e_complete_rx_cqe(rq, cqe, cqe_bcnt, skb); napi_gro_receive(rq->cq.napi, skb); // 注册接收skb，后续就一步一步送到协议栈了 free_wqe: // 释放接收WQE mlx5e_free_rx_wqe(rq, wi, true); wq_cyc_pop: // 从工作队列中移除已处理的元素 mlx5_wq_cyc_pop(wq); }

enum { MLX5_COMP_EQ_SIZE = 1024, };

nent = MLX5_COMP_EQ_SIZE; // 省略 err = mlx5_create_map_eq(dev, eq, i + MLX5_EQ_VEC_COMP_BASE, nent, 0, name, MLX5_EQ_TYPE_COMP);

enum { MLX5_NUM_SPARE_EQE = 0x80, // 十进制为128 MLX5_NUM_ASYNC_EQE = 0x1000, MLX5_NUM_CMD_EQE = 32, MLX5_NUM_PF_DRAIN = 64, };

enum { MLX5_EQE_SIZE = sizeof(struct mlx5_eqe), MLX5_EQE_OWNER_INIT_VAL = 0x1, };

eq->type = type; // 1024+128再roundup_pow_of_two后得到2048 eq->nent = roundup_pow_of_two(nent + MLX5_NUM_SPARE_EQE); eq->cons_index = 0; // 2048*64=131072字节 err = mlx5_buf_alloc(dev, eq->nent * MLX5_EQE_SIZE, &eq->buf); if (err) return err;

static const struct neigh_ops arp_generic_ops = { .family = AF_INET, .solicit = arp_solicit, .error_report = arp_error_report, .output = neigh_resolve_output, .connected_output = neigh_connected_output, }; static const struct neigh_ops arp_hh_ops = { .family = AF_INET, .solicit = arp_solicit, .error_report = arp_error_report, .output = neigh_resolve_output, .connected_output = neigh_resolve_output, }; static const struct neigh_ops arp_direct_ops = { .family = AF_INET, .output = neigh_direct_output, .connected_output = neigh_direct_output, };