|
[Date Prev][Date Next][Thread Prev][Thread Next][Date Index][Thread Index] [Xen-devel] [7/9] [NET] back: Transmit SG packets if supported
Hi: [NET] back: Transmit SG packets if supported This patch adds scatter-and-gather transmission support to the backend. This allows the MTU to be raised right now and the potential for TSO in future. Signed-off-by: Herbert Xu <herbert@xxxxxxxxxxxxxxxxxxx> Cheers, -- Visit Openswan at http://www.openswan.org/ Email: Herbert Xu ~{PmV>HI~} <herbert@xxxxxxxxxxxxxxxxxxx> Home Page: http://gondor.apana.org.au/~herbert/ PGP Key: http://gondor.apana.org.au/~herbert/pubkey.txt -- diff -r 05d6393e9d18 -r 4f008474675a linux-2.6-xen-sparse/drivers/xen/netback/common.h --- a/linux-2.6-xen-sparse/drivers/xen/netback/common.h Fri Jul 28 18:20:45 2006 +1000 +++ b/linux-2.6-xen-sparse/drivers/xen/netback/common.h Fri Jul 28 18:22:22 2006 +1000 @@ -129,4 +129,10 @@ static inline int netbk_can_queue(struct return netif->can_queue; } +static inline int netbk_can_sg(struct net_device *dev) +{ + netif_t *netif = netdev_priv(dev); + return netif->features & NETIF_F_SG; +} + #endif /* __NETIF__BACKEND__COMMON_H__ */ diff -r 05d6393e9d18 -r 4f008474675a linux-2.6-xen-sparse/drivers/xen/netback/interface.c --- a/linux-2.6-xen-sparse/drivers/xen/netback/interface.c Fri Jul 28 18:20:45 2006 +1000 +++ b/linux-2.6-xen-sparse/drivers/xen/netback/interface.c Fri Jul 28 18:22:22 2006 +1000 @@ -62,10 +62,34 @@ static int net_close(struct net_device * return 0; } +static int netbk_change_mtu(struct net_device *dev, int mtu) +{ + int max = netbk_can_sg(dev) ? 65535 - ETH_HLEN : ETH_DATA_LEN; + + if (mtu > max) + return -EINVAL; + dev->mtu = mtu; + return 0; +} + +static int netbk_set_sg(struct net_device *dev, u32 data) +{ + if (data) { + netif_t *netif = netdev_priv(dev); + + if (!(netif->features & NETIF_F_SG)) + return -ENOSYS; + } + + return ethtool_op_set_sg(dev, data); +} + static struct ethtool_ops network_ethtool_ops = { .get_tx_csum = ethtool_op_get_tx_csum, .set_tx_csum = ethtool_op_set_tx_csum, + .get_sg = ethtool_op_get_sg, + .set_sg = netbk_set_sg, .get_link = ethtool_op_get_link, }; @@ -101,6 +125,7 @@ netif_t *netif_alloc(domid_t domid, unsi dev->get_stats = netif_be_get_stats; dev->open = net_open; dev->stop = net_close; + dev->change_mtu = netbk_change_mtu; dev->features = NETIF_F_IP_CSUM; SET_ETHTOOL_OPS(dev, &network_ethtool_ops); diff -r 05d6393e9d18 -r 4f008474675a linux-2.6-xen-sparse/drivers/xen/netback/netback.c --- a/linux-2.6-xen-sparse/drivers/xen/netback/netback.c Fri Jul 28 18:20:45 2006 +1000 +++ b/linux-2.6-xen-sparse/drivers/xen/netback/netback.c Fri Jul 28 18:22:22 2006 +1000 @@ -40,6 +40,11 @@ /*#define NETBE_DEBUG_INTERRUPT*/ +struct netbk_rx_meta { + skb_frag_t frag; + int id; +}; + static void netif_idx_release(u16 pending_idx); static void netif_page_release(struct page *page); static void make_tx_response(netif_t *netif, @@ -100,21 +105,27 @@ static unsigned long mfn_list[MAX_MFN_AL static unsigned long mfn_list[MAX_MFN_ALLOC]; static unsigned int alloc_index = 0; -static unsigned long alloc_mfn(void) -{ - unsigned long mfn = 0; +static inline unsigned long alloc_mfn(void) +{ + return mfn_list[--alloc_index]; +} + +static int check_mfn(int nr) +{ struct xen_memory_reservation reservation = { - .nr_extents = MAX_MFN_ALLOC, .extent_order = 0, .domid = DOMID_SELF }; - set_xen_guest_handle(reservation.extent_start, mfn_list); - if ( unlikely(alloc_index == 0) ) - alloc_index = HYPERVISOR_memory_op( - XENMEM_increase_reservation, &reservation); - if ( alloc_index != 0 ) - mfn = mfn_list[--alloc_index]; - return mfn; + + if (likely(alloc_index >= nr)) + return 0; + + set_xen_guest_handle(reservation.extent_start, mfn_list + alloc_index); + reservation.nr_extents = MAX_MFN_ALLOC - alloc_index; + alloc_index += HYPERVISOR_memory_op(XENMEM_increase_reservation, + &reservation); + + return alloc_index >= nr ? 0 : -ENOMEM; } static inline void maybe_schedule_tx_action(void) @@ -136,12 +147,87 @@ static inline int is_xen_skb(struct sk_b return (cp == skbuff_cachep); } +static struct sk_buff *netbk_copy_skb(struct sk_buff *skb) +{ + struct skb_shared_info *ninfo; + struct sk_buff *nskb; + unsigned long offset; + int ret; + int len; + int headlen; + + nskb = alloc_skb(SKB_MAX_HEAD(0), GFP_ATOMIC); + if (unlikely(!nskb)) + goto err; + + skb_reserve(nskb, 16); + headlen = nskb->end - nskb->data; + if (headlen > skb_headlen(skb)) + headlen = skb_headlen(skb); + ret = skb_copy_bits(skb, 0, __skb_put(nskb, headlen), headlen); + BUG_ON(ret); + + ninfo = skb_shinfo(nskb); + ninfo->gso_size = skb_shinfo(skb)->gso_size; + ninfo->gso_type = skb_shinfo(skb)->gso_type; + + offset = headlen; + len = skb->len - headlen; + + nskb->len = skb->len; + nskb->data_len = len; + nskb->truesize += len; + + while (len) { + struct page *page; + int copy; + int zero; + + if (unlikely(ninfo->nr_frags >= MAX_SKB_FRAGS)) { + dump_stack(); + goto err_free; + } + + copy = len >= PAGE_SIZE ? PAGE_SIZE : len; + zero = len >= PAGE_SIZE ? 0 : __GFP_ZERO; + + page = alloc_page(GFP_ATOMIC | zero); + if (unlikely(!page)) + goto err_free; + + ret = skb_copy_bits(skb, offset, page_address(page), copy); + BUG_ON(ret); + + ninfo->frags[ninfo->nr_frags].page = page; + ninfo->frags[ninfo->nr_frags].page_offset = 0; + ninfo->frags[ninfo->nr_frags].size = copy; + ninfo->nr_frags++; + + offset += copy; + len -= copy; + } + + offset = nskb->data - skb->data; + + nskb->h.raw = skb->h.raw + offset; + nskb->nh.raw = skb->nh.raw + offset; + nskb->mac.raw = skb->mac.raw + offset; + + return nskb; + +err_free: + kfree_skb(nskb); +err: + return NULL; +} + static inline int netbk_queue_full(netif_t *netif) { RING_IDX peek = netif->rx_req_cons_peek; - return netif->rx.sring->req_prod - peek <= 0 || - netif->rx.rsp_prod_pvt + NET_RX_RING_SIZE - peek <= 0; + return netif->rx.sring->req_prod - peek <= MAX_SKB_FRAGS || + netif->rx.rsp_prod_pvt + NET_RX_RING_SIZE - peek <= + MAX_SKB_FRAGS; } int netif_be_start_xmit(struct sk_buff *skb, struct net_device *dev) @@ -163,20 +249,12 @@ int netif_be_start_xmit(struct sk_buff * * We do not copy the packet unless: * 1. The data is shared; or * 2. The data is not allocated from our special cache. - * NB. We also couldn't cope with fragmented packets, but we won't get - * any because we not advertise the NETIF_F_SG feature. + * 3. The data is fragmented. */ - if (skb_shared(skb) || skb_cloned(skb) || !is_xen_skb(skb)) { - int hlen = skb->data - skb->head; - int ret; - struct sk_buff *nskb = dev_alloc_skb(hlen + skb->len); + if (skb_cloned(skb) || skb_is_nonlinear(skb) || !is_xen_skb(skb)) { + struct sk_buff *nskb = netbk_copy_skb(skb); if ( unlikely(nskb == NULL) ) goto drop; - skb_reserve(nskb, hlen); - __skb_put(nskb, skb->len); - ret = skb_copy_bits(skb, -hlen, nskb->data - hlen, - skb->len + hlen); - BUG_ON(ret); /* Copy only the header fields we use in this driver. */ nskb->dev = skb->dev; nskb->ip_summed = skb->ip_summed; @@ -185,7 +263,7 @@ int netif_be_start_xmit(struct sk_buff * skb = nskb; } - netif->rx_req_cons_peek++; + netif->rx_req_cons_peek += skb_shinfo(skb)->nr_frags + 1; netif_get(netif); if (netbk_can_queue(dev) && netbk_queue_full(netif)) @@ -221,116 +299,80 @@ int xen_network_done(void) } #endif -static void net_rx_action(unsigned long unused) -{ - netif_t *netif = NULL; - s8 status; - u16 size, id, irq, flags; - multicall_entry_t *mcl; - mmu_update_t *mmu; - gnttab_transfer_t *gop; - unsigned long vdata, old_mfn, new_mfn; - struct sk_buff_head rxq; - struct sk_buff *skb; - int notify_nr = 0; - int ret; +static u16 netbk_gop_frag(netif_t *netif, struct page *page, int count, int i) +{ + multicall_entry_t *mcl = rx_mcl + count; + mmu_update_t *mmu = rx_mmu + count; + gnttab_transfer_t *gop = grant_rx_op + count; + netif_rx_request_t *req; + unsigned long old_mfn, new_mfn; + + old_mfn = virt_to_mfn(page_address(page)); + + if (!xen_feature(XENFEAT_auto_translated_physmap)) { + new_mfn = alloc_mfn(); + + /* + * Set the new P2M table entry before reassigning + * the old data page. Heed the comment in + * pgtable-2level.h:pte_page(). :-) + */ + set_phys_to_machine(page_to_pfn(page), new_mfn); + + MULTI_update_va_mapping(mcl, (unsigned long)page_address(page), + pfn_pte_ma(new_mfn, PAGE_KERNEL), 0); + + mmu->ptr = ((maddr_t)new_mfn << PAGE_SHIFT) | + MMU_MACHPHYS_UPDATE; + mmu->val = page_to_pfn(page); + } + + req = RING_GET_REQUEST(&netif->rx, netif->rx.req_cons + i); + gop->mfn = old_mfn; + gop->domid = netif->domid; + gop->ref = req->gref; + return req->id; +} + +static void netbk_gop_skb(struct sk_buff *skb, struct netbk_rx_meta *meta, + int count) +{ + netif_t *netif = netdev_priv(skb->dev); + int nr_frags = skb_shinfo(skb)->nr_frags; + int i; + + for (i = 0; i < nr_frags; i++) { + meta[++count].frag = skb_shinfo(skb)->frags[i]; + meta[count].id = netbk_gop_frag(netif, meta[count].frag.page, + count, i + 1); + } + /* - * Putting hundreds of bytes on the stack is considered rude. - * Static works because a tasklet can only be on one CPU at any time. + * This must occur at the end to ensure that we don't trash + * skb_shinfo until we're done. */ - static u16 notify_list[NET_RX_RING_SIZE]; - - skb_queue_head_init(&rxq); - - mcl = rx_mcl; - mmu = rx_mmu; - gop = grant_rx_op; - - while ((skb = skb_dequeue(&rx_queue)) != NULL) { - netif = netdev_priv(skb->dev); - vdata = (unsigned long)skb->data; - old_mfn = virt_to_mfn(vdata); - - if (!xen_feature(XENFEAT_auto_translated_physmap)) { - /* Memory squeeze? Back off for an arbitrary while. */ - if ((new_mfn = alloc_mfn()) == 0) { - if ( net_ratelimit() ) - WPRINTK("Memory squeeze in netback " - "driver.\n"); - mod_timer(&net_timer, jiffies + HZ); - skb_queue_head(&rx_queue, skb); - break; - } - /* - * Set the new P2M table entry before reassigning - * the old data page. Heed the comment in - * pgtable-2level.h:pte_page(). :-) - */ - set_phys_to_machine( - __pa(skb->data) >> PAGE_SHIFT, - new_mfn); - - MULTI_update_va_mapping(mcl, vdata, - pfn_pte_ma(new_mfn, - PAGE_KERNEL), 0); - mcl++; - - mmu->ptr = ((maddr_t)new_mfn << PAGE_SHIFT) | - MMU_MACHPHYS_UPDATE; - mmu->val = __pa(vdata) >> PAGE_SHIFT; - mmu++; - } - - gop->mfn = old_mfn; - gop->domid = netif->domid; - gop->ref = RING_GET_REQUEST( - &netif->rx, netif->rx.req_cons)->gref; - netif->rx.req_cons++; - gop++; - - __skb_queue_tail(&rxq, skb); - - /* Filled the batch queue? */ - if ((gop - grant_rx_op) == ARRAY_SIZE(grant_rx_op)) - break; - } - - if (!xen_feature(XENFEAT_auto_translated_physmap)) { - if (mcl == rx_mcl) - return; - - mcl[-1].args[MULTI_UVMFLAGS_INDEX] = UVMF_TLB_FLUSH|UVMF_ALL; - - if (mmu - rx_mmu) { - mcl->op = __HYPERVISOR_mmu_update; - mcl->args[0] = (unsigned long)rx_mmu; - mcl->args[1] = mmu - rx_mmu; - mcl->args[2] = 0; - mcl->args[3] = DOMID_SELF; - mcl++; - } - - ret = HYPERVISOR_multicall(rx_mcl, mcl - rx_mcl); - BUG_ON(ret != 0); - } - - ret = HYPERVISOR_grant_table_op(GNTTABOP_transfer, grant_rx_op, - gop - grant_rx_op); - BUG_ON(ret != 0); - - mcl = rx_mcl; - gop = grant_rx_op; - while ((skb = __skb_dequeue(&rxq)) != NULL) { - netif = netdev_priv(skb->dev); - size = skb->tail - skb->data; - - atomic_set(&(skb_shinfo(skb)->dataref), 1); - skb_shinfo(skb)->nr_frags = 0; - skb_shinfo(skb)->frag_list = NULL; - - netif->stats.tx_bytes += size; - netif->stats.tx_packets++; - + meta[count - nr_frags].id = netbk_gop_frag(netif, + virt_to_page(skb->data), + count - nr_frags, 0); + netif->rx.req_cons += nr_frags + 1; +} + +static inline void netbk_free_pages(int nr_frags, struct netbk_rx_meta *meta) +{ + int i; + + for (i = 0; i < nr_frags; i++) + put_page(meta[i].frag.page); +} + +static int netbk_check_gop(int nr_frags, domid_t domid, int count) +{ + multicall_entry_t *mcl = rx_mcl + count; + gnttab_transfer_t *gop = grant_rx_op + count; + int status = NETIF_RSP_OKAY; + int i; + + for (i = 0; i <= nr_frags; i++) { if (!xen_feature(XENFEAT_auto_translated_physmap)) { /* The update_va_mapping() must not fail. */ BUG_ON(mcl->result != 0); @@ -338,10 +380,9 @@ static void net_rx_action(unsigned long } /* Check the reassignment error code. */ - status = NETIF_RSP_OKAY; if (gop->status != 0) { DPRINTK("Bad status %d from grant transfer to DOM%u\n", - gop->status, netif->domid); + gop->status, domid); /* * Page no longer belongs to us unless GNTST_bad_page, * but that should be a fatal error anyway. @@ -349,17 +390,128 @@ static void net_rx_action(unsigned long BUG_ON(gop->status == GNTST_bad_page); status = NETIF_RSP_ERROR; } - irq = netif->irq; - id = RING_GET_REQUEST(&netif->rx, netif->rx.rsp_prod_pvt)->id; - flags = 0; + gop++; + } + + return status; +} + +static void netbk_add_frag_responses(netif_t *netif, int status, + struct netbk_rx_meta *meta, int nr_frags) +{ + int i; + + for (i = 0; i < nr_frags; i++) { + int id = meta[i].id; + int flags = (i == nr_frags - 1) ? 0 : NETRXF_more_data; + + make_rx_response(netif, id, status, meta[i].frag.page_offset, + meta[i].frag.size, flags); + } +} + +static void net_rx_action(unsigned long unused) +{ + netif_t *netif = NULL; + s8 status; + u16 id, irq, flags; + multicall_entry_t *mcl; + struct sk_buff_head rxq; + struct sk_buff *skb; + int notify_nr = 0; + int ret; + int nr_frags; + int count; + + /* + * Putting hundreds of bytes on the stack is considered rude. + * Static works because a tasklet can only be on one CPU at any time. + */ + static u16 notify_list[NET_RX_RING_SIZE]; + static struct netbk_rx_meta meta[NET_RX_RING_SIZE]; + + skb_queue_head_init(&rxq); + + count = 0; + + while ((skb = skb_dequeue(&rx_queue)) != NULL) { + nr_frags = skb_shinfo(skb)->nr_frags; + *(int *)skb->cb = nr_frags; + + if (!xen_feature(XENFEAT_auto_translated_physmap) && + check_mfn(nr_frags + 1)) { + /* Memory squeeze? Back off for an arbitrary while. */ + if ( net_ratelimit() ) + WPRINTK("Memory squeeze in netback " + "driver.\n"); + mod_timer(&net_timer, jiffies + HZ); + skb_queue_head(&rx_queue, skb); + break; + } + + netbk_gop_skb(skb, meta, count); + + count += nr_frags + 1; + + __skb_queue_tail(&rxq, skb); + + /* Filled the batch queue? */ + if (count + MAX_SKB_FRAGS >= NET_RX_RING_SIZE) + break; + } + + if (!count) + return; + + if (!xen_feature(XENFEAT_auto_translated_physmap)) { + mcl = rx_mcl + count; + + mcl[-1].args[MULTI_UVMFLAGS_INDEX] = UVMF_TLB_FLUSH|UVMF_ALL; + + mcl->op = __HYPERVISOR_mmu_update; + mcl->args[0] = (unsigned long)rx_mmu; + mcl->args[1] = count; + mcl->args[2] = 0; + mcl->args[3] = DOMID_SELF; + + ret = HYPERVISOR_multicall(rx_mcl, count + 1); + BUG_ON(ret != 0); + } + + ret = HYPERVISOR_grant_table_op(GNTTABOP_transfer, grant_rx_op, count); + BUG_ON(ret != 0); + + count = 0; + while ((skb = __skb_dequeue(&rxq)) != NULL) { + nr_frags = *(int *)skb->cb; + + atomic_set(&(skb_shinfo(skb)->dataref), 1); + skb_shinfo(skb)->nr_frags = 0; + skb_shinfo(skb)->frag_list = NULL; + + netif = netdev_priv(skb->dev); + netif->stats.tx_bytes += skb->len; + netif->stats.tx_packets++; + + netbk_free_pages(nr_frags, meta + count + 1); + status = netbk_check_gop(nr_frags, netif->domid, count); + + id = meta[count].id; + flags = nr_frags ? NETRXF_more_data : 0; + if (skb->ip_summed == CHECKSUM_HW) /* local packet? */ flags |= NETRXF_csum_blank | NETRXF_data_validated; else if (skb->proto_data_valid) /* remote but checksummed? */ flags |= NETRXF_data_validated; - if (make_rx_response(netif, id, status, - (unsigned long)skb->data & ~PAGE_MASK, - size, flags) && - (rx_notify[irq] == 0)) { + + make_rx_response(netif, id, status, offset_in_page(skb->data), + skb_headlen(skb), flags); + netbk_add_frag_responses(netif, status, meta + count + 1, + nr_frags); + + RING_PUSH_RESPONSES_AND_CHECK_NOTIFY(&netif->rx, ret); + irq = netif->irq; + if (ret && !rx_notify[irq]) { rx_notify[irq] = 1; notify_list[notify_nr++] = irq; } @@ -370,7 +522,7 @@ static void net_rx_action(unsigned long netif_put(netif); dev_kfree_skb(skb); - gop++; + count += nr_frags + 1; } while (notify_nr != 0) { @@ -1040,7 +1192,6 @@ static int make_rx_response(netif_t *net { RING_IDX i = netif->rx.rsp_prod_pvt; netif_rx_response_t *resp; - int notify; resp = RING_GET_RESPONSE(&netif->rx, i); resp->offset = offset; @@ -1051,9 +1202,8 @@ static int make_rx_response(netif_t *net resp->status = (s16)st; netif->rx.rsp_prod_pvt = ++i; - RING_PUSH_RESPONSES_AND_CHECK_NOTIFY(&netif->rx, notify); - - return notify; + + return 0; } #ifdef NETBE_DEBUG_INTERRUPT diff -r 05d6393e9d18 -r 4f008474675a linux-2.6-xen-sparse/drivers/xen/netback/xenbus.c --- a/linux-2.6-xen-sparse/drivers/xen/netback/xenbus.c Fri Jul 28 18:20:45 2006 +1000 +++ b/linux-2.6-xen-sparse/drivers/xen/netback/xenbus.c Fri Jul 28 18:22:22 2006 +1000 @@ -377,6 +377,13 @@ static int connect_rings(struct backend_ /* Must be non-zero for pfifo_fast to work. */ be->netif->dev->tx_queue_len = 1; + if (xenbus_scanf(XBT_NIL, dev->otherend, "feature-sg", "%d", &val) < 0) + val = 0; + if (val) { + be->netif->features |= NETIF_F_SG; + be->netif->dev->features |= NETIF_F_SG; + } + /* Map the shared frame, irq etc. */ err = netif_map(be->netif, tx_ring_ref, rx_ring_ref, evtchn); if (err) { _______________________________________________ Xen-devel mailing list Xen-devel@xxxxxxxxxxxxxxxxxxx http://lists.xensource.com/xen-devel
|
 |
Lists.xenproject.org is hosted with RackSpace, monitoring our |