[Xen-devel] [3/3] [NET] back: Add lazy copying

[Herbert Xu <herbert@xxxxxxxxxxxxxxxxxxx>]

Hi:

[NET] back: Add lazy copying

This patch adds lazy copying using the new unmap_and_replace grant
table operation.

We keep a list of pending entries sorted by arrival order.  We'll
process this list every time net_tx_action is invoked.  We ensure
that net_tx_action is invoked within one second of the arrival of
the first packet in the list.

When we process the list any entry that has been around for more
than half a second is copied.  This allows up to free the grant
table entry and return it to domU.

If the new grant table operation is not available (e.g., old HV
or architectures that don't support it yet) we simply copy each
packet as we receive them using skb_linearize.  We also disable
SG/TSO if this is the case.

By default the new code is disabled.  In order to enable it,
the module needs to be loaded with the argument copy_skb=1.

Signed-off-by: Herbert Xu <herbert@xxxxxxxxxxxxxxxxxxx> 

Cheers,
-- 
Visit Openswan at http://www.openswan.org/
Email: Herbert Xu ~{PmV>HI~} <herbert@xxxxxxxxxxxxxxxxxxx>
Home Page: http://gondor.apana.org.au/~herbert/
PGP Key: http://gondor.apana.org.au/~herbert/pubkey.txt
--
diff -r 3ef0510e44d0 linux-2.6-xen-sparse/drivers/xen/netback/common.h
--- a/linux-2.6-xen-sparse/drivers/xen/netback/common.h Tue May 08 10:21:23 
2007 +0100
+++ b/linux-2.6-xen-sparse/drivers/xen/netback/common.h Wed May 16 22:31:20 
2007 +1000
@@ -114,6 +114,14 @@ typedef struct netif_st {
 #define netback_carrier_off(netif)     ((netif)->carrier = 0)
 #define netback_carrier_ok(netif)      ((netif)->carrier)
 
+enum {
+       NETBK_DONT_COPY_SKB,
+       NETBK_DELAYED_COPY_SKB,
+       NETBK_ALWAYS_COPY_SKB,
+};
+
+extern int netbk_copy_skb_mode;
+
 #define NET_TX_RING_SIZE __RING_SIZE((netif_tx_sring_t *)0, PAGE_SIZE)
 #define NET_RX_RING_SIZE __RING_SIZE((netif_rx_sring_t *)0, PAGE_SIZE)
 
diff -r 3ef0510e44d0 linux-2.6-xen-sparse/drivers/xen/netback/netback.c
--- a/linux-2.6-xen-sparse/drivers/xen/netback/netback.c        Tue May 08 
10:21:23 2007 +0100
+++ b/linux-2.6-xen-sparse/drivers/xen/netback/netback.c        Wed May 16 
22:31:20 2007 +1000
@@ -49,6 +49,11 @@ struct netbk_rx_meta {
        int copy:1;
 };
 
+struct netbk_tx_pending_inuse {
+       struct list_head list;
+       unsigned long alloc_time;
+};
+
 static void netif_idx_release(u16 pending_idx);
 static void netif_page_release(struct page *page);
 static void make_tx_response(netif_t *netif, 
@@ -68,15 +73,21 @@ static DECLARE_TASKLET(net_rx_tasklet, n
 static DECLARE_TASKLET(net_rx_tasklet, net_rx_action, 0);
 
 static struct timer_list net_timer;
+static struct timer_list netbk_tx_pending_timer;
 
 #define MAX_PENDING_REQS 256
 
 static struct sk_buff_head rx_queue;
 
 static struct page **mmap_pages;
+static inline unsigned long idx_to_pfn(unsigned int idx)
+{
+       return page_to_pfn(mmap_pages[idx]);
+}
+
 static inline unsigned long idx_to_kaddr(unsigned int idx)
 {
-       return (unsigned long)pfn_to_kaddr(page_to_pfn(mmap_pages[idx]));
+       return (unsigned long)pfn_to_kaddr(idx_to_pfn(idx));
 }
 
 #define PKT_PROT_LEN 64
@@ -95,6 +106,10 @@ static u16 dealloc_ring[MAX_PENDING_REQS
 static u16 dealloc_ring[MAX_PENDING_REQS];
 static PEND_RING_IDX dealloc_prod, dealloc_cons;
 
+/* Doubly-linked list of in-use pending entries. */
+static struct netbk_tx_pending_inuse pending_inuse[MAX_PENDING_REQS];
+static LIST_HEAD(pending_inuse_head);
+
 static struct sk_buff_head tx_queue;
 
 static grant_handle_t grant_tx_handle[MAX_PENDING_REQS];
@@ -107,6 +122,13 @@ static spinlock_t net_schedule_list_lock
 #define MAX_MFN_ALLOC 64
 static unsigned long mfn_list[MAX_MFN_ALLOC];
 static unsigned int alloc_index = 0;
+
+/* Setting this allows the safe use of this driver without netloop. */
+static int MODPARM_copy_skb;
+module_param_named(copy_skb, MODPARM_copy_skb, bool, 0);
+MODULE_PARM_DESC(copy_skb, "Copy data received from netfront without netloop");
+
+int netbk_copy_skb_mode;
 
 static inline unsigned long alloc_mfn(void)
 {
@@ -719,6 +741,11 @@ static void net_alarm(unsigned long unus
        tasklet_schedule(&net_rx_tasklet);
 }
 
+static void netbk_tx_pending_timeout(unsigned long unused)
+{
+       tasklet_schedule(&net_tx_tasklet);
+}
+
 struct net_device_stats *netif_be_get_stats(struct net_device *dev)
 {
        netif_t *netif = netdev_priv(dev);
@@ -812,46 +839,97 @@ static void tx_credit_callback(unsigned 
        netif_schedule_work(netif);
 }
 
+static inline int copy_pending_req(PEND_RING_IDX pending_idx)
+{
+       return gnttab_copy_grant_page(grant_tx_handle[pending_idx],
+                                     &mmap_pages[pending_idx]);
+}
+
 inline static void net_tx_action_dealloc(void)
 {
+       struct netbk_tx_pending_inuse *inuse, *n;
        gnttab_unmap_grant_ref_t *gop;
        u16 pending_idx;
        PEND_RING_IDX dc, dp;
        netif_t *netif;
        int ret;
+       LIST_HEAD(list);
 
        dc = dealloc_cons;
-       dp = dealloc_prod;
-
-       /* Ensure we see all indexes enqueued by netif_idx_release(). */
-       smp_rmb();
+       gop = tx_unmap_ops;
 
        /*
         * Free up any grants we have finished using
         */
-       gop = tx_unmap_ops;
-       while (dc != dp) {
-               pending_idx = dealloc_ring[MASK_PEND_IDX(dc++)];
-               gnttab_set_unmap_op(gop, idx_to_kaddr(pending_idx),
-                                   GNTMAP_host_map,
-                                   grant_tx_handle[pending_idx]);
-               gop++;
-       }
+       do {
+               dp = dealloc_prod;
+
+               /* Ensure we see all indices enqueued by netif_idx_release(). */
+               smp_rmb();
+
+               while (dc != dp) {
+                       unsigned long pfn;
+
+                       pending_idx = dealloc_ring[MASK_PEND_IDX(dc++)];
+                       list_move_tail(&pending_inuse[pending_idx].list, &list);
+
+                       pfn = idx_to_pfn(pending_idx);
+                       /* Already unmapped? */
+                       if (!phys_to_machine_mapping_valid(pfn))
+                               continue;
+
+                       gnttab_set_unmap_op(gop, idx_to_kaddr(pending_idx),
+                                           GNTMAP_host_map,
+                                           grant_tx_handle[pending_idx]);
+                       gop++;
+               }
+
+               if (netbk_copy_skb_mode != NETBK_DELAYED_COPY_SKB ||
+                   list_empty(&pending_inuse_head))
+                       break;
+
+               /* Copy any entries that have been pending for too long. */
+               list_for_each_entry_safe(inuse, n, &pending_inuse_head, list) {
+                       if (time_after(inuse->alloc_time + HZ / 2, jiffies))
+                               break;
+
+                       switch (copy_pending_req(inuse - pending_inuse)) {
+                       case 0:
+                               list_move_tail(&inuse->list, &list);
+                               continue;
+                       case -EBUSY:
+                               list_del_init(&inuse->list);
+                               continue;
+                       case -ENOENT:
+                               continue;
+                       }
+
+                       break;
+               }
+       } while (dp != dealloc_prod);
+
+       dealloc_cons = dc;
+
        ret = HYPERVISOR_grant_table_op(
                GNTTABOP_unmap_grant_ref, tx_unmap_ops, gop - tx_unmap_ops);
        BUG_ON(ret);
 
-       while (dealloc_cons != dp) {
-               pending_idx = dealloc_ring[MASK_PEND_IDX(dealloc_cons++)];
+       list_for_each_entry_safe(inuse, n, &list, list) {
+               pending_idx = inuse - pending_inuse;
 
                netif = pending_tx_info[pending_idx].netif;
 
                make_tx_response(netif, &pending_tx_info[pending_idx].req, 
                                 NETIF_RSP_OKAY);
 
+               /* Ready for next use. */
+               gnttab_reset_grant_page(mmap_pages[pending_idx]);
+
                pending_ring[MASK_PEND_IDX(pending_prod++)] = pending_idx;
 
                netif_put(netif);
+
+               list_del_init(&inuse->list);
        }
 }
 
@@ -1023,6 +1101,11 @@ static void netbk_fill_frags(struct sk_b
                unsigned long pending_idx;
 
                pending_idx = (unsigned long)frag->page;
+
+               pending_inuse[pending_idx].alloc_time = jiffies;
+               list_add_tail(&pending_inuse[pending_idx].list,
+                             &pending_inuse_head);
+
                txp = &pending_tx_info[pending_idx].req;
                frag->page = virt_to_page(idx_to_kaddr(pending_idx));
                frag->size = txp->size;
@@ -1311,8 +1394,24 @@ static void net_tx_action(unsigned long 
                netif->stats.rx_bytes += skb->len;
                netif->stats.rx_packets++;
 
+               if (unlikely(netbk_copy_skb_mode == NETBK_ALWAYS_COPY_SKB) &&
+                   unlikely(skb_linearize(skb))) {
+                       DPRINTK("Can't linearize skb in net_tx_action.\n");
+                       kfree_skb(skb);
+                       continue;
+               }
+
                netif_rx(skb);
                netif->dev->last_rx = jiffies;
+       }
+
+       if (netbk_copy_skb_mode == NETBK_DELAYED_COPY_SKB &&
+           !list_empty(&pending_inuse_head)) {
+               struct netbk_tx_pending_inuse *oldest;
+
+               oldest = list_entry(pending_inuse_head.next,
+                                   struct netbk_tx_pending_inuse, list);
+               mod_timer(&netbk_tx_pending_timer, oldest->alloc_time + HZ);
        }
 }
 
@@ -1333,9 +1432,6 @@ static void netif_idx_release(u16 pendin
 
 static void netif_page_release(struct page *page)
 {
-       /* Ready for next use. */
-       init_page_count(page);
-
        netif_idx_release(netif_page_index(page));
 }
 
@@ -1457,6 +1553,10 @@ static int __init netback_init(void)
        net_timer.data = 0;
        net_timer.function = net_alarm;
 
+       init_timer(&netbk_tx_pending_timer);
+       netbk_tx_pending_timer.data = 0;
+       netbk_tx_pending_timer.function = netbk_tx_pending_timeout;
+
        mmap_pages = alloc_empty_pages_and_pagevec(MAX_PENDING_REQS);
        if (mmap_pages == NULL) {
                printk("%s: out of memory\n", __FUNCTION__);
@@ -1467,6 +1567,7 @@ static int __init netback_init(void)
                page = mmap_pages[i];
                SetPageForeign(page, netif_page_release);
                netif_page_index(page) = i;
+               INIT_LIST_HEAD(&pending_inuse[i].list);
        }
 
        pending_cons = 0;
@@ -1476,6 +1577,15 @@ static int __init netback_init(void)
 
        spin_lock_init(&net_schedule_list_lock);
        INIT_LIST_HEAD(&net_schedule_list);
+
+       netbk_copy_skb_mode = NETBK_DONT_COPY_SKB;
+       if (MODPARM_copy_skb) {
+               if (HYPERVISOR_grant_table_op(GNTTABOP_unmap_and_replace,
+                                             NULL, 0))
+                       netbk_copy_skb_mode = NETBK_ALWAYS_COPY_SKB;
+               else
+                       netbk_copy_skb_mode = NETBK_DELAYED_COPY_SKB;
+       }
 
        netif_xenbus_init();
 
diff -r 3ef0510e44d0 linux-2.6-xen-sparse/drivers/xen/netback/xenbus.c
--- a/linux-2.6-xen-sparse/drivers/xen/netback/xenbus.c Tue May 08 10:21:23 
2007 +0100
+++ b/linux-2.6-xen-sparse/drivers/xen/netback/xenbus.c Wed May 16 22:31:20 
2007 +1000
@@ -62,6 +62,7 @@ static int netback_probe(struct xenbus_d
        const char *message;
        struct xenbus_transaction xbt;
        int err;
+       int sg;
        struct backend_info *be = kzalloc(sizeof(struct backend_info),
                                          GFP_KERNEL);
        if (!be) {
@@ -73,6 +74,10 @@ static int netback_probe(struct xenbus_d
        be->dev = dev;
        dev->dev.driver_data = be;
 
+       sg = 1;
+       if (netbk_copy_skb_mode == NETBK_ALWAYS_COPY_SKB)
+               sg = 0;
+
        do {
                err = xenbus_transaction_start(&xbt);
                if (err) {
@@ -80,14 +85,14 @@ static int netback_probe(struct xenbus_d
                        goto fail;
                }
 
-               err = xenbus_printf(xbt, dev->nodename, "feature-sg", "%d", 1);
+               err = xenbus_printf(xbt, dev->nodename, "feature-sg", "%d", sg);
                if (err) {
                        message = "writing feature-sg";
                        goto abort_transaction;
                }
 
                err = xenbus_printf(xbt, dev->nodename, "feature-gso-tcpv4",
-                                   "%d", 1);
+                                   "%d", sg);
                if (err) {
                        message = "writing feature-gso-tcpv4";
                        goto abort_transaction;

_______________________________________________
Xen-devel mailing list
Xen-devel@xxxxxxxxxxxxxxxxxxx
http://lists.xensource.com/xen-devel