[Date Prev][Date Next][Thread Prev][Thread Next][Date Index][Thread Index]

[Minios-devel] [UNIKRAFT PATCH v3 3/6] lib/ukblkdev: Blkdev initialization



This patch introduces the API for initializing a Unikraft block device.
The initialization is done in the following order:
    (1) Configure main aspects of device (e.g., number of queues)
    (2) Configure each queue
    (3) Start the device.

Signed-off-by: Roxana Nicolescu <nicolescu.roxana1996@xxxxxxxxx>
Reviewed-by: Simon Kuenzer <simon.kuenzer@xxxxxxxxx>
---
 lib/ukblkdev/Config.uk                |  24 ++-
 lib/ukblkdev/blkdev.c                 | 243 ++++++++++++++++++++++++++
 lib/ukblkdev/exportsyms.uk            |   5 +
 lib/ukblkdev/include/uk/blkdev.h      | 100 +++++++++++
 lib/ukblkdev/include/uk/blkdev_core.h | 133 ++++++++++++++
 5 files changed, 504 insertions(+), 1 deletion(-)

diff --git a/lib/ukblkdev/Config.uk b/lib/ukblkdev/Config.uk
index b17d6167..f0c89654 100644
--- a/lib/ukblkdev/Config.uk
+++ b/lib/ukblkdev/Config.uk
@@ -1,6 +1,28 @@
-config LIBUKBLKDEV
+menuconfig LIBUKBLKDEV
        bool "ukblkdev: Block driver interface"
        default n
        select LIBUKALLOC
        select LIBNOLIBC if !HAVE_LIBC
        select LIBUKDEBUG
+
+if LIBUKBLKDEV
+       config LIBUKBLKDEV_MAXNBQUEUES
+                int "Maximum number of request-response queues"
+                default 1
+               help
+                       Upper limit for supported number of request-response
+                       queues with the ukblkdev API.
+
+       config LIBUKBLKDEV_DISPATCHERTHREADS
+                bool "Dispatcher threads for event callbacks"
+                default n
+                select LIBUKSCHED
+                select LIBUKLOCK
+               select LIBUKLOCK_SEMAPHORE
+               help
+                       Event callbacks are dispatched in a bottom half
+                       thread context instead of the device interrupt context.
+                       When this option is enabled a dispatcher thread is
+                       allocated for each configured queue.
+                       libuksched is required for this option.
+endif
diff --git a/lib/ukblkdev/blkdev.c b/lib/ukblkdev/blkdev.c
index d68ecdb7..f696dd5f 100644
--- a/lib/ukblkdev/blkdev.c
+++ b/lib/ukblkdev/blkdev.c
@@ -77,6 +77,13 @@ int uk_blkdev_drv_register(struct uk_blkdev *dev, struct 
uk_alloc *a,
 
        /* Data must be unallocated. */
        UK_ASSERT(PTRISERR(dev->_data));
+       /* Assert mandatory configuration. */
+       UK_ASSERT(dev->dev_ops);
+       UK_ASSERT(dev->dev_ops->dev_configure);
+       UK_ASSERT(dev->dev_ops->dev_start);
+       UK_ASSERT(dev->dev_ops->queue_setup);
+       UK_ASSERT(dev->dev_ops->get_info);
+       UK_ASSERT(dev->dev_ops->queue_get_info);
 
        dev->_data = _alloc_data(a, blkdev_count,  drv_name);
        if (!dev->_data)
@@ -131,3 +138,239 @@ enum uk_blkdev_state uk_blkdev_state_get(struct uk_blkdev 
*dev)
 
        return dev->_data->state;
 }
+
+int uk_blkdev_get_info(struct uk_blkdev *dev,
+               struct uk_blkdev_info *dev_info)
+{
+       int rc = 0;
+
+       UK_ASSERT(dev);
+       UK_ASSERT(dev->dev_ops);
+       UK_ASSERT(dev->dev_ops->get_info);
+       UK_ASSERT(dev_info);
+
+       /* Clear values before querying driver for capabilities */
+       memset(dev_info, 0, sizeof(*dev_info));
+       dev->dev_ops->get_info(dev, dev_info);
+
+       /* Limit the maximum number of queues
+        * according to the API configuration
+        */
+       dev_info->max_queues = MIN(CONFIG_LIBUKBLKDEV_MAXNBQUEUES,
+                       dev_info->max_queues);
+
+       return rc;
+}
+
+int uk_blkdev_configure(struct uk_blkdev *dev,
+               const struct uk_blkdev_conf *conf)
+{
+       int rc = 0;
+       struct uk_blkdev_info dev_info;
+
+       UK_ASSERT(dev);
+       UK_ASSERT(dev->_data);
+       UK_ASSERT(dev->dev_ops);
+       UK_ASSERT(dev->dev_ops->dev_configure);
+       UK_ASSERT(conf);
+
+       rc = uk_blkdev_get_info(dev, &dev_info);
+       if (rc) {
+               uk_pr_err("blkdev-%"PRIu16": Failed to get initial info: %d\n",
+                               dev->_data->id, rc);
+               return rc;
+       }
+
+       if (conf->nb_queues > dev_info.max_queues)
+               return -EINVAL;
+
+       rc = dev->dev_ops->dev_configure(dev, conf);
+       if (!rc) {
+               uk_pr_info("blkdev%"PRIu16": Configured interface\n",
+                               dev->_data->id);
+               dev->_data->state = UK_BLKDEV_CONFIGURED;
+       } else
+               uk_pr_err("blkdev%"PRIu16": Failed to configure interface %d\n",
+                               dev->_data->id, rc);
+
+       return rc;
+}
+
+#if CONFIG_LIBUKBLKDEV_DISPATCHERTHREADS
+static void _dispatcher(void *args)
+{
+       struct uk_blkdev_event_handler *handler =
+               (struct uk_blkdev_event_handler *) args;
+
+       UK_ASSERT(handler);
+       UK_ASSERT(handler->callback);
+
+       while (1) {
+               uk_semaphore_down(&handler->events);
+               handler->callback(handler->dev,
+                               handler->queue_id, handler->cookie);
+       }
+}
+#endif
+
+static int _create_event_handler(uk_blkdev_queue_event_t callback,
+               void *cookie,
+#if CONFIG_LIBUKBLKDEV_DISPATCHERTHREADS
+               struct uk_blkdev *dev, uint16_t queue_id,
+               struct uk_sched *s,
+#endif
+               struct uk_blkdev_event_handler *event_handler)
+{
+       UK_ASSERT(event_handler);
+       UK_ASSERT(callback || (!callback && !cookie));
+
+       event_handler->callback = callback;
+       event_handler->cookie = cookie;
+
+#if CONFIG_LIBUKBLKDEV_DISPATCHERTHREADS
+       /* If we do not have a callback, we do not need a thread */
+       if (!callback)
+               return 0;
+
+       event_handler->dev = dev;
+       event_handler->queue_id = queue_id;
+       uk_semaphore_init(&event_handler->events, 0);
+       event_handler->dispatcher_s = s;
+
+       /* Create a name for the dispatcher thread.
+        * In case of errors, we just continue without a name
+        */
+       if (asprintf(&event_handler->dispatcher_name,
+                               "blkdev%"PRIu16"-q%"PRIu16"]",
+                               dev->_data->id, queue_id) < 0) {
+               event_handler->dispatcher_name = NULL;
+       }
+
+       /* Create thread */
+       event_handler->dispatcher = uk_sched_thread_create(
+                       event_handler->dispatcher_s,
+                       event_handler->dispatcher_name, NULL,
+                       _dispatcher, (void *)event_handler);
+       if (event_handler->dispatcher == NULL) {
+               if (event_handler->dispatcher_name) {
+                       free(event_handler->dispatcher);
+                       event_handler->dispatcher = NULL;
+               }
+
+               return -ENOMEM;
+       }
+#endif
+
+       return 0;
+}
+
+static void _destroy_event_handler(struct uk_blkdev_event_handler *h
+               __maybe_unused)
+{
+       UK_ASSERT(h);
+
+#if CONFIG_LIBUKBLKDEV_DISPATCHERTHREADS
+       if (h->dispatcher) {
+               uk_semaphore_up(&h->events);
+               UK_ASSERT(h->dispatcher_s);
+               uk_thread_kill(h->dispatcher);
+               uk_thread_wait(h->dispatcher);
+               h->dispatcher = NULL;
+       }
+
+       if (h->dispatcher_name) {
+               free(h->dispatcher_name);
+               h->dispatcher_name = NULL;
+       }
+#endif
+}
+
+int uk_blkdev_queue_get_info(struct uk_blkdev *dev, uint16_t queue_id,
+               struct uk_blkdev_queue_info *q_info)
+{
+       UK_ASSERT(dev);
+       UK_ASSERT(queue_id < CONFIG_LIBUKBLKDEV_MAXNBQUEUES);
+       UK_ASSERT(q_info);
+
+       /* Clear values before querying driver for queue capabilities */
+       memset(q_info, 0, sizeof(*q_info));
+       return dev->dev_ops->queue_get_info(dev, queue_id, q_info);
+}
+
+int uk_blkdev_queue_configure(struct uk_blkdev *dev, uint16_t queue_id,
+               uint16_t nb_desc,
+               const struct uk_blkdev_queue_conf *queue_conf)
+{
+       int err = 0;
+
+       UK_ASSERT(dev);
+       UK_ASSERT(dev->_data);
+       UK_ASSERT(dev->dev_ops);
+       UK_ASSERT(dev->dev_ops->queue_setup);
+       UK_ASSERT(dev->finish_reqs);
+       UK_ASSERT(queue_id < CONFIG_LIBUKBLKDEV_MAXNBQUEUES);
+       UK_ASSERT(queue_conf);
+
+#if CONFIG_LIBUKBLKDEV_DISPATCHERTHREADS
+       UK_ASSERT((queue_conf->callback && queue_conf->s)
+                       || !queue_conf->callback);
+#endif
+
+       if (dev->_data->state != UK_BLKDEV_CONFIGURED)
+               return -EINVAL;
+
+       /* Make sure that we are not initializing this queue a second time */
+       if (!PTRISERR(dev->_queue[queue_id]))
+               return -EBUSY;
+
+       err = _create_event_handler(queue_conf->callback,
+                       queue_conf->callback_cookie,
+#if CONFIG_LIBUKBLKDEV_DISPATCHERTHREADS
+                       dev, queue_id, queue_conf->s,
+#endif
+                       &dev->_data->queue_handler[queue_id]);
+       if (err)
+               goto err_out;
+
+       dev->_queue[queue_id] = dev->dev_ops->queue_setup(dev, queue_id,
+                       nb_desc,
+                       queue_conf);
+       if (PTRISERR(dev->_queue[queue_id])) {
+               err = PTR2ERR(dev->_queue[queue_id]);
+               uk_pr_err("blkdev%"PRIu16"-q%"PRIu16": Failed to configure: 
%d\n",
+                               dev->_data->id, queue_id, err);
+               goto err_destroy_handler;
+       }
+
+       uk_pr_info("blkdev%"PRIu16": Configured queue %"PRIu16"\n",
+                       dev->_data->id, queue_id);
+       return 0;
+
+err_destroy_handler:
+       _destroy_event_handler(&dev->_data->queue_handler[queue_id]);
+err_out:
+       return err;
+}
+
+int uk_blkdev_start(struct uk_blkdev *dev)
+{
+       int rc = 0;
+
+       UK_ASSERT(dev);
+       UK_ASSERT(dev->_data);
+       UK_ASSERT(dev->dev_ops);
+       UK_ASSERT(dev->dev_ops->dev_start);
+       UK_ASSERT(dev->_data->state == UK_BLKDEV_CONFIGURED);
+
+       rc = dev->dev_ops->dev_start(dev);
+       if (rc)
+               uk_pr_err("blkdev%"PRIu16": Failed to start interface %d\n",
+                               dev->_data->id, rc);
+       else {
+               uk_pr_info("blkdev%"PRIu16": Started interface\n",
+                               dev->_data->id);
+               dev->_data->state = UK_BLKDEV_RUNNING;
+       }
+
+       return rc;
+}
diff --git a/lib/ukblkdev/exportsyms.uk b/lib/ukblkdev/exportsyms.uk
index 34e22426..077994f0 100644
--- a/lib/ukblkdev/exportsyms.uk
+++ b/lib/ukblkdev/exportsyms.uk
@@ -4,3 +4,8 @@ uk_blkdev_get
 uk_blkdev_id_get
 uk_blkdev_drv_name_get
 uk_blkdev_state_get
+uk_blkdev_get_info
+uk_blkdev_configure
+uk_blkdev_queue_get_info
+uk_blkdev_queue_configure
+uk_blkdev_start
diff --git a/lib/ukblkdev/include/uk/blkdev.h b/lib/ukblkdev/include/uk/blkdev.h
index 21b04117..375407ed 100644
--- a/lib/ukblkdev/include/uk/blkdev.h
+++ b/lib/ukblkdev/include/uk/blkdev.h
@@ -48,6 +48,12 @@
  * to uk_blkdev_get(). The block app should store this reference and
  * use it for all subsequent API calls.
  *
+ * The functions exported by the Unikraft BLK API to setup a device
+ * designated by its ID must be invoked in the following order:
+ *      - uk_blkdev_configure()
+ *      - uk_blkdev_queue_setup()
+ *      - uk_blkdev_start()
+ *
  * There are 4 states in which a block device can be found:
  *      - UK_BLKDEV_UNREGISTERED
  *      - UK_BLKDEV_UNCONFIGURED
@@ -122,6 +128,100 @@ const char *uk_blkdev_drv_name_get(struct uk_blkdev *dev);
  */
 enum uk_blkdev_state uk_blkdev_state_get(struct uk_blkdev *dev);
 
+/**
+ * Query device capabilities.
+ * Information that is useful for device initialization (e.g.,
+ * maximum number of supported queues).
+ *
+ * @param dev
+ *     The Unikraft Block Device.
+ * @param dev_info
+ *     A pointer to a structure of type *uk_blkdev_info* to be filled with
+ *     the contextual information of a block device.
+ * @return
+ *     - 0: Success
+ *     - <0: Error in driver
+ */
+int uk_blkdev_get_info(struct uk_blkdev *dev,
+               struct uk_blkdev_info *dev_info);
+
+/**
+ * Configure an Unikraft block device.
+ * This function must be invoked first before any other function in the
+ * Unikraft BLK API. This function can also be re-invoked when a device is
+ * in the stopped state.
+ *
+ * @param dev
+ *     The Unikraft Block Device.
+
+ * @return
+ *     - 0: Success, device configured.
+ *     - <0: Error code returned by the driver configuration function.
+ */
+int uk_blkdev_configure(struct uk_blkdev *dev,
+               const struct uk_blkdev_conf *conf);
+
+/**
+ * Query device queue capabilities.
+ * Information that is useful for device queue initialization (e.g.,
+ * maximum number of supported descriptors on queues).
+ *
+ * @param dev
+ *   The Unikraft Block Device in configured state.
+ * @param queue_id
+ *   The index of the queue to set up.
+ *   The value must be in the range [0, nb_queue - 1] previously supplied
+ *   to uk_blkdev_configure().
+ * @param queue_info
+ *   A pointer to a structure of type *uk_blkdev_queue_info* to be filled out.
+ * @return
+ *   - (0): Success, queue_info is filled out.
+ *   - (<0): Error code of the drivers function.
+ */
+int uk_blkdev_queue_get_info(struct uk_blkdev *dev, uint16_t queue_id,
+               struct uk_blkdev_queue_info *q_info);
+
+/**
+ * Allocate and set up a queue for an Unikraft block device.
+ * The queue is responsible for both requests and responses.
+ *
+ * @param dev
+ *     The Unikraft Block Device.
+ * @param queue_id
+ *     The index of the queue to set up.
+ *     The value must be in range [0, nb_queue -1] previously supplied
+ *     to uk_blkdev_configure()
+ * @param nb_desc
+ *     Number of descriptors for the queue. Inspect uk_blkdev_queue_get_info()
+ *     to retrieve limitations.
+ * @param queue_conf
+ *     The pointer to the configuration data to be used for the queue.
+ *     This can be shared across multiple queue setups.
+ * @return
+ *     - 0: Success, receive queue correctly set up.
+ *     - <0: Unable to allocate and set up the ring descriptors.
+ */
+int uk_blkdev_queue_configure(struct uk_blkdev *dev, uint16_t queue_id,
+               uint16_t nb_desc,
+               const struct uk_blkdev_queue_conf *queue_conf);
+
+/**
+ * Start a Block device.
+ *
+ * The device start step is the last one and consists of setting the configured
+ * offload features and in starting the transmit and the receive units of the
+ * device.
+ * On success, all basic functions exported by the Unikraft BLK API
+ * can be invoked.
+ *
+ * @param dev
+ *     The Unikraft Block Device.
+ * @return
+ *     - 0: Success, Unikraft block device started.
+ *     - <0: Error code of the driver device start function.
+ */
+int uk_blkdev_start(struct uk_blkdev *dev);
+
 #ifdef __cplusplus
 }
 #endif
diff --git a/lib/ukblkdev/include/uk/blkdev_core.h 
b/lib/ukblkdev/include/uk/blkdev_core.h
index 50d4dc4f..c4c458ab 100644
--- a/lib/ukblkdev/include/uk/blkdev_core.h
+++ b/lib/ukblkdev/include/uk/blkdev_core.h
@@ -38,6 +38,10 @@
 
 #include <uk/list.h>
 #include <uk/config.h>
+#if defined(CONFIG_LIBUKBLKDEV_DISPATCHERTHREADS)
+#include <uk/sched.h>
+#include <uk/semaphore.h>
+#endif
 
 /**
  * Unikraft block API common declarations.
@@ -71,6 +75,128 @@ enum uk_blkdev_state {
        UK_BLKDEV_RUNNING,
 };
 
+/**
+ * Structure used to configure an Unikraft block device.
+ */
+struct uk_blkdev_conf {
+       uint16_t nb_queues;
+};
+
+/**
+ * Structure used to  describe block device capabilities
+ * before negotiation
+ */
+struct uk_blkdev_info {
+       /* Max nb of supported queues by device. */
+       uint16_t max_queues;
+};
+
+/**
+ * Structure used to describe device descriptor ring limitations.
+ */
+struct uk_blkdev_queue_info {
+       /* Max allowed number of descriptors. */
+       uint16_t nb_max;
+       /* Min allowed number of descriptors. */
+       uint16_t nb_min;
+       /* Number should be a multiple of nb_align. */
+       uint16_t nb_align;
+       /* Number should be a power of two. */
+       int nb_is_power_of_two;
+};
+
+/**
+ * Queue Structure used for both requests and responses.
+ * This is private to the drivers.
+ * In the API, this structure is used only for type checking.
+ */
+struct uk_blkdev_queue;
+
+/**
+ * Function type used for queue event callbacks.
+ *
+ * @param dev
+ *   The Unikraft Block Device.
+ * @param queue
+ *   The queue on the Unikraft block device on which the event happened.
+ * @param argp
+ *   Extra argument that can be defined on callback registration.
+ */
+typedef void (*uk_blkdev_queue_event_t)(struct uk_blkdev *dev,
+               uint16_t queue_id, void *argp);
+
+/**
+ * Structure used to configure an Unikraft block device queue.
+ *
+ */
+struct uk_blkdev_queue_conf {
+       /* Allocator used for descriptor rings */
+       struct uk_alloc *a;
+       /* Event callback function */
+       uk_blkdev_queue_event_t callback;
+       /* Argument pointer for callback*/
+       void *callback_cookie;
+
+#if CONFIG_LIBUKBLKDEV_DISPATCHERTHREADS
+       /* Scheduler for dispatcher. */
+       struct uk_sched *s;
+#endif
+};
+
+/** Driver callback type to get initial device capabilities */
+typedef void (*uk_blkdev_get_info_t)(struct uk_blkdev *dev,
+               struct uk_blkdev_info *dev_info);
+
+/** Driver callback type to configure a block device. */
+typedef int (*uk_blkdev_configure_t)(struct uk_blkdev *dev,
+               const struct uk_blkdev_conf *conf);
+
+/* Driver callback type to get info about a device queue */
+typedef int (*uk_blkdev_queue_get_info_t)(struct uk_blkdev *dev,
+               uint16_t queue_id, struct uk_blkdev_queue_info *q_info);
+
+/** Driver callback type to set up a queue of an Unikraft block device. */
+typedef struct uk_blkdev_queue * (*uk_blkdev_queue_configure_t)(
+               struct uk_blkdev *dev, uint16_t queue_id, uint16_t nb_desc,
+               const struct uk_blkdev_queue_conf *queue_conf);
+
+/** Driver callback type to start a configured Unikraft block device. */
+typedef int (*uk_blkdev_start_t)(struct uk_blkdev *dev);
+
+struct uk_blkdev_ops {
+       uk_blkdev_get_info_t                            get_info;
+       uk_blkdev_configure_t                           dev_configure;
+       uk_blkdev_queue_get_info_t                      queue_get_info;
+       uk_blkdev_queue_configure_t                     queue_setup;
+       uk_blkdev_start_t                               dev_start;
+};
+
+/**
+ * @internal
+ * Event handler configuration (internal to libukblkdev)
+ */
+struct uk_blkdev_event_handler {
+       /* Callback */
+       uk_blkdev_queue_event_t callback;
+       /* Parameter for callback */
+       void *cookie;
+
+#if CONFIG_LIBUKBLKDEV_DISPATCHERTHREADS
+       /* Semaphore to trigger events. */
+       struct uk_semaphore events;
+       /* Reference to blk device. */
+       struct uk_blkdev    *dev;
+       /* Queue id which caused event. */
+       uint16_t            queue_id;
+       /* Dispatcher thread. */
+       struct uk_thread    *dispatcher;
+       /* Reference to thread name. */
+       char    *dispatcher_name;
+       /* Scheduler for dispatcher. */
+       struct uk_sched     *dispatcher_s;
+#endif
+};
+
 /**
  * @internal
  * libukblkdev internal data associated with each block device.
@@ -80,6 +206,9 @@ struct uk_blkdev_data {
        const uint16_t id;
        /* Device state */
        enum uk_blkdev_state state;
+       /* Event handler for each queue */
+       struct uk_blkdev_event_handler
+               queue_handler[CONFIG_LIBUKBLKDEV_MAXNBQUEUES];
        /* Name of device*/
        const char *drv_name;
        /* Allocator */
@@ -89,6 +218,10 @@ struct uk_blkdev_data {
 struct uk_blkdev {
        /* Pointer to API-internal state data. */
        struct uk_blkdev_data *_data;
+       /* Functions callbacks by driver. */
+       const struct uk_blkdev_ops *dev_ops;
+       /* Pointers to queues (API-private) */
+       struct uk_blkdev_queue *_queue[CONFIG_LIBUKBLKDEV_MAXNBQUEUES];
        /* Entry for list of block devices */
        UK_TAILQ_ENTRY(struct uk_blkdev) _list;
 };
-- 
2.17.1


_______________________________________________
Minios-devel mailing list
Minios-devel@xxxxxxxxxxxxxxxxxxxx
https://lists.xenproject.org/mailman/listinfo/minios-devel

 


Rackspace

Lists.xenproject.org is hosted with RackSpace, monitoring our
servers 24x7x365 and backed by RackSpace's Fanatical Support®.