[Date Prev][Date Next][Thread Prev][Thread Next][Date Index][Thread Index]
Re: [Xen-devel] [PATCH v2 09/13] optee: add support for arbitrary shared memory
Hi,
On 10.09.18 17:02, Julien Grall wrote:
Hi,
On 03/09/18 17:54, Volodymyr Babchuk wrote:
Shared memory is widely used by NW to communicate with
TAs in OP-TEE. NW can share part of own memory with
TA or OP-TEE core, by registering it OP-TEE, or by providing
a temporal refernce. Anyways, information about such memory
buffers are sent to OP-TEE as a list of pages. This mechanism
is descripted optee_msg.h.
Mediator should step in when NW tries to share memory with
OP-TEE for two reasons:
1. Do address translation from IPA to PA.
2. Pin domain pages till they are mapped into OP-TEE or TA
address space, so domain can't transfer this pages to
other domain or baloon out them.
s/baloon/balloon/
Address translation is done by translate_noncontig(...) function.
It allocates new buffer from xenheap and then walks on guest
provided list of pages, translates addresses and stores PAs into
newly allocated buffer. This buffer will be provided to OP-TEE
instead of original buffer from the guest. This buffer will
be free at the end of sdandard call.
In the same time this function pins pages and stores them in
struct shm_buf object. This object will live all the time,
when given SHM buffer is known to OP-TEE. It will be freed
after guest unregisters shared buffer. At this time pages
will be unpinned.
Signed-off-by: Volodymyr Babchuk <volodymyr_babchuk@xxxxxxxx>
---
xen/arch/arm/tee/optee.c | 245
++++++++++++++++++++++++++++++++++++++++++++++-
1 file changed, 244 insertions(+), 1 deletion(-)
diff --git a/xen/arch/arm/tee/optee.c b/xen/arch/arm/tee/optee.c
index 6d6b51d..8bfcfdc 100644
--- a/xen/arch/arm/tee/optee.c
+++ b/xen/arch/arm/tee/optee.c
@@ -22,6 +22,8 @@
#define MAX_STD_CALLS 16
#define MAX_RPC_SHMS 16
+#define MAX_TOTAL_SMH_BUF_PG 16384
So that's 64MB worth of guest memory. Do we expect them to be mapped in
Xen? Or just pinned?
Just pinned. We are not interested in contents of this memory.
+#define MAX_NONCONTIG_ENTRIES 5
/*
* Call context. OP-TEE can issue multiple RPC returns during one call.
@@ -31,6 +33,9 @@ struct std_call_ctx {
struct list_head list;
struct optee_msg_arg *guest_arg;
struct optee_msg_arg *xen_arg;
+ /* Buffer for translated page addresses, shared with OP-TEE */
+ void *non_contig[MAX_NONCONTIG_ENTRIES];
+ int non_contig_order[MAX_NONCONTIG_ENTRIES];
Can you please introduce a structure with the order and mapping?
mfn_t guest_arg_mfn;
int optee_thread_id;
int rpc_op;
@@ -45,13 +50,24 @@ struct shm_rpc {
uint64_t cookie;
};
+/* Shared memory buffer for arbitrary data */
+struct shm_buf {
+ struct list_head list;
+ uint64_t cookie;
+ int max_page_cnt;
+ int page_cnt;
AFAICT, max_page_cnt and page_cnt should never but negative. If so, then
they should be unsigned.
+ struct page_info *pages[];
+};
+
struct domain_ctx {
struct list_head list;
struct list_head call_ctx_list;
struct list_head shm_rpc_list;
+ struct list_head shm_buf_list;
struct domain *domain;
atomic_t call_ctx_count;
atomic_t shm_rpc_count;
+ atomic_t shm_buf_pages;
spinlock_t lock;
};
@@ -158,9 +174,12 @@ static int optee_enable(struct domain *d)
ctx->domain = d;
INIT_LIST_HEAD(&ctx->call_ctx_list);
INIT_LIST_HEAD(&ctx->shm_rpc_list);
+ INIT_LIST_HEAD(&ctx->shm_buf_list);
atomic_set(&ctx->call_ctx_count, 0);
atomic_set(&ctx->shm_rpc_count, 0);
+ atomic_set(&ctx->shm_buf_pages, 0);
+
spin_lock_init(&ctx->lock);
spin_lock(&domain_ctx_list_lock);
@@ -339,12 +358,76 @@ static void free_shm_rpc(struct domain_ctx *ctx,
uint64_t cookie)
xfree(shm_rpc);
}
+static struct shm_buf *allocate_shm_buf(struct domain_ctx *ctx,
+ uint64_t cookie,
+ int pages_cnt)
Ditto.
+{
+ struct shm_buf *shm_buf;
+
+ while(1)
+ {
+ int old = atomic_read(&ctx->shm_buf_pages);
+ int new = old + pages_cnt;
+ if ( new >= MAX_TOTAL_SMH_BUF_PG )
+ return NULL;
+ if ( likely(old == atomic_cmpxchg(&ctx->shm_buf_pages, old,
new)) )
+ break;
+ }
+
+ shm_buf = xzalloc_bytes(sizeof(struct shm_buf) +
+ pages_cnt * sizeof(struct page *));
+ if ( !shm_buf ) {
Coding style:
if ( ... )
{
+ atomic_sub(pages_cnt, &ctx->shm_buf_pages);
+ return NULL;
+ }
+
+ shm_buf->cookie = cookie;
+ shm_buf->max_page_cnt = pages_cnt;
+
+ spin_lock(&ctx->lock);
+ list_add_tail(&shm_buf->list, &ctx->shm_buf_list);
+ spin_unlock(&ctx->lock);
+
+ return shm_buf;
+}
+
+static void free_shm_buf(struct domain_ctx *ctx, uint64_t cookie)
+{
+ struct shm_buf *shm_buf; > + bool found = false;
+
+ spin_lock(&ctx->lock);
+ list_for_each_entry( shm_buf, &ctx->shm_buf_list, list )
+ {
+ if ( shm_buf->cookie == cookie )
What does guarantee you the cookie will be uniq?
+ {
+ found = true;
+ list_del(&shm_buf->list);
+ break;
+ }
+ }
+ spin_unlock(&ctx->lock);
At this point you have a shm_buf in hand to free. But what does
guarantee you no-one will use it?
+
+ if ( !found ) {
+ return;
+ }
+
+ for ( int i = 0; i < shm_buf->page_cnt; i++ )
Please define int i before hand.
+ if ( shm_buf->pages[i] )
+ put_page(shm_buf->pages[i]);
+
+ atomic_sub(shm_buf->max_page_cnt, &ctx->shm_buf_pages);
+
+ xfree(shm_buf);
+}
+
static void optee_domain_destroy(struct domain *d)
{
struct arm_smccc_res resp;
struct domain_ctx *ctx;
struct std_call_ctx *call, *call_tmp;
struct shm_rpc *shm_rpc, *shm_rpc_tmp;
+ struct shm_buf *shm_buf, *shm_buf_tmp;
bool found = false;
/* At this time all domain VCPUs should be stopped */
@@ -377,12 +460,163 @@ static void optee_domain_destroy(struct domain *d)
list_for_each_entry_safe( shm_rpc, shm_rpc_tmp,
&ctx->shm_rpc_list, list )
free_shm_rpc(ctx, shm_rpc->cookie);
+ list_for_each_entry_safe( shm_buf, shm_buf_tmp,
&ctx->shm_buf_list, list )
+ free_shm_buf(ctx, shm_buf->cookie);
+
ASSERT(!atomic_read(&ctx->call_ctx_count));
ASSERT(!atomic_read(&ctx->shm_rpc_count));
+ ASSERT(!atomic_read(&ctx->shm_buf_pages));
xfree(ctx);
}
+#define PAGELIST_ENTRIES_PER_PAGE \
+ ((OPTEE_MSG_NONCONTIG_PAGE_SIZE / sizeof(u64)) - 1)
+
+static size_t get_pages_list_size(size_t num_entries)
+{
+ int pages = DIV_ROUND_UP(num_entries, PAGELIST_ENTRIES_PER_PAGE);
+
+ return pages * OPTEE_MSG_NONCONTIG_PAGE_SIZE;
+}
+
+static bool translate_noncontig(struct domain_ctx *ctx,
+ struct std_call_ctx *call,
+ struct optee_msg_param *param,
+ int idx)
Most likely this should be unsigned.
+{
+ /*
+ * Refer to OPTEE_MSG_ATTR_NONCONTIG description in optee_msg.h
for details.
+ */
+ uint64_t size;
+ int page_offset;
+ int num_pages;
+ int order;
+ int entries_on_page = 0;
+ paddr_t gaddr;
+ mfn_t guest_mfn;
I don't like the terminology guest_mfn. This is misleading because of
the past usage in Xen. It would be better if you call this just mfn.
Okay, will do.
+ struct {
+ uint64_t pages_list[PAGELIST_ENTRIES_PER_PAGE];
+ uint64_t next_page_data;
+ } *pages_data_guest, *pages_data_xen, *pages_data_xen_start;
+ struct shm_buf *shm_buf;
+
+ page_offset = param->u.tmem.buf_ptr &
(OPTEE_MSG_NONCONTIG_PAGE_SIZE - 1);
+
+ size = ROUNDUP(param->u.tmem.size + page_offset,
+ OPTEE_MSG_NONCONTIG_PAGE_SIZE);
+
+ num_pages = DIV_ROUND_UP(size, OPTEE_MSG_NONCONTIG_PAGE_SIZE);
+
+ order = get_order_from_bytes(get_pages_list_size(num_pages));
+
+ pages_data_xen_start = alloc_xenheap_pages(order, 0);
This could be replaced by a _xmalloc and would avoid to allocate more
memory than necessary when the order is getting bigger.
Thanks. Would it allocate page-aligned buffer? This is crucial in this
case. I can't find any documentation on it so I don't know which
alignment it guarantees.
+ if ( !pages_data_xen_start )
+ return false;
+
+ shm_buf = allocate_shm_buf(ctx, param->u.tmem.shm_ref, num_pages);
In alocate_shm_buf you are now globally limiting the number of pages (
(16384) to pin. However, this does not limit per call.
With the current limit, you would could call up to 16384 times
lookup_and_pin_guest_ram_addr(...). On Arm, for p2m related operation,
we limit to 512 iterations in one go before checking the preemption.
So I think 16384 times is far too much.
So, in other words, I can translate only 2MB buffer (if 4096KB pages are
used), is it right?
I think, it will be okay to implement such limitation for this initial
version of mediator. In the future, it would be possible to do RPC
return from XEN (as OP-TEE does) to finish this request later.
+ if ( !shm_buf )
+ goto err_free;
+
+ gaddr = param->u.tmem.buf_ptr & ~(OPTEE_MSG_NONCONTIG_PAGE_SIZE -
1);
+ guest_mfn = lookup_and_pin_guest_ram_addr(gaddr, NULL);
+ if ( mfn_eq(guest_mfn, INVALID_MFN) )
+ goto err_free;
+
+ pages_data_guest = map_domain_page(guest_mfn);
+ if ( !pages_data_guest )
+ goto err_free;
+
+ pages_data_xen = pages_data_xen_start;
+ while ( num_pages ) {
+ struct page_info *page;
+ mfn_t entry_mfn = lookup_and_pin_guest_ram_addr(
+ pages_data_guest->pages_list[entries_on_page], &page);
+
+ if ( mfn_eq(entry_mfn, INVALID_MFN) )
+ goto err_unmap;
+
+ shm_buf->pages[shm_buf->page_cnt++] = page;
+ pages_data_xen->pages_list[entries_on_page] =
mfn_to_maddr(entry_mfn);
+ entries_on_page++;
+
+ if ( entries_on_page == PAGELIST_ENTRIES_PER_PAGE ) {
+ pages_data_xen->next_page_data =
virt_to_maddr(pages_data_xen + 1);
+ pages_data_xen++;
+ gaddr = pages_data_guest->next_page_data;
next_page_data is not a guest address but a machine address. For
anything related to address, the variable should be named accordingly to
avoid confusion.
Why? In this case it is IPA that comes from the guest.
+
+ unmap_domain_page(pages_data_guest);
+ unpin_guest_ram_addr(guest_mfn);
+
+ guest_mfn = lookup_and_pin_guest_ram_addr(gaddr, NULL);
+ if ( mfn_eq(guest_mfn, INVALID_MFN) )
+ goto err_free;
+
+ pages_data_guest = map_domain_page(guest_mfn);
+ if ( !pages_data_guest )
+ goto err_free;
+ /* Roll over to the next page */
+ entries_on_page = 0;
+ }
+ num_pages--;
+ }
+
+ param->u.tmem.buf_ptr = virt_to_maddr(pages_data_xen_start) |
page_offset;
+
+ call->non_contig[idx] = pages_data_xen_start;
+ call->non_contig_order[idx] = order;
If you use _xmalloc, then you don't need to store the order. But, who is
going to free non_contig?
Thank you for pointing out. It should be done in free_std_call_ctx() but
seems
I missed this part of the code when split the big patch into smaller ones.
+
+ unmap_domain_page(pages_data_guest);
+ unpin_guest_ram_addr(guest_mfn);
+ return true;
+
+err_unmap:
+ unmap_domain_page(pages_data_guest);
+ unpin_guest_ram_addr(guest_mfn);
+ free_shm_buf(ctx, shm_buf->cookie);
+
+err_free:
+ free_xenheap_pages(pages_data_xen_start, order);
+
+ return false;
+}
+
+static bool translate_params(struct domain_ctx *ctx,
+ struct std_call_ctx *call)
+{
+ unsigned int i;
+ uint32_t attr;
+
+ for ( i = 0; i < call->xen_arg->num_params; i++ ) {
Please pay attention to Xen coding style. I haven't pointed out
everywhere, but I would all of them to be fixed in the next version.
Yes, I'm sorry for that. I simultaneously work with different projects
and sometimes it is hard to track coding style. I'll fix all such
problems.
+ attr = call->xen_arg->params[i].attr;
+
+ switch ( attr & OPTEE_MSG_ATTR_TYPE_MASK ) {
+ case OPTEE_MSG_ATTR_TYPE_TMEM_INPUT:
+ case OPTEE_MSG_ATTR_TYPE_TMEM_OUTPUT:
+ case OPTEE_MSG_ATTR_TYPE_TMEM_INOUT:
+ if ( attr & OPTEE_MSG_ATTR_NONCONTIG ) {
+ if ( !translate_noncontig(ctx, call,
+ call->xen_arg->params + i,
i) )
+ return false;
+ }
+ else {
+ gprintk(XENLOG_WARNING, "Guest tries to use old tmem
arg\n");
+ return false;
+ }
+ break;
+ case OPTEE_MSG_ATTR_TYPE_NONE:
+ case OPTEE_MSG_ATTR_TYPE_VALUE_INPUT:
+ case OPTEE_MSG_ATTR_TYPE_VALUE_OUTPUT:
+ case OPTEE_MSG_ATTR_TYPE_VALUE_INOUT:
+ case OPTEE_MSG_ATTR_TYPE_RMEM_INPUT:
+ case OPTEE_MSG_ATTR_TYPE_RMEM_OUTPUT:
+ case OPTEE_MSG_ATTR_TYPE_RMEM_INOUT:
+ continue;
+ }
+ }
+ return true;
+}
+
/*
* Copy command buffer into xen memory to:
* 1) Hide translated addresses from guest
@@ -488,6 +722,15 @@ static bool execute_std_call(struct domain_ctx *ctx,
copy_std_request_back(ctx, regs, call);
+ /*
+ * If guest successfully unregistered own shared memory,
+ * then we can unpin it's pages
+ */
+ if ( call->xen_arg->cmd == OPTEE_MSG_CMD_UNREGISTER_SHM &&
+ call->xen_arg->ret == 0 ) {
+ free_shm_buf(ctx, call->xen_arg->params[0].u.rmem.shm_ref);
+ }
+
free_std_call_ctx(ctx, call);
return true;
@@ -522,7 +765,7 @@ static bool handle_std_call(struct domain_ctx
*ctx, struct cpu_user_regs *regs)
case OPTEE_MSG_CMD_CANCEL:
case OPTEE_MSG_CMD_REGISTER_SHM:
case OPTEE_MSG_CMD_UNREGISTER_SHM:
- ret = true;
+ ret = translate_params(ctx, call);
break;
default:
ret = false;
Cheers,
--
Volodymyr Babchuk
_______________________________________________
Xen-devel mailing list
Xen-devel@xxxxxxxxxxxxxxxxxxxx
https://lists.xenproject.org/mailman/listinfo/xen-devel
|
