diff options
author | Whi copybara merger <whitechapel-automerger@google.com> | 2022-04-08 05:43:32 +0000 |
---|---|---|
committer | Copybara-Service <copybara-worker@google.com> | 2022-04-15 18:12:50 -0700 |
commit | 262ab56ba3e4df3e23e549de20f319f00237a919 (patch) | |
tree | 1dc9a47c7f698589c0272898798c02436c8a72a4 | |
parent | 52eebd33cb649712fe418a18bdcc204bea1fc672 (diff) | |
download | janeiro-262ab56ba3e4df3e23e549de20f319f00237a919.tar.gz |
[Copybara Auto Merge] Merge branch 'pro' into android13-gs-pixel-5.10
edgetpu: fix system out of memory locking pages log message
edgetpu: add edgetpu_domain_pool
Bug: 217454089
GitOrigin-RevId: 831d96abe4484715e9fd38135706b121d02af109
Change-Id: I5362c287e511b3fdb51743f38fb568c2913eaa35
-rw-r--r-- | drivers/edgetpu/Kbuild | 2 | ||||
-rw-r--r-- | drivers/edgetpu/Makefile | 3 | ||||
-rw-r--r-- | drivers/edgetpu/edgetpu-device-group.c | 3 | ||||
-rw-r--r-- | drivers/edgetpu/edgetpu-domain-pool.c | 102 | ||||
-rw-r--r-- | drivers/edgetpu/edgetpu-domain-pool.h | 53 | ||||
-rw-r--r-- | drivers/edgetpu/edgetpu-google-iommu.c | 56 | ||||
-rw-r--r-- | drivers/edgetpu/janeiro/config.h | 3 |
7 files changed, 198 insertions, 24 deletions
diff --git a/drivers/edgetpu/Kbuild b/drivers/edgetpu/Kbuild index bb899dd..d0826ab 100644 --- a/drivers/edgetpu/Kbuild +++ b/drivers/edgetpu/Kbuild @@ -10,7 +10,7 @@ else ccflags-y += -DGIT_REPO_TAG=\"Not\ a\ git\ repository\" endif -edgetpu-objs := edgetpu-mailbox.o edgetpu-kci.o edgetpu-telemetry.o edgetpu-mapping.o edgetpu-dmabuf.o edgetpu-async.o edgetpu-iremap-pool.o edgetpu-sw-watchdog.o edgetpu-firmware.o edgetpu-firmware-util.o +edgetpu-objs := edgetpu-mailbox.o edgetpu-kci.o edgetpu-telemetry.o edgetpu-mapping.o edgetpu-dmabuf.o edgetpu-async.o edgetpu-iremap-pool.o edgetpu-sw-watchdog.o edgetpu-firmware.o edgetpu-firmware-util.o edgetpu-domain-pool.o janeiro-y := janeiro-device.o janeiro-device-group.o janeiro-fs.o janeiro-core.o janeiro-platform.o janeiro-firmware.o janeiro-thermal.o janeiro-pm.o janeiro-debug-dump.o janeiro-usage-stats.o janeiro-iommu.o janeiro-wakelock.o janeiro-external.o $(edgetpu-objs) diff --git a/drivers/edgetpu/Makefile b/drivers/edgetpu/Makefile index 4be6164..6b428ea 100644 --- a/drivers/edgetpu/Makefile +++ b/drivers/edgetpu/Makefile @@ -16,7 +16,8 @@ endif edgetpu-objs := edgetpu-async.o edgetpu-dmabuf.o edgetpu-iremap-pool.o \ edgetpu-kci.o edgetpu-mailbox.o edgetpu-mapping.o \ edgetpu-sw-watchdog.o edgetpu-telemetry.o \ - edgetpu-firmware-util.o edgetpu-firmware.o + edgetpu-firmware-util.o edgetpu-firmware.o \ + edgetpu-domain-pool.o janeiro-objs := janeiro-core.o janeiro-debug-dump.o janeiro-device-group.o \ janeiro-device.o janeiro-firmware.o janeiro-fs.o \ diff --git a/drivers/edgetpu/edgetpu-device-group.c b/drivers/edgetpu/edgetpu-device-group.c index 49c672d..066bb9c 100644 --- a/drivers/edgetpu/edgetpu-device-group.c +++ b/drivers/edgetpu/edgetpu-device-group.c @@ -1236,12 +1236,11 @@ static struct page **edgetpu_pin_user_pages(struct edgetpu_device_group *group, etdev_dbg(etdev, "pin_user_pages failed %u:%pK-%u: %d", group->workload_id, (void *)host_addr, num_pages, ret); - num_pages = 0; - if (ret == -ENOMEM) etdev_err(etdev, "system out of memory locking %u pages", num_pages); + num_pages = 0; goto error; } if (ret < num_pages) { diff --git a/drivers/edgetpu/edgetpu-domain-pool.c b/drivers/edgetpu/edgetpu-domain-pool.c new file mode 100644 index 0000000..2989254 --- /dev/null +++ b/drivers/edgetpu/edgetpu-domain-pool.c @@ -0,0 +1,102 @@ +// SPDX-License-Identifier: GPL-2.0 +/* + * EdgeTPU IOMMU domain allocator. + * + * Copyright (C) 2022 Google, LLC. + */ + +#include <linux/idr.h> +#include <linux/iommu.h> +#include <linux/slab.h> + +#include "edgetpu-domain-pool.h" +#include "edgetpu-internal.h" + +int edgetpu_domain_pool_init(struct edgetpu_dev *etdev, struct edgetpu_domain_pool *pool, + unsigned int size) +{ + unsigned int i; + struct iommu_domain *domain; + + pool->size = size; + pool->etdev = etdev; + + if (!size) + return 0; + + etdev_dbg(pool->etdev, "Initializing domain pool with %u domains\n", size); + + ida_init(&pool->idp); + pool->array = vzalloc(sizeof(*pool->array) * size); + if (!pool->array) { + etdev_err(etdev, "Failed to allocate memory for domain pool array\n"); + return -ENOMEM; + } + for (i = 0; i < size; i++) { + domain = iommu_domain_alloc(pool->etdev->dev->bus); + if (!domain) { + etdev_err(pool->etdev, "Failed to allocate iommu domain %d of %u\n", i + 1, + size); + edgetpu_domain_pool_destroy(pool); + return -ENOMEM; + } + pool->array[i] = domain; + } + return 0; +} + +struct iommu_domain *edgetpu_domain_pool_alloc(struct edgetpu_domain_pool *pool) +{ + int id; + + if (!pool->size) + return iommu_domain_alloc(pool->etdev->dev->bus); + + id = ida_alloc_max(&pool->idp, pool->size - 1, GFP_KERNEL); + + if (id < 0) { + etdev_err(pool->etdev, "No more domains available from pool of size %u\n", + pool->size); + return NULL; + } + + etdev_dbg(pool->etdev, "Allocated domain from pool with id = %d\n", id); + + return pool->array[id]; +} + +void edgetpu_domain_pool_free(struct edgetpu_domain_pool *pool, struct iommu_domain *domain) +{ + int id; + + if (!pool->size) { + iommu_domain_free(domain); + return; + } + for (id = 0; id < pool->size; id++) { + if (pool->array[id] == domain) { + etdev_dbg(pool->etdev, "Released domain from pool with id = %d\n", id); + ida_free(&pool->idp, id); + return; + } + } + etdev_err(pool->etdev, "%s: domain not found in pool", __func__); +} + +void edgetpu_domain_pool_destroy(struct edgetpu_domain_pool *pool) +{ + int i; + + if (!pool->size) + return; + + etdev_dbg(pool->etdev, "Destroying domain pool with %u domains\n", pool->size); + + for (i = 0; i < pool->size; i++) { + if (pool->array[i]) + iommu_domain_free(pool->array[i]); + } + + ida_destroy(&pool->idp); + vfree(pool->array); +} diff --git a/drivers/edgetpu/edgetpu-domain-pool.h b/drivers/edgetpu/edgetpu-domain-pool.h new file mode 100644 index 0000000..3dd19d3 --- /dev/null +++ b/drivers/edgetpu/edgetpu-domain-pool.h @@ -0,0 +1,53 @@ +/* SPDX-License-Identifier: GPL-2.0 */ +/* + * IOMMU domain allocator for edgetpu + * + * Copyright (C) 2022 Google LLC + */ + +#ifndef __EDGETPU_DOMAIN_POOL_H__ +#define __EDGETPU_DOMAIN_POOL_H__ + +#include <linux/idr.h> +#include <linux/iommu.h> + +#include "edgetpu-internal.h" + +struct edgetpu_domain_pool { + struct ida idp; /* ID allocator to keep track of used domains. */ + /* + * Size of the pool. Can be set to 0, in which case the implementation will fall back to + * dynamic domain allocation using the IOMMU API directly. + */ + unsigned int size; + struct iommu_domain **array; /* Array holding the pointers to pre-allocated domains. */ + struct edgetpu_dev *etdev; /* The edgetpu device used for logging warnings/errors. */ +}; + + +/* + * Initializes a domain pool. + * + * @etdev: pointer to edgeptu device. + * @pool: caller-allocated pool structure. + * @size: size of the pre-allocated domains pool. + * Set to zero to fall back to dynamically allocated domains. + * + * returns 0 on success or negative error value. + */ +int edgetpu_domain_pool_init(struct edgetpu_dev *etdev, struct edgetpu_domain_pool *pool, + unsigned int size); + +/* + * Allocates a domain from the pool + * returns NULL on error. + */ +struct iommu_domain *edgetpu_domain_pool_alloc(struct edgetpu_domain_pool *pool); + +/* Releases a domain from the pool. */ +void edgetpu_domain_pool_free(struct edgetpu_domain_pool *pool, struct iommu_domain *domain); + +/* Cleans up all resources used by the domain pool. */ +void edgetpu_domain_pool_destroy(struct edgetpu_domain_pool *pool); + +#endif /* __EDGETPU_DOMAIN_POOL_H__ */ diff --git a/drivers/edgetpu/edgetpu-google-iommu.c b/drivers/edgetpu/edgetpu-google-iommu.c index 0f28918..95578b5 100644 --- a/drivers/edgetpu/edgetpu-google-iommu.c +++ b/drivers/edgetpu/edgetpu-google-iommu.c @@ -14,10 +14,15 @@ #include <linux/types.h> #include "edgetpu-config.h" +#include "edgetpu-domain-pool.h" #include "edgetpu-internal.h" #include "edgetpu-mapping.h" #include "edgetpu-mmu.h" +#if !defined(EDGETPU_NUM_PREALLOCATED_DOMAINS) +#define EDGETPU_NUM_PREALLOCATED_DOMAINS 0 +#endif + struct edgetpu_iommu { struct iommu_group *iommu_group; /* @@ -26,13 +31,20 @@ struct edgetpu_iommu { */ struct iommu_domain *domains[EDGETPU_NCONTEXTS]; /* - * Records all domains currently allocated, to support IOMMU (un)mapping + * Records IDs for all domains currently allocated, to support IOMMU (un)mapping * when the domain is not attached. */ - struct idr domain_pool; - struct mutex pool_lock; /* protects access of @domain_pool */ + struct idr domain_id_pool; + struct mutex pool_lock; /* protects access of @domain_id_pool */ bool context_0_default; /* is context 0 domain the default? */ bool aux_enabled; + /* + * Holds a pool of pre-allocated IOMMU domains if the chip config specifies this is + * required. + * The implementation will fall back to dynamically allocated domains otherwise. + */ + struct edgetpu_domain_pool domain_pool; + }; struct edgetpu_iommu_map_params { @@ -57,7 +69,7 @@ static struct iommu_domain *get_domain_by_token(struct edgetpu_iommu *etiommu, struct iommu_domain *domain; mutex_lock(&etiommu->pool_lock); - domain = idr_find(&etiommu->domain_pool, token); + domain = idr_find(&etiommu->domain_id_pool, token); mutex_unlock(&etiommu->pool_lock); return domain; } @@ -131,8 +143,9 @@ static int edgetpu_unregister_iommu_device_fault_handler(struct edgetpu_dev *etd static int edgetpu_idr_free_domain_callback(int id, void *p, void *data) { struct iommu_domain *domain = p; + struct edgetpu_iommu *etiommu = data; - iommu_domain_free(domain); + edgetpu_domain_pool_free(&etiommu->domain_pool, domain); return 0; } @@ -181,7 +194,7 @@ static int check_default_domain(struct edgetpu_dev *etdev, if (!etiommu->aux_enabled) return -EINVAL; - domain = iommu_domain_alloc(etdev->dev->bus); + domain = edgetpu_domain_pool_alloc(&etiommu->domain_pool); if (!domain) { etdev_warn(etdev, "iommu domain alloc failed"); return -EINVAL; @@ -189,7 +202,7 @@ static int check_default_domain(struct edgetpu_dev *etdev, ret = iommu_aux_attach_device(domain, etdev->dev); if (ret) { etdev_warn(etdev, "Attach IOMMU aux failed: %d", ret); - iommu_domain_free(domain); + edgetpu_domain_pool_free(&etiommu->domain_pool, domain); return ret; } pasid = iommu_aux_get_pasid(domain, etdev->dev); @@ -198,7 +211,7 @@ static int check_default_domain(struct edgetpu_dev *etdev, etdev_warn(etdev, "Invalid PASID %d returned from iommu\n", pasid); iommu_aux_detach_device(domain, etdev->dev); - iommu_domain_free(domain); + edgetpu_domain_pool_free(&etiommu->domain_pool, domain); return -EINVAL; } out: @@ -215,7 +228,9 @@ int edgetpu_mmu_attach(struct edgetpu_dev *etdev, void *mmu_info) etiommu = kzalloc(sizeof(*etiommu), GFP_KERNEL); if (!etiommu) return -ENOMEM; - idr_init(&etiommu->domain_pool); + ret = edgetpu_domain_pool_init(etdev, &etiommu->domain_pool, + EDGETPU_NUM_PREALLOCATED_DOMAINS); + idr_init(&etiommu->domain_id_pool); mutex_init(&etiommu->pool_lock); etiommu->iommu_group = iommu_group_get(etdev->dev); if (etiommu->iommu_group) @@ -278,11 +293,12 @@ void edgetpu_mmu_detach(struct edgetpu_dev *etdev) /* free the domain if the context 0 domain is not default */ if (!etiommu->context_0_default && etiommu->domains[0]) - iommu_domain_free(etiommu->domains[0]); + edgetpu_domain_pool_free(&etiommu->domain_pool, etiommu->domains[0]); - idr_for_each(&etiommu->domain_pool, edgetpu_idr_free_domain_callback, - NULL); - idr_destroy(&etiommu->domain_pool); + idr_for_each(&etiommu->domain_id_pool, edgetpu_idr_free_domain_callback, + etiommu); + idr_destroy(&etiommu->domain_id_pool); + edgetpu_domain_pool_destroy(&etiommu->domain_pool); kfree(etiommu); etdev->mmu_cookie = NULL; } @@ -611,26 +627,26 @@ struct edgetpu_iommu_domain *edgetpu_mmu_alloc_domain(struct edgetpu_dev *etdev) if (!etiommu->aux_enabled) return &invalid_etdomain; - domain = iommu_domain_alloc(etdev->dev->bus); + domain = edgetpu_domain_pool_alloc(&etiommu->domain_pool); if (!domain) { - etdev_warn(etdev, "iommu domain alloc failed"); + etdev_warn(etdev, "iommu domain allocation failed"); return NULL; } etdomain = kzalloc(sizeof(*etdomain), GFP_KERNEL); if (!etdomain) { - iommu_domain_free(domain); + edgetpu_domain_pool_free(&etiommu->domain_pool, domain); return NULL; } mutex_lock(&etiommu->pool_lock); - token = idr_alloc(&etiommu->domain_pool, domain, 0, + token = idr_alloc(&etiommu->domain_id_pool, domain, 0, EDGETPU_DOMAIN_TOKEN_END, GFP_KERNEL); mutex_unlock(&etiommu->pool_lock); if (token < 0) { etdev_warn(etdev, "alloc iommu domain token failed: %d", token); kfree(etdomain); - iommu_domain_free(domain); + edgetpu_domain_pool_free(&etiommu->domain_pool, domain); return NULL; } @@ -650,9 +666,9 @@ void edgetpu_mmu_free_domain(struct edgetpu_dev *etdev, edgetpu_mmu_detach_domain(etdev, etdomain); } mutex_lock(&etiommu->pool_lock); - idr_remove(&etiommu->domain_pool, etdomain->token); + idr_remove(&etiommu->domain_id_pool, etdomain->token); mutex_unlock(&etiommu->pool_lock); - iommu_domain_free(etdomain->iommu_domain); + edgetpu_domain_pool_free(&etiommu->domain_pool, etdomain->iommu_domain); kfree(etdomain); } diff --git a/drivers/edgetpu/janeiro/config.h b/drivers/edgetpu/janeiro/config.h index 063f705..f179ca9 100644 --- a/drivers/edgetpu/janeiro/config.h +++ b/drivers/edgetpu/janeiro/config.h @@ -30,6 +30,9 @@ /* Reserved VCID that uses the extra partition. */ #define EDGETPU_VCID_EXTRA_PARTITION 0 +/* Pre-allocate 1 IOMMU domain per VCID */ +#define EDGETPU_NUM_PREALLOCATED_DOMAINS EDGETPU_NUM_VCIDS + /* Is a "mobile" style device. */ #define EDGETPU_FEATURE_MOBILE #define EDGETPU_HAS_WAKELOCK |