From 7bb5660317e125dee80c6adbb27d1a3b136c15f4 Mon Sep 17 00:00:00 2001 From: Zuma copybara merger Date: Thu, 16 Mar 2023 13:18:05 +0000 Subject: [Copybara Auto Merge] Merge branch zuma into android14-gs-pixel-5.15 Revert "edgetpu: create debugfs symlinks for non-default device names" edgetpu: create debugfs symlinks for non-default device names gcip: Match gcip_image_config struct with firmware Bug: 263852751 gcip: Add state2power_table Bug: 271305993 edgetpu: Make group required for creating fence Bug: 258868303 edgetpu: Adopt GCIP DMA fence Bug: 264971968 edgetpu: Add etdev as private data for syncfences Bug: 264971968 (repeat) Signed-off-by: Zuma copybara merger GitOrigin-RevId: 04818a72f70612a750d21d85633efc7f9e679bb8 Change-Id: I8d3a6f072acb3e96802b46fe19783cfbb1c3ffa5 --- drivers/edgetpu/edgetpu-dmabuf.c | 260 ++++++--------------- drivers/edgetpu/edgetpu-dmabuf.h | 4 +- drivers/edgetpu/edgetpu-fs.c | 42 ++-- drivers/edgetpu/edgetpu-internal.h | 2 + drivers/edgetpu/edgetpu-mobile-platform.c | 7 + .../gcip-kernel-driver/drivers/gcip/gcip-thermal.c | 22 ++ .../include/gcip/gcip-image-config.h | 5 +- 7 files changed, 127 insertions(+), 215 deletions(-) diff --git a/drivers/edgetpu/edgetpu-dmabuf.c b/drivers/edgetpu/edgetpu-dmabuf.c index d6a438d..4ca8c40 100644 --- a/drivers/edgetpu/edgetpu-dmabuf.c +++ b/drivers/edgetpu/edgetpu-dmabuf.c @@ -21,6 +21,8 @@ #include #include +#include + #include "edgetpu-device-group.h" #include "edgetpu-dmabuf.h" #include "edgetpu-internal.h" @@ -28,6 +30,8 @@ #include "edgetpu-mmu.h" #include "edgetpu.h" +#define to_etfence(gfence) container_of(gfence, struct edgetpu_dma_fence, gfence) + /* * Records objects for mapping a dma-buf to an edgetpu_dev. */ @@ -57,29 +61,16 @@ struct edgetpu_dmabuf_map { /* * edgetpu implementation of DMA fence * - * @fence: the base DMA fence - * @lock: spinlock protecting updates to @fence - * @timeline_name: name of the timeline associated with the fence + * @gfence: GCIP DMA fence * @group: owning device group - * @etfence_list: global list of all edgetpu DMA fences * @group_list: list of DMA fences owned by the same group - * - * It is likely timelines will become a separate object in the future, - * but for now there's a unique named timeline associated with each fence. */ struct edgetpu_dma_fence { - struct dma_fence fence; - spinlock_t lock; - char timeline_name[EDGETPU_SYNC_TIMELINE_NAME_LEN]; + struct gcip_dma_fence gfence; struct edgetpu_device_group *group; - struct list_head etfence_list; struct list_head group_list; }; -/* List of all edgetpu fence objects for debugging. */ -static LIST_HEAD(etfence_list_head); -static DEFINE_SPINLOCK(etfence_list_lock); - static const struct dma_fence_ops edgetpu_dma_fence_ops; /* @@ -459,15 +450,16 @@ out_unlock: return ret; } -static struct edgetpu_dma_fence *to_etfence(struct dma_fence *fence) +int edgetpu_sync_fence_manager_create(struct edgetpu_dev *etdev) { - struct edgetpu_dma_fence *etfence; + struct gcip_dma_fence_manager *gfence_mgr = gcip_dma_fence_manager_create(etdev->dev); - etfence = container_of(fence, struct edgetpu_dma_fence, fence); - if (fence->ops != &edgetpu_dma_fence_ops) - return NULL; + if (IS_ERR(gfence_mgr)) + return PTR_ERR(gfence_mgr); - return etfence; + etdev->gfence_mgr = gfence_mgr; + + return 0; } static const char *edgetpu_dma_fence_get_driver_name(struct dma_fence *fence) @@ -475,152 +467,74 @@ static const char *edgetpu_dma_fence_get_driver_name(struct dma_fence *fence) return "edgetpu"; } -static const char *edgetpu_dma_fence_get_timeline_name(struct dma_fence *fence) -{ - struct edgetpu_dma_fence *etfence = to_etfence(fence); - - return etfence->timeline_name; -} - static void edgetpu_dma_fence_release(struct dma_fence *fence) { - struct edgetpu_dma_fence *etfence = to_etfence(fence); - struct edgetpu_device_group *group; - unsigned long flags; - - if (!etfence) - return; - - spin_lock_irqsave(&etfence_list_lock, flags); - list_del(&etfence->etfence_list); - spin_unlock_irqrestore(&etfence_list_lock, flags); - - /* TODO(b/258868303): Don't remove this check when group required, might not yet be set. */ - group = etfence->group; - if (group) { - mutex_lock(&group->lock); - list_del(&etfence->group_list); - mutex_unlock(&group->lock); - /* Release this fence's reference on the owning group. */ - edgetpu_device_group_put(group); - } + struct gcip_dma_fence *gfence = to_gcip_fence(fence); + struct edgetpu_dma_fence *etfence = to_etfence(gfence); + struct edgetpu_device_group *group = etfence->group; + mutex_lock(&group->lock); + list_del(&etfence->group_list); + mutex_unlock(&group->lock); + /* Release this fence's reference on the owning group. */ + edgetpu_device_group_put(group); + gcip_dma_fence_exit(gfence); kfree(etfence); } -static bool edgetpu_dma_fence_enable_signaling(struct dma_fence *fence) -{ - return true; -} - static const struct dma_fence_ops edgetpu_dma_fence_ops = { .get_driver_name = edgetpu_dma_fence_get_driver_name, - .get_timeline_name = edgetpu_dma_fence_get_timeline_name, + .get_timeline_name = gcip_dma_fence_get_timeline_name, .wait = dma_fence_default_wait, - .enable_signaling = edgetpu_dma_fence_enable_signaling, + .enable_signaling = gcip_dma_fence_always_true, .release = edgetpu_dma_fence_release, }; -int edgetpu_sync_fence_create(struct edgetpu_device_group *group, - struct edgetpu_create_sync_fence_data *datap) +static int edgetpu_dma_fence_after_init(struct gcip_dma_fence *gfence) { - int fd = get_unused_fd_flags(O_CLOEXEC); - int ret; - struct edgetpu_dma_fence *etfence; - struct sync_file *sync_file; - unsigned long flags; + struct edgetpu_dma_fence *etfence = to_etfence(gfence); + struct edgetpu_device_group *group = etfence->group; - if (fd < 0) - return fd; - etfence = kzalloc(sizeof(*etfence), GFP_KERNEL); - if (!etfence) { - ret = -ENOMEM; - goto err_put_fd; - } - - spin_lock_init(&etfence->lock); - /* - * If sync_file_create() fails, fence release is called on dma_fence_put(). A valid - * list_head is needed for list_del(). - */ - INIT_LIST_HEAD(&etfence->etfence_list); - INIT_LIST_HEAD(&etfence->group_list); - memcpy(&etfence->timeline_name, &datap->timeline_name, - EDGETPU_SYNC_TIMELINE_NAME_LEN - 1); - - dma_fence_init(&etfence->fence, &edgetpu_dma_fence_ops, - &etfence->lock, dma_fence_context_alloc(1), - datap->seqno); - - sync_file = sync_file_create(&etfence->fence); - dma_fence_put(&etfence->fence); - if (!sync_file) { - ret = -ENOMEM; - /* doesn't need kfree(etfence) here: dma_fence_put does it for us */ - goto err_put_fd; - } - - spin_lock_irqsave(&etfence_list_lock, flags); - list_add_tail(&etfence->etfence_list, &etfence_list_head); - spin_unlock_irqrestore(&etfence_list_lock, flags); - - /* TODO(b/258868303): Make group required, disallow creating fence we can't track. */ - if (group) { - etfence->group = edgetpu_device_group_get(group); - mutex_lock(&group->lock); - list_add_tail(&etfence->group_list, &group->dma_fence_list); - mutex_unlock(&group->lock); - } + mutex_lock(&group->lock); + list_add_tail(&etfence->group_list, &group->dma_fence_list); + mutex_unlock(&group->lock); - fd_install(fd, sync_file->file); - datap->fence = fd; return 0; - -err_put_fd: - put_unused_fd(fd); - return ret; } -static int _edgetpu_sync_fence_signal(struct dma_fence *fence, int errno, bool ignore_signaled) +int edgetpu_sync_fence_create(struct edgetpu_dev *etdev, struct edgetpu_device_group *group, + struct edgetpu_create_sync_fence_data *datap) { + struct gcip_dma_fence_data data = { + .timeline_name = datap->timeline_name, + .ops = &edgetpu_dma_fence_ops, + .seqno = datap->seqno, + .after_init = edgetpu_dma_fence_after_init, + }; + struct edgetpu_dma_fence *etfence = kzalloc(sizeof(*etfence), GFP_KERNEL); int ret; - spin_lock_irq(fence->lock); - /* don't signal fence twice */ - if (unlikely(test_bit(DMA_FENCE_FLAG_SIGNALED_BIT, &fence->flags))) { - ret = ignore_signaled ? 0 : -EINVAL; - goto out_unlock; - } - pr_debug("%s: %s-%s%llu-%llu errno=%d\n", __func__, fence->ops->get_driver_name(fence), - fence->ops->get_timeline_name(fence), fence->context, fence->seqno, errno); - if (errno) - dma_fence_set_error(fence, errno); - ret = dma_fence_signal_locked(fence); + if (!etfence) + return -ENOMEM; + + INIT_LIST_HEAD(&etfence->group_list); + etfence->group = edgetpu_device_group_get(group); + + ret = gcip_dma_fence_init(etdev->gfence_mgr, &etfence->gfence, &data); + if (!ret) + datap->fence = data.fence; + + /* + * We don't need to kfree(etfence) on error because that's called in + * edgetpu_dma_fence_release. + */ -out_unlock: - spin_unlock_irq(fence->lock); return ret; } int edgetpu_sync_fence_signal(struct edgetpu_signal_sync_fence_data *datap) { - struct dma_fence *fence; - int errno; - int ret; - - errno = datap->error; - if (errno > 0) - errno = -errno; - if (errno < -MAX_ERRNO) - return -EINVAL; - - fence = sync_file_get_fence(datap->fence); - if (!fence) - return -EINVAL; - - ret = _edgetpu_sync_fence_signal(fence, errno, false); - dma_fence_put(fence); - return ret; + return gcip_dma_fence_signal(datap->fence, datap->error, false); } /* Caller holds group lock. */ @@ -633,75 +547,39 @@ void edgetpu_sync_fence_group_shutdown(struct edgetpu_device_group *group) list_for_each(pos, &group->dma_fence_list) { struct edgetpu_dma_fence *etfence = container_of(pos, struct edgetpu_dma_fence, group_list); - struct dma_fence *fence = &etfence->fence; - ret = _edgetpu_sync_fence_signal(fence, -EPIPE, true); - if (ret) + ret = gcip_dma_fenceptr_signal(&etfence->gfence, -EPIPE, true); + if (ret) { + struct dma_fence *fence = &etfence->gfence.fence; + etdev_warn(group->etdev, "error %d signaling fence %s-%s %llu-%llu", ret, fence->ops->get_driver_name(fence), fence->ops->get_timeline_name(fence), fence->context, fence->seqno); + } } } int edgetpu_sync_fence_status(struct edgetpu_sync_fence_status *datap) { - struct dma_fence *fence; - - fence = sync_file_get_fence(datap->fence); - if (!fence) - return -EINVAL; - - datap->status = dma_fence_get_status(fence); - dma_fence_put(fence); - return 0; -} - -static const char *sync_status_str(int status) -{ - if (status < 0) - return "error"; - - if (status > 0) - return "signaled"; - - return "active"; + return gcip_dma_fence_status(datap->fence, &datap->status); } int edgetpu_sync_fence_debugfs_show(struct seq_file *s, void *unused) { - struct list_head *pos; - - spin_lock_irq(&etfence_list_lock); - list_for_each(pos, &etfence_list_head) { - struct edgetpu_dma_fence *etfence = - container_of(pos, struct edgetpu_dma_fence, - etfence_list); - struct dma_fence *fence = &etfence->fence; - - spin_lock_irq(&etfence->lock); - seq_printf(s, "%s-%s %llu-%llu %s", fence->ops->get_driver_name(fence), - fence->ops->get_timeline_name(fence), fence->context, fence->seqno, - sync_status_str(dma_fence_get_status_locked(fence))); - - if (test_bit(DMA_FENCE_FLAG_TIMESTAMP_BIT, &fence->flags)) { - struct timespec64 ts64 = - ktime_to_timespec64(fence->timestamp); + struct edgetpu_dev *etdev = s->private; + struct gcip_dma_fence *gfence; + unsigned long flags; - seq_printf(s, " @%lld.%09ld", (s64)ts64.tv_sec, - ts64.tv_nsec); - } + GCIP_DMA_FENCE_LIST_LOCK(etdev->gfence_mgr, flags); + gcip_for_each_fence(etdev->gfence_mgr, gfence) { + struct edgetpu_dma_fence *etfence = to_etfence(gfence); - if (fence->error) - seq_printf(s, " err=%d", fence->error); - /* TODO(b/258868303): Remove check when group is required. */ - if (etfence->group) - seq_printf(s, " group=%u", etfence->group->workload_id); - seq_putc(s, '\n'); - spin_unlock_irq(&etfence->lock); + gcip_dma_fence_show(gfence, s); + seq_printf(s, " group=%u\n", etfence->group->workload_id); } + GCIP_DMA_FENCE_LIST_UNLOCK(etdev->gfence_mgr, flags); - spin_unlock_irq(&etfence_list_lock); return 0; } diff --git a/drivers/edgetpu/edgetpu-dmabuf.h b/drivers/edgetpu/edgetpu-dmabuf.h index 70af6fd..a4225a4 100644 --- a/drivers/edgetpu/edgetpu-dmabuf.h +++ b/drivers/edgetpu/edgetpu-dmabuf.h @@ -25,8 +25,10 @@ int edgetpu_map_dmabuf(struct edgetpu_device_group *group, /* unmap the dma-buf backed buffer from a device group */ int edgetpu_unmap_dmabuf(struct edgetpu_device_group *group, tpu_addr_t tpu_addr); +/* Creates a DMA sync fence manager. */ +int edgetpu_sync_fence_manager_create(struct edgetpu_dev *etdev); /* Create a DMA sync fence via ioctl */ -int edgetpu_sync_fence_create(struct edgetpu_device_group *group, +int edgetpu_sync_fence_create(struct edgetpu_dev *etdev, struct edgetpu_device_group *group, struct edgetpu_create_sync_fence_data *datap); /* Signal a DMA sync fence, optionally specifying error status */ int edgetpu_sync_fence_signal(struct edgetpu_signal_sync_fence_data *datap); diff --git a/drivers/edgetpu/edgetpu-fs.c b/drivers/edgetpu/edgetpu-fs.c index a3ecb11..888e301 100644 --- a/drivers/edgetpu/edgetpu-fs.c +++ b/drivers/edgetpu/edgetpu-fs.c @@ -387,11 +387,12 @@ static int edgetpu_ioctl_sync_fence_create( if (copy_from_user(&data, (void __user *)datap, sizeof(data))) return -EFAULT; LOCK(client); - if (!client->group) - /* TODO(b/258868303): Require a group, disallow creating a fence we can't track. */ - etdev_warn(client->etdev, - "client creating sync fence not joined to a device group"); - ret = edgetpu_sync_fence_create(client->group, &data); + if (!client->group) { + etdev_err(client->etdev, "client creating sync fence not joined to a device group"); + UNLOCK(client); + return -EINVAL; + } + ret = edgetpu_sync_fence_create(client->etdev, client->group, &data); UNLOCK(client); if (ret) return ret; @@ -800,6 +801,19 @@ static const struct file_operations mappings_ops = { .release = single_release, }; +static int syncfences_open(struct inode *inode, struct file *file) +{ + return single_open(file, edgetpu_sync_fence_debugfs_show, inode->i_private); +} + +static const struct file_operations syncfences_ops = { + .open = syncfences_open, + .read = seq_read, + .llseek = seq_lseek, + .owner = THIS_MODULE, + .release = single_release, +}; + static int edgetpu_pm_debugfs_set_wakelock(void *data, u64 val) { struct edgetpu_dev *etdev = data; @@ -824,6 +838,7 @@ static void edgetpu_fs_setup_debugfs(struct edgetpu_dev *etdev) } debugfs_create_file("mappings", 0440, etdev->d_entry, etdev, &mappings_ops); + debugfs_create_file("syncfences", 0440, etdev->d_entry, etdev, &syncfences_ops); debugfs_create_file("wakelock", 0220, etdev->d_entry, etdev, &fops_wakelock); } @@ -1062,20 +1077,6 @@ void edgetpu_fs_remove(struct edgetpu_dev *etdev) debugfs_remove_recursive(etdev->d_entry); } -static int syncfences_open(struct inode *inode, struct file *file) -{ - return single_open(file, edgetpu_sync_fence_debugfs_show, - inode->i_private); -} - -static const struct file_operations syncfences_ops = { - .open = syncfences_open, - .read = seq_read, - .llseek = seq_lseek, - .owner = THIS_MODULE, - .release = single_release, -}; - static void edgetpu_debugfs_global_setup(void) { edgetpu_debugfs_dir = debugfs_create_dir("edgetpu", NULL); @@ -1083,9 +1084,6 @@ static void edgetpu_debugfs_global_setup(void) pr_warn(DRIVER_NAME " error creating edgetpu debugfs dir\n"); return; } - - debugfs_create_file("syncfences", 0440, edgetpu_debugfs_dir, NULL, - &syncfences_ops); } int __init edgetpu_fs_init(void) diff --git a/drivers/edgetpu/edgetpu-internal.h b/drivers/edgetpu/edgetpu-internal.h index caf8ef4..1179ee4 100644 --- a/drivers/edgetpu/edgetpu-internal.h +++ b/drivers/edgetpu/edgetpu-internal.h @@ -30,6 +30,7 @@ #include #include +#include #include #include #include @@ -205,6 +206,7 @@ struct edgetpu_dev { /* Memory pool in instruction remap region */ struct edgetpu_mempool *iremap_pool; struct edgetpu_sw_wdt *etdev_sw_wdt; /* software watchdog */ + struct gcip_dma_fence_manager *gfence_mgr; /* DMA sync fences manager */ /* version read from the firmware binary file */ struct edgetpu_fw_version fw_version; atomic_t job_count; /* times joined to a device group */ diff --git a/drivers/edgetpu/edgetpu-mobile-platform.c b/drivers/edgetpu/edgetpu-mobile-platform.c index 0f38c29..e9d9a5e 100644 --- a/drivers/edgetpu/edgetpu-mobile-platform.c +++ b/drivers/edgetpu/edgetpu-mobile-platform.c @@ -16,6 +16,7 @@ #include #include "edgetpu-config.h" +#include "edgetpu-dmabuf.h" #include "edgetpu-internal.h" #include "edgetpu-iremap-pool.h" #include "edgetpu-mmu.h" @@ -421,6 +422,12 @@ static int edgetpu_mobile_platform_probe(struct platform_device *pdev, goto out_destroy_thermal; } + ret = edgetpu_sync_fence_manager_create(etdev); + if (ret) { + etdev_err(etdev, "Failed to create DMA fence manager: %d", ret); + goto out_destroy_thermal; + } + if (etmdev->after_probe) { ret = etmdev->after_probe(etmdev); if (ret) { diff --git a/drivers/edgetpu/gcip-kernel-driver/drivers/gcip/gcip-thermal.c b/drivers/edgetpu/gcip-kernel-driver/drivers/gcip/gcip-thermal.c index bc06cd5..ed56de2 100644 --- a/drivers/edgetpu/gcip-kernel-driver/drivers/gcip/gcip-thermal.c +++ b/drivers/edgetpu/gcip-kernel-driver/drivers/gcip/gcip-thermal.c @@ -272,6 +272,24 @@ struct notifier_block *gcip_thermal_get_notifier_block(struct gcip_thermal *ther return &thermal->nb; } +static ssize_t state2power_table_show(struct device *dev, struct device_attribute *attr, char *buf) +{ + struct gcip_thermal *thermal = to_gcip_thermal(dev); + ssize_t count = 0; + int i; + u32 power; + + for (i = 0; i < thermal->num_states; i++) { + gcip_thermal_state2power(thermal->cdev, i, &power); + count += sysfs_emit_at(buf, count, "%u ", power); + } + count += sysfs_emit_at(buf, count, "\n"); + + return count; +} + +static DEVICE_ATTR_RO(state2power_table); + void gcip_thermal_destroy(struct gcip_thermal *thermal) { if (IS_ERR_OR_NULL(thermal)) @@ -391,6 +409,10 @@ static int gcip_thermal_cooling_register(struct gcip_thermal *thermal, const cha if (ret) thermal_cooling_device_unregister(thermal->cdev); + ret = device_create_file(&thermal->cdev->device, &dev_attr_state2power_table); + if (ret) + thermal_cooling_device_unregister(thermal->cdev); + return ret; } diff --git a/drivers/edgetpu/gcip-kernel-driver/include/gcip/gcip-image-config.h b/drivers/edgetpu/gcip-kernel-driver/include/gcip/gcip-image-config.h index df09d39..5a2e4a6 100644 --- a/drivers/edgetpu/gcip-kernel-driver/include/gcip/gcip-image-config.h +++ b/drivers/edgetpu/gcip-kernel-driver/include/gcip/gcip-image-config.h @@ -13,8 +13,9 @@ #include #define GCIP_FW_NUM_VERSIONS 4 -#define GCIP_IMG_CFG_MAX_IOMMU_MAPPINGS 22 +#define GCIP_IMG_CFG_MAX_IOMMU_MAPPINGS 20 #define GCIP_IMG_CFG_MAX_NS_IOMMU_MAPPINGS 5 +#define GCIP_IMG_CFG_MAX_PROTECTED_MEMORY_MAPPINGS 3 #define GCIP_FW_PRIV_LEVEL_GSA 0 #define GCIP_FW_PRIV_LEVEL_TZ 1 @@ -43,6 +44,8 @@ struct gcip_image_config { */ __u32 image_config_value; } iommu_mappings[GCIP_IMG_CFG_MAX_IOMMU_MAPPINGS]; + __u32 protected_memory_regions[GCIP_IMG_CFG_MAX_PROTECTED_MEMORY_MAPPINGS]; + __u32 secure_telemetry_region_start; __u32 remapped_data_start; __u32 remapped_data_size; __u32 num_ns_iommu_mappings; -- cgit v1.2.3