msm: camera: ife: check cdm hang in the ife config timeout

IFE driver submit the config packet to cdm and wait for call back.
If call back has not received with in 30ms, then ife return failure
which cause ife stream on failure. Some time cdm call back is not coming
due to cdm worker thread execution delay. So add mechanism to check if
cdm worker thread delay happened then wait for some more time
and check it again.

Bug: 159088802
Test: GCA, CTS/ITS

CRs-Fixed: 2672759
Change-Id: Ic28299517e563d39151d56b01943930b541176da
Signed-off-by: Chandan Kumar Jha <cjha@codeaurora.org>

7 files changed, 121 insertions, 8 deletions

diff --git a/drivers/cam_cdm/cam_cdm.h b/drivers/cam_cdm/cam_cdm.h
index c20e0d6..54882fb 100644
--- a/drivers/cam_cdm/cam_cdm.h
+++ b/drivers/cam_cdm/cam_cdm.h
@@ -1,6 +1,6 @@
 /* SPDX-License-Identifier: GPL-2.0-only */
 /*
- * Copyright (c) 2017-2019, The Linux Foundation. All rights reserved.
+ * Copyright (c) 2017-2020, The Linux Foundation. All rights reserved.
  */
 
 #ifndef _CAM_CDM_H_
@@ -52,6 +52,7 @@ enum cam_cdm_hw_process_intf_cmd {
 	CAM_CDM_HW_INTF_CMD_RELEASE,
 	CAM_CDM_HW_INTF_CMD_SUBMIT_BL,
 	CAM_CDM_HW_INTF_CMD_RESET_HW,
+	CAM_CDM_HW_INTF_CMD_HANG_DETECT,
 	CAM_CDM_HW_INTF_CMD_INVALID,
 };
 
@@ -217,6 +218,7 @@ struct cam_cdm {
 	atomic_t bl_done;
 	struct cam_cdm_hw_mem gen_irq;
 	uint32_t cpas_handle;
+	atomic_t work_record;
 };
 
 /* struct cam_cdm_private_dt_data - CDM hw custom dt data */
diff --git a/drivers/cam_cdm/cam_cdm_core_common.c b/drivers/cam_cdm/cam_cdm_core_common.c
index e903dc8..ee9f494 100644
--- a/drivers/cam_cdm/cam_cdm_core_common.c
+++ b/drivers/cam_cdm/cam_cdm_core_common.c
@@ -1,6 +1,6 @@
 // SPDX-License-Identifier: GPL-2.0-only
 /*
- * Copyright (c) 2017-2019, The Linux Foundation. All rights reserved.
+ * Copyright (c) 2017-2020, The Linux Foundation. All rights reserved.
  */
 
 #include <linux/delay.h>
@@ -579,6 +579,31 @@ int cam_cdm_process_cmd(void *hw_priv,
 			*((uint32_t *)cmd_args));
 		break;
 	}
+	case CAM_CDM_HW_INTF_CMD_HANG_DETECT: {
+		uint32_t *handle = cmd_args;
+		int idx;
+		struct cam_cdm_client *client;
+
+		if (sizeof(uint32_t) != arg_size) {
+			CAM_ERR(CAM_CDM,
+				"Invalid CDM cmd %d size=%x for handle=%x",
+				cmd, arg_size, *handle);
+				return -EINVAL;
+		}
+
+		idx = CAM_CDM_GET_CLIENT_IDX(*handle);
+		mutex_lock(&cdm_hw->hw_mutex);
+		client = core->clients[idx];
+		if ((!client) || (*handle != client->handle)) {
+			CAM_ERR(CAM_CDM, "Invalid client %pK hdl=%x",
+				client, *handle);
+			mutex_unlock(&cdm_hw->hw_mutex);
+			break;
+		}
+		rc = cam_hw_cdm_hang_detect(cdm_hw, *handle);
+		mutex_unlock(&cdm_hw->hw_mutex);
+		break;
+	}
 	default:
 		CAM_ERR(CAM_CDM, "CDM HW intf command not valid =%d", cmd);
 		break;
diff --git a/drivers/cam_cdm/cam_cdm_core_common.h b/drivers/cam_cdm/cam_cdm_core_common.h
index 8dcbe8e..64dc52d 100644
--- a/drivers/cam_cdm/cam_cdm_core_common.h
+++ b/drivers/cam_cdm/cam_cdm_core_common.h
@@ -1,6 +1,6 @@
 /* SPDX-License-Identifier: GPL-2.0-only */
 /*
- * Copyright (c) 2017-2019, The Linux Foundation. All rights reserved.
+ * Copyright (c) 2017-2020, The Linux Foundation. All rights reserved.
  */
 
 #ifndef _CAM_CDM_CORE_COMMON_H_
@@ -37,6 +37,7 @@ int cam_virtual_cdm_submit_bl(struct cam_hw_info *cdm_hw,
 int cam_hw_cdm_submit_bl(struct cam_hw_info *cdm_hw,
 	struct cam_cdm_hw_intf_cmd_submit_bl *req,
 	struct cam_cdm_client *client);
+int cam_hw_cdm_hang_detect(struct cam_hw_info *cdm_hw, uint32_t handle);
 struct cam_cdm_bl_cb_request_entry *cam_cdm_find_request_by_bl_tag(
 	uint32_t tag, struct list_head *bl_list);
 void cam_cdm_notify_clients(struct cam_hw_info *cdm_hw,
diff --git a/drivers/cam_cdm/cam_cdm_hw_core.c b/drivers/cam_cdm/cam_cdm_hw_core.c
index 92f30ec..f68c393 100644
--- a/drivers/cam_cdm/cam_cdm_hw_core.c
+++ b/drivers/cam_cdm/cam_cdm_hw_core.c
@@ -1,6 +1,6 @@
 // SPDX-License-Identifier: GPL-2.0-only
 /*
- * Copyright (c) 2017-2019, The Linux Foundation. All rights reserved.
+ * Copyright (c) 2017-2020, The Linux Foundation. All rights reserved.
  */
 
 #include <linux/delay.h>
@@ -563,6 +563,10 @@ static void cam_hw_cdm_work(struct work_struct *work)
 			CAM_DBG(CAM_CDM, "inline IRQ data=0x%x",
 				payload->irq_data);
 			mutex_lock(&cdm_hw->hw_mutex);
+
+			if (atomic_read(&core->work_record))
+				atomic_dec(&core->work_record);
+
 			list_for_each_entry_safe(node, tnode,
 					&core->bl_request_list, entry) {
 				if (node->request_type ==
@@ -689,6 +693,9 @@ irqreturn_t cam_hw_cdm_irq(int irq_num, void *data)
 		payload->hw = cdm_hw;
 		INIT_WORK((struct work_struct *)&payload->work,
 			cam_hw_cdm_work);
+		if (payload->irq_status &
+			CAM_CDM_IRQ_STATUS_INFO_INLINE_IRQ_MASK)
+			atomic_inc(&cdm_core->work_record);
 		work_status = queue_work(cdm_core->work_queue, &payload->work);
 		if (work_status == false) {
 			CAM_ERR(CAM_CDM, "Failed to queue work for irq=0x%x",
@@ -700,6 +707,22 @@ irqreturn_t cam_hw_cdm_irq(int irq_num, void *data)
 	return IRQ_HANDLED;
 }
 
+int cam_hw_cdm_hang_detect(struct cam_hw_info *cdm_hw, uint32_t handle)
+{
+	struct cam_cdm *cdm_core = NULL;
+	int rc = -1;
+
+	cdm_core = (struct cam_cdm *)cdm_hw->core_info;
+
+	if (atomic_read(&cdm_core->work_record)) {
+		CAM_WARN(CAM_CDM,
+			"workqueue got delayed, work_record :%u",
+			 atomic_read(&cdm_core->work_record));
+		rc = 0;
+	}
+	return rc;
+}
+
 int cam_hw_cdm_alloc_genirq_mem(void *hw_priv)
 {
 	struct cam_hw_info *cdm_hw = hw_priv;
@@ -775,6 +798,7 @@ int cam_hw_cdm_init(void *hw_priv,
 	CAM_DBG(CAM_CDM, "Enable soc done");
 
 /* Before triggering the reset to HW, clear the reset complete */
+	atomic_set(&cdm_core->work_record, 0);
 	atomic_set(&cdm_core->error, 0);
 	atomic_set(&cdm_core->bl_done, 0);
 	reinit_completion(&cdm_core->reset_complete);
@@ -824,6 +848,7 @@ int cam_hw_cdm_deinit(void *hw_priv,
 
 	soc_info = &cdm_hw->soc_info;
 	cdm_core = cdm_hw->core_info;
+	atomic_set(&cdm_core->work_record, 0);
 	rc = cam_soc_util_disable_platform_resource(soc_info, true, true);
 	if (rc) {
 		CAM_ERR(CAM_CDM, "disable platform failed");
@@ -888,6 +913,7 @@ int cam_hw_cdm_probe(struct platform_device *pdev)
 		cdm_core->flags = CAM_CDM_FLAG_PRIVATE_CDM;
 
 	cdm_core->bl_tag = 0;
+	atomic_set(&cdm_core->work_record, 0);
 	cdm_core->id = cam_hw_cdm_get_id_by_name(cdm_core->name);
 	if (cdm_core->id >= CAM_CDM_MAX) {
 		CAM_ERR(CAM_CDM, "Failed to get CDM HW name for %s",
diff --git a/drivers/cam_cdm/cam_cdm_intf.c b/drivers/cam_cdm/cam_cdm_intf.c
index 7694463..599991e 100644
--- a/drivers/cam_cdm/cam_cdm_intf.c
+++ b/drivers/cam_cdm/cam_cdm_intf.c
@@ -1,6 +1,6 @@
 // SPDX-License-Identifier: GPL-2.0-only
 /*
- * Copyright (c) 2017-2019, The Linux Foundation. All rights reserved.
+ * Copyright (c) 2017-2020, The Linux Foundation. All rights reserved.
  */
 
 #include <linux/delay.h>
@@ -379,6 +379,33 @@ int cam_cdm_reset_hw(uint32_t handle)
 }
 EXPORT_SYMBOL(cam_cdm_reset_hw);
 
+int cam_cdm_detect_hang_error(uint32_t handle)
+{
+	uint32_t hw_index;
+	int rc = -EINVAL;
+	struct cam_hw_intf *hw;
+
+	if (get_cdm_mgr_refcount()) {
+		CAM_ERR(CAM_CDM, "CDM intf mgr get refcount failed");
+		rc = -EPERM;
+		return rc;
+	}
+
+	hw_index = CAM_CDM_GET_HW_IDX(handle);
+	if (hw_index < CAM_CDM_INTF_MGR_MAX_SUPPORTED_CDM) {
+		hw = cdm_mgr.nodes[hw_index].device;
+		if (hw && hw->hw_ops.process_cmd)
+			rc = hw->hw_ops.process_cmd(hw->hw_priv,
+				CAM_CDM_HW_INTF_CMD_HANG_DETECT,
+				&handle,
+				sizeof(handle));
+	}
+	put_cdm_mgr_refcount();
+
+	return rc;
+}
+EXPORT_SYMBOL(cam_cdm_detect_hang_error);
+
 int cam_cdm_intf_register_hw_cdm(struct cam_hw_intf *hw,
 	struct cam_cdm_private_dt_data *data, enum cam_cdm_type type,
 	uint32_t *index)
diff --git a/drivers/cam_cdm/cam_cdm_intf_api.h b/drivers/cam_cdm/cam_cdm_intf_api.h
index 3e89b22..466ead3 100644
--- a/drivers/cam_cdm/cam_cdm_intf_api.h
+++ b/drivers/cam_cdm/cam_cdm_intf_api.h
@@ -1,6 +1,6 @@
 /* SPDX-License-Identifier: GPL-2.0-only */
 /*
- * Copyright (c) 2017-2018, The Linux Foundation. All rights reserved.
+ * Copyright (c) 2017-2020, The Linux Foundation. All rights reserved.
  */
 
 #ifndef _CAM_CDM_API_H_
@@ -199,4 +199,13 @@ int cam_cdm_stream_off(uint32_t handle);
  */
 int cam_cdm_reset_hw(uint32_t handle);
 
+/**
+ * @brief : API to detect hang in previously acquired CDM,
+ *          this should be only performed only if the CDM is private.
+ *
+ * @handle : Input handle of the CDM to detect hang
+ *
+ * @return 0 on success
+ */
+int cam_cdm_detect_hang_error(uint32_t handle);
 #endif /* _CAM_CDM_API_H_ */
diff --git a/drivers/cam_isp/isp_hw_mgr/cam_ife_hw_mgr.c b/drivers/cam_isp/isp_hw_mgr/cam_ife_hw_mgr.c
index 89b4c44..a6a99e0 100644
--- a/drivers/cam_isp/isp_hw_mgr/cam_ife_hw_mgr.c
+++ b/drivers/cam_isp/isp_hw_mgr/cam_ife_hw_mgr.c
@@ -25,6 +25,7 @@
 #include "cam_common_util.h"
 
 #define CAM_IFE_HW_ENTRIES_MAX  20
+#define CAM_IFE_HW_CONFIG_WAIT_MAX_TRY  3
 
 #define TZ_SVC_SMMU_PROGRAM 0x15
 #define TZ_SAFE_SYSCALL_ID  0x3
@@ -3738,23 +3739,45 @@ static int cam_ife_mgr_config_hw(void *hw_mgr_priv,
 			return rc;
 		}
 
-		if (cfg->init_packet) {
+		if (!cfg->init_packet)
+			goto end;
+
+		for (i = 0; i < CAM_IFE_HW_CONFIG_WAIT_MAX_TRY; i++) {
 			rem_jiffies = wait_for_completion_timeout(
 				&ctx->config_done_complete,
 				msecs_to_jiffies(30));
 			if (rem_jiffies == 0) {
+				if (!cam_cdm_detect_hang_error(
+						ctx->cdm_handle)) {
+					CAM_INFO(CAM_ISP,
+						"CDM workqueue delay detected, wait for some more time req_id=%llu rc=%d ctx_index %d",
+						cfg->request_id, rc,
+						ctx->ctx_index);
+					continue;
+				}
 				CAM_ERR(CAM_ISP,
 					"config done completion timeout for req_id=%llu ctx_index %d",
 					cfg->request_id, ctx->ctx_index);
 				rc = -ETIMEDOUT;
-			} else
+				goto end;
+			} else {
+				rc = 0;
 				CAM_DBG(CAM_ISP,
 					"config done Success for req_id=%llu ctx_index %d",
 					cfg->request_id, ctx->ctx_index);
+				break;
+			}
+		}
+		if ((i == CAM_IFE_HW_CONFIG_WAIT_MAX_TRY) && (rc == 0)) {
+			CAM_ERR(CAM_ISP,
+				"config done completion timeout for req_id=%llu ctx_index %d",
+				cfg->request_id, ctx->ctx_index);
+			rc = -ETIMEDOUT;
 		}
 	} else {
 		CAM_ERR(CAM_ISP, "No commands to config");
 	}
+end:
 	CAM_DBG(CAM_ISP, "Exit: Config Done: %llu",  cfg->request_id);
 
 	return rc;