path: root/modules/ocl/cl_post_image_processor.cpp

/*
 * cl_post_image_processor.cpp - CL post image processor
 *
 *  Copyright (c) 2015 Intel Corporation
 *
 * Licensed under the Apache License, Version 2.0 (the "License");
 * you may not use this file except in compliance with the License.
 * You may obtain a copy of the License at
 *
 *      http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing, software
 * distributed under the License is distributed on an "AS IS" BASIS,
 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 * See the License for the specific language governing permissions and
 * limitations under the License.
 *
 * Author: Wind Yuan <feng.yuan@intel.com>
 * Author: Yinhang Liu <yinhangx.liu@intel.com>
 */

#include "cl_post_image_processor.h"
#include "cl_context.h"

#include "cl_tnr_handler.h"
#include "cl_retinex_handler.h"
#include "cl_defog_dcp_handler.h"
#include "cl_wavelet_denoise_handler.h"
#include "cl_newwavelet_denoise_handler.h"
#include "cl_3d_denoise_handler.h"
#include "cl_image_scaler.h"
#include "cl_wire_frame_handler.h"
#include "cl_csc_handler.h"
#include "cl_image_warp_handler.h"
#include "cl_image_360_stitch.h"
#include "cl_video_stabilizer.h"

#define XCAM_CL_POST_IMAGE_DEFAULT_POOL_SIZE 6
#define XCAM_CL_POST_IMAGE_MAX_POOL_SIZE 12

namespace XCam {

CLPostImageProcessor::CLPostImageProcessor ()
    : CLImageProcessor ("CLPostImageProcessor")
    , _output_fourcc (V4L2_PIX_FMT_NV12)
    , _out_sample_type (OutSampleYuv)
    , _scaler_factor (1.0)
    , _tnr_mode (TnrYuv)
    , _defog_mode (CLPostImageProcessor::DefogDisabled)
    , _wavelet_basis (CL_WAVELET_DISABLED)
    , _wavelet_channel (CL_IMAGE_CHANNEL_UV)
    , _wavelet_bayes_shrink (false)
    , _3d_denoise_mode (CLPostImageProcessor::Denoise3DDisabled)
    , _3d_denoise_ref_count (3)
    , _enable_scaler (false)
    , _enable_wireframe (false)
    , _enable_image_warp (false)
    , _enable_stitch (false)
    , _stitch_enable_seam (false)
    , _stitch_fisheye_map (false)
    , _stitch_lsc (false)
    , _stitch_scale_mode (CLBlenderScaleLocal)
    , _stitch_width (0)
    , _stitch_height (0)
    , _stitch_res_mode (0)
    , _surround_mode (SphereView)
{
    XCAM_LOG_DEBUG ("CLPostImageProcessor constructed");
}

CLPostImageProcessor::~CLPostImageProcessor ()
{
    XCAM_LOG_DEBUG ("CLPostImageProcessor destructed");
}

bool
CLPostImageProcessor::set_output_format (uint32_t fourcc)
{
    switch (fourcc) {
    case XCAM_PIX_FMT_RGBA64:
    case V4L2_PIX_FMT_XBGR32:
    case V4L2_PIX_FMT_ABGR32:
    case V4L2_PIX_FMT_BGR32:
    case V4L2_PIX_FMT_RGB32:
    case V4L2_PIX_FMT_ARGB32:
    case V4L2_PIX_FMT_XRGB32:
        _out_sample_type = OutSampleRGB;
        break;
    case V4L2_PIX_FMT_NV12:
        _out_sample_type = OutSampleYuv;
        break;
    default:
        XCAM_LOG_WARNING (
            "cl post processor doesn't support output format: %s",
            xcam_fourcc_to_string(fourcc));
        return false;
    }

    _output_fourcc = fourcc;
    return true;
}

void
CLPostImageProcessor::set_stats_callback (const SmartPtr<StatsCallback> &callback)
{
    XCAM_ASSERT (callback.ptr ());
    _stats_callback = callback;
}

bool
CLPostImageProcessor::set_scaler_factor (const double factor)
{
    _scaler_factor = factor;

    return true;
}

bool
CLPostImageProcessor::can_process_result (SmartPtr < X3aResult > & result)
{
    if (!result.ptr ())
        return false;

    switch (result->get_type ()) {
    case XCAM_3A_RESULT_TEMPORAL_NOISE_REDUCTION_YUV:
    case XCAM_3A_RESULT_3D_NOISE_REDUCTION:
    case XCAM_3A_RESULT_WAVELET_NOISE_REDUCTION:
    case XCAM_3A_RESULT_FACE_DETECTION:
    case XCAM_3A_RESULT_DVS:
        return true;
    default:
        return false;
    }

    return false;
}

XCamReturn
CLPostImageProcessor::apply_3a_results (X3aResultList &results)
{
    XCamReturn ret = XCAM_RETURN_NO_ERROR;

    for (X3aResultList::iterator iter = results.begin (); iter != results.end (); ++iter)
    {
        SmartPtr<X3aResult> &result = *iter;
        ret = apply_3a_result (result);
        if (ret != XCAM_RETURN_NO_ERROR)
            break;
    }

    return ret;
}

XCamReturn
CLPostImageProcessor::apply_3a_result (SmartPtr<X3aResult> &result)
{
    STREAM_LOCK;

    if (!result.ptr ())
        return XCAM_RETURN_BYPASS;

    uint32_t res_type = result->get_type ();

    switch (res_type) {
    case XCAM_3A_RESULT_TEMPORAL_NOISE_REDUCTION_YUV: {
        SmartPtr<X3aTemporalNoiseReduction> tnr_res = result.dynamic_cast_ptr<X3aTemporalNoiseReduction> ();
        XCAM_ASSERT (tnr_res.ptr ());
        if (_tnr.ptr ()) {
            if (_defog_mode != CLPostImageProcessor::DefogDisabled) {
                XCam3aResultTemporalNoiseReduction config;
                xcam_mem_clear (config);
                // isp processor
                // config.gain = 0.12;

                // cl processor
                config.gain = 0.22;

                config.threshold [0] = 0.00081;
                config.threshold [1] = 0.00072;
                _tnr->set_yuv_config (config);
            } else {
                _tnr->set_yuv_config (tnr_res->get_standard_result ());
            }
        }
        break;
    }
    case XCAM_3A_RESULT_3D_NOISE_REDUCTION: {
        SmartPtr<X3aTemporalNoiseReduction> nr_res = result.dynamic_cast_ptr<X3aTemporalNoiseReduction> ();
        XCAM_ASSERT (nr_res.ptr ());
        if (_3d_denoise.ptr ()) {
            _3d_denoise->set_denoise_config (nr_res->get_standard_result ());
        }
        break;
    }
    case XCAM_3A_RESULT_WAVELET_NOISE_REDUCTION: {
        SmartPtr<X3aWaveletNoiseReduction> wavelet_res = result.dynamic_cast_ptr<X3aWaveletNoiseReduction> ();
        XCAM_ASSERT (wavelet_res.ptr ());
        if (_wavelet.ptr()) {
            _wavelet->set_denoise_config (wavelet_res->get_standard_result ());
        }
        if (_newwavelet.ptr()) {
            _newwavelet->set_denoise_config (wavelet_res->get_standard_result ());
        }
        break;
    }
    case XCAM_3A_RESULT_FACE_DETECTION: {
        SmartPtr<X3aFaceDetectionResult> fd_res = result.dynamic_cast_ptr<X3aFaceDetectionResult> ();
        XCAM_ASSERT (fd_res.ptr ());
        if (_wireframe.ptr ()) {
            _wireframe->set_wire_frame_config (fd_res->get_standard_result_ptr (), get_scaler_factor ());
        }
        break;
    }
    case XCAM_3A_RESULT_DVS: {
        SmartPtr<X3aDVSResult> dvs_res = result.dynamic_cast_ptr<X3aDVSResult> ();
        XCAM_ASSERT (dvs_res.ptr ());
        if (_image_warp.ptr ()) {
            _image_warp->set_warp_config (dvs_res->get_standard_result ());
        }
        break;
    }
    default:
        XCAM_LOG_WARNING ("CLPostImageProcessor unknown 3a result: %d", res_type);
        break;
    }

    return XCAM_RETURN_NO_ERROR;
}


XCamReturn
CLPostImageProcessor::create_handlers ()
{
    SmartPtr<CLImageHandler> image_handler;
    SmartPtr<CLContext> context = get_cl_context ();

    XCAM_ASSERT (context.ptr ());

    /* defog: retinex */
    image_handler = create_cl_retinex_image_handler (context);
    _retinex = image_handler.dynamic_cast_ptr<CLRetinexImageHandler> ();
    XCAM_FAIL_RETURN (
        WARNING,
        _retinex.ptr (),
        XCAM_RETURN_ERROR_CL,
        "CLPostImageProcessor create retinex handler failed");
    _retinex->enable_handler (_defog_mode == CLPostImageProcessor::DefogRetinex);
    image_handler->set_pool_type (CLImageHandler::CLVideoPoolType);
    image_handler->set_pool_size (XCAM_CL_POST_IMAGE_MAX_POOL_SIZE);
    add_handler (image_handler);

    /* defog: dark channel prior */
    image_handler = create_cl_defog_dcp_image_handler (context);
    _defog_dcp = image_handler.dynamic_cast_ptr<CLDefogDcpImageHandler> ();
    XCAM_FAIL_RETURN (
        WARNING,
        _defog_dcp.ptr (),
        XCAM_RETURN_ERROR_CL,
        "CLPostImageProcessor create defog handler failed");
    _defog_dcp->enable_handler (_defog_mode == CLPostImageProcessor::DefogDarkChannelPrior);
    image_handler->set_pool_type (CLImageHandler::CLVideoPoolType);
    image_handler->set_pool_size (XCAM_CL_POST_IMAGE_MAX_POOL_SIZE);
    add_handler (image_handler);

    /* Temporal Noise Reduction */
    if (_defog_mode != CLPostImageProcessor::DefogDisabled) {
        switch (_tnr_mode) {
        case TnrYuv: {
            image_handler = create_cl_tnr_image_handler (context, CL_TNR_TYPE_YUV);
            _tnr = image_handler.dynamic_cast_ptr<CLTnrImageHandler> ();
            XCAM_FAIL_RETURN (
                WARNING,
                _tnr.ptr (),
                XCAM_RETURN_ERROR_CL,
                "CLPostImageProcessor create tnr handler failed");
            image_handler->set_pool_type (CLImageHandler::CLVideoPoolType);
            image_handler->set_pool_size (XCAM_CL_POST_IMAGE_DEFAULT_POOL_SIZE);
            add_handler (image_handler);
            break;
        }
        case TnrDisable:
            XCAM_LOG_DEBUG ("CLPostImageProcessor disable tnr");
            break;
        default:
            XCAM_LOG_WARNING ("CLPostImageProcessor unknown tnr mode (%d)", _tnr_mode);
            break;
        }
    }

    /* wavelet denoise */
    switch (_wavelet_basis) {
    case CL_WAVELET_HAT: {
        image_handler = create_cl_wavelet_denoise_image_handler (context, _wavelet_channel);
        _wavelet = image_handler.dynamic_cast_ptr<CLWaveletDenoiseImageHandler> ();
        XCAM_FAIL_RETURN (
            WARNING,
            _wavelet.ptr (),
            XCAM_RETURN_ERROR_CL,
            "CLPostImageProcessor create wavelet denoise handler failed");
        _wavelet->enable_handler (true);
        image_handler->set_pool_type (CLImageHandler::CLVideoPoolType);
        image_handler->set_pool_size (XCAM_CL_POST_IMAGE_DEFAULT_POOL_SIZE);
        add_handler (image_handler);
        break;
    }
    case CL_WAVELET_HAAR: {
        image_handler = create_cl_newwavelet_denoise_image_handler (context, _wavelet_channel, _wavelet_bayes_shrink);
        _newwavelet = image_handler.dynamic_cast_ptr<CLNewWaveletDenoiseImageHandler> ();
        XCAM_FAIL_RETURN (
            WARNING,
            _newwavelet.ptr (),
            XCAM_RETURN_ERROR_CL,
            "CLPostImageProcessor create new wavelet denoise handler failed");
        _newwavelet->enable_handler (true);
        image_handler->set_pool_type (CLImageHandler::CLVideoPoolType);
        image_handler->set_pool_size (XCAM_CL_POST_IMAGE_DEFAULT_POOL_SIZE);
        add_handler (image_handler);
        break;
    }
    case CL_WAVELET_DISABLED:
    default :
        XCAM_LOG_DEBUG ("unknown or disable wavelet (%d)", _wavelet_basis);
        break;
    }

    /* 3D noise reduction */
    if (_3d_denoise_mode != CLPostImageProcessor::Denoise3DDisabled) {
        uint32_t denoise_channel = CL_IMAGE_CHANNEL_UV;

        if (_3d_denoise_mode == CLPostImageProcessor::Denoise3DUV) {
            denoise_channel = CL_IMAGE_CHANNEL_UV;
        } else if (_3d_denoise_mode == CLPostImageProcessor::Denoise3DYuv) {
            denoise_channel = CL_IMAGE_CHANNEL_Y | CL_IMAGE_CHANNEL_UV;
        }

        image_handler = create_cl_3d_denoise_image_handler (context, denoise_channel, _3d_denoise_ref_count);
        _3d_denoise = image_handler.dynamic_cast_ptr<CL3DDenoiseImageHandler> ();
        XCAM_FAIL_RETURN (
            WARNING,
            _3d_denoise.ptr (),
            XCAM_RETURN_ERROR_CL,
            "CL3aImageProcessor create 3D noise reduction handler failed");
        image_handler->set_pool_type (CLImageHandler::CLVideoPoolType);
        image_handler->set_pool_size (XCAM_CL_POST_IMAGE_MAX_POOL_SIZE);
        image_handler->enable_handler (true);
        add_handler (image_handler);
    }

    /* image scaler */
    image_handler = create_cl_image_scaler_handler (context, V4L2_PIX_FMT_NV12);
    _scaler = image_handler.dynamic_cast_ptr<CLImageScaler> ();
    XCAM_FAIL_RETURN (
        WARNING,
        _scaler.ptr (),
        XCAM_RETURN_ERROR_CL,
        "CLPostImageProcessor create scaler handler failed");
    _scaler->set_scaler_factor (_scaler_factor, _scaler_factor);
    _scaler->set_buffer_callback (_stats_callback);
    image_handler->set_pool_type (CLImageHandler::CLVideoPoolType);
    image_handler->enable_handler (_enable_scaler);
    add_handler (image_handler);

    /* wire frame */
    image_handler = create_cl_wire_frame_image_handler (context);
    _wireframe = image_handler.dynamic_cast_ptr<CLWireFrameImageHandler> ();
    XCAM_FAIL_RETURN (
        WARNING,
        _wireframe.ptr (),
        XCAM_RETURN_ERROR_CL,
        "CLPostImageProcessor create wire frame handler failed");
    _wireframe->enable_handler (_enable_wireframe);
    image_handler->set_pool_type (CLImageHandler::CLVideoPoolType);
    image_handler->set_pool_size (XCAM_CL_POST_IMAGE_DEFAULT_POOL_SIZE);
    add_handler (image_handler);

    /* image warp */
    image_handler = create_cl_image_warp_handler (context);
    _image_warp = image_handler.dynamic_cast_ptr<CLImageWarpHandler> ();
    XCAM_FAIL_RETURN (
        WARNING,
        _image_warp.ptr (),
        XCAM_RETURN_ERROR_CL,
        "CLPostImageProcessor create image warp handler failed");
    _image_warp->enable_handler (_enable_image_warp);
    image_handler->set_pool_type (CLImageHandler::CLVideoPoolType);
    image_handler->set_pool_size (XCAM_CL_POST_IMAGE_MAX_POOL_SIZE);
    add_handler (image_handler);

    /* video stabilization */
    image_handler = create_cl_video_stab_handler (context);
    _video_stab = image_handler.dynamic_cast_ptr<CLVideoStabilizer> ();
    XCAM_FAIL_RETURN (
        WARNING,
        _video_stab.ptr (),
        XCAM_RETURN_ERROR_CL,
        "CLPostImageProcessor create video stabilizer failed");
    _video_stab->enable_handler (false);
    image_handler->set_pool_type (CLImageHandler::CLVideoPoolType);
    image_handler->set_pool_size (XCAM_CL_POST_IMAGE_MAX_POOL_SIZE);
    add_handler (image_handler);

    /* image stitch */
    image_handler =
        create_image_360_stitch (context, _stitch_enable_seam, _stitch_scale_mode,
                                 _stitch_fisheye_map, _stitch_lsc, (SurroundMode) _surround_mode, (StitchResMode) _stitch_res_mode);
    _stitch = image_handler.dynamic_cast_ptr<CLImage360Stitch> ();
    XCAM_FAIL_RETURN (
        WARNING,
        _stitch.ptr (),
        XCAM_RETURN_ERROR_CL,
        "CLPostImageProcessor create image stitch handler failed");
    _stitch->set_output_size (_stitch_width, _stitch_height);
    image_handler->set_pool_type (CLImageHandler::CLVideoPoolType);
    image_handler->set_pool_size (XCAM_CL_POST_IMAGE_MAX_POOL_SIZE);
    image_handler->enable_handler (_enable_stitch);
    add_handler (image_handler);

    /* csc (nv12torgba) */
    image_handler = create_cl_csc_image_handler (context, CL_CSC_TYPE_NV12TORGBA);
    _csc = image_handler.dynamic_cast_ptr<CLCscImageHandler> ();
    XCAM_FAIL_RETURN (
        WARNING,
        _csc .ptr (),
        XCAM_RETURN_ERROR_CL,
        "CLPostImageProcessor create csc handler failed");
    _csc->enable_handler (_out_sample_type == OutSampleRGB);
    _csc->set_output_format (_output_fourcc);
    image_handler->set_pool_type (CLImageHandler::CLVideoPoolType);
    image_handler->set_pool_size (XCAM_CL_POST_IMAGE_DEFAULT_POOL_SIZE);
    add_handler (image_handler);

    return XCAM_RETURN_NO_ERROR;
}

bool
CLPostImageProcessor::set_tnr (CLTnrMode mode)
{
    _tnr_mode = mode;

    STREAM_LOCK;

    return true;
}

bool
CLPostImageProcessor::set_defog_mode (CLDefogMode mode)
{
    _defog_mode = mode;

    STREAM_LOCK;

    return true;
}

bool
CLPostImageProcessor::set_wavelet (CLWaveletBasis basis, uint32_t channel, bool bayes_shrink)
{
    _wavelet_basis = basis;
    _wavelet_channel = (CLImageChannel) channel;
    _wavelet_bayes_shrink = bayes_shrink;

    STREAM_LOCK;

    return true;
}

bool
CLPostImageProcessor::set_3ddenoise_mode (CL3DDenoiseMode mode, uint8_t ref_frame_count)
{
    _3d_denoise_mode = mode;
    _3d_denoise_ref_count = ref_frame_count;

    STREAM_LOCK;

    return true;
}

bool
CLPostImageProcessor::set_scaler (bool enable)
{
    _enable_scaler = enable;

    STREAM_LOCK;

    return true;
}

bool
CLPostImageProcessor::set_wireframe (bool enable)
{
    _enable_wireframe = enable;

    STREAM_LOCK;

    return true;
}

bool
CLPostImageProcessor::set_image_warp (bool enable)
{
    _enable_image_warp = enable;

    STREAM_LOCK;

    return true;
}

bool
CLPostImageProcessor::set_image_stitch (
    bool enable_stitch, bool enable_seam, CLBlenderScaleMode scale_mode, bool enable_fisheye_map,
    bool lsc, uint32_t stitch_width, uint32_t stitch_height, uint32_t res_mode)
{
    XCAM_ASSERT (scale_mode < CLBlenderScaleMax);

    _enable_stitch = enable_stitch;
    if (enable_stitch)
        _stitch_enable_seam = enable_seam;
    else
        _stitch_enable_seam = false;

    _stitch_scale_mode = scale_mode;
    _stitch_fisheye_map = enable_fisheye_map;
    _stitch_lsc = lsc;
    _stitch_width = stitch_width;
    _stitch_height = stitch_height;
    _stitch_res_mode = res_mode;

    STREAM_LOCK;

    return true;
}

};