Documentation for Parameters Estimation¶

class AffineLeastMeanSquares(BaseEstimator):
    """Affine Least Mean Squares (ALMS) filter for parameter estimation.

    The ALMS filter is an adaptive filter used to estimate the parameters of a model.
    It incorporates an offset covariance factor to improve the stability and convergence
    of the parameter estimation process.

    Parameters
    ----------
    mu : float, default=0.01
        The learning rate or step size for the LMS algorithm.
    offset_covariance : float, default=0.2
        The offset covariance factor of the affine least mean squares filter.
    unbiased : bool, optional
        If True, applies an unbiased estimator. Default is False.
    uiter : int, optional
        Number of iterations for the unbiased estimator. Default is 30.

    Attributes
    ----------
    mu : float
        The learning rate or step size for the LMS algorithm.
    offset_covariance : float
        The offset covariance factor of the affine least mean squares filter.
    xi : np.ndarray or None
        The estimation error at each iteration. Initialized as None and updated during
        optimization.

    Methods
    -------
    optimize(psi: np.ndarray, y: np.ndarray) -> np.ndarray
        Estimate the model parameters using the ALMS filter.

    References
    ----------
    - Poularikas, A. D. (2017). Adaptive filtering: Fundamentals of least mean squares
    with MATLAB®. CRC Press.
    """

    def __init__(
        self,
        *,
        mu: float = 0.01,
        offset_covariance: float = 0.2,
        unbiased: bool = False,
        uiter: int = 30,
    ):
        self.mu = mu
        self.offset_covariance = offset_covariance
        self.uiter = uiter
        self.unbiased = unbiased
        _validate_params(vars(self))
        self.xi: Optional[np.ndarray] = None

    def optimize(self, psi: np.ndarray, y: np.ndarray) -> np.ndarray:
        r"""Estimate the model parameters using the Affine Least Mean Squares.

        The ALMS method updates the parameter estimates recursively as follows:

        1. Compute the estimation error:

           $$
           \xi = y - \psi \theta_{i-1}
           $$

        2. Update the parameter vector:

           $$
           \theta_i = \theta_{i-1} + \mu \psi (\psi^T \psi + \text{offset_covariance}
           \cdot I)^{-1} \xi
           $$

        Parameters
        ----------
        psi : ndarray of floats
            The information matrix of the model.
        y : array-like of shape (n_samples, 1)
            The data used to train the model.

        Returns
        -------
        theta : array-like of shape (n_features, 1)
            The estimated parameters of the model.

        Notes
        -----
        A more in-depth documentation of all methods for parameters estimation
        will be available soon. For now, please refer to the mentioned references.
        """
        n_theta, n, theta, self.xi = _initial_values(psi)

        for i in range(n_theta, n):
            self.xi = y - psi.dot(theta[:, i - 1].reshape(-1, 1))
            aux = (
                self.mu
                * psi
                @ np.linalg.pinv(psi.T @ psi + self.offset_covariance * np.eye(n_theta))
            )
            tmp_list = theta[:, i - 1].reshape(-1, 1) + aux.T.dot(self.xi)
            theta[:, i] = tmp_list.flatten()

        return theta[:, -1].reshape(-1, 1)

class BoundedVariableLeastSquares(BaseEstimator):
    """Solve a linear least-squares problem with bounds on the variables.

    This is a wrapper class for the `scipy.optimize.lsq_linear` method.

    Given a m-by-n design matrix A and a target vector b with m elements,
    `lsq_linear` solves the following optimization problem::

        minimize 0.5 * ||A x - b||**2
        subject to lb <= x <= ub

    This optimization problem is convex, hence a found minimum (if iterations
    have converged) is guaranteed to be global.

    Parameters
    ----------
    unbiased : bool
        Indicates whether an unbiased estimator is applied.
    uiter : int
        Number of iterations for the unbiased estimator.
    method : 'trf' or 'bvls', optional
        Method to perform minimization.

            * 'trf' : Trust Region Reflective algorithm adapted for a linear
              least-squares problem. This is an interior-point-like method
              and the required number of iterations is weakly correlated with
              the number of variables.
            * 'bvls' : Bounded-variable least-squares algorithm. This is
              an active set method, which requires the number of iterations
              comparable to the number of variables. Can't be used when `A` is
              sparse or LinearOperator.

        Default is 'trf'.
    tol : float, optional
        Tolerance parameter. The algorithm terminates if a relative change
        of the cost function is less than `tol` on the last iteration.
        Additionally, the first-order optimality measure is considered:

            * ``method='trf'`` terminates if the uniform norm of the gradient,
              scaled to account for the presence of the bounds, is less than
              `tol`.
            * ``method='bvls'`` terminates if Karush-Kuhn-Tucker conditions
              are satisfied within `tol` tolerance.

    lsq_solver : {None, 'exact', 'lsmr'}, optional
        Method of solving unbounded least-squares problems throughout
        iterations:

            * 'exact' : Use dense QR or SVD decomposition approach. Can't be
              used when `A` is sparse or LinearOperator.
            * 'lsmr' : Use `scipy.sparse.linalg.lsmr` iterative procedure
              which requires only matrix-vector product evaluations. Can't
              be used with ``method='bvls'``.

        If None (default), the solver is chosen based on type of `A`.
    lsmr_tol : None, float or 'auto', optional
        Tolerance parameters 'atol' and 'btol' for `scipy.sparse.linalg.lsmr`
        If None (default), it is set to ``1e-2 * tol``. If 'auto', the
        tolerance will be adjusted based on the optimality of the current
        iterate, which can speed up the optimization process, but is not always
        reliable.
    max_iter : None or int, optional
        Maximum number of iterations before termination. If None (default), it
        is set to 100 for ``method='trf'`` or to the number of variables for
        ``method='bvls'`` (not counting iterations for 'bvls' initialization).
    verbose : {0, 1, 2}, optional
        Level of algorithm's verbosity:

            * 0 : work silently (default).
            * 1 : display a termination report.
            * 2 : display progress during iterations.
    lsmr_maxiter : None or int, optional
        Maximum number of iterations for the lsmr least squares solver,
        if it is used (by setting ``lsq_solver='lsmr'``). If None (default), it
        uses lsmr's default of ``min(m, n)`` where ``m`` and ``n`` are the
        number of rows and columns of `A`, respectively. Has no effect if
        ``lsq_solver='exact'``.

    References
    ----------
    M. A. Branch, T. F. Coleman, and Y. Li, "A Subspace, Interior,
        and Conjugate Gradient Method for Large-Scale Bound-Constrained
        Minimization Problems," SIAM Journal on Scientific Computing,
        Vol. 21, Number 1, pp 1-23, 1999.
    P. B. Start and R. L. Parker, "Bounded-Variable Least-Squares:
        an Algorithm and Applications", Computational Statistics, 10,
        129-141, 1995.

    Notes
    -----
    This docstring is adapted from the `scipy.optimize.lsq_linear` method.

    Examples
    --------
    In this example, a problem with a large sparse matrix and bounds on the
    variables is solved.

    >>> import numpy as np
    >>> from scipy.sparse import rand
    >>> from sysidentpy.parameter_estimation import BoundedVariableLeastSquares
    >>> rng = np.random.default_rng()
    ...
    >>> m = 20000
    >>> n = 10000
    ...
    >>> A = rand(m, n, density=1e-4, random_state=rng)
    >>> b = rng.standard_normal(m)
    ...
    >>> lb = rng.standard_normal(n)
    >>> ub = lb + 1
    ...
    >>> res = BoundedVariableLeastSquares(A, b, bounds=(lb, ub), lsmr_tol='auto',
    verbose=1)
    The relative change of the cost function is less than `tol`.
    Number of iterations 16, initial cost 1.5039e+04, final cost 1.1112e+04,
    first-order optimality 4.66e-08.
    """

    def __init__(
        self,
        *,
        unbiased: bool = False,
        uiter: int = 30,
        bounds=(-np.inf, np.inf),
        method="trf",
        tol=1e-10,
        lsq_solver=None,
        lsmr_tol=None,
        max_iter=None,
        verbose=0,
        lsmr_maxiter=None,
    ):
        self.unbiased = unbiased
        self.uiter = uiter
        self.max_iter = max_iter
        self.bounds = bounds
        self.method = method
        self.tol = tol
        self.lsq_solver = lsq_solver
        self.lsmr_tol = lsmr_tol
        self.verbose = verbose
        self.lsmr_maxiter = lsmr_maxiter

    def optimize(self, psi, y):
        """Parameter estimation using the BoundedVariableLeastSquares algorithm.

        Parameters
        ----------
        psi : ndarray of floats
            The information matrix of the model.
        y : ndarray of floats of shape (n_samples, 1)
            The data used to train the model.

        Returns
        -------
        theta : array-like of shape = number_of_model_elements
            The estimated parameters of the model.

        Notes
        -----
        This is a wrapper class for the `scipy.optimize.lsq_linear` method.

        References
        ----------
        .. [1] scipy, https://docs.scipy.org/doc/scipy/reference/generated/scipy.optimize.lsq_linear.html
        """
        theta = lsq_linear(
            psi,
            y.ravel(),
            bounds=self.bounds,
            method=self.method,
            tol=self.tol,
            lsq_solver=self.lsq_solver,
            lsmr_tol=self.lsmr_tol,
            max_iter=self.max_iter,
            verbose=self.verbose,
            lsmr_maxiter=self.lsmr_maxiter,
        )
        return theta.x.reshape(-1, 1)

class EstimatorError(Exception):
    """Generic Python-exception-derived object raised by estimator functions.

    General purpose exception class, derived from Python's ValueError
    class, programmatically raised in estimators functions when a Estimator-related
    condition would prevent further correct execution of the function.

    Parameters
    ----------
    None

    """

class LeastMeanSquareMixedNorm(BaseEstimator):
    """Least Mean Square Mixed Norm (LMS-MN) Adaptive Filter.

    This class implements the Mixed-norm Least Mean Square (LMS) adaptive filter
    algorithm, which incorporates an additional weight factor to control the
    proportions of the error norms, thus providing an extra degree of freedom
    in the adaptation process.

    Parameters
    ----------
    mu : float, optional
        The adaptation step size. Default is 0.01.
    weight : float, optional
        The weight factor for mixed-norm control. This factor controls the
        proportions of the error norms and offers an extra degree of freedom
        within the adaptation of the LMS mixed norm method.
    unbiased : bool, optional
        If True, applies an unbiased estimator. Default is False.
    uiter : int, optional
        Number of iterations for the unbiased estimator. Default is 30.

    Attributes
    ----------
    mu : float
        The adaptation step size.
    weight : float
        The weight factor for mixed-norm control.
    xi : ndarray or None
        The error signal, initialized to None.

    Methods
    -------
    optimize(psi: np.ndarray, y: np.ndarray) -> np.ndarray
        Estimate the model parameters using the LMSF filter.

    References
    ----------
    - Chambers, J. A., Tanrikulu, O., & Constantinides, A. G. (1994).
      Least mean mixed-norm adaptive filtering.
      Electronics letters, 30(19), 1574-1575.
      https://ieeexplore.ieee.org/document/326382
    - Dissertation (Portuguese): Zipf, J. G. F. (2011). Classificação,
      análise estatística e novas estratégias de algoritmos LMS de passo
      variável.
    - Wikipedia entry on Least Mean Squares
      https://en.wikipedia.org/wiki/Least_mean_squares_filter
    """

    def __init__(
        self,
        *,
        mu: float = 0.01,
        weight: float = 0.02,
        unbiased: bool = False,
        uiter: int = 30,
    ):
        self.mu = mu
        self.weight = weight
        self.unbiased = unbiased
        self.uiter = uiter
        _validate_params(vars(self))
        self.xi: Optional[np.ndarray] = None

    def optimize(self, psi: np.ndarray, y: np.ndarray) -> np.ndarray:
        r"""Parameter estimation using the Mixed-norm LMS filter.

        The LMS-MN algorithm updates the parameter estimates recursively as follows:

        1. Compute the estimation error:

           $$
           \xi_i = y_i - \psi_i^T \theta_{i-1}
           $$

        2. Update the parameter vector:

           $$
           \theta_i = \theta_{i-1} + \mu \psi_i \xi_i (\text{weight}
           + (1 - \text{weight}) \xi_i^2)
           $$

        Parameters
        ----------
        psi : ndarray of floats
            The information matrix of the model.
        y : array-like of shape (n_samples, 1)
            The data used to train the model.

        Returns
        -------
        theta : array-like of shape (n_features, 1)
            The estimated parameters of the model.

        Notes
        -----
        A more in-depth documentation of all methods for parameter estimation
        will be available soon. For now, please refer to the mentioned references.
        """
        n_theta, n, theta, self.xi = _initial_values(psi)

        for i in range(n_theta, n):
            psi_tmp = psi[i, :].reshape(-1, 1)
            self.xi[i, 0] = y[i, 0] - np.dot(psi_tmp.T, theta[:, i - 1])[0]
            tmp_list = theta[:, i - 1].reshape(-1, 1) + self.mu * psi_tmp * self.xi[
                i, 0
            ] * (self.weight + (1 - self.weight) * self.xi[i, 0] ** 2)
            theta[:, i] = tmp_list.flatten()

        return theta[:, -1].reshape(-1, 1)

class LeastMeanSquares(BaseEstimator):
    """Least Mean Squares (LMS) filter for parameter estimation in adaptive filtering.

    The LMS algorithm is an adaptive filter used to estimate the parameters of a model
    by minimizing the mean square error between the observed and predicted values.

    Parameters
    ----------
    mu : float, default=0.01
        The learning rate or step size for the LMS algorithm.
    unbiased : bool, optional
        If True, applies an unbiased estimator. Default is False.
    uiter : int, optional
        Number of iterations for the unbiased estimator. Default is 30.

    Attributes
    ----------
    mu : float
        The learning rate or step size for the LMS algorithm.
    unbiased : bool
        Indicates whether an unbiased estimator is applied.
    uiter : int
        Number of iterations for the unbiased estimator.
    xi : np.ndarray or None
        The estimation error at each iteration. Initialized as None and updated during
        optimization.

    Methods
    -------
    optimize(psi: np.ndarray, y: np.ndarray) -> np.ndarray
        Estimate the model parameters using the LMS filter.

    References
    ----------
    - Haykin, S., & Widrow, B. (Eds.). (2003). Least-mean-square adaptive filters
    (Vol. 31). John Wiley & Sons.
    - Zipf, J. G. F. (2011). Classificação, análise estatística e novas estratégias de
    algoritmos LMS de passo variável.
    - Wikipedia entry on Least Mean Squares: https://en.wikipedia.org/wiki/Least_mean_squares_filter
    """

    def __init__(self, *, mu: float = 0.01, unbiased: bool = False, uiter: int = 30):
        self.mu = mu
        self.unbiased = unbiased
        self.uiter = uiter
        _validate_params(vars(self))
        self.xi: Optional[np.ndarray] = None

    def optimize(self, psi: np.ndarray, y: np.ndarray) -> np.ndarray:
        r"""Estimate the model parameters using the Least Mean Squares filter.

        The LMS algorithm updates the parameter estimates recursively as follows:

        1. Compute the estimation error:

           $$
           \xi_i = y_i - \psi_i^T \theta_{i-1}
           $$

        2. Update the parameter vector:

           $$
           \theta_i = \theta_{i-1} + 2 \mu \xi_i \psi_i
           $$

        Parameters
        ----------
        psi : ndarray of floats
            The information matrix of the model.
        y : array-like of shape (n_samples, 1)
            The data used to train the model.

        Returns
        -------
        theta : array-like of shape (n_features, 1)
            The estimated parameters of the model.
        """
        n_theta, n, theta, self.xi = _initial_values(psi)

        for i in range(n_theta, n):
            psi_tmp = psi[i, :].reshape(-1, 1)
            self.xi[i, 0] = y[i, 0] - np.dot(psi_tmp.T, theta[:, i - 1])[0]
            tmp_list = (
                theta[:, i - 1].reshape(-1, 1) + 2 * self.mu * self.xi[i, 0] * psi_tmp
            )
            theta[:, i] = tmp_list.flatten()

        return theta[:, -1].reshape(-1, 1)

class LeastMeanSquaresFourth(BaseEstimator):
    """Least Mean Squares Fourth (LMSF) filter for parameter estimation.

    The LMSF algorithm is an adaptive filter used to estimate the parameters of a model
    by using the mean fourth error cost function to eliminate the noise effectively.

    Parameters
    ----------
    mu : float, default=0.5
        The learning rate or step size for the LMS algorithm.
    unbiased : bool, optional
        If True, applies an unbiased estimator. Default is False.
    uiter : int, optional
        Number of iterations for the unbiased estimator. Default is 30.

    Attributes
    ----------
    mu : float
        The learning rate or step size for the LMS algorithm.
    unbiased : bool
        Indicates whether an unbiased estimator is applied.
    uiter : int
        Number of iterations for the unbiased estimator.
    xi : np.ndarray or None
        The estimation error at each iteration. Initialized as None and updated during
        optimization.

    Methods
    -------
    optimize(psi: np.ndarray, y: np.ndarray) -> np.ndarray
        Estimate the model parameters using the LMSF filter.

    References
    ----------
    - Hayes, M. H. (2009). Statistical digital signal processing and modeling.
      John Wiley & Sons.
    - Zipf, J. G. F. (2011). Classificação, análise estatística e novas estratégias de
      algoritmos LMS de passo variável.
    - Gui, G., Mehbodniya, A., & Adachi, F. (2013). Least mean square/fourth algorithm
      with application to sparse channel estimation. arXiv preprint arXiv:1304.3911.
      https://arxiv.org/pdf/1304.3911.pdf
    - Nascimento, V. H., & Bermudez, J. C. M. (2005, March). When is the least-mean
      fourth algorithm mean-square stable? In Proceedings.(ICASSP'05). IEEE
      International Conference on Acoustics, Speech, and Signal Processing, 2005.
      (Vol. 4, pp. iv-341). IEEE. http://www.lps.usp.br/vitor/artigos/icassp05.pdf
    - Wikipedia entry on Least Mean Squares: https://en.wikipedia.org/wiki/Least_mean_squares_filter
    """

    def __init__(self, *, mu: float = 0.5, unbiased: bool = False, uiter: int = 30):
        self.mu = mu
        self.unbiased = unbiased
        self.uiter = uiter
        _validate_params(vars(self))
        self.xi: Optional[np.ndarray] = None

    def optimize(self, psi: np.ndarray, y: np.ndarray) -> np.ndarray:
        r"""Parameter estimation using the LMS Fourth filter.

        The LMSF algorithm updates the parameter estimates recursively as follows:

        1. Compute the estimation error:

           $$
           \xi_i = y_i - \psi_i^T \theta_{i-1}
           $$

        2. Update the parameter vector:

           $$
           \theta_i = \theta_{i-1} + \mu \psi_i \xi_i^3
           $$

        Parameters
        ----------
        psi : ndarray of floats
            The information matrix of the model.
        y : ndarray of floats of shape (n_samples, 1)
            The data used to train the model.

        Returns
        -------
        theta : ndarray of floats of shape (n_features, 1)
            The estimated parameters of the model.
        """
        n_theta, n, theta, self.xi = _initial_values(psi)

        for i in range(n_theta, n):
            psi_tmp = psi[i, :].reshape(-1, 1)
            self.xi[i, 0] = y[i, 0] - np.dot(psi_tmp.T, theta[:, i - 1])[0]
            tmp_list = (
                theta[:, i - 1].reshape(-1, 1) + self.mu * psi_tmp * self.xi[i, 0] ** 3
            )
            theta[:, i] = tmp_list.flatten()

        return theta[:, -1].reshape(-1, 1)

class LeastMeanSquaresLeaky(BaseEstimator):
    """Least Mean Squares Leaky (LMSL) filter for parameter estimation.

    The LMSL algorithm is an adaptive filter used to estimate the parameters of a model
    by minimizing the mean square error between the observed and predicted values. The
    leakage factor helps to prevent coefficient drift.

    Parameters
    ----------
    mu : float, default=0.01
        The learning rate or step size for the LMS algorithm.
    gama : float, default=0.2
        The leakage factor of the Leaky LMS method.
    unbiased : bool, optional
        If True, applies an unbiased estimator. Default is False.
    uiter : int, optional
        Number of iterations for the unbiased estimator. Default is 30.

    Attributes
    ----------
    mu : float
        The learning rate or step size for the LMS algorithm.
    gama : float, default=0.2
        The leakage factor of the Leaky LMS method.
    xi : np.ndarray or None
        The estimation error at each iteration. Initialized as None and updated during
        optimization.

    Methods
    -------
    optimize(psi: np.ndarray, y: np.ndarray) -> np.ndarray
        Estimate the model parameters using the LMSL filter.

    References
    ----------
    - Hayes, M. H. (2009). Statistical digital signal processing and modeling.
      John Wiley & Sons.
    - Zipf, J. G. F. (2011). Classificação, análise estatística e novas estratégias de
      algoritmos LMS de passo variável.
    - Wikipedia entry on Least Mean Squares: https://en.wikipedia.org/wiki/Least_mean_squares_filter
    """

    def __init__(
        self,
        *,
        mu: float = 0.01,
        gama: float = 0.001,
        unbiased: bool = False,
        uiter: int = 30,
    ):
        self.mu = mu
        self.gama = gama
        self.unbiased = unbiased
        self.uiter = uiter
        _validate_params(vars(self))
        self.xi: Optional[np.ndarray] = None

    def optimize(self, psi: np.ndarray, y: np.ndarray) -> np.ndarray:
        r"""Parameter estimation using the Leaky LMS filter.

        The LMSL algorithm updates the parameter estimates recursively as follows:

        1. Compute the estimation error:

           $$
           \xi_i = y_i - \psi_i^T \theta_{i-1}
           $$

        2. Update the parameter vector:

           $$
           \theta_i = \theta_{i-1} (1 - \mu \gamma) + \mu \xi_i \psi_i
           $$

        When the leakage factor, $\gamma$, is set to 0, there is no leakage in the
        estimation process.

        Parameters
        ----------
        psi : ndarray of floats
            The information matrix of the model.
        y : array-like of shape (n_samples, 1)
            The data used to train the model.

        Returns
        -------
        theta : array-like of shape (n_features, 1)
            The estimated parameters of the model.

        References
        ----------
        - Hayes, M. H. (2009). Statistical digital signal processing and modeling.
          John Wiley & Sons.
        - Zipf, J. G. F. (2011). Classificação, análise estatística e novas estratégias
          de algoritmos LMS de passo variável.
        - Wikipedia entry on Least Mean Squares: https://en.wikipedia.org/wiki/Least_mean_squares_filter
        """
        n_theta, n, theta, self.xi = _initial_values(psi)

        for i in range(n_theta, n):
            psi_tmp = psi[i, :].reshape(-1, 1)
            self.xi[i, 0] = y[i, 0] - np.dot(psi_tmp.T, theta[:, i - 1])[0]
            tmp_list = (
                theta[:, i - 1].reshape(-1, 1) * (1 - self.mu * self.gama)
                + self.mu * self.xi[i, 0] * psi_tmp
            )
            theta[:, i] = tmp_list.flatten()

        return theta[:, -1].reshape(-1, 1)

class LeastMeanSquaresNormalizedLeaky(BaseEstimator):
    """Normalized Least Mean Squares Leaky (NLMSL) filter for parameter estimation.

    The NLMSL algorithm is an adaptive filter used to estimate the parameters of a model
    by minimizing the mean square error between the observed and predicted values. The
    normalization is used to avoid numerical instability when updating the estimated
    parameters, and the leakage factor helps to prevent coefficient drift.

    Parameters
    ----------
    mu : float, default=0.01
        The learning rate or step size for the LMS algorithm.
    eps : float, default=np.finfo(np.float64).eps
        Normalization factor of the normalized filters.
    gama : float, default=0.2
        The leakage factor of the Leaky LMS method.

    Attributes
    ----------
    mu : float
        The learning rate or step size for the LMS algorithm.
    eps : float, default=np.finfo(np.float64).eps
        Normalization factor of the normalized filters.
    gama : float, default=0.2
        The leakage factor of the Leaky LMS method.
    xi : np.ndarray or None
        The estimation error at each iteration. Initialized as None and updated during
        optimization.

    Methods
    -------
    optimize(psi: np.ndarray, y: np.ndarray) -> np.ndarray
        Estimate the model parameters using the NLMSL filter.

    References
    ----------
    - Hayes, M. H. (2009). Statistical digital signal processing and modeling.
      John Wiley & Sons.
    - Zipf, J. G. F. (2011). Classificação, análise estatística e novas estratégias de
      algoritmos LMS de passo variável.
    - Wikipedia entry on Least Mean Squares: https://en.wikipedia.org/wiki/Least_mean_squares_filter
    """

    def __init__(
        self,
        *,
        mu: float = 0.01,
        gama: float = 0.2,
        eps: np.float64 = np.finfo(np.float64).eps,
        unbiased: bool = False,
        uiter: int = 30,
    ):
        self.mu = mu
        self.eps = eps
        self.gama = gama
        self.unbiased = unbiased
        self.uiter = uiter
        _validate_params(vars(self))
        self.xi: Optional[np.ndarray] = None

    def optimize(self, psi: np.ndarray, y: np.ndarray) -> np.ndarray:
        r"""Parameter estimation using the Normalized Leaky LMS filter.

        The NLMSL algorithm updates the parameter estimates recursively as follows:

        1. Compute the estimation error:

           $$
           \xi_i = y_i - \psi_i^T \theta_{i-1}
           $$

        2. Update the parameter vector:

           $$
           \theta_i = \theta_{i-1} (1 - \mu \gamma) + \mu \frac{\xi_i \psi_i}{\epsilon
           + \psi_i^T \psi_i}
           $$

        When the leakage factor, $\gamma$, is set to 0, there is no leakage in the
        estimation process.

        Parameters
        ----------
        psi : ndarray of floats
            The information matrix of the model.
        y : array-like of shape (n_samples, 1)
            The data used to train the model.

        Returns
        -------
        theta : array-like of shape (n_features, 1)
            The estimated parameters of the model.

        References
        ----------
        - Hayes, M. H. (2009). Statistical digital signal processing and modeling.
          John Wiley & Sons.
        - Zipf, J. G. F. (2011). Classificação, análise estatística e novas estratégias
          de algoritmos LMS de passo variável.
        - Wikipedia entry on Least Mean Squares: https://en.wikipedia.org/wiki/Least_mean_squares_filter
        """
        n_theta, n, theta, self.xi = _initial_values(psi)

        for i in range(n_theta, n):
            psi_tmp = psi[i, :].reshape(-1, 1)
            self.xi[i, 0] = y[i, 0] - np.dot(psi_tmp.T, theta[:, i - 1])[0]
            tmp_list = theta[:, i - 1].reshape(-1, 1) * (
                1 - self.mu * self.gama
            ) + self.mu * self.xi[i, 0] * psi_tmp / (
                self.eps + np.dot(psi_tmp.T, psi_tmp)
            )
            theta[:, i] = tmp_list.flatten()

        return theta[:, -1].reshape(-1, 1)