Documentation for Parameters Estimation¶

class AffineLeastMeanSquares(BaseEstimator):
    """Affine Least Mean Squares (ALMS) filter for parameter estimation.

    Parameters
    ----------
    mu : float, default=0.01
        The learning rate or step size for the LMS algorithm.
    offset_covariance : float, default=0.2
        The offset covariance factor of the affine least mean squares
        filter.

    Attributes
    ----------
    mu : float
        The learning rate or step size for the LMS algorithm.
    offset_covariance : float, default=0.2
        The offset covariance factor of the affine least mean squares
        filter.
    xi : np.ndarray or None
        The estimation error at each iteration. Initialized as None and updated during
        optimization.

    Methods
    -------
    optimize(psi: np.ndarray, y: np.ndarray) -> np.ndarray
        Estimate the model parameters using the LMS filter.
    """

    def __init__(
        self,
        *,
        mu: float = 0.01,
        offset_covariance: float = 0.2,
        unbiased: bool = False,
        uiter: int = 30,
    ):
        self.mu = mu
        self.offset_covariance = offset_covariance
        self.uiter = uiter
        self.unbiased = unbiased
        self._validate_params(vars(self))
        self.xi: np.ndarray

    def optimize(self, psi: np.ndarray, y: np.ndarray) -> np.ndarray:
        """Estimate the model parameters using the Affine Least Mean Squares.

        Parameters
        ----------
        psi : ndarray of floats
            The information matrix of the model.
        y : array-like of shape = y_training
            The data used to training the model.

        Returns
        -------
        theta : array-like of shape = number_of_model_elements
            The estimated parameters of the model.

        Notes
        -----
        A more in-depth documentation of all methods for parameters estimation
        will be available soon. For now, please refer to the mentioned
        references.

        References
        ----------
        - Book: Poularikas, A. D. (2017). Adaptive filtering: Fundamentals
           of least mean squares with MATLAB®. CRC Press.

        """
        n_theta, n, theta, self.xi = self._initial_values(psi)

        for i in range(n_theta, n):
            self.xi = y - psi.dot(theta[:, i - 1].reshape(-1, 1))
            aux = (
                self.mu
                * psi
                @ np.linalg.pinv(psi.T @ psi + self.offset_covariance * np.eye(n_theta))
            )
            tmp_list = theta[:, i - 1].reshape(-1, 1) + aux.T.dot(self.xi)
            theta[:, i] = tmp_list.flatten()

        return theta[:, -1].reshape(-1, 1)

class BoundedVariableLeastSquares(BaseEstimator):
    """Solve a linear least-squares problem with bounds on the variables.

    This is a wrapper class for the `scipy.optimize.lsq_linear` method.

    Given a m-by-n design matrix A and a target vector b with m elements,
    `lsq_linear` solves the following optimization problem::

        minimize 0.5 * ||A x - b||**2
        subject to lb <= x <= ub

    This optimization problem is convex, hence a found minimum (if iterations
    have converged) is guaranteed to be global.

    Attributes
    ----------
    unbiased : bool
        Indicates whether an unbiased estimator is applied.
    uiter : int
        Number of iterations for the unbiased estimator.
    method : 'trf' or 'bvls', optional
        Method to perform minimization.

            * 'trf' : Trust Region Reflective algorithm adapted for a linear
              least-squares problem. This is an interior-point-like method
              and the required number of iterations is weakly correlated with
              the number of variables.
            * 'bvls' : Bounded-variable least-squares algorithm. This is
              an active set method, which requires the number of iterations
              comparable to the number of variables. Can't be used when `A` is
              sparse or LinearOperator.

        Default is 'trf'.
    tol : float, optional
        Tolerance parameter. The algorithm terminates if a relative change
        of the cost function is less than `tol` on the last iteration.
        Additionally, the first-order optimality measure is considered:

            * ``method='trf'`` terminates if the uniform norm of the gradient,
              scaled to account for the presence of the bounds, is less than
              `tol`.
            * ``method='bvls'`` terminates if Karush-Kuhn-Tucker conditions
              are satisfied within `tol` tolerance.

    lsq_solver : {None, 'exact', 'lsmr'}, optional
        Method of solving unbounded least-squares problems throughout
        iterations:

            * 'exact' : Use dense QR or SVD decomposition approach. Can't be
              used when `A` is sparse or LinearOperator.
            * 'lsmr' : Use `scipy.sparse.linalg.lsmr` iterative procedure
              which requires only matrix-vector product evaluations. Can't
              be used with ``method='bvls'``.

        If None (default), the solver is chosen based on type of `A`.
    lsmr_tol : None, float or 'auto', optional
        Tolerance parameters 'atol' and 'btol' for `scipy.sparse.linalg.lsmr`
        If None (default), it is set to ``1e-2 * tol``. If 'auto', the
        tolerance will be adjusted based on the optimality of the current
        iterate, which can speed up the optimization process, but is not always
        reliable.
    max_iter : None or int, optional
        Maximum number of iterations before termination. If None (default), it
        is set to 100 for ``method='trf'`` or to the number of variables for
        ``method='bvls'`` (not counting iterations for 'bvls' initialization).
    verbose : {0, 1, 2}, optional
        Level of algorithm's verbosity:

            * 0 : work silently (default).
            * 1 : display a termination report.
            * 2 : display progress during iterations.
    lsmr_maxiter : None or int, optional
        Maximum number of iterations for the lsmr least squares solver,
        if it is used (by setting ``lsq_solver='lsmr'``). If None (default), it
        uses lsmr's default of ``min(m, n)`` where ``m`` and ``n`` are the
        number of rows and columns of `A`, respectively. Has no effect if
        ``lsq_solver='exact'``.

    References
    ----------
    .. [STIR] M. A. Branch, T. F. Coleman, and Y. Li, "A Subspace, Interior,
              and Conjugate Gradient Method for Large-Scale Bound-Constrained
              Minimization Problems," SIAM Journal on Scientific Computing,
              Vol. 21, Number 1, pp 1-23, 1999.
    .. [BVLS] P. B. Start and R. L. Parker, "Bounded-Variable Least-Squares:
              an Algorithm and Applications", Computational Statistics, 10,
              129-141, 1995.


    Notes
    -----
    This docstring is adapted from the `scipy.optimize.lsq_linear` method.

    Examples
    --------
    In this example, a problem with a large sparse matrix and bounds on the
    variables is solved.

    >>> import numpy as np
    >>> from scipy.sparse import rand
    >>> from sysidentpy.parameter_estimation import BoundedVariableLeastSquares
    >>> rng = np.random.default_rng()
    ...
    >>> m = 20000
    >>> n = 10000
    ...
    >>> A = rand(m, n, density=1e-4, random_state=rng)
    >>> b = rng.standard_normal(m)
    ...
    >>> lb = rng.standard_normal(n)
    >>> ub = lb + 1
    ...
    >>> res = BoundedVariableLeastSquares(A, b, bounds=(lb, ub), lsmr_tol='auto',
    verbose=1)
    The relative change of the cost function is less than `tol`.
    Number of iterations 16, initial cost 1.5039e+04, final cost 1.1112e+04,
    first-order optimality 4.66e-08.
    """

    def __init__(
        self,
        *,
        unbiased: bool = False,
        uiter: int = 30,
        bounds=(-np.inf, np.inf),
        method="trf",
        tol=1e-10,
        lsq_solver=None,
        lsmr_tol=None,
        max_iter=None,
        verbose=0,
        lsmr_maxiter=None,
    ):
        self.unbiased = unbiased
        self.uiter = uiter
        self.max_iter = max_iter
        self.bounds = bounds
        self.method = method
        self.tol = tol
        self.lsq_solver = lsq_solver
        self.lsmr_tol = lsmr_tol
        self.verbose = verbose
        self.lsmr_maxiter = lsmr_maxiter

    def optimize(self, psi, y):
        """Parameter estimation using the BoundedVariableLeastSquares algorithm.

        Parameters
        ----------
        psi : ndarray of floats
            The information matrix of the model.
        y : array-like
            The data used to training the model.

        Returns
        -------
        theta : array-like of shape = number_of_model_elements
            The estimated parameters of the model.

        Notes
        -----
        This is a wrapper class for the `scipy.optimize.lsq_linear` method.

        References
        ----------
        .. [1] scipy, https://docs.scipy.org/doc/scipy/reference/generated/scipy.optimize.lsq_linear.html
        """
        theta = lsq_linear(
            psi,
            y.ravel(),
            bounds=self.bounds,
            method=self.method,
            tol=self.tol,
            lsq_solver=self.lsq_solver,
            lsmr_tol=self.lsmr_tol,
            max_iter=self.max_iter,
            verbose=self.verbose,
            lsmr_maxiter=self.lsmr_maxiter,
        )
        return theta.x.reshape(-1, 1)

class EstimatorError(Exception):
    """Generic Python-exception-derived object raised by estimator functions.

    General purpose exception class, derived from Python's ValueError
    class, programmatically raised in estimators functions when a Estimator-related
    condition would prevent further correct execution of the function.

    Parameters
    ----------
    None

    """

class LeastMeanSquareMixedNorm(BaseEstimator):
    """Least Mean Square Mixed Norm (LMS-MN) Adaptive Filter.

    This class implements the Mixed-norm Least Mean Square (LMS) adaptive filter
    algorithm, which incorporates an additional weight factor to control the
    proportions of the error norms, thus providing an extra degree of freedom
    in the adaptation process.

    Parameters
    ----------
    mu : float, optional
        The adaptation step size. Default is 0.01.
    weight : float, optional
        The weight factor for mixed-norm control. Weight factor to control the
        proportions of the error norms and offers an extra degree of freedom within
        the adaptation of the LMS mixed norm method.

    Attributes
    ----------
    mu : float
        The adaptation step size.
    weight : float
        The weight factor for mixed-norm control. Weight factor to control the
        proportions of the error norms and offers an extra degree of freedom within
        the adaptation of the LMS mixed norm method.
    xi : ndarray or None
        The error signal, initialized to None.
    """

    def __init__(
        self,
        *,
        mu: float = 0.01,
        weight: float = 0.02,
        unbiased: bool = False,
        uiter: int = 30,
    ):
        self.mu = mu
        self.weight = weight
        self.unbiased = unbiased
        self.uiter = uiter
        self._validate_params(vars(self))
        self.xi: np.ndarray

    def optimize(self, psi: np.ndarray, y: np.ndarray) -> np.ndarray:
        """Parameter estimation using the Mixed-norm LMS filter.

        Parameters
        ----------
        psi : ndarray of floats
            The information matrix of the model.
        y : array-like of shape = y_training
            The data used to training the model.

        Returns
        -------
        theta : array-like of shape = number_of_model_elements
            The estimated parameters of the model.

        Notes
        -----
        A more in-depth documentation of all methods for parameters estimation
        will be available soon. For now, please refer to the mentioned
        references.

        References
        ----------
        - Chambers, J. A., Tanrikulu, O., & Constantinides, A. G. (1994).
           Least mean mixed-norm adaptive filtering.
           Electronics letters, 30(19), 1574-1575.
           https://ieeexplore.ieee.org/document/326382
        - Dissertation (Portuguese): Zipf, J. G. F. (2011). Classificação,
           análise estatística e novas estratégias de algoritmos LMS de passo
           variável.
        - Wikipedia entry on Least Mean Squares
           https://en.wikipedia.org/wiki/Least_mean_squares_filter

        """
        n_theta, n, theta, self.xi = self._initial_values(psi)

        for i in range(n_theta, n):
            psi_tmp = psi[i, :].reshape(-1, 1)
            self.xi[i, 0] = y[i, 0] - np.dot(psi_tmp.T, theta[:, i - 1])[0]
            tmp_list = theta[:, i - 1].reshape(-1, 1) + self.mu * psi_tmp * self.xi[
                i, 0
            ] * (self.weight + (1 - self.weight) * self.xi[i, 0] ** 2)
            theta[:, i] = tmp_list.flatten()

        return theta[:, -1].reshape(-1, 1)

class LeastMeanSquares(BaseEstimator):
    """Least Mean Squares (LMS) filter for parameter estimation in adaptive filtering.

    Parameters
    ----------
    mu : float, default=0.01
        The learning rate or step size for the LMS algorithm.

    Attributes
    ----------
    mu : float
        The learning rate or step size for the LMS algorithm.
    xi : np.ndarray or None
        The estimation error at each iteration. Initialized as None and updated during
        optimization.

    Methods
    -------
    optimize(psi: np.ndarray, y: np.ndarray) -> np.ndarray
        Estimate the model parameters using the LMS filter.
    """

    def __init__(self, *, mu: float = 0.01, unbiased: bool = False, uiter: int = 30):
        self.mu = mu
        self.unbiased = unbiased
        self.uiter = uiter
        self._validate_params(vars(self))
        self.xi: np.ndarray

    def optimize(self, psi: np.ndarray, y: np.ndarray) -> np.ndarray:
        """Estimate the model parameters using the Least Mean Squares filter.

        Parameters
        ----------
        psi : ndarray of floats
            The information matrix of the model.
        y : array-like of shape = y_training
            The data used to training the model.

        Returns
        -------
        theta : array-like of shape = number_of_model_elements
            The estimated parameters of the model.

        Notes
        -----
        A more in-depth documentation of all methods for parameters estimation
        will be available soon. For now, please refer to the mentioned
        references.

        References
        ----------
        - Book: Haykin, S., & Widrow, B. (Eds.). (2003). Least-mean-square
           adaptive filters (Vol. 31). John Wiley & Sons.
        - Dissertation (Portuguese): Zipf, J. G. F. (2011). Classificação,
           análise estatística e novas estratégias de algoritmos LMS de passo
           variável.
        - Wikipedia entry on Least Mean Squares
           https://en.wikipedia.org/wiki/Least_mean_squares_filter

        """
        n_theta, n, theta, self.xi = self._initial_values(psi)

        for i in range(n_theta, n):
            psi_tmp = psi[i, :].reshape(-1, 1)
            self.xi[i, 0] = y[i, 0] - np.dot(psi_tmp.T, theta[:, i - 1])[0]
            tmp_list = (
                theta[:, i - 1].reshape(-1, 1) + 2 * self.mu * self.xi[i, 0] * psi_tmp
            )
            theta[:, i] = tmp_list.flatten()

        return theta[:, -1].reshape(-1, 1)

class LeastMeanSquaresFourth(BaseEstimator):
    """Least Mean Squares Fourth(LMSF) filter for parameter estimation.

    Parameters
    ----------
    mu : float, default=0.01
        The learning rate or step size for the LMS algorithm.

    Attributes
    ----------
    mu : float
        The learning rate or step size for the LMS algorithm.
    xi : np.ndarray or None
        The estimation error at each iteration. Initialized as None and updated during
        optimization.

    Methods
    -------
    optimize(psi: np.ndarray, y: np.ndarray) -> np.ndarray
        Estimate the model parameters using the LMS filter.
    """

    def __init__(self, *, mu: float = 0.5, unbiased: bool = False, uiter: int = 30):
        self.mu = mu
        self.unbiased = unbiased
        self.uiter = uiter
        self._validate_params(vars(self))
        self.xi: np.ndarray

    def optimize(self, psi: np.ndarray, y: np.ndarray) -> np.ndarray:
        """Parameter estimation using the  LMS Fourth filter.

        When the leakage factor, gama, is set to 0 then there is no
        leakage in the estimation process.

        Parameters
        ----------
        psi : ndarray of floats
            The information matrix of the model.
        y : ndarray of floats of shape = y_training
            The data used to training the model.

        Returns
        -------
        theta : ndarray of floats of shape = number_of_model_elements
            The estimated parameters of the model.

        Notes
        -----
        A more in-depth documentation of all methods for parameters estimation
        will be available soon. For now, please refer to the mentioned
        references.

        References
        ----------
        - Book: Hayes, M. H. (2009). Statistical digital signal processing
           and modeling. John Wiley & Sons.
        - Dissertation (Portuguese): Zipf, J. G. F. (2011). Classificação,
           análise estatística e novas estratégias de algoritmos LMS de passo
           variável.
        - Manuscript:Gui, G., Mehbodniya, A., & Adachi, F. (2013).
           Least mean square/fourth algorithm with application to sparse
           channel estimation. arXiv preprint arXiv:1304.3911.
           https://arxiv.org/pdf/1304.3911.pdf
        - Manuscript: Nascimento, V. H., & Bermudez, J. C. M. (2005, March).
           When is the least-mean fourth algorithm mean-square stable?
           In Proceedings.(ICASSP'05). IEEE International Conference on
           Acoustics, Speech, and Signal Processing, 2005.
           (Vol. 4, pp. iv-341). IEEE.
           http://www.lps.usp.br/vitor/artigos/icassp05.pdf
        - Wikipedia entry on Least Mean Squares
           https://en.wikipedia.org/wiki/Least_mean_squares_filter

        """
        n_theta, n, theta, self.xi = self._initial_values(psi)

        for i in range(n_theta, n):
            psi_tmp = psi[i, :].reshape(-1, 1)
            self.xi[i, 0] = y[i, 0] - np.dot(psi_tmp.T, theta[:, i - 1])[0]
            tmp_list = (
                theta[:, i - 1].reshape(-1, 1) + self.mu * psi_tmp * self.xi[i, 0] ** 3
            )
            theta[:, i] = tmp_list.flatten()

        return theta[:, -1].reshape(-1, 1)

class LeastMeanSquaresLeaky(BaseEstimator):
    """Least Mean Squares Leaky(LMSL) filter for parameter estimation.

    Parameters
    ----------
    mu : float, default=0.01
        The learning rate or step size for the LMS algorithm.
    gama : float, default=0.2
        The leakage factor of the Leaky LMS method.

    Attributes
    ----------
    mu : float
        The learning rate or step size for the LMS algorithm.
    gama : float, default=0.2
        The leakage factor of the Leaky LMS method.
    xi : np.ndarray or None
        The estimation error at each iteration. Initialized as None and updated during
        optimization.

    Methods
    -------
    optimize(psi: np.ndarray, y: np.ndarray) -> np.ndarray
        Estimate the model parameters using the LMS filter.
    """

    def __init__(
        self,
        *,
        mu: float = 0.01,
        gama: float = 0.001,
        unbiased: bool = False,
        uiter: int = 30,
    ):
        self.mu = mu
        self.gama = gama
        self.unbiased = unbiased
        self.uiter = uiter
        self._validate_params(vars(self))
        self.xi: np.ndarray

    def optimize(self, psi: np.ndarray, y: np.ndarray) -> np.ndarray:
        """Parameter estimation using the  Leaky LMS filter.

        When the leakage factor, gama, is set to 0 then there is no
        leakage in the estimation process.

        Parameters
        ----------
        psi : ndarray of floats
            The information matrix of the model.
        y : array-like of shape = y_training
            The data used to training the model.

        Returns
        -------
        theta : array-like of shape = number_of_model_elements
            The estimated parameters of the model.

        Notes
        -----
        A more in-depth documentation of all methods for parameters estimation
        will be available soon. For now, please refer to the mentioned
        references.

        References
        ----------
        - Book: Hayes, M. H. (2009). Statistical digital signal processing
           and modeling. John Wiley & Sons.
        - Dissertation (Portuguese): Zipf, J. G. F. (2011). Classificação,
           análise estatística e novas estratégias de algoritmos LMS de passo
           variável.
        - Wikipedia entry on Least Mean Squares
           https://en.wikipedia.org/wiki/Least_mean_squares_filter

        """
        n_theta, n, theta, self.xi = self._initial_values(psi)

        for i in range(n_theta, n):
            psi_tmp = psi[i, :].reshape(-1, 1)
            self.xi[i, 0] = y[i, 0] - np.dot(psi_tmp.T, theta[:, i - 1])[0]
            tmp_list = (
                theta[:, i - 1].reshape(-1, 1) * (1 - self.mu * self.gama)
                + self.mu * self.xi[i, 0] * psi_tmp
            )
            theta[:, i] = tmp_list.flatten()

        return theta[:, -1].reshape(-1, 1)

class LeastMeanSquaresNormalizedLeaky(BaseEstimator):
    """Normalized Least Mean Squares Leaky(NLMSL) filter for parameter estimation.

    Parameters
    ----------
    mu : float, default=0.01
        The learning rate or step size for the LMS algorithm.
    eps : float, default=np.finfo(np.float64).eps
        Normalization factor of the normalized filters.
    gama : float, default=0.2
        The leakage factor of the Leaky LMS method.

    Attributes
    ----------
    mu : float
        The learning rate or step size for the LMS algorithm.
    eps : float, default=np.finfo(np.float64).eps
        Normalization factor of the normalized filters.
    gama : float, default=0.2
        The leakage factor of the Leaky LMS method.
    xi : np.ndarray or None
        The estimation error at each iteration. Initialized as None and updated during
        optimization.

    Methods
    -------
    optimize(psi: np.ndarray, y: np.ndarray) -> np.ndarray
        Estimate the model parameters using the LMS filter.
    """

    def __init__(
        self,
        *,
        mu: float = 0.01,
        gama: float = 0.2,
        eps: np.float64 = np.finfo(np.float64).eps,
        unbiased: bool = False,
        uiter: int = 30,
    ):
        self.mu = mu
        self.eps = eps
        self.gama = gama
        self.unbiased = unbiased
        self.uiter = uiter
        self._validate_params(vars(self))
        self.xi: np.ndarray

    def optimize(self, psi: np.ndarray, y: np.ndarray) -> np.ndarray:
        """Parameter estimation using the  Normalized Leaky LMS filter.

        When the leakage factor, gama, is set to 0 then there is no
        leakage in the estimation process.

        Parameters
        ----------
        psi : ndarray of floats
            The information matrix of the model.
        y : array-like of shape = y_training
            The data used to training the model.

        Returns
        -------
        theta : array-like of shape = number_of_model_elements
            The estimated parameters of the model.

        Notes
        -----
        A more in-depth documentation of all methods for parameters estimation
        will be available soon. For now, please refer to the mentioned
        references.

        References
        ----------
        - Book: Hayes, M. H. (2009). Statistical digital signal processing
           and modeling. John Wiley & Sons.
        - Dissertation (Portuguese): Zipf, J. G. F. (2011). Classificação,
           análise estatística e novas estratégias de algoritmos LMS de passo
           variável.
        - Wikipedia entry on Least Mean Squares
           https://en.wikipedia.org/wiki/Least_mean_squares_filter

        """
        n_theta, n, theta, self.xi = self._initial_values(psi)

        for i in range(n_theta, n):
            psi_tmp = psi[i, :].reshape(-1, 1)
            self.xi[i, 0] = y[i, 0] - np.dot(psi_tmp.T, theta[:, i - 1])[0]
            tmp_list = theta[:, i - 1].reshape(-1, 1) * (
                1 - self.mu * self.gama
            ) + self.mu * self.xi[i, 0] * psi_tmp / (
                self.eps + np.dot(psi_tmp.T, psi_tmp)
            )
            theta[:, i] = tmp_list.flatten()

        return theta[:, -1].reshape(-1, 1)

class LeastMeanSquaresNormalizedSignRegressor(BaseEstimator):
    """Normalized Least Mean Squares SignRegressor filter for parameter estimation.

    Parameters
    ----------
    mu : float, default=0.01
        The learning rate or step size for the LMS algorithm.
    eps : float, default=np.finfo(np.float64).eps
        Normalization factor of the normalized filters.

    Attributes
    ----------
    mu : float
        The learning rate or step size for the LMS algorithm.
    eps : float, default=np.finfo(np.float64).eps
        Normalization factor of the normalized filters.
    xi : np.ndarray or None
        The estimation error at each iteration. Initialized as None and updated during
        optimization.

    Methods
    -------
    optimize(psi: np.ndarray, y: np.ndarray) -> np.ndarray
        Estimate the model parameters using the LMS filter.
    """

    def __init__(
        self,
        *,
        mu: float = 0.01,
        eps: np.float64 = np.finfo(np.float64).eps,
        unbiased: bool = False,
        uiter: int = 30,
    ):
        self.mu = mu
        self.eps = eps
        self.unbiased = unbiased
        self.uiter = uiter
        self._validate_params(vars(self))
        self.xi: np.ndarray

    def optimize(self, psi: np.ndarray, y: np.ndarray) -> np.ndarray:
        """Parameter estimation using the Normalized Sign-Regressor LMS filter.

        The normalization is used to avoid numerical instability when updating
        the estimated parameters and the sign of the information matrix is
        used to change the filter coefficients.

        Parameters
        ----------
        psi : ndarray of floats
            The information matrix of the model.
        y : array-like of shape = y_training
            The data used to training the model.

        Returns
        -------
        theta : array-like of shape = number_of_model_elements
            The estimated parameters of the model.

        Notes
        -----
        A more in-depth documentation of all methods for parameters estimation
        will be available soon. For now, please refer to the mentioned
        references.

        References
        ----------
        .. [1] Book: Hayes, M. H. (2009). Statistical digital signal processing
           and modeling. John Wiley & Sons.
        .. [2] Dissertation (Portuguese): Zipf, J. G. F. (2011). Classificação,
           análise estatística e novas estratégias de algoritmos LMS de passo
           variável.
        .. [3] Wikipedia entry on Least Mean Squares
           https://en.wikipedia.org/wiki/Least_mean_squares_filter

        """
        n_theta, n, theta, self.xi = self._initial_values(psi)

        for i in range(n_theta, n):
            psi_tmp = psi[i, :].reshape(-1, 1)
            self.xi[i, 0] = y[i, 0] - np.dot(psi_tmp.T, theta[:, i - 1])[0]
            tmp_list = theta[:, i - 1].reshape(-1, 1) + self.mu * self.xi[i, 0] * (
                np.sign(psi_tmp) / (self.eps + np.dot(psi_tmp.T, psi_tmp))
            )
            theta[:, i] = tmp_list.flatten()

        return theta[:, -1].reshape(-1, 1)