Uploaded Test files

venv/Lib/site-packages/sklearn/utils/optimize.py

@@ -0,0 +1,256 @@
+"""
+Our own implementation of the Newton algorithm
+
+Unlike the scipy.optimize version, this version of the Newton conjugate
+gradient solver uses only one function call to retrieve the
+func value, the gradient value and a callable for the Hessian matvec
+product. If the function call is very expensive (e.g. for logistic
+regression with large design matrix), this approach gives very
+significant speedups.
+"""
+# This is a modified file from scipy.optimize
+# Original authors: Travis Oliphant, Eric Jones
+# Modifications by Gael Varoquaux, Mathieu Blondel and Tom Dupre la Tour
+# License: BSD
+
+import numpy as np
+import warnings
+from scipy.optimize.linesearch import line_search_wolfe2, line_search_wolfe1
+
+from ..exceptions import ConvergenceWarning
+from . import deprecated
+
+
+class _LineSearchError(RuntimeError):
+    pass
+
+
+def _line_search_wolfe12(f, fprime, xk, pk, gfk, old_fval, old_old_fval,
+                         **kwargs):
+    """
+    Same as line_search_wolfe1, but fall back to line_search_wolfe2 if
+    suitable step length is not found, and raise an exception if a
+    suitable step length is not found.
+
+    Raises
+    ------
+    _LineSearchError
+        If no suitable step size is found
+
+    """
+    ret = line_search_wolfe1(f, fprime, xk, pk, gfk,
+                             old_fval, old_old_fval,
+                             **kwargs)
+
+    if ret[0] is None:
+        # line search failed: try different one.
+        ret = line_search_wolfe2(f, fprime, xk, pk, gfk,
+                                 old_fval, old_old_fval, **kwargs)
+
+    if ret[0] is None:
+        raise _LineSearchError()
+
+    return ret
+
+
+def _cg(fhess_p, fgrad, maxiter, tol):
+    """
+    Solve iteratively the linear system 'fhess_p . xsupi = fgrad'
+    with a conjugate gradient descent.
+
+    Parameters
+    ----------
+    fhess_p : callable
+        Function that takes the gradient as a parameter and returns the
+        matrix product of the Hessian and gradient
+
+    fgrad : ndarray, shape (n_features,) or (n_features + 1,)
+        Gradient vector
+
+    maxiter : int
+        Number of CG iterations.
+
+    tol : float
+        Stopping criterion.
+
+    Returns
+    -------
+    xsupi : ndarray, shape (n_features,) or (n_features + 1,)
+        Estimated solution
+    """
+    xsupi = np.zeros(len(fgrad), dtype=fgrad.dtype)
+    ri = fgrad
+    psupi = -ri
+    i = 0
+    dri0 = np.dot(ri, ri)
+
+    while i <= maxiter:
+        if np.sum(np.abs(ri)) <= tol:
+            break
+
+        Ap = fhess_p(psupi)
+        # check curvature
+        curv = np.dot(psupi, Ap)
+        if 0 <= curv <= 3 * np.finfo(np.float64).eps:
+            break
+        elif curv < 0:
+            if i > 0:
+                break
+            else:
+                # fall back to steepest descent direction
+                xsupi += dri0 / curv * psupi
+                break
+        alphai = dri0 / curv
+        xsupi += alphai * psupi
+        ri = ri + alphai * Ap
+        dri1 = np.dot(ri, ri)
+        betai = dri1 / dri0
+        psupi = -ri + betai * psupi
+        i = i + 1
+        dri0 = dri1          # update np.dot(ri,ri) for next time.
+
+    return xsupi
+
+
+@deprecated("newton_cg is deprecated in version "
+            "0.22 and will be removed in version 0.24.")
+def newton_cg(grad_hess, func, grad, x0, args=(), tol=1e-4,
+              maxiter=100, maxinner=200, line_search=True, warn=True):
+    return _newton_cg(grad_hess, func, grad, x0, args, tol, maxiter,
+                      maxinner, line_search, warn)
+
+
+def _newton_cg(grad_hess, func, grad, x0, args=(), tol=1e-4,
+               maxiter=100, maxinner=200, line_search=True, warn=True):
+    """
+    Minimization of scalar function of one or more variables using the
+    Newton-CG algorithm.
+
+    Parameters
+    ----------
+    grad_hess : callable
+        Should return the gradient and a callable returning the matvec product
+        of the Hessian.
+
+    func : callable
+        Should return the value of the function.
+
+    grad : callable
+        Should return the function value and the gradient. This is used
+        by the linesearch functions.
+
+    x0 : array of float
+        Initial guess.
+
+    args : tuple, optional
+        Arguments passed to func_grad_hess, func and grad.
+
+    tol : float
+        Stopping criterion. The iteration will stop when
+        ``max{|g_i | i = 1, ..., n} <= tol``
+        where ``g_i`` is the i-th component of the gradient.
+
+    maxiter : int
+        Number of Newton iterations.
+
+    maxinner : int
+        Number of CG iterations.
+
+    line_search : boolean
+        Whether to use a line search or not.
+
+    warn : boolean
+        Whether to warn when didn't converge.
+
+    Returns
+    -------
+    xk : ndarray of float
+        Estimated minimum.
+    """
+    x0 = np.asarray(x0).flatten()
+    xk = x0
+    k = 0
+
+    if line_search:
+        old_fval = func(x0, *args)
+        old_old_fval = None
+
+    # Outer loop: our Newton iteration
+    while k < maxiter:
+        # Compute a search direction pk by applying the CG method to
+        #  del2 f(xk) p = - fgrad f(xk) starting from 0.
+        fgrad, fhess_p = grad_hess(xk, *args)
+
+        absgrad = np.abs(fgrad)
+        if np.max(absgrad) <= tol:
+            break
+
+        maggrad = np.sum(absgrad)
+        eta = min([0.5, np.sqrt(maggrad)])
+        termcond = eta * maggrad
+
+        # Inner loop: solve the Newton update by conjugate gradient, to
+        # avoid inverting the Hessian
+        xsupi = _cg(fhess_p, fgrad, maxiter=maxinner, tol=termcond)
+
+        alphak = 1.0
+
+        if line_search:
+            try:
+                alphak, fc, gc, old_fval, old_old_fval, gfkp1 = \
+                    _line_search_wolfe12(func, grad, xk, xsupi, fgrad,
+                                         old_fval, old_old_fval, args=args)
+            except _LineSearchError:
+                warnings.warn('Line Search failed')
+                break
+
+        xk = xk + alphak * xsupi        # upcast if necessary
+        k += 1
+
+    if warn and k >= maxiter:
+        warnings.warn("newton-cg failed to converge. Increase the "
+                      "number of iterations.", ConvergenceWarning)
+    return xk, k
+
+
+def _check_optimize_result(solver, result, max_iter=None,
+                           extra_warning_msg=None):
+    """Check the OptimizeResult for successful convergence
+
+    Parameters
+    ----------
+    solver: str
+       solver name. Currently only `lbfgs` is supported.
+    result: OptimizeResult
+       result of the scipy.optimize.minimize function
+    max_iter: {int, None}
+       expected maximum number of iterations
+
+    Returns
+    -------
+    n_iter: int
+       number of iterations
+    """
+    # handle both scipy and scikit-learn solver names
+    if solver == "lbfgs":
+        if result.status != 0:
+            warning_msg = (
+                "{} failed to converge (status={}):\n{}.\n\n"
+                "Increase the number of iterations (max_iter) "
+                "or scale the data as shown in:\n"
+                "    https://scikit-learn.org/stable/modules/"
+                "preprocessing.html"
+            ).format(solver, result.status, result.message.decode("latin1"))
+            if extra_warning_msg is not None:
+                warning_msg += "\n" + extra_warning_msg
+            warnings.warn(warning_msg, ConvergenceWarning, stacklevel=2)
+        if max_iter is not None:
+            # In scipy <= 1.0.0, nit may exceed maxiter for lbfgs.
+            # See https://github.com/scipy/scipy/issues/7854
+            n_iter_i = min(result.nit, max_iter)
+        else:
+            n_iter_i = result.nit
+    else:
+        raise NotImplementedError
+
+    return n_iter_i