Fixed database typo and removed unnecessary class identifier.

venv/Lib/site-packages/scipy/optimize/tests/test__basinhopping.py

@@ -0,0 +1,466 @@
+"""
+Unit tests for the basin hopping global minimization algorithm.
+"""
+import copy
+
+from numpy.testing import assert_almost_equal, assert_equal, assert_
+import pytest
+from pytest import raises as assert_raises
+import numpy as np
+from numpy import cos, sin
+
+from scipy.optimize import basinhopping, OptimizeResult
+from scipy.optimize._basinhopping import (
+    Storage, RandomDisplacement, Metropolis, AdaptiveStepsize)
+from scipy._lib._pep440 import Version
+
+
+def func1d(x):
+    f = cos(14.5 * x - 0.3) + (x + 0.2) * x
+    df = np.array(-14.5 * sin(14.5 * x - 0.3) + 2. * x + 0.2)
+    return f, df
+
+
+def func2d_nograd(x):
+    f = cos(14.5 * x[0] - 0.3) + (x[1] + 0.2) * x[1] + (x[0] + 0.2) * x[0]
+    return f
+
+
+def func2d(x):
+    f = cos(14.5 * x[0] - 0.3) + (x[1] + 0.2) * x[1] + (x[0] + 0.2) * x[0]
+    df = np.zeros(2)
+    df[0] = -14.5 * sin(14.5 * x[0] - 0.3) + 2. * x[0] + 0.2
+    df[1] = 2. * x[1] + 0.2
+    return f, df
+
+
+def func2d_easyderiv(x):
+    f = 2.0*x[0]**2 + 2.0*x[0]*x[1] + 2.0*x[1]**2 - 6.0*x[0]
+    df = np.zeros(2)
+    df[0] = 4.0*x[0] + 2.0*x[1] - 6.0
+    df[1] = 2.0*x[0] + 4.0*x[1]
+
+    return f, df
+
+
+class MyTakeStep1(RandomDisplacement):
+    """use a copy of displace, but have it set a special parameter to
+    make sure it's actually being used."""
+    def __init__(self):
+        self.been_called = False
+        super(MyTakeStep1, self).__init__()
+
+    def __call__(self, x):
+        self.been_called = True
+        return super(MyTakeStep1, self).__call__(x)
+
+
+def myTakeStep2(x):
+    """redo RandomDisplacement in function form without the attribute stepsize
+    to make sure everything still works ok
+    """
+    s = 0.5
+    x += np.random.uniform(-s, s, np.shape(x))
+    return x
+
+
+class MyAcceptTest(object):
+    """pass a custom accept test
+
+    This does nothing but make sure it's being used and ensure all the
+    possible return values are accepted
+    """
+    def __init__(self):
+        self.been_called = False
+        self.ncalls = 0
+        self.testres = [False, 'force accept', True, np.bool_(True),
+                        np.bool_(False), [], {}, 0, 1]
+
+    def __call__(self, **kwargs):
+        self.been_called = True
+        self.ncalls += 1
+        if self.ncalls - 1 < len(self.testres):
+            return self.testres[self.ncalls - 1]
+        else:
+            return True
+
+
+class MyCallBack(object):
+    """pass a custom callback function
+
+    This makes sure it's being used. It also returns True after 10
+    steps to ensure that it's stopping early.
+
+    """
+    def __init__(self):
+        self.been_called = False
+        self.ncalls = 0
+
+    def __call__(self, x, f, accepted):
+        self.been_called = True
+        self.ncalls += 1
+        if self.ncalls == 10:
+            return True
+
+
+class TestBasinHopping(object):
+
+    def setup_method(self):
+        """ Tests setup.
+
+        Run tests based on the 1-D and 2-D functions described above.
+        """
+        self.x0 = (1.0, [1.0, 1.0])
+        self.sol = (-0.195, np.array([-0.195, -0.1]))
+
+        self.tol = 3  # number of decimal places
+
+        self.niter = 100
+        self.disp = False
+
+        # fix random seed
+        np.random.seed(1234)
+
+        self.kwargs = {"method": "L-BFGS-B", "jac": True}
+        self.kwargs_nograd = {"method": "L-BFGS-B"}
+
+    def test_TypeError(self):
+        # test the TypeErrors are raised on bad input
+        i = 1
+        # if take_step is passed, it must be callable
+        assert_raises(TypeError, basinhopping, func2d, self.x0[i],
+                      take_step=1)
+        # if accept_test is passed, it must be callable
+        assert_raises(TypeError, basinhopping, func2d, self.x0[i],
+                      accept_test=1)
+
+    def test_1d_grad(self):
+        # test 1-D minimizations with gradient
+        i = 0
+        res = basinhopping(func1d, self.x0[i], minimizer_kwargs=self.kwargs,
+                           niter=self.niter, disp=self.disp)
+        assert_almost_equal(res.x, self.sol[i], self.tol)
+
+    def test_2d(self):
+        # test 2d minimizations with gradient
+        i = 1
+        res = basinhopping(func2d, self.x0[i], minimizer_kwargs=self.kwargs,
+                           niter=self.niter, disp=self.disp)
+        assert_almost_equal(res.x, self.sol[i], self.tol)
+        assert_(res.nfev > 0)
+
+    def test_njev(self):
+        # test njev is returned correctly
+        i = 1
+        minimizer_kwargs = self.kwargs.copy()
+        # L-BFGS-B doesn't use njev, but BFGS does
+        minimizer_kwargs["method"] = "BFGS"
+        res = basinhopping(func2d, self.x0[i],
+                           minimizer_kwargs=minimizer_kwargs, niter=self.niter,
+                           disp=self.disp)
+        assert_(res.nfev > 0)
+        assert_equal(res.nfev, res.njev)
+
+    def test_jac(self):
+        # test Jacobian returned
+        minimizer_kwargs = self.kwargs.copy()
+        # BFGS returns a Jacobian
+        minimizer_kwargs["method"] = "BFGS"
+
+        res = basinhopping(func2d_easyderiv, [0.0, 0.0],
+                           minimizer_kwargs=minimizer_kwargs, niter=self.niter,
+                           disp=self.disp)
+
+        assert_(hasattr(res.lowest_optimization_result, "jac"))
+
+        # in this case, the Jacobian is just [df/dx, df/dy]
+        _, jacobian = func2d_easyderiv(res.x)
+        assert_almost_equal(res.lowest_optimization_result.jac, jacobian,
+                            self.tol)
+
+    def test_2d_nograd(self):
+        # test 2-D minimizations without gradient
+        i = 1
+        res = basinhopping(func2d_nograd, self.x0[i],
+                           minimizer_kwargs=self.kwargs_nograd,
+                           niter=self.niter, disp=self.disp)
+        assert_almost_equal(res.x, self.sol[i], self.tol)
+
+    def test_all_minimizers(self):
+        # Test 2-D minimizations with gradient. Nelder-Mead, Powell, and COBYLA
+        # don't accept jac=True, so aren't included here.
+        i = 1
+        methods = ['CG', 'BFGS', 'Newton-CG', 'L-BFGS-B', 'TNC', 'SLSQP']
+        minimizer_kwargs = copy.copy(self.kwargs)
+        for method in methods:
+            minimizer_kwargs["method"] = method
+            res = basinhopping(func2d, self.x0[i],
+                               minimizer_kwargs=minimizer_kwargs,
+                               niter=self.niter, disp=self.disp)
+            assert_almost_equal(res.x, self.sol[i], self.tol)
+
+    def test_all_nograd_minimizers(self):
+        # Test 2-D minimizations without gradient. Newton-CG requires jac=True,
+        # so not included here.
+        i = 1
+        methods = ['CG', 'BFGS', 'L-BFGS-B', 'TNC', 'SLSQP',
+                   'Nelder-Mead', 'Powell', 'COBYLA']
+        minimizer_kwargs = copy.copy(self.kwargs_nograd)
+        for method in methods:
+            minimizer_kwargs["method"] = method
+            res = basinhopping(func2d_nograd, self.x0[i],
+                               minimizer_kwargs=minimizer_kwargs,
+                               niter=self.niter, disp=self.disp)
+            tol = self.tol
+            if method == 'COBYLA':
+                tol = 2
+            assert_almost_equal(res.x, self.sol[i], decimal=tol)
+
+    def test_pass_takestep(self):
+        # test that passing a custom takestep works
+        # also test that the stepsize is being adjusted
+        takestep = MyTakeStep1()
+        initial_step_size = takestep.stepsize
+        i = 1
+        res = basinhopping(func2d, self.x0[i], minimizer_kwargs=self.kwargs,
+                           niter=self.niter, disp=self.disp,
+                           take_step=takestep)
+        assert_almost_equal(res.x, self.sol[i], self.tol)
+        assert_(takestep.been_called)
+        # make sure that the build in adaptive step size has been used
+        assert_(initial_step_size != takestep.stepsize)
+
+    def test_pass_simple_takestep(self):
+        # test that passing a custom takestep without attribute stepsize
+        takestep = myTakeStep2
+        i = 1
+        res = basinhopping(func2d_nograd, self.x0[i],
+                           minimizer_kwargs=self.kwargs_nograd,
+                           niter=self.niter, disp=self.disp,
+                           take_step=takestep)
+        assert_almost_equal(res.x, self.sol[i], self.tol)
+
+    def test_pass_accept_test(self):
+        # test passing a custom accept test
+        # makes sure it's being used and ensures all the possible return values
+        # are accepted.
+        accept_test = MyAcceptTest()
+        i = 1
+        # there's no point in running it more than a few steps.
+        basinhopping(func2d, self.x0[i], minimizer_kwargs=self.kwargs,
+                     niter=10, disp=self.disp, accept_test=accept_test)
+        assert_(accept_test.been_called)
+
+    def test_pass_callback(self):
+        # test passing a custom callback function
+        # This makes sure it's being used. It also returns True after 10 steps
+        # to ensure that it's stopping early.
+        callback = MyCallBack()
+        i = 1
+        # there's no point in running it more than a few steps.
+        res = basinhopping(func2d, self.x0[i], minimizer_kwargs=self.kwargs,
+                           niter=30, disp=self.disp, callback=callback)
+        assert_(callback.been_called)
+        assert_("callback" in res.message[0])
+        assert_equal(res.nit, 10)
+
+    def test_minimizer_fail(self):
+        # test if a minimizer fails
+        i = 1
+        self.kwargs["options"] = dict(maxiter=0)
+        self.niter = 10
+        res = basinhopping(func2d, self.x0[i], minimizer_kwargs=self.kwargs,
+                           niter=self.niter, disp=self.disp)
+        # the number of failed minimizations should be the number of
+        # iterations + 1
+        assert_equal(res.nit + 1, res.minimization_failures)
+
+    def test_niter_zero(self):
+        # gh5915, what happens if you call basinhopping with niter=0
+        i = 0
+        basinhopping(func1d, self.x0[i], minimizer_kwargs=self.kwargs,
+                     niter=0, disp=self.disp)
+
+    def test_seed_reproducibility(self):
+        # seed should ensure reproducibility between runs
+        minimizer_kwargs = {"method": "L-BFGS-B", "jac": True}
+
+        f_1 = []
+
+        def callback(x, f, accepted):
+            f_1.append(f)
+
+        basinhopping(func2d, [1.0, 1.0], minimizer_kwargs=minimizer_kwargs,
+                     niter=10, callback=callback, seed=10)
+
+        f_2 = []
+
+        def callback2(x, f, accepted):
+            f_2.append(f)
+
+        basinhopping(func2d, [1.0, 1.0], minimizer_kwargs=minimizer_kwargs,
+                     niter=10, callback=callback2, seed=10)
+        assert_equal(np.array(f_1), np.array(f_2))
+
+    @pytest.mark.skipif(Version(np.__version__) < Version('1.17'),
+                        reason='Generator not available for numpy, < 1.17')
+    def test_random_gen(self):
+        # check that np.random.Generator can be used (numpy >= 1.17)
+        rng = np.random.default_rng(1)
+
+        minimizer_kwargs = {"method": "L-BFGS-B", "jac": True}
+
+        res1 = basinhopping(func2d, [1.0, 1.0],
+                            minimizer_kwargs=minimizer_kwargs,
+                            niter=10, seed=rng)
+
+        rng = np.random.default_rng(1)
+        res2 = basinhopping(func2d, [1.0, 1.0],
+                            minimizer_kwargs=minimizer_kwargs,
+                            niter=10, seed=rng)
+        assert_equal(res1.x, res2.x)
+
+    def test_monotonic_basin_hopping(self):
+        # test 1-D minimizations with gradient and T=0
+        i = 0
+        res = basinhopping(func1d, self.x0[i], minimizer_kwargs=self.kwargs,
+                           niter=self.niter, disp=self.disp, T=0)
+        assert_almost_equal(res.x, self.sol[i], self.tol)
+
+
+class Test_Storage(object):
+    def setup_method(self):
+        self.x0 = np.array(1)
+        self.f0 = 0
+
+        minres = OptimizeResult()
+        minres.x = self.x0
+        minres.fun = self.f0
+
+        self.storage = Storage(minres)
+
+    def test_higher_f_rejected(self):
+        new_minres = OptimizeResult()
+        new_minres.x = self.x0 + 1
+        new_minres.fun = self.f0 + 1
+
+        ret = self.storage.update(new_minres)
+        minres = self.storage.get_lowest()
+        assert_equal(self.x0, minres.x)
+        assert_equal(self.f0, minres.fun)
+        assert_(not ret)
+
+    def test_lower_f_accepted(self):
+        new_minres = OptimizeResult()
+        new_minres.x = self.x0 + 1
+        new_minres.fun = self.f0 - 1
+
+        ret = self.storage.update(new_minres)
+        minres = self.storage.get_lowest()
+        assert_(self.x0 != minres.x)
+        assert_(self.f0 != minres.fun)
+        assert_(ret)
+
+
+class Test_RandomDisplacement(object):
+    def setup_method(self):
+        self.stepsize = 1.0
+        self.displace = RandomDisplacement(stepsize=self.stepsize)
+        self.N = 300000
+        self.x0 = np.zeros([self.N])
+
+    def test_random(self):
+        # the mean should be 0
+        # the variance should be (2*stepsize)**2 / 12
+        # note these tests are random, they will fail from time to time
+        x = self.displace(self.x0)
+        v = (2. * self.stepsize) ** 2 / 12
+        assert_almost_equal(np.mean(x), 0., 1)
+        assert_almost_equal(np.var(x), v, 1)
+
+
+class Test_Metropolis(object):
+    def setup_method(self):
+        self.T = 2.
+        self.met = Metropolis(self.T)
+
+    def test_boolean_return(self):
+        # the return must be a bool, else an error will be raised in
+        # basinhopping
+        ret = self.met(f_new=0., f_old=1.)
+        assert isinstance(ret, bool)
+
+    def test_lower_f_accepted(self):
+        assert_(self.met(f_new=0., f_old=1.))
+
+    def test_KeyError(self):
+        # should raise KeyError if kwargs f_old or f_new is not passed
+        assert_raises(KeyError, self.met, f_old=1.)
+        assert_raises(KeyError, self.met, f_new=1.)
+
+    def test_accept(self):
+        # test that steps are randomly accepted for f_new > f_old
+        one_accept = False
+        one_reject = False
+        for i in range(1000):
+            if one_accept and one_reject:
+                break
+            ret = self.met(f_new=1., f_old=0.5)
+            if ret:
+                one_accept = True
+            else:
+                one_reject = True
+        assert_(one_accept)
+        assert_(one_reject)
+
+    def test_GH7495(self):
+        # an overflow in exp was producing a RuntimeWarning
+        # create own object here in case someone changes self.T
+        met = Metropolis(2)
+        with np.errstate(over='raise'):
+            met.accept_reject(0, 2000)
+
+
+class Test_AdaptiveStepsize(object):
+    def setup_method(self):
+        self.stepsize = 1.
+        self.ts = RandomDisplacement(stepsize=self.stepsize)
+        self.target_accept_rate = 0.5
+        self.takestep = AdaptiveStepsize(takestep=self.ts, verbose=False,
+                                         accept_rate=self.target_accept_rate)
+
+    def test_adaptive_increase(self):
+        # if few steps are rejected, the stepsize should increase
+        x = 0.
+        self.takestep(x)
+        self.takestep.report(False)
+        for i in range(self.takestep.interval):
+            self.takestep(x)
+            self.takestep.report(True)
+        assert_(self.ts.stepsize > self.stepsize)
+
+    def test_adaptive_decrease(self):
+        # if few steps are rejected, the stepsize should increase
+        x = 0.
+        self.takestep(x)
+        self.takestep.report(True)
+        for i in range(self.takestep.interval):
+            self.takestep(x)
+            self.takestep.report(False)
+        assert_(self.ts.stepsize < self.stepsize)
+
+    def test_all_accepted(self):
+        # test that everything works OK if all steps were accepted
+        x = 0.
+        for i in range(self.takestep.interval + 1):
+            self.takestep(x)
+            self.takestep.report(True)
+        assert_(self.ts.stepsize > self.stepsize)
+
+    def test_all_rejected(self):
+        # test that everything works OK if all steps were rejected
+        x = 0.
+        for i in range(self.takestep.interval + 1):
+            self.takestep(x)
+            self.takestep.report(False)
+        assert_(self.ts.stepsize < self.stepsize)

venv/Lib/site-packages/scipy/optimize/tests/test__dual_annealing.py

@@ -0,0 +1,260 @@
+# Dual annealing unit tests implementation.
+# Copyright (c) 2018 Sylvain Gubian <sylvain.gubian@pmi.com>,
+# Yang Xiang <yang.xiang@pmi.com>
+# Author: Sylvain Gubian, PMP S.A.
+"""
+Unit tests for the dual annealing global optimizer
+"""
+from scipy.optimize import dual_annealing
+from scipy.optimize._dual_annealing import VisitingDistribution
+from scipy.optimize._dual_annealing import ObjectiveFunWrapper
+from scipy.optimize._dual_annealing import EnergyState
+from scipy.optimize._dual_annealing import LocalSearchWrapper
+from scipy.optimize import rosen, rosen_der
+import pytest
+import numpy as np
+from numpy.testing import assert_equal, assert_allclose, assert_array_less
+from pytest import raises as assert_raises
+from scipy._lib._util import check_random_state
+from scipy._lib._pep440 import Version
+
+
+class TestDualAnnealing:
+
+    def setup_method(self):
+        # A function that returns always infinity for initialization tests
+        self.weirdfunc = lambda x: np.inf
+        # 2-D bounds for testing function
+        self.ld_bounds = [(-5.12, 5.12)] * 2
+        # 4-D bounds for testing function
+        self.hd_bounds = self.ld_bounds * 4
+        # Number of values to be generated for testing visit function
+        self.nbtestvalues = 5000
+        self.high_temperature = 5230
+        self.low_temperature = 0.1
+        self.qv = 2.62
+        self.seed = 1234
+        self.rs = check_random_state(self.seed)
+        self.nb_fun_call = 0
+        self.ngev = 0
+
+    def callback(self, x, f, context):
+        # For testing callback mechanism. Should stop for e <= 1 as
+        # the callback function returns True
+        if f <= 1.0:
+            return True
+
+    def func(self, x, args=()):
+        # Using Rastrigin function for performing tests
+        if args:
+            shift = args
+        else:
+            shift = 0
+        y = np.sum((x - shift) ** 2 - 10 * np.cos(2 * np.pi * (
+            x - shift))) + 10 * np.size(x) + shift
+        self.nb_fun_call += 1
+        return y
+
+    def rosen_der_wrapper(self, x, args=()):
+        self.ngev += 1
+        return rosen_der(x, *args)
+
+    def test_visiting_stepping(self):
+        lu = list(zip(*self.ld_bounds))
+        lower = np.array(lu[0])
+        upper = np.array(lu[1])
+        dim = lower.size
+        vd = VisitingDistribution(lower, upper, self.qv, self.rs)
+        values = np.zeros(dim)
+        x_step_low = vd.visiting(values, 0, self.high_temperature)
+        # Make sure that only the first component is changed
+        assert_equal(np.not_equal(x_step_low, 0), True)
+        values = np.zeros(dim)
+        x_step_high = vd.visiting(values, dim, self.high_temperature)
+        # Make sure that component other than at dim has changed
+        assert_equal(np.not_equal(x_step_high[0], 0), True)
+
+    def test_visiting_dist_high_temperature(self):
+        lu = list(zip(*self.ld_bounds))
+        lower = np.array(lu[0])
+        upper = np.array(lu[1])
+        vd = VisitingDistribution(lower, upper, self.qv, self.rs)
+        # values = np.zeros(self.nbtestvalues)
+        # for i in np.arange(self.nbtestvalues):
+        #     values[i] = vd.visit_fn(self.high_temperature)
+        values = vd.visit_fn(self.high_temperature, self.nbtestvalues)
+
+        # Visiting distribution is a distorted version of Cauchy-Lorentz
+        # distribution, and as no 1st and higher moments (no mean defined,
+        # no variance defined).
+        # Check that big tails values are generated
+        assert_array_less(np.min(values), 1e-10)
+        assert_array_less(1e+10, np.max(values))
+
+    def test_reset(self):
+        owf = ObjectiveFunWrapper(self.weirdfunc)
+        lu = list(zip(*self.ld_bounds))
+        lower = np.array(lu[0])
+        upper = np.array(lu[1])
+        es = EnergyState(lower, upper)
+        assert_raises(ValueError, es.reset, owf, check_random_state(None))
+
+    def test_low_dim(self):
+        ret = dual_annealing(
+            self.func, self.ld_bounds, seed=self.seed)
+        assert_allclose(ret.fun, 0., atol=1e-12)
+        assert ret.success
+
+    def test_high_dim(self):
+        ret = dual_annealing(self.func, self.hd_bounds, seed=self.seed)
+        assert_allclose(ret.fun, 0., atol=1e-12)
+        assert ret.success
+
+    def test_low_dim_no_ls(self):
+        ret = dual_annealing(self.func, self.ld_bounds,
+                             no_local_search=True, seed=self.seed)
+        assert_allclose(ret.fun, 0., atol=1e-4)
+
+    def test_high_dim_no_ls(self):
+        ret = dual_annealing(self.func, self.hd_bounds,
+                             no_local_search=True, seed=self.seed)
+        assert_allclose(ret.fun, 0., atol=1e-4)
+
+    def test_nb_fun_call(self):
+        ret = dual_annealing(self.func, self.ld_bounds, seed=self.seed)
+        assert_equal(self.nb_fun_call, ret.nfev)
+
+    def test_nb_fun_call_no_ls(self):
+        ret = dual_annealing(self.func, self.ld_bounds,
+                             no_local_search=True, seed=self.seed)
+        assert_equal(self.nb_fun_call, ret.nfev)
+
+    def test_max_reinit(self):
+        assert_raises(ValueError, dual_annealing, self.weirdfunc,
+                      self.ld_bounds)
+
+    def test_reproduce(self):
+        res1 = dual_annealing(self.func, self.ld_bounds, seed=self.seed)
+        res2 = dual_annealing(self.func, self.ld_bounds, seed=self.seed)
+        res3 = dual_annealing(self.func, self.ld_bounds, seed=self.seed)
+        # If we have reproducible results, x components found has to
+        # be exactly the same, which is not the case with no seeding
+        assert_equal(res1.x, res2.x)
+        assert_equal(res1.x, res3.x)
+
+    @pytest.mark.skipif(Version(np.__version__) < Version('1.17'),
+                        reason='Generator not available for numpy, < 1.17')
+    def test_rand_gen(self):
+        # check that np.random.Generator can be used (numpy >= 1.17)
+        # obtain a np.random.Generator object
+        rng = np.random.default_rng(1)
+
+        res1 = dual_annealing(self.func, self.ld_bounds, seed=rng)
+        # seed again
+        rng = np.random.default_rng(1)
+        res2 = dual_annealing(self.func, self.ld_bounds, seed=rng)
+        # If we have reproducible results, x components found has to
+        # be exactly the same, which is not the case with no seeding
+        assert_equal(res1.x, res2.x)
+
+    def test_bounds_integrity(self):
+        wrong_bounds = [(-5.12, 5.12), (1, 0), (5.12, 5.12)]
+        assert_raises(ValueError, dual_annealing, self.func,
+                      wrong_bounds)
+
+    def test_bound_validity(self):
+        invalid_bounds = [(-5, 5), (-np.inf, 0), (-5, 5)]
+        assert_raises(ValueError, dual_annealing, self.func,
+                      invalid_bounds)
+        invalid_bounds = [(-5, 5), (0, np.inf), (-5, 5)]
+        assert_raises(ValueError, dual_annealing, self.func,
+                      invalid_bounds)
+        invalid_bounds = [(-5, 5), (0, np.nan), (-5, 5)]
+        assert_raises(ValueError, dual_annealing, self.func,
+                      invalid_bounds)
+
+    def test_max_fun_ls(self):
+        ret = dual_annealing(self.func, self.ld_bounds, maxfun=100,
+                             seed=self.seed)
+
+        ls_max_iter = min(max(
+            len(self.ld_bounds) * LocalSearchWrapper.LS_MAXITER_RATIO,
+            LocalSearchWrapper.LS_MAXITER_MIN),
+            LocalSearchWrapper.LS_MAXITER_MAX)
+        assert ret.nfev <= 100 + ls_max_iter
+        assert not ret.success
+
+    def test_max_fun_no_ls(self):
+        ret = dual_annealing(self.func, self.ld_bounds,
+                             no_local_search=True, maxfun=500, seed=self.seed)
+        assert ret.nfev <= 500
+        assert not ret.success
+
+    def test_maxiter(self):
+        ret = dual_annealing(self.func, self.ld_bounds, maxiter=700,
+                             seed=self.seed)
+        assert ret.nit <= 700
+
+    # Testing that args are passed correctly for dual_annealing
+    def test_fun_args_ls(self):
+        ret = dual_annealing(self.func, self.ld_bounds,
+                             args=((3.14159,)), seed=self.seed)
+        assert_allclose(ret.fun, 3.14159, atol=1e-6)
+
+    # Testing that args are passed correctly for pure simulated annealing
+    def test_fun_args_no_ls(self):
+        ret = dual_annealing(self.func, self.ld_bounds,
+                             args=((3.14159, )), no_local_search=True,
+                             seed=self.seed)
+        assert_allclose(ret.fun, 3.14159, atol=1e-4)
+
+    def test_callback_stop(self):
+        # Testing that callback make the algorithm stop for
+        # fun value <= 1.0 (see callback method)
+        ret = dual_annealing(self.func, self.ld_bounds,
+                             callback=self.callback, seed=self.seed)
+        assert ret.fun <= 1.0
+        assert 'stop early' in ret.message[0]
+        assert not ret.success
+
+    @pytest.mark.parametrize('method, atol', [
+        ('Nelder-Mead', 2e-5),
+        ('COBYLA', 1e-5),
+        ('Powell', 1e-8),
+        ('CG', 1e-8),
+        ('BFGS', 1e-8),
+        ('TNC', 1e-8),
+        ('SLSQP', 2e-7),
+    ])
+    def test_multi_ls_minimizer(self, method, atol):
+        ret = dual_annealing(self.func, self.ld_bounds,
+                             local_search_options=dict(method=method),
+                             seed=self.seed)
+        assert_allclose(ret.fun, 0., atol=atol)
+
+    def test_wrong_restart_temp(self):
+        assert_raises(ValueError, dual_annealing, self.func,
+                      self.ld_bounds, restart_temp_ratio=1)
+        assert_raises(ValueError, dual_annealing, self.func,
+                      self.ld_bounds, restart_temp_ratio=0)
+
+    def test_gradient_gnev(self):
+        minimizer_opts = {
+            'jac': self.rosen_der_wrapper,
+        }
+        ret = dual_annealing(rosen, self.ld_bounds,
+                             local_search_options=minimizer_opts,
+                             seed=self.seed)
+        assert ret.njev == self.ngev
+
+    def test_from_docstring(self):
+        func = lambda x: np.sum(x * x - 10 * np.cos(2 * np.pi * x)) + 10 * np.size(x)
+        lw = [-5.12] * 10
+        up = [5.12] * 10
+        ret = dual_annealing(func, bounds=list(zip(lw, up)), seed=1234)
+        assert_allclose(ret.x,
+                        [-4.26437714e-09, -3.91699361e-09, -1.86149218e-09,
+                         -3.97165720e-09, -6.29151648e-09, -6.53145322e-09,
+                         -3.93616815e-09, -6.55623025e-09, -6.05775280e-09,
+                         -5.00668935e-09], atol=4e-8)
+        assert_allclose(ret.fun, 0.000000, atol=5e-13)

venv/Lib/site-packages/scipy/optimize/tests/test__linprog_clean_inputs.py

@@ -0,0 +1,297 @@
+"""
+Unit test for Linear Programming via Simplex Algorithm.
+"""
+import numpy as np
+from numpy.testing import assert_, assert_allclose, assert_equal
+from pytest import raises as assert_raises
+from scipy.optimize._linprog_util import _clean_inputs, _LPProblem
+from copy import deepcopy
+from datetime import date
+
+
+def test_aliasing():
+    """
+    Test for ensuring that no objects referred to by `lp` attributes,
+    `c`, `A_ub`, `b_ub`, `A_eq`, `b_eq`, `bounds`, have been modified
+    by `_clean_inputs` as a side effect.
+    """
+    lp = _LPProblem(
+        c=1,
+        A_ub=[[1]],
+        b_ub=[1],
+        A_eq=[[1]],
+        b_eq=[1],
+        bounds=(-np.inf, np.inf)
+    )
+    lp_copy = deepcopy(lp)
+
+    _clean_inputs(lp)
+
+    assert_(lp.c == lp_copy.c, "c modified by _clean_inputs")
+    assert_(lp.A_ub == lp_copy.A_ub, "A_ub modified by _clean_inputs")
+    assert_(lp.b_ub == lp_copy.b_ub, "b_ub modified by _clean_inputs")
+    assert_(lp.A_eq == lp_copy.A_eq, "A_eq modified by _clean_inputs")
+    assert_(lp.b_eq == lp_copy.b_eq, "b_eq modified by _clean_inputs")
+    assert_(lp.bounds == lp_copy.bounds, "bounds modified by _clean_inputs")
+
+
+def test_aliasing2():
+    """
+    Similar purpose as `test_aliasing` above.
+    """
+    lp = _LPProblem(
+        c=np.array([1, 1]),
+        A_ub=np.array([[1, 1], [2, 2]]),
+        b_ub=np.array([[1], [1]]),
+        A_eq=np.array([[1, 1]]),
+        b_eq=np.array([1]),
+        bounds=[(-np.inf, np.inf), (None, 1)]
+    )
+    lp_copy = deepcopy(lp)
+
+    _clean_inputs(lp)
+
+    assert_allclose(lp.c, lp_copy.c, err_msg="c modified by _clean_inputs")
+    assert_allclose(lp.A_ub, lp_copy.A_ub, err_msg="A_ub modified by _clean_inputs")
+    assert_allclose(lp.b_ub, lp_copy.b_ub, err_msg="b_ub modified by _clean_inputs")
+    assert_allclose(lp.A_eq, lp_copy.A_eq, err_msg="A_eq modified by _clean_inputs")
+    assert_allclose(lp.b_eq, lp_copy.b_eq, err_msg="b_eq modified by _clean_inputs")
+    assert_(lp.bounds == lp_copy.bounds, "bounds modified by _clean_inputs")
+
+
+def test_missing_inputs():
+    c = [1, 2]
+    A_ub = np.array([[1, 1], [2, 2]])
+    b_ub = np.array([1, 1])
+    A_eq = np.array([[1, 1], [2, 2]])
+    b_eq = np.array([1, 1])
+
+    assert_raises(TypeError, _clean_inputs)
+    assert_raises(TypeError, _clean_inputs, _LPProblem(c=None))
+    assert_raises(ValueError, _clean_inputs, _LPProblem(c=c, A_ub=A_ub))
+    assert_raises(ValueError, _clean_inputs, _LPProblem(c=c, A_ub=A_ub, b_ub=None))
+    assert_raises(ValueError, _clean_inputs, _LPProblem(c=c, b_ub=b_ub))
+    assert_raises(ValueError, _clean_inputs, _LPProblem(c=c, A_ub=None, b_ub=b_ub))
+    assert_raises(ValueError, _clean_inputs, _LPProblem(c=c, A_eq=A_eq))
+    assert_raises(ValueError, _clean_inputs, _LPProblem(c=c, A_eq=A_eq, b_eq=None))
+    assert_raises(ValueError, _clean_inputs, _LPProblem(c=c, b_eq=b_eq))
+    assert_raises(ValueError, _clean_inputs, _LPProblem(c=c, A_eq=None, b_eq=b_eq))
+
+
+def test_too_many_dimensions():
+    cb = [1, 2, 3, 4]
+    A = np.random.rand(4, 4)
+    bad2D = [[1, 2], [3, 4]]
+    bad3D = np.random.rand(4, 4, 4)
+    assert_raises(ValueError, _clean_inputs, _LPProblem(c=bad2D, A_ub=A, b_ub=cb))
+    assert_raises(ValueError, _clean_inputs, _LPProblem(c=cb, A_ub=bad3D, b_ub=cb))
+    assert_raises(ValueError, _clean_inputs, _LPProblem(c=cb, A_ub=A, b_ub=bad2D))
+    assert_raises(ValueError, _clean_inputs, _LPProblem(c=cb, A_eq=bad3D, b_eq=cb))
+    assert_raises(ValueError, _clean_inputs, _LPProblem(c=cb, A_eq=A, b_eq=bad2D))
+
+
+def test_too_few_dimensions():
+    bad = np.random.rand(4, 4).ravel()
+    cb = np.random.rand(4)
+    assert_raises(ValueError, _clean_inputs, _LPProblem(c=cb, A_ub=bad, b_ub=cb))
+    assert_raises(ValueError, _clean_inputs, _LPProblem(c=cb, A_eq=bad, b_eq=cb))
+
+
+def test_inconsistent_dimensions():
+    m = 2
+    n = 4
+    c = [1, 2, 3, 4]
+
+    Agood = np.random.rand(m, n)
+    Abad = np.random.rand(m, n + 1)
+    bgood = np.random.rand(m)
+    bbad = np.random.rand(m + 1)
+    boundsbad = [(0, 1)] * (n + 1)
+    assert_raises(ValueError, _clean_inputs, _LPProblem(c=c, A_ub=Abad, b_ub=bgood))
+    assert_raises(ValueError, _clean_inputs, _LPProblem(c=c, A_ub=Agood, b_ub=bbad))
+    assert_raises(ValueError, _clean_inputs, _LPProblem(c=c, A_eq=Abad, b_eq=bgood))
+    assert_raises(ValueError, _clean_inputs, _LPProblem(c=c, A_eq=Agood, b_eq=bbad))
+    assert_raises(ValueError, _clean_inputs, _LPProblem(c=c, bounds=boundsbad))
+    assert_raises(ValueError, _clean_inputs, _LPProblem(c=c, bounds=[[1, 2], [2, 3], [3, 4], [4, 5, 6]]))
+
+
+def test_type_errors():
+    lp = _LPProblem(
+        c=[1, 2],
+        A_ub=np.array([[1, 1], [2, 2]]),
+        b_ub=np.array([1, 1]),
+        A_eq=np.array([[1, 1], [2, 2]]),
+        b_eq=np.array([1, 1]),
+        bounds=[(0, 1)]
+    )
+    bad = "hello"
+
+    assert_raises(TypeError, _clean_inputs, lp._replace(c=bad))
+    assert_raises(TypeError, _clean_inputs, lp._replace(A_ub=bad))
+    assert_raises(TypeError, _clean_inputs, lp._replace(b_ub=bad))
+    assert_raises(TypeError, _clean_inputs, lp._replace(A_eq=bad))
+    assert_raises(TypeError, _clean_inputs, lp._replace(b_eq=bad))
+
+    assert_raises(ValueError, _clean_inputs, lp._replace(bounds=bad))
+    assert_raises(ValueError, _clean_inputs, lp._replace(bounds="hi"))
+    assert_raises(ValueError, _clean_inputs, lp._replace(bounds=["hi"]))
+    assert_raises(ValueError, _clean_inputs, lp._replace(bounds=[("hi")]))
+    assert_raises(ValueError, _clean_inputs, lp._replace(bounds=[(1, "")]))
+    assert_raises(ValueError, _clean_inputs, lp._replace(bounds=[(1, 2), (1, "")]))
+    assert_raises(TypeError, _clean_inputs, lp._replace(bounds=[(1, date(2020, 2, 29))]))
+    assert_raises(ValueError, _clean_inputs, lp._replace(bounds=[[[1, 2]]]))
+
+
+def test_non_finite_errors():
+    lp = _LPProblem(
+        c=[1, 2],
+        A_ub=np.array([[1, 1], [2, 2]]),
+        b_ub=np.array([1, 1]),
+        A_eq=np.array([[1, 1], [2, 2]]),
+        b_eq=np.array([1, 1]),
+        bounds=[(0, 1)]
+    )
+    assert_raises(ValueError, _clean_inputs, lp._replace(c=[0, None]))
+    assert_raises(ValueError, _clean_inputs, lp._replace(c=[np.inf, 0]))
+    assert_raises(ValueError, _clean_inputs, lp._replace(c=[0, -np.inf]))
+    assert_raises(ValueError, _clean_inputs, lp._replace(c=[np.nan, 0]))
+
+    assert_raises(ValueError, _clean_inputs, lp._replace(A_ub=[[1, 2], [None, 1]]))
+    assert_raises(ValueError, _clean_inputs, lp._replace(b_ub=[np.inf, 1]))
+    assert_raises(ValueError, _clean_inputs, lp._replace(A_eq=[[1, 2], [1, -np.inf]]))
+    assert_raises(ValueError, _clean_inputs, lp._replace(b_eq=[1, np.nan]))
+
+
+def test__clean_inputs1():
+    lp = _LPProblem(
+        c=[1, 2],
+        A_ub=[[1, 1], [2, 2]],
+        b_ub=[1, 1],
+        A_eq=[[1, 1], [2, 2]],
+        b_eq=[1, 1],
+        bounds=None
+    )
+
+    lp_cleaned = _clean_inputs(lp)
+
+    assert_allclose(lp_cleaned.c, np.array(lp.c))
+    assert_allclose(lp_cleaned.A_ub, np.array(lp.A_ub))
+    assert_allclose(lp_cleaned.b_ub, np.array(lp.b_ub))
+    assert_allclose(lp_cleaned.A_eq, np.array(lp.A_eq))
+    assert_allclose(lp_cleaned.b_eq, np.array(lp.b_eq))
+    assert_equal(lp_cleaned.bounds, [(0, np.inf)] * 2)
+
+    assert_(lp_cleaned.c.shape == (2,), "")
+    assert_(lp_cleaned.A_ub.shape == (2, 2), "")
+    assert_(lp_cleaned.b_ub.shape == (2,), "")
+    assert_(lp_cleaned.A_eq.shape == (2, 2), "")
+    assert_(lp_cleaned.b_eq.shape == (2,), "")
+
+
+def test__clean_inputs2():
+    lp = _LPProblem(
+        c=1,
+        A_ub=[[1]],
+        b_ub=1,
+        A_eq=[[1]],
+        b_eq=1,
+        bounds=(0, 1)
+    )
+
+    lp_cleaned = _clean_inputs(lp)
+
+    assert_allclose(lp_cleaned.c, np.array(lp.c))
+    assert_allclose(lp_cleaned.A_ub, np.array(lp.A_ub))
+    assert_allclose(lp_cleaned.b_ub, np.array(lp.b_ub))
+    assert_allclose(lp_cleaned.A_eq, np.array(lp.A_eq))
+    assert_allclose(lp_cleaned.b_eq, np.array(lp.b_eq))
+    assert_equal(lp_cleaned.bounds, [(0, 1)])
+
+    assert_(lp_cleaned.c.shape == (1,), "")
+    assert_(lp_cleaned.A_ub.shape == (1, 1), "")
+    assert_(lp_cleaned.b_ub.shape == (1,), "")
+    assert_(lp_cleaned.A_eq.shape == (1, 1), "")
+    assert_(lp_cleaned.b_eq.shape == (1,), "")
+
+
+def test__clean_inputs3():
+    lp = _LPProblem(
+        c=[[1, 2]],
+        A_ub=np.random.rand(2, 2),
+        b_ub=[[1], [2]],
+        A_eq=np.random.rand(2, 2),
+        b_eq=[[1], [2]],
+        bounds=[(0, 1)]
+    )
+
+    lp_cleaned = _clean_inputs(lp)
+
+    assert_allclose(lp_cleaned.c, np.array([1, 2]))
+    assert_allclose(lp_cleaned.b_ub, np.array([1, 2]))
+    assert_allclose(lp_cleaned.b_eq, np.array([1, 2]))
+    assert_equal(lp_cleaned.bounds, [(0, 1)] * 2)
+
+    assert_(lp_cleaned.c.shape == (2,), "")
+    assert_(lp_cleaned.b_ub.shape == (2,), "")
+    assert_(lp_cleaned.b_eq.shape == (2,), "")
+
+
+def test_bad_bounds():
+    lp = _LPProblem(c=[1, 2])
+
+    assert_raises(ValueError, _clean_inputs, lp._replace(bounds=(1, 2, 2)))
+    assert_raises(ValueError, _clean_inputs, lp._replace(bounds=[(1, 2, 2)]))
+    assert_raises(ValueError, _clean_inputs, lp._replace(bounds=[(1, 2), (1, 2, 2)]))
+    assert_raises(ValueError, _clean_inputs, lp._replace(bounds=[(1, 2), (1, 2), (1, 2)]))
+
+    lp = _LPProblem(c=[1, 2, 3, 4])
+
+    assert_raises(ValueError, _clean_inputs, lp._replace(bounds=[(1, 2, 3, 4), (1, 2, 3, 4)]))
+
+
+def test_good_bounds():
+    lp = _LPProblem(c=[1, 2])
+
+    lp_cleaned = _clean_inputs(lp)  # lp.bounds is None by default
+    assert_equal(lp_cleaned.bounds, [(0, np.inf)] * 2)
+
+    lp_cleaned = _clean_inputs(lp._replace(bounds=[]))
+    assert_equal(lp_cleaned.bounds, [(0, np.inf)] * 2)
+
+    lp_cleaned = _clean_inputs(lp._replace(bounds=[[]]))
+    assert_equal(lp_cleaned.bounds, [(0, np.inf)] * 2)
+
+    lp_cleaned = _clean_inputs(lp._replace(bounds=(1, 2)))
+    assert_equal(lp_cleaned.bounds, [(1, 2)] * 2)
+
+    lp_cleaned = _clean_inputs(lp._replace(bounds=[(1, 2)]))
+    assert_equal(lp_cleaned.bounds, [(1, 2)] * 2)
+
+    lp_cleaned = _clean_inputs(lp._replace(bounds=[(1, None)]))
+    assert_equal(lp_cleaned.bounds, [(1, np.inf)] * 2)
+
+    lp_cleaned = _clean_inputs(lp._replace(bounds=[(None, 1)]))
+    assert_equal(lp_cleaned.bounds, [(-np.inf, 1)] * 2)
+
+    lp_cleaned = _clean_inputs(lp._replace(bounds=[(None, None), (-np.inf, None)]))
+    assert_equal(lp_cleaned.bounds, [(-np.inf, np.inf)] * 2)
+
+    lp = _LPProblem(c=[1, 2, 3, 4])
+
+    lp_cleaned = _clean_inputs(lp)  # lp.bounds is None by default
+    assert_equal(lp_cleaned.bounds, [(0, np.inf)] * 4)
+
+    lp_cleaned = _clean_inputs(lp._replace(bounds=(1, 2)))
+    assert_equal(lp_cleaned.bounds, [(1, 2)] * 4)
+
+    lp_cleaned = _clean_inputs(lp._replace(bounds=[(1, 2)]))
+    assert_equal(lp_cleaned.bounds, [(1, 2)] * 4)
+
+    lp_cleaned = _clean_inputs(lp._replace(bounds=[(1, None)]))
+    assert_equal(lp_cleaned.bounds, [(1, np.inf)] * 4)
+
+    lp_cleaned = _clean_inputs(lp._replace(bounds=[(None, 1)]))
+    assert_equal(lp_cleaned.bounds, [(-np.inf, 1)] * 4)
+
+    lp_cleaned = _clean_inputs(lp._replace(bounds=[(None, None), (-np.inf, None), (None, np.inf), (-np.inf, np.inf)]))
+    assert_equal(lp_cleaned.bounds, [(-np.inf, np.inf)] * 4)

venv/Lib/site-packages/scipy/optimize/tests/test__numdiff.py

@@ -0,0 +1,677 @@
+import math
+from itertools import product
+
+import numpy as np
+from numpy.testing import assert_allclose, assert_equal, assert_
+from pytest import raises as assert_raises
+
+from scipy.sparse import csr_matrix, csc_matrix, lil_matrix
+
+from scipy.optimize._numdiff import (
+    _adjust_scheme_to_bounds, approx_derivative, check_derivative,
+    group_columns)
+
+
+def test_group_columns():
+    structure = [
+        [1, 1, 0, 0, 0, 0],
+        [1, 1, 1, 0, 0, 0],
+        [0, 1, 1, 1, 0, 0],
+        [0, 0, 1, 1, 1, 0],
+        [0, 0, 0, 1, 1, 1],
+        [0, 0, 0, 0, 1, 1],
+        [0, 0, 0, 0, 0, 0]
+    ]
+    for transform in [np.asarray, csr_matrix, csc_matrix, lil_matrix]:
+        A = transform(structure)
+        order = np.arange(6)
+        groups_true = np.array([0, 1, 2, 0, 1, 2])
+        groups = group_columns(A, order)
+        assert_equal(groups, groups_true)
+
+        order = [1, 2, 4, 3, 5, 0]
+        groups_true = np.array([2, 0, 1, 2, 0, 1])
+        groups = group_columns(A, order)
+        assert_equal(groups, groups_true)
+
+    # Test repeatability.
+    groups_1 = group_columns(A)
+    groups_2 = group_columns(A)
+    assert_equal(groups_1, groups_2)
+
+
+class TestAdjustSchemeToBounds(object):
+    def test_no_bounds(self):
+        x0 = np.zeros(3)
+        h = np.full(3, 1e-2)
+        inf_lower = np.empty_like(x0)
+        inf_upper = np.empty_like(x0)
+        inf_lower.fill(-np.inf)
+        inf_upper.fill(np.inf)
+
+        h_adjusted, one_sided = _adjust_scheme_to_bounds(
+            x0, h, 1, '1-sided', inf_lower, inf_upper)
+        assert_allclose(h_adjusted, h)
+        assert_(np.all(one_sided))
+
+        h_adjusted, one_sided = _adjust_scheme_to_bounds(
+            x0, h, 2, '1-sided', inf_lower, inf_upper)
+        assert_allclose(h_adjusted, h)
+        assert_(np.all(one_sided))
+
+        h_adjusted, one_sided = _adjust_scheme_to_bounds(
+            x0, h, 1, '2-sided', inf_lower, inf_upper)
+        assert_allclose(h_adjusted, h)
+        assert_(np.all(~one_sided))
+
+        h_adjusted, one_sided = _adjust_scheme_to_bounds(
+            x0, h, 2, '2-sided', inf_lower, inf_upper)
+        assert_allclose(h_adjusted, h)
+        assert_(np.all(~one_sided))
+
+    def test_with_bound(self):
+        x0 = np.array([0.0, 0.85, -0.85])
+        lb = -np.ones(3)
+        ub = np.ones(3)
+        h = np.array([1, 1, -1]) * 1e-1
+
+        h_adjusted, _ = _adjust_scheme_to_bounds(x0, h, 1, '1-sided', lb, ub)
+        assert_allclose(h_adjusted, h)
+
+        h_adjusted, _ = _adjust_scheme_to_bounds(x0, h, 2, '1-sided', lb, ub)
+        assert_allclose(h_adjusted, np.array([1, -1, 1]) * 1e-1)
+
+        h_adjusted, one_sided = _adjust_scheme_to_bounds(
+            x0, h, 1, '2-sided', lb, ub)
+        assert_allclose(h_adjusted, np.abs(h))
+        assert_(np.all(~one_sided))
+
+        h_adjusted, one_sided = _adjust_scheme_to_bounds(
+            x0, h, 2, '2-sided', lb, ub)
+        assert_allclose(h_adjusted, np.array([1, -1, 1]) * 1e-1)
+        assert_equal(one_sided, np.array([False, True, True]))
+
+    def test_tight_bounds(self):
+        lb = np.array([-0.03, -0.03])
+        ub = np.array([0.05, 0.05])
+        x0 = np.array([0.0, 0.03])
+        h = np.array([-0.1, -0.1])
+
+        h_adjusted, _ = _adjust_scheme_to_bounds(x0, h, 1, '1-sided', lb, ub)
+        assert_allclose(h_adjusted, np.array([0.05, -0.06]))
+
+        h_adjusted, _ = _adjust_scheme_to_bounds(x0, h, 2, '1-sided', lb, ub)
+        assert_allclose(h_adjusted, np.array([0.025, -0.03]))
+
+        h_adjusted, one_sided = _adjust_scheme_to_bounds(
+            x0, h, 1, '2-sided', lb, ub)
+        assert_allclose(h_adjusted, np.array([0.03, -0.03]))
+        assert_equal(one_sided, np.array([False, True]))
+
+        h_adjusted, one_sided = _adjust_scheme_to_bounds(
+            x0, h, 2, '2-sided', lb, ub)
+        assert_allclose(h_adjusted, np.array([0.015, -0.015]))
+        assert_equal(one_sided, np.array([False, True]))
+
+
+class TestApproxDerivativesDense(object):
+    def fun_scalar_scalar(self, x):
+        return np.sinh(x)
+
+    def jac_scalar_scalar(self, x):
+        return np.cosh(x)
+
+    def fun_scalar_vector(self, x):
+        return np.array([x[0]**2, np.tan(x[0]), np.exp(x[0])])
+
+    def jac_scalar_vector(self, x):
+        return np.array(
+            [2 * x[0], np.cos(x[0]) ** -2, np.exp(x[0])]).reshape(-1, 1)
+
+    def fun_vector_scalar(self, x):
+        return np.sin(x[0] * x[1]) * np.log(x[0])
+
+    def wrong_dimensions_fun(self, x):
+        return np.array([x**2, np.tan(x), np.exp(x)])
+
+    def jac_vector_scalar(self, x):
+        return np.array([
+            x[1] * np.cos(x[0] * x[1]) * np.log(x[0]) +
+            np.sin(x[0] * x[1]) / x[0],
+            x[0] * np.cos(x[0] * x[1]) * np.log(x[0])
+        ])
+
+    def fun_vector_vector(self, x):
+        return np.array([
+            x[0] * np.sin(x[1]),
+            x[1] * np.cos(x[0]),
+            x[0] ** 3 * x[1] ** -0.5
+        ])
+
+    def jac_vector_vector(self, x):
+        return np.array([
+            [np.sin(x[1]), x[0] * np.cos(x[1])],
+            [-x[1] * np.sin(x[0]), np.cos(x[0])],
+            [3 * x[0] ** 2 * x[1] ** -0.5, -0.5 * x[0] ** 3 * x[1] ** -1.5]
+        ])
+
+    def fun_parametrized(self, x, c0, c1=1.0):
+        return np.array([np.exp(c0 * x[0]), np.exp(c1 * x[1])])
+
+    def jac_parametrized(self, x, c0, c1=0.1):
+        return np.array([
+            [c0 * np.exp(c0 * x[0]), 0],
+            [0, c1 * np.exp(c1 * x[1])]
+        ])
+
+    def fun_with_nan(self, x):
+        return x if np.abs(x) <= 1e-8 else np.nan
+
+    def jac_with_nan(self, x):
+        return 1.0 if np.abs(x) <= 1e-8 else np.nan
+
+    def fun_zero_jacobian(self, x):
+        return np.array([x[0] * x[1], np.cos(x[0] * x[1])])
+
+    def jac_zero_jacobian(self, x):
+        return np.array([
+            [x[1], x[0]],
+            [-x[1] * np.sin(x[0] * x[1]), -x[0] * np.sin(x[0] * x[1])]
+        ])
+
+    def fun_non_numpy(self, x):
+        return math.exp(x)
+
+    def jac_non_numpy(self, x):
+        return math.exp(x)
+
+    def test_scalar_scalar(self):
+        x0 = 1.0
+        jac_diff_2 = approx_derivative(self.fun_scalar_scalar, x0,
+                                       method='2-point')
+        jac_diff_3 = approx_derivative(self.fun_scalar_scalar, x0)
+        jac_diff_4 = approx_derivative(self.fun_scalar_scalar, x0,
+                                       method='cs')
+        jac_true = self.jac_scalar_scalar(x0)
+        assert_allclose(jac_diff_2, jac_true, rtol=1e-6)
+        assert_allclose(jac_diff_3, jac_true, rtol=1e-9)
+        assert_allclose(jac_diff_4, jac_true, rtol=1e-12)
+
+    def test_scalar_scalar_abs_step(self):
+        # can approx_derivative use abs_step?
+        x0 = 1.0
+        jac_diff_2 = approx_derivative(self.fun_scalar_scalar, x0,
+                                       method='2-point', abs_step=1.49e-8)
+        jac_diff_3 = approx_derivative(self.fun_scalar_scalar, x0,
+                                       abs_step=1.49e-8)
+        jac_diff_4 = approx_derivative(self.fun_scalar_scalar, x0,
+                                       method='cs', abs_step=1.49e-8)
+        jac_true = self.jac_scalar_scalar(x0)
+        assert_allclose(jac_diff_2, jac_true, rtol=1e-6)
+        assert_allclose(jac_diff_3, jac_true, rtol=1e-9)
+        assert_allclose(jac_diff_4, jac_true, rtol=1e-12)
+
+    def test_scalar_vector(self):
+        x0 = 0.5
+        jac_diff_2 = approx_derivative(self.fun_scalar_vector, x0,
+                                       method='2-point')
+        jac_diff_3 = approx_derivative(self.fun_scalar_vector, x0)
+        jac_diff_4 = approx_derivative(self.fun_scalar_vector, x0,
+                                       method='cs')
+        jac_true = self.jac_scalar_vector(np.atleast_1d(x0))
+        assert_allclose(jac_diff_2, jac_true, rtol=1e-6)
+        assert_allclose(jac_diff_3, jac_true, rtol=1e-9)
+        assert_allclose(jac_diff_4, jac_true, rtol=1e-12)
+
+    def test_vector_scalar(self):
+        x0 = np.array([100.0, -0.5])
+        jac_diff_2 = approx_derivative(self.fun_vector_scalar, x0,
+                                       method='2-point')
+        jac_diff_3 = approx_derivative(self.fun_vector_scalar, x0)
+        jac_diff_4 = approx_derivative(self.fun_vector_scalar, x0,
+                                       method='cs')
+        jac_true = self.jac_vector_scalar(x0)
+        assert_allclose(jac_diff_2, jac_true, rtol=1e-6)
+        assert_allclose(jac_diff_3, jac_true, rtol=1e-7)
+        assert_allclose(jac_diff_4, jac_true, rtol=1e-12)
+
+    def test_vector_scalar_abs_step(self):
+        # can approx_derivative use abs_step?
+        x0 = np.array([100.0, -0.5])
+        jac_diff_2 = approx_derivative(self.fun_vector_scalar, x0,
+                                       method='2-point', abs_step=1.49e-8)
+        jac_diff_3 = approx_derivative(self.fun_vector_scalar, x0,
+                                       abs_step=1.49e-8, rel_step=np.inf)
+        jac_diff_4 = approx_derivative(self.fun_vector_scalar, x0,
+                                       method='cs', abs_step=1.49e-8)
+        jac_true = self.jac_vector_scalar(x0)
+        assert_allclose(jac_diff_2, jac_true, rtol=1e-6)
+        assert_allclose(jac_diff_3, jac_true, rtol=3e-9)
+        assert_allclose(jac_diff_4, jac_true, rtol=1e-12)
+
+    def test_vector_vector(self):
+        x0 = np.array([-100.0, 0.2])
+        jac_diff_2 = approx_derivative(self.fun_vector_vector, x0,
+                                       method='2-point')
+        jac_diff_3 = approx_derivative(self.fun_vector_vector, x0)
+        jac_diff_4 = approx_derivative(self.fun_vector_vector, x0,
+                                       method='cs')
+        jac_true = self.jac_vector_vector(x0)
+        assert_allclose(jac_diff_2, jac_true, rtol=1e-5)
+        assert_allclose(jac_diff_3, jac_true, rtol=1e-6)
+        assert_allclose(jac_diff_4, jac_true, rtol=1e-12)
+
+    def test_wrong_dimensions(self):
+        x0 = 1.0
+        assert_raises(RuntimeError, approx_derivative,
+                      self.wrong_dimensions_fun, x0)
+        f0 = self.wrong_dimensions_fun(np.atleast_1d(x0))
+        assert_raises(ValueError, approx_derivative,
+                      self.wrong_dimensions_fun, x0, f0=f0)
+
+    def test_custom_rel_step(self):
+        x0 = np.array([-0.1, 0.1])
+        jac_diff_2 = approx_derivative(self.fun_vector_vector, x0,
+                                       method='2-point', rel_step=1e-4)
+        jac_diff_3 = approx_derivative(self.fun_vector_vector, x0,
+                                       rel_step=1e-4)
+        jac_true = self.jac_vector_vector(x0)
+        assert_allclose(jac_diff_2, jac_true, rtol=1e-2)
+        assert_allclose(jac_diff_3, jac_true, rtol=1e-4)
+
+    def test_options(self):
+        x0 = np.array([1.0, 1.0])
+        c0 = -1.0
+        c1 = 1.0
+        lb = 0.0
+        ub = 2.0
+        f0 = self.fun_parametrized(x0, c0, c1=c1)
+        rel_step = np.array([-1e-6, 1e-7])
+        jac_true = self.jac_parametrized(x0, c0, c1)
+        jac_diff_2 = approx_derivative(
+            self.fun_parametrized, x0, method='2-point', rel_step=rel_step,
+            f0=f0, args=(c0,), kwargs=dict(c1=c1), bounds=(lb, ub))
+        jac_diff_3 = approx_derivative(
+            self.fun_parametrized, x0, rel_step=rel_step,
+            f0=f0, args=(c0,), kwargs=dict(c1=c1), bounds=(lb, ub))
+        assert_allclose(jac_diff_2, jac_true, rtol=1e-6)
+        assert_allclose(jac_diff_3, jac_true, rtol=1e-9)
+
+    def test_with_bounds_2_point(self):
+        lb = -np.ones(2)
+        ub = np.ones(2)
+
+        x0 = np.array([-2.0, 0.2])
+        assert_raises(ValueError, approx_derivative,
+                      self.fun_vector_vector, x0, bounds=(lb, ub))
+
+        x0 = np.array([-1.0, 1.0])
+        jac_diff = approx_derivative(self.fun_vector_vector, x0,
+                                     method='2-point', bounds=(lb, ub))
+        jac_true = self.jac_vector_vector(x0)
+        assert_allclose(jac_diff, jac_true, rtol=1e-6)
+
+    def test_with_bounds_3_point(self):
+        lb = np.array([1.0, 1.0])
+        ub = np.array([2.0, 2.0])
+
+        x0 = np.array([1.0, 2.0])
+        jac_true = self.jac_vector_vector(x0)
+
+        jac_diff = approx_derivative(self.fun_vector_vector, x0)
+        assert_allclose(jac_diff, jac_true, rtol=1e-9)
+
+        jac_diff = approx_derivative(self.fun_vector_vector, x0,
+                                     bounds=(lb, np.inf))
+        assert_allclose(jac_diff, jac_true, rtol=1e-9)
+
+        jac_diff = approx_derivative(self.fun_vector_vector, x0,
+                                     bounds=(-np.inf, ub))
+        assert_allclose(jac_diff, jac_true, rtol=1e-9)
+
+        jac_diff = approx_derivative(self.fun_vector_vector, x0,
+                                     bounds=(lb, ub))
+        assert_allclose(jac_diff, jac_true, rtol=1e-9)
+
+    def test_tight_bounds(self):
+        x0 = np.array([10.0, 10.0])
+        lb = x0 - 3e-9
+        ub = x0 + 2e-9
+        jac_true = self.jac_vector_vector(x0)
+        jac_diff = approx_derivative(
+            self.fun_vector_vector, x0, method='2-point', bounds=(lb, ub))
+        assert_allclose(jac_diff, jac_true, rtol=1e-6)
+        jac_diff = approx_derivative(
+            self.fun_vector_vector, x0, method='2-point',
+            rel_step=1e-6, bounds=(lb, ub))
+        assert_allclose(jac_diff, jac_true, rtol=1e-6)
+
+        jac_diff = approx_derivative(
+            self.fun_vector_vector, x0, bounds=(lb, ub))
+        assert_allclose(jac_diff, jac_true, rtol=1e-6)
+        jac_diff = approx_derivative(
+            self.fun_vector_vector, x0, rel_step=1e-6, bounds=(lb, ub))
+        assert_allclose(jac_true, jac_diff, rtol=1e-6)
+
+    def test_bound_switches(self):
+        lb = -1e-8
+        ub = 1e-8
+        x0 = 0.0
+        jac_true = self.jac_with_nan(x0)
+        jac_diff_2 = approx_derivative(
+            self.fun_with_nan, x0, method='2-point', rel_step=1e-6,
+            bounds=(lb, ub))
+        jac_diff_3 = approx_derivative(
+            self.fun_with_nan, x0, rel_step=1e-6, bounds=(lb, ub))
+        assert_allclose(jac_diff_2, jac_true, rtol=1e-6)
+        assert_allclose(jac_diff_3, jac_true, rtol=1e-9)
+
+        x0 = 1e-8
+        jac_true = self.jac_with_nan(x0)
+        jac_diff_2 = approx_derivative(
+            self.fun_with_nan, x0, method='2-point', rel_step=1e-6,
+            bounds=(lb, ub))
+        jac_diff_3 = approx_derivative(
+            self.fun_with_nan, x0, rel_step=1e-6, bounds=(lb, ub))
+        assert_allclose(jac_diff_2, jac_true, rtol=1e-6)
+        assert_allclose(jac_diff_3, jac_true, rtol=1e-9)
+
+    def test_non_numpy(self):
+        x0 = 1.0
+        jac_true = self.jac_non_numpy(x0)
+        jac_diff_2 = approx_derivative(self.jac_non_numpy, x0,
+                                       method='2-point')
+        jac_diff_3 = approx_derivative(self.jac_non_numpy, x0)
+        assert_allclose(jac_diff_2, jac_true, rtol=1e-6)
+        assert_allclose(jac_diff_3, jac_true, rtol=1e-8)
+
+        # math.exp cannot handle complex arguments, hence this raises
+        assert_raises(TypeError, approx_derivative, self.jac_non_numpy, x0,
+                      **dict(method='cs'))
+
+    def test_check_derivative(self):
+        x0 = np.array([-10.0, 10])
+        accuracy = check_derivative(self.fun_vector_vector,
+                                    self.jac_vector_vector, x0)
+        assert_(accuracy < 1e-9)
+        accuracy = check_derivative(self.fun_vector_vector,
+                                    self.jac_vector_vector, x0)
+        assert_(accuracy < 1e-6)
+
+        x0 = np.array([0.0, 0.0])
+        accuracy = check_derivative(self.fun_zero_jacobian,
+                                    self.jac_zero_jacobian, x0)
+        assert_(accuracy == 0)
+        accuracy = check_derivative(self.fun_zero_jacobian,
+                                    self.jac_zero_jacobian, x0)
+        assert_(accuracy == 0)
+
+
+class TestApproxDerivativeSparse(object):
+    # Example from Numerical Optimization 2nd edition, p. 198.
+    def setup_method(self):
+        np.random.seed(0)
+        self.n = 50
+        self.lb = -0.1 * (1 + np.arange(self.n))
+        self.ub = 0.1 * (1 + np.arange(self.n))
+        self.x0 = np.empty(self.n)
+        self.x0[::2] = (1 - 1e-7) * self.lb[::2]
+        self.x0[1::2] = (1 - 1e-7) * self.ub[1::2]
+
+        self.J_true = self.jac(self.x0)
+
+    def fun(self, x):
+        e = x[1:]**3 - x[:-1]**2
+        return np.hstack((0, 3 * e)) + np.hstack((2 * e, 0))
+
+    def jac(self, x):
+        n = x.size
+        J = np.zeros((n, n))
+        J[0, 0] = -4 * x[0]
+        J[0, 1] = 6 * x[1]**2
+        for i in range(1, n - 1):
+            J[i, i - 1] = -6 * x[i-1]
+            J[i, i] = 9 * x[i]**2 - 4 * x[i]
+            J[i, i + 1] = 6 * x[i+1]**2
+        J[-1, -1] = 9 * x[-1]**2
+        J[-1, -2] = -6 * x[-2]
+
+        return J
+
+    def structure(self, n):
+        A = np.zeros((n, n), dtype=int)
+        A[0, 0] = 1
+        A[0, 1] = 1
+        for i in range(1, n - 1):
+            A[i, i - 1: i + 2] = 1
+        A[-1, -1] = 1
+        A[-1, -2] = 1
+
+        return A
+
+    def test_all(self):
+        A = self.structure(self.n)
+        order = np.arange(self.n)
+        groups_1 = group_columns(A, order)
+        np.random.shuffle(order)
+        groups_2 = group_columns(A, order)
+
+        for method, groups, l, u in product(
+                ['2-point', '3-point', 'cs'], [groups_1, groups_2],
+                [-np.inf, self.lb], [np.inf, self.ub]):
+            J = approx_derivative(self.fun, self.x0, method=method,
+                                  bounds=(l, u), sparsity=(A, groups))
+            assert_(isinstance(J, csr_matrix))
+            assert_allclose(J.toarray(), self.J_true, rtol=1e-6)
+
+            rel_step = np.full_like(self.x0, 1e-8)
+            rel_step[::2] *= -1
+            J = approx_derivative(self.fun, self.x0, method=method,
+                                  rel_step=rel_step, sparsity=(A, groups))
+            assert_allclose(J.toarray(), self.J_true, rtol=1e-5)
+
+    def test_no_precomputed_groups(self):
+        A = self.structure(self.n)
+        J = approx_derivative(self.fun, self.x0, sparsity=A)
+        assert_allclose(J.toarray(), self.J_true, rtol=1e-6)
+
+    def test_equivalence(self):
+        structure = np.ones((self.n, self.n), dtype=int)
+        groups = np.arange(self.n)
+        for method in ['2-point', '3-point', 'cs']:
+            J_dense = approx_derivative(self.fun, self.x0, method=method)
+            J_sparse = approx_derivative(
+                self.fun, self.x0, sparsity=(structure, groups), method=method)
+            assert_equal(J_dense, J_sparse.toarray())
+
+    def test_check_derivative(self):
+        def jac(x):
+            return csr_matrix(self.jac(x))
+
+        accuracy = check_derivative(self.fun, jac, self.x0,
+                                    bounds=(self.lb, self.ub))
+        assert_(accuracy < 1e-9)
+
+        accuracy = check_derivative(self.fun, jac, self.x0,
+                                    bounds=(self.lb, self.ub))
+        assert_(accuracy < 1e-9)
+
+
+class TestApproxDerivativeLinearOperator(object):
+
+    def fun_scalar_scalar(self, x):
+        return np.sinh(x)
+
+    def jac_scalar_scalar(self, x):
+        return np.cosh(x)
+
+    def fun_scalar_vector(self, x):
+        return np.array([x[0]**2, np.tan(x[0]), np.exp(x[0])])
+
+    def jac_scalar_vector(self, x):
+        return np.array(
+            [2 * x[0], np.cos(x[0]) ** -2, np.exp(x[0])]).reshape(-1, 1)
+
+    def fun_vector_scalar(self, x):
+        return np.sin(x[0] * x[1]) * np.log(x[0])
+
+    def jac_vector_scalar(self, x):
+        return np.array([
+            x[1] * np.cos(x[0] * x[1]) * np.log(x[0]) +
+            np.sin(x[0] * x[1]) / x[0],
+            x[0] * np.cos(x[0] * x[1]) * np.log(x[0])
+        ])
+
+    def fun_vector_vector(self, x):
+        return np.array([
+            x[0] * np.sin(x[1]),
+            x[1] * np.cos(x[0]),
+            x[0] ** 3 * x[1] ** -0.5
+        ])
+
+    def jac_vector_vector(self, x):
+        return np.array([
+            [np.sin(x[1]), x[0] * np.cos(x[1])],
+            [-x[1] * np.sin(x[0]), np.cos(x[0])],
+            [3 * x[0] ** 2 * x[1] ** -0.5, -0.5 * x[0] ** 3 * x[1] ** -1.5]
+        ])
+
+    def test_scalar_scalar(self):
+        x0 = 1.0
+        jac_diff_2 = approx_derivative(self.fun_scalar_scalar, x0,
+                                       method='2-point',
+                                       as_linear_operator=True)
+        jac_diff_3 = approx_derivative(self.fun_scalar_scalar, x0,
+                                       as_linear_operator=True)
+        jac_diff_4 = approx_derivative(self.fun_scalar_scalar, x0,
+                                       method='cs',
+                                       as_linear_operator=True)
+        jac_true = self.jac_scalar_scalar(x0)
+        np.random.seed(1)
+        for i in range(10):
+            p = np.random.uniform(-10, 10, size=(1,))
+            assert_allclose(jac_diff_2.dot(p), jac_true*p,
+                            rtol=1e-5)
+            assert_allclose(jac_diff_3.dot(p), jac_true*p,
+                            rtol=5e-6)
+            assert_allclose(jac_diff_4.dot(p), jac_true*p,
+                            rtol=5e-6)
+
+    def test_scalar_vector(self):
+        x0 = 0.5
+        jac_diff_2 = approx_derivative(self.fun_scalar_vector, x0,
+                                       method='2-point',
+                                       as_linear_operator=True)
+        jac_diff_3 = approx_derivative(self.fun_scalar_vector, x0,
+                                       as_linear_operator=True)
+        jac_diff_4 = approx_derivative(self.fun_scalar_vector, x0,
+                                       method='cs',
+                                       as_linear_operator=True)
+        jac_true = self.jac_scalar_vector(np.atleast_1d(x0))
+        np.random.seed(1)
+        for i in range(10):
+            p = np.random.uniform(-10, 10, size=(1,))
+            assert_allclose(jac_diff_2.dot(p), jac_true.dot(p),
+                            rtol=1e-5)
+            assert_allclose(jac_diff_3.dot(p), jac_true.dot(p),
+                            rtol=5e-6)
+            assert_allclose(jac_diff_4.dot(p), jac_true.dot(p),
+                            rtol=5e-6)
+
+    def test_vector_scalar(self):
+        x0 = np.array([100.0, -0.5])
+        jac_diff_2 = approx_derivative(self.fun_vector_scalar, x0,
+                                       method='2-point',
+                                       as_linear_operator=True)
+        jac_diff_3 = approx_derivative(self.fun_vector_scalar, x0,
+                                       as_linear_operator=True)
+        jac_diff_4 = approx_derivative(self.fun_vector_scalar, x0,
+                                       method='cs',
+                                       as_linear_operator=True)
+        jac_true = self.jac_vector_scalar(x0)
+        np.random.seed(1)
+        for i in range(10):
+            p = np.random.uniform(-10, 10, size=x0.shape)
+            assert_allclose(jac_diff_2.dot(p), np.atleast_1d(jac_true.dot(p)),
+                            rtol=1e-5)
+            assert_allclose(jac_diff_3.dot(p), np.atleast_1d(jac_true.dot(p)),
+                            rtol=5e-6)
+            assert_allclose(jac_diff_4.dot(p), np.atleast_1d(jac_true.dot(p)),
+                            rtol=1e-7)
+
+    def test_vector_vector(self):
+        x0 = np.array([-100.0, 0.2])
+        jac_diff_2 = approx_derivative(self.fun_vector_vector, x0,
+                                       method='2-point',
+                                       as_linear_operator=True)
+        jac_diff_3 = approx_derivative(self.fun_vector_vector, x0,
+                                       as_linear_operator=True)
+        jac_diff_4 = approx_derivative(self.fun_vector_vector, x0,
+                                       method='cs',
+                                       as_linear_operator=True)
+        jac_true = self.jac_vector_vector(x0)
+        np.random.seed(1)
+        for i in range(10):
+            p = np.random.uniform(-10, 10, size=x0.shape)
+            assert_allclose(jac_diff_2.dot(p), jac_true.dot(p), rtol=1e-5)
+            assert_allclose(jac_diff_3.dot(p), jac_true.dot(p), rtol=1e-6)
+            assert_allclose(jac_diff_4.dot(p), jac_true.dot(p), rtol=1e-7)
+
+    def test_exception(self):
+        x0 = np.array([-100.0, 0.2])
+        assert_raises(ValueError, approx_derivative,
+                      self.fun_vector_vector, x0,
+                      method='2-point', bounds=(1, np.inf))
+
+
+def test_absolute_step():
+    # test for gh12487
+    # if an absolute step is specified for 2-point differences make sure that
+    # the side corresponds to the step. i.e. if step is positive then forward
+    # differences should be used, if step is negative then backwards
+    # differences should be used.
+
+    # function has double discontinuity at x = [-1, -1]
+    # first component is \/, second component is /\
+    def f(x):
+        return -np.abs(x[0] + 1) + np.abs(x[1] + 1)
+
+    # check that the forward difference is used
+    grad = approx_derivative(f, [-1, -1], method='2-point', abs_step=1e-8)
+    assert_allclose(grad, [-1.0, 1.0])
+
+    # check that the backwards difference is used
+    grad = approx_derivative(f, [-1, -1], method='2-point', abs_step=-1e-8)
+    assert_allclose(grad, [1.0, -1.0])
+
+    # check that the forwards difference is used with a step for both
+    # parameters
+    grad = approx_derivative(
+        f, [-1, -1], method='2-point', abs_step=[1e-8, 1e-8]
+    )
+    assert_allclose(grad, [-1.0, 1.0])
+
+    # check that we can mix forward/backwards steps.
+    grad = approx_derivative(
+        f, [-1, -1], method='2-point', abs_step=[1e-8, -1e-8]
+     )
+    assert_allclose(grad, [-1.0, -1.0])
+    grad = approx_derivative(
+        f, [-1, -1], method='2-point', abs_step=[-1e-8, 1e-8]
+    )
+    assert_allclose(grad, [1.0, 1.0])
+
+    # the forward step should reverse to a backwards step if it runs into a
+    # bound
+    # This is kind of tested in TestAdjustSchemeToBounds, but only for a lower level
+    # function.
+    grad = approx_derivative(
+        f, [-1, -1], method='2-point', abs_step=1e-8,
+        bounds=(-np.inf, -1)
+    )
+    assert_allclose(grad, [1.0, -1.0])
+
+    grad = approx_derivative(
+        f, [-1, -1], method='2-point', abs_step=-1e-8, bounds=(-1, np.inf)
+    )
+    assert_allclose(grad, [-1.0, 1.0])

venv/Lib/site-packages/scipy/optimize/tests/test__remove_redundancy.py

@@ -0,0 +1,240 @@
+"""
+Unit test for Linear Programming via Simplex Algorithm.
+"""
+
+# TODO: add tests for:
+# https://github.com/scipy/scipy/issues/5400
+# https://github.com/scipy/scipy/issues/6690
+
+import numpy as np
+from numpy.testing import (
+    assert_,
+    assert_allclose,
+    assert_equal)
+
+from .test_linprog import magic_square
+from scipy.optimize._remove_redundancy import _remove_redundancy
+from scipy.optimize._remove_redundancy import _remove_redundancy_dense
+from scipy.optimize._remove_redundancy import _remove_redundancy_sparse
+from scipy.sparse import csc_matrix
+
+
+def setup_module():
+    np.random.seed(2017)
+
+
+def _assert_success(
+        res,
+        desired_fun=None,
+        desired_x=None,
+        rtol=1e-7,
+        atol=1e-7):
+    # res: linprog result object
+    # desired_fun: desired objective function value or None
+    # desired_x: desired solution or None
+    assert_(res.success)
+    assert_equal(res.status, 0)
+    if desired_fun is not None:
+        assert_allclose(
+            res.fun,
+            desired_fun,
+            err_msg="converged to an unexpected objective value",
+            rtol=rtol,
+            atol=atol)
+    if desired_x is not None:
+        assert_allclose(
+            res.x,
+            desired_x,
+            err_msg="converged to an unexpected solution",
+            rtol=rtol,
+            atol=atol)
+
+class RRCommonTests(object):
+    def test_no_redundancy(self):
+        m, n = 10, 10
+        A0 = np.random.rand(m, n)
+        b0 = np.random.rand(m)
+        A1, b1, status, message = self.rr(A0, b0)
+        assert_allclose(A0, A1)
+        assert_allclose(b0, b1)
+        assert_equal(status, 0)
+
+    def test_infeasible_zero_row(self):
+        A = np.eye(3)
+        A[1, :] = 0
+        b = np.random.rand(3)
+        A1, b1, status, message = self.rr(A, b)
+        assert_equal(status, 2)
+
+    def test_remove_zero_row(self):
+        A = np.eye(3)
+        A[1, :] = 0
+        b = np.random.rand(3)
+        b[1] = 0
+        A1, b1, status, message = self.rr(A, b)
+        assert_equal(status, 0)
+        assert_allclose(A1, A[[0, 2], :])
+        assert_allclose(b1, b[[0, 2]])
+
+    def test_infeasible_m_gt_n(self):
+        m, n = 20, 10
+        A0 = np.random.rand(m, n)
+        b0 = np.random.rand(m)
+        A1, b1, status, message = self.rr(A0, b0)
+        assert_equal(status, 2)
+
+    def test_infeasible_m_eq_n(self):
+        m, n = 10, 10
+        A0 = np.random.rand(m, n)
+        b0 = np.random.rand(m)
+        A0[-1, :] = 2 * A0[-2, :]
+        A1, b1, status, message = self.rr(A0, b0)
+        assert_equal(status, 2)
+
+    def test_infeasible_m_lt_n(self):
+        m, n = 9, 10
+        A0 = np.random.rand(m, n)
+        b0 = np.random.rand(m)
+        A0[-1, :] = np.arange(m - 1).dot(A0[:-1])
+        A1, b1, status, message = self.rr(A0, b0)
+        assert_equal(status, 2)
+
+    def test_m_gt_n(self):
+        np.random.seed(2032)
+        m, n = 20, 10
+        A0 = np.random.rand(m, n)
+        b0 = np.random.rand(m)
+        x = np.linalg.solve(A0[:n, :], b0[:n])
+        b0[n:] = A0[n:, :].dot(x)
+        A1, b1, status, message = self.rr(A0, b0)
+        assert_equal(status, 0)
+        assert_equal(A1.shape[0], n)
+        assert_equal(np.linalg.matrix_rank(A1), n)
+
+    def test_m_gt_n_rank_deficient(self):
+        m, n = 20, 10
+        A0 = np.zeros((m, n))
+        A0[:, 0] = 1
+        b0 = np.ones(m)
+        A1, b1, status, message = self.rr(A0, b0)
+        assert_equal(status, 0)
+        assert_allclose(A1, A0[0:1, :])
+        assert_allclose(b1, b0[0])
+
+    def test_m_lt_n_rank_deficient(self):
+        m, n = 9, 10
+        A0 = np.random.rand(m, n)
+        b0 = np.random.rand(m)
+        A0[-1, :] = np.arange(m - 1).dot(A0[:-1])
+        b0[-1] = np.arange(m - 1).dot(b0[:-1])
+        A1, b1, status, message = self.rr(A0, b0)
+        assert_equal(status, 0)
+        assert_equal(A1.shape[0], 8)
+        assert_equal(np.linalg.matrix_rank(A1), 8)
+
+    def test_dense1(self):
+        A = np.ones((6, 6))
+        A[0, :3] = 0
+        A[1, 3:] = 0
+        A[3:, ::2] = -1
+        A[3, :2] = 0
+        A[4, 2:] = 0
+        b = np.zeros(A.shape[0])
+
+        A2 = A[[0, 1, 3, 4], :]
+        b2 = np.zeros(4)
+
+        A1, b1, status, message = self.rr(A, b)
+        assert_allclose(A1, A2)
+        assert_allclose(b1, b2)
+        assert_equal(status, 0)
+
+    def test_dense2(self):
+        A = np.eye(6)
+        A[-2, -1] = 1
+        A[-1, :] = 1
+        b = np.zeros(A.shape[0])
+        A1, b1, status, message = self.rr(A, b)
+        assert_allclose(A1, A[:-1, :])
+        assert_allclose(b1, b[:-1])
+        assert_equal(status, 0)
+
+    def test_dense3(self):
+        A = np.eye(6)
+        A[-2, -1] = 1
+        A[-1, :] = 1
+        b = np.random.rand(A.shape[0])
+        b[-1] = np.sum(b[:-1])
+        A1, b1, status, message = self.rr(A, b)
+        assert_allclose(A1, A[:-1, :])
+        assert_allclose(b1, b[:-1])
+        assert_equal(status, 0)
+
+    def test_m_gt_n_sparse(self):
+        np.random.seed(2013)
+        m, n = 20, 5
+        p = 0.1
+        A = np.random.rand(m, n)
+        A[np.random.rand(m, n) > p] = 0
+        rank = np.linalg.matrix_rank(A)
+        b = np.zeros(A.shape[0])
+        A1, b1, status, message = self.rr(A, b)
+        assert_equal(status, 0)
+        assert_equal(A1.shape[0], rank)
+        assert_equal(np.linalg.matrix_rank(A1), rank)
+
+    def test_m_lt_n_sparse(self):
+        np.random.seed(2017)
+        m, n = 20, 50
+        p = 0.05
+        A = np.random.rand(m, n)
+        A[np.random.rand(m, n) > p] = 0
+        rank = np.linalg.matrix_rank(A)
+        b = np.zeros(A.shape[0])
+        A1, b1, status, message = self.rr(A, b)
+        assert_equal(status, 0)
+        assert_equal(A1.shape[0], rank)
+        assert_equal(np.linalg.matrix_rank(A1), rank)
+
+    def test_m_eq_n_sparse(self):
+        np.random.seed(2017)
+        m, n = 100, 100
+        p = 0.01
+        A = np.random.rand(m, n)
+        A[np.random.rand(m, n) > p] = 0
+        rank = np.linalg.matrix_rank(A)
+        b = np.zeros(A.shape[0])
+        A1, b1, status, message = self.rr(A, b)
+        assert_equal(status, 0)
+        assert_equal(A1.shape[0], rank)
+        assert_equal(np.linalg.matrix_rank(A1), rank)
+
+    def test_magic_square(self):
+        A, b, c, numbers = magic_square(3)
+        A1, b1, status, message = self.rr(A, b)
+        assert_equal(status, 0)
+        assert_equal(A1.shape[0], 23)
+        assert_equal(np.linalg.matrix_rank(A1), 23)
+
+    def test_magic_square2(self):
+        A, b, c, numbers = magic_square(4)
+        A1, b1, status, message = self.rr(A, b)
+        assert_equal(status, 0)
+        assert_equal(A1.shape[0], 39)
+        assert_equal(np.linalg.matrix_rank(A1), 39)
+
+
+class TestRRSVD(RRCommonTests):
+    def rr(self, A, b):
+        return _remove_redundancy(A, b)
+
+
+class TestRRPivotDense(RRCommonTests):
+    def rr(self, A, b):
+        return _remove_redundancy_dense(A, b)
+
+
+class TestRRPivotSparse(RRCommonTests):
+    def rr(self, A, b):
+        A1, b1, status, message = _remove_redundancy_sparse(csc_matrix(A), b)
+        return A1.toarray(), b1, status, message

venv/Lib/site-packages/scipy/optimize/tests/test__root.py

@@ -0,0 +1,69 @@
+"""
+Unit tests for optimization routines from _root.py.
+"""
+from numpy.testing import assert_
+from pytest import raises as assert_raises
+import numpy as np
+
+from scipy.optimize import root
+
+
+class TestRoot(object):
+    def test_tol_parameter(self):
+        # Check that the minimize() tol= argument does something
+        def func(z):
+            x, y = z
+            return np.array([x**3 - 1, y**3 - 1])
+
+        def dfunc(z):
+            x, y = z
+            return np.array([[3*x**2, 0], [0, 3*y**2]])
+
+        for method in ['hybr', 'lm', 'broyden1', 'broyden2', 'anderson',
+                       'diagbroyden', 'krylov']:
+            if method in ('linearmixing', 'excitingmixing'):
+                # doesn't converge
+                continue
+
+            if method in ('hybr', 'lm'):
+                jac = dfunc
+            else:
+                jac = None
+
+            sol1 = root(func, [1.1,1.1], jac=jac, tol=1e-4, method=method)
+            sol2 = root(func, [1.1,1.1], jac=jac, tol=0.5, method=method)
+            msg = "%s: %s vs. %s" % (method, func(sol1.x), func(sol2.x))
+            assert_(sol1.success, msg)
+            assert_(sol2.success, msg)
+            assert_(abs(func(sol1.x)).max() < abs(func(sol2.x)).max(),
+                    msg)
+
+    def test_minimize_scalar_coerce_args_param(self):
+        # github issue #3503
+        def func(z, f=1):
+            x, y = z
+            return np.array([x**3 - 1, y**3 - f])
+        root(func, [1.1, 1.1], args=1.5)
+
+    def test_f_size(self):
+        # gh8320
+        # check that decreasing the size of the returned array raises an error
+        # and doesn't segfault
+        class fun(object):
+            def __init__(self):
+                self.count = 0
+
+            def __call__(self, x):
+                self.count += 1
+
+                if not (self.count % 5):
+                    ret = x[0] + 0.5 * (x[0] - x[1]) ** 3 - 1.0
+                else:
+                    ret = ([x[0] + 0.5 * (x[0] - x[1]) ** 3 - 1.0,
+                           0.5 * (x[1] - x[0]) ** 3 + x[1]])
+
+                return ret
+
+        F = fun()
+        with assert_raises(ValueError):
+            root(F, [0.1, 0.0], method='lm')

venv/Lib/site-packages/scipy/optimize/tests/test__shgo.py

@@ -0,0 +1,761 @@
+import logging
+import numpy
+import pytest
+from pytest import raises as assert_raises, warns
+from scipy.optimize import shgo
+from scipy.optimize._shgo import SHGO
+
+
+class StructTestFunction(object):
+    def __init__(self, bounds, expected_x, expected_fun=None,
+                 expected_xl=None, expected_funl=None):
+        self.bounds = bounds
+        self.expected_x = expected_x
+        self.expected_fun = expected_fun
+        self.expected_xl = expected_xl
+        self.expected_funl = expected_funl
+
+
+def wrap_constraints(g):
+    cons = []
+    if g is not None:
+        if (type(g) is not tuple) and (type(g) is not list):
+            g = (g,)
+        else:
+            pass
+        for g in g:
+            cons.append({'type': 'ineq',
+                         'fun': g})
+        cons = tuple(cons)
+    else:
+        cons = None
+    return cons
+
+
+class StructTest1(StructTestFunction):
+    def f(self, x):
+        return x[0] ** 2 + x[1] ** 2
+
+    def g(x):
+        return -(numpy.sum(x, axis=0) - 6.0)
+
+    cons = wrap_constraints(g)
+
+
+test1_1 = StructTest1(bounds=[(-1, 6), (-1, 6)],
+                      expected_x=[0, 0])
+test1_2 = StructTest1(bounds=[(0, 1), (0, 1)],
+                      expected_x=[0, 0])
+test1_3 = StructTest1(bounds=[(None, None), (None, None)],
+                      expected_x=[0, 0])
+
+
+class StructTest2(StructTestFunction):
+    """
+    Scalar function with several minima to test all minimizer retrievals
+    """
+
+    def f(self, x):
+        return (x - 30) * numpy.sin(x)
+
+    def g(x):
+        return 58 - numpy.sum(x, axis=0)
+
+    cons = wrap_constraints(g)
+
+
+test2_1 = StructTest2(bounds=[(0, 60)],
+                      expected_x=[1.53567906],
+                      expected_fun=-28.44677132,
+                      # Important: test that funl return is in the correct order
+                      expected_xl=numpy.array([[1.53567906],
+                                               [55.01782167],
+                                               [7.80894889],
+                                               [48.74797493],
+                                               [14.07445705],
+                                               [42.4913859],
+                                               [20.31743841],
+                                               [36.28607535],
+                                               [26.43039605],
+                                               [30.76371366]]),
+
+                      expected_funl=numpy.array([-28.44677132, -24.99785984,
+                                                 -22.16855376, -18.72136195,
+                                                 -15.89423937, -12.45154942,
+                                                 -9.63133158, -6.20801301,
+                                                 -3.43727232, -0.46353338])
+                      )
+
+test2_2 = StructTest2(bounds=[(0, 4.5)],
+                      expected_x=[1.53567906],
+                      expected_fun=[-28.44677132],
+                      expected_xl=numpy.array([[1.53567906]]),
+                      expected_funl=numpy.array([-28.44677132])
+                      )
+
+
+class StructTest3(StructTestFunction):
+    """
+    Hock and Schittkowski 18 problem (HS18). Hoch and Schittkowski (1981)
+    http://www.ai7.uni-bayreuth.de/test_problem_coll.pdf
+    Minimize: f = 0.01 * (x_1)**2 + (x_2)**2
+
+    Subject to: x_1 * x_2 - 25.0 >= 0,
+                (x_1)**2 + (x_2)**2 - 25.0 >= 0,
+                2 <= x_1 <= 50,
+                0 <= x_2 <= 50.
+
+    Approx. Answer:
+        f([(250)**0.5 , (2.5)**0.5]) = 5.0
+
+
+    """
+
+    def f(self, x):
+        return 0.01 * (x[0]) ** 2 + (x[1]) ** 2
+
+    def g1(x):
+        return x[0] * x[1] - 25.0
+
+    def g2(x):
+        return x[0] ** 2 + x[1] ** 2 - 25.0
+
+    g = (g1, g2)
+
+    cons = wrap_constraints(g)
+
+
+test3_1 = StructTest3(bounds=[(2, 50), (0, 50)],
+                      expected_x=[250 ** 0.5, 2.5 ** 0.5],
+                      expected_fun=5.0
+                      )
+
+
+class StructTest4(StructTestFunction):
+    """
+    Hock and Schittkowski 11 problem (HS11). Hoch and Schittkowski (1981)
+
+    NOTE: Did not find in original reference to HS collection, refer to
+          Henderson (2015) problem 7 instead. 02.03.2016
+    """
+
+    def f(self, x):
+        return ((x[0] - 10) ** 2 + 5 * (x[1] - 12) ** 2 + x[2] ** 4
+                + 3 * (x[3] - 11) ** 2 + 10 * x[4] ** 6 + 7 * x[5] ** 2 + x[
+                    6] ** 4
+                - 4 * x[5] * x[6] - 10 * x[5] - 8 * x[6]
+                )
+
+    def g1(x):
+        return -(2 * x[0] ** 2 + 3 * x[1] ** 4 + x[2] + 4 * x[3] ** 2
+                 + 5 * x[4] - 127)
+
+    def g2(x):
+        return -(7 * x[0] + 3 * x[1] + 10 * x[2] ** 2 + x[3] - x[4] - 282.0)
+
+    def g3(x):
+        return -(23 * x[0] + x[1] ** 2 + 6 * x[5] ** 2 - 8 * x[6] - 196)
+
+    def g4(x):
+        return -(4 * x[0] ** 2 + x[1] ** 2 - 3 * x[0] * x[1] + 2 * x[2] ** 2
+                 + 5 * x[5] - 11 * x[6])
+
+    g = (g1, g2, g3, g4)
+
+    cons = wrap_constraints(g)
+
+
+test4_1 = StructTest4(bounds=[(-10, 10), ] * 7,
+                      expected_x=[2.330499, 1.951372, -0.4775414,
+                                  4.365726, -0.6244870, 1.038131, 1.594227],
+                      expected_fun=680.6300573
+                      )
+
+
+class StructTest5(StructTestFunction):
+    def f(self, x):
+        return (-(x[1] + 47.0)
+                * numpy.sin(numpy.sqrt(abs(x[0] / 2.0 + (x[1] + 47.0))))
+                - x[0] * numpy.sin(numpy.sqrt(abs(x[0] - (x[1] + 47.0))))
+                )
+
+    g = None
+    cons = wrap_constraints(g)
+
+
+test5_1 = StructTest5(bounds=[(-512, 512), (-512, 512)],
+                      expected_fun=[-959.64066272085051],
+                      expected_x=[512., 404.23180542])
+
+
+class StructTestLJ(StructTestFunction):
+    """
+    LennardJones objective function. Used to test symmetry constraints settings.
+    """
+
+    def f(self, x, *args):
+        self.N = args[0]
+        k = int(self.N / 3)
+        s = 0.0
+
+        for i in range(k - 1):
+            for j in range(i + 1, k):
+                a = 3 * i
+                b = 3 * j
+                xd = x[a] - x[b]
+                yd = x[a + 1] - x[b + 1]
+                zd = x[a + 2] - x[b + 2]
+                ed = xd * xd + yd * yd + zd * zd
+                ud = ed * ed * ed
+                if ed > 0.0:
+                    s += (1.0 / ud - 2.0) / ud
+
+        return s
+
+    g = None
+    cons = wrap_constraints(g)
+
+
+N = 6
+boundsLJ = list(zip([-4.0] * 6, [4.0] * 6))
+
+testLJ = StructTestLJ(bounds=boundsLJ,
+                      expected_fun=[-1.0],
+                      expected_x=[-2.71247337e-08,
+                                  -2.71247337e-08,
+                                  -2.50000222e+00,
+                                  -2.71247337e-08,
+                                  -2.71247337e-08,
+                                  -1.50000222e+00]
+                      )
+
+
+class StructTestTable(StructTestFunction):
+    def f(self, x):
+        if x[0] == 3.0 and x[1] == 3.0:
+            return 50
+        else:
+            return 100
+
+    g = None
+    cons = wrap_constraints(g)
+
+
+test_table = StructTestTable(bounds=[(-10, 10), (-10, 10)],
+                             expected_fun=[50],
+                             expected_x=[3.0, 3.0])
+
+
+class StructTestInfeasible(StructTestFunction):
+    """
+    Test function with no feasible domain.
+    """
+
+    def f(self, x, *args):
+        return x[0] ** 2 + x[1] ** 2
+
+    def g1(x):
+        return x[0] + x[1] - 1
+
+    def g2(x):
+        return -(x[0] + x[1] - 1)
+
+    def g3(x):
+        return -x[0] + x[1] - 1
+
+    def g4(x):
+        return -(-x[0] + x[1] - 1)
+
+    g = (g1, g2, g3, g4)
+    cons = wrap_constraints(g)
+
+
+test_infeasible = StructTestInfeasible(bounds=[(2, 50), (-1, 1)],
+                                       expected_fun=None,
+                                       expected_x=None
+                                       )
+
+
+def run_test(test, args=(), test_atol=1e-5, n=100, iters=None,
+             callback=None, minimizer_kwargs=None, options=None,
+             sampling_method='sobol'):
+    res = shgo(test.f, test.bounds, args=args, constraints=test.cons,
+               n=n, iters=iters, callback=callback,
+               minimizer_kwargs=minimizer_kwargs, options=options,
+               sampling_method=sampling_method)
+
+    logging.info(res)
+
+    if test.expected_x is not None:
+        numpy.testing.assert_allclose(res.x, test.expected_x,
+                                      rtol=test_atol,
+                                      atol=test_atol)
+
+    # (Optional tests)
+    if test.expected_fun is not None:
+        numpy.testing.assert_allclose(res.fun,
+                                      test.expected_fun,
+                                      atol=test_atol)
+
+    if test.expected_xl is not None:
+        numpy.testing.assert_allclose(res.xl,
+                                      test.expected_xl,
+                                      atol=test_atol)
+
+    if test.expected_funl is not None:
+        numpy.testing.assert_allclose(res.funl,
+                                      test.expected_funl,
+                                      atol=test_atol)
+    return
+
+
+# Base test functions:
+class TestShgoSobolTestFunctions(object):
+    """
+    Global optimization tests with Sobol sampling:
+    """
+
+    # Sobol algorithm
+    def test_f1_1_sobol(self):
+        """Multivariate test function 1:
+        x[0]**2 + x[1]**2 with bounds=[(-1, 6), (-1, 6)]"""
+        run_test(test1_1)
+
+    def test_f1_2_sobol(self):
+        """Multivariate test function 1:
+         x[0]**2 + x[1]**2 with bounds=[(0, 1), (0, 1)]"""
+        run_test(test1_2)
+
+    def test_f1_3_sobol(self):
+        """Multivariate test function 1:
+        x[0]**2 + x[1]**2 with bounds=[(None, None),(None, None)]"""
+        run_test(test1_3)
+
+    def test_f2_1_sobol(self):
+        """Univariate test function on
+        f(x) = (x - 30) * sin(x) with bounds=[(0, 60)]"""
+        run_test(test2_1)
+
+    def test_f2_2_sobol(self):
+        """Univariate test function on
+        f(x) = (x - 30) * sin(x) bounds=[(0, 4.5)]"""
+        run_test(test2_2)
+
+    def test_f3_sobol(self):
+        """NLP: Hock and Schittkowski problem 18"""
+        run_test(test3_1)
+
+    @pytest.mark.slow
+    def test_f4_sobol(self):
+        """NLP: (High-dimensional) Hock and Schittkowski 11 problem (HS11)"""
+        # run_test(test4_1, n=500)
+        # run_test(test4_1, n=800)
+        options = {'infty_constraints': False}
+        run_test(test4_1, n=990, options=options)
+
+    def test_f5_1_sobol(self):
+        """NLP: Eggholder, multimodal"""
+        run_test(test5_1, n=30)
+
+    def test_f5_2_sobol(self):
+        """NLP: Eggholder, multimodal"""
+        # run_test(test5_1, n=60, iters=5)
+        run_test(test5_1, n=60, iters=5)
+
+        # def test_t911(self):
+        #    """1-D tabletop function"""
+        #    run_test(test11_1)
+
+
+class TestShgoSimplicialTestFunctions(object):
+    """
+    Global optimization tests with Simplicial sampling:
+    """
+
+    def test_f1_1_simplicial(self):
+        """Multivariate test function 1:
+        x[0]**2 + x[1]**2 with bounds=[(-1, 6), (-1, 6)]"""
+        run_test(test1_1, n=1, sampling_method='simplicial')
+
+    def test_f1_2_simplicial(self):
+        """Multivariate test function 1:
+        x[0]**2 + x[1]**2 with bounds=[(0, 1), (0, 1)]"""
+        run_test(test1_2, n=1, sampling_method='simplicial')
+
+    def test_f1_3_simplicial(self):
+        """Multivariate test function 1: x[0]**2 + x[1]**2
+        with bounds=[(None, None),(None, None)]"""
+        run_test(test1_3, n=1, sampling_method='simplicial')
+
+    def test_f2_1_simplicial(self):
+        """Univariate test function on
+        f(x) = (x - 30) * sin(x) with bounds=[(0, 60)]"""
+        options = {'minimize_every_iter': False}
+        run_test(test2_1, iters=7, options=options,
+                 sampling_method='simplicial')
+
+    def test_f2_2_simplicial(self):
+        """Univariate test function on
+        f(x) = (x - 30) * sin(x) bounds=[(0, 4.5)]"""
+        run_test(test2_2, n=1, sampling_method='simplicial')
+
+    def test_f3_simplicial(self):
+        """NLP: Hock and Schittkowski problem 18"""
+        run_test(test3_1, n=1, sampling_method='simplicial')
+
+    @pytest.mark.slow
+    def test_f4_simplicial(self):
+        """NLP: (High-dimensional) Hock and Schittkowski 11 problem (HS11)"""
+        run_test(test4_1, n=1, sampling_method='simplicial')
+
+    def test_lj_symmetry(self):
+        """LJ: Symmetry-constrained test function"""
+        options = {'symmetry': True,
+                   'disp': True}
+        args = (6,)  # Number of atoms
+        run_test(testLJ, args=args, n=None,
+                 options=options, iters=4,
+                 sampling_method='simplicial')
+
+
+# Argument test functions
+class TestShgoArguments(object):
+    def test_1_1_simpl_iter(self):
+        """Iterative simplicial sampling on TestFunction 1 (multivariate)"""
+        run_test(test1_2, n=None, iters=2, sampling_method='simplicial')
+
+    def test_1_2_simpl_iter(self):
+        """Iterative simplicial on TestFunction 2 (univariate)"""
+        options = {'minimize_every_iter': False}
+        run_test(test2_1, n=None, iters=7, options=options,
+                 sampling_method='simplicial')
+
+    def test_2_1_sobol_iter(self):
+        """Iterative Sobol sampling on TestFunction 1 (multivariate)"""
+        run_test(test1_2, n=None, iters=1, sampling_method='sobol')
+
+    def test_2_2_sobol_iter(self):
+        """Iterative Sobol sampling on TestFunction 2 (univariate)"""
+        res = shgo(test2_1.f, test2_1.bounds, constraints=test2_1.cons,
+                   n=None, iters=1, sampling_method='sobol')
+
+        numpy.testing.assert_allclose(res.x, test2_1.expected_x, rtol=1e-5,
+                                      atol=1e-5)
+        numpy.testing.assert_allclose(res.fun, test2_1.expected_fun, atol=1e-5)
+
+    def test_3_1_disp_simplicial(self):
+        """Iterative sampling on TestFunction 1 and 2  (multi- and univariate)"""
+
+        def callback_func(x):
+            print("Local minimization callback test")
+
+        for test in [test1_1, test2_1]:
+            shgo(test.f, test.bounds, iters=1,
+                 sampling_method='simplicial',
+                 callback=callback_func, options={'disp': True})
+            shgo(test.f, test.bounds, n=1, sampling_method='simplicial',
+                 callback=callback_func, options={'disp': True})
+
+    def test_3_2_disp_sobol(self):
+        """Iterative sampling on TestFunction 1 and 2 (multi- and univariate)"""
+
+        def callback_func(x):
+            print("Local minimization callback test")
+
+        for test in [test1_1, test2_1]:
+            shgo(test.f, test.bounds, iters=1, sampling_method='sobol',
+                 callback=callback_func, options={'disp': True})
+
+            shgo(test.f, test.bounds, n=1, sampling_method='simplicial',
+                 callback=callback_func, options={'disp': True})
+
+    @pytest.mark.slow
+    def test_4_1_known_f_min(self):
+        """Test known function minima stopping criteria"""
+        # Specify known function value
+        options = {'f_min': test4_1.expected_fun,
+                   'f_tol': 1e-6,
+                   'minimize_every_iter': True}
+        # TODO: Make default n higher for faster tests
+        run_test(test4_1, n=None, test_atol=1e-5, options=options,
+                 sampling_method='simplicial')
+
+    @pytest.mark.slow
+    def test_4_2_known_f_min(self):
+        """Test Global mode limiting local evalutions"""
+        options = {  # Specify known function value
+            'f_min': test4_1.expected_fun,
+            'f_tol': 1e-6,
+            # Specify number of local iterations to perform
+            'minimize_every_iter': True,
+            'local_iter': 1}
+
+        run_test(test4_1, n=None, test_atol=1e-5, options=options,
+                 sampling_method='simplicial')
+
+    @pytest.mark.slow
+    def test_4_3_known_f_min(self):
+        """Test Global mode limiting local evalutions"""
+        options = {  # Specify known function value
+            'f_min': test4_1.expected_fun,
+            'f_tol': 1e-6,
+            # Specify number of local iterations to perform+
+            'minimize_every_iter': True,
+            'local_iter': 1,
+            'infty_constraints': False}
+
+        run_test(test4_1, n=300, test_atol=1e-5, options=options,
+                 sampling_method='sobol')
+
+    def test_4_4_known_f_min(self):
+        """Test Global mode limiting local evalutions for 1-D functions"""
+        options = {  # Specify known function value
+            'f_min': test2_1.expected_fun,
+            'f_tol': 1e-6,
+            # Specify number of local iterations to perform+
+            'minimize_every_iter': True,
+            'local_iter': 1,
+            'infty_constraints': False}
+
+        res = shgo(test2_1.f, test2_1.bounds, constraints=test2_1.cons,
+                   n=None, iters=None, options=options,
+                   sampling_method='sobol')
+        numpy.testing.assert_allclose(res.x, test2_1.expected_x, rtol=1e-5,
+                                      atol=1e-5)
+
+    def test_5_1_simplicial_argless(self):
+        """Test Default simplicial sampling settings on TestFunction 1"""
+        res = shgo(test1_1.f, test1_1.bounds, constraints=test1_1.cons)
+        numpy.testing.assert_allclose(res.x, test1_1.expected_x, rtol=1e-5,
+                                      atol=1e-5)
+
+    def test_5_2_sobol_argless(self):
+        """Test Default sobol sampling settings on TestFunction 1"""
+        res = shgo(test1_1.f, test1_1.bounds, constraints=test1_1.cons,
+                   sampling_method='sobol')
+        numpy.testing.assert_allclose(res.x, test1_1.expected_x, rtol=1e-5,
+                                      atol=1e-5)
+
+    def test_6_1_simplicial_max_iter(self):
+        """Test that maximum iteration option works on TestFunction 3"""
+        options = {'max_iter': 2}
+        res = shgo(test3_1.f, test3_1.bounds, constraints=test3_1.cons,
+                   options=options, sampling_method='simplicial')
+        numpy.testing.assert_allclose(res.x, test3_1.expected_x, rtol=1e-5,
+                                      atol=1e-5)
+        numpy.testing.assert_allclose(res.fun, test3_1.expected_fun, atol=1e-5)
+
+    def test_6_2_simplicial_min_iter(self):
+        """Test that maximum iteration option works on TestFunction 3"""
+        options = {'min_iter': 2}
+        res = shgo(test3_1.f, test3_1.bounds, constraints=test3_1.cons,
+                   options=options, sampling_method='simplicial')
+        numpy.testing.assert_allclose(res.x, test3_1.expected_x, rtol=1e-5,
+                                      atol=1e-5)
+        numpy.testing.assert_allclose(res.fun, test3_1.expected_fun, atol=1e-5)
+
+    def test_7_1_minkwargs(self):
+        """Test the minimizer_kwargs arguments for solvers with constraints"""
+        # Test solvers
+        for solver in ['COBYLA', 'SLSQP']:
+            # Note that passing global constraints to SLSQP is tested in other
+            # unittests which run test4_1 normally
+            minimizer_kwargs = {'method': solver,
+                                'constraints': test3_1.cons}
+            print("Solver = {}".format(solver))
+            print("=" * 100)
+            run_test(test3_1, n=100, test_atol=1e-3,
+                     minimizer_kwargs=minimizer_kwargs, sampling_method='sobol')
+
+    def test_7_2_minkwargs(self):
+        """Test the minimizer_kwargs default inits"""
+        minimizer_kwargs = {'ftol': 1e-5}
+        options = {'disp': True}  # For coverage purposes
+        SHGO(test3_1.f, test3_1.bounds, constraints=test3_1.cons[0],
+             minimizer_kwargs=minimizer_kwargs, options=options)
+
+    def test_7_3_minkwargs(self):
+        """Test minimizer_kwargs arguments for solvers without constraints"""
+        for solver in ['Nelder-Mead', 'Powell', 'CG', 'BFGS', 'Newton-CG',
+                       'L-BFGS-B', 'TNC', 'dogleg', 'trust-ncg', 'trust-exact',
+                       'trust-krylov']:
+            def jac(x):
+                return numpy.array([2 * x[0], 2 * x[1]]).T
+
+            def hess(x):
+                return numpy.array([[2, 0], [0, 2]])
+
+            minimizer_kwargs = {'method': solver,
+                                'jac': jac,
+                                'hess': hess}
+            logging.info("Solver = {}".format(solver))
+            logging.info("=" * 100)
+            run_test(test1_1, n=100, test_atol=1e-3,
+                     minimizer_kwargs=minimizer_kwargs, sampling_method='sobol')
+
+    def test_8_homology_group_diff(self):
+        options = {'minhgrd': 1,
+                   'minimize_every_iter': True}
+
+        run_test(test1_1, n=None, iters=None, options=options,
+                 sampling_method='simplicial')
+
+    def test_9_cons_g(self):
+        """Test single function constraint passing"""
+        SHGO(test3_1.f, test3_1.bounds, constraints=test3_1.cons[0])
+
+    def test_10_finite_time(self):
+        """Test single function constraint passing"""
+        options = {'maxtime': 1e-15}
+        shgo(test1_1.f, test1_1.bounds, n=1, iters=None,
+             options=options, sampling_method='sobol')
+
+    def test_11_f_min_time(self):
+        """Test to cover the case where f_lowest == 0"""
+        options = {'maxtime': 1e-15,
+                   'f_min': 0.0}
+        shgo(test1_2.f, test1_2.bounds, n=1, iters=None,
+             options=options, sampling_method='sobol')
+
+    def test_12_sobol_inf_cons(self):
+        """Test to cover the case where f_lowest == 0"""
+        options = {'maxtime': 1e-15,
+                   'f_min': 0.0}
+        shgo(test1_2.f, test1_2.bounds, n=1, iters=None,
+             options=options, sampling_method='sobol')
+
+    def test_13_high_sobol(self):
+        """Test init of high-dimensional sobol sequences"""
+
+        def f(x):
+            return 0
+
+        bounds = [(None, None), ] * 41
+        SHGOc = SHGO(f, bounds)
+        SHGOc.sobol_points(2, 50)
+
+    def test_14_local_iter(self):
+        """Test limited local iterations for a pseudo-global mode"""
+        options = {'local_iter': 4}
+        run_test(test5_1, n=30, options=options)
+
+    def test_15_min_every_iter(self):
+        """Test minimize every iter options and cover function cache"""
+        options = {'minimize_every_iter': True}
+        run_test(test1_1, n=1, iters=7, options=options,
+                 sampling_method='sobol')
+
+    def test_16_disp_bounds_minimizer(self):
+        """Test disp=True with minimizers that do not support bounds """
+        options = {'disp': True}
+        minimizer_kwargs = {'method': 'nelder-mead'}
+        run_test(test1_2, sampling_method='simplicial',
+                 options=options, minimizer_kwargs=minimizer_kwargs)
+
+    def test_17_custom_sampling(self):
+        """Test the functionality to add custom sampling methods to shgo"""
+        def sample(n, d):
+            return numpy.random.uniform(size=(n,d))
+
+        run_test(test1_1, n=30, sampling_method=sample)
+
+# Failure test functions
+class TestShgoFailures(object):
+    def test_1_maxiter(self):
+        """Test failure on insufficient iterations"""
+        options = {'maxiter': 2}
+        res = shgo(test4_1.f, test4_1.bounds, n=2, iters=None,
+                   options=options, sampling_method='sobol')
+
+        numpy.testing.assert_equal(False, res.success)
+        numpy.testing.assert_equal(4, res.nfev)
+
+    def test_2_sampling(self):
+        """Rejection of unknown sampling method"""
+        assert_raises(ValueError, shgo, test1_1.f, test1_1.bounds,
+                      sampling_method='not_Sobol')
+
+    def test_3_1_no_min_pool_sobol(self):
+        """Check that the routine stops when no minimiser is found
+           after maximum specified function evaluations"""
+        options = {'maxfev': 10,
+                   'disp': True}
+        res = shgo(test_table.f, test_table.bounds, n=3, options=options,
+                   sampling_method='sobol')
+        numpy.testing.assert_equal(False, res.success)
+        # numpy.testing.assert_equal(9, res.nfev)
+        numpy.testing.assert_equal(12, res.nfev)
+
+    def test_3_2_no_min_pool_simplicial(self):
+        """Check that the routine stops when no minimiser is found
+           after maximum specified sampling evaluations"""
+        options = {'maxev': 10,
+                   'disp': True}
+        res = shgo(test_table.f, test_table.bounds, n=3, options=options,
+                   sampling_method='simplicial')
+        numpy.testing.assert_equal(False, res.success)
+
+    def test_4_1_bound_err(self):
+        """Specified bounds ub > lb"""
+        bounds = [(6, 3), (3, 5)]
+        assert_raises(ValueError, shgo, test1_1.f, bounds)
+
+    def test_4_2_bound_err(self):
+        """Specified bounds are of the form (lb, ub)"""
+        bounds = [(3, 5, 5), (3, 5)]
+        assert_raises(ValueError, shgo, test1_1.f, bounds)
+
+    def test_5_1_1_infeasible_sobol(self):
+        """Ensures the algorithm terminates on infeasible problems
+           after maxev is exceeded. Use infty constraints option"""
+        options = {'maxev': 100,
+                   'disp': True}
+
+        res = shgo(test_infeasible.f, test_infeasible.bounds,
+                   constraints=test_infeasible.cons, n=100, options=options,
+                   sampling_method='sobol')
+
+        numpy.testing.assert_equal(False, res.success)
+
+    def test_5_1_2_infeasible_sobol(self):
+        """Ensures the algorithm terminates on infeasible problems
+           after maxev is exceeded. Do not use infty constraints option"""
+        options = {'maxev': 100,
+                   'disp': True,
+                   'infty_constraints': False}
+
+        res = shgo(test_infeasible.f, test_infeasible.bounds,
+                   constraints=test_infeasible.cons, n=100, options=options,
+                   sampling_method='sobol')
+
+        numpy.testing.assert_equal(False, res.success)
+
+    def test_5_2_infeasible_simplicial(self):
+        """Ensures the algorithm terminates on infeasible problems
+           after maxev is exceeded."""
+        options = {'maxev': 1000,
+                   'disp': False}
+
+        res = shgo(test_infeasible.f, test_infeasible.bounds,
+                   constraints=test_infeasible.cons, n=100, options=options,
+                   sampling_method='simplicial')
+
+        numpy.testing.assert_equal(False, res.success)
+
+    def test_6_1_lower_known_f_min(self):
+        """Test Global mode limiting local evalutions with f* too high"""
+        options = {  # Specify known function value
+            'f_min': test2_1.expected_fun + 2.0,
+            'f_tol': 1e-6,
+            # Specify number of local iterations to perform+
+            'minimize_every_iter': True,
+            'local_iter': 1,
+            'infty_constraints': False}
+        args = (test2_1.f, test2_1.bounds)
+        kwargs = {'constraints': test2_1.cons,
+                  'n': None,
+                  'iters': None,
+                  'options': options,
+                  'sampling_method': 'sobol'
+                  }
+        warns(UserWarning, shgo, *args, **kwargs)

venv/Lib/site-packages/scipy/optimize/tests/test__spectral.py

@@ -0,0 +1,208 @@
+import itertools
+
+import numpy as np
+from numpy import exp
+from numpy.testing import assert_, assert_equal
+
+from scipy.optimize import root
+
+
+def test_performance():
+    # Compare performance results to those listed in
+    # [Cheng & Li, IMA J. Num. An. 29, 814 (2008)]
+    # and
+    # [W. La Cruz, J.M. Martinez, M. Raydan, Math. Comp. 75, 1429 (2006)].
+    # and those produced by dfsane.f from M. Raydan's website.
+    #
+    # Where the results disagree, the largest limits are taken.
+
+    e_a = 1e-5
+    e_r = 1e-4
+
+    table_1 = [
+        dict(F=F_1, x0=x0_1, n=1000, nit=5, nfev=5),
+        dict(F=F_1, x0=x0_1, n=10000, nit=2, nfev=2),
+        dict(F=F_2, x0=x0_2, n=500, nit=11, nfev=11),
+        dict(F=F_2, x0=x0_2, n=2000, nit=11, nfev=11),
+        # dict(F=F_4, x0=x0_4, n=999, nit=243, nfev=1188),  removed: too sensitive to rounding errors
+        dict(F=F_6, x0=x0_6, n=100, nit=6, nfev=6),  # Results from dfsane.f; papers list nit=3, nfev=3
+        dict(F=F_7, x0=x0_7, n=99, nit=23, nfev=29),  # Must have n%3==0, typo in papers?
+        dict(F=F_7, x0=x0_7, n=999, nit=23, nfev=29),  # Must have n%3==0, typo in papers?
+        dict(F=F_9, x0=x0_9, n=100, nit=12, nfev=18),  # Results from dfsane.f; papers list nit=nfev=6?
+        dict(F=F_9, x0=x0_9, n=1000, nit=12, nfev=18),
+        dict(F=F_10, x0=x0_10, n=1000, nit=5, nfev=5),  # Results from dfsane.f; papers list nit=2, nfev=12
+    ]
+
+    # Check also scaling invariance
+    for xscale, yscale, line_search in itertools.product([1.0, 1e-10, 1e10], [1.0, 1e-10, 1e10],
+                                                         ['cruz', 'cheng']):
+        for problem in table_1:
+            n = problem['n']
+            func = lambda x, n: yscale*problem['F'](x/xscale, n)
+            args = (n,)
+            x0 = problem['x0'](n) * xscale
+
+            fatol = np.sqrt(n) * e_a * yscale + e_r * np.linalg.norm(func(x0, n))
+
+            sigma_eps = 1e-10 * min(yscale/xscale, xscale/yscale)
+            sigma_0 = xscale/yscale
+
+            with np.errstate(over='ignore'):
+                sol = root(func, x0, args=args,
+                           options=dict(ftol=0, fatol=fatol, maxfev=problem['nfev'] + 1,
+                                        sigma_0=sigma_0, sigma_eps=sigma_eps,
+                                        line_search=line_search),
+                           method='DF-SANE')
+
+            err_msg = repr([xscale, yscale, line_search, problem, np.linalg.norm(func(sol.x, n)),
+                            fatol, sol.success, sol.nit, sol.nfev])
+            assert_(sol.success, err_msg)
+            assert_(sol.nfev <= problem['nfev'] + 1, err_msg)  # nfev+1: dfsane.f doesn't count first eval
+            assert_(sol.nit <= problem['nit'], err_msg)
+            assert_(np.linalg.norm(func(sol.x, n)) <= fatol, err_msg)
+
+
+def test_complex():
+    def func(z):
+        return z**2 - 1 + 2j
+    x0 = 2.0j
+
+    ftol = 1e-4
+    sol = root(func, x0, tol=ftol, method='DF-SANE')
+
+    assert_(sol.success)
+
+    f0 = np.linalg.norm(func(x0))
+    fx = np.linalg.norm(func(sol.x))
+    assert_(fx <= ftol*f0)
+
+
+def test_linear_definite():
+    # The DF-SANE paper proves convergence for "strongly isolated"
+    # solutions.
+    #
+    # For linear systems F(x) = A x - b = 0, with A positive or
+    # negative definite, the solution is strongly isolated.
+
+    def check_solvability(A, b, line_search='cruz'):
+        func = lambda x: A.dot(x) - b
+        xp = np.linalg.solve(A, b)
+        eps = np.linalg.norm(func(xp)) * 1e3
+        sol = root(func, b, options=dict(fatol=eps, ftol=0, maxfev=17523, line_search=line_search),
+                   method='DF-SANE')
+        assert_(sol.success)
+        assert_(np.linalg.norm(func(sol.x)) <= eps)
+
+    n = 90
+
+    # Test linear pos.def. system
+    np.random.seed(1234)
+    A = np.arange(n*n).reshape(n, n)
+    A = A + n*n * np.diag(1 + np.arange(n))
+    assert_(np.linalg.eigvals(A).min() > 0)
+    b = np.arange(n) * 1.0
+    check_solvability(A, b, 'cruz')
+    check_solvability(A, b, 'cheng')
+
+    # Test linear neg.def. system
+    check_solvability(-A, b, 'cruz')
+    check_solvability(-A, b, 'cheng')
+
+
+def test_shape():
+    def f(x, arg):
+        return x - arg
+
+    for dt in [float, complex]:
+        x = np.zeros([2,2])
+        arg = np.ones([2,2], dtype=dt)
+
+        sol = root(f, x, args=(arg,), method='DF-SANE')
+        assert_(sol.success)
+        assert_equal(sol.x.shape, x.shape)
+
+
+# Some of the test functions and initial guesses listed in
+# [W. La Cruz, M. Raydan. Optimization Methods and Software, 18, 583 (2003)]
+
+def F_1(x, n):
+    g = np.zeros([n])
+    i = np.arange(2, n+1)
+    g[0] = exp(x[0] - 1) - 1
+    g[1:] = i*(exp(x[1:] - 1) - x[1:])
+    return g
+
+def x0_1(n):
+    x0 = np.empty([n])
+    x0.fill(n/(n-1))
+    return x0
+
+def F_2(x, n):
+    g = np.zeros([n])
+    i = np.arange(2, n+1)
+    g[0] = exp(x[0]) - 1
+    g[1:] = 0.1*i*(exp(x[1:]) + x[:-1] - 1)
+    return g
+
+def x0_2(n):
+    x0 = np.empty([n])
+    x0.fill(1/n**2)
+    return x0
+
+def F_4(x, n):
+    assert_equal(n % 3, 0)
+    g = np.zeros([n])
+    # Note: the first line is typoed in some of the references;
+    # correct in original [Gasparo, Optimization Meth. 13, 79 (2000)]
+    g[::3] = 0.6 * x[::3] + 1.6 * x[1::3]**3 - 7.2 * x[1::3]**2 + 9.6 * x[1::3] - 4.8
+    g[1::3] = 0.48 * x[::3] - 0.72 * x[1::3]**3 + 3.24 * x[1::3]**2 - 4.32 * x[1::3] - x[2::3] + 0.2 * x[2::3]**3 + 2.16
+    g[2::3] = 1.25 * x[2::3] - 0.25*x[2::3]**3
+    return g
+
+def x0_4(n):
+    assert_equal(n % 3, 0)
+    x0 = np.array([-1, 1/2, -1] * (n//3))
+    return x0
+
+def F_6(x, n):
+    c = 0.9
+    mu = (np.arange(1, n+1) - 0.5)/n
+    return x - 1/(1 - c/(2*n) * (mu[:,None]*x / (mu[:,None] + mu)).sum(axis=1))
+
+def x0_6(n):
+    return np.ones([n])
+
+def F_7(x, n):
+    assert_equal(n % 3, 0)
+
+    def phi(t):
+        v = 0.5*t - 2
+        v[t > -1] = ((-592*t**3 + 888*t**2 + 4551*t - 1924)/1998)[t > -1]
+        v[t >= 2] = (0.5*t + 2)[t >= 2]
+        return v
+    g = np.zeros([n])
+    g[::3] = 1e4 * x[1::3]**2 - 1
+    g[1::3] = exp(-x[::3]) + exp(-x[1::3]) - 1.0001
+    g[2::3] = phi(x[2::3])
+    return g
+
+def x0_7(n):
+    assert_equal(n % 3, 0)
+    return np.array([1e-3, 18, 1] * (n//3))
+
+def F_9(x, n):
+    g = np.zeros([n])
+    i = np.arange(2, n)
+    g[0] = x[0]**3/3 + x[1]**2/2
+    g[1:-1] = -x[1:-1]**2/2 + i*x[1:-1]**3/3 + x[2:]**2/2
+    g[-1] = -x[-1]**2/2 + n*x[-1]**3/3
+    return g
+
+def x0_9(n):
+    return np.ones([n])
+
+def F_10(x, n):
+    return np.log(1 + x) - x/n
+
+def x0_10(n):
+    return np.ones([n])

venv/Lib/site-packages/scipy/optimize/tests/test_cobyla.py

@@ -0,0 +1,113 @@
+import math
+import numpy as np
+
+from numpy.testing import assert_allclose, assert_
+
+from scipy.optimize import fmin_cobyla, minimize
+
+
+class TestCobyla(object):
+    def setup_method(self):
+        self.x0 = [4.95, 0.66]
+        self.solution = [math.sqrt(25 - (2.0/3)**2), 2.0/3]
+        self.opts = {'disp': False, 'rhobeg': 1, 'tol': 1e-5,
+                     'maxiter': 100}
+
+    def fun(self, x):
+        return x[0]**2 + abs(x[1])**3
+
+    def con1(self, x):
+        return x[0]**2 + x[1]**2 - 25
+
+    def con2(self, x):
+        return -self.con1(x)
+
+    def test_simple(self):
+        # use disp=True as smoke test for gh-8118
+        x = fmin_cobyla(self.fun, self.x0, [self.con1, self.con2], rhobeg=1,
+                        rhoend=1e-5, maxfun=100, disp=True)
+        assert_allclose(x, self.solution, atol=1e-4)
+
+    def test_minimize_simple(self):
+        # Minimize with method='COBYLA'
+        cons = ({'type': 'ineq', 'fun': self.con1},
+                {'type': 'ineq', 'fun': self.con2})
+        sol = minimize(self.fun, self.x0, method='cobyla', constraints=cons,
+                       options=self.opts)
+        assert_allclose(sol.x, self.solution, atol=1e-4)
+        assert_(sol.success, sol.message)
+        assert_(sol.maxcv < 1e-5, sol)
+        assert_(sol.nfev < 70, sol)
+        assert_(sol.fun < self.fun(self.solution) + 1e-3, sol)
+
+    def test_minimize_constraint_violation(self):
+        np.random.seed(1234)
+        pb = np.random.rand(10, 10)
+        spread = np.random.rand(10)
+
+        def p(w):
+            return pb.dot(w)
+
+        def f(w):
+            return -(w * spread).sum()
+
+        def c1(w):
+            return 500 - abs(p(w)).sum()
+
+        def c2(w):
+            return 5 - abs(p(w).sum())
+
+        def c3(w):
+            return 5 - abs(p(w)).max()
+
+        cons = ({'type': 'ineq', 'fun': c1},
+                {'type': 'ineq', 'fun': c2},
+                {'type': 'ineq', 'fun': c3})
+        w0 = np.zeros((10, 1))
+        sol = minimize(f, w0, method='cobyla', constraints=cons,
+                       options={'catol': 1e-6})
+        assert_(sol.maxcv > 1e-6)
+        assert_(not sol.success)
+
+
+def test_vector_constraints():
+    # test that fmin_cobyla and minimize can take a combination
+    # of constraints, some returning a number and others an array
+    def fun(x):
+        return (x[0] - 1)**2 + (x[1] - 2.5)**2
+
+    def fmin(x):
+        return fun(x) - 1
+
+    def cons1(x):
+        a = np.array([[1, -2, 2], [-1, -2, 6], [-1, 2, 2]])
+        return np.array([a[i, 0] * x[0] + a[i, 1] * x[1] +
+                         a[i, 2] for i in range(len(a))])
+
+    def cons2(x):
+        return x     # identity, acts as bounds x > 0
+
+    x0 = np.array([2, 0])
+    cons_list = [fun, cons1, cons2]
+
+    xsol = [1.4, 1.7]
+    fsol = 0.8
+
+    # testing fmin_cobyla
+    sol = fmin_cobyla(fun, x0, cons_list, rhoend=1e-5)
+    assert_allclose(sol, xsol, atol=1e-4)
+
+    sol = fmin_cobyla(fun, x0, fmin, rhoend=1e-5)
+    assert_allclose(fun(sol), 1, atol=1e-4)
+
+    # testing minimize
+    constraints = [{'type': 'ineq', 'fun': cons} for cons in cons_list]
+    sol = minimize(fun, x0, constraints=constraints, tol=1e-5)
+    assert_allclose(sol.x, xsol, atol=1e-4)
+    assert_(sol.success, sol.message)
+    assert_allclose(sol.fun, fsol, atol=1e-4)
+
+    constraints = {'type': 'ineq', 'fun': fmin}
+    sol = minimize(fun, x0, constraints=constraints, tol=1e-5)
+    assert_allclose(sol.fun, 1, atol=1e-4)
+

venv/Lib/site-packages/scipy/optimize/tests/test_constraint_conversion.py

@@ -0,0 +1,267 @@
+"""
+Unit test for constraint conversion
+"""
+
+import numpy as np
+from numpy.testing import (assert_array_almost_equal,
+                           assert_allclose, assert_warns, suppress_warnings)
+import pytest
+from scipy.optimize import (NonlinearConstraint, LinearConstraint,
+                            OptimizeWarning, minimize, BFGS)
+from .test_minimize_constrained import (Maratos, HyperbolicIneq, Rosenbrock,
+                                        IneqRosenbrock, EqIneqRosenbrock,
+                                        BoundedRosenbrock, Elec)
+
+
+class TestOldToNew(object):
+    x0 = (2, 0)
+    bnds = ((0, None), (0, None))
+    method = "trust-constr"
+
+    def test_constraint_dictionary_1(self):
+        fun = lambda x: (x[0] - 1)**2 + (x[1] - 2.5)**2
+        cons = ({'type': 'ineq', 'fun': lambda x: x[0] - 2 * x[1] + 2},
+                {'type': 'ineq', 'fun': lambda x: -x[0] - 2 * x[1] + 6},
+                {'type': 'ineq', 'fun': lambda x: -x[0] + 2 * x[1] + 2})
+
+        with suppress_warnings() as sup:
+            sup.filter(UserWarning, "delta_grad == 0.0")
+            res = minimize(fun, self.x0, method=self.method,
+                           bounds=self.bnds, constraints=cons)
+        assert_allclose(res.x, [1.4, 1.7], rtol=1e-4)
+        assert_allclose(res.fun, 0.8, rtol=1e-4)
+
+    def test_constraint_dictionary_2(self):
+        fun = lambda x: (x[0] - 1)**2 + (x[1] - 2.5)**2
+        cons = {'type': 'eq',
+                'fun': lambda x, p1, p2: p1*x[0] - p2*x[1],
+                'args': (1, 1.1),
+                'jac': lambda x, p1, p2: np.array([[p1, -p2]])}
+        with suppress_warnings() as sup:
+            sup.filter(UserWarning, "delta_grad == 0.0")
+            res = minimize(fun, self.x0, method=self.method,
+                           bounds=self.bnds, constraints=cons)
+        assert_allclose(res.x, [1.7918552, 1.62895927])
+        assert_allclose(res.fun, 1.3857466063348418)
+
+    def test_constraint_dictionary_3(self):
+        fun = lambda x: (x[0] - 1)**2 + (x[1] - 2.5)**2
+        cons = [{'type': 'ineq', 'fun': lambda x: x[0] - 2 * x[1] + 2},
+                NonlinearConstraint(lambda x: x[0] - x[1], 0, 0)]
+
+        with suppress_warnings() as sup:
+            sup.filter(UserWarning, "delta_grad == 0.0")
+            res = minimize(fun, self.x0, method=self.method,
+                           bounds=self.bnds, constraints=cons)
+        assert_allclose(res.x, [1.75, 1.75], rtol=1e-4)
+        assert_allclose(res.fun, 1.125, rtol=1e-4)
+
+
+class TestNewToOld(object):
+
+    def test_multiple_constraint_objects(self):
+        fun = lambda x: (x[0] - 1)**2 + (x[1] - 2.5)**2 + (x[2] - 0.75)**2
+        x0 = [2, 0, 1]
+        coni = []  # only inequality constraints (can use cobyla)
+        methods = ["slsqp", "cobyla", "trust-constr"]
+
+        # mixed old and new
+        coni.append([{'type': 'ineq', 'fun': lambda x: x[0] - 2 * x[1] + 2},
+                     NonlinearConstraint(lambda x: x[0] - x[1], -1, 1)])
+
+        coni.append([LinearConstraint([1, -2, 0], -2, np.inf),
+                     NonlinearConstraint(lambda x: x[0] - x[1], -1, 1)])
+
+        coni.append([NonlinearConstraint(lambda x: x[0] - 2 * x[1] + 2, 0, np.inf),
+                     NonlinearConstraint(lambda x: x[0] - x[1], -1, 1)])
+
+        for con in coni:
+            funs = {}
+            for method in methods:
+                with suppress_warnings() as sup:
+                    sup.filter(UserWarning)
+                    result = minimize(fun, x0, method=method, constraints=con)
+                    funs[method] = result.fun
+            assert_allclose(funs['slsqp'], funs['trust-constr'], rtol=1e-4)
+            assert_allclose(funs['cobyla'], funs['trust-constr'], rtol=1e-4)
+
+    def test_individual_constraint_objects(self):
+        fun = lambda x: (x[0] - 1)**2 + (x[1] - 2.5)**2 + (x[2] - 0.75)**2
+        x0 = [2, 0, 1]
+
+        cone = []  # with equality constraints (can't use cobyla)
+        coni = []  # only inequality constraints (can use cobyla)
+        methods = ["slsqp", "cobyla", "trust-constr"]
+
+        # nonstandard data types for constraint equality bounds
+        cone.append(NonlinearConstraint(lambda x: x[0] - x[1], 1, 1))
+        cone.append(NonlinearConstraint(lambda x: x[0] - x[1], [1.21], [1.21]))
+        cone.append(NonlinearConstraint(lambda x: x[0] - x[1],
+                                        1.21, np.array([1.21])))
+
+        # multiple equalities
+        cone.append(NonlinearConstraint(
+                    lambda x: [x[0] - x[1], x[1] - x[2]],
+                    1.21, 1.21))  # two same equalities
+        cone.append(NonlinearConstraint(
+                    lambda x: [x[0] - x[1], x[1] - x[2]],
+                    [1.21, 1.4], [1.21, 1.4]))  # two different equalities
+        cone.append(NonlinearConstraint(
+                    lambda x: [x[0] - x[1], x[1] - x[2]],
+                    [1.21, 1.21], 1.21))  # equality specified two ways
+        cone.append(NonlinearConstraint(
+                    lambda x: [x[0] - x[1], x[1] - x[2]],
+                    [1.21, -np.inf], [1.21, np.inf]))  # equality + unbounded
+
+        # nonstandard data types for constraint inequality bounds
+        coni.append(NonlinearConstraint(lambda x: x[0] - x[1], 1.21, np.inf))
+        coni.append(NonlinearConstraint(lambda x: x[0] - x[1], [1.21], np.inf))
+        coni.append(NonlinearConstraint(lambda x: x[0] - x[1],
+                                        1.21, np.array([np.inf])))
+        coni.append(NonlinearConstraint(lambda x: x[0] - x[1], -np.inf, -3))
+        coni.append(NonlinearConstraint(lambda x: x[0] - x[1],
+                                        np.array(-np.inf), -3))
+
+        # multiple inequalities/equalities
+        coni.append(NonlinearConstraint(
+                    lambda x: [x[0] - x[1], x[1] - x[2]],
+                    1.21, np.inf))  # two same inequalities
+        cone.append(NonlinearConstraint(
+                    lambda x: [x[0] - x[1], x[1] - x[2]],
+                    [1.21, -np.inf], [1.21, 1.4]))  # mixed equality/inequality
+        coni.append(NonlinearConstraint(
+                    lambda x: [x[0] - x[1], x[1] - x[2]],
+                    [1.1, .8], [1.2, 1.4]))  # bounded above and below
+        coni.append(NonlinearConstraint(
+                    lambda x: [x[0] - x[1], x[1] - x[2]],
+                    [-1.2, -1.4], [-1.1, -.8]))  # - bounded above and below
+
+        # quick check of LinearConstraint class (very little new code to test)
+        cone.append(LinearConstraint([1, -1, 0], 1.21, 1.21))
+        cone.append(LinearConstraint([[1, -1, 0], [0, 1, -1]], 1.21, 1.21))
+        cone.append(LinearConstraint([[1, -1, 0], [0, 1, -1]],
+                                     [1.21, -np.inf], [1.21, 1.4]))
+
+        for con in coni:
+            funs = {}
+            for method in methods:
+                with suppress_warnings() as sup:
+                    sup.filter(UserWarning)
+                    result = minimize(fun, x0, method=method, constraints=con)
+                    funs[method] = result.fun
+            assert_allclose(funs['slsqp'], funs['trust-constr'], rtol=1e-3)
+            assert_allclose(funs['cobyla'], funs['trust-constr'], rtol=1e-3)
+
+        for con in cone:
+            funs = {}
+            for method in methods[::2]:  # skip cobyla
+                with suppress_warnings() as sup:
+                    sup.filter(UserWarning)
+                    result = minimize(fun, x0, method=method, constraints=con)
+                    funs[method] = result.fun
+            assert_allclose(funs['slsqp'], funs['trust-constr'], rtol=1e-3)
+
+
+class TestNewToOldSLSQP(object):
+    method = 'slsqp'
+    elec = Elec(n_electrons=2)
+    elec.x_opt = np.array([-0.58438468, 0.58438466, 0.73597047,
+                           -0.73597044, 0.34180668, -0.34180667])
+    brock = BoundedRosenbrock()
+    brock.x_opt = [0, 0]
+    list_of_problems = [Maratos(),
+                        HyperbolicIneq(),
+                        Rosenbrock(),
+                        IneqRosenbrock(),
+                        EqIneqRosenbrock(),
+                        elec,
+                        brock
+                        ]
+
+    def test_list_of_problems(self):
+
+        for prob in self.list_of_problems:
+
+            with suppress_warnings() as sup:
+                sup.filter(UserWarning)
+                result = minimize(prob.fun, prob.x0,
+                                  method=self.method,
+                                  bounds=prob.bounds,
+                                  constraints=prob.constr)
+
+            assert_array_almost_equal(result.x, prob.x_opt, decimal=3)
+
+    def test_warn_mixed_constraints(self):
+        # warns about inefficiency of mixed equality/inequality constraints
+        fun = lambda x: (x[0] - 1)**2 + (x[1] - 2.5)**2 + (x[2] - 0.75)**2
+        cons = NonlinearConstraint(lambda x: [x[0]**2 - x[1], x[1] - x[2]],
+                                   [1.1, .8], [1.1, 1.4])
+        bnds = ((0, None), (0, None), (0, None))
+        with suppress_warnings() as sup:
+            sup.filter(UserWarning, "delta_grad == 0.0")
+            assert_warns(OptimizeWarning, minimize, fun, (2, 0, 1),
+                         method=self.method, bounds=bnds, constraints=cons)
+
+    def test_warn_ignored_options(self):
+        # warns about constraint options being ignored
+        fun = lambda x: (x[0] - 1)**2 + (x[1] - 2.5)**2 + (x[2] - 0.75)**2
+        x0 = (2, 0, 1)
+
+        if self.method == "slsqp":
+            bnds = ((0, None), (0, None), (0, None))
+        else:
+            bnds = None
+
+        cons = NonlinearConstraint(lambda x: x[0], 2, np.inf)
+        res = minimize(fun, x0, method=self.method,
+                       bounds=bnds, constraints=cons)
+        # no warnings without constraint options
+        assert_allclose(res.fun, 1)
+
+        cons = LinearConstraint([1, 0, 0], 2, np.inf)
+        res = minimize(fun, x0, method=self.method,
+                       bounds=bnds, constraints=cons)
+        # no warnings without constraint options
+        assert_allclose(res.fun, 1)
+
+        cons = []
+        cons.append(NonlinearConstraint(lambda x: x[0]**2, 2, np.inf,
+                                        keep_feasible=True))
+        cons.append(NonlinearConstraint(lambda x: x[0]**2, 2, np.inf,
+                                        hess=BFGS()))
+        cons.append(NonlinearConstraint(lambda x: x[0]**2, 2, np.inf,
+                                        finite_diff_jac_sparsity=42))
+        cons.append(NonlinearConstraint(lambda x: x[0]**2, 2, np.inf,
+                                        finite_diff_rel_step=42))
+        cons.append(LinearConstraint([1, 0, 0], 2, np.inf,
+                                     keep_feasible=True))
+        for con in cons:
+            assert_warns(OptimizeWarning, minimize, fun, x0,
+                         method=self.method, bounds=bnds, constraints=cons)
+
+
+class TestNewToOldCobyla(object):
+    method = 'cobyla'
+
+    list_of_problems = [
+                        Elec(n_electrons=2),
+                        Elec(n_electrons=4),
+                        ]
+
+    @pytest.mark.slow
+    def test_list_of_problems(self):
+
+        for prob in self.list_of_problems:
+
+            with suppress_warnings() as sup:
+                sup.filter(UserWarning)
+                truth = minimize(prob.fun, prob.x0,
+                                 method='trust-constr',
+                                 bounds=prob.bounds,
+                                 constraints=prob.constr)
+                result = minimize(prob.fun, prob.x0,
+                                  method=self.method,
+                                  bounds=prob.bounds,
+                                  constraints=prob.constr)
+
+            assert_allclose(result.fun, truth.fun, rtol=1e-3)

venv/Lib/site-packages/scipy/optimize/tests/test_constraints.py

@@ -0,0 +1,183 @@
+import pytest
+import numpy as np
+from numpy.testing import TestCase, assert_array_equal
+import scipy.sparse as sps
+from scipy.optimize._constraints import (
+    Bounds, LinearConstraint, NonlinearConstraint, PreparedConstraint,
+    new_bounds_to_old, old_bound_to_new, strict_bounds)
+
+
+class TestStrictBounds(TestCase):
+    def test_scalarvalue_unique_enforce_feasibility(self):
+        m = 3
+        lb = 2
+        ub = 4
+        enforce_feasibility = False
+        strict_lb, strict_ub = strict_bounds(lb, ub,
+                                             enforce_feasibility,
+                                             m)
+        assert_array_equal(strict_lb, [-np.inf, -np.inf, -np.inf])
+        assert_array_equal(strict_ub, [np.inf, np.inf, np.inf])
+
+        enforce_feasibility = True
+        strict_lb, strict_ub = strict_bounds(lb, ub,
+                                             enforce_feasibility,
+                                             m)
+        assert_array_equal(strict_lb, [2, 2, 2])
+        assert_array_equal(strict_ub, [4, 4, 4])
+
+    def test_vectorvalue_unique_enforce_feasibility(self):
+        m = 3
+        lb = [1, 2, 3]
+        ub = [4, 5, 6]
+        enforce_feasibility = False
+        strict_lb, strict_ub = strict_bounds(lb, ub,
+                                              enforce_feasibility,
+                                              m)
+        assert_array_equal(strict_lb, [-np.inf, -np.inf, -np.inf])
+        assert_array_equal(strict_ub, [np.inf, np.inf, np.inf])
+
+        enforce_feasibility = True
+        strict_lb, strict_ub = strict_bounds(lb, ub,
+                                              enforce_feasibility,
+                                              m)
+        assert_array_equal(strict_lb, [1, 2, 3])
+        assert_array_equal(strict_ub, [4, 5, 6])
+
+    def test_scalarvalue_vector_enforce_feasibility(self):
+        m = 3
+        lb = 2
+        ub = 4
+        enforce_feasibility = [False, True, False]
+        strict_lb, strict_ub = strict_bounds(lb, ub,
+                                             enforce_feasibility,
+                                             m)
+        assert_array_equal(strict_lb, [-np.inf, 2, -np.inf])
+        assert_array_equal(strict_ub, [np.inf, 4, np.inf])
+
+    def test_vectorvalue_vector_enforce_feasibility(self):
+        m = 3
+        lb = [1, 2, 3]
+        ub = [4, 6, np.inf]
+        enforce_feasibility = [True, False, True]
+        strict_lb, strict_ub = strict_bounds(lb, ub,
+                                             enforce_feasibility,
+                                             m)
+        assert_array_equal(strict_lb, [1, -np.inf, 3])
+        assert_array_equal(strict_ub, [4, np.inf, np.inf])
+
+
+def test_prepare_constraint_infeasible_x0():
+    lb = np.array([0, 20, 30])
+    ub = np.array([0.5, np.inf, 70])
+    x0 = np.array([1, 2, 3])
+    enforce_feasibility = np.array([False, True, True], dtype=bool)
+    bounds = Bounds(lb, ub, enforce_feasibility)
+    pytest.raises(ValueError, PreparedConstraint, bounds, x0)
+
+    pc = PreparedConstraint(Bounds(lb, ub), [1, 2, 3])
+    assert (pc.violation([1, 2, 3]) > 0).any()
+    assert (pc.violation([0.25, 21, 31]) == 0).all()
+
+    x0 = np.array([1, 2, 3, 4])
+    A = np.array([[1, 2, 3, 4], [5, 0, 0, 6], [7, 0, 8, 0]])
+    enforce_feasibility = np.array([True, True, True], dtype=bool)
+    linear = LinearConstraint(A, -np.inf, 0, enforce_feasibility)
+    pytest.raises(ValueError, PreparedConstraint, linear, x0)
+
+    pc = PreparedConstraint(LinearConstraint(A, -np.inf, 0),
+                            [1, 2, 3, 4])
+    assert (pc.violation([1, 2, 3, 4]) > 0).any()
+    assert (pc.violation([-10, 2, -10, 4]) == 0).all()
+
+    def fun(x):
+        return A.dot(x)
+
+    def jac(x):
+        return A
+
+    def hess(x, v):
+        return sps.csr_matrix((4, 4))
+
+    nonlinear = NonlinearConstraint(fun, -np.inf, 0, jac, hess,
+                                    enforce_feasibility)
+    pytest.raises(ValueError, PreparedConstraint, nonlinear, x0)
+
+    pc = PreparedConstraint(nonlinear, [-10, 2, -10, 4])
+    assert (pc.violation([1, 2, 3, 4]) > 0).any()
+    assert (pc.violation([-10, 2, -10, 4]) == 0).all()
+
+
+def test_violation():
+    def cons_f(x):
+        return np.array([x[0] ** 2 + x[1], x[0] ** 2 - x[1]])
+
+    nlc = NonlinearConstraint(cons_f, [-1, -0.8500], [2, 2])
+    pc = PreparedConstraint(nlc, [0.5, 1])
+
+    assert_array_equal(pc.violation([0.5, 1]), [0., 0.])
+
+    np.testing.assert_almost_equal(pc.violation([0.5, 1.2]), [0., 0.1])
+
+    np.testing.assert_almost_equal(pc.violation([1.2, 1.2]), [0.64, 0])
+
+    np.testing.assert_almost_equal(pc.violation([0.1, -1.2]), [0.19, 0])
+
+    np.testing.assert_almost_equal(pc.violation([0.1, 2]), [0.01, 1.14])
+
+
+def test_new_bounds_to_old():
+    lb = np.array([-np.inf, 2, 3])
+    ub = np.array([3, np.inf, 10])
+
+    bounds = [(None, 3), (2, None), (3, 10)]
+    assert_array_equal(new_bounds_to_old(lb, ub, 3), bounds)
+
+    bounds_single_lb = [(-1, 3), (-1, None), (-1, 10)]
+    assert_array_equal(new_bounds_to_old(-1, ub, 3), bounds_single_lb)
+
+    bounds_no_lb = [(None, 3), (None, None), (None, 10)]
+    assert_array_equal(new_bounds_to_old(-np.inf, ub, 3), bounds_no_lb)
+
+    bounds_single_ub = [(None, 20), (2, 20), (3, 20)]
+    assert_array_equal(new_bounds_to_old(lb, 20, 3), bounds_single_ub)
+
+    bounds_no_ub = [(None, None), (2, None), (3, None)]
+    assert_array_equal(new_bounds_to_old(lb, np.inf, 3), bounds_no_ub)
+
+    bounds_single_both = [(1, 2), (1, 2), (1, 2)]
+    assert_array_equal(new_bounds_to_old(1, 2, 3), bounds_single_both)
+
+    bounds_no_both = [(None, None), (None, None), (None, None)]
+    assert_array_equal(new_bounds_to_old(-np.inf, np.inf, 3), bounds_no_both)
+
+
+def test_old_bounds_to_new():
+    bounds = ([1, 2], (None, 3), (-1, None))
+    lb_true = np.array([1, -np.inf, -1])
+    ub_true = np.array([2, 3, np.inf])
+
+    lb, ub = old_bound_to_new(bounds)
+    assert_array_equal(lb, lb_true)
+    assert_array_equal(ub, ub_true)
+
+    bounds = [(-np.inf, np.inf), (np.array([1]), np.array([1]))]
+    lb, ub = old_bound_to_new(bounds)
+
+    assert_array_equal(lb, [-np.inf, 1])
+    assert_array_equal(ub, [np.inf, 1])
+
+
+def test_bounds_repr():
+    from numpy import array, inf  # so that eval works
+    for args in (
+        (-1.0, 5.0),
+        (-1.0, np.inf, True),
+        (np.array([1.0, -np.inf]), np.array([2.0, np.inf])),
+        (np.array([1.0, -np.inf]), np.array([2.0, np.inf]), np.array([True, False])),
+    ):
+        bounds = Bounds(*args)
+        bounds2 = eval(repr(Bounds(*args)))
+        assert_array_equal(bounds.lb, bounds2.lb)
+        assert_array_equal(bounds.ub, bounds2.ub)
+        assert_array_equal(bounds.keep_feasible, bounds2.keep_feasible)

venv/Lib/site-packages/scipy/optimize/tests/test_cython_optimize.py

@@ -0,0 +1,92 @@
+"""
+Test Cython optimize zeros API functions: ``bisect``, ``ridder``, ``brenth``,
+and ``brentq`` in `scipy.optimize.cython_optimize`, by finding the roots of a
+3rd order polynomial given a sequence of constant terms, ``a0``, and fixed 1st,
+2nd, and 3rd order terms in ``args``.
+
+.. math::
+
+    f(x, a0, args) =  ((args[2]*x + args[1])*x + args[0])*x + a0
+
+The 3rd order polynomial function is written in Cython and called in a Python
+wrapper named after the zero function. See the private ``_zeros`` Cython module
+in `scipy.optimize.cython_optimze` for more information.
+"""
+
+import numpy.testing as npt
+from scipy.optimize.cython_optimize import _zeros
+
+# CONSTANTS
+# Solve x**3 - A0 = 0  for A0 = [2.0, 2.1, ..., 2.9].
+# The ARGS have 3 elements just to show how this could be done for any cubic
+# polynomial.
+A0 = tuple(-2.0 - x/10.0 for x in range(10))  # constant term
+ARGS = (0.0, 0.0, 1.0)  # 1st, 2nd, and 3rd order terms
+XLO, XHI = 0.0, 2.0  # first and second bounds of zeros functions
+# absolute and relative tolerances and max iterations for zeros functions
+XTOL, RTOL, MITR = 0.001, 0.001, 10
+EXPECTED = [(-a0) ** (1.0/3.0) for a0 in A0]
+# = [1.2599210498948732,
+#    1.2805791649874942,
+#    1.300591446851387,
+#    1.3200061217959123,
+#    1.338865900164339,
+#    1.3572088082974532,
+#    1.375068867074141,
+#    1.3924766500838337,
+#    1.4094597464129783,
+#    1.4260431471424087]
+
+
+# test bisect
+def test_bisect():
+    npt.assert_allclose(
+        EXPECTED,
+        list(
+            _zeros.loop_example('bisect', A0, ARGS, XLO, XHI, XTOL, RTOL, MITR)
+        ),
+        rtol=RTOL, atol=XTOL
+    )
+
+
+# test ridder
+def test_ridder():
+    npt.assert_allclose(
+        EXPECTED,
+        list(
+            _zeros.loop_example('ridder', A0, ARGS, XLO, XHI, XTOL, RTOL, MITR)
+        ),
+        rtol=RTOL, atol=XTOL
+    )
+
+
+# test brenth
+def test_brenth():
+    npt.assert_allclose(
+        EXPECTED,
+        list(
+            _zeros.loop_example('brenth', A0, ARGS, XLO, XHI, XTOL, RTOL, MITR)
+        ),
+        rtol=RTOL, atol=XTOL
+    )
+
+
+# test brentq
+def test_brentq():
+    npt.assert_allclose(
+        EXPECTED,
+        list(
+            _zeros.loop_example('brentq', A0, ARGS, XLO, XHI, XTOL, RTOL, MITR)
+        ),
+        rtol=RTOL, atol=XTOL
+    )
+
+
+# test brentq with full output
+def test_brentq_full_output():
+    output = _zeros.full_output_example(
+        (A0[0],) + ARGS, XLO, XHI, XTOL, RTOL, MITR)
+    npt.assert_allclose(EXPECTED[0], output['root'], rtol=RTOL, atol=XTOL)
+    npt.assert_equal(6, output['iterations'])
+    npt.assert_equal(7, output['funcalls'])
+    npt.assert_equal(0, output['error_num'])

venv/Lib/site-packages/scipy/optimize/tests/test_differentiable_functions.py

@@ -0,0 +1,695 @@
+import pytest
+import numpy as np
+from numpy.testing import (TestCase, assert_array_almost_equal,
+                           assert_array_equal, assert_, assert_allclose,
+                           assert_equal)
+from scipy.sparse import csr_matrix
+from scipy.sparse.linalg import LinearOperator
+from scipy.optimize._differentiable_functions import (ScalarFunction,
+                                                      VectorFunction,
+                                                      LinearVectorFunction,
+                                                      IdentityVectorFunction)
+from scipy.optimize._hessian_update_strategy import BFGS
+
+
+class ExScalarFunction:
+
+    def __init__(self):
+        self.nfev = 0
+        self.ngev = 0
+        self.nhev = 0
+
+    def fun(self, x):
+        self.nfev += 1
+        return 2*(x[0]**2 + x[1]**2 - 1) - x[0]
+
+    def grad(self, x):
+        self.ngev += 1
+        return np.array([4*x[0]-1, 4*x[1]])
+
+    def hess(self, x):
+        self.nhev += 1
+        return 4*np.eye(2)
+
+
+class TestScalarFunction(TestCase):
+
+    def test_finite_difference_grad(self):
+        ex = ExScalarFunction()
+        nfev = 0
+        ngev = 0
+
+        x0 = [1.0, 0.0]
+        analit = ScalarFunction(ex.fun, x0, (), ex.grad,
+                                ex.hess, None, (-np.inf, np.inf))
+        nfev += 1
+        ngev += 1
+        assert_array_equal(ex.nfev, nfev)
+        assert_array_equal(analit.nfev, nfev)
+        assert_array_equal(ex.ngev, ngev)
+        assert_array_equal(analit.ngev, nfev)
+        approx = ScalarFunction(ex.fun, x0, (), '2-point',
+                                ex.hess, None, (-np.inf, np.inf))
+        nfev += 3
+        ngev += 1
+        assert_array_equal(ex.nfev, nfev)
+        assert_array_equal(analit.nfev+approx.nfev, nfev)
+        assert_array_equal(analit.ngev+approx.ngev, ngev)
+        assert_array_equal(analit.f, approx.f)
+        assert_array_almost_equal(analit.g, approx.g)
+
+        x = [10, 0.3]
+        f_analit = analit.fun(x)
+        g_analit = analit.grad(x)
+        nfev += 1
+        ngev += 1
+        assert_array_equal(ex.nfev, nfev)
+        assert_array_equal(analit.nfev+approx.nfev, nfev)
+        assert_array_equal(analit.ngev+approx.ngev, ngev)
+        f_approx = approx.fun(x)
+        g_approx = approx.grad(x)
+        nfev += 3
+        ngev += 1
+        assert_array_equal(ex.nfev, nfev)
+        assert_array_equal(analit.nfev+approx.nfev, nfev)
+        assert_array_equal(analit.ngev+approx.ngev, ngev)
+        assert_array_almost_equal(f_analit, f_approx)
+        assert_array_almost_equal(g_analit, g_approx)
+
+        x = [2.0, 1.0]
+        g_analit = analit.grad(x)
+        ngev += 1
+        assert_array_equal(ex.nfev, nfev)
+        assert_array_equal(analit.nfev+approx.nfev, nfev)
+        assert_array_equal(analit.ngev+approx.ngev, ngev)
+
+        g_approx = approx.grad(x)
+        nfev += 3
+        ngev += 1
+        assert_array_equal(ex.nfev, nfev)
+        assert_array_equal(analit.nfev+approx.nfev, nfev)
+        assert_array_equal(analit.ngev+approx.ngev, ngev)
+        assert_array_almost_equal(g_analit, g_approx)
+
+        x = [2.5, 0.3]
+        f_analit = analit.fun(x)
+        g_analit = analit.grad(x)
+        nfev += 1
+        ngev += 1
+        assert_array_equal(ex.nfev, nfev)
+        assert_array_equal(analit.nfev+approx.nfev, nfev)
+        assert_array_equal(analit.ngev+approx.ngev, ngev)
+        f_approx = approx.fun(x)
+        g_approx = approx.grad(x)
+        nfev += 3
+        ngev += 1
+        assert_array_equal(ex.nfev, nfev)
+        assert_array_equal(analit.nfev+approx.nfev, nfev)
+        assert_array_equal(analit.ngev+approx.ngev, ngev)
+        assert_array_almost_equal(f_analit, f_approx)
+        assert_array_almost_equal(g_analit, g_approx)
+
+        x = [2, 0.3]
+        f_analit = analit.fun(x)
+        g_analit = analit.grad(x)
+        nfev += 1
+        ngev += 1
+        assert_array_equal(ex.nfev, nfev)
+        assert_array_equal(analit.nfev+approx.nfev, nfev)
+        assert_array_equal(analit.ngev+approx.ngev, ngev)
+        f_approx = approx.fun(x)
+        g_approx = approx.grad(x)
+        nfev += 3
+        ngev += 1
+        assert_array_equal(ex.nfev, nfev)
+        assert_array_equal(analit.nfev+approx.nfev, nfev)
+        assert_array_equal(analit.ngev+approx.ngev, ngev)
+        assert_array_almost_equal(f_analit, f_approx)
+        assert_array_almost_equal(g_analit, g_approx)
+
+    def test_fun_and_grad(self):
+        ex = ExScalarFunction()
+
+        def fg_allclose(x, y):
+            assert_allclose(x[0], y[0])
+            assert_allclose(x[1], y[1])
+
+        # with analytic gradient
+        x0 = [2.0, 0.3]
+        analit = ScalarFunction(ex.fun, x0, (), ex.grad,
+                                ex.hess, None, (-np.inf, np.inf))
+
+        fg = ex.fun(x0), ex.grad(x0)
+        fg_allclose(analit.fun_and_grad(x0), fg)
+        assert(analit.ngev == 1)
+
+        x0[1] = 1.
+        fg = ex.fun(x0), ex.grad(x0)
+        fg_allclose(analit.fun_and_grad(x0), fg)
+
+        # with finite difference gradient
+        x0 = [2.0, 0.3]
+        sf = ScalarFunction(ex.fun, x0, (), '3-point',
+                                ex.hess, None, (-np.inf, np.inf))
+        assert(sf.ngev == 1)
+        fg = ex.fun(x0), ex.grad(x0)
+        fg_allclose(sf.fun_and_grad(x0), fg)
+        assert(sf.ngev == 1)
+
+        x0[1] = 1.
+        fg = ex.fun(x0), ex.grad(x0)
+        fg_allclose(sf.fun_and_grad(x0), fg)
+
+    def test_finite_difference_hess_linear_operator(self):
+        ex = ExScalarFunction()
+        nfev = 0
+        ngev = 0
+        nhev = 0
+
+        x0 = [1.0, 0.0]
+        analit = ScalarFunction(ex.fun, x0, (), ex.grad,
+                                ex.hess, None, (-np.inf, np.inf))
+        nfev += 1
+        ngev += 1
+        nhev += 1
+        assert_array_equal(ex.nfev, nfev)
+        assert_array_equal(analit.nfev, nfev)
+        assert_array_equal(ex.ngev, ngev)
+        assert_array_equal(analit.ngev, ngev)
+        assert_array_equal(ex.nhev, nhev)
+        assert_array_equal(analit.nhev, nhev)
+        approx = ScalarFunction(ex.fun, x0, (), ex.grad,
+                                '2-point', None, (-np.inf, np.inf))
+        assert_(isinstance(approx.H, LinearOperator))
+        for v in ([1.0, 2.0], [3.0, 4.0], [5.0, 2.0]):
+            assert_array_equal(analit.f, approx.f)
+            assert_array_almost_equal(analit.g, approx.g)
+            assert_array_almost_equal(analit.H.dot(v), approx.H.dot(v))
+        nfev += 1
+        ngev += 4
+        assert_array_equal(ex.nfev, nfev)
+        assert_array_equal(analit.nfev+approx.nfev, nfev)
+        assert_array_equal(ex.ngev, ngev)
+        assert_array_equal(analit.ngev+approx.ngev, ngev)
+        assert_array_equal(ex.nhev, nhev)
+        assert_array_equal(analit.nhev+approx.nhev, nhev)
+
+        x = [2.0, 1.0]
+        H_analit = analit.hess(x)
+        nhev += 1
+        assert_array_equal(ex.nfev, nfev)
+        assert_array_equal(analit.nfev+approx.nfev, nfev)
+        assert_array_equal(ex.ngev, ngev)
+        assert_array_equal(analit.ngev+approx.ngev, ngev)
+        assert_array_equal(ex.nhev, nhev)
+        assert_array_equal(analit.nhev+approx.nhev, nhev)
+        H_approx = approx.hess(x)
+        assert_(isinstance(H_approx, LinearOperator))
+        for v in ([1.0, 2.0], [3.0, 4.0], [5.0, 2.0]):
+            assert_array_almost_equal(H_analit.dot(v), H_approx.dot(v))
+        ngev += 4
+        assert_array_equal(ex.nfev, nfev)
+        assert_array_equal(analit.nfev+approx.nfev, nfev)
+        assert_array_equal(ex.ngev, ngev)
+        assert_array_equal(analit.ngev+approx.ngev, ngev)
+        assert_array_equal(ex.nhev, nhev)
+        assert_array_equal(analit.nhev+approx.nhev, nhev)
+
+        x = [2.1, 1.2]
+        H_analit = analit.hess(x)
+        nhev += 1
+        assert_array_equal(ex.nfev, nfev)
+        assert_array_equal(analit.nfev+approx.nfev, nfev)
+        assert_array_equal(ex.ngev, ngev)
+        assert_array_equal(analit.ngev+approx.ngev, ngev)
+        assert_array_equal(ex.nhev, nhev)
+        assert_array_equal(analit.nhev+approx.nhev, nhev)
+        H_approx = approx.hess(x)
+        assert_(isinstance(H_approx, LinearOperator))
+        for v in ([1.0, 2.0], [3.0, 4.0], [5.0, 2.0]):
+            assert_array_almost_equal(H_analit.dot(v), H_approx.dot(v))
+        ngev += 4
+        assert_array_equal(ex.nfev, nfev)
+        assert_array_equal(analit.nfev+approx.nfev, nfev)
+        assert_array_equal(ex.ngev, ngev)
+        assert_array_equal(analit.ngev+approx.ngev, ngev)
+        assert_array_equal(ex.nhev, nhev)
+        assert_array_equal(analit.nhev+approx.nhev, nhev)
+
+        x = [2.5, 0.3]
+        _ = analit.grad(x)
+        H_analit = analit.hess(x)
+        ngev += 1
+        nhev += 1
+        assert_array_equal(ex.nfev, nfev)
+        assert_array_equal(analit.nfev+approx.nfev, nfev)
+        assert_array_equal(ex.ngev, ngev)
+        assert_array_equal(analit.ngev+approx.ngev, ngev)
+        assert_array_equal(ex.nhev, nhev)
+        assert_array_equal(analit.nhev+approx.nhev, nhev)
+        _ = approx.grad(x)
+        H_approx = approx.hess(x)
+        assert_(isinstance(H_approx, LinearOperator))
+        for v in ([1.0, 2.0], [3.0, 4.0], [5.0, 2.0]):
+            assert_array_almost_equal(H_analit.dot(v), H_approx.dot(v))
+        ngev += 4
+        assert_array_equal(ex.nfev, nfev)
+        assert_array_equal(analit.nfev+approx.nfev, nfev)
+        assert_array_equal(ex.ngev, ngev)
+        assert_array_equal(analit.ngev+approx.ngev, ngev)
+        assert_array_equal(ex.nhev, nhev)
+        assert_array_equal(analit.nhev+approx.nhev, nhev)
+
+        x = [5.2, 2.3]
+        _ = analit.grad(x)
+        H_analit = analit.hess(x)
+        ngev += 1
+        nhev += 1
+        assert_array_equal(ex.nfev, nfev)
+        assert_array_equal(analit.nfev+approx.nfev, nfev)
+        assert_array_equal(ex.ngev, ngev)
+        assert_array_equal(analit.ngev+approx.ngev, ngev)
+        assert_array_equal(ex.nhev, nhev)
+        assert_array_equal(analit.nhev+approx.nhev, nhev)
+        _ = approx.grad(x)
+        H_approx = approx.hess(x)
+        assert_(isinstance(H_approx, LinearOperator))
+        for v in ([1.0, 2.0], [3.0, 4.0], [5.0, 2.0]):
+            assert_array_almost_equal(H_analit.dot(v), H_approx.dot(v))
+        ngev += 4
+        assert_array_equal(ex.nfev, nfev)
+        assert_array_equal(analit.nfev+approx.nfev, nfev)
+        assert_array_equal(ex.ngev, ngev)
+        assert_array_equal(analit.ngev+approx.ngev, ngev)
+        assert_array_equal(ex.nhev, nhev)
+        assert_array_equal(analit.nhev+approx.nhev, nhev)
+
+    def test_x_storage_overlap(self):
+        # Scalar_Function should not store references to arrays, it should
+        # store copies - this checks that updating an array in-place causes
+        # Scalar_Function.x to be updated.
+
+        def f(x):
+            return np.sum(np.asarray(x) ** 2)
+
+        x = np.array([1., 2., 3.])
+        sf = ScalarFunction(f, x, (), '3-point', lambda x: x, None, (-np.inf, np.inf))
+
+        assert x is not sf.x
+        assert_equal(sf.fun(x), 14.0)
+        assert x is not sf.x
+
+        x[0] = 0.
+        f1 = sf.fun(x)
+        assert_equal(f1, 13.0)
+
+        x[0] = 1
+        f2 = sf.fun(x)
+        assert_equal(f2, 14.0)
+        assert x is not sf.x
+
+        # now test with a HessianUpdate strategy specified
+        hess = BFGS()
+        x = np.array([1., 2., 3.])
+        sf = ScalarFunction(f, x, (), '3-point', hess, None, (-np.inf, np.inf))
+
+        assert x is not sf.x
+        assert_equal(sf.fun(x), 14.0)
+        assert x is not sf.x
+
+        x[0] = 0.
+        f1 = sf.fun(x)
+        assert_equal(f1, 13.0)
+
+        x[0] = 1
+        f2 = sf.fun(x)
+        assert_equal(f2, 14.0)
+        assert x is not sf.x
+
+
+class ExVectorialFunction:
+
+    def __init__(self):
+        self.nfev = 0
+        self.njev = 0
+        self.nhev = 0
+
+    def fun(self, x):
+        self.nfev += 1
+        return np.array([2*(x[0]**2 + x[1]**2 - 1) - x[0],
+                         4*(x[0]**3 + x[1]**2 - 4) - 3*x[0]])
+
+    def jac(self, x):
+        self.njev += 1
+        return np.array([[4*x[0]-1, 4*x[1]],
+                         [12*x[0]**2-3, 8*x[1]]])
+
+    def hess(self, x, v):
+        self.nhev += 1
+        return v[0]*4*np.eye(2) + v[1]*np.array([[24*x[0], 0],
+                                                 [0, 8]])
+
+
+class TestVectorialFunction(TestCase):
+
+    def test_finite_difference_jac(self):
+        ex = ExVectorialFunction()
+        nfev = 0
+        njev = 0
+
+        x0 = [1.0, 0.0]
+        analit = VectorFunction(ex.fun, x0, ex.jac, ex.hess, None, None,
+                                (-np.inf, np.inf), None)
+        nfev += 1
+        njev += 1
+        assert_array_equal(ex.nfev, nfev)
+        assert_array_equal(analit.nfev, nfev)
+        assert_array_equal(ex.njev, njev)
+        assert_array_equal(analit.njev, njev)
+        approx = VectorFunction(ex.fun, x0, '2-point', ex.hess, None, None,
+                                (-np.inf, np.inf), None)
+        nfev += 3
+        assert_array_equal(ex.nfev, nfev)
+        assert_array_equal(analit.nfev+approx.nfev, nfev)
+        assert_array_equal(ex.njev, njev)
+        assert_array_equal(analit.njev+approx.njev, njev)
+        assert_array_equal(analit.f, approx.f)
+        assert_array_almost_equal(analit.J, approx.J)
+
+        x = [10, 0.3]
+        f_analit = analit.fun(x)
+        J_analit = analit.jac(x)
+        nfev += 1
+        njev += 1
+        assert_array_equal(ex.nfev, nfev)
+        assert_array_equal(analit.nfev+approx.nfev, nfev)
+        assert_array_equal(ex.njev, njev)
+        assert_array_equal(analit.njev+approx.njev, njev)
+        f_approx = approx.fun(x)
+        J_approx = approx.jac(x)
+        nfev += 3
+        assert_array_equal(ex.nfev, nfev)
+        assert_array_equal(analit.nfev+approx.nfev, nfev)
+        assert_array_equal(ex.njev, njev)
+        assert_array_equal(analit.njev+approx.njev, njev)
+        assert_array_almost_equal(f_analit, f_approx)
+        assert_array_almost_equal(J_analit, J_approx, decimal=4)
+
+        x = [2.0, 1.0]
+        J_analit = analit.jac(x)
+        njev += 1
+        assert_array_equal(ex.nfev, nfev)
+        assert_array_equal(analit.nfev+approx.nfev, nfev)
+        assert_array_equal(ex.njev, njev)
+        assert_array_equal(analit.njev+approx.njev, njev)
+        J_approx = approx.jac(x)
+        nfev += 3
+        assert_array_equal(ex.nfev, nfev)
+        assert_array_equal(analit.nfev+approx.nfev, nfev)
+        assert_array_equal(ex.njev, njev)
+        assert_array_equal(analit.njev+approx.njev, njev)
+        assert_array_almost_equal(J_analit, J_approx)
+
+        x = [2.5, 0.3]
+        f_analit = analit.fun(x)
+        J_analit = analit.jac(x)
+        nfev += 1
+        njev += 1
+        assert_array_equal(ex.nfev, nfev)
+        assert_array_equal(analit.nfev+approx.nfev, nfev)
+        assert_array_equal(ex.njev, njev)
+        assert_array_equal(analit.njev+approx.njev, njev)
+        f_approx = approx.fun(x)
+        J_approx = approx.jac(x)
+        nfev += 3
+        assert_array_equal(ex.nfev, nfev)
+        assert_array_equal(analit.nfev+approx.nfev, nfev)
+        assert_array_equal(ex.njev, njev)
+        assert_array_equal(analit.njev+approx.njev, njev)
+        assert_array_almost_equal(f_analit, f_approx)
+        assert_array_almost_equal(J_analit, J_approx)
+
+        x = [2, 0.3]
+        f_analit = analit.fun(x)
+        J_analit = analit.jac(x)
+        nfev += 1
+        njev += 1
+        assert_array_equal(ex.nfev, nfev)
+        assert_array_equal(analit.nfev+approx.nfev, nfev)
+        assert_array_equal(ex.njev, njev)
+        assert_array_equal(analit.njev+approx.njev, njev)
+        f_approx = approx.fun(x)
+        J_approx = approx.jac(x)
+        nfev += 3
+        assert_array_equal(ex.nfev, nfev)
+        assert_array_equal(analit.nfev+approx.nfev, nfev)
+        assert_array_equal(ex.njev, njev)
+        assert_array_equal(analit.njev+approx.njev, njev)
+        assert_array_almost_equal(f_analit, f_approx)
+        assert_array_almost_equal(J_analit, J_approx)
+
+    def test_finite_difference_hess_linear_operator(self):
+        ex = ExVectorialFunction()
+        nfev = 0
+        njev = 0
+        nhev = 0
+
+        x0 = [1.0, 0.0]
+        v0 = [1.0, 2.0]
+        analit = VectorFunction(ex.fun, x0, ex.jac, ex.hess, None, None,
+                                (-np.inf, np.inf), None)
+        nfev += 1
+        njev += 1
+        nhev += 1
+        assert_array_equal(ex.nfev, nfev)
+        assert_array_equal(analit.nfev, nfev)
+        assert_array_equal(ex.njev, njev)
+        assert_array_equal(analit.njev, njev)
+        assert_array_equal(ex.nhev, nhev)
+        assert_array_equal(analit.nhev, nhev)
+        approx = VectorFunction(ex.fun, x0, ex.jac, '2-point', None, None,
+                                (-np.inf, np.inf), None)
+        assert_(isinstance(approx.H, LinearOperator))
+        for p in ([1.0, 2.0], [3.0, 4.0], [5.0, 2.0]):
+            assert_array_equal(analit.f, approx.f)
+            assert_array_almost_equal(analit.J, approx.J)
+            assert_array_almost_equal(analit.H.dot(p), approx.H.dot(p))
+        nfev += 1
+        njev += 4
+        assert_array_equal(ex.nfev, nfev)
+        assert_array_equal(analit.nfev+approx.nfev, nfev)
+        assert_array_equal(ex.njev, njev)
+        assert_array_equal(analit.njev+approx.njev, njev)
+        assert_array_equal(ex.nhev, nhev)
+        assert_array_equal(analit.nhev+approx.nhev, nhev)
+
+        x = [2.0, 1.0]
+        H_analit = analit.hess(x, v0)
+        nhev += 1
+        assert_array_equal(ex.nfev, nfev)
+        assert_array_equal(analit.nfev+approx.nfev, nfev)
+        assert_array_equal(ex.njev, njev)
+        assert_array_equal(analit.njev+approx.njev, njev)
+        assert_array_equal(ex.nhev, nhev)
+        assert_array_equal(analit.nhev+approx.nhev, nhev)
+        H_approx = approx.hess(x, v0)
+        assert_(isinstance(H_approx, LinearOperator))
+        for p in ([1.0, 2.0], [3.0, 4.0], [5.0, 2.0]):
+            assert_array_almost_equal(H_analit.dot(p), H_approx.dot(p),
+                                      decimal=5)
+        njev += 4
+        assert_array_equal(ex.nfev, nfev)
+        assert_array_equal(analit.nfev+approx.nfev, nfev)
+        assert_array_equal(ex.njev, njev)
+        assert_array_equal(analit.njev+approx.njev, njev)
+        assert_array_equal(ex.nhev, nhev)
+        assert_array_equal(analit.nhev+approx.nhev, nhev)
+
+        x = [2.1, 1.2]
+        v = [1.0, 1.0]
+        H_analit = analit.hess(x, v)
+        nhev += 1
+        assert_array_equal(ex.nfev, nfev)
+        assert_array_equal(analit.nfev+approx.nfev, nfev)
+        assert_array_equal(ex.njev, njev)
+        assert_array_equal(analit.njev+approx.njev, njev)
+        assert_array_equal(ex.nhev, nhev)
+        assert_array_equal(analit.nhev+approx.nhev, nhev)
+        H_approx = approx.hess(x, v)
+        assert_(isinstance(H_approx, LinearOperator))
+        for v in ([1.0, 2.0], [3.0, 4.0], [5.0, 2.0]):
+            assert_array_almost_equal(H_analit.dot(v), H_approx.dot(v))
+        njev += 4
+        assert_array_equal(ex.nfev, nfev)
+        assert_array_equal(analit.nfev+approx.nfev, nfev)
+        assert_array_equal(ex.njev, njev)
+        assert_array_equal(analit.njev+approx.njev, njev)
+        assert_array_equal(ex.nhev, nhev)
+        assert_array_equal(analit.nhev+approx.nhev, nhev)
+
+        x = [2.5, 0.3]
+        _ = analit.jac(x)
+        H_analit = analit.hess(x, v0)
+        njev += 1
+        nhev += 1
+        assert_array_equal(ex.nfev, nfev)
+        assert_array_equal(analit.nfev+approx.nfev, nfev)
+        assert_array_equal(ex.njev, njev)
+        assert_array_equal(analit.njev+approx.njev, njev)
+        assert_array_equal(ex.nhev, nhev)
+        assert_array_equal(analit.nhev+approx.nhev, nhev)
+        _ = approx.jac(x)
+        H_approx = approx.hess(x, v0)
+        assert_(isinstance(H_approx, LinearOperator))
+        for v in ([1.0, 2.0], [3.0, 4.0], [5.0, 2.0]):
+            assert_array_almost_equal(H_analit.dot(v), H_approx.dot(v), decimal=4)
+        njev += 4
+        assert_array_equal(ex.nfev, nfev)
+        assert_array_equal(analit.nfev+approx.nfev, nfev)
+        assert_array_equal(ex.njev, njev)
+        assert_array_equal(analit.njev+approx.njev, njev)
+        assert_array_equal(ex.nhev, nhev)
+        assert_array_equal(analit.nhev+approx.nhev, nhev)
+
+        x = [5.2, 2.3]
+        v = [2.3, 5.2]
+        _ = analit.jac(x)
+        H_analit = analit.hess(x, v)
+        njev += 1
+        nhev += 1
+        assert_array_equal(ex.nfev, nfev)
+        assert_array_equal(analit.nfev+approx.nfev, nfev)
+        assert_array_equal(ex.njev, njev)
+        assert_array_equal(analit.njev+approx.njev, njev)
+        assert_array_equal(ex.nhev, nhev)
+        assert_array_equal(analit.nhev+approx.nhev, nhev)
+        _ = approx.jac(x)
+        H_approx = approx.hess(x, v)
+        assert_(isinstance(H_approx, LinearOperator))
+        for v in ([1.0, 2.0], [3.0, 4.0], [5.0, 2.0]):
+            assert_array_almost_equal(H_analit.dot(v), H_approx.dot(v), decimal=4)
+        njev += 4
+        assert_array_equal(ex.nfev, nfev)
+        assert_array_equal(analit.nfev+approx.nfev, nfev)
+        assert_array_equal(ex.njev, njev)
+        assert_array_equal(analit.njev+approx.njev, njev)
+        assert_array_equal(ex.nhev, nhev)
+        assert_array_equal(analit.nhev+approx.nhev, nhev)
+
+    def test_x_storage_overlap(self):
+        # VectorFunction should not store references to arrays, it should
+        # store copies - this checks that updating an array in-place causes
+        # Scalar_Function.x to be updated.
+        ex = ExVectorialFunction()
+        x0 = np.array([1.0, 0.0])
+
+        vf = VectorFunction(ex.fun, x0, '3-point', ex.hess, None, None,
+                            (-np.inf, np.inf), None)
+
+        assert x0 is not vf.x
+        assert_equal(vf.fun(x0), ex.fun(x0))
+        assert x0 is not vf.x
+
+        x0[0] = 2.
+        assert_equal(vf.fun(x0), ex.fun(x0))
+        assert x0 is not vf.x
+
+        x0[0] = 1.
+        assert_equal(vf.fun(x0), ex.fun(x0))
+        assert x0 is not vf.x
+
+        # now test with a HessianUpdate strategy specified
+        hess = BFGS()
+        x0 = np.array([1.0, 0.0])
+        vf = VectorFunction(ex.fun, x0, '3-point', hess, None, None,
+                            (-np.inf, np.inf), None)
+
+        with pytest.warns(UserWarning):
+            # filter UserWarning because ExVectorialFunction is linear and
+            # a quasi-Newton approximation is used for the Hessian.
+            assert x0 is not vf.x
+            assert_equal(vf.fun(x0), ex.fun(x0))
+            assert x0 is not vf.x
+
+            x0[0] = 2.
+            assert_equal(vf.fun(x0), ex.fun(x0))
+            assert x0 is not vf.x
+
+            x0[0] = 1.
+            assert_equal(vf.fun(x0), ex.fun(x0))
+            assert x0 is not vf.x
+
+
+def test_LinearVectorFunction():
+    A_dense = np.array([
+        [-1, 2, 0],
+        [0, 4, 2]
+    ])
+    x0 = np.zeros(3)
+    A_sparse = csr_matrix(A_dense)
+    x = np.array([1, -1, 0])
+    v = np.array([-1, 1])
+    Ax = np.array([-3, -4])
+
+    f1 = LinearVectorFunction(A_dense, x0, None)
+    assert_(not f1.sparse_jacobian)
+
+    f2 = LinearVectorFunction(A_dense, x0, True)
+    assert_(f2.sparse_jacobian)
+
+    f3 = LinearVectorFunction(A_dense, x0, False)
+    assert_(not f3.sparse_jacobian)
+
+    f4 = LinearVectorFunction(A_sparse, x0, None)
+    assert_(f4.sparse_jacobian)
+
+    f5 = LinearVectorFunction(A_sparse, x0, True)
+    assert_(f5.sparse_jacobian)
+
+    f6 = LinearVectorFunction(A_sparse, x0, False)
+    assert_(not f6.sparse_jacobian)
+
+    assert_array_equal(f1.fun(x), Ax)
+    assert_array_equal(f2.fun(x), Ax)
+    assert_array_equal(f1.jac(x), A_dense)
+    assert_array_equal(f2.jac(x).toarray(), A_sparse.toarray())
+    assert_array_equal(f1.hess(x, v).toarray(), np.zeros((3, 3)))
+
+
+def test_LinearVectorFunction_memoization():
+    A = np.array([[-1, 2, 0], [0, 4, 2]])
+    x0 = np.array([1, 2, -1])
+    fun = LinearVectorFunction(A, x0, False)
+
+    assert_array_equal(x0, fun.x)
+    assert_array_equal(A.dot(x0), fun.f)
+
+    x1 = np.array([-1, 3, 10])
+    assert_array_equal(A, fun.jac(x1))
+    assert_array_equal(x1, fun.x)
+    assert_array_equal(A.dot(x0), fun.f)
+    assert_array_equal(A.dot(x1), fun.fun(x1))
+    assert_array_equal(A.dot(x1), fun.f)
+
+
+def test_IdentityVectorFunction():
+    x0 = np.zeros(3)
+
+    f1 = IdentityVectorFunction(x0, None)
+    f2 = IdentityVectorFunction(x0, False)
+    f3 = IdentityVectorFunction(x0, True)
+
+    assert_(f1.sparse_jacobian)
+    assert_(not f2.sparse_jacobian)
+    assert_(f3.sparse_jacobian)
+
+    x = np.array([-1, 2, 1])
+    v = np.array([-2, 3, 0])
+
+    assert_array_equal(f1.fun(x), x)
+    assert_array_equal(f2.fun(x), x)
+
+    assert_array_equal(f1.jac(x).toarray(), np.eye(3))
+    assert_array_equal(f2.jac(x), np.eye(3))
+
+    assert_array_equal(f1.hess(x, v).toarray(), np.zeros((3, 3)))

venv/Lib/site-packages/scipy/optimize/tests/test_hessian_update_strategy.py

@@ -0,0 +1,212 @@
+import numpy as np
+from copy import deepcopy
+from numpy.linalg import norm
+from numpy.testing import (TestCase, assert_array_almost_equal,
+                           assert_array_equal, assert_array_less)
+from scipy.optimize import (BFGS, SR1)
+
+
+class Rosenbrock:
+    """Rosenbrock function.
+
+    The following optimization problem:
+        minimize sum(100.0*(x[1:] - x[:-1]**2.0)**2.0 + (1 - x[:-1])**2.0)
+    """
+
+    def __init__(self, n=2, random_state=0):
+        rng = np.random.RandomState(random_state)
+        self.x0 = rng.uniform(-1, 1, n)
+        self.x_opt = np.ones(n)
+
+    def fun(self, x):
+        x = np.asarray(x)
+        r = np.sum(100.0 * (x[1:] - x[:-1]**2.0)**2.0 + (1 - x[:-1])**2.0,
+                   axis=0)
+        return r
+
+    def grad(self, x):
+        x = np.asarray(x)
+        xm = x[1:-1]
+        xm_m1 = x[:-2]
+        xm_p1 = x[2:]
+        der = np.zeros_like(x)
+        der[1:-1] = (200 * (xm - xm_m1**2) -
+                     400 * (xm_p1 - xm**2) * xm - 2 * (1 - xm))
+        der[0] = -400 * x[0] * (x[1] - x[0]**2) - 2 * (1 - x[0])
+        der[-1] = 200 * (x[-1] - x[-2]**2)
+        return der
+
+    def hess(self, x):
+        x = np.atleast_1d(x)
+        H = np.diag(-400 * x[:-1], 1) - np.diag(400 * x[:-1], -1)
+        diagonal = np.zeros(len(x), dtype=x.dtype)
+        diagonal[0] = 1200 * x[0]**2 - 400 * x[1] + 2
+        diagonal[-1] = 200
+        diagonal[1:-1] = 202 + 1200 * x[1:-1]**2 - 400 * x[2:]
+        H = H + np.diag(diagonal)
+        return H
+
+
+class TestHessianUpdateStrategy(TestCase):
+
+    def test_hessian_initialization(self):
+        quasi_newton = (BFGS(), SR1())
+
+        for qn in quasi_newton:
+            qn.initialize(5, 'hess')
+            B = qn.get_matrix()
+
+            assert_array_equal(B, np.eye(5))
+
+    # For this list of points, it is known
+    # that no exception occur during the
+    # Hessian update. Hence no update is
+    # skiped or damped.
+    def test_rosenbrock_with_no_exception(self):
+        # Define auxiliar problem
+        prob = Rosenbrock(n=5)
+        # Define iteration points
+        x_list = [[0.0976270, 0.4303787, 0.2055267, 0.0897663, -0.15269040],
+                  [0.1847239, 0.0505757, 0.2123832, 0.0255081, 0.00083286],
+                  [0.2142498, -0.0188480, 0.0503822, 0.0347033, 0.03323606],
+                  [0.2071680, -0.0185071, 0.0341337, -0.0139298, 0.02881750],
+                  [0.1533055, -0.0322935, 0.0280418, -0.0083592, 0.01503699],
+                  [0.1382378, -0.0276671, 0.0266161, -0.0074060, 0.02801610],
+                  [0.1651957, -0.0049124, 0.0269665, -0.0040025, 0.02138184],
+                  [0.2354930, 0.0443711, 0.0173959, 0.0041872, 0.00794563],
+                  [0.4168118, 0.1433867, 0.0111714, 0.0126265, -0.00658537],
+                  [0.4681972, 0.2153273, 0.0225249, 0.0152704, -0.00463809],
+                  [0.6023068, 0.3346815, 0.0731108, 0.0186618, -0.00371541],
+                  [0.6415743, 0.3985468, 0.1324422, 0.0214160, -0.00062401],
+                  [0.7503690, 0.5447616, 0.2804541, 0.0539851, 0.00242230],
+                  [0.7452626, 0.5644594, 0.3324679, 0.0865153, 0.00454960],
+                  [0.8059782, 0.6586838, 0.4229577, 0.1452990, 0.00976702],
+                  [0.8549542, 0.7226562, 0.4991309, 0.2420093, 0.02772661],
+                  [0.8571332, 0.7285741, 0.5279076, 0.2824549, 0.06030276],
+                  [0.8835633, 0.7727077, 0.5957984, 0.3411303, 0.09652185],
+                  [0.9071558, 0.8299587, 0.6771400, 0.4402896, 0.17469338],
+                  [0.9190793, 0.8486480, 0.7163332, 0.5083780, 0.26107691],
+                  [0.9371223, 0.8762177, 0.7653702, 0.5773109, 0.32181041],
+                  [0.9554613, 0.9119893, 0.8282687, 0.6776178, 0.43162744],
+                  [0.9545744, 0.9099264, 0.8270244, 0.6822220, 0.45237623],
+                  [0.9688112, 0.9351710, 0.8730961, 0.7546601, 0.56622448],
+                  [0.9743227, 0.9491953, 0.9005150, 0.8086497, 0.64505437],
+                  [0.9807345, 0.9638853, 0.9283012, 0.8631675, 0.73812581],
+                  [0.9886746, 0.9777760, 0.9558950, 0.9123417, 0.82726553],
+                  [0.9899096, 0.9803828, 0.9615592, 0.9255600, 0.85822149],
+                  [0.9969510, 0.9935441, 0.9864657, 0.9726775, 0.94358663],
+                  [0.9979533, 0.9960274, 0.9921724, 0.9837415, 0.96626288],
+                  [0.9995981, 0.9989171, 0.9974178, 0.9949954, 0.99023356],
+                  [1.0002640, 1.0005088, 1.0010594, 1.0021161, 1.00386912],
+                  [0.9998903, 0.9998459, 0.9997795, 0.9995484, 0.99916305],
+                  [1.0000008, 0.9999905, 0.9999481, 0.9998903, 0.99978047],
+                  [1.0000004, 0.9999983, 1.0000001, 1.0000031, 1.00000297],
+                  [0.9999995, 1.0000003, 1.0000005, 1.0000001, 1.00000032],
+                  [0.9999999, 0.9999997, 0.9999994, 0.9999989, 0.99999786],
+                  [0.9999999, 0.9999999, 0.9999999, 0.9999999, 0.99999991]]
+        # Get iteration points
+        grad_list = [prob.grad(x) for x in x_list]
+        delta_x = [np.array(x_list[i+1])-np.array(x_list[i])
+                   for i in range(len(x_list)-1)]
+        delta_grad = [grad_list[i+1]-grad_list[i]
+                      for i in range(len(grad_list)-1)]
+        # Check curvature condition
+        for i in range(len(delta_x)):
+            s = delta_x[i]
+            y = delta_grad[i]
+            if np.dot(s, y) <= 0:
+                raise ArithmeticError()
+        # Define QuasiNewton update
+        for quasi_newton in (BFGS(init_scale=1, min_curvature=1e-4),
+                             SR1(init_scale=1)):
+            hess = deepcopy(quasi_newton)
+            inv_hess = deepcopy(quasi_newton)
+            hess.initialize(len(x_list[0]), 'hess')
+            inv_hess.initialize(len(x_list[0]), 'inv_hess')
+            # Compare the hessian and its inverse
+            for i in range(len(delta_x)):
+                s = delta_x[i]
+                y = delta_grad[i]
+                hess.update(s, y)
+                inv_hess.update(s, y)
+                B = hess.get_matrix()
+                H = inv_hess.get_matrix()
+                assert_array_almost_equal(np.linalg.inv(B), H, decimal=10)
+            B_true = prob.hess(x_list[i+1])
+            assert_array_less(norm(B - B_true)/norm(B_true), 0.1)
+
+    def test_SR1_skip_update(self):
+        # Define auxiliary problem
+        prob = Rosenbrock(n=5)
+        # Define iteration points
+        x_list = [[0.0976270, 0.4303787, 0.2055267, 0.0897663, -0.15269040],
+                  [0.1847239, 0.0505757, 0.2123832, 0.0255081, 0.00083286],
+                  [0.2142498, -0.0188480, 0.0503822, 0.0347033, 0.03323606],
+                  [0.2071680, -0.0185071, 0.0341337, -0.0139298, 0.02881750],
+                  [0.1533055, -0.0322935, 0.0280418, -0.0083592, 0.01503699],
+                  [0.1382378, -0.0276671, 0.0266161, -0.0074060, 0.02801610],
+                  [0.1651957, -0.0049124, 0.0269665, -0.0040025, 0.02138184],
+                  [0.2354930, 0.0443711, 0.0173959, 0.0041872, 0.00794563],
+                  [0.4168118, 0.1433867, 0.0111714, 0.0126265, -0.00658537],
+                  [0.4681972, 0.2153273, 0.0225249, 0.0152704, -0.00463809],
+                  [0.6023068, 0.3346815, 0.0731108, 0.0186618, -0.00371541],
+                  [0.6415743, 0.3985468, 0.1324422, 0.0214160, -0.00062401],
+                  [0.7503690, 0.5447616, 0.2804541, 0.0539851, 0.00242230],
+                  [0.7452626, 0.5644594, 0.3324679, 0.0865153, 0.00454960],
+                  [0.8059782, 0.6586838, 0.4229577, 0.1452990, 0.00976702],
+                  [0.8549542, 0.7226562, 0.4991309, 0.2420093, 0.02772661],
+                  [0.8571332, 0.7285741, 0.5279076, 0.2824549, 0.06030276],
+                  [0.8835633, 0.7727077, 0.5957984, 0.3411303, 0.09652185],
+                  [0.9071558, 0.8299587, 0.6771400, 0.4402896, 0.17469338]]
+        # Get iteration points
+        grad_list = [prob.grad(x) for x in x_list]
+        delta_x = [np.array(x_list[i+1])-np.array(x_list[i])
+                   for i in range(len(x_list)-1)]
+        delta_grad = [grad_list[i+1]-grad_list[i]
+                      for i in range(len(grad_list)-1)]
+        hess = SR1(init_scale=1, min_denominator=1e-2)
+        hess.initialize(len(x_list[0]), 'hess')
+        # Compare the Hessian and its inverse
+        for i in range(len(delta_x)-1):
+            s = delta_x[i]
+            y = delta_grad[i]
+            hess.update(s, y)
+        # Test skip update
+        B = np.copy(hess.get_matrix())
+        s = delta_x[17]
+        y = delta_grad[17]
+        hess.update(s, y)
+        B_updated = np.copy(hess.get_matrix())
+        assert_array_equal(B, B_updated)
+
+    def test_BFGS_skip_update(self):
+        # Define auxiliar problem
+        prob = Rosenbrock(n=5)
+        # Define iteration points
+        x_list = [[0.0976270, 0.4303787, 0.2055267, 0.0897663, -0.15269040],
+                  [0.1847239, 0.0505757, 0.2123832, 0.0255081, 0.00083286],
+                  [0.2142498, -0.0188480, 0.0503822, 0.0347033, 0.03323606],
+                  [0.2071680, -0.0185071, 0.0341337, -0.0139298, 0.02881750],
+                  [0.1533055, -0.0322935, 0.0280418, -0.0083592, 0.01503699],
+                  [0.1382378, -0.0276671, 0.0266161, -0.0074060, 0.02801610],
+                  [0.1651957, -0.0049124, 0.0269665, -0.0040025, 0.02138184]]
+        # Get iteration points
+        grad_list = [prob.grad(x) for x in x_list]
+        delta_x = [np.array(x_list[i+1])-np.array(x_list[i])
+                   for i in range(len(x_list)-1)]
+        delta_grad = [grad_list[i+1]-grad_list[i]
+                      for i in range(len(grad_list)-1)]
+        hess = BFGS(init_scale=1, min_curvature=10)
+        hess.initialize(len(x_list[0]), 'hess')
+        # Compare the Hessian and its inverse
+        for i in range(len(delta_x)-1):
+            s = delta_x[i]
+            y = delta_grad[i]
+            hess.update(s, y)
+        # Test skip update
+        B = np.copy(hess.get_matrix())
+        s = delta_x[5]
+        y = delta_grad[5]
+        hess.update(s, y)
+        B_updated = np.copy(hess.get_matrix())
+        assert_array_equal(B, B_updated)

venv/Lib/site-packages/scipy/optimize/tests/test_lbfgsb_hessinv.py

@@ -0,0 +1,43 @@
+import numpy as np
+from numpy.testing import assert_allclose
+import scipy.linalg
+from scipy.optimize import minimize
+
+
+def test_1():
+    def f(x):
+        return x**4, 4*x**3
+
+    for gtol in [1e-8, 1e-12, 1e-20]:
+        for maxcor in range(20, 35):
+            result = minimize(fun=f, jac=True, method='L-BFGS-B', x0=20,
+                options={'gtol': gtol, 'maxcor': maxcor})
+
+            H1 = result.hess_inv(np.array([1])).reshape(1,1)
+            H2 = result.hess_inv.todense()
+
+            assert_allclose(H1, H2)
+
+
+def test_2():
+    H0 = [[3, 0], [1, 2]]
+
+    def f(x):
+        return np.dot(x, np.dot(scipy.linalg.inv(H0), x))
+
+    result1 = minimize(fun=f, method='L-BFGS-B', x0=[10, 20])
+    result2 = minimize(fun=f, method='BFGS', x0=[10, 20])
+
+    H1 = result1.hess_inv.todense()
+
+    H2 = np.vstack((
+        result1.hess_inv(np.array([1, 0])),
+        result1.hess_inv(np.array([0, 1]))))
+
+    assert_allclose(
+        result1.hess_inv(np.array([1, 0]).reshape(2,1)).reshape(-1),
+        result1.hess_inv(np.array([1, 0])))
+    assert_allclose(H1, H2)
+    assert_allclose(H1, result2.hess_inv, rtol=1e-2, atol=0.03)
+
+

venv/Lib/site-packages/scipy/optimize/tests/test_lbfgsb_setulb.py

@@ -0,0 +1,116 @@
+import numpy as np
+from scipy.optimize import _lbfgsb
+
+
+def objfun(x):
+    """simplified objective func to test lbfgsb bound violation"""
+    x0 = [0.8750000000000278,
+          0.7500000000000153,
+          0.9499999999999722,
+          0.8214285714285992,
+          0.6363636363636085]
+    x1 = [1.0, 0.0, 1.0, 0.0, 0.0]
+    x2 = [1.0,
+          0.0,
+          0.9889733043149325,
+          0.0,
+          0.026353554421041155]
+    x3 = [1.0,
+          0.0,
+          0.9889917442915558,
+          0.0,
+          0.020341986743231205]
+
+    f0 = 5163.647901211178
+    f1 = 5149.8181642072905
+    f2 = 5149.379332309634
+    f3 = 5149.374490771297
+
+    g0 = np.array([-0.5934820547965749,
+                   1.6251549718258351,
+                   -71.99168459202559,
+                   5.346636965797545,
+                   37.10732723092604])
+    g1 = np.array([-0.43295349282641515,
+                   1.008607936794592,
+                   18.223666726602975,
+                   31.927010036981997,
+                   -19.667512518739386])
+    g2 = np.array([-0.4699874455100256,
+                   0.9466285353668347,
+                   -0.016874360242016825,
+                   48.44999161133457,
+                   5.819631620590712])
+    g3 = np.array([-0.46970678696829116,
+                   0.9612719312174818,
+                   0.006129809488833699,
+                   48.43557729419473,
+                   6.005481418498221])
+
+    if np.allclose(x, x0):
+        f = f0
+        g = g0
+    elif np.allclose(x, x1):
+        f = f1
+        g = g1
+    elif np.allclose(x, x2):
+        f = f2
+        g = g2
+    elif np.allclose(x, x3):
+        f = f3
+        g = g3
+    else:
+        raise ValueError(
+            'Simplified objective function not defined '
+            'at requested point')
+    return (np.copy(f), np.copy(g))
+
+
+def test_setulb_floatround():
+    """test if setulb() violates bounds
+
+    checks for violation due to floating point rounding error
+    """
+
+    n = 5
+    m = 10
+    factr = 1e7
+    pgtol = 1e-5
+    maxls = 20
+    iprint = -1
+    nbd = np.full((n,), 2)
+    low_bnd = np.zeros(n, np.float64)
+    upper_bnd = np.ones(n, np.float64)
+
+    x0 = np.array(
+        [0.8750000000000278,
+         0.7500000000000153,
+         0.9499999999999722,
+         0.8214285714285992,
+         0.6363636363636085])
+    x = np.copy(x0)
+
+    f = np.array(0.0, np.float64)
+    g = np.zeros(n, np.float64)
+
+    fortran_int = _lbfgsb.types.intvar.dtype
+
+    wa = np.zeros(2*m*n + 5*n + 11*m*m + 8*m, np.float64)
+    iwa = np.zeros(3*n, fortran_int)
+    task = np.zeros(1, 'S60')
+    csave = np.zeros(1, 'S60')
+    lsave = np.zeros(4, fortran_int)
+    isave = np.zeros(44, fortran_int)
+    dsave = np.zeros(29, np.float64)
+
+    task[:] = b'START'
+
+    for n_iter in range(7):  # 7 steps required to reproduce error
+        f, g = objfun(x)
+
+        _lbfgsb.setulb(m, x, low_bnd, upper_bnd, nbd, f, g, factr,
+                       pgtol, wa, iwa, task, iprint, csave, lsave,
+                       isave, dsave, maxls)
+
+        assert (x <= upper_bnd).all() and (x >= low_bnd).all(), (
+            "_lbfgsb.setulb() stepped to a point outside of the bounds")

venv/Lib/site-packages/scipy/optimize/tests/test_least_squares.py

@@ -0,0 +1,756 @@
+from itertools import product
+
+import numpy as np
+from numpy.linalg import norm
+from numpy.testing import (assert_, assert_allclose,
+                           assert_equal, suppress_warnings)
+from pytest import raises as assert_raises
+from scipy.sparse import issparse, lil_matrix
+from scipy.sparse.linalg import aslinearoperator
+
+from scipy.optimize import least_squares
+from scipy.optimize._lsq.least_squares import IMPLEMENTED_LOSSES
+from scipy.optimize._lsq.common import EPS, make_strictly_feasible
+
+
+def fun_trivial(x, a=0):
+    return (x - a)**2 + 5.0
+
+
+def jac_trivial(x, a=0.0):
+    return 2 * (x - a)
+
+
+def fun_2d_trivial(x):
+    return np.array([x[0], x[1]])
+
+
+def jac_2d_trivial(x):
+    return np.identity(2)
+
+
+def fun_rosenbrock(x):
+    return np.array([10 * (x[1] - x[0]**2), (1 - x[0])])
+
+
+def jac_rosenbrock(x):
+    return np.array([
+        [-20 * x[0], 10],
+        [-1, 0]
+    ])
+
+
+def jac_rosenbrock_bad_dim(x):
+    return np.array([
+        [-20 * x[0], 10],
+        [-1, 0],
+        [0.0, 0.0]
+    ])
+
+
+def fun_rosenbrock_cropped(x):
+    return fun_rosenbrock(x)[0]
+
+
+def jac_rosenbrock_cropped(x):
+    return jac_rosenbrock(x)[0]
+
+
+# When x is 1-D array, return is 2-D array.
+def fun_wrong_dimensions(x):
+    return np.array([x, x**2, x**3])
+
+
+def jac_wrong_dimensions(x, a=0.0):
+    return np.atleast_3d(jac_trivial(x, a=a))
+
+
+def fun_bvp(x):
+    n = int(np.sqrt(x.shape[0]))
+    u = np.zeros((n + 2, n + 2))
+    x = x.reshape((n, n))
+    u[1:-1, 1:-1] = x
+    y = u[:-2, 1:-1] + u[2:, 1:-1] + u[1:-1, :-2] + u[1:-1, 2:] - 4 * x + x**3
+    return y.ravel()
+
+
+class BroydenTridiagonal(object):
+    def __init__(self, n=100, mode='sparse'):
+        np.random.seed(0)
+
+        self.n = n
+
+        self.x0 = -np.ones(n)
+        self.lb = np.linspace(-2, -1.5, n)
+        self.ub = np.linspace(-0.8, 0.0, n)
+
+        self.lb += 0.1 * np.random.randn(n)
+        self.ub += 0.1 * np.random.randn(n)
+
+        self.x0 += 0.1 * np.random.randn(n)
+        self.x0 = make_strictly_feasible(self.x0, self.lb, self.ub)
+
+        if mode == 'sparse':
+            self.sparsity = lil_matrix((n, n), dtype=int)
+            i = np.arange(n)
+            self.sparsity[i, i] = 1
+            i = np.arange(1, n)
+            self.sparsity[i, i - 1] = 1
+            i = np.arange(n - 1)
+            self.sparsity[i, i + 1] = 1
+
+            self.jac = self._jac
+        elif mode == 'operator':
+            self.jac = lambda x: aslinearoperator(self._jac(x))
+        elif mode == 'dense':
+            self.sparsity = None
+            self.jac = lambda x: self._jac(x).toarray()
+        else:
+            assert_(False)
+
+    def fun(self, x):
+        f = (3 - x) * x + 1
+        f[1:] -= x[:-1]
+        f[:-1] -= 2 * x[1:]
+        return f
+
+    def _jac(self, x):
+        J = lil_matrix((self.n, self.n))
+        i = np.arange(self.n)
+        J[i, i] = 3 - 2 * x
+        i = np.arange(1, self.n)
+        J[i, i - 1] = -1
+        i = np.arange(self.n - 1)
+        J[i, i + 1] = -2
+        return J
+
+
+class ExponentialFittingProblem(object):
+    """Provide data and function for exponential fitting in the form
+    y = a + exp(b * x) + noise."""
+
+    def __init__(self, a, b, noise, n_outliers=1, x_range=(-1, 1),
+                 n_points=11, random_seed=None):
+        np.random.seed(random_seed)
+        self.m = n_points
+        self.n = 2
+
+        self.p0 = np.zeros(2)
+        self.x = np.linspace(x_range[0], x_range[1], n_points)
+
+        self.y = a + np.exp(b * self.x)
+        self.y += noise * np.random.randn(self.m)
+
+        outliers = np.random.randint(0, self.m, n_outliers)
+        self.y[outliers] += 50 * noise * np.random.rand(n_outliers)
+
+        self.p_opt = np.array([a, b])
+
+    def fun(self, p):
+        return p[0] + np.exp(p[1] * self.x) - self.y
+
+    def jac(self, p):
+        J = np.empty((self.m, self.n))
+        J[:, 0] = 1
+        J[:, 1] = self.x * np.exp(p[1] * self.x)
+        return J
+
+
+def cubic_soft_l1(z):
+    rho = np.empty((3, z.size))
+
+    t = 1 + z
+    rho[0] = 3 * (t**(1/3) - 1)
+    rho[1] = t ** (-2/3)
+    rho[2] = -2/3 * t**(-5/3)
+
+    return rho
+
+
+LOSSES = list(IMPLEMENTED_LOSSES.keys()) + [cubic_soft_l1]
+
+
+class BaseMixin(object):
+    def test_basic(self):
+        # Test that the basic calling sequence works.
+        res = least_squares(fun_trivial, 2., method=self.method)
+        assert_allclose(res.x, 0, atol=1e-4)
+        assert_allclose(res.fun, fun_trivial(res.x))
+
+    def test_args_kwargs(self):
+        # Test that args and kwargs are passed correctly to the functions.
+        a = 3.0
+        for jac in ['2-point', '3-point', 'cs', jac_trivial]:
+            with suppress_warnings() as sup:
+                sup.filter(UserWarning,
+                           "jac='(3-point|cs)' works equivalently to '2-point' for method='lm'")
+                res = least_squares(fun_trivial, 2.0, jac, args=(a,),
+                                    method=self.method)
+                res1 = least_squares(fun_trivial, 2.0, jac, kwargs={'a': a},
+                                    method=self.method)
+
+            assert_allclose(res.x, a, rtol=1e-4)
+            assert_allclose(res1.x, a, rtol=1e-4)
+
+            assert_raises(TypeError, least_squares, fun_trivial, 2.0,
+                          args=(3, 4,), method=self.method)
+            assert_raises(TypeError, least_squares, fun_trivial, 2.0,
+                          kwargs={'kaboom': 3}, method=self.method)
+
+    def test_jac_options(self):
+        for jac in ['2-point', '3-point', 'cs', jac_trivial]:
+            with suppress_warnings() as sup:
+                sup.filter(UserWarning,
+                           "jac='(3-point|cs)' works equivalently to '2-point' for method='lm'")
+                res = least_squares(fun_trivial, 2.0, jac, method=self.method)
+            assert_allclose(res.x, 0, atol=1e-4)
+
+        assert_raises(ValueError, least_squares, fun_trivial, 2.0, jac='oops',
+                      method=self.method)
+
+    def test_nfev_options(self):
+        for max_nfev in [None, 20]:
+            res = least_squares(fun_trivial, 2.0, max_nfev=max_nfev,
+                                method=self.method)
+            assert_allclose(res.x, 0, atol=1e-4)
+
+    def test_x_scale_options(self):
+        for x_scale in [1.0, np.array([0.5]), 'jac']:
+            res = least_squares(fun_trivial, 2.0, x_scale=x_scale)
+            assert_allclose(res.x, 0)
+        assert_raises(ValueError, least_squares, fun_trivial,
+                      2.0, x_scale='auto', method=self.method)
+        assert_raises(ValueError, least_squares, fun_trivial,
+                      2.0, x_scale=-1.0, method=self.method)
+        assert_raises(ValueError, least_squares, fun_trivial,
+                      2.0, x_scale=None, method=self.method)
+        assert_raises(ValueError, least_squares, fun_trivial,
+                      2.0, x_scale=1.0+2.0j, method=self.method)
+
+    def test_diff_step(self):
+        # res1 and res2 should be equivalent.
+        # res2 and res3 should be different.
+        res1 = least_squares(fun_trivial, 2.0, diff_step=1e-1,
+                             method=self.method)
+        res2 = least_squares(fun_trivial, 2.0, diff_step=-1e-1,
+                             method=self.method)
+        res3 = least_squares(fun_trivial, 2.0,
+                             diff_step=None, method=self.method)
+        assert_allclose(res1.x, 0, atol=1e-4)
+        assert_allclose(res2.x, 0, atol=1e-4)
+        assert_allclose(res3.x, 0, atol=1e-4)
+        assert_equal(res1.x, res2.x)
+        assert_equal(res1.nfev, res2.nfev)
+        assert_(res2.nfev != res3.nfev)
+
+    def test_incorrect_options_usage(self):
+        assert_raises(TypeError, least_squares, fun_trivial, 2.0,
+                      method=self.method, options={'no_such_option': 100})
+        assert_raises(TypeError, least_squares, fun_trivial, 2.0,
+                      method=self.method, options={'max_nfev': 100})
+
+    def test_full_result(self):
+        # MINPACK doesn't work very well with factor=100 on this problem,
+        # thus using low 'atol'.
+        res = least_squares(fun_trivial, 2.0, method=self.method)
+        assert_allclose(res.x, 0, atol=1e-4)
+        assert_allclose(res.cost, 12.5)
+        assert_allclose(res.fun, 5)
+        assert_allclose(res.jac, 0, atol=1e-4)
+        assert_allclose(res.grad, 0, atol=1e-2)
+        assert_allclose(res.optimality, 0, atol=1e-2)
+        assert_equal(res.active_mask, 0)
+        if self.method == 'lm':
+            assert_(res.nfev < 30)
+            assert_(res.njev is None)
+        else:
+            assert_(res.nfev < 10)
+            assert_(res.njev < 10)
+        assert_(res.status > 0)
+        assert_(res.success)
+
+    def test_full_result_single_fev(self):
+        # MINPACK checks the number of nfev after the iteration,
+        # so it's hard to tell what he is going to compute.
+        if self.method == 'lm':
+            return
+
+        res = least_squares(fun_trivial, 2.0, method=self.method,
+                            max_nfev=1)
+        assert_equal(res.x, np.array([2]))
+        assert_equal(res.cost, 40.5)
+        assert_equal(res.fun, np.array([9]))
+        assert_equal(res.jac, np.array([[4]]))
+        assert_equal(res.grad, np.array([36]))
+        assert_equal(res.optimality, 36)
+        assert_equal(res.active_mask, np.array([0]))
+        assert_equal(res.nfev, 1)
+        assert_equal(res.njev, 1)
+        assert_equal(res.status, 0)
+        assert_equal(res.success, 0)
+
+    def test_rosenbrock(self):
+        x0 = [-2, 1]
+        x_opt = [1, 1]
+        for jac, x_scale, tr_solver in product(
+                ['2-point', '3-point', 'cs', jac_rosenbrock],
+                [1.0, np.array([1.0, 0.2]), 'jac'],
+                ['exact', 'lsmr']):
+            with suppress_warnings() as sup:
+                sup.filter(UserWarning,
+                           "jac='(3-point|cs)' works equivalently to '2-point' for method='lm'")
+                res = least_squares(fun_rosenbrock, x0, jac, x_scale=x_scale,
+                                    tr_solver=tr_solver, method=self.method)
+            assert_allclose(res.x, x_opt)
+
+    def test_rosenbrock_cropped(self):
+        x0 = [-2, 1]
+        if self.method == 'lm':
+            assert_raises(ValueError, least_squares, fun_rosenbrock_cropped,
+                          x0, method='lm')
+        else:
+            for jac, x_scale, tr_solver in product(
+                    ['2-point', '3-point', 'cs', jac_rosenbrock_cropped],
+                    [1.0, np.array([1.0, 0.2]), 'jac'],
+                    ['exact', 'lsmr']):
+                res = least_squares(
+                    fun_rosenbrock_cropped, x0, jac, x_scale=x_scale,
+                    tr_solver=tr_solver, method=self.method)
+                assert_allclose(res.cost, 0, atol=1e-14)
+
+    def test_fun_wrong_dimensions(self):
+        assert_raises(ValueError, least_squares, fun_wrong_dimensions,
+                      2.0, method=self.method)
+
+    def test_jac_wrong_dimensions(self):
+        assert_raises(ValueError, least_squares, fun_trivial,
+                      2.0, jac_wrong_dimensions, method=self.method)
+
+    def test_fun_and_jac_inconsistent_dimensions(self):
+        x0 = [1, 2]
+        assert_raises(ValueError, least_squares, fun_rosenbrock, x0,
+                      jac_rosenbrock_bad_dim, method=self.method)
+
+    def test_x0_multidimensional(self):
+        x0 = np.ones(4).reshape(2, 2)
+        assert_raises(ValueError, least_squares, fun_trivial, x0,
+                      method=self.method)
+
+    def test_x0_complex_scalar(self):
+        x0 = 2.0 + 0.0*1j
+        assert_raises(ValueError, least_squares, fun_trivial, x0,
+                      method=self.method)
+
+    def test_x0_complex_array(self):
+        x0 = [1.0, 2.0 + 0.0*1j]
+        assert_raises(ValueError, least_squares, fun_trivial, x0,
+                      method=self.method)
+
+    def test_bvp(self):
+        # This test was introduced with fix #5556. It turned out that
+        # dogbox solver had a bug with trust-region radius update, which
+        # could block its progress and create an infinite loop. And this
+        # discrete boundary value problem is the one which triggers it.
+        n = 10
+        x0 = np.ones(n**2)
+        if self.method == 'lm':
+            max_nfev = 5000  # To account for Jacobian estimation.
+        else:
+            max_nfev = 100
+        res = least_squares(fun_bvp, x0, ftol=1e-2, method=self.method,
+                            max_nfev=max_nfev)
+
+        assert_(res.nfev < max_nfev)
+        assert_(res.cost < 0.5)
+
+    def test_error_raised_when_all_tolerances_below_eps(self):
+        # Test that all 0 tolerances are not allowed.
+        assert_raises(ValueError, least_squares, fun_trivial, 2.0,
+                      method=self.method, ftol=None, xtol=None, gtol=None)
+
+    def test_convergence_with_only_one_tolerance_enabled(self):
+        if self.method == 'lm':
+            return  # should not do test
+        x0 = [-2, 1]
+        x_opt = [1, 1]
+        for ftol, xtol, gtol in [(1e-8, None, None),
+                                  (None, 1e-8, None),
+                                  (None, None, 1e-8)]:
+            res = least_squares(fun_rosenbrock, x0, jac=jac_rosenbrock,
+                                ftol=ftol, gtol=gtol, xtol=xtol,
+                                method=self.method)
+            assert_allclose(res.x, x_opt)
+
+
+class BoundsMixin(object):
+    def test_inconsistent(self):
+        assert_raises(ValueError, least_squares, fun_trivial, 2.0,
+                      bounds=(10.0, 0.0), method=self.method)
+
+    def test_infeasible(self):
+        assert_raises(ValueError, least_squares, fun_trivial, 2.0,
+                      bounds=(3., 4), method=self.method)
+
+    def test_wrong_number(self):
+        assert_raises(ValueError, least_squares, fun_trivial, 2.,
+                      bounds=(1., 2, 3), method=self.method)
+
+    def test_inconsistent_shape(self):
+        assert_raises(ValueError, least_squares, fun_trivial, 2.0,
+                      bounds=(1.0, [2.0, 3.0]), method=self.method)
+        # 1-D array wont't be broadcasted
+        assert_raises(ValueError, least_squares, fun_rosenbrock, [1.0, 2.0],
+                      bounds=([0.0], [3.0, 4.0]), method=self.method)
+
+    def test_in_bounds(self):
+        for jac in ['2-point', '3-point', 'cs', jac_trivial]:
+            res = least_squares(fun_trivial, 2.0, jac=jac,
+                                bounds=(-1.0, 3.0), method=self.method)
+            assert_allclose(res.x, 0.0, atol=1e-4)
+            assert_equal(res.active_mask, [0])
+            assert_(-1 <= res.x <= 3)
+            res = least_squares(fun_trivial, 2.0, jac=jac,
+                                bounds=(0.5, 3.0), method=self.method)
+            assert_allclose(res.x, 0.5, atol=1e-4)
+            assert_equal(res.active_mask, [-1])
+            assert_(0.5 <= res.x <= 3)
+
+    def test_bounds_shape(self):
+        for jac in ['2-point', '3-point', 'cs', jac_2d_trivial]:
+            x0 = [1.0, 1.0]
+            res = least_squares(fun_2d_trivial, x0, jac=jac)
+            assert_allclose(res.x, [0.0, 0.0])
+            res = least_squares(fun_2d_trivial, x0, jac=jac,
+                                bounds=(0.5, [2.0, 2.0]), method=self.method)
+            assert_allclose(res.x, [0.5, 0.5])
+            res = least_squares(fun_2d_trivial, x0, jac=jac,
+                                bounds=([0.3, 0.2], 3.0), method=self.method)
+            assert_allclose(res.x, [0.3, 0.2])
+            res = least_squares(
+                fun_2d_trivial, x0, jac=jac, bounds=([-1, 0.5], [1.0, 3.0]),
+                method=self.method)
+            assert_allclose(res.x, [0.0, 0.5], atol=1e-5)
+
+    def test_rosenbrock_bounds(self):
+        x0_1 = np.array([-2.0, 1.0])
+        x0_2 = np.array([2.0, 2.0])
+        x0_3 = np.array([-2.0, 2.0])
+        x0_4 = np.array([0.0, 2.0])
+        x0_5 = np.array([-1.2, 1.0])
+        problems = [
+            (x0_1, ([-np.inf, -1.5], np.inf)),
+            (x0_2, ([-np.inf, 1.5], np.inf)),
+            (x0_3, ([-np.inf, 1.5], np.inf)),
+            (x0_4, ([-np.inf, 1.5], [1.0, np.inf])),
+            (x0_2, ([1.0, 1.5], [3.0, 3.0])),
+            (x0_5, ([-50.0, 0.0], [0.5, 100]))
+        ]
+        for x0, bounds in problems:
+            for jac, x_scale, tr_solver in product(
+                    ['2-point', '3-point', 'cs', jac_rosenbrock],
+                    [1.0, [1.0, 0.5], 'jac'],
+                    ['exact', 'lsmr']):
+                res = least_squares(fun_rosenbrock, x0, jac, bounds,
+                                    x_scale=x_scale, tr_solver=tr_solver,
+                                    method=self.method)
+                assert_allclose(res.optimality, 0.0, atol=1e-5)
+
+
+class SparseMixin(object):
+    def test_exact_tr_solver(self):
+        p = BroydenTridiagonal()
+        assert_raises(ValueError, least_squares, p.fun, p.x0, p.jac,
+                      tr_solver='exact', method=self.method)
+        assert_raises(ValueError, least_squares, p.fun, p.x0,
+                      tr_solver='exact', jac_sparsity=p.sparsity,
+                      method=self.method)
+
+    def test_equivalence(self):
+        sparse = BroydenTridiagonal(mode='sparse')
+        dense = BroydenTridiagonal(mode='dense')
+        res_sparse = least_squares(
+            sparse.fun, sparse.x0, jac=sparse.jac,
+            method=self.method)
+        res_dense = least_squares(
+            dense.fun, dense.x0, jac=sparse.jac,
+            method=self.method)
+        assert_equal(res_sparse.nfev, res_dense.nfev)
+        assert_allclose(res_sparse.x, res_dense.x, atol=1e-20)
+        assert_allclose(res_sparse.cost, 0, atol=1e-20)
+        assert_allclose(res_dense.cost, 0, atol=1e-20)
+
+    def test_tr_options(self):
+        p = BroydenTridiagonal()
+        res = least_squares(p.fun, p.x0, p.jac, method=self.method,
+                            tr_options={'btol': 1e-10})
+        assert_allclose(res.cost, 0, atol=1e-20)
+
+    def test_wrong_parameters(self):
+        p = BroydenTridiagonal()
+        assert_raises(ValueError, least_squares, p.fun, p.x0, p.jac,
+                      tr_solver='best', method=self.method)
+        assert_raises(TypeError, least_squares, p.fun, p.x0, p.jac,
+                      tr_solver='lsmr', tr_options={'tol': 1e-10})
+
+    def test_solver_selection(self):
+        sparse = BroydenTridiagonal(mode='sparse')
+        dense = BroydenTridiagonal(mode='dense')
+        res_sparse = least_squares(sparse.fun, sparse.x0, jac=sparse.jac,
+                                   method=self.method)
+        res_dense = least_squares(dense.fun, dense.x0, jac=dense.jac,
+                                  method=self.method)
+        assert_allclose(res_sparse.cost, 0, atol=1e-20)
+        assert_allclose(res_dense.cost, 0, atol=1e-20)
+        assert_(issparse(res_sparse.jac))
+        assert_(isinstance(res_dense.jac, np.ndarray))
+
+    def test_numerical_jac(self):
+        p = BroydenTridiagonal()
+        for jac in ['2-point', '3-point', 'cs']:
+            res_dense = least_squares(p.fun, p.x0, jac, method=self.method)
+            res_sparse = least_squares(
+                p.fun, p.x0, jac,method=self.method,
+                jac_sparsity=p.sparsity)
+            assert_equal(res_dense.nfev, res_sparse.nfev)
+            assert_allclose(res_dense.x, res_sparse.x, atol=1e-20)
+            assert_allclose(res_dense.cost, 0, atol=1e-20)
+            assert_allclose(res_sparse.cost, 0, atol=1e-20)
+
+    def test_with_bounds(self):
+        p = BroydenTridiagonal()
+        for jac, jac_sparsity in product(
+                [p.jac, '2-point', '3-point', 'cs'], [None, p.sparsity]):
+            res_1 = least_squares(
+                p.fun, p.x0, jac, bounds=(p.lb, np.inf),
+                method=self.method,jac_sparsity=jac_sparsity)
+            res_2 = least_squares(
+                p.fun, p.x0, jac, bounds=(-np.inf, p.ub),
+                method=self.method, jac_sparsity=jac_sparsity)
+            res_3 = least_squares(
+                p.fun, p.x0, jac, bounds=(p.lb, p.ub),
+                method=self.method, jac_sparsity=jac_sparsity)
+            assert_allclose(res_1.optimality, 0, atol=1e-10)
+            assert_allclose(res_2.optimality, 0, atol=1e-10)
+            assert_allclose(res_3.optimality, 0, atol=1e-10)
+
+    def test_wrong_jac_sparsity(self):
+        p = BroydenTridiagonal()
+        sparsity = p.sparsity[:-1]
+        assert_raises(ValueError, least_squares, p.fun, p.x0,
+                      jac_sparsity=sparsity, method=self.method)
+
+    def test_linear_operator(self):
+        p = BroydenTridiagonal(mode='operator')
+        res = least_squares(p.fun, p.x0, p.jac, method=self.method)
+        assert_allclose(res.cost, 0.0, atol=1e-20)
+        assert_raises(ValueError, least_squares, p.fun, p.x0, p.jac,
+                      method=self.method, tr_solver='exact')
+
+    def test_x_scale_jac_scale(self):
+        p = BroydenTridiagonal()
+        res = least_squares(p.fun, p.x0, p.jac, method=self.method,
+                            x_scale='jac')
+        assert_allclose(res.cost, 0.0, atol=1e-20)
+
+        p = BroydenTridiagonal(mode='operator')
+        assert_raises(ValueError, least_squares, p.fun, p.x0, p.jac,
+                      method=self.method, x_scale='jac')
+
+
+class LossFunctionMixin(object):
+    def test_options(self):
+        for loss in LOSSES:
+            res = least_squares(fun_trivial, 2.0, loss=loss,
+                                method=self.method)
+            assert_allclose(res.x, 0, atol=1e-15)
+
+        assert_raises(ValueError, least_squares, fun_trivial, 2.0,
+                      loss='hinge', method=self.method)
+
+    def test_fun(self):
+        # Test that res.fun is actual residuals, and not modified by loss
+        # function stuff.
+        for loss in LOSSES:
+            res = least_squares(fun_trivial, 2.0, loss=loss,
+                                method=self.method)
+            assert_equal(res.fun, fun_trivial(res.x))
+
+    def test_grad(self):
+        # Test that res.grad is true gradient of loss function at the
+        # solution. Use max_nfev = 1, to avoid reaching minimum.
+        x = np.array([2.0])  # res.x will be this.
+
+        res = least_squares(fun_trivial, x, jac_trivial, loss='linear',
+                            max_nfev=1, method=self.method)
+        assert_equal(res.grad, 2 * x * (x**2 + 5))
+
+        res = least_squares(fun_trivial, x, jac_trivial, loss='huber',
+                            max_nfev=1, method=self.method)
+        assert_equal(res.grad, 2 * x)
+
+        res = least_squares(fun_trivial, x, jac_trivial, loss='soft_l1',
+                            max_nfev=1, method=self.method)
+        assert_allclose(res.grad,
+                        2 * x * (x**2 + 5) / (1 + (x**2 + 5)**2)**0.5)
+
+        res = least_squares(fun_trivial, x, jac_trivial, loss='cauchy',
+                            max_nfev=1, method=self.method)
+        assert_allclose(res.grad, 2 * x * (x**2 + 5) / (1 + (x**2 + 5)**2))
+
+        res = least_squares(fun_trivial, x, jac_trivial, loss='arctan',
+                            max_nfev=1, method=self.method)
+        assert_allclose(res.grad, 2 * x * (x**2 + 5) / (1 + (x**2 + 5)**4))
+
+        res = least_squares(fun_trivial, x, jac_trivial, loss=cubic_soft_l1,
+                            max_nfev=1, method=self.method)
+        assert_allclose(res.grad,
+                        2 * x * (x**2 + 5) / (1 + (x**2 + 5)**2)**(2/3))
+
+    def test_jac(self):
+        # Test that res.jac.T.dot(res.jac) gives Gauss-Newton approximation
+        # of Hessian. This approximation is computed by doubly differentiating
+        # the cost function and dropping the part containing second derivative
+        # of f. For a scalar function it is computed as
+        # H = (rho' + 2 * rho'' * f**2) * f'**2, if the expression inside the
+        # brackets is less than EPS it is replaced by EPS. Here, we check
+        # against the root of H.
+
+        x = 2.0  # res.x will be this.
+        f = x**2 + 5  # res.fun will be this.
+
+        res = least_squares(fun_trivial, x, jac_trivial, loss='linear',
+                            max_nfev=1, method=self.method)
+        assert_equal(res.jac, 2 * x)
+
+        # For `huber` loss the Jacobian correction is identically zero
+        # in outlier region, in such cases it is modified to be equal EPS**0.5.
+        res = least_squares(fun_trivial, x, jac_trivial, loss='huber',
+                            max_nfev=1, method=self.method)
+        assert_equal(res.jac, 2 * x * EPS**0.5)
+
+        # Now, let's apply `loss_scale` to turn the residual into an inlier.
+        # The loss function becomes linear.
+        res = least_squares(fun_trivial, x, jac_trivial, loss='huber',
+                            f_scale=10, max_nfev=1)
+        assert_equal(res.jac, 2 * x)
+
+        # 'soft_l1' always gives a positive scaling.
+        res = least_squares(fun_trivial, x, jac_trivial, loss='soft_l1',
+                            max_nfev=1, method=self.method)
+        assert_allclose(res.jac, 2 * x * (1 + f**2)**-0.75)
+
+        # For 'cauchy' the correction term turns out to be negative, and it
+        # replaced by EPS**0.5.
+        res = least_squares(fun_trivial, x, jac_trivial, loss='cauchy',
+                            max_nfev=1, method=self.method)
+        assert_allclose(res.jac, 2 * x * EPS**0.5)
+
+        # Now use scaling to turn the residual to inlier.
+        res = least_squares(fun_trivial, x, jac_trivial, loss='cauchy',
+                            f_scale=10, max_nfev=1, method=self.method)
+        fs = f / 10
+        assert_allclose(res.jac, 2 * x * (1 - fs**2)**0.5 / (1 + fs**2))
+
+        # 'arctan' gives an outlier.
+        res = least_squares(fun_trivial, x, jac_trivial, loss='arctan',
+                            max_nfev=1, method=self.method)
+        assert_allclose(res.jac, 2 * x * EPS**0.5)
+
+        # Turn to inlier.
+        res = least_squares(fun_trivial, x, jac_trivial, loss='arctan',
+                            f_scale=20.0, max_nfev=1, method=self.method)
+        fs = f / 20
+        assert_allclose(res.jac, 2 * x * (1 - 3 * fs**4)**0.5 / (1 + fs**4))
+
+        # cubic_soft_l1 will give an outlier.
+        res = least_squares(fun_trivial, x, jac_trivial, loss=cubic_soft_l1,
+                            max_nfev=1)
+        assert_allclose(res.jac, 2 * x * EPS**0.5)
+
+        # Turn to inlier.
+        res = least_squares(fun_trivial, x, jac_trivial,
+                            loss=cubic_soft_l1, f_scale=6, max_nfev=1)
+        fs = f / 6
+        assert_allclose(res.jac,
+                        2 * x * (1 - fs**2 / 3)**0.5 * (1 + fs**2)**(-5/6))
+
+    def test_robustness(self):
+        for noise in [0.1, 1.0]:
+            p = ExponentialFittingProblem(1, 0.1, noise, random_seed=0)
+
+            for jac in ['2-point', '3-point', 'cs', p.jac]:
+                res_lsq = least_squares(p.fun, p.p0, jac=jac,
+                                        method=self.method)
+                assert_allclose(res_lsq.optimality, 0, atol=1e-2)
+                for loss in LOSSES:
+                    if loss == 'linear':
+                        continue
+                    res_robust = least_squares(
+                        p.fun, p.p0, jac=jac, loss=loss, f_scale=noise,
+                        method=self.method)
+                    assert_allclose(res_robust.optimality, 0, atol=1e-2)
+                    assert_(norm(res_robust.x - p.p_opt) <
+                            norm(res_lsq.x - p.p_opt))
+
+
+class TestDogbox(BaseMixin, BoundsMixin, SparseMixin, LossFunctionMixin):
+    method = 'dogbox'
+
+
+class TestTRF(BaseMixin, BoundsMixin, SparseMixin, LossFunctionMixin):
+    method = 'trf'
+
+    def test_lsmr_regularization(self):
+        p = BroydenTridiagonal()
+        for regularize in [True, False]:
+            res = least_squares(p.fun, p.x0, p.jac, method='trf',
+                                tr_options={'regularize': regularize})
+            assert_allclose(res.cost, 0, atol=1e-20)
+
+
+class TestLM(BaseMixin):
+    method = 'lm'
+
+    def test_bounds_not_supported(self):
+        assert_raises(ValueError, least_squares, fun_trivial,
+                      2.0, bounds=(-3.0, 3.0), method='lm')
+
+    def test_m_less_n_not_supported(self):
+        x0 = [-2, 1]
+        assert_raises(ValueError, least_squares, fun_rosenbrock_cropped, x0,
+                      method='lm')
+
+    def test_sparse_not_supported(self):
+        p = BroydenTridiagonal()
+        assert_raises(ValueError, least_squares, p.fun, p.x0, p.jac,
+                      method='lm')
+
+    def test_jac_sparsity_not_supported(self):
+        assert_raises(ValueError, least_squares, fun_trivial, 2.0,
+                      jac_sparsity=[1], method='lm')
+
+    def test_LinearOperator_not_supported(self):
+        p = BroydenTridiagonal(mode="operator")
+        assert_raises(ValueError, least_squares, p.fun, p.x0, p.jac,
+                      method='lm')
+
+    def test_loss(self):
+        res = least_squares(fun_trivial, 2.0, loss='linear', method='lm')
+        assert_allclose(res.x, 0.0, atol=1e-4)
+
+        assert_raises(ValueError, least_squares, fun_trivial, 2.0,
+                      method='lm', loss='huber')
+
+
+def test_basic():
+    # test that 'method' arg is really optional
+    res = least_squares(fun_trivial, 2.0)
+    assert_allclose(res.x, 0, atol=1e-10)
+
+
+def test_small_tolerances_for_lm():
+    for ftol, xtol, gtol in [(None, 1e-13, 1e-13),
+                             (1e-13, None, 1e-13),
+                             (1e-13, 1e-13, None)]:
+        assert_raises(ValueError, least_squares, fun_trivial, 2.0, xtol=xtol,
+                      ftol=ftol, gtol=gtol, method='lm')

venv/Lib/site-packages/scipy/optimize/tests/test_linear_assignment.py

@@ -0,0 +1,102 @@
+# Author: Brian M. Clapper, G. Varoquaux, Lars Buitinck
+# License: BSD
+
+from numpy.testing import assert_array_equal
+from pytest import raises as assert_raises
+
+import numpy as np
+
+from scipy.optimize import linear_sum_assignment
+from scipy.sparse.sputils import matrix
+
+
+def test_linear_sum_assignment():
+    for sign in [-1, 1]:
+        for cost_matrix, expected_cost in [
+            # Square
+            ([[400, 150, 400],
+              [400, 450, 600],
+              [300, 225, 300]],
+             [150, 400, 300]
+             ),
+
+            # Rectangular variant
+            ([[400, 150, 400, 1],
+              [400, 450, 600, 2],
+              [300, 225, 300, 3]],
+             [150, 2, 300]),
+
+            # Square
+            ([[10, 10, 8],
+              [9, 8, 1],
+              [9, 7, 4]],
+             [10, 1, 7]),
+
+            # Rectangular variant
+            ([[10, 10, 8, 11],
+              [9, 8, 1, 1],
+              [9, 7, 4, 10]],
+             [10, 1, 4]),
+
+            # n == 2, m == 0 matrix
+            ([[], []],
+             []),
+
+            # Square with positive infinities
+            ([[10, float("inf"), float("inf")],
+              [float("inf"), float("inf"), 1],
+              [float("inf"), 7, float("inf")]],
+             [10, 1, 7]),
+        ]:
+
+            maximize = sign == -1
+            cost_matrix = sign * np.array(cost_matrix)
+            expected_cost = sign * np.array(expected_cost)
+
+            row_ind, col_ind = linear_sum_assignment(cost_matrix,
+                                                     maximize=maximize)
+            assert_array_equal(row_ind, np.sort(row_ind))
+            assert_array_equal(expected_cost, cost_matrix[row_ind, col_ind])
+
+            cost_matrix = cost_matrix.T
+            row_ind, col_ind = linear_sum_assignment(cost_matrix,
+                                                     maximize=maximize)
+            assert_array_equal(row_ind, np.sort(row_ind))
+            assert_array_equal(np.sort(expected_cost),
+                               np.sort(cost_matrix[row_ind, col_ind]))
+
+
+def test_linear_sum_assignment_input_validation():
+
+    assert_raises(ValueError, linear_sum_assignment, [1, 2, 3])
+
+    C = [[1, 2, 3], [4, 5, 6]]
+    assert_array_equal(linear_sum_assignment(C),
+                       linear_sum_assignment(np.asarray(C)))
+    assert_array_equal(linear_sum_assignment(C),
+                       linear_sum_assignment(matrix(C)))
+
+    I = np.identity(3)
+    assert_array_equal(linear_sum_assignment(I.astype(np.bool_)),
+                       linear_sum_assignment(I))
+    assert_raises(ValueError, linear_sum_assignment, I.astype(str))
+
+    I[0][0] = np.nan
+    assert_raises(ValueError, linear_sum_assignment, I)
+
+    I = np.identity(3)
+    I[1][1] = -np.inf
+    assert_raises(ValueError, linear_sum_assignment, I)
+
+    I = np.identity(3)
+    I[:, 0] = np.inf
+    assert_raises(ValueError, linear_sum_assignment, I)
+
+
+def test_constant_cost_matrix():
+    # Fixes #11602
+    n = 8
+    C = np.ones((n, n))
+    row_ind, col_ind = linear_sum_assignment(C)
+    assert_array_equal(row_ind, np.arange(n))
+    assert_array_equal(col_ind, np.arange(n))

venv/Lib/site-packages/scipy/optimize/tests/test_linesearch.py

@@ -0,0 +1,292 @@
+"""
+Tests for line search routines
+"""
+from numpy.testing import (assert_, assert_equal, assert_array_almost_equal,
+                           assert_array_almost_equal_nulp, assert_warns,
+                           suppress_warnings)
+import scipy.optimize.linesearch as ls
+from scipy.optimize.linesearch import LineSearchWarning
+import numpy as np
+
+
+def assert_wolfe(s, phi, derphi, c1=1e-4, c2=0.9, err_msg=""):
+    """
+    Check that strong Wolfe conditions apply
+    """
+    phi1 = phi(s)
+    phi0 = phi(0)
+    derphi0 = derphi(0)
+    derphi1 = derphi(s)
+    msg = "s = %s; phi(0) = %s; phi(s) = %s; phi'(0) = %s; phi'(s) = %s; %s" % (
+        s, phi0, phi1, derphi0, derphi1, err_msg)
+
+    assert_(phi1 <= phi0 + c1*s*derphi0, "Wolfe 1 failed: " + msg)
+    assert_(abs(derphi1) <= abs(c2*derphi0), "Wolfe 2 failed: " + msg)
+
+
+def assert_armijo(s, phi, c1=1e-4, err_msg=""):
+    """
+    Check that Armijo condition applies
+    """
+    phi1 = phi(s)
+    phi0 = phi(0)
+    msg = "s = %s; phi(0) = %s; phi(s) = %s; %s" % (s, phi0, phi1, err_msg)
+    assert_(phi1 <= (1 - c1*s)*phi0, msg)
+
+
+def assert_line_wolfe(x, p, s, f, fprime, **kw):
+    assert_wolfe(s, phi=lambda sp: f(x + p*sp),
+                 derphi=lambda sp: np.dot(fprime(x + p*sp), p), **kw)
+
+
+def assert_line_armijo(x, p, s, f, **kw):
+    assert_armijo(s, phi=lambda sp: f(x + p*sp), **kw)
+
+
+def assert_fp_equal(x, y, err_msg="", nulp=50):
+    """Assert two arrays are equal, up to some floating-point rounding error"""
+    try:
+        assert_array_almost_equal_nulp(x, y, nulp)
+    except AssertionError as e:
+        raise AssertionError("%s\n%s" % (e, err_msg))
+
+
+class TestLineSearch(object):
+    # -- scalar functions; must have dphi(0.) < 0
+    def _scalar_func_1(self, s):
+        self.fcount += 1
+        p = -s - s**3 + s**4
+        dp = -1 - 3*s**2 + 4*s**3
+        return p, dp
+
+    def _scalar_func_2(self, s):
+        self.fcount += 1
+        p = np.exp(-4*s) + s**2
+        dp = -4*np.exp(-4*s) + 2*s
+        return p, dp
+
+    def _scalar_func_3(self, s):
+        self.fcount += 1
+        p = -np.sin(10*s)
+        dp = -10*np.cos(10*s)
+        return p, dp
+
+    # -- n-d functions
+
+    def _line_func_1(self, x):
+        self.fcount += 1
+        f = np.dot(x, x)
+        df = 2*x
+        return f, df
+
+    def _line_func_2(self, x):
+        self.fcount += 1
+        f = np.dot(x, np.dot(self.A, x)) + 1
+        df = np.dot(self.A + self.A.T, x)
+        return f, df
+
+    # --
+
+    def setup_method(self):
+        self.scalar_funcs = []
+        self.line_funcs = []
+        self.N = 20
+        self.fcount = 0
+
+        def bind_index(func, idx):
+            # Remember Python's closure semantics!
+            return lambda *a, **kw: func(*a, **kw)[idx]
+
+        for name in sorted(dir(self)):
+            if name.startswith('_scalar_func_'):
+                value = getattr(self, name)
+                self.scalar_funcs.append(
+                    (name, bind_index(value, 0), bind_index(value, 1)))
+            elif name.startswith('_line_func_'):
+                value = getattr(self, name)
+                self.line_funcs.append(
+                    (name, bind_index(value, 0), bind_index(value, 1)))
+
+        np.random.seed(1234)
+        self.A = np.random.randn(self.N, self.N)
+
+    def scalar_iter(self):
+        for name, phi, derphi in self.scalar_funcs:
+            for old_phi0 in np.random.randn(3):
+                yield name, phi, derphi, old_phi0
+
+    def line_iter(self):
+        for name, f, fprime in self.line_funcs:
+            k = 0
+            while k < 9:
+                x = np.random.randn(self.N)
+                p = np.random.randn(self.N)
+                if np.dot(p, fprime(x)) >= 0:
+                    # always pick a descent direction
+                    continue
+                k += 1
+                old_fv = float(np.random.randn())
+                yield name, f, fprime, x, p, old_fv
+
+    # -- Generic scalar searches
+
+    def test_scalar_search_wolfe1(self):
+        c = 0
+        for name, phi, derphi, old_phi0 in self.scalar_iter():
+            c += 1
+            s, phi1, phi0 = ls.scalar_search_wolfe1(phi, derphi, phi(0),
+                                                    old_phi0, derphi(0))
+            assert_fp_equal(phi0, phi(0), name)
+            assert_fp_equal(phi1, phi(s), name)
+            assert_wolfe(s, phi, derphi, err_msg=name)
+
+        assert_(c > 3)  # check that the iterator really works...
+
+    def test_scalar_search_wolfe2(self):
+        for name, phi, derphi, old_phi0 in self.scalar_iter():
+            s, phi1, phi0, derphi1 = ls.scalar_search_wolfe2(
+                phi, derphi, phi(0), old_phi0, derphi(0))
+            assert_fp_equal(phi0, phi(0), name)
+            assert_fp_equal(phi1, phi(s), name)
+            if derphi1 is not None:
+                assert_fp_equal(derphi1, derphi(s), name)
+            assert_wolfe(s, phi, derphi, err_msg="%s %g" % (name, old_phi0))
+
+    def test_scalar_search_wolfe2_with_low_amax(self):
+        def phi(alpha):
+            return (alpha - 5) ** 2
+
+        def derphi(alpha):
+            return 2 * (alpha - 5)
+
+        s, _, _, _ = assert_warns(LineSearchWarning,
+                                  ls.scalar_search_wolfe2, phi, derphi, amax=0.001)
+        assert_(s is None)
+
+    def test_scalar_search_armijo(self):
+        for name, phi, derphi, old_phi0 in self.scalar_iter():
+            s, phi1 = ls.scalar_search_armijo(phi, phi(0), derphi(0))
+            assert_fp_equal(phi1, phi(s), name)
+            assert_armijo(s, phi, err_msg="%s %g" % (name, old_phi0))
+
+    # -- Generic line searches
+
+    def test_line_search_wolfe1(self):
+        c = 0
+        smax = 100
+        for name, f, fprime, x, p, old_f in self.line_iter():
+            f0 = f(x)
+            g0 = fprime(x)
+            self.fcount = 0
+            s, fc, gc, fv, ofv, gv = ls.line_search_wolfe1(f, fprime, x, p,
+                                                           g0, f0, old_f,
+                                                           amax=smax)
+            assert_equal(self.fcount, fc+gc)
+            assert_fp_equal(ofv, f(x))
+            if s is None:
+                continue
+            assert_fp_equal(fv, f(x + s*p))
+            assert_array_almost_equal(gv, fprime(x + s*p), decimal=14)
+            if s < smax:
+                c += 1
+                assert_line_wolfe(x, p, s, f, fprime, err_msg=name)
+
+        assert_(c > 3)  # check that the iterator really works...
+
+    def test_line_search_wolfe2(self):
+        c = 0
+        smax = 512
+        for name, f, fprime, x, p, old_f in self.line_iter():
+            f0 = f(x)
+            g0 = fprime(x)
+            self.fcount = 0
+            with suppress_warnings() as sup:
+                sup.filter(LineSearchWarning,
+                           "The line search algorithm could not find a solution")
+                sup.filter(LineSearchWarning,
+                           "The line search algorithm did not converge")
+                s, fc, gc, fv, ofv, gv = ls.line_search_wolfe2(f, fprime, x, p,
+                                                               g0, f0, old_f,
+                                                               amax=smax)
+            assert_equal(self.fcount, fc+gc)
+            assert_fp_equal(ofv, f(x))
+            assert_fp_equal(fv, f(x + s*p))
+            if gv is not None:
+                assert_array_almost_equal(gv, fprime(x + s*p), decimal=14)
+            if s < smax:
+                c += 1
+                assert_line_wolfe(x, p, s, f, fprime, err_msg=name)
+        assert_(c > 3)  # check that the iterator really works...
+
+    def test_line_search_wolfe2_bounds(self):
+        # See gh-7475
+
+        # For this f and p, starting at a point on axis 0, the strong Wolfe
+        # condition 2 is met if and only if the step length s satisfies
+        # |x + s| <= c2 * |x|
+        f = lambda x: np.dot(x, x)
+        fp = lambda x: 2 * x
+        p = np.array([1, 0])
+
+        # Smallest s satisfying strong Wolfe conditions for these arguments is 30
+        x = -60 * p
+        c2 = 0.5
+
+        s, _, _, _, _, _ = ls.line_search_wolfe2(f, fp, x, p, amax=30, c2=c2)
+        assert_line_wolfe(x, p, s, f, fp)
+
+        s, _, _, _, _, _ = assert_warns(LineSearchWarning,
+                                        ls.line_search_wolfe2, f, fp, x, p,
+                                        amax=29, c2=c2)
+        assert_(s is None)
+
+        # s=30 will only be tried on the 6th iteration, so this won't converge
+        assert_warns(LineSearchWarning, ls.line_search_wolfe2, f, fp, x, p,
+                     c2=c2, maxiter=5)
+
+    def test_line_search_armijo(self):
+        c = 0
+        for name, f, fprime, x, p, old_f in self.line_iter():
+            f0 = f(x)
+            g0 = fprime(x)
+            self.fcount = 0
+            s, fc, fv = ls.line_search_armijo(f, x, p, g0, f0)
+            c += 1
+            assert_equal(self.fcount, fc)
+            assert_fp_equal(fv, f(x + s*p))
+            assert_line_armijo(x, p, s, f, err_msg=name)
+        assert_(c >= 9)
+
+    # -- More specific tests
+
+    def test_armijo_terminate_1(self):
+        # Armijo should evaluate the function only once if the trial step
+        # is already suitable
+        count = [0]
+
+        def phi(s):
+            count[0] += 1
+            return -s + 0.01*s**2
+        s, phi1 = ls.scalar_search_armijo(phi, phi(0), -1, alpha0=1)
+        assert_equal(s, 1)
+        assert_equal(count[0], 2)
+        assert_armijo(s, phi)
+
+    def test_wolfe_terminate(self):
+        # wolfe1 and wolfe2 should also evaluate the function only a few
+        # times if the trial step is already suitable
+
+        def phi(s):
+            count[0] += 1
+            return -s + 0.05*s**2
+
+        def derphi(s):
+            count[0] += 1
+            return -1 + 0.05*2*s
+
+        for func in [ls.scalar_search_wolfe1, ls.scalar_search_wolfe2]:
+            count = [0]
+            r = func(phi, derphi, phi(0), None, derphi(0))
+            assert_(r[0] is not None, (r, func))
+            assert_(count[0] <= 2 + 2, (count, func))
+            assert_wolfe(r[0], phi, derphi, err_msg=str(func))

venv/Lib/site-packages/scipy/optimize/tests/test_lsq_common.py

@@ -0,0 +1,297 @@
+from numpy.testing import assert_, assert_allclose, assert_equal
+from pytest import raises as assert_raises
+import numpy as np
+
+from scipy.optimize._lsq.common import (
+    step_size_to_bound, find_active_constraints, make_strictly_feasible,
+    CL_scaling_vector, intersect_trust_region, build_quadratic_1d,
+    minimize_quadratic_1d, evaluate_quadratic, reflective_transformation,
+    left_multiplied_operator, right_multiplied_operator)
+
+
+class TestBounds(object):
+    def test_step_size_to_bounds(self):
+        lb = np.array([-1.0, 2.5, 10.0])
+        ub = np.array([1.0, 5.0, 100.0])
+        x = np.array([0.0, 2.5, 12.0])
+
+        s = np.array([0.1, 0.0, 0.0])
+        step, hits = step_size_to_bound(x, s, lb, ub)
+        assert_equal(step, 10)
+        assert_equal(hits, [1, 0, 0])
+
+        s = np.array([0.01, 0.05, -1.0])
+        step, hits = step_size_to_bound(x, s, lb, ub)
+        assert_equal(step, 2)
+        assert_equal(hits, [0, 0, -1])
+
+        s = np.array([10.0, -0.0001, 100.0])
+        step, hits = step_size_to_bound(x, s, lb, ub)
+        assert_equal(step, np.array(-0))
+        assert_equal(hits, [0, -1, 0])
+
+        s = np.array([1.0, 0.5, -2.0])
+        step, hits = step_size_to_bound(x, s, lb, ub)
+        assert_equal(step, 1.0)
+        assert_equal(hits, [1, 0, -1])
+
+        s = np.zeros(3)
+        step, hits = step_size_to_bound(x, s, lb, ub)
+        assert_equal(step, np.inf)
+        assert_equal(hits, [0, 0, 0])
+
+    def test_find_active_constraints(self):
+        lb = np.array([0.0, -10.0, 1.0])
+        ub = np.array([1.0, 0.0, 100.0])
+
+        x = np.array([0.5, -5.0, 2.0])
+        active = find_active_constraints(x, lb, ub)
+        assert_equal(active, [0, 0, 0])
+
+        x = np.array([0.0, 0.0, 10.0])
+        active = find_active_constraints(x, lb, ub)
+        assert_equal(active, [-1, 1, 0])
+
+        active = find_active_constraints(x, lb, ub, rtol=0)
+        assert_equal(active, [-1, 1, 0])
+
+        x = np.array([1e-9, -1e-8, 100 - 1e-9])
+        active = find_active_constraints(x, lb, ub)
+        assert_equal(active, [0, 0, 1])
+
+        active = find_active_constraints(x, lb, ub, rtol=1.5e-9)
+        assert_equal(active, [-1, 0, 1])
+
+        lb = np.array([1.0, -np.inf, -np.inf])
+        ub = np.array([np.inf, 10.0, np.inf])
+
+        x = np.ones(3)
+        active = find_active_constraints(x, lb, ub)
+        assert_equal(active, [-1, 0, 0])
+
+        # Handles out-of-bound cases.
+        x = np.array([0.0, 11.0, 0.0])
+        active = find_active_constraints(x, lb, ub)
+        assert_equal(active, [-1, 1, 0])
+
+        active = find_active_constraints(x, lb, ub, rtol=0)
+        assert_equal(active, [-1, 1, 0])
+
+    def test_make_strictly_feasible(self):
+        lb = np.array([-0.5, -0.8, 2.0])
+        ub = np.array([0.8, 1.0, 3.0])
+
+        x = np.array([-0.5, 0.0, 2 + 1e-10])
+
+        x_new = make_strictly_feasible(x, lb, ub, rstep=0)
+        assert_(x_new[0] > -0.5)
+        assert_equal(x_new[1:], x[1:])
+
+        x_new = make_strictly_feasible(x, lb, ub, rstep=1e-4)
+        assert_equal(x_new, [-0.5 + 1e-4, 0.0, 2 * (1 + 1e-4)])
+
+        x = np.array([-0.5, -1, 3.1])
+        x_new = make_strictly_feasible(x, lb, ub)
+        assert_(np.all((x_new >= lb) & (x_new <= ub)))
+
+        x_new = make_strictly_feasible(x, lb, ub, rstep=0)
+        assert_(np.all((x_new >= lb) & (x_new <= ub)))
+
+        lb = np.array([-1, 100.0])
+        ub = np.array([1, 100.0 + 1e-10])
+        x = np.array([0, 100.0])
+        x_new = make_strictly_feasible(x, lb, ub, rstep=1e-8)
+        assert_equal(x_new, [0, 100.0 + 0.5e-10])
+
+    def test_scaling_vector(self):
+        lb = np.array([-np.inf, -5.0, 1.0, -np.inf])
+        ub = np.array([1.0, np.inf, 10.0, np.inf])
+        x = np.array([0.5, 2.0, 5.0, 0.0])
+        g = np.array([1.0, 0.1, -10.0, 0.0])
+        v, dv = CL_scaling_vector(x, g, lb, ub)
+        assert_equal(v, [1.0, 7.0, 5.0, 1.0])
+        assert_equal(dv, [0.0, 1.0, -1.0, 0.0])
+
+
+class TestQuadraticFunction(object):
+    def setup_method(self):
+        self.J = np.array([
+            [0.1, 0.2],
+            [-1.0, 1.0],
+            [0.5, 0.2]])
+        self.g = np.array([0.8, -2.0])
+        self.diag = np.array([1.0, 2.0])
+
+    def test_build_quadratic_1d(self):
+        s = np.zeros(2)
+        a, b = build_quadratic_1d(self.J, self.g, s)
+        assert_equal(a, 0)
+        assert_equal(b, 0)
+
+        a, b = build_quadratic_1d(self.J, self.g, s, diag=self.diag)
+        assert_equal(a, 0)
+        assert_equal(b, 0)
+
+        s = np.array([1.0, -1.0])
+        a, b = build_quadratic_1d(self.J, self.g, s)
+        assert_equal(a, 2.05)
+        assert_equal(b, 2.8)
+
+        a, b = build_quadratic_1d(self.J, self.g, s, diag=self.diag)
+        assert_equal(a, 3.55)
+        assert_equal(b, 2.8)
+
+        s0 = np.array([0.5, 0.5])
+        a, b, c = build_quadratic_1d(self.J, self.g, s, diag=self.diag, s0=s0)
+        assert_equal(a, 3.55)
+        assert_allclose(b, 2.39)
+        assert_allclose(c, -0.1525)
+
+    def test_minimize_quadratic_1d(self):
+        a = 5
+        b = -1
+
+        t, y = minimize_quadratic_1d(a, b, 1, 2)
+        assert_equal(t, 1)
+        assert_allclose(y, a * t**2 + b * t, rtol=1e-15)
+
+        t, y = minimize_quadratic_1d(a, b, -2, -1)
+        assert_equal(t, -1)
+        assert_allclose(y, a * t**2 + b * t, rtol=1e-15)
+
+        t, y = minimize_quadratic_1d(a, b, -1, 1)
+        assert_equal(t, 0.1)
+        assert_allclose(y, a * t**2 + b * t, rtol=1e-15)
+
+        c = 10
+        t, y = minimize_quadratic_1d(a, b, -1, 1, c=c)
+        assert_equal(t, 0.1)
+        assert_allclose(y, a * t**2 + b * t + c, rtol=1e-15)
+
+        t, y = minimize_quadratic_1d(a, b, -np.inf, np.inf, c=c)
+        assert_equal(t, 0.1)
+        assert_allclose(y, a * t ** 2 + b * t + c, rtol=1e-15)
+
+        t, y = minimize_quadratic_1d(a, b, 0, np.inf, c=c)
+        assert_equal(t, 0.1)
+        assert_allclose(y, a * t ** 2 + b * t + c, rtol=1e-15)
+
+        t, y = minimize_quadratic_1d(a, b, -np.inf, 0, c=c)
+        assert_equal(t, 0)
+        assert_allclose(y, a * t ** 2 + b * t + c, rtol=1e-15)
+
+        a = -1
+        b = 0.2
+        t, y = minimize_quadratic_1d(a, b, -np.inf, np.inf)
+        assert_equal(y, -np.inf)
+
+        t, y = minimize_quadratic_1d(a, b, 0, np.inf)
+        assert_equal(t, np.inf)
+        assert_equal(y, -np.inf)
+
+        t, y = minimize_quadratic_1d(a, b, -np.inf, 0)
+        assert_equal(t, -np.inf)
+        assert_equal(y, -np.inf)
+
+    def test_evaluate_quadratic(self):
+        s = np.array([1.0, -1.0])
+
+        value = evaluate_quadratic(self.J, self.g, s)
+        assert_equal(value, 4.85)
+
+        value = evaluate_quadratic(self.J, self.g, s, diag=self.diag)
+        assert_equal(value, 6.35)
+
+        s = np.array([[1.0, -1.0],
+                     [1.0, 1.0],
+                     [0.0, 0.0]])
+
+        values = evaluate_quadratic(self.J, self.g, s)
+        assert_allclose(values, [4.85, -0.91, 0.0])
+
+        values = evaluate_quadratic(self.J, self.g, s, diag=self.diag)
+        assert_allclose(values, [6.35, 0.59, 0.0])
+
+
+class TestTrustRegion(object):
+    def test_intersect(self):
+        Delta = 1.0
+
+        x = np.zeros(3)
+        s = np.array([1.0, 0.0, 0.0])
+        t_neg, t_pos = intersect_trust_region(x, s, Delta)
+        assert_equal(t_neg, -1)
+        assert_equal(t_pos, 1)
+
+        s = np.array([-1.0, 1.0, -1.0])
+        t_neg, t_pos = intersect_trust_region(x, s, Delta)
+        assert_allclose(t_neg, -3**-0.5)
+        assert_allclose(t_pos, 3**-0.5)
+
+        x = np.array([0.5, -0.5, 0])
+        s = np.array([0, 0, 1.0])
+        t_neg, t_pos = intersect_trust_region(x, s, Delta)
+        assert_allclose(t_neg, -2**-0.5)
+        assert_allclose(t_pos, 2**-0.5)
+
+        x = np.ones(3)
+        assert_raises(ValueError, intersect_trust_region, x, s, Delta)
+
+        x = np.zeros(3)
+        s = np.zeros(3)
+        assert_raises(ValueError, intersect_trust_region, x, s, Delta)
+
+
+def test_reflective_transformation():
+    lb = np.array([-1, -2], dtype=float)
+    ub = np.array([5, 3], dtype=float)
+
+    y = np.array([0, 0])
+    x, g = reflective_transformation(y, lb, ub)
+    assert_equal(x, y)
+    assert_equal(g, np.ones(2))
+
+    y = np.array([-4, 4], dtype=float)
+
+    x, g = reflective_transformation(y, lb, np.array([np.inf, np.inf]))
+    assert_equal(x, [2, 4])
+    assert_equal(g, [-1, 1])
+
+    x, g = reflective_transformation(y, np.array([-np.inf, -np.inf]), ub)
+    assert_equal(x, [-4, 2])
+    assert_equal(g, [1, -1])
+
+    x, g = reflective_transformation(y, lb, ub)
+    assert_equal(x, [2, 2])
+    assert_equal(g, [-1, -1])
+
+    lb = np.array([-np.inf, -2])
+    ub = np.array([5, np.inf])
+    y = np.array([10, 10], dtype=float)
+    x, g = reflective_transformation(y, lb, ub)
+    assert_equal(x, [0, 10])
+    assert_equal(g, [-1, 1])
+
+
+def test_linear_operators():
+    A = np.arange(6).reshape((3, 2))
+
+    d_left = np.array([-1, 2, 5])
+    DA = np.diag(d_left).dot(A)
+    J_left = left_multiplied_operator(A, d_left)
+
+    d_right = np.array([5, 10])
+    AD = A.dot(np.diag(d_right))
+    J_right = right_multiplied_operator(A, d_right)
+
+    x = np.array([-2, 3])
+    X = -2 * np.arange(2, 8).reshape((2, 3))
+    xt = np.array([0, -2, 15])
+
+    assert_allclose(DA.dot(x), J_left.matvec(x))
+    assert_allclose(DA.dot(X), J_left.matmat(X))
+    assert_allclose(DA.T.dot(xt), J_left.rmatvec(xt))
+
+    assert_allclose(AD.dot(x), J_right.matvec(x))
+    assert_allclose(AD.dot(X), J_right.matmat(X))
+    assert_allclose(AD.T.dot(xt), J_right.rmatvec(xt))

venv/Lib/site-packages/scipy/optimize/tests/test_lsq_linear.py

@@ -0,0 +1,162 @@
+import numpy as np
+from numpy.linalg import lstsq
+from numpy.testing import assert_allclose, assert_equal, assert_
+
+from scipy.sparse import rand
+from scipy.sparse.linalg import aslinearoperator
+from scipy.optimize import lsq_linear
+
+
+A = np.array([
+    [0.171, -0.057],
+    [-0.049, -0.248],
+    [-0.166, 0.054],
+])
+b = np.array([0.074, 1.014, -0.383])
+
+
+class BaseMixin(object):
+    def setup_method(self):
+        self.rnd = np.random.RandomState(0)
+
+    def test_dense_no_bounds(self):
+        for lsq_solver in self.lsq_solvers:
+            res = lsq_linear(A, b, method=self.method, lsq_solver=lsq_solver)
+            assert_allclose(res.x, lstsq(A, b, rcond=-1)[0])
+
+    def test_dense_bounds(self):
+        # Solutions for comparison are taken from MATLAB.
+        lb = np.array([-1, -10])
+        ub = np.array([1, 0])
+        for lsq_solver in self.lsq_solvers:
+            res = lsq_linear(A, b, (lb, ub), method=self.method,
+                             lsq_solver=lsq_solver)
+            assert_allclose(res.x, lstsq(A, b, rcond=-1)[0])
+
+        lb = np.array([0.0, -np.inf])
+        for lsq_solver in self.lsq_solvers:
+            res = lsq_linear(A, b, (lb, np.inf), method=self.method,
+                             lsq_solver=lsq_solver)
+            assert_allclose(res.x, np.array([0.0, -4.084174437334673]),
+                            atol=1e-6)
+
+        lb = np.array([-1, 0])
+        for lsq_solver in self.lsq_solvers:
+            res = lsq_linear(A, b, (lb, np.inf), method=self.method,
+                             lsq_solver=lsq_solver)
+            assert_allclose(res.x, np.array([0.448427311733504, 0]),
+                            atol=1e-15)
+
+        ub = np.array([np.inf, -5])
+        for lsq_solver in self.lsq_solvers:
+            res = lsq_linear(A, b, (-np.inf, ub), method=self.method,
+                             lsq_solver=lsq_solver)
+            assert_allclose(res.x, np.array([-0.105560998682388, -5]))
+
+        ub = np.array([-1, np.inf])
+        for lsq_solver in self.lsq_solvers:
+            res = lsq_linear(A, b, (-np.inf, ub), method=self.method,
+                             lsq_solver=lsq_solver)
+            assert_allclose(res.x, np.array([-1, -4.181102129483254]))
+
+        lb = np.array([0, -4])
+        ub = np.array([1, 0])
+        for lsq_solver in self.lsq_solvers:
+            res = lsq_linear(A, b, (lb, ub), method=self.method,
+                             lsq_solver=lsq_solver)
+            assert_allclose(res.x, np.array([0.005236663400791, -4]))
+
+    def test_dense_rank_deficient(self):
+        A = np.array([[-0.307, -0.184]])
+        b = np.array([0.773])
+        lb = [-0.1, -0.1]
+        ub = [0.1, 0.1]
+        for lsq_solver in self.lsq_solvers:
+            res = lsq_linear(A, b, (lb, ub), method=self.method,
+                             lsq_solver=lsq_solver)
+            assert_allclose(res.x, [-0.1, -0.1])
+
+        A = np.array([
+            [0.334, 0.668],
+            [-0.516, -1.032],
+            [0.192, 0.384],
+        ])
+        b = np.array([-1.436, 0.135, 0.909])
+        lb = [0, -1]
+        ub = [1, -0.5]
+        for lsq_solver in self.lsq_solvers:
+            res = lsq_linear(A, b, (lb, ub), method=self.method,
+                             lsq_solver=lsq_solver)
+            assert_allclose(res.optimality, 0, atol=1e-11)
+
+    def test_full_result(self):
+        lb = np.array([0, -4])
+        ub = np.array([1, 0])
+        res = lsq_linear(A, b, (lb, ub), method=self.method)
+
+        assert_allclose(res.x, [0.005236663400791, -4])
+
+        r = A.dot(res.x) - b
+        assert_allclose(res.cost, 0.5 * np.dot(r, r))
+        assert_allclose(res.fun, r)
+
+        assert_allclose(res.optimality, 0.0, atol=1e-12)
+        assert_equal(res.active_mask, [0, -1])
+        assert_(res.nit < 15)
+        assert_(res.status == 1 or res.status == 3)
+        assert_(isinstance(res.message, str))
+        assert_(res.success)
+
+    # This is a test for issue #9982.
+    def test_almost_singular(self):
+        A = np.array(
+            [[0.8854232310355122, 0.0365312146937765, 0.0365312146836789],
+             [0.3742460132129041, 0.0130523214078376, 0.0130523214077873],
+             [0.9680633871281361, 0.0319366128718639, 0.0319366128718388]])
+
+        b = np.array(
+            [0.0055029366538097, 0.0026677442422208, 0.0066612514782381])
+
+        result = lsq_linear(A, b, method=self.method)
+        assert_(result.cost < 1.1e-8)
+
+
+class SparseMixin(object):
+    def test_sparse_and_LinearOperator(self):
+        m = 5000
+        n = 1000
+        A = rand(m, n, random_state=0)
+        b = self.rnd.randn(m)
+        res = lsq_linear(A, b)
+        assert_allclose(res.optimality, 0, atol=1e-6)
+
+        A = aslinearoperator(A)
+        res = lsq_linear(A, b)
+        assert_allclose(res.optimality, 0, atol=1e-6)
+
+    def test_sparse_bounds(self):
+        m = 5000
+        n = 1000
+        A = rand(m, n, random_state=0)
+        b = self.rnd.randn(m)
+        lb = self.rnd.randn(n)
+        ub = lb + 1
+        res = lsq_linear(A, b, (lb, ub))
+        assert_allclose(res.optimality, 0.0, atol=1e-6)
+
+        res = lsq_linear(A, b, (lb, ub), lsmr_tol=1e-13)
+        assert_allclose(res.optimality, 0.0, atol=1e-6)
+
+        res = lsq_linear(A, b, (lb, ub), lsmr_tol='auto')
+        assert_allclose(res.optimality, 0.0, atol=1e-6)
+
+
+class TestTRF(BaseMixin, SparseMixin):
+    method = 'trf'
+    lsq_solvers = ['exact', 'lsmr']
+
+
+class TestBVLS(BaseMixin):
+    method = 'bvls'
+    lsq_solvers = ['exact']
+

venv/Lib/site-packages/scipy/optimize/tests/test_minimize_constrained.py

@@ -0,0 +1,678 @@
+import numpy as np
+import pytest
+from scipy.linalg import block_diag
+from scipy.sparse import csc_matrix
+from numpy.testing import (TestCase, assert_array_almost_equal,
+                           assert_array_less, assert_,
+                           suppress_warnings)
+from pytest import raises
+from scipy.optimize import (NonlinearConstraint,
+                            LinearConstraint,
+                            Bounds,
+                            minimize,
+                            BFGS,
+                            SR1)
+
+
+class Maratos:
+    """Problem 15.4 from Nocedal and Wright
+
+    The following optimization problem:
+        minimize 2*(x[0]**2 + x[1]**2 - 1) - x[0]
+        Subject to: x[0]**2 + x[1]**2 - 1 = 0
+    """
+
+    def __init__(self, degrees=60, constr_jac=None, constr_hess=None):
+        rads = degrees/180*np.pi
+        self.x0 = [np.cos(rads), np.sin(rads)]
+        self.x_opt = np.array([1.0, 0.0])
+        self.constr_jac = constr_jac
+        self.constr_hess = constr_hess
+        self.bounds = None
+
+    def fun(self, x):
+        return 2*(x[0]**2 + x[1]**2 - 1) - x[0]
+
+    def grad(self, x):
+        return np.array([4*x[0]-1, 4*x[1]])
+
+    def hess(self, x):
+        return 4*np.eye(2)
+
+    @property
+    def constr(self):
+        def fun(x):
+            return x[0]**2 + x[1]**2
+
+        if self.constr_jac is None:
+            def jac(x):
+                return [[2*x[0], 2*x[1]]]
+        else:
+            jac = self.constr_jac
+
+        if self.constr_hess is None:
+            def hess(x, v):
+                return 2*v[0]*np.eye(2)
+        else:
+            hess = self.constr_hess
+
+        return NonlinearConstraint(fun, 1, 1, jac, hess)
+
+
+class MaratosTestArgs:
+    """Problem 15.4 from Nocedal and Wright
+
+    The following optimization problem:
+        minimize 2*(x[0]**2 + x[1]**2 - 1) - x[0]
+        Subject to: x[0]**2 + x[1]**2 - 1 = 0
+    """
+
+    def __init__(self, a, b, degrees=60, constr_jac=None, constr_hess=None):
+        rads = degrees/180*np.pi
+        self.x0 = [np.cos(rads), np.sin(rads)]
+        self.x_opt = np.array([1.0, 0.0])
+        self.constr_jac = constr_jac
+        self.constr_hess = constr_hess
+        self.a = a
+        self.b = b
+        self.bounds = None
+
+    def _test_args(self, a, b):
+        if self.a != a or self.b != b:
+            raise ValueError()
+
+    def fun(self, x, a, b):
+        self._test_args(a, b)
+        return 2*(x[0]**2 + x[1]**2 - 1) - x[0]
+
+    def grad(self, x, a, b):
+        self._test_args(a, b)
+        return np.array([4*x[0]-1, 4*x[1]])
+
+    def hess(self, x, a, b):
+        self._test_args(a, b)
+        return 4*np.eye(2)
+
+    @property
+    def constr(self):
+        def fun(x):
+            return x[0]**2 + x[1]**2
+
+        if self.constr_jac is None:
+            def jac(x):
+                return [[4*x[0], 4*x[1]]]
+        else:
+            jac = self.constr_jac
+
+        if self.constr_hess is None:
+            def hess(x, v):
+                return 2*v[0]*np.eye(2)
+        else:
+            hess = self.constr_hess
+
+        return NonlinearConstraint(fun, 1, 1, jac, hess)
+
+
+class MaratosGradInFunc:
+    """Problem 15.4 from Nocedal and Wright
+
+    The following optimization problem:
+        minimize 2*(x[0]**2 + x[1]**2 - 1) - x[0]
+        Subject to: x[0]**2 + x[1]**2 - 1 = 0
+    """
+
+    def __init__(self, degrees=60, constr_jac=None, constr_hess=None):
+        rads = degrees/180*np.pi
+        self.x0 = [np.cos(rads), np.sin(rads)]
+        self.x_opt = np.array([1.0, 0.0])
+        self.constr_jac = constr_jac
+        self.constr_hess = constr_hess
+        self.bounds = None
+
+    def fun(self, x):
+        return (2*(x[0]**2 + x[1]**2 - 1) - x[0],
+                np.array([4*x[0]-1, 4*x[1]]))
+
+    @property
+    def grad(self):
+        return True
+
+    def hess(self, x):
+        return 4*np.eye(2)
+
+    @property
+    def constr(self):
+        def fun(x):
+            return x[0]**2 + x[1]**2
+
+        if self.constr_jac is None:
+            def jac(x):
+                return [[4*x[0], 4*x[1]]]
+        else:
+            jac = self.constr_jac
+
+        if self.constr_hess is None:
+            def hess(x, v):
+                return 2*v[0]*np.eye(2)
+        else:
+            hess = self.constr_hess
+
+        return NonlinearConstraint(fun, 1, 1, jac, hess)
+
+
+class HyperbolicIneq:
+    """Problem 15.1 from Nocedal and Wright
+
+    The following optimization problem:
+        minimize 1/2*(x[0] - 2)**2 + 1/2*(x[1] - 1/2)**2
+        Subject to: 1/(x[0] + 1) - x[1] >= 1/4
+                                   x[0] >= 0
+                                   x[1] >= 0
+    """
+    def __init__(self, constr_jac=None, constr_hess=None):
+        self.x0 = [0, 0]
+        self.x_opt = [1.952823, 0.088659]
+        self.constr_jac = constr_jac
+        self.constr_hess = constr_hess
+        self.bounds = Bounds(0, np.inf)
+
+    def fun(self, x):
+        return 1/2*(x[0] - 2)**2 + 1/2*(x[1] - 1/2)**2
+
+    def grad(self, x):
+        return [x[0] - 2, x[1] - 1/2]
+
+    def hess(self, x):
+        return np.eye(2)
+
+    @property
+    def constr(self):
+        def fun(x):
+            return 1/(x[0] + 1) - x[1]
+
+        if self.constr_jac is None:
+            def jac(x):
+                return [[-1/(x[0] + 1)**2, -1]]
+        else:
+            jac = self.constr_jac
+
+        if self.constr_hess is None:
+            def hess(x, v):
+                return 2*v[0]*np.array([[1/(x[0] + 1)**3, 0],
+                                        [0, 0]])
+        else:
+            hess = self.constr_hess
+
+        return NonlinearConstraint(fun, 0.25, np.inf, jac, hess)
+
+
+class Rosenbrock:
+    """Rosenbrock function.
+
+    The following optimization problem:
+        minimize sum(100.0*(x[1:] - x[:-1]**2.0)**2.0 + (1 - x[:-1])**2.0)
+    """
+
+    def __init__(self, n=2, random_state=0):
+        rng = np.random.RandomState(random_state)
+        self.x0 = rng.uniform(-1, 1, n)
+        self.x_opt = np.ones(n)
+        self.bounds = None
+
+    def fun(self, x):
+        x = np.asarray(x)
+        r = np.sum(100.0 * (x[1:] - x[:-1]**2.0)**2.0 + (1 - x[:-1])**2.0,
+                   axis=0)
+        return r
+
+    def grad(self, x):
+        x = np.asarray(x)
+        xm = x[1:-1]
+        xm_m1 = x[:-2]
+        xm_p1 = x[2:]
+        der = np.zeros_like(x)
+        der[1:-1] = (200 * (xm - xm_m1**2) -
+                     400 * (xm_p1 - xm**2) * xm - 2 * (1 - xm))
+        der[0] = -400 * x[0] * (x[1] - x[0]**2) - 2 * (1 - x[0])
+        der[-1] = 200 * (x[-1] - x[-2]**2)
+        return der
+
+    def hess(self, x):
+        x = np.atleast_1d(x)
+        H = np.diag(-400 * x[:-1], 1) - np.diag(400 * x[:-1], -1)
+        diagonal = np.zeros(len(x), dtype=x.dtype)
+        diagonal[0] = 1200 * x[0]**2 - 400 * x[1] + 2
+        diagonal[-1] = 200
+        diagonal[1:-1] = 202 + 1200 * x[1:-1]**2 - 400 * x[2:]
+        H = H + np.diag(diagonal)
+        return H
+
+    @property
+    def constr(self):
+        return ()
+
+
+class IneqRosenbrock(Rosenbrock):
+    """Rosenbrock subject to inequality constraints.
+
+    The following optimization problem:
+        minimize sum(100.0*(x[1] - x[0]**2)**2.0 + (1 - x[0])**2)
+        subject to: x[0] + 2 x[1] <= 1
+
+    Taken from matlab ``fmincon`` documentation.
+    """
+    def __init__(self, random_state=0):
+        Rosenbrock.__init__(self, 2, random_state)
+        self.x0 = [-1, -0.5]
+        self.x_opt = [0.5022, 0.2489]
+        self.bounds = None
+
+    @property
+    def constr(self):
+        A = [[1, 2]]
+        b = 1
+        return LinearConstraint(A, -np.inf, b)
+
+
+class BoundedRosenbrock(Rosenbrock):
+    """Rosenbrock subject to inequality constraints.
+
+    The following optimization problem:
+        minimize sum(100.0*(x[1] - x[0]**2)**2.0 + (1 - x[0])**2)
+        subject to:  -2 <= x[0] <= 0
+                      0 <= x[1] <= 2
+
+    Taken from matlab ``fmincon`` documentation.
+    """
+    def __init__(self, random_state=0):
+        Rosenbrock.__init__(self, 2, random_state)
+        self.x0 = [-0.2, 0.2]
+        self.x_opt = None
+        self.bounds = Bounds([-2, 0], [0, 2])
+
+
+class EqIneqRosenbrock(Rosenbrock):
+    """Rosenbrock subject to equality and inequality constraints.
+
+    The following optimization problem:
+        minimize sum(100.0*(x[1] - x[0]**2)**2.0 + (1 - x[0])**2)
+        subject to: x[0] + 2 x[1] <= 1
+                    2 x[0] + x[1] = 1
+
+    Taken from matlab ``fimincon`` documentation.
+    """
+    def __init__(self, random_state=0):
+        Rosenbrock.__init__(self, 2, random_state)
+        self.x0 = [-1, -0.5]
+        self.x_opt = [0.41494, 0.17011]
+        self.bounds = None
+
+    @property
+    def constr(self):
+        A_ineq = [[1, 2]]
+        b_ineq = 1
+        A_eq = [[2, 1]]
+        b_eq = 1
+        return (LinearConstraint(A_ineq, -np.inf, b_ineq),
+                LinearConstraint(A_eq, b_eq, b_eq))
+
+
+class Elec:
+    """Distribution of electrons on a sphere.
+
+    Problem no 2 from COPS collection [2]_. Find
+    the equilibrium state distribution (of minimal
+    potential) of the electrons positioned on a
+    conducting sphere.
+
+    References
+    ----------
+    .. [1] E. D. Dolan, J. J. Mor\'{e}, and T. S. Munson,
+           "Benchmarking optimization software with COPS 3.0.",
+            Argonne National Lab., Argonne, IL (US), 2004.
+    """
+    def __init__(self, n_electrons=200, random_state=0,
+                 constr_jac=None, constr_hess=None):
+        self.n_electrons = n_electrons
+        self.rng = np.random.RandomState(random_state)
+        # Initial Guess
+        phi = self.rng.uniform(0, 2 * np.pi, self.n_electrons)
+        theta = self.rng.uniform(-np.pi, np.pi, self.n_electrons)
+        x = np.cos(theta) * np.cos(phi)
+        y = np.cos(theta) * np.sin(phi)
+        z = np.sin(theta)
+        self.x0 = np.hstack((x, y, z))
+        self.x_opt = None
+        self.constr_jac = constr_jac
+        self.constr_hess = constr_hess
+        self.bounds = None
+
+    def _get_cordinates(self, x):
+        x_coord = x[:self.n_electrons]
+        y_coord = x[self.n_electrons:2 * self.n_electrons]
+        z_coord = x[2 * self.n_electrons:]
+        return x_coord, y_coord, z_coord
+
+    def _compute_coordinate_deltas(self, x):
+        x_coord, y_coord, z_coord = self._get_cordinates(x)
+        dx = x_coord[:, None] - x_coord
+        dy = y_coord[:, None] - y_coord
+        dz = z_coord[:, None] - z_coord
+        return dx, dy, dz
+
+    def fun(self, x):
+        dx, dy, dz = self._compute_coordinate_deltas(x)
+        with np.errstate(divide='ignore'):
+            dm1 = (dx**2 + dy**2 + dz**2) ** -0.5
+        dm1[np.diag_indices_from(dm1)] = 0
+        return 0.5 * np.sum(dm1)
+
+    def grad(self, x):
+        dx, dy, dz = self._compute_coordinate_deltas(x)
+
+        with np.errstate(divide='ignore'):
+            dm3 = (dx**2 + dy**2 + dz**2) ** -1.5
+        dm3[np.diag_indices_from(dm3)] = 0
+
+        grad_x = -np.sum(dx * dm3, axis=1)
+        grad_y = -np.sum(dy * dm3, axis=1)
+        grad_z = -np.sum(dz * dm3, axis=1)
+
+        return np.hstack((grad_x, grad_y, grad_z))
+
+    def hess(self, x):
+        dx, dy, dz = self._compute_coordinate_deltas(x)
+        d = (dx**2 + dy**2 + dz**2) ** 0.5
+
+        with np.errstate(divide='ignore'):
+            dm3 = d ** -3
+            dm5 = d ** -5
+
+        i = np.arange(self.n_electrons)
+        dm3[i, i] = 0
+        dm5[i, i] = 0
+
+        Hxx = dm3 - 3 * dx**2 * dm5
+        Hxx[i, i] = -np.sum(Hxx, axis=1)
+
+        Hxy = -3 * dx * dy * dm5
+        Hxy[i, i] = -np.sum(Hxy, axis=1)
+
+        Hxz = -3 * dx * dz * dm5
+        Hxz[i, i] = -np.sum(Hxz, axis=1)
+
+        Hyy = dm3 - 3 * dy**2 * dm5
+        Hyy[i, i] = -np.sum(Hyy, axis=1)
+
+        Hyz = -3 * dy * dz * dm5
+        Hyz[i, i] = -np.sum(Hyz, axis=1)
+
+        Hzz = dm3 - 3 * dz**2 * dm5
+        Hzz[i, i] = -np.sum(Hzz, axis=1)
+
+        H = np.vstack((
+            np.hstack((Hxx, Hxy, Hxz)),
+            np.hstack((Hxy, Hyy, Hyz)),
+            np.hstack((Hxz, Hyz, Hzz))
+        ))
+
+        return H
+
+    @property
+    def constr(self):
+        def fun(x):
+            x_coord, y_coord, z_coord = self._get_cordinates(x)
+            return x_coord**2 + y_coord**2 + z_coord**2 - 1
+
+        if self.constr_jac is None:
+            def jac(x):
+                x_coord, y_coord, z_coord = self._get_cordinates(x)
+                Jx = 2 * np.diag(x_coord)
+                Jy = 2 * np.diag(y_coord)
+                Jz = 2 * np.diag(z_coord)
+                return csc_matrix(np.hstack((Jx, Jy, Jz)))
+        else:
+            jac = self.constr_jac
+
+        if self.constr_hess is None:
+            def hess(x, v):
+                D = 2 * np.diag(v)
+                return block_diag(D, D, D)
+        else:
+            hess = self.constr_hess
+
+        return NonlinearConstraint(fun, -np.inf, 0, jac, hess)
+
+
+class TestTrustRegionConstr(TestCase):
+
+    @pytest.mark.slow
+    def test_list_of_problems(self):
+        list_of_problems = [Maratos(),
+                            Maratos(constr_hess='2-point'),
+                            Maratos(constr_hess=SR1()),
+                            Maratos(constr_jac='2-point', constr_hess=SR1()),
+                            MaratosGradInFunc(),
+                            HyperbolicIneq(),
+                            HyperbolicIneq(constr_hess='3-point'),
+                            HyperbolicIneq(constr_hess=BFGS()),
+                            HyperbolicIneq(constr_jac='3-point',
+                                           constr_hess=BFGS()),
+                            Rosenbrock(),
+                            IneqRosenbrock(),
+                            EqIneqRosenbrock(),
+                            BoundedRosenbrock(),
+                            Elec(n_electrons=2),
+                            Elec(n_electrons=2, constr_hess='2-point'),
+                            Elec(n_electrons=2, constr_hess=SR1()),
+                            Elec(n_electrons=2, constr_jac='3-point',
+                                 constr_hess=SR1())]
+
+        for prob in list_of_problems:
+            for grad in (prob.grad, '3-point', False):
+                for hess in (prob.hess,
+                             '3-point',
+                             SR1(),
+                             BFGS(exception_strategy='damp_update'),
+                             BFGS(exception_strategy='skip_update')):
+
+                    # Remove exceptions
+                    if grad in ('2-point', '3-point', 'cs', False) and \
+                       hess in ('2-point', '3-point', 'cs'):
+                        continue
+                    if prob.grad is True and grad in ('3-point', False):
+                        continue
+                    with suppress_warnings() as sup:
+                        sup.filter(UserWarning, "delta_grad == 0.0")
+                        result = minimize(prob.fun, prob.x0,
+                                          method='trust-constr',
+                                          jac=grad, hess=hess,
+                                          bounds=prob.bounds,
+                                          constraints=prob.constr)
+
+                    if prob.x_opt is not None:
+                        assert_array_almost_equal(result.x, prob.x_opt,
+                                                  decimal=5)
+                        # gtol
+                        if result.status == 1:
+                            assert_array_less(result.optimality, 1e-8)
+                    # xtol
+                    if result.status == 2:
+                        assert_array_less(result.tr_radius, 1e-8)
+
+                        if result.method == "tr_interior_point":
+                            assert_array_less(result.barrier_parameter, 1e-8)
+                    # max iter
+                    if result.status in (0, 3):
+                        raise RuntimeError("Invalid termination condition.")
+
+    def test_default_jac_and_hess(self):
+        def fun(x):
+            return (x - 1) ** 2
+        bounds = [(-2, 2)]
+        res = minimize(fun, x0=[-1.5], bounds=bounds, method='trust-constr')
+        assert_array_almost_equal(res.x, 1, decimal=5)
+
+    def test_default_hess(self):
+        def fun(x):
+            return (x - 1) ** 2
+        bounds = [(-2, 2)]
+        res = minimize(fun, x0=[-1.5], bounds=bounds, method='trust-constr',
+                       jac='2-point')
+        assert_array_almost_equal(res.x, 1, decimal=5)
+
+    def test_no_constraints(self):
+        prob = Rosenbrock()
+        result = minimize(prob.fun, prob.x0,
+                          method='trust-constr',
+                          jac=prob.grad, hess=prob.hess)
+        result1 = minimize(prob.fun, prob.x0,
+                           method='L-BFGS-B',
+                           jac='2-point')
+
+        result2 = minimize(prob.fun, prob.x0,
+                           method='L-BFGS-B',
+                           jac='3-point')
+        assert_array_almost_equal(result.x, prob.x_opt, decimal=5)
+        assert_array_almost_equal(result1.x, prob.x_opt, decimal=5)
+        assert_array_almost_equal(result2.x, prob.x_opt, decimal=5)
+
+    def test_hessp(self):
+        prob = Maratos()
+
+        def hessp(x, p):
+            H = prob.hess(x)
+            return H.dot(p)
+
+        result = minimize(prob.fun, prob.x0,
+                          method='trust-constr',
+                          jac=prob.grad, hessp=hessp,
+                          bounds=prob.bounds,
+                          constraints=prob.constr)
+
+        if prob.x_opt is not None:
+            assert_array_almost_equal(result.x, prob.x_opt, decimal=2)
+
+        # gtol
+        if result.status == 1:
+            assert_array_less(result.optimality, 1e-8)
+        # xtol
+        if result.status == 2:
+            assert_array_less(result.tr_radius, 1e-8)
+
+            if result.method == "tr_interior_point":
+                assert_array_less(result.barrier_parameter, 1e-8)
+        # max iter
+        if result.status in (0, 3):
+            raise RuntimeError("Invalid termination condition.")
+
+    def test_args(self):
+        prob = MaratosTestArgs("a", 234)
+
+        result = minimize(prob.fun, prob.x0, ("a", 234),
+                          method='trust-constr',
+                          jac=prob.grad, hess=prob.hess,
+                          bounds=prob.bounds,
+                          constraints=prob.constr)
+
+        if prob.x_opt is not None:
+            assert_array_almost_equal(result.x, prob.x_opt, decimal=2)
+
+        # gtol
+        if result.status == 1:
+            assert_array_less(result.optimality, 1e-8)
+        # xtol
+        if result.status == 2:
+            assert_array_less(result.tr_radius, 1e-8)
+            if result.method == "tr_interior_point":
+                assert_array_less(result.barrier_parameter, 1e-8)
+        # max iter
+        if result.status in (0, 3):
+            raise RuntimeError("Invalid termination condition.")
+
+    def test_raise_exception(self):
+        prob = Maratos()
+
+        raises(ValueError, minimize, prob.fun, prob.x0, method='trust-constr',
+               jac='2-point', hess='2-point', constraints=prob.constr)
+
+    def test_issue_9044(self):
+        # https://github.com/scipy/scipy/issues/9044
+        # Test the returned `OptimizeResult` contains keys consistent with
+        # other solvers.
+
+        def callback(x, info):
+            assert_('nit' in info)
+            assert_('niter' in info)
+
+        result = minimize(lambda x: x**2, [0], jac=lambda x: 2*x,
+                          hess=lambda x: 2, callback=callback,
+                          method='trust-constr')
+        assert_(result.get('success'))
+        assert_(result.get('nit', -1) == 1)
+
+        # Also check existence of the 'niter' attribute, for backward
+        # compatibility
+        assert_(result.get('niter', -1) == 1)
+
+class TestEmptyConstraint(TestCase):
+    """
+    Here we minimize x^2+y^2 subject to x^2-y^2>1.
+    The actual minimum is at (0, 0) which fails the constraint.
+    Therefore we will find a minimum on the boundary at (+/-1, 0).
+
+    When minimizing on the boundary, optimize uses a set of
+    constraints that removes the constraint that sets that
+    boundary.  In our case, there's only one constraint, so
+    the result is an empty constraint.
+
+    This tests that the empty constraint works.
+    """
+    def test_empty_constraint(self):
+
+        def function(x):
+            return x[0]**2 + x[1]**2
+
+        def functionjacobian(x):
+            return np.array([2.*x[0], 2.*x[1]])
+
+        def functionhvp(x, v):
+            return 2.*v
+
+        def constraint(x):
+            return np.array([x[0]**2 - x[1]**2])
+
+        def constraintjacobian(x):
+            return np.array([[2*x[0], -2*x[1]]])
+
+        def constraintlcoh(x, v):
+            return np.array([[2., 0.], [0., -2.]]) * v[0]
+
+        constraint = NonlinearConstraint(constraint, 1., np.inf, constraintjacobian, constraintlcoh)
+
+        startpoint = [1., 2.]
+
+        bounds = Bounds([-np.inf, -np.inf], [np.inf, np.inf])
+
+        result = minimize(
+          function,
+          startpoint,
+          method='trust-constr',
+          jac=functionjacobian,
+          hessp=functionhvp,
+          constraints=[constraint],
+          bounds=bounds,
+        )
+
+        assert_array_almost_equal(abs(result.x), np.array([1, 0]), decimal=4)
+
+
+def test_bug_11886():
+    def opt(x):
+        return x[0]**2+x[1]**2
+
+    with np.testing.suppress_warnings() as sup:
+        sup.filter(PendingDeprecationWarning)
+        A = np.matrix(np.diag([1, 1]))
+    lin_cons = LinearConstraint(A, -1, np.inf)
+    minimize(opt, 2*[1], constraints = lin_cons)  # just checking that there are no errors

venv/Lib/site-packages/scipy/optimize/tests/test_minpack.py

@@ -0,0 +1,882 @@
+"""
+Unit tests for optimization routines from minpack.py.
+"""
+import warnings
+
+from numpy.testing import (assert_, assert_almost_equal, assert_array_equal,
+                           assert_array_almost_equal, assert_allclose,
+                           assert_warns, suppress_warnings)
+from pytest import raises as assert_raises
+import numpy as np
+from numpy import array, float64
+from multiprocessing.pool import ThreadPool
+
+from scipy import optimize
+from scipy.special import lambertw
+from scipy.optimize.minpack import leastsq, curve_fit, fixed_point
+from scipy.optimize import OptimizeWarning
+
+
+class ReturnShape(object):
+    """This class exists to create a callable that does not have a '__name__' attribute.
+
+    __init__ takes the argument 'shape', which should be a tuple of ints. When an instance
+    is called with a single argument 'x', it returns numpy.ones(shape).
+    """
+
+    def __init__(self, shape):
+        self.shape = shape
+
+    def __call__(self, x):
+        return np.ones(self.shape)
+
+
+def dummy_func(x, shape):
+    """A function that returns an array of ones of the given shape.
+    `x` is ignored.
+    """
+    return np.ones(shape)
+
+
+def sequence_parallel(fs):
+    pool = ThreadPool(len(fs))
+    try:
+        return pool.map(lambda f: f(), fs)
+    finally:
+        pool.terminate()
+
+
+# Function and Jacobian for tests of solvers for systems of nonlinear
+# equations
+
+
+def pressure_network(flow_rates, Qtot, k):
+    """Evaluate non-linear equation system representing
+    the pressures and flows in a system of n parallel pipes::
+
+        f_i = P_i - P_0, for i = 1..n
+        f_0 = sum(Q_i) - Qtot
+
+    where Q_i is the flow rate in pipe i and P_i the pressure in that pipe.
+    Pressure is modeled as a P=kQ**2 where k is a valve coefficient and
+    Q is the flow rate.
+
+    Parameters
+    ----------
+    flow_rates : float
+        A 1-D array of n flow rates [kg/s].
+    k : float
+        A 1-D array of n valve coefficients [1/kg m].
+    Qtot : float
+        A scalar, the total input flow rate [kg/s].
+
+    Returns
+    -------
+    F : float
+        A 1-D array, F[i] == f_i.
+
+    """
+    P = k * flow_rates**2
+    F = np.hstack((P[1:] - P[0], flow_rates.sum() - Qtot))
+    return F
+
+
+def pressure_network_jacobian(flow_rates, Qtot, k):
+    """Return the jacobian of the equation system F(flow_rates)
+    computed by `pressure_network` with respect to
+    *flow_rates*. See `pressure_network` for the detailed
+    description of parrameters.
+
+    Returns
+    -------
+    jac : float
+        *n* by *n* matrix ``df_i/dQ_i`` where ``n = len(flow_rates)``
+        and *f_i* and *Q_i* are described in the doc for `pressure_network`
+    """
+    n = len(flow_rates)
+    pdiff = np.diag(flow_rates[1:] * 2 * k[1:] - 2 * flow_rates[0] * k[0])
+
+    jac = np.empty((n, n))
+    jac[:n-1, :n-1] = pdiff * 0
+    jac[:n-1, n-1] = 0
+    jac[n-1, :] = np.ones(n)
+
+    return jac
+
+
+def pressure_network_fun_and_grad(flow_rates, Qtot, k):
+    return (pressure_network(flow_rates, Qtot, k),
+            pressure_network_jacobian(flow_rates, Qtot, k))
+
+
+class TestFSolve(object):
+    def test_pressure_network_no_gradient(self):
+        # fsolve without gradient, equal pipes -> equal flows.
+        k = np.full(4, 0.5)
+        Qtot = 4
+        initial_guess = array([2., 0., 2., 0.])
+        final_flows, info, ier, mesg = optimize.fsolve(
+            pressure_network, initial_guess, args=(Qtot, k),
+            full_output=True)
+        assert_array_almost_equal(final_flows, np.ones(4))
+        assert_(ier == 1, mesg)
+
+    def test_pressure_network_with_gradient(self):
+        # fsolve with gradient, equal pipes -> equal flows
+        k = np.full(4, 0.5)
+        Qtot = 4
+        initial_guess = array([2., 0., 2., 0.])
+        final_flows = optimize.fsolve(
+            pressure_network, initial_guess, args=(Qtot, k),
+            fprime=pressure_network_jacobian)
+        assert_array_almost_equal(final_flows, np.ones(4))
+
+    def test_wrong_shape_func_callable(self):
+        func = ReturnShape(1)
+        # x0 is a list of two elements, but func will return an array with
+        # length 1, so this should result in a TypeError.
+        x0 = [1.5, 2.0]
+        assert_raises(TypeError, optimize.fsolve, func, x0)
+
+    def test_wrong_shape_func_function(self):
+        # x0 is a list of two elements, but func will return an array with
+        # length 1, so this should result in a TypeError.
+        x0 = [1.5, 2.0]
+        assert_raises(TypeError, optimize.fsolve, dummy_func, x0, args=((1,),))
+
+    def test_wrong_shape_fprime_callable(self):
+        func = ReturnShape(1)
+        deriv_func = ReturnShape((2,2))
+        assert_raises(TypeError, optimize.fsolve, func, x0=[0,1], fprime=deriv_func)
+
+    def test_wrong_shape_fprime_function(self):
+        func = lambda x: dummy_func(x, (2,))
+        deriv_func = lambda x: dummy_func(x, (3,3))
+        assert_raises(TypeError, optimize.fsolve, func, x0=[0,1], fprime=deriv_func)
+
+    def test_func_can_raise(self):
+        def func(*args):
+            raise ValueError('I raised')
+
+        with assert_raises(ValueError, match='I raised'):
+            optimize.fsolve(func, x0=[0])
+
+    def test_Dfun_can_raise(self):
+        func = lambda x: x - np.array([10])
+
+        def deriv_func(*args):
+            raise ValueError('I raised')
+
+        with assert_raises(ValueError, match='I raised'):
+            optimize.fsolve(func, x0=[0], fprime=deriv_func)
+
+    def test_float32(self):
+        func = lambda x: np.array([x[0] - 100, x[1] - 1000], dtype=np.float32)**2
+        p = optimize.fsolve(func, np.array([1, 1], np.float32))
+        assert_allclose(func(p), [0, 0], atol=1e-3)
+
+    def test_reentrant_func(self):
+        def func(*args):
+            self.test_pressure_network_no_gradient()
+            return pressure_network(*args)
+
+        # fsolve without gradient, equal pipes -> equal flows.
+        k = np.full(4, 0.5)
+        Qtot = 4
+        initial_guess = array([2., 0., 2., 0.])
+        final_flows, info, ier, mesg = optimize.fsolve(
+            func, initial_guess, args=(Qtot, k),
+            full_output=True)
+        assert_array_almost_equal(final_flows, np.ones(4))
+        assert_(ier == 1, mesg)
+
+    def test_reentrant_Dfunc(self):
+        def deriv_func(*args):
+            self.test_pressure_network_with_gradient()
+            return pressure_network_jacobian(*args)
+
+        # fsolve with gradient, equal pipes -> equal flows
+        k = np.full(4, 0.5)
+        Qtot = 4
+        initial_guess = array([2., 0., 2., 0.])
+        final_flows = optimize.fsolve(
+            pressure_network, initial_guess, args=(Qtot, k),
+            fprime=deriv_func)
+        assert_array_almost_equal(final_flows, np.ones(4))
+
+    def test_concurrent_no_gradient(self):
+        return sequence_parallel([self.test_pressure_network_no_gradient] * 10)
+
+    def test_concurrent_with_gradient(self):
+        return sequence_parallel([self.test_pressure_network_with_gradient] * 10)
+
+
+class TestRootHybr(object):
+    def test_pressure_network_no_gradient(self):
+        # root/hybr without gradient, equal pipes -> equal flows
+        k = np.full(4, 0.5)
+        Qtot = 4
+        initial_guess = array([2., 0., 2., 0.])
+        final_flows = optimize.root(pressure_network, initial_guess,
+                                    method='hybr', args=(Qtot, k)).x
+        assert_array_almost_equal(final_flows, np.ones(4))
+
+    def test_pressure_network_with_gradient(self):
+        # root/hybr with gradient, equal pipes -> equal flows
+        k = np.full(4, 0.5)
+        Qtot = 4
+        initial_guess = array([[2., 0., 2., 0.]])
+        final_flows = optimize.root(pressure_network, initial_guess,
+                                    args=(Qtot, k), method='hybr',
+                                    jac=pressure_network_jacobian).x
+        assert_array_almost_equal(final_flows, np.ones(4))
+
+    def test_pressure_network_with_gradient_combined(self):
+        # root/hybr with gradient and function combined, equal pipes -> equal
+        # flows
+        k = np.full(4, 0.5)
+        Qtot = 4
+        initial_guess = array([2., 0., 2., 0.])
+        final_flows = optimize.root(pressure_network_fun_and_grad,
+                                    initial_guess, args=(Qtot, k),
+                                    method='hybr', jac=True).x
+        assert_array_almost_equal(final_flows, np.ones(4))
+
+
+class TestRootLM(object):
+    def test_pressure_network_no_gradient(self):
+        # root/lm without gradient, equal pipes -> equal flows
+        k = np.full(4, 0.5)
+        Qtot = 4
+        initial_guess = array([2., 0., 2., 0.])
+        final_flows = optimize.root(pressure_network, initial_guess,
+                                    method='lm', args=(Qtot, k)).x
+        assert_array_almost_equal(final_flows, np.ones(4))
+
+
+class TestLeastSq(object):
+    def setup_method(self):
+        x = np.linspace(0, 10, 40)
+        a,b,c = 3.1, 42, -304.2
+        self.x = x
+        self.abc = a,b,c
+        y_true = a*x**2 + b*x + c
+        np.random.seed(0)
+        self.y_meas = y_true + 0.01*np.random.standard_normal(y_true.shape)
+
+    def residuals(self, p, y, x):
+        a,b,c = p
+        err = y-(a*x**2 + b*x + c)
+        return err
+
+    def residuals_jacobian(self, _p, _y, x):
+        return -np.vstack([x**2, x, np.ones_like(x)]).T
+
+    def test_basic(self):
+        p0 = array([0,0,0])
+        params_fit, ier = leastsq(self.residuals, p0,
+                                  args=(self.y_meas, self.x))
+        assert_(ier in (1,2,3,4), 'solution not found (ier=%d)' % ier)
+        # low precision due to random
+        assert_array_almost_equal(params_fit, self.abc, decimal=2)
+
+    def test_basic_with_gradient(self):
+        p0 = array([0,0,0])
+        params_fit, ier = leastsq(self.residuals, p0,
+                                  args=(self.y_meas, self.x),
+                                  Dfun=self.residuals_jacobian)
+        assert_(ier in (1,2,3,4), 'solution not found (ier=%d)' % ier)
+        # low precision due to random
+        assert_array_almost_equal(params_fit, self.abc, decimal=2)
+
+    def test_full_output(self):
+        p0 = array([[0,0,0]])
+        full_output = leastsq(self.residuals, p0,
+                              args=(self.y_meas, self.x),
+                              full_output=True)
+        params_fit, cov_x, infodict, mesg, ier = full_output
+        assert_(ier in (1,2,3,4), 'solution not found: %s' % mesg)
+
+    def test_input_untouched(self):
+        p0 = array([0,0,0],dtype=float64)
+        p0_copy = array(p0, copy=True)
+        full_output = leastsq(self.residuals, p0,
+                              args=(self.y_meas, self.x),
+                              full_output=True)
+        params_fit, cov_x, infodict, mesg, ier = full_output
+        assert_(ier in (1,2,3,4), 'solution not found: %s' % mesg)
+        assert_array_equal(p0, p0_copy)
+
+    def test_wrong_shape_func_callable(self):
+        func = ReturnShape(1)
+        # x0 is a list of two elements, but func will return an array with
+        # length 1, so this should result in a TypeError.
+        x0 = [1.5, 2.0]
+        assert_raises(TypeError, optimize.leastsq, func, x0)
+
+    def test_wrong_shape_func_function(self):
+        # x0 is a list of two elements, but func will return an array with
+        # length 1, so this should result in a TypeError.
+        x0 = [1.5, 2.0]
+        assert_raises(TypeError, optimize.leastsq, dummy_func, x0, args=((1,),))
+
+    def test_wrong_shape_Dfun_callable(self):
+        func = ReturnShape(1)
+        deriv_func = ReturnShape((2,2))
+        assert_raises(TypeError, optimize.leastsq, func, x0=[0,1], Dfun=deriv_func)
+
+    def test_wrong_shape_Dfun_function(self):
+        func = lambda x: dummy_func(x, (2,))
+        deriv_func = lambda x: dummy_func(x, (3,3))
+        assert_raises(TypeError, optimize.leastsq, func, x0=[0,1], Dfun=deriv_func)
+
+    def test_float32(self):
+        # Regression test for gh-1447
+        def func(p,x,y):
+            q = p[0]*np.exp(-(x-p[1])**2/(2.0*p[2]**2))+p[3]
+            return q - y
+
+        x = np.array([1.475,1.429,1.409,1.419,1.455,1.519,1.472, 1.368,1.286,
+                       1.231], dtype=np.float32)
+        y = np.array([0.0168,0.0193,0.0211,0.0202,0.0171,0.0151,0.0185,0.0258,
+                      0.034,0.0396], dtype=np.float32)
+        p0 = np.array([1.0,1.0,1.0,1.0])
+        p1, success = optimize.leastsq(func, p0, args=(x,y))
+
+        assert_(success in [1,2,3,4])
+        assert_((func(p1,x,y)**2).sum() < 1e-4 * (func(p0,x,y)**2).sum())
+
+    def test_func_can_raise(self):
+        def func(*args):
+            raise ValueError('I raised')
+
+        with assert_raises(ValueError, match='I raised'):
+            optimize.leastsq(func, x0=[0])
+
+    def test_Dfun_can_raise(self):
+        func = lambda x: x - np.array([10])
+
+        def deriv_func(*args):
+            raise ValueError('I raised')
+
+        with assert_raises(ValueError, match='I raised'):
+            optimize.leastsq(func, x0=[0], Dfun=deriv_func)
+
+    def test_reentrant_func(self):
+        def func(*args):
+            self.test_basic()
+            return self.residuals(*args)
+
+        p0 = array([0,0,0])
+        params_fit, ier = leastsq(func, p0,
+                                  args=(self.y_meas, self.x))
+        assert_(ier in (1,2,3,4), 'solution not found (ier=%d)' % ier)
+        # low precision due to random
+        assert_array_almost_equal(params_fit, self.abc, decimal=2)
+
+    def test_reentrant_Dfun(self):
+        def deriv_func(*args):
+            self.test_basic()
+            return self.residuals_jacobian(*args)
+
+        p0 = array([0,0,0])
+        params_fit, ier = leastsq(self.residuals, p0,
+                                  args=(self.y_meas, self.x),
+                                  Dfun=deriv_func)
+        assert_(ier in (1,2,3,4), 'solution not found (ier=%d)' % ier)
+        # low precision due to random
+        assert_array_almost_equal(params_fit, self.abc, decimal=2)
+
+    def test_concurrent_no_gradient(self):
+        return sequence_parallel([self.test_basic] * 10)
+
+    def test_concurrent_with_gradient(self):
+        return sequence_parallel([self.test_basic_with_gradient] * 10)
+
+
+class TestCurveFit(object):
+    def setup_method(self):
+        self.y = array([1.0, 3.2, 9.5, 13.7])
+        self.x = array([1.0, 2.0, 3.0, 4.0])
+
+    def test_one_argument(self):
+        def func(x,a):
+            return x**a
+        popt, pcov = curve_fit(func, self.x, self.y)
+        assert_(len(popt) == 1)
+        assert_(pcov.shape == (1,1))
+        assert_almost_equal(popt[0], 1.9149, decimal=4)
+        assert_almost_equal(pcov[0,0], 0.0016, decimal=4)
+
+        # Test if we get the same with full_output. Regression test for #1415.
+        # Also test if check_finite can be turned off.
+        res = curve_fit(func, self.x, self.y,
+                        full_output=1, check_finite=False)
+        (popt2, pcov2, infodict, errmsg, ier) = res
+        assert_array_almost_equal(popt, popt2)
+
+    def test_two_argument(self):
+        def func(x, a, b):
+            return b*x**a
+        popt, pcov = curve_fit(func, self.x, self.y)
+        assert_(len(popt) == 2)
+        assert_(pcov.shape == (2,2))
+        assert_array_almost_equal(popt, [1.7989, 1.1642], decimal=4)
+        assert_array_almost_equal(pcov, [[0.0852, -0.1260], [-0.1260, 0.1912]],
+                                  decimal=4)
+
+    def test_func_is_classmethod(self):
+        class test_self(object):
+            """This class tests if curve_fit passes the correct number of
+               arguments when the model function is a class instance method.
+            """
+
+            def func(self, x, a, b):
+                return b * x**a
+
+        test_self_inst = test_self()
+        popt, pcov = curve_fit(test_self_inst.func, self.x, self.y)
+        assert_(pcov.shape == (2,2))
+        assert_array_almost_equal(popt, [1.7989, 1.1642], decimal=4)
+        assert_array_almost_equal(pcov, [[0.0852, -0.1260], [-0.1260, 0.1912]],
+                                  decimal=4)
+
+    def test_regression_2639(self):
+        # This test fails if epsfcn in leastsq is too large.
+        x = [574.14200000000005, 574.154, 574.16499999999996,
+             574.17700000000002, 574.18799999999999, 574.19899999999996,
+             574.21100000000001, 574.22199999999998, 574.23400000000004,
+             574.245]
+        y = [859.0, 997.0, 1699.0, 2604.0, 2013.0, 1964.0, 2435.0,
+             1550.0, 949.0, 841.0]
+        guess = [574.1861428571428, 574.2155714285715, 1302.0, 1302.0,
+                 0.0035019999999983615, 859.0]
+        good = [5.74177150e+02, 5.74209188e+02, 1.74187044e+03, 1.58646166e+03,
+                1.0068462e-02, 8.57450661e+02]
+
+        def f_double_gauss(x, x0, x1, A0, A1, sigma, c):
+            return (A0*np.exp(-(x-x0)**2/(2.*sigma**2))
+                    + A1*np.exp(-(x-x1)**2/(2.*sigma**2)) + c)
+        popt, pcov = curve_fit(f_double_gauss, x, y, guess, maxfev=10000)
+        assert_allclose(popt, good, rtol=1e-5)
+
+    def test_pcov(self):
+        xdata = np.array([0, 1, 2, 3, 4, 5])
+        ydata = np.array([1, 1, 5, 7, 8, 12])
+        sigma = np.array([1, 2, 1, 2, 1, 2])
+
+        def f(x, a, b):
+            return a*x + b
+
+        for method in ['lm', 'trf', 'dogbox']:
+            popt, pcov = curve_fit(f, xdata, ydata, p0=[2, 0], sigma=sigma,
+                                   method=method)
+            perr_scaled = np.sqrt(np.diag(pcov))
+            assert_allclose(perr_scaled, [0.20659803, 0.57204404], rtol=1e-3)
+
+            popt, pcov = curve_fit(f, xdata, ydata, p0=[2, 0], sigma=3*sigma,
+                                   method=method)
+            perr_scaled = np.sqrt(np.diag(pcov))
+            assert_allclose(perr_scaled, [0.20659803, 0.57204404], rtol=1e-3)
+
+            popt, pcov = curve_fit(f, xdata, ydata, p0=[2, 0], sigma=sigma,
+                                   absolute_sigma=True, method=method)
+            perr = np.sqrt(np.diag(pcov))
+            assert_allclose(perr, [0.30714756, 0.85045308], rtol=1e-3)
+
+            popt, pcov = curve_fit(f, xdata, ydata, p0=[2, 0], sigma=3*sigma,
+                                   absolute_sigma=True, method=method)
+            perr = np.sqrt(np.diag(pcov))
+            assert_allclose(perr, [3*0.30714756, 3*0.85045308], rtol=1e-3)
+
+        # infinite variances
+
+        def f_flat(x, a, b):
+            return a*x
+
+        pcov_expected = np.array([np.inf]*4).reshape(2, 2)
+
+        with suppress_warnings() as sup:
+            sup.filter(OptimizeWarning,
+                       "Covariance of the parameters could not be estimated")
+            popt, pcov = curve_fit(f_flat, xdata, ydata, p0=[2, 0], sigma=sigma)
+            popt1, pcov1 = curve_fit(f, xdata[:2], ydata[:2], p0=[2, 0])
+
+        assert_(pcov.shape == (2, 2))
+        assert_array_equal(pcov, pcov_expected)
+
+        assert_(pcov1.shape == (2, 2))
+        assert_array_equal(pcov1, pcov_expected)
+
+    def test_array_like(self):
+        # Test sequence input. Regression test for gh-3037.
+        def f_linear(x, a, b):
+            return a*x + b
+
+        x = [1, 2, 3, 4]
+        y = [3, 5, 7, 9]
+        assert_allclose(curve_fit(f_linear, x, y)[0], [2, 1], atol=1e-10)
+
+    def test_indeterminate_covariance(self):
+        # Test that a warning is returned when pcov is indeterminate
+        xdata = np.array([1, 2, 3, 4, 5, 6])
+        ydata = np.array([1, 2, 3, 4, 5.5, 6])
+        assert_warns(OptimizeWarning, curve_fit,
+                     lambda x, a, b: a*x, xdata, ydata)
+
+    def test_NaN_handling(self):
+        # Test for correct handling of NaNs in input data: gh-3422
+
+        # create input with NaNs
+        xdata = np.array([1, np.nan, 3])
+        ydata = np.array([1, 2, 3])
+
+        assert_raises(ValueError, curve_fit,
+                      lambda x, a, b: a*x + b, xdata, ydata)
+        assert_raises(ValueError, curve_fit,
+                      lambda x, a, b: a*x + b, ydata, xdata)
+
+        assert_raises(ValueError, curve_fit, lambda x, a, b: a*x + b,
+                      xdata, ydata, **{"check_finite": True})
+
+    def test_empty_inputs(self):
+        # Test both with and without bounds (regression test for gh-9864)
+        assert_raises(ValueError, curve_fit, lambda x, a: a*x, [], [])
+        assert_raises(ValueError, curve_fit, lambda x, a: a*x, [], [],
+                      bounds=(1, 2))
+        assert_raises(ValueError, curve_fit, lambda x, a: a*x, [1], [])
+        assert_raises(ValueError, curve_fit, lambda x, a: a*x, [2], [],
+                      bounds=(1, 2))
+
+    def test_function_zero_params(self):
+        # Fit args is zero, so "Unable to determine number of fit parameters."
+        assert_raises(ValueError, curve_fit, lambda x: x, [1, 2], [3, 4])
+
+    def test_None_x(self):  # Added in GH10196
+        popt, pcov = curve_fit(lambda _, a: a * np.arange(10),
+                               None, 2 * np.arange(10))
+        assert_allclose(popt, [2.])
+
+    def test_method_argument(self):
+        def f(x, a, b):
+            return a * np.exp(-b*x)
+
+        xdata = np.linspace(0, 1, 11)
+        ydata = f(xdata, 2., 2.)
+
+        for method in ['trf', 'dogbox', 'lm', None]:
+            popt, pcov = curve_fit(f, xdata, ydata, method=method)
+            assert_allclose(popt, [2., 2.])
+
+        assert_raises(ValueError, curve_fit, f, xdata, ydata, method='unknown')
+
+    def test_bounds(self):
+        def f(x, a, b):
+            return a * np.exp(-b*x)
+
+        xdata = np.linspace(0, 1, 11)
+        ydata = f(xdata, 2., 2.)
+
+        # The minimum w/out bounds is at [2., 2.],
+        # and with bounds it's at [1.5, smth].
+        bounds = ([1., 0], [1.5, 3.])
+        for method in [None, 'trf', 'dogbox']:
+            popt, pcov = curve_fit(f, xdata, ydata, bounds=bounds,
+                                   method=method)
+            assert_allclose(popt[0], 1.5)
+
+        # With bounds, the starting estimate is feasible.
+        popt, pcov = curve_fit(f, xdata, ydata, method='trf',
+                               bounds=([0., 0], [0.6, np.inf]))
+        assert_allclose(popt[0], 0.6)
+
+        # method='lm' doesn't support bounds.
+        assert_raises(ValueError, curve_fit, f, xdata, ydata, bounds=bounds,
+                      method='lm')
+
+    def test_bounds_p0(self):
+        # This test is for issue #5719. The problem was that an initial guess
+        # was ignored when 'trf' or 'dogbox' methods were invoked.
+        def f(x, a):
+            return np.sin(x + a)
+
+        xdata = np.linspace(-2*np.pi, 2*np.pi, 40)
+        ydata = np.sin(xdata)
+        bounds = (-3 * np.pi, 3 * np.pi)
+        for method in ['trf', 'dogbox']:
+            popt_1, _ = curve_fit(f, xdata, ydata, p0=2.1*np.pi)
+            popt_2, _ = curve_fit(f, xdata, ydata, p0=2.1*np.pi,
+                                  bounds=bounds, method=method)
+
+            # If the initial guess is ignored, then popt_2 would be close 0.
+            assert_allclose(popt_1, popt_2)
+
+    def test_jac(self):
+        # Test that Jacobian callable is handled correctly and
+        # weighted if sigma is provided.
+        def f(x, a, b):
+            return a * np.exp(-b*x)
+
+        def jac(x, a, b):
+            e = np.exp(-b*x)
+            return np.vstack((e, -a * x * e)).T
+
+        xdata = np.linspace(0, 1, 11)
+        ydata = f(xdata, 2., 2.)
+
+        # Test numerical options for least_squares backend.
+        for method in ['trf', 'dogbox']:
+            for scheme in ['2-point', '3-point', 'cs']:
+                popt, pcov = curve_fit(f, xdata, ydata, jac=scheme,
+                                       method=method)
+                assert_allclose(popt, [2, 2])
+
+        # Test the analytic option.
+        for method in ['lm', 'trf', 'dogbox']:
+            popt, pcov = curve_fit(f, xdata, ydata, method=method, jac=jac)
+            assert_allclose(popt, [2, 2])
+
+        # Now add an outlier and provide sigma.
+        ydata[5] = 100
+        sigma = np.ones(xdata.shape[0])
+        sigma[5] = 200
+        for method in ['lm', 'trf', 'dogbox']:
+            popt, pcov = curve_fit(f, xdata, ydata, sigma=sigma, method=method,
+                                   jac=jac)
+            # Still the optimization process is influenced somehow,
+            # have to set rtol=1e-3.
+            assert_allclose(popt, [2, 2], rtol=1e-3)
+
+    def test_maxfev_and_bounds(self):
+        # gh-6340: with no bounds, curve_fit accepts parameter maxfev (via leastsq)
+        # but with bounds, the parameter is `max_nfev` (via least_squares)
+        x = np.arange(0, 10)
+        y = 2*x
+        popt1, _ = curve_fit(lambda x,p: p*x, x, y, bounds=(0, 3), maxfev=100)
+        popt2, _ = curve_fit(lambda x,p: p*x, x, y, bounds=(0, 3), max_nfev=100)
+
+        assert_allclose(popt1, 2, atol=1e-14)
+        assert_allclose(popt2, 2, atol=1e-14)
+
+    def test_curvefit_simplecovariance(self):
+
+        def func(x, a, b):
+            return a * np.exp(-b*x)
+
+        def jac(x, a, b):
+            e = np.exp(-b*x)
+            return np.vstack((e, -a * x * e)).T
+
+        np.random.seed(0)
+        xdata = np.linspace(0, 4, 50)
+        y = func(xdata, 2.5, 1.3)
+        ydata = y + 0.2 * np.random.normal(size=len(xdata))
+
+        sigma = np.zeros(len(xdata)) + 0.2
+        covar = np.diag(sigma**2)
+
+        for jac1, jac2 in [(jac, jac), (None, None)]:
+            for absolute_sigma in [False, True]:
+                popt1, pcov1 = curve_fit(func, xdata, ydata, sigma=sigma,
+                        jac=jac1, absolute_sigma=absolute_sigma)
+                popt2, pcov2 = curve_fit(func, xdata, ydata, sigma=covar,
+                        jac=jac2, absolute_sigma=absolute_sigma)
+
+                assert_allclose(popt1, popt2, atol=1e-14)
+                assert_allclose(pcov1, pcov2, atol=1e-14)
+
+    def test_curvefit_covariance(self):
+
+        def funcp(x, a, b):
+            rotn = np.array([[1./np.sqrt(2), -1./np.sqrt(2), 0], [1./np.sqrt(2), 1./np.sqrt(2), 0], [0, 0, 1.0]])
+            return rotn.dot(a * np.exp(-b*x))
+
+        def jacp(x, a, b):
+            rotn = np.array([[1./np.sqrt(2), -1./np.sqrt(2), 0], [1./np.sqrt(2), 1./np.sqrt(2), 0], [0, 0, 1.0]])
+            e = np.exp(-b*x)
+            return rotn.dot(np.vstack((e, -a * x * e)).T)
+
+        def func(x, a, b):
+            return a * np.exp(-b*x)
+
+        def jac(x, a, b):
+            e = np.exp(-b*x)
+            return np.vstack((e, -a * x * e)).T
+
+        np.random.seed(0)
+        xdata = np.arange(1, 4)
+        y = func(xdata, 2.5, 1.0)
+        ydata = y + 0.2 * np.random.normal(size=len(xdata))
+        sigma = np.zeros(len(xdata)) + 0.2
+        covar = np.diag(sigma**2)
+        # Get a rotation matrix, and obtain ydatap = R ydata
+        # Chisq = ydata^T C^{-1} ydata
+        #       = ydata^T R^T R C^{-1} R^T R ydata
+        #       = ydatap^T Cp^{-1} ydatap
+        # Cp^{-1} = R C^{-1} R^T
+        # Cp      = R C R^T, since R^-1 = R^T
+        rotn = np.array([[1./np.sqrt(2), -1./np.sqrt(2), 0], [1./np.sqrt(2), 1./np.sqrt(2), 0], [0, 0, 1.0]])
+        ydatap = rotn.dot(ydata)
+        covarp = rotn.dot(covar).dot(rotn.T)
+
+        for jac1, jac2 in [(jac, jacp), (None, None)]:
+            for absolute_sigma in [False, True]:
+                popt1, pcov1 = curve_fit(func, xdata, ydata, sigma=sigma,
+                        jac=jac1, absolute_sigma=absolute_sigma)
+                popt2, pcov2 = curve_fit(funcp, xdata, ydatap, sigma=covarp,
+                        jac=jac2, absolute_sigma=absolute_sigma)
+
+                assert_allclose(popt1, popt2, rtol=1.2e-7, atol=1e-14)
+                assert_allclose(pcov1, pcov2, rtol=1.2e-7, atol=1e-14)
+
+    def test_dtypes(self):
+        # regression test for gh-9581: curve_fit fails if x and y dtypes differ
+        x = np.arange(-3, 5)
+        y = 1.5*x + 3.0 + 0.5*np.sin(x)
+
+        def func(x, a, b):
+            return a*x + b
+
+        for method in ['lm', 'trf', 'dogbox']:
+            for dtx in [np.float32, np.float64]:
+                for dty in [np.float32, np.float64]:
+                    x = x.astype(dtx)
+                    y = y.astype(dty)
+
+                with warnings.catch_warnings():
+                    warnings.simplefilter("error", OptimizeWarning)
+                    p, cov = curve_fit(func, x, y, method=method)
+
+                    assert np.isfinite(cov).all()
+                    assert not np.allclose(p, 1)   # curve_fit's initial value
+
+    def test_dtypes2(self):
+        # regression test for gh-7117: curve_fit fails if
+        # both inputs are float32
+        def hyperbola(x, s_1, s_2, o_x, o_y, c):
+            b_2 = (s_1 + s_2) / 2
+            b_1 = (s_2 - s_1) / 2
+            return o_y + b_1*(x-o_x) + b_2*np.sqrt((x-o_x)**2 + c**2/4)
+
+        min_fit = np.array([-3.0, 0.0, -2.0, -10.0, 0.0])
+        max_fit = np.array([0.0, 3.0, 3.0, 0.0, 10.0])
+        guess = np.array([-2.5/3.0, 4/3.0, 1.0, -4.0, 0.5])
+
+        params = [-2, .4, -1, -5, 9.5]
+        xdata = np.array([-32, -16, -8, 4, 4, 8, 16, 32])
+        ydata = hyperbola(xdata, *params)
+
+        # run optimization twice, with xdata being float32 and float64
+        popt_64, _ = curve_fit(f=hyperbola, xdata=xdata, ydata=ydata, p0=guess,
+                               bounds=(min_fit, max_fit))
+
+        xdata = xdata.astype(np.float32)
+        ydata = hyperbola(xdata, *params)
+
+        popt_32, _ = curve_fit(f=hyperbola, xdata=xdata, ydata=ydata, p0=guess,
+                               bounds=(min_fit, max_fit))
+
+        assert_allclose(popt_32, popt_64, atol=2e-5)
+
+    def test_broadcast_y(self):
+        xdata = np.arange(10)
+        target = 4.7 * xdata ** 2 + 3.5 * xdata + np.random.rand(len(xdata))
+        fit_func = lambda x, a, b: a*x**2 + b*x - target
+        for method in ['lm', 'trf', 'dogbox']:
+            popt0, pcov0 = curve_fit(fit_func,
+                                     xdata=xdata,
+                                     ydata=np.zeros_like(xdata),
+                                     method=method)
+            popt1, pcov1 = curve_fit(fit_func,
+                                     xdata=xdata,
+                                     ydata=0,
+                                     method=method)
+            assert_allclose(pcov0, pcov1)
+
+    def test_args_in_kwargs(self):
+        # Ensure that `args` cannot be passed as keyword argument to `curve_fit`
+
+        def func(x, a, b):
+            return a * x + b
+
+        with assert_raises(ValueError):
+            curve_fit(func,
+                      xdata=[1, 2, 3, 4],
+                      ydata=[5, 9, 13, 17],
+                      p0=[1],
+                      args=(1,))
+
+
+class TestFixedPoint(object):
+
+    def test_scalar_trivial(self):
+        # f(x) = 2x; fixed point should be x=0
+        def func(x):
+            return 2.0*x
+        x0 = 1.0
+        x = fixed_point(func, x0)
+        assert_almost_equal(x, 0.0)
+
+    def test_scalar_basic1(self):
+        # f(x) = x**2; x0=1.05; fixed point should be x=1
+        def func(x):
+            return x**2
+        x0 = 1.05
+        x = fixed_point(func, x0)
+        assert_almost_equal(x, 1.0)
+
+    def test_scalar_basic2(self):
+        # f(x) = x**0.5; x0=1.05; fixed point should be x=1
+        def func(x):
+            return x**0.5
+        x0 = 1.05
+        x = fixed_point(func, x0)
+        assert_almost_equal(x, 1.0)
+
+    def test_array_trivial(self):
+        def func(x):
+            return 2.0*x
+        x0 = [0.3, 0.15]
+        with np.errstate(all='ignore'):
+            x = fixed_point(func, x0)
+        assert_almost_equal(x, [0.0, 0.0])
+
+    def test_array_basic1(self):
+        # f(x) = c * x**2; fixed point should be x=1/c
+        def func(x, c):
+            return c * x**2
+        c = array([0.75, 1.0, 1.25])
+        x0 = [1.1, 1.15, 0.9]
+        with np.errstate(all='ignore'):
+            x = fixed_point(func, x0, args=(c,))
+        assert_almost_equal(x, 1.0/c)
+
+    def test_array_basic2(self):
+        # f(x) = c * x**0.5; fixed point should be x=c**2
+        def func(x, c):
+            return c * x**0.5
+        c = array([0.75, 1.0, 1.25])
+        x0 = [0.8, 1.1, 1.1]
+        x = fixed_point(func, x0, args=(c,))
+        assert_almost_equal(x, c**2)
+
+    def test_lambertw(self):
+        # python-list/2010-December/594592.html
+        xxroot = fixed_point(lambda xx: np.exp(-2.0*xx)/2.0, 1.0,
+                args=(), xtol=1e-12, maxiter=500)
+        assert_allclose(xxroot, np.exp(-2.0*xxroot)/2.0)
+        assert_allclose(xxroot, lambertw(1)/2)
+
+    def test_no_acceleration(self):
+        # github issue 5460
+        ks = 2
+        kl = 6
+        m = 1.3
+        n0 = 1.001
+        i0 = ((m-1)/m)*(kl/ks/m)**(1/(m-1))
+
+        def func(n):
+            return np.log(kl/ks/n) / np.log((i0*n/(n - 1))) + 1
+
+        n = fixed_point(func, n0, method='iteration')
+        assert_allclose(n, m)

venv/Lib/site-packages/scipy/optimize/tests/test_nnls.py

@@ -0,0 +1,34 @@
+""" Unit tests for nonnegative least squares
+Author: Uwe Schmitt
+Sep 2008
+"""
+import numpy as np
+
+from numpy.testing import assert_
+from pytest import raises as assert_raises
+
+from scipy.optimize import nnls
+from numpy import arange, dot
+from numpy.linalg import norm
+
+
+class TestNNLS(object):
+
+    def test_nnls(self):
+        a = arange(25.0).reshape(-1,5)
+        x = arange(5.0)
+        y = dot(a,x)
+        x, res = nnls(a,y)
+        assert_(res < 1e-7)
+        assert_(norm(dot(a,x)-y) < 1e-7)
+
+    def test_maxiter(self):
+        # test that maxiter argument does stop iterations
+        # NB: did not manage to find a test case where the default value
+        # of maxiter is not sufficient, so use a too-small value
+        rndm = np.random.RandomState(1234)
+        a = rndm.uniform(size=(100, 100))
+        b = rndm.uniform(size=100)
+        with assert_raises(RuntimeError):
+            nnls(a, b, maxiter=1)
+

venv/Lib/site-packages/scipy/optimize/tests/test_nonlin.py

@@ -0,0 +1,445 @@
+""" Unit tests for nonlinear solvers
+Author: Ondrej Certik
+May 2007
+"""
+from numpy.testing import assert_
+import pytest
+
+from scipy.optimize import nonlin, root
+from numpy import diag, dot
+from numpy.linalg import inv
+import numpy as np
+
+from .test_minpack import pressure_network
+
+SOLVERS = {'anderson': nonlin.anderson, 'diagbroyden': nonlin.diagbroyden,
+           'linearmixing': nonlin.linearmixing, 'excitingmixing': nonlin.excitingmixing,
+           'broyden1': nonlin.broyden1, 'broyden2': nonlin.broyden2,
+           'krylov': nonlin.newton_krylov}
+MUST_WORK = {'anderson': nonlin.anderson, 'broyden1': nonlin.broyden1,
+             'broyden2': nonlin.broyden2, 'krylov': nonlin.newton_krylov}
+
+#-------------------------------------------------------------------------------
+# Test problems
+#-------------------------------------------------------------------------------
+
+
+def F(x):
+    x = np.asarray(x).T
+    d = diag([3,2,1.5,1,0.5])
+    c = 0.01
+    f = -d @ x - c * float(x.T @ x) * x
+    return f
+
+
+F.xin = [1,1,1,1,1]
+F.KNOWN_BAD = {}
+
+
+def F2(x):
+    return x
+
+
+F2.xin = [1,2,3,4,5,6]
+F2.KNOWN_BAD = {'linearmixing': nonlin.linearmixing,
+                'excitingmixing': nonlin.excitingmixing}
+
+
+def F2_lucky(x):
+    return x
+
+
+F2_lucky.xin = [0,0,0,0,0,0]
+F2_lucky.KNOWN_BAD = {}
+
+
+def F3(x):
+    A = np.array([[-2, 1, 0.], [1, -2, 1], [0, 1, -2]])
+    b = np.array([1, 2, 3.])
+    return A @ x - b
+
+
+F3.xin = [1,2,3]
+F3.KNOWN_BAD = {}
+
+
+def F4_powell(x):
+    A = 1e4
+    return [A*x[0]*x[1] - 1, np.exp(-x[0]) + np.exp(-x[1]) - (1 + 1/A)]
+
+
+F4_powell.xin = [-1, -2]
+F4_powell.KNOWN_BAD = {'linearmixing': nonlin.linearmixing,
+                       'excitingmixing': nonlin.excitingmixing,
+                       'diagbroyden': nonlin.diagbroyden}
+
+
+def F5(x):
+    return pressure_network(x, 4, np.array([.5, .5, .5, .5]))
+
+
+F5.xin = [2., 0, 2, 0]
+F5.KNOWN_BAD = {'excitingmixing': nonlin.excitingmixing,
+                'linearmixing': nonlin.linearmixing,
+                'diagbroyden': nonlin.diagbroyden}
+
+
+def F6(x):
+    x1, x2 = x
+    J0 = np.array([[-4.256, 14.7],
+                [0.8394989, 0.59964207]])
+    v = np.array([(x1 + 3) * (x2**5 - 7) + 3*6,
+                  np.sin(x2 * np.exp(x1) - 1)])
+    return -np.linalg.solve(J0, v)
+
+
+F6.xin = [-0.5, 1.4]
+F6.KNOWN_BAD = {'excitingmixing': nonlin.excitingmixing,
+                'linearmixing': nonlin.linearmixing,
+                'diagbroyden': nonlin.diagbroyden}
+
+
+#-------------------------------------------------------------------------------
+# Tests
+#-------------------------------------------------------------------------------
+
+
+class TestNonlin(object):
+    """
+    Check the Broyden methods for a few test problems.
+
+    broyden1, broyden2, and newton_krylov must succeed for
+    all functions. Some of the others don't -- tests in KNOWN_BAD are skipped.
+
+    """
+
+    def _check_nonlin_func(self, f, func, f_tol=1e-2):
+        x = func(f, f.xin, f_tol=f_tol, maxiter=200, verbose=0)
+        assert_(np.absolute(f(x)).max() < f_tol)
+
+    def _check_root(self, f, method, f_tol=1e-2):
+        res = root(f, f.xin, method=method,
+                   options={'ftol': f_tol, 'maxiter': 200, 'disp': 0})
+        assert_(np.absolute(res.fun).max() < f_tol)
+
+    @pytest.mark.xfail
+    def _check_func_fail(self, *a, **kw):
+        pass
+
+    def test_problem_nonlin(self):
+        for f in [F, F2, F2_lucky, F3, F4_powell, F5, F6]:
+            for func in SOLVERS.values():
+                if func in f.KNOWN_BAD.values():
+                    if func in MUST_WORK.values():
+                        self._check_func_fail(f, func)
+                    continue
+                self._check_nonlin_func(f, func)
+
+    def test_tol_norm_called(self):
+        # Check that supplying tol_norm keyword to nonlin_solve works
+        self._tol_norm_used = False
+
+        def local_norm_func(x):
+            self._tol_norm_used = True
+            return np.absolute(x).max()
+
+        nonlin.newton_krylov(F, F.xin, f_tol=1e-2, maxiter=200, verbose=0,
+             tol_norm=local_norm_func)
+        assert_(self._tol_norm_used)
+
+    def test_problem_root(self):
+        for f in [F, F2, F2_lucky, F3, F4_powell, F5, F6]:
+            for meth in SOLVERS:
+                if meth in f.KNOWN_BAD:
+                    if meth in MUST_WORK:
+                        self._check_func_fail(f, meth)
+                    continue
+                self._check_root(f, meth)
+
+
+class TestSecant(object):
+    """Check that some Jacobian approximations satisfy the secant condition"""
+
+    xs = [np.array([1,2,3,4,5], float),
+          np.array([2,3,4,5,1], float),
+          np.array([3,4,5,1,2], float),
+          np.array([4,5,1,2,3], float),
+          np.array([9,1,9,1,3], float),
+          np.array([0,1,9,1,3], float),
+          np.array([5,5,7,1,1], float),
+          np.array([1,2,7,5,1], float),]
+    fs = [x**2 - 1 for x in xs]
+
+    def _check_secant(self, jac_cls, npoints=1, **kw):
+        """
+        Check that the given Jacobian approximation satisfies secant
+        conditions for last `npoints` points.
+        """
+        jac = jac_cls(**kw)
+        jac.setup(self.xs[0], self.fs[0], None)
+        for j, (x, f) in enumerate(zip(self.xs[1:], self.fs[1:])):
+            jac.update(x, f)
+
+            for k in range(min(npoints, j+1)):
+                dx = self.xs[j-k+1] - self.xs[j-k]
+                df = self.fs[j-k+1] - self.fs[j-k]
+                assert_(np.allclose(dx, jac.solve(df)))
+
+            # Check that the `npoints` secant bound is strict
+            if j >= npoints:
+                dx = self.xs[j-npoints+1] - self.xs[j-npoints]
+                df = self.fs[j-npoints+1] - self.fs[j-npoints]
+                assert_(not np.allclose(dx, jac.solve(df)))
+
+    def test_broyden1(self):
+        self._check_secant(nonlin.BroydenFirst)
+
+    def test_broyden2(self):
+        self._check_secant(nonlin.BroydenSecond)
+
+    def test_broyden1_update(self):
+        # Check that BroydenFirst update works as for a dense matrix
+        jac = nonlin.BroydenFirst(alpha=0.1)
+        jac.setup(self.xs[0], self.fs[0], None)
+
+        B = np.identity(5) * (-1/0.1)
+
+        for last_j, (x, f) in enumerate(zip(self.xs[1:], self.fs[1:])):
+            df = f - self.fs[last_j]
+            dx = x - self.xs[last_j]
+            B += (df - dot(B, dx))[:,None] * dx[None,:] / dot(dx, dx)
+            jac.update(x, f)
+            assert_(np.allclose(jac.todense(), B, rtol=1e-10, atol=1e-13))
+
+    def test_broyden2_update(self):
+        # Check that BroydenSecond update works as for a dense matrix
+        jac = nonlin.BroydenSecond(alpha=0.1)
+        jac.setup(self.xs[0], self.fs[0], None)
+
+        H = np.identity(5) * (-0.1)
+
+        for last_j, (x, f) in enumerate(zip(self.xs[1:], self.fs[1:])):
+            df = f - self.fs[last_j]
+            dx = x - self.xs[last_j]
+            H += (dx - dot(H, df))[:,None] * df[None,:] / dot(df, df)
+            jac.update(x, f)
+            assert_(np.allclose(jac.todense(), inv(H), rtol=1e-10, atol=1e-13))
+
+    def test_anderson(self):
+        # Anderson mixing (with w0=0) satisfies secant conditions
+        # for the last M iterates, see [Ey]_
+        #
+        # .. [Ey] V. Eyert, J. Comp. Phys., 124, 271 (1996).
+        self._check_secant(nonlin.Anderson, M=3, w0=0, npoints=3)
+
+
+class TestLinear(object):
+    """Solve a linear equation;
+    some methods find the exact solution in a finite number of steps"""
+
+    def _check(self, jac, N, maxiter, complex=False, **kw):
+        np.random.seed(123)
+
+        A = np.random.randn(N, N)
+        if complex:
+            A = A + 1j*np.random.randn(N, N)
+        b = np.random.randn(N)
+        if complex:
+            b = b + 1j*np.random.randn(N)
+
+        def func(x):
+            return dot(A, x) - b
+
+        sol = nonlin.nonlin_solve(func, np.zeros(N), jac, maxiter=maxiter,
+                                  f_tol=1e-6, line_search=None, verbose=0)
+        assert_(np.allclose(dot(A, sol), b, atol=1e-6))
+
+    def test_broyden1(self):
+        # Broyden methods solve linear systems exactly in 2*N steps
+        self._check(nonlin.BroydenFirst(alpha=1.0), 20, 41, False)
+        self._check(nonlin.BroydenFirst(alpha=1.0), 20, 41, True)
+
+    def test_broyden2(self):
+        # Broyden methods solve linear systems exactly in 2*N steps
+        self._check(nonlin.BroydenSecond(alpha=1.0), 20, 41, False)
+        self._check(nonlin.BroydenSecond(alpha=1.0), 20, 41, True)
+
+    def test_anderson(self):
+        # Anderson is rather similar to Broyden, if given enough storage space
+        self._check(nonlin.Anderson(M=50, alpha=1.0), 20, 29, False)
+        self._check(nonlin.Anderson(M=50, alpha=1.0), 20, 29, True)
+
+    def test_krylov(self):
+        # Krylov methods solve linear systems exactly in N inner steps
+        self._check(nonlin.KrylovJacobian, 20, 2, False, inner_m=10)
+        self._check(nonlin.KrylovJacobian, 20, 2, True, inner_m=10)
+
+
+class TestJacobianDotSolve(object):
+    """Check that solve/dot methods in Jacobian approximations are consistent"""
+
+    def _func(self, x):
+        return x**2 - 1 + np.dot(self.A, x)
+
+    def _check_dot(self, jac_cls, complex=False, tol=1e-6, **kw):
+        np.random.seed(123)
+
+        N = 7
+
+        def rand(*a):
+            q = np.random.rand(*a)
+            if complex:
+                q = q + 1j*np.random.rand(*a)
+            return q
+
+        def assert_close(a, b, msg):
+            d = abs(a - b).max()
+            f = tol + abs(b).max()*tol
+            if d > f:
+                raise AssertionError('%s: err %g' % (msg, d))
+
+        self.A = rand(N, N)
+
+        # initialize
+        x0 = np.random.rand(N)
+        jac = jac_cls(**kw)
+        jac.setup(x0, self._func(x0), self._func)
+
+        # check consistency
+        for k in range(2*N):
+            v = rand(N)
+
+            if hasattr(jac, '__array__'):
+                Jd = np.array(jac)
+                if hasattr(jac, 'solve'):
+                    Gv = jac.solve(v)
+                    Gv2 = np.linalg.solve(Jd, v)
+                    assert_close(Gv, Gv2, 'solve vs array')
+                if hasattr(jac, 'rsolve'):
+                    Gv = jac.rsolve(v)
+                    Gv2 = np.linalg.solve(Jd.T.conj(), v)
+                    assert_close(Gv, Gv2, 'rsolve vs array')
+                if hasattr(jac, 'matvec'):
+                    Jv = jac.matvec(v)
+                    Jv2 = np.dot(Jd, v)
+                    assert_close(Jv, Jv2, 'dot vs array')
+                if hasattr(jac, 'rmatvec'):
+                    Jv = jac.rmatvec(v)
+                    Jv2 = np.dot(Jd.T.conj(), v)
+                    assert_close(Jv, Jv2, 'rmatvec vs array')
+
+            if hasattr(jac, 'matvec') and hasattr(jac, 'solve'):
+                Jv = jac.matvec(v)
+                Jv2 = jac.solve(jac.matvec(Jv))
+                assert_close(Jv, Jv2, 'dot vs solve')
+
+            if hasattr(jac, 'rmatvec') and hasattr(jac, 'rsolve'):
+                Jv = jac.rmatvec(v)
+                Jv2 = jac.rmatvec(jac.rsolve(Jv))
+                assert_close(Jv, Jv2, 'rmatvec vs rsolve')
+
+            x = rand(N)
+            jac.update(x, self._func(x))
+
+    def test_broyden1(self):
+        self._check_dot(nonlin.BroydenFirst, complex=False)
+        self._check_dot(nonlin.BroydenFirst, complex=True)
+
+    def test_broyden2(self):
+        self._check_dot(nonlin.BroydenSecond, complex=False)
+        self._check_dot(nonlin.BroydenSecond, complex=True)
+
+    def test_anderson(self):
+        self._check_dot(nonlin.Anderson, complex=False)
+        self._check_dot(nonlin.Anderson, complex=True)
+
+    def test_diagbroyden(self):
+        self._check_dot(nonlin.DiagBroyden, complex=False)
+        self._check_dot(nonlin.DiagBroyden, complex=True)
+
+    def test_linearmixing(self):
+        self._check_dot(nonlin.LinearMixing, complex=False)
+        self._check_dot(nonlin.LinearMixing, complex=True)
+
+    def test_excitingmixing(self):
+        self._check_dot(nonlin.ExcitingMixing, complex=False)
+        self._check_dot(nonlin.ExcitingMixing, complex=True)
+
+    def test_krylov(self):
+        self._check_dot(nonlin.KrylovJacobian, complex=False, tol=1e-3)
+        self._check_dot(nonlin.KrylovJacobian, complex=True, tol=1e-3)
+
+
+class TestNonlinOldTests(object):
+    """ Test case for a simple constrained entropy maximization problem
+    (the machine translation example of Berger et al in
+    Computational Linguistics, vol 22, num 1, pp 39--72, 1996.)
+    """
+
+    def test_broyden1(self):
+        x = nonlin.broyden1(F,F.xin,iter=12,alpha=1)
+        assert_(nonlin.norm(x) < 1e-9)
+        assert_(nonlin.norm(F(x)) < 1e-9)
+
+    def test_broyden2(self):
+        x = nonlin.broyden2(F,F.xin,iter=12,alpha=1)
+        assert_(nonlin.norm(x) < 1e-9)
+        assert_(nonlin.norm(F(x)) < 1e-9)
+
+    def test_anderson(self):
+        x = nonlin.anderson(F,F.xin,iter=12,alpha=0.03,M=5)
+        assert_(nonlin.norm(x) < 0.33)
+
+    def test_linearmixing(self):
+        x = nonlin.linearmixing(F,F.xin,iter=60,alpha=0.5)
+        assert_(nonlin.norm(x) < 1e-7)
+        assert_(nonlin.norm(F(x)) < 1e-7)
+
+    def test_exciting(self):
+        x = nonlin.excitingmixing(F,F.xin,iter=20,alpha=0.5)
+        assert_(nonlin.norm(x) < 1e-5)
+        assert_(nonlin.norm(F(x)) < 1e-5)
+
+    def test_diagbroyden(self):
+        x = nonlin.diagbroyden(F,F.xin,iter=11,alpha=1)
+        assert_(nonlin.norm(x) < 1e-8)
+        assert_(nonlin.norm(F(x)) < 1e-8)
+
+    def test_root_broyden1(self):
+        res = root(F, F.xin, method='broyden1',
+                   options={'nit': 12, 'jac_options': {'alpha': 1}})
+        assert_(nonlin.norm(res.x) < 1e-9)
+        assert_(nonlin.norm(res.fun) < 1e-9)
+
+    def test_root_broyden2(self):
+        res = root(F, F.xin, method='broyden2',
+                   options={'nit': 12, 'jac_options': {'alpha': 1}})
+        assert_(nonlin.norm(res.x) < 1e-9)
+        assert_(nonlin.norm(res.fun) < 1e-9)
+
+    def test_root_anderson(self):
+        res = root(F, F.xin, method='anderson',
+                   options={'nit': 12,
+                            'jac_options': {'alpha': 0.03, 'M': 5}})
+        assert_(nonlin.norm(res.x) < 0.33)
+
+    def test_root_linearmixing(self):
+        res = root(F, F.xin, method='linearmixing',
+                   options={'nit': 60,
+                            'jac_options': {'alpha': 0.5}})
+        assert_(nonlin.norm(res.x) < 1e-7)
+        assert_(nonlin.norm(res.fun) < 1e-7)
+
+    def test_root_excitingmixing(self):
+        res = root(F, F.xin, method='excitingmixing',
+                   options={'nit': 20,
+                            'jac_options': {'alpha': 0.5}})
+        assert_(nonlin.norm(res.x) < 1e-5)
+        assert_(nonlin.norm(res.fun) < 1e-5)
+
+    def test_root_diagbroyden(self):
+        res = root(F, F.xin, method='diagbroyden',
+                   options={'nit': 11,
+                            'jac_options': {'alpha': 1}})
+        assert_(nonlin.norm(res.x) < 1e-8)
+        assert_(nonlin.norm(res.fun) < 1e-8)

venv/Lib/site-packages/scipy/optimize/tests/test_regression.py

@@ -0,0 +1,40 @@
+"""Regression tests for optimize.
+
+"""
+import numpy as np
+from numpy.testing import assert_almost_equal
+from pytest import raises as assert_raises
+
+import scipy.optimize
+
+
+class TestRegression(object):
+
+    def test_newton_x0_is_0(self):
+        # Regression test for gh-1601
+        tgt = 1
+        res = scipy.optimize.newton(lambda x: x - 1, 0)
+        assert_almost_equal(res, tgt)
+
+    def test_newton_integers(self):
+        # Regression test for gh-1741
+        root = scipy.optimize.newton(lambda x: x**2 - 1, x0=2,
+                                    fprime=lambda x: 2*x)
+        assert_almost_equal(root, 1.0)
+
+    def test_lmdif_errmsg(self):
+        # This shouldn't cause a crash on Python 3
+        class SomeError(Exception):
+            pass
+        counter = [0]
+
+        def func(x):
+            counter[0] += 1
+            if counter[0] < 3:
+                return x**2 - np.array([9, 10, 11])
+            else:
+                raise SomeError()
+        assert_raises(SomeError,
+                      scipy.optimize.leastsq,
+                      func, [1, 2, 3])
+

venv/Lib/site-packages/scipy/optimize/tests/test_slsqp.py

@@ -0,0 +1,563 @@
+"""
+Unit test for SLSQP optimization.
+"""
+from numpy.testing import (assert_, assert_array_almost_equal,
+                           assert_allclose, assert_equal)
+from pytest import raises as assert_raises
+import numpy as np
+
+from scipy.optimize import fmin_slsqp, minimize, Bounds
+
+
+class MyCallBack(object):
+    """pass a custom callback function
+
+    This makes sure it's being used.
+    """
+    def __init__(self):
+        self.been_called = False
+        self.ncalls = 0
+
+    def __call__(self, x):
+        self.been_called = True
+        self.ncalls += 1
+
+
+class TestSLSQP(object):
+    """
+    Test SLSQP algorithm using Example 14.4 from Numerical Methods for
+    Engineers by Steven Chapra and Raymond Canale.
+    This example maximizes the function f(x) = 2*x*y + 2*x - x**2 - 2*y**2,
+    which has a maximum at x=2, y=1.
+    """
+    def setup_method(self):
+        self.opts = {'disp': False}
+
+    def fun(self, d, sign=1.0):
+        """
+        Arguments:
+        d     - A list of two elements, where d[0] represents x and d[1] represents y
+                 in the following equation.
+        sign - A multiplier for f. Since we want to optimize it, and the SciPy
+               optimizers can only minimize functions, we need to multiply it by
+               -1 to achieve the desired solution
+        Returns:
+        2*x*y + 2*x - x**2 - 2*y**2
+
+        """
+        x = d[0]
+        y = d[1]
+        return sign*(2*x*y + 2*x - x**2 - 2*y**2)
+
+    def jac(self, d, sign=1.0):
+        """
+        This is the derivative of fun, returning a NumPy array
+        representing df/dx and df/dy.
+
+        """
+        x = d[0]
+        y = d[1]
+        dfdx = sign*(-2*x + 2*y + 2)
+        dfdy = sign*(2*x - 4*y)
+        return np.array([dfdx, dfdy], float)
+
+    def fun_and_jac(self, d, sign=1.0):
+        return self.fun(d, sign), self.jac(d, sign)
+
+    def f_eqcon(self, x, sign=1.0):
+        """ Equality constraint """
+        return np.array([x[0] - x[1]])
+
+    def fprime_eqcon(self, x, sign=1.0):
+        """ Equality constraint, derivative """
+        return np.array([[1, -1]])
+
+    def f_eqcon_scalar(self, x, sign=1.0):
+        """ Scalar equality constraint """
+        return self.f_eqcon(x, sign)[0]
+
+    def fprime_eqcon_scalar(self, x, sign=1.0):
+        """ Scalar equality constraint, derivative """
+        return self.fprime_eqcon(x, sign)[0].tolist()
+
+    def f_ieqcon(self, x, sign=1.0):
+        """ Inequality constraint """
+        return np.array([x[0] - x[1] - 1.0])
+
+    def fprime_ieqcon(self, x, sign=1.0):
+        """ Inequality constraint, derivative """
+        return np.array([[1, -1]])
+
+    def f_ieqcon2(self, x):
+        """ Vector inequality constraint """
+        return np.asarray(x)
+
+    def fprime_ieqcon2(self, x):
+        """ Vector inequality constraint, derivative """
+        return np.identity(x.shape[0])
+
+    # minimize
+    def test_minimize_unbounded_approximated(self):
+        # Minimize, method='SLSQP': unbounded, approximated jacobian.
+        jacs = [None, False, '2-point', '3-point']
+        for jac in jacs:
+            res = minimize(self.fun, [-1.0, 1.0], args=(-1.0, ),
+                           jac=jac, method='SLSQP',
+                           options=self.opts)
+            assert_(res['success'], res['message'])
+            assert_allclose(res.x, [2, 1])
+
+    def test_minimize_unbounded_given(self):
+        # Minimize, method='SLSQP': unbounded, given Jacobian.
+        res = minimize(self.fun, [-1.0, 1.0], args=(-1.0, ),
+                       jac=self.jac, method='SLSQP', options=self.opts)
+        assert_(res['success'], res['message'])
+        assert_allclose(res.x, [2, 1])
+
+    def test_minimize_bounded_approximated(self):
+        # Minimize, method='SLSQP': bounded, approximated jacobian.
+        jacs = [None, False, '2-point', '3-point']
+        for jac in jacs:
+            with np.errstate(invalid='ignore'):
+                res = minimize(self.fun, [-1.0, 1.0], args=(-1.0, ),
+                               jac=jac,
+                               bounds=((2.5, None), (None, 0.5)),
+                               method='SLSQP', options=self.opts)
+            assert_(res['success'], res['message'])
+            assert_allclose(res.x, [2.5, 0.5])
+            assert_(2.5 <= res.x[0])
+            assert_(res.x[1] <= 0.5)
+
+    def test_minimize_unbounded_combined(self):
+        # Minimize, method='SLSQP': unbounded, combined function and Jacobian.
+        res = minimize(self.fun_and_jac, [-1.0, 1.0], args=(-1.0, ),
+                       jac=True, method='SLSQP', options=self.opts)
+        assert_(res['success'], res['message'])
+        assert_allclose(res.x, [2, 1])
+
+    def test_minimize_equality_approximated(self):
+        # Minimize with method='SLSQP': equality constraint, approx. jacobian.
+        jacs = [None, False, '2-point', '3-point']
+        for jac in jacs:
+            res = minimize(self.fun, [-1.0, 1.0], args=(-1.0, ),
+                           jac=jac,
+                           constraints={'type': 'eq',
+                                        'fun': self.f_eqcon,
+                                        'args': (-1.0, )},
+                           method='SLSQP', options=self.opts)
+            assert_(res['success'], res['message'])
+            assert_allclose(res.x, [1, 1])
+
+    def test_minimize_equality_given(self):
+        # Minimize with method='SLSQP': equality constraint, given Jacobian.
+        res = minimize(self.fun, [-1.0, 1.0], jac=self.jac,
+                       method='SLSQP', args=(-1.0,),
+                       constraints={'type': 'eq', 'fun':self.f_eqcon,
+                                    'args': (-1.0, )},
+                       options=self.opts)
+        assert_(res['success'], res['message'])
+        assert_allclose(res.x, [1, 1])
+
+    def test_minimize_equality_given2(self):
+        # Minimize with method='SLSQP': equality constraint, given Jacobian
+        # for fun and const.
+        res = minimize(self.fun, [-1.0, 1.0], method='SLSQP',
+                       jac=self.jac, args=(-1.0,),
+                       constraints={'type': 'eq',
+                                    'fun': self.f_eqcon,
+                                    'args': (-1.0, ),
+                                    'jac': self.fprime_eqcon},
+                       options=self.opts)
+        assert_(res['success'], res['message'])
+        assert_allclose(res.x, [1, 1])
+
+    def test_minimize_equality_given_cons_scalar(self):
+        # Minimize with method='SLSQP': scalar equality constraint, given
+        # Jacobian for fun and const.
+        res = minimize(self.fun, [-1.0, 1.0], method='SLSQP',
+                       jac=self.jac, args=(-1.0,),
+                       constraints={'type': 'eq',
+                                    'fun': self.f_eqcon_scalar,
+                                    'args': (-1.0, ),
+                                    'jac': self.fprime_eqcon_scalar},
+                       options=self.opts)
+        assert_(res['success'], res['message'])
+        assert_allclose(res.x, [1, 1])
+
+    def test_minimize_inequality_given(self):
+        # Minimize with method='SLSQP': inequality constraint, given Jacobian.
+        res = minimize(self.fun, [-1.0, 1.0], method='SLSQP',
+                       jac=self.jac, args=(-1.0, ),
+                       constraints={'type': 'ineq',
+                                    'fun': self.f_ieqcon,
+                                    'args': (-1.0, )},
+                       options=self.opts)
+        assert_(res['success'], res['message'])
+        assert_allclose(res.x, [2, 1], atol=1e-3)
+
+    def test_minimize_inequality_given_vector_constraints(self):
+        # Minimize with method='SLSQP': vector inequality constraint, given
+        # Jacobian.
+        res = minimize(self.fun, [-1.0, 1.0], jac=self.jac,
+                       method='SLSQP', args=(-1.0,),
+                       constraints={'type': 'ineq',
+                                    'fun': self.f_ieqcon2,
+                                    'jac': self.fprime_ieqcon2},
+                       options=self.opts)
+        assert_(res['success'], res['message'])
+        assert_allclose(res.x, [2, 1])
+
+    def test_minimize_bound_equality_given2(self):
+        # Minimize with method='SLSQP': bounds, eq. const., given jac. for
+        # fun. and const.
+        res = minimize(self.fun, [-1.0, 1.0], method='SLSQP',
+                       jac=self.jac, args=(-1.0, ),
+                       bounds=[(-0.8, 1.), (-1, 0.8)],
+                       constraints={'type': 'eq',
+                                    'fun': self.f_eqcon,
+                                    'args': (-1.0, ),
+                                    'jac': self.fprime_eqcon},
+                       options=self.opts)
+        assert_(res['success'], res['message'])
+        assert_allclose(res.x, [0.8, 0.8], atol=1e-3)
+        assert_(-0.8 <= res.x[0] <= 1)
+        assert_(-1 <= res.x[1] <= 0.8)
+
+    # fmin_slsqp
+    def test_unbounded_approximated(self):
+        # SLSQP: unbounded, approximated Jacobian.
+        res = fmin_slsqp(self.fun, [-1.0, 1.0], args=(-1.0, ),
+                         iprint = 0, full_output = 1)
+        x, fx, its, imode, smode = res
+        assert_(imode == 0, imode)
+        assert_array_almost_equal(x, [2, 1])
+
+    def test_unbounded_given(self):
+        # SLSQP: unbounded, given Jacobian.
+        res = fmin_slsqp(self.fun, [-1.0, 1.0], args=(-1.0, ),
+                         fprime = self.jac, iprint = 0,
+                         full_output = 1)
+        x, fx, its, imode, smode = res
+        assert_(imode == 0, imode)
+        assert_array_almost_equal(x, [2, 1])
+
+    def test_equality_approximated(self):
+        # SLSQP: equality constraint, approximated Jacobian.
+        res = fmin_slsqp(self.fun,[-1.0,1.0], args=(-1.0,),
+                         eqcons = [self.f_eqcon],
+                         iprint = 0, full_output = 1)
+        x, fx, its, imode, smode = res
+        assert_(imode == 0, imode)
+        assert_array_almost_equal(x, [1, 1])
+
+    def test_equality_given(self):
+        # SLSQP: equality constraint, given Jacobian.
+        res = fmin_slsqp(self.fun, [-1.0, 1.0],
+                         fprime=self.jac, args=(-1.0,),
+                         eqcons = [self.f_eqcon], iprint = 0,
+                         full_output = 1)
+        x, fx, its, imode, smode = res
+        assert_(imode == 0, imode)
+        assert_array_almost_equal(x, [1, 1])
+
+    def test_equality_given2(self):
+        # SLSQP: equality constraint, given Jacobian for fun and const.
+        res = fmin_slsqp(self.fun, [-1.0, 1.0],
+                         fprime=self.jac, args=(-1.0,),
+                         f_eqcons = self.f_eqcon,
+                         fprime_eqcons = self.fprime_eqcon,
+                         iprint = 0,
+                         full_output = 1)
+        x, fx, its, imode, smode = res
+        assert_(imode == 0, imode)
+        assert_array_almost_equal(x, [1, 1])
+
+    def test_inequality_given(self):
+        # SLSQP: inequality constraint, given Jacobian.
+        res = fmin_slsqp(self.fun, [-1.0, 1.0],
+                         fprime=self.jac, args=(-1.0, ),
+                         ieqcons = [self.f_ieqcon],
+                         iprint = 0, full_output = 1)
+        x, fx, its, imode, smode = res
+        assert_(imode == 0, imode)
+        assert_array_almost_equal(x, [2, 1], decimal=3)
+
+    def test_bound_equality_given2(self):
+        # SLSQP: bounds, eq. const., given jac. for fun. and const.
+        res = fmin_slsqp(self.fun, [-1.0, 1.0],
+                         fprime=self.jac, args=(-1.0, ),
+                         bounds = [(-0.8, 1.), (-1, 0.8)],
+                         f_eqcons = self.f_eqcon,
+                         fprime_eqcons = self.fprime_eqcon,
+                         iprint = 0, full_output = 1)
+        x, fx, its, imode, smode = res
+        assert_(imode == 0, imode)
+        assert_array_almost_equal(x, [0.8, 0.8], decimal=3)
+        assert_(-0.8 <= x[0] <= 1)
+        assert_(-1 <= x[1] <= 0.8)
+
+    def test_scalar_constraints(self):
+        # Regression test for gh-2182
+        x = fmin_slsqp(lambda z: z**2, [3.],
+                       ieqcons=[lambda z: z[0] - 1],
+                       iprint=0)
+        assert_array_almost_equal(x, [1.])
+
+        x = fmin_slsqp(lambda z: z**2, [3.],
+                       f_ieqcons=lambda z: [z[0] - 1],
+                       iprint=0)
+        assert_array_almost_equal(x, [1.])
+
+    def test_integer_bounds(self):
+        # This should not raise an exception
+        fmin_slsqp(lambda z: z**2 - 1, [0], bounds=[[0, 1]], iprint=0)
+
+    def test_array_bounds(self):
+        # This should pass (1-element arrays are castable to floats in numpy)
+        bounds = [(-np.inf, np.inf), (np.array([2]), np.array([3]))]
+        x = fmin_slsqp(lambda z: np.sum(z**2 - 1), [2.5, 2.5], bounds=bounds, iprint=0)
+        assert_array_almost_equal(x, [0, 2])
+
+    def test_obj_must_return_scalar(self):
+        # Regression test for Github Issue #5433
+        # If objective function does not return a scalar, raises ValueError
+        with assert_raises(ValueError):
+            fmin_slsqp(lambda x: [0, 1], [1, 2, 3])
+
+    def test_obj_returns_scalar_in_list(self):
+        # Test for Github Issue #5433 and PR #6691
+        # Objective function should be able to return length-1 Python list
+        #  containing the scalar
+        fmin_slsqp(lambda x: [0], [1, 2, 3], iprint=0)
+
+    def test_callback(self):
+        # Minimize, method='SLSQP': unbounded, approximated jacobian. Check for callback
+        callback = MyCallBack()
+        res = minimize(self.fun, [-1.0, 1.0], args=(-1.0, ),
+                       method='SLSQP', callback=callback, options=self.opts)
+        assert_(res['success'], res['message'])
+        assert_(callback.been_called)
+        assert_equal(callback.ncalls, res['nit'])
+
+    def test_inconsistent_linearization(self):
+        # SLSQP must be able to solve this problem, even if the
+        # linearized problem at the starting point is infeasible.
+
+        # Linearized constraints are
+        #
+        #    2*x0[0]*x[0] >= 1
+        #
+        # At x0 = [0, 1], the second constraint is clearly infeasible.
+        # This triggers a call with n2==1 in the LSQ subroutine.
+        x = [0, 1]
+        f1 = lambda x: x[0] + x[1] - 2
+        f2 = lambda x: x[0]**2 - 1
+        sol = minimize(
+            lambda x: x[0]**2 + x[1]**2,
+            x,
+            constraints=({'type':'eq','fun': f1},
+                         {'type':'ineq','fun': f2}),
+            bounds=((0,None), (0,None)),
+            method='SLSQP')
+        x = sol.x
+
+        assert_allclose(f1(x), 0, atol=1e-8)
+        assert_(f2(x) >= -1e-8)
+        assert_(sol.success, sol)
+
+    def test_regression_5743(self):
+        # SLSQP must not indicate success for this problem,
+        # which is infeasible.
+        x = [1, 2]
+        sol = minimize(
+            lambda x: x[0]**2 + x[1]**2,
+            x,
+            constraints=({'type':'eq','fun': lambda x: x[0]+x[1]-1},
+                         {'type':'ineq','fun': lambda x: x[0]-2}),
+            bounds=((0,None), (0,None)),
+            method='SLSQP')
+        assert_(not sol.success, sol)
+
+    def test_gh_6676(self):
+        def func(x):
+            return (x[0] - 1)**2 + 2*(x[1] - 1)**2 + 0.5*(x[2] - 1)**2
+
+        sol = minimize(func, [0, 0, 0], method='SLSQP')
+        assert_(sol.jac.shape == (3,))
+
+    def test_invalid_bounds(self):
+        # Raise correct error when lower bound is greater than upper bound.
+        # See Github issue 6875.
+        bounds_list = [
+            ((1, 2), (2, 1)),
+            ((2, 1), (1, 2)),
+            ((2, 1), (2, 1)),
+            ((np.inf, 0), (np.inf, 0)),
+            ((1, -np.inf), (0, 1)),
+        ]
+        for bounds in bounds_list:
+            with assert_raises(ValueError):
+                minimize(self.fun, [-1.0, 1.0], bounds=bounds, method='SLSQP')
+
+    def test_bounds_clipping(self):
+        #
+        # SLSQP returns bogus results for initial guess out of bounds, gh-6859
+        #
+        def f(x):
+            return (x[0] - 1)**2
+
+        sol = minimize(f, [10], method='slsqp', bounds=[(None, 0)])
+        assert_(sol.success)
+        assert_allclose(sol.x, 0, atol=1e-10)
+
+        sol = minimize(f, [-10], method='slsqp', bounds=[(2, None)])
+        assert_(sol.success)
+        assert_allclose(sol.x, 2, atol=1e-10)
+
+        sol = minimize(f, [-10], method='slsqp', bounds=[(None, 0)])
+        assert_(sol.success)
+        assert_allclose(sol.x, 0, atol=1e-10)
+
+        sol = minimize(f, [10], method='slsqp', bounds=[(2, None)])
+        assert_(sol.success)
+        assert_allclose(sol.x, 2, atol=1e-10)
+
+        sol = minimize(f, [-0.5], method='slsqp', bounds=[(-1, 0)])
+        assert_(sol.success)
+        assert_allclose(sol.x, 0, atol=1e-10)
+
+        sol = minimize(f, [10], method='slsqp', bounds=[(-1, 0)])
+        assert_(sol.success)
+        assert_allclose(sol.x, 0, atol=1e-10)
+
+    def test_infeasible_initial(self):
+        # Check SLSQP behavior with infeasible initial point
+        def f(x):
+            x, = x
+            return x*x - 2*x + 1
+
+        cons_u = [{'type': 'ineq', 'fun': lambda x: 0 - x}]
+        cons_l = [{'type': 'ineq', 'fun': lambda x: x - 2}]
+        cons_ul = [{'type': 'ineq', 'fun': lambda x: 0 - x},
+                   {'type': 'ineq', 'fun': lambda x: x + 1}]
+
+        sol = minimize(f, [10], method='slsqp', constraints=cons_u)
+        assert_(sol.success)
+        assert_allclose(sol.x, 0, atol=1e-10)
+
+        sol = minimize(f, [-10], method='slsqp', constraints=cons_l)
+        assert_(sol.success)
+        assert_allclose(sol.x, 2, atol=1e-10)
+
+        sol = minimize(f, [-10], method='slsqp', constraints=cons_u)
+        assert_(sol.success)
+        assert_allclose(sol.x, 0, atol=1e-10)
+
+        sol = minimize(f, [10], method='slsqp', constraints=cons_l)
+        assert_(sol.success)
+        assert_allclose(sol.x, 2, atol=1e-10)
+
+        sol = minimize(f, [-0.5], method='slsqp', constraints=cons_ul)
+        assert_(sol.success)
+        assert_allclose(sol.x, 0, atol=1e-10)
+
+        sol = minimize(f, [10], method='slsqp', constraints=cons_ul)
+        assert_(sol.success)
+        assert_allclose(sol.x, 0, atol=1e-10)
+
+    def test_inconsistent_inequalities(self):
+        # gh-7618
+
+        def cost(x):
+            return -1 * x[0] + 4 * x[1]
+
+        def ineqcons1(x):
+            return x[1] - x[0] - 1
+
+        def ineqcons2(x):
+            return x[0] - x[1]
+
+        # The inequalities are inconsistent, so no solution can exist:
+        #
+        # x1 >= x0 + 1
+        # x0 >= x1
+
+        x0 = (1,5)
+        bounds = ((-5, 5), (-5, 5))
+        cons = (dict(type='ineq', fun=ineqcons1), dict(type='ineq', fun=ineqcons2))
+        res = minimize(cost, x0, method='SLSQP', bounds=bounds, constraints=cons)
+
+        assert_(not res.success)
+
+    def test_new_bounds_type(self):
+        f = lambda x: x[0]**2 + x[1]**2
+        bounds = Bounds([1, 0], [np.inf, np.inf])
+        sol = minimize(f, [0, 0], method='slsqp', bounds=bounds)
+        assert_(sol.success)
+        assert_allclose(sol.x, [1, 0])
+
+    def test_nested_minimization(self):
+
+        class NestedProblem():
+
+            def __init__(self):
+                self.F_outer_count = 0
+
+            def F_outer(self, x):
+                self.F_outer_count += 1
+                if self.F_outer_count > 1000:
+                    raise Exception("Nested minimization failed to terminate.")
+                inner_res = minimize(self.F_inner, (3, 4), method="SLSQP")
+                assert_(inner_res.success)
+                assert_allclose(inner_res.x, [1, 1])
+                return x[0]**2 + x[1]**2 + x[2]**2
+
+            def F_inner(self, x):
+                return (x[0] - 1)**2 + (x[1] - 1)**2
+
+            def solve(self):
+                outer_res = minimize(self.F_outer, (5, 5, 5), method="SLSQP")
+                assert_(outer_res.success)
+                assert_allclose(outer_res.x, [0, 0, 0])
+
+        problem = NestedProblem()
+        problem.solve()
+
+    def test_gh1758(self):
+        # the test suggested in gh1758
+        # https://nlopt.readthedocs.io/en/latest/NLopt_Tutorial/
+        # implement two equality constraints, in R^2.
+        def fun(x):
+            return np.sqrt(x[1])
+
+        def f_eqcon(x):
+            """ Equality constraint """
+            return x[1] - (2 * x[0]) ** 3
+
+        def f_eqcon2(x):
+            """ Equality constraint """
+            return x[1] - (-x[0] + 1) ** 3
+
+        c1 = {'type': 'eq', 'fun': f_eqcon}
+        c2 = {'type': 'eq', 'fun': f_eqcon2}
+
+        res = minimize(fun, [8, 0.25], method='SLSQP',
+                       constraints=[c1, c2], bounds=[(-0.5, 1), (0, 8)])
+
+        np.testing.assert_allclose(res.fun, 0.5443310539518)
+        np.testing.assert_allclose(res.x, [0.33333333, 0.2962963])
+        assert res.success
+
+    def test_gh9640(self):
+        np.random.seed(10)
+        cons = ({'type': 'ineq', 'fun': lambda x: -x[0] - x[1] - 3},
+                {'type': 'ineq', 'fun': lambda x: x[1] + x[2] - 2})
+        bnds = ((-2, 2), (-2, 2), (-2, 2))
+
+        target = lambda x: 1
+        x0 = [-1.8869783504471584, -0.640096352696244, -0.8174212253407696]
+        res = minimize(target, x0, method='SLSQP', bounds=bnds, constraints=cons,
+                       options={'disp':False, 'maxiter':10000})
+
+        # The problem is infeasible, so it cannot succeed
+        assert not res.success

venv/Lib/site-packages/scipy/optimize/tests/test_tnc.py

@@ -0,0 +1,303 @@
+"""
+Unit tests for TNC optimization routine from tnc.py
+"""
+
+from numpy.testing import assert_allclose, assert_equal
+
+import numpy as np
+from math import pow
+
+from scipy import optimize
+from scipy.sparse.sputils import matrix
+
+
+class TestTnc(object):
+    """TNC non-linear optimization.
+
+    These tests are taken from Prof. K. Schittkowski's test examples
+    for constrained non-linear programming.
+
+    http://www.uni-bayreuth.de/departments/math/~kschittkowski/home.htm
+
+    """
+    def setup_method(self):
+        # options for minimize
+        self.opts = {'disp': False, 'maxfun': 200}
+
+    # objective functions and Jacobian for each test
+    def f1(self, x, a=100.0):
+        return a * pow((x[1] - pow(x[0], 2)), 2) + pow(1.0 - x[0], 2)
+
+    def g1(self, x, a=100.0):
+        dif = [0, 0]
+        dif[1] = 2 * a * (x[1] - pow(x[0], 2))
+        dif[0] = -2.0 * (x[0] * (dif[1] - 1.0) + 1.0)
+        return dif
+
+    def fg1(self, x, a=100.0):
+        return self.f1(x, a), self.g1(x, a)
+
+    def f3(self, x):
+        return x[1] + pow(x[1] - x[0], 2) * 1.0e-5
+
+    def g3(self, x):
+        dif = [0, 0]
+        dif[0] = -2.0 * (x[1] - x[0]) * 1.0e-5
+        dif[1] = 1.0 - dif[0]
+        return dif
+
+    def fg3(self, x):
+        return self.f3(x), self.g3(x)
+
+    def f4(self, x):
+        return pow(x[0] + 1.0, 3) / 3.0 + x[1]
+
+    def g4(self, x):
+        dif = [0, 0]
+        dif[0] = pow(x[0] + 1.0, 2)
+        dif[1] = 1.0
+        return dif
+
+    def fg4(self, x):
+        return self.f4(x), self.g4(x)
+
+    def f5(self, x):
+        return np.sin(x[0] + x[1]) + pow(x[0] - x[1], 2) - \
+                1.5 * x[0] + 2.5 * x[1] + 1.0
+
+    def g5(self, x):
+        dif = [0, 0]
+        v1 = np.cos(x[0] + x[1])
+        v2 = 2.0*(x[0] - x[1])
+
+        dif[0] = v1 + v2 - 1.5
+        dif[1] = v1 - v2 + 2.5
+        return dif
+
+    def fg5(self, x):
+        return self.f5(x), self.g5(x)
+
+    def f38(self, x):
+        return (100.0 * pow(x[1] - pow(x[0], 2), 2) +
+                pow(1.0 - x[0], 2) + 90.0 * pow(x[3] - pow(x[2], 2), 2) +
+                pow(1.0 - x[2], 2) + 10.1 * (pow(x[1] - 1.0, 2) +
+                                             pow(x[3] - 1.0, 2)) +
+                19.8 * (x[1] - 1.0) * (x[3] - 1.0)) * 1.0e-5
+
+    def g38(self, x):
+        dif = [0, 0, 0, 0]
+        dif[0] = (-400.0 * x[0] * (x[1] - pow(x[0], 2)) -
+                  2.0 * (1.0 - x[0])) * 1.0e-5
+        dif[1] = (200.0 * (x[1] - pow(x[0], 2)) + 20.2 * (x[1] - 1.0) +
+                  19.8 * (x[3] - 1.0)) * 1.0e-5
+        dif[2] = (- 360.0 * x[2] * (x[3] - pow(x[2], 2)) -
+                  2.0 * (1.0 - x[2])) * 1.0e-5
+        dif[3] = (180.0 * (x[3] - pow(x[2], 2)) + 20.2 * (x[3] - 1.0) +
+                  19.8 * (x[1] - 1.0)) * 1.0e-5
+        return dif
+
+    def fg38(self, x):
+        return self.f38(x), self.g38(x)
+
+    def f45(self, x):
+        return 2.0 - x[0] * x[1] * x[2] * x[3] * x[4] / 120.0
+
+    def g45(self, x):
+        dif = [0] * 5
+        dif[0] = - x[1] * x[2] * x[3] * x[4] / 120.0
+        dif[1] = - x[0] * x[2] * x[3] * x[4] / 120.0
+        dif[2] = - x[0] * x[1] * x[3] * x[4] / 120.0
+        dif[3] = - x[0] * x[1] * x[2] * x[4] / 120.0
+        dif[4] = - x[0] * x[1] * x[2] * x[3] / 120.0
+        return dif
+
+    def fg45(self, x):
+        return self.f45(x), self.g45(x)
+
+    # tests
+    # minimize with method=TNC
+    def test_minimize_tnc1(self):
+        x0, bnds = [-2, 1], ([-np.inf, None], [-1.5, None])
+        xopt = [1, 1]
+        iterx = []  # to test callback
+
+        res = optimize.minimize(self.f1, x0, method='TNC', jac=self.g1,
+                                bounds=bnds, options=self.opts,
+                                callback=iterx.append)
+        assert_allclose(res.fun, self.f1(xopt), atol=1e-8)
+        assert_equal(len(iterx), res.nit)
+
+    def test_minimize_tnc1b(self):
+        x0, bnds = matrix([-2, 1]), ([-np.inf, None],[-1.5, None])
+        xopt = [1, 1]
+        x = optimize.minimize(self.f1, x0, method='TNC',
+                              bounds=bnds, options=self.opts).x
+        assert_allclose(self.f1(x), self.f1(xopt), atol=1e-4)
+
+    def test_minimize_tnc1c(self):
+        x0, bnds = [-2, 1], ([-np.inf, None],[-1.5, None])
+        xopt = [1, 1]
+        x = optimize.minimize(self.fg1, x0, method='TNC',
+                              jac=True, bounds=bnds,
+                              options=self.opts).x
+        assert_allclose(self.f1(x), self.f1(xopt), atol=1e-8)
+
+    def test_minimize_tnc2(self):
+        x0, bnds = [-2, 1], ([-np.inf, None], [1.5, None])
+        xopt = [-1.2210262419616387, 1.5]
+        x = optimize.minimize(self.f1, x0, method='TNC',
+                              jac=self.g1, bounds=bnds,
+                              options=self.opts).x
+        assert_allclose(self.f1(x), self.f1(xopt), atol=1e-8)
+
+    def test_minimize_tnc3(self):
+        x0, bnds = [10, 1], ([-np.inf, None], [0.0, None])
+        xopt = [0, 0]
+        x = optimize.minimize(self.f3, x0, method='TNC',
+                              jac=self.g3, bounds=bnds,
+                              options=self.opts).x
+        assert_allclose(self.f3(x), self.f3(xopt), atol=1e-8)
+
+    def test_minimize_tnc4(self):
+        x0,bnds = [1.125, 0.125], [(1, None), (0, None)]
+        xopt = [1, 0]
+        x = optimize.minimize(self.f4, x0, method='TNC',
+                              jac=self.g4, bounds=bnds,
+                              options=self.opts).x
+        assert_allclose(self.f4(x), self.f4(xopt), atol=1e-8)
+
+    def test_minimize_tnc5(self):
+        x0, bnds = [0, 0], [(-1.5, 4),(-3, 3)]
+        xopt = [-0.54719755119659763, -1.5471975511965976]
+        x = optimize.minimize(self.f5, x0, method='TNC',
+                              jac=self.g5, bounds=bnds,
+                              options=self.opts).x
+        assert_allclose(self.f5(x), self.f5(xopt), atol=1e-8)
+
+    def test_minimize_tnc38(self):
+        x0, bnds = np.array([-3, -1, -3, -1]), [(-10, 10)]*4
+        xopt = [1]*4
+        x = optimize.minimize(self.f38, x0, method='TNC',
+                              jac=self.g38, bounds=bnds,
+                              options=self.opts).x
+        assert_allclose(self.f38(x), self.f38(xopt), atol=1e-8)
+
+    def test_minimize_tnc45(self):
+        x0, bnds = [2] * 5, [(0, 1), (0, 2), (0, 3), (0, 4), (0, 5)]
+        xopt = [1, 2, 3, 4, 5]
+        x = optimize.minimize(self.f45, x0, method='TNC',
+                              jac=self.g45, bounds=bnds,
+                              options=self.opts).x
+        assert_allclose(self.f45(x), self.f45(xopt), atol=1e-8)
+
+    # fmin_tnc
+    def test_tnc1(self):
+        fg, x, bounds = self.fg1, [-2, 1], ([-np.inf, None], [-1.5, None])
+        xopt = [1, 1]
+
+        x, nf, rc = optimize.fmin_tnc(fg, x, bounds=bounds, args=(100.0, ),
+                                      messages=optimize.tnc.MSG_NONE,
+                                      maxfun=200)
+
+        assert_allclose(self.f1(x), self.f1(xopt), atol=1e-8,
+                        err_msg="TNC failed with status: " +
+                                optimize.tnc.RCSTRINGS[rc])
+
+    def test_tnc1b(self):
+        x, bounds = [-2, 1], ([-np.inf, None], [-1.5, None])
+        xopt = [1, 1]
+
+        x, nf, rc = optimize.fmin_tnc(self.f1, x, approx_grad=True,
+                                      bounds=bounds,
+                                      messages=optimize.tnc.MSG_NONE,
+                                      maxfun=200)
+
+        assert_allclose(self.f1(x), self.f1(xopt), atol=1e-4,
+                        err_msg="TNC failed with status: " +
+                                optimize.tnc.RCSTRINGS[rc])
+
+    def test_tnc1c(self):
+        x, bounds = [-2, 1], ([-np.inf, None], [-1.5, None])
+        xopt = [1, 1]
+
+        x, nf, rc = optimize.fmin_tnc(self.f1, x, fprime=self.g1,
+                                      bounds=bounds,
+                                      messages=optimize.tnc.MSG_NONE,
+                                      maxfun=200)
+
+        assert_allclose(self.f1(x), self.f1(xopt), atol=1e-8,
+                        err_msg="TNC failed with status: " +
+                                optimize.tnc.RCSTRINGS[rc])
+
+    def test_tnc2(self):
+        fg, x, bounds = self.fg1, [-2, 1], ([-np.inf, None], [1.5, None])
+        xopt = [-1.2210262419616387, 1.5]
+
+        x, nf, rc = optimize.fmin_tnc(fg, x, bounds=bounds,
+                                      messages=optimize.tnc.MSG_NONE,
+                                      maxfun=200)
+
+        assert_allclose(self.f1(x), self.f1(xopt), atol=1e-8,
+                        err_msg="TNC failed with status: " +
+                                optimize.tnc.RCSTRINGS[rc])
+
+    def test_tnc3(self):
+        fg, x, bounds = self.fg3, [10, 1], ([-np.inf, None], [0.0, None])
+        xopt = [0, 0]
+
+        x, nf, rc = optimize.fmin_tnc(fg, x, bounds=bounds,
+                                      messages=optimize.tnc.MSG_NONE,
+                                      maxfun=200)
+
+        assert_allclose(self.f3(x), self.f3(xopt), atol=1e-8,
+                        err_msg="TNC failed with status: " +
+                                optimize.tnc.RCSTRINGS[rc])
+
+    def test_tnc4(self):
+        fg, x, bounds = self.fg4, [1.125, 0.125], [(1, None), (0, None)]
+        xopt = [1, 0]
+
+        x, nf, rc = optimize.fmin_tnc(fg, x, bounds=bounds,
+                                      messages=optimize.tnc.MSG_NONE,
+                                      maxfun=200)
+
+        assert_allclose(self.f4(x), self.f4(xopt), atol=1e-8,
+                        err_msg="TNC failed with status: " +
+                                optimize.tnc.RCSTRINGS[rc])
+
+    def test_tnc5(self):
+        fg, x, bounds = self.fg5, [0, 0], [(-1.5, 4),(-3, 3)]
+        xopt = [-0.54719755119659763, -1.5471975511965976]
+
+        x, nf, rc = optimize.fmin_tnc(fg, x, bounds=bounds,
+                                      messages=optimize.tnc.MSG_NONE,
+                                      maxfun=200)
+
+        assert_allclose(self.f5(x), self.f5(xopt), atol=1e-8,
+                        err_msg="TNC failed with status: " +
+                                optimize.tnc.RCSTRINGS[rc])
+
+    def test_tnc38(self):
+        fg, x, bounds = self.fg38, np.array([-3, -1, -3, -1]), [(-10, 10)]*4
+        xopt = [1]*4
+
+        x, nf, rc = optimize.fmin_tnc(fg, x, bounds=bounds,
+                                      messages=optimize.tnc.MSG_NONE,
+                                      maxfun=200)
+
+        assert_allclose(self.f38(x), self.f38(xopt), atol=1e-8,
+                        err_msg="TNC failed with status: " +
+                                optimize.tnc.RCSTRINGS[rc])
+
+    def test_tnc45(self):
+        fg, x, bounds = self.fg45, [2] * 5, [(0, 1), (0, 2), (0, 3),
+                                             (0, 4), (0, 5)]
+        xopt = [1, 2, 3, 4, 5]
+
+        x, nf, rc = optimize.fmin_tnc(fg, x, bounds=bounds,
+                                      messages=optimize.tnc.MSG_NONE,
+                                      maxfun=200)
+
+        assert_allclose(self.f45(x), self.f45(xopt), atol=1e-8,
+                        err_msg="TNC failed with status: " +
+                                optimize.tnc.RCSTRINGS[rc])

venv/Lib/site-packages/scipy/optimize/tests/test_trustregion.py

@@ -0,0 +1,104 @@
+"""
+Unit tests for trust-region optimization routines.
+
+To run it in its simplest form::
+  nosetests test_optimize.py
+
+"""
+import numpy as np
+from scipy.optimize import (minimize, rosen, rosen_der, rosen_hess,
+                            rosen_hess_prod)
+from numpy.testing import assert_, assert_equal, assert_allclose
+
+
+class Accumulator:
+    """ This is for testing callbacks."""
+    def __init__(self):
+        self.count = 0
+        self.accum = None
+
+    def __call__(self, x):
+        self.count += 1
+        if self.accum is None:
+            self.accum = np.array(x)
+        else:
+            self.accum += x
+
+
+class TestTrustRegionSolvers(object):
+
+    def setup_method(self):
+        self.x_opt = [1.0, 1.0]
+        self.easy_guess = [2.0, 2.0]
+        self.hard_guess = [-1.2, 1.0]
+
+    def test_dogleg_accuracy(self):
+        # test the accuracy and the return_all option
+        x0 = self.hard_guess
+        r = minimize(rosen, x0, jac=rosen_der, hess=rosen_hess, tol=1e-8,
+                     method='dogleg', options={'return_all': True},)
+        assert_allclose(x0, r['allvecs'][0])
+        assert_allclose(r['x'], r['allvecs'][-1])
+        assert_allclose(r['x'], self.x_opt)
+
+    def test_dogleg_callback(self):
+        # test the callback mechanism and the maxiter and return_all options
+        accumulator = Accumulator()
+        maxiter = 5
+        r = minimize(rosen, self.hard_guess, jac=rosen_der, hess=rosen_hess,
+                     callback=accumulator, method='dogleg',
+                     options={'return_all': True, 'maxiter': maxiter},)
+        assert_equal(accumulator.count, maxiter)
+        assert_equal(len(r['allvecs']), maxiter+1)
+        assert_allclose(r['x'], r['allvecs'][-1])
+        assert_allclose(sum(r['allvecs'][1:]), accumulator.accum)
+
+    def test_solver_concordance(self):
+        # Assert that dogleg uses fewer iterations than ncg on the Rosenbrock
+        # test function, although this does not necessarily mean
+        # that dogleg is faster or better than ncg even for this function
+        # and especially not for other test functions.
+        f = rosen
+        g = rosen_der
+        h = rosen_hess
+        for x0 in (self.easy_guess, self.hard_guess):
+            r_dogleg = minimize(f, x0, jac=g, hess=h, tol=1e-8,
+                                method='dogleg', options={'return_all': True})
+            r_trust_ncg = minimize(f, x0, jac=g, hess=h, tol=1e-8,
+                                   method='trust-ncg',
+                                   options={'return_all': True})
+            r_trust_krylov = minimize(f, x0, jac=g, hess=h, tol=1e-8,
+                                   method='trust-krylov',
+                                   options={'return_all': True})
+            r_ncg = minimize(f, x0, jac=g, hess=h, tol=1e-8,
+                             method='newton-cg', options={'return_all': True})
+            r_iterative = minimize(f, x0, jac=g, hess=h, tol=1e-8,
+                                   method='trust-exact',
+                                   options={'return_all': True})
+            assert_allclose(self.x_opt, r_dogleg['x'])
+            assert_allclose(self.x_opt, r_trust_ncg['x'])
+            assert_allclose(self.x_opt, r_trust_krylov['x'])
+            assert_allclose(self.x_opt, r_ncg['x'])
+            assert_allclose(self.x_opt, r_iterative['x'])
+            assert_(len(r_dogleg['allvecs']) < len(r_ncg['allvecs']))
+
+    def test_trust_ncg_hessp(self):
+        for x0 in (self.easy_guess, self.hard_guess, self.x_opt):
+            r = minimize(rosen, x0, jac=rosen_der, hessp=rosen_hess_prod,
+                         tol=1e-8, method='trust-ncg')
+            assert_allclose(self.x_opt, r['x'])
+
+    def test_trust_ncg_start_in_optimum(self):
+        r = minimize(rosen, x0=self.x_opt, jac=rosen_der, hess=rosen_hess,
+                     tol=1e-8, method='trust-ncg')
+        assert_allclose(self.x_opt, r['x'])
+
+    def test_trust_krylov_start_in_optimum(self):
+        r = minimize(rosen, x0=self.x_opt, jac=rosen_der, hess=rosen_hess,
+                     tol=1e-8, method='trust-krylov')
+        assert_allclose(self.x_opt, r['x'])
+
+    def test_trust_exact_start_in_optimum(self):
+        r = minimize(rosen, x0=self.x_opt, jac=rosen_der, hess=rosen_hess,
+                     tol=1e-8, method='trust-exact')
+        assert_allclose(self.x_opt, r['x'])

venv/Lib/site-packages/scipy/optimize/tests/test_trustregion_exact.py

@@ -0,0 +1,352 @@
+"""
+Unit tests for trust-region iterative subproblem.
+
+To run it in its simplest form::
+  nosetests test_optimize.py
+
+"""
+import numpy as np
+from scipy.optimize._trustregion_exact import (
+    estimate_smallest_singular_value,
+    singular_leading_submatrix,
+    IterativeSubproblem)
+from scipy.linalg import (svd, get_lapack_funcs, det, qr, norm)
+from numpy.testing import (assert_array_equal,
+                           assert_equal, assert_array_almost_equal)
+
+
+def random_entry(n, min_eig, max_eig, case):
+
+    # Generate random matrix
+    rand = np.random.uniform(-1, 1, (n, n))
+
+    # QR decomposition
+    Q, _, _ = qr(rand, pivoting='True')
+
+    # Generate random eigenvalues
+    eigvalues = np.random.uniform(min_eig, max_eig, n)
+    eigvalues = np.sort(eigvalues)[::-1]
+
+    # Generate matrix
+    Qaux = np.multiply(eigvalues, Q)
+    A = np.dot(Qaux, Q.T)
+
+    # Generate gradient vector accordingly
+    # to the case is being tested.
+    if case == 'hard':
+        g = np.zeros(n)
+        g[:-1] = np.random.uniform(-1, 1, n-1)
+        g = np.dot(Q, g)
+    elif case == 'jac_equal_zero':
+        g = np.zeros(n)
+    else:
+        g = np.random.uniform(-1, 1, n)
+
+    return A, g
+
+
+class TestEstimateSmallestSingularValue(object):
+
+    def test_for_ill_condiotioned_matrix(self):
+
+        # Ill-conditioned triangular matrix
+        C = np.array([[1, 2, 3, 4],
+                      [0, 0.05, 60, 7],
+                      [0, 0, 0.8, 9],
+                      [0, 0, 0, 10]])
+
+        # Get svd decomposition
+        U, s, Vt = svd(C)
+
+        # Get smallest singular value and correspondent right singular vector.
+        smin_svd = s[-1]
+        zmin_svd = Vt[-1, :]
+
+        # Estimate smallest singular value
+        smin, zmin = estimate_smallest_singular_value(C)
+
+        # Check the estimation
+        assert_array_almost_equal(smin, smin_svd, decimal=8)
+        assert_array_almost_equal(abs(zmin), abs(zmin_svd), decimal=8)
+
+
+class TestSingularLeadingSubmatrix(object):
+
+    def test_for_already_singular_leading_submatrix(self):
+
+        # Define test matrix A.
+        # Note that the leading 2x2 submatrix is singular.
+        A = np.array([[1, 2, 3],
+                      [2, 4, 5],
+                      [3, 5, 6]])
+
+        # Get Cholesky from lapack functions
+        cholesky, = get_lapack_funcs(('potrf',), (A,))
+
+        # Compute Cholesky Decomposition
+        c, k = cholesky(A, lower=False, overwrite_a=False, clean=True)
+
+        delta, v = singular_leading_submatrix(A, c, k)
+
+        A[k-1, k-1] += delta
+
+        # Check if the leading submatrix is singular.
+        assert_array_almost_equal(det(A[:k, :k]), 0)
+
+        # Check if `v` fullfil the specified properties
+        quadratic_term = np.dot(v, np.dot(A, v))
+        assert_array_almost_equal(quadratic_term, 0)
+
+    def test_for_simetric_indefinite_matrix(self):
+
+        # Define test matrix A.
+        # Note that the leading 5x5 submatrix is indefinite.
+        A = np.asarray([[1, 2, 3, 7, 8],
+                        [2, 5, 5, 9, 0],
+                        [3, 5, 11, 1, 2],
+                        [7, 9, 1, 7, 5],
+                        [8, 0, 2, 5, 8]])
+
+        # Get Cholesky from lapack functions
+        cholesky, = get_lapack_funcs(('potrf',), (A,))
+
+        # Compute Cholesky Decomposition
+        c, k = cholesky(A, lower=False, overwrite_a=False, clean=True)
+
+        delta, v = singular_leading_submatrix(A, c, k)
+
+        A[k-1, k-1] += delta
+
+        # Check if the leading submatrix is singular.
+        assert_array_almost_equal(det(A[:k, :k]), 0)
+
+        # Check if `v` fullfil the specified properties
+        quadratic_term = np.dot(v, np.dot(A, v))
+        assert_array_almost_equal(quadratic_term, 0)
+
+    def test_for_first_element_equal_to_zero(self):
+
+        # Define test matrix A.
+        # Note that the leading 2x2 submatrix is singular.
+        A = np.array([[0, 3, 11],
+                      [3, 12, 5],
+                      [11, 5, 6]])
+
+        # Get Cholesky from lapack functions
+        cholesky, = get_lapack_funcs(('potrf',), (A,))
+
+        # Compute Cholesky Decomposition
+        c, k = cholesky(A, lower=False, overwrite_a=False, clean=True)
+
+        delta, v = singular_leading_submatrix(A, c, k)
+
+        A[k-1, k-1] += delta
+
+        # Check if the leading submatrix is singular
+        assert_array_almost_equal(det(A[:k, :k]), 0)
+
+        # Check if `v` fullfil the specified properties
+        quadratic_term = np.dot(v, np.dot(A, v))
+        assert_array_almost_equal(quadratic_term, 0)
+
+
+class TestIterativeSubproblem(object):
+
+    def test_for_the_easy_case(self):
+
+        # `H` is chosen such that `g` is not orthogonal to the
+        # eigenvector associated with the smallest eigenvalue `s`.
+        H = [[10, 2, 3, 4],
+             [2, 1, 7, 1],
+             [3, 7, 1, 7],
+             [4, 1, 7, 2]]
+        g = [1, 1, 1, 1]
+
+        # Trust Radius
+        trust_radius = 1
+
+        # Solve Subproblem
+        subprob = IterativeSubproblem(x=0,
+                                      fun=lambda x: 0,
+                                      jac=lambda x: np.array(g),
+                                      hess=lambda x: np.array(H),
+                                      k_easy=1e-10,
+                                      k_hard=1e-10)
+        p, hits_boundary = subprob.solve(trust_radius)
+
+        assert_array_almost_equal(p, [0.00393332, -0.55260862,
+                                      0.67065477, -0.49480341])
+        assert_array_almost_equal(hits_boundary, True)
+
+    def test_for_the_hard_case(self):
+
+        # `H` is chosen such that `g` is orthogonal to the
+        # eigenvector associated with the smallest eigenvalue `s`.
+        H = [[10, 2, 3, 4],
+             [2, 1, 7, 1],
+             [3, 7, 1, 7],
+             [4, 1, 7, 2]]
+        g = [6.4852641521327437, 1, 1, 1]
+        s = -8.2151519874416614
+
+        # Trust Radius
+        trust_radius = 1
+
+        # Solve Subproblem
+        subprob = IterativeSubproblem(x=0,
+                                      fun=lambda x: 0,
+                                      jac=lambda x: np.array(g),
+                                      hess=lambda x: np.array(H),
+                                      k_easy=1e-10,
+                                      k_hard=1e-10)
+        p, hits_boundary = subprob.solve(trust_radius)
+
+        assert_array_almost_equal(-s, subprob.lambda_current)
+
+    def test_for_interior_convergence(self):
+
+        H = [[1.812159, 0.82687265, 0.21838879, -0.52487006, 0.25436988],
+             [0.82687265, 2.66380283, 0.31508988, -0.40144163, 0.08811588],
+             [0.21838879, 0.31508988, 2.38020726, -0.3166346, 0.27363867],
+             [-0.52487006, -0.40144163, -0.3166346, 1.61927182, -0.42140166],
+             [0.25436988, 0.08811588, 0.27363867, -0.42140166, 1.33243101]]
+
+        g = [0.75798952, 0.01421945, 0.33847612, 0.83725004, -0.47909534]
+
+        # Solve Subproblem
+        subprob = IterativeSubproblem(x=0,
+                                      fun=lambda x: 0,
+                                      jac=lambda x: np.array(g),
+                                      hess=lambda x: np.array(H))
+        p, hits_boundary = subprob.solve(1.1)
+
+        assert_array_almost_equal(p, [-0.68585435, 0.1222621, -0.22090999,
+                                      -0.67005053, 0.31586769])
+        assert_array_almost_equal(hits_boundary, False)
+        assert_array_almost_equal(subprob.lambda_current, 0)
+        assert_array_almost_equal(subprob.niter, 1)
+
+    def test_for_jac_equal_zero(self):
+
+        H = [[0.88547534, 2.90692271, 0.98440885, -0.78911503, -0.28035809],
+             [2.90692271, -0.04618819, 0.32867263, -0.83737945, 0.17116396],
+             [0.98440885, 0.32867263, -0.87355957, -0.06521957, -1.43030957],
+             [-0.78911503, -0.83737945, -0.06521957, -1.645709, -0.33887298],
+             [-0.28035809, 0.17116396, -1.43030957, -0.33887298, -1.68586978]]
+
+        g = [0, 0, 0, 0, 0]
+
+        # Solve Subproblem
+        subprob = IterativeSubproblem(x=0,
+                                      fun=lambda x: 0,
+                                      jac=lambda x: np.array(g),
+                                      hess=lambda x: np.array(H),
+                                      k_easy=1e-10,
+                                      k_hard=1e-10)
+        p, hits_boundary = subprob.solve(1.1)
+
+        assert_array_almost_equal(p, [0.06910534, -0.01432721,
+                                      -0.65311947, -0.23815972,
+                                      -0.84954934])
+        assert_array_almost_equal(hits_boundary, True)
+
+    def test_for_jac_very_close_to_zero(self):
+
+        H = [[0.88547534, 2.90692271, 0.98440885, -0.78911503, -0.28035809],
+             [2.90692271, -0.04618819, 0.32867263, -0.83737945, 0.17116396],
+             [0.98440885, 0.32867263, -0.87355957, -0.06521957, -1.43030957],
+             [-0.78911503, -0.83737945, -0.06521957, -1.645709, -0.33887298],
+             [-0.28035809, 0.17116396, -1.43030957, -0.33887298, -1.68586978]]
+
+        g = [0, 0, 0, 0, 1e-15]
+
+        # Solve Subproblem
+        subprob = IterativeSubproblem(x=0,
+                                      fun=lambda x: 0,
+                                      jac=lambda x: np.array(g),
+                                      hess=lambda x: np.array(H),
+                                      k_easy=1e-10,
+                                      k_hard=1e-10)
+        p, hits_boundary = subprob.solve(1.1)
+
+        assert_array_almost_equal(p, [0.06910534, -0.01432721,
+                                      -0.65311947, -0.23815972,
+                                      -0.84954934])
+        assert_array_almost_equal(hits_boundary, True)
+
+    def test_for_random_entries(self):
+        # Seed
+        np.random.seed(1)
+
+        # Dimension
+        n = 5
+
+        for case in ('easy', 'hard', 'jac_equal_zero'):
+
+            eig_limits = [(-20, -15),
+                          (-10, -5),
+                          (-10, 0),
+                          (-5, 5),
+                          (-10, 10),
+                          (0, 10),
+                          (5, 10),
+                          (15, 20)]
+
+            for min_eig, max_eig in eig_limits:
+                # Generate random symmetric matrix H with
+                # eigenvalues between min_eig and max_eig.
+                H, g = random_entry(n, min_eig, max_eig, case)
+
+                # Trust radius
+                trust_radius_list = [0.1, 0.3, 0.6, 0.8, 1, 1.2, 3.3, 5.5, 10]
+
+                for trust_radius in trust_radius_list:
+                    # Solve subproblem with very high accuracy
+                    subprob_ac = IterativeSubproblem(0,
+                                                     lambda x: 0,
+                                                     lambda x: g,
+                                                     lambda x: H,
+                                                     k_easy=1e-10,
+                                                     k_hard=1e-10)
+
+                    p_ac, hits_boundary_ac = subprob_ac.solve(trust_radius)
+
+                    # Compute objective function value
+                    J_ac = 1/2*np.dot(p_ac, np.dot(H, p_ac))+np.dot(g, p_ac)
+
+                    stop_criteria = [(0.1, 2),
+                                     (0.5, 1.1),
+                                     (0.9, 1.01)]
+
+                    for k_opt, k_trf in stop_criteria:
+
+                        # k_easy and k_hard computed in function
+                        # of k_opt and k_trf accordingly to
+                        # Conn, A. R., Gould, N. I., & Toint, P. L. (2000).
+                        # "Trust region methods". Siam. p. 197.
+                        k_easy = min(k_trf-1,
+                                     1-np.sqrt(k_opt))
+                        k_hard = 1-k_opt
+
+                        # Solve subproblem
+                        subprob = IterativeSubproblem(0,
+                                                      lambda x: 0,
+                                                      lambda x: g,
+                                                      lambda x: H,
+                                                      k_easy=k_easy,
+                                                      k_hard=k_hard)
+                        p, hits_boundary = subprob.solve(trust_radius)
+
+                        # Compute objective function value
+                        J = 1/2*np.dot(p, np.dot(H, p))+np.dot(g, p)
+
+                        # Check if it respect k_trf
+                        if hits_boundary:
+                            assert_array_equal(np.abs(norm(p)-trust_radius) <=
+                                               (k_trf-1)*trust_radius, True)
+                        else:
+                            assert_equal(norm(p) <= trust_radius, True)
+
+                        # Check if it respect k_opt
+                        assert_equal(J <= k_opt*J_ac, True)
+

venv/Lib/site-packages/scipy/optimize/tests/test_trustregion_krylov.py

@@ -0,0 +1,170 @@
+"""
+Unit tests for Krylov space trust-region subproblem solver.
+
+To run it in its simplest form::
+  nosetests test_optimize.py
+
+"""
+import numpy as np
+from scipy.optimize._trlib import (get_trlib_quadratic_subproblem)
+from numpy.testing import (assert_,
+                           assert_almost_equal,
+                           assert_equal, assert_array_almost_equal)
+
+KrylovQP = get_trlib_quadratic_subproblem(tol_rel_i=1e-8, tol_rel_b=1e-6)
+KrylovQP_disp = get_trlib_quadratic_subproblem(tol_rel_i=1e-8, tol_rel_b=1e-6, disp=True)
+
+class TestKrylovQuadraticSubproblem(object):
+
+    def test_for_the_easy_case(self):
+
+        # `H` is chosen such that `g` is not orthogonal to the
+        # eigenvector associated with the smallest eigenvalue.
+        H = np.array([[1.0, 0.0, 4.0],
+                      [0.0, 2.0, 0.0],
+                      [4.0, 0.0, 3.0]])
+        g = np.array([5.0, 0.0, 4.0])
+
+        # Trust Radius
+        trust_radius = 1.0
+
+        # Solve Subproblem
+        subprob = KrylovQP(x=0,
+                           fun=lambda x: 0,
+                           jac=lambda x: g,
+                           hess=lambda x: None,
+                           hessp=lambda x, y: H.dot(y))
+        p, hits_boundary = subprob.solve(trust_radius)
+
+        assert_array_almost_equal(p, np.array([-1.0, 0.0, 0.0]))
+        assert_equal(hits_boundary, True)
+        # check kkt satisfaction
+        assert_almost_equal(
+                np.linalg.norm(H.dot(p) + subprob.lam * p + g),
+                0.0)
+        # check trust region constraint
+        assert_almost_equal(np.linalg.norm(p), trust_radius)
+
+        trust_radius = 0.5
+        p, hits_boundary = subprob.solve(trust_radius)
+
+        assert_array_almost_equal(p,
+                np.array([-0.46125446, 0., -0.19298788]))
+        assert_equal(hits_boundary, True)
+        # check kkt satisfaction
+        assert_almost_equal(
+                np.linalg.norm(H.dot(p) + subprob.lam * p + g),
+                0.0)
+        # check trust region constraint
+        assert_almost_equal(np.linalg.norm(p), trust_radius)
+
+    def test_for_the_hard_case(self):
+
+        # `H` is chosen such that `g` is orthogonal to the
+        # eigenvector associated with the smallest eigenvalue.
+        H = np.array([[1.0, 0.0, 4.0],
+                      [0.0, 2.0, 0.0],
+                      [4.0, 0.0, 3.0]])
+        g = np.array([0.0, 2.0, 0.0])
+
+        # Trust Radius
+        trust_radius = 1.0
+
+        # Solve Subproblem
+        subprob = KrylovQP(x=0,
+                           fun=lambda x: 0,
+                           jac=lambda x: g,
+                           hess=lambda x: None,
+                           hessp=lambda x, y: H.dot(y))
+        p, hits_boundary = subprob.solve(trust_radius)
+
+        assert_array_almost_equal(p, np.array([0.0, -1.0, 0.0]))
+        # check kkt satisfaction
+        assert_almost_equal(
+                np.linalg.norm(H.dot(p) + subprob.lam * p + g),
+                0.0)
+        # check trust region constraint
+        assert_almost_equal(np.linalg.norm(p), trust_radius)
+
+        trust_radius = 0.5
+        p, hits_boundary = subprob.solve(trust_radius)
+
+        assert_array_almost_equal(p, np.array([0.0, -0.5, 0.0]))
+        # check kkt satisfaction
+        assert_almost_equal(
+                np.linalg.norm(H.dot(p) + subprob.lam * p + g),
+                0.0)
+        # check trust region constraint
+        assert_almost_equal(np.linalg.norm(p), trust_radius)
+
+    def test_for_interior_convergence(self):
+
+        H = np.array([[1.812159, 0.82687265, 0.21838879, -0.52487006, 0.25436988],
+                      [0.82687265, 2.66380283, 0.31508988, -0.40144163, 0.08811588],
+                      [0.21838879, 0.31508988, 2.38020726, -0.3166346, 0.27363867],
+                      [-0.52487006, -0.40144163, -0.3166346, 1.61927182, -0.42140166],
+                      [0.25436988, 0.08811588, 0.27363867, -0.42140166, 1.33243101]])
+        g = np.array([0.75798952, 0.01421945, 0.33847612, 0.83725004, -0.47909534])
+        trust_radius = 1.1
+
+        # Solve Subproblem
+        subprob = KrylovQP(x=0,
+                           fun=lambda x: 0,
+                           jac=lambda x: g,
+                           hess=lambda x: None,
+                           hessp=lambda x, y: H.dot(y))
+        p, hits_boundary = subprob.solve(trust_radius)
+
+        # check kkt satisfaction
+        assert_almost_equal(
+                np.linalg.norm(H.dot(p) + subprob.lam * p + g),
+                0.0)
+
+        assert_array_almost_equal(p, [-0.68585435, 0.1222621, -0.22090999,
+                                      -0.67005053, 0.31586769])
+        assert_array_almost_equal(hits_boundary, False)
+
+    def test_for_very_close_to_zero(self):
+
+        H = np.array([[0.88547534, 2.90692271, 0.98440885, -0.78911503, -0.28035809],
+                      [2.90692271, -0.04618819, 0.32867263, -0.83737945, 0.17116396],
+                      [0.98440885, 0.32867263, -0.87355957, -0.06521957, -1.43030957],
+                      [-0.78911503, -0.83737945, -0.06521957, -1.645709, -0.33887298],
+                      [-0.28035809, 0.17116396, -1.43030957, -0.33887298, -1.68586978]])
+        g = np.array([0, 0, 0, 0, 1e-6])
+        trust_radius = 1.1
+
+        # Solve Subproblem
+        subprob = KrylovQP(x=0,
+                           fun=lambda x: 0,
+                           jac=lambda x: g,
+                           hess=lambda x: None,
+                           hessp=lambda x, y: H.dot(y))
+        p, hits_boundary = subprob.solve(trust_radius)
+
+        # check kkt satisfaction
+        assert_almost_equal(
+                np.linalg.norm(H.dot(p) + subprob.lam * p + g),
+                0.0)
+        # check trust region constraint
+        assert_almost_equal(np.linalg.norm(p), trust_radius)
+
+        assert_array_almost_equal(p, [0.06910534, -0.01432721,
+                                      -0.65311947, -0.23815972,
+                                      -0.84954934])
+        assert_array_almost_equal(hits_boundary, True)
+
+    def test_disp(self, capsys):
+        H = -np.eye(5)
+        g = np.array([0, 0, 0, 0, 1e-6])
+        trust_radius = 1.1
+
+        subprob = KrylovQP_disp(x=0,
+                                fun=lambda x: 0,
+                                jac=lambda x: g,
+                                hess=lambda x: None,
+                                hessp=lambda x, y: H.dot(y))
+        p, hits_boundary = subprob.solve(trust_radius)
+        out, err = capsys.readouterr()
+        assert_(out.startswith(' TR Solving trust region problem'), repr(out))
+

venv/Lib/site-packages/scipy/optimize/tests/test_zeros.py

@@ -0,0 +1,755 @@
+import pytest
+
+from math import sqrt, exp, sin, cos
+from functools import lru_cache
+
+from numpy.testing import (assert_warns, assert_,
+                           assert_allclose,
+                           assert_equal,
+                           assert_array_equal,
+                           suppress_warnings)
+import numpy as np
+from numpy import finfo, power, nan, isclose
+
+
+from scipy.optimize import zeros, newton, root_scalar
+
+from scipy._lib._util import getfullargspec_no_self as _getfullargspec
+
+# Import testing parameters
+from scipy.optimize._tstutils import get_tests, functions as tstutils_functions, fstrings as tstutils_fstrings
+
+TOL = 4*np.finfo(float).eps  # tolerance
+
+_FLOAT_EPS = finfo(float).eps
+
+# A few test functions used frequently:
+# # A simple quadratic, (x-1)^2 - 1
+def f1(x):
+    return x ** 2 - 2 * x - 1
+
+
+def f1_1(x):
+    return 2 * x - 2
+
+
+def f1_2(x):
+    return 2.0 + 0 * x
+
+
+def f1_and_p_and_pp(x):
+    return f1(x), f1_1(x), f1_2(x)
+
+
+# Simple transcendental function
+def f2(x):
+    return exp(x) - cos(x)
+
+
+def f2_1(x):
+    return exp(x) + sin(x)
+
+
+def f2_2(x):
+    return exp(x) + cos(x)
+
+
+# lru cached function
+@lru_cache()
+def f_lrucached(x):
+    return x
+
+
+class TestBasic(object):
+
+    def run_check_by_name(self, name, smoothness=0, **kwargs):
+        a = .5
+        b = sqrt(3)
+        xtol = 4*np.finfo(float).eps
+        rtol = 4*np.finfo(float).eps
+        for function, fname in zip(tstutils_functions, tstutils_fstrings):
+            if smoothness > 0 and fname in ['f4', 'f5', 'f6']:
+                continue
+            r = root_scalar(function, method=name, bracket=[a, b], x0=a,
+                            xtol=xtol, rtol=rtol, **kwargs)
+            zero = r.root
+            assert_(r.converged)
+            assert_allclose(zero, 1.0, atol=xtol, rtol=rtol,
+                            err_msg='method %s, function %s' % (name, fname))
+
+    def run_check(self, method, name):
+        a = .5
+        b = sqrt(3)
+        xtol = 4 * _FLOAT_EPS
+        rtol = 4 * _FLOAT_EPS
+        for function, fname in zip(tstutils_functions, tstutils_fstrings):
+            zero, r = method(function, a, b, xtol=xtol, rtol=rtol,
+                             full_output=True)
+            assert_(r.converged)
+            assert_allclose(zero, 1.0, atol=xtol, rtol=rtol,
+                            err_msg='method %s, function %s' % (name, fname))
+
+    def run_check_lru_cached(self, method, name):
+        # check that https://github.com/scipy/scipy/issues/10846 is fixed
+        a = -1
+        b = 1
+        zero, r = method(f_lrucached, a, b, full_output=True)
+        assert_(r.converged)
+        assert_allclose(zero, 0,
+                        err_msg='method %s, function %s' % (name, 'f_lrucached'))
+
+    def _run_one_test(self, tc, method, sig_args_keys=None,
+                      sig_kwargs_keys=None, **kwargs):
+        method_args = []
+        for k in sig_args_keys or []:
+            if k not in tc:
+                # If a,b not present use x0, x1. Similarly for f and func
+                k = {'a': 'x0', 'b': 'x1', 'func': 'f'}.get(k, k)
+            method_args.append(tc[k])
+
+        method_kwargs = dict(**kwargs)
+        method_kwargs.update({'full_output': True, 'disp': False})
+        for k in sig_kwargs_keys or []:
+            method_kwargs[k] = tc[k]
+
+        root = tc.get('root')
+        func_args = tc.get('args', ())
+
+        try:
+            r, rr = method(*method_args, args=func_args, **method_kwargs)
+            return root, rr, tc
+        except Exception:
+            return root, zeros.RootResults(nan, -1, -1, zeros._EVALUEERR), tc
+
+    def run_tests(self, tests, method, name,
+                  xtol=4 * _FLOAT_EPS, rtol=4 * _FLOAT_EPS,
+                  known_fail=None, **kwargs):
+        r"""Run test-cases using the specified method and the supplied signature.
+
+        Extract the arguments for the method call from the test case
+        dictionary using the supplied keys for the method's signature."""
+        # The methods have one of two base signatures:
+        # (f, a, b, **kwargs)  # newton
+        # (func, x0, **kwargs)  # bisect/brentq/...
+        sig = _getfullargspec(method)  # FullArgSpec with args, varargs, varkw, defaults, ...
+        assert_(not sig.kwonlyargs)
+        nDefaults = len(sig.defaults)
+        nRequired = len(sig.args) - nDefaults
+        sig_args_keys = sig.args[:nRequired]
+        sig_kwargs_keys = []
+        if name in ['secant', 'newton', 'halley']:
+            if name in ['newton', 'halley']:
+                sig_kwargs_keys.append('fprime')
+                if name in ['halley']:
+                    sig_kwargs_keys.append('fprime2')
+            kwargs['tol'] = xtol
+        else:
+            kwargs['xtol'] = xtol
+            kwargs['rtol'] = rtol
+
+        results = [list(self._run_one_test(
+            tc, method, sig_args_keys=sig_args_keys,
+            sig_kwargs_keys=sig_kwargs_keys, **kwargs)) for tc in tests]
+        # results= [[true root, full output, tc], ...]
+
+        known_fail = known_fail or []
+        notcvgd = [elt for elt in results if not elt[1].converged]
+        notcvgd = [elt for elt in notcvgd if elt[-1]['ID'] not in known_fail]
+        notcvged_IDS = [elt[-1]['ID'] for elt in notcvgd]
+        assert_equal([len(notcvged_IDS), notcvged_IDS], [0, []])
+
+        # The usable xtol and rtol depend on the test
+        tols = {'xtol': 4 * _FLOAT_EPS, 'rtol': 4 * _FLOAT_EPS}
+        tols.update(**kwargs)
+        rtol = tols['rtol']
+        atol = tols.get('tol', tols['xtol'])
+
+        cvgd = [elt for elt in results if elt[1].converged]
+        approx = [elt[1].root for elt in cvgd]
+        correct = [elt[0] for elt in cvgd]
+        notclose = [[a] + elt for a, c, elt in zip(approx, correct, cvgd) if
+                    not isclose(a, c, rtol=rtol, atol=atol)
+                    and elt[-1]['ID'] not in known_fail]
+        # Evaluate the function and see if is 0 at the purported root
+        fvs = [tc['f'](aroot, *(tc['args'])) for aroot, c, fullout, tc in notclose]
+        notclose = [[fv] + elt for fv, elt in zip(fvs, notclose) if fv != 0]
+        assert_equal([notclose, len(notclose)], [[], 0])
+
+    def run_collection(self, collection, method, name, smoothness=None,
+                       known_fail=None,
+                       xtol=4 * _FLOAT_EPS, rtol=4 * _FLOAT_EPS,
+                       **kwargs):
+        r"""Run a collection of tests using the specified method.
+
+        The name is used to determine some optional arguments."""
+        tests = get_tests(collection, smoothness=smoothness)
+        self.run_tests(tests, method, name, xtol=xtol, rtol=rtol,
+                       known_fail=known_fail, **kwargs)
+
+    def test_bisect(self):
+        self.run_check(zeros.bisect, 'bisect')
+        self.run_check_lru_cached(zeros.bisect, 'bisect')
+        self.run_check_by_name('bisect')
+        self.run_collection('aps', zeros.bisect, 'bisect', smoothness=1)
+
+    def test_ridder(self):
+        self.run_check(zeros.ridder, 'ridder')
+        self.run_check_lru_cached(zeros.ridder, 'ridder')
+        self.run_check_by_name('ridder')
+        self.run_collection('aps', zeros.ridder, 'ridder', smoothness=1)
+
+    def test_brentq(self):
+        self.run_check(zeros.brentq, 'brentq')
+        self.run_check_lru_cached(zeros.brentq, 'brentq')
+        self.run_check_by_name('brentq')
+        # Brentq/h needs a lower tolerance to be specified
+        self.run_collection('aps', zeros.brentq, 'brentq', smoothness=1,
+                            xtol=1e-14, rtol=1e-14)
+
+    def test_brenth(self):
+        self.run_check(zeros.brenth, 'brenth')
+        self.run_check_lru_cached(zeros.brenth, 'brenth')
+        self.run_check_by_name('brenth')
+        self.run_collection('aps', zeros.brenth, 'brenth', smoothness=1,
+                            xtol=1e-14, rtol=1e-14)
+
+    def test_toms748(self):
+        self.run_check(zeros.toms748, 'toms748')
+        self.run_check_lru_cached(zeros.toms748, 'toms748')
+        self.run_check_by_name('toms748')
+        self.run_collection('aps', zeros.toms748, 'toms748', smoothness=1)
+
+    def test_newton_collections(self):
+        known_fail = ['aps.13.00']
+        known_fail += ['aps.12.05', 'aps.12.17']  # fails under Windows Py27
+        for collection in ['aps', 'complex']:
+            self.run_collection(collection, zeros.newton, 'newton',
+                                smoothness=2, known_fail=known_fail)
+
+    def test_halley_collections(self):
+        known_fail = ['aps.12.06', 'aps.12.07', 'aps.12.08', 'aps.12.09',
+                      'aps.12.10', 'aps.12.11', 'aps.12.12', 'aps.12.13',
+                      'aps.12.14', 'aps.12.15', 'aps.12.16', 'aps.12.17',
+                      'aps.12.18', 'aps.13.00']
+        for collection in ['aps', 'complex']:
+            self.run_collection(collection, zeros.newton, 'halley',
+                                smoothness=2, known_fail=known_fail)
+
+    @staticmethod
+    def f1(x):
+        return x**2 - 2*x - 1  # == (x-1)**2 - 2
+
+    @staticmethod
+    def f1_1(x):
+        return 2*x - 2
+
+    @staticmethod
+    def f1_2(x):
+        return 2.0 + 0*x
+
+    @staticmethod
+    def f2(x):
+        return exp(x) - cos(x)
+
+    @staticmethod
+    def f2_1(x):
+        return exp(x) + sin(x)
+
+    @staticmethod
+    def f2_2(x):
+        return exp(x) + cos(x)
+
+    def test_newton(self):
+        for f, f_1, f_2 in [(self.f1, self.f1_1, self.f1_2),
+                            (self.f2, self.f2_1, self.f2_2)]:
+            x = zeros.newton(f, 3, tol=1e-6)
+            assert_allclose(f(x), 0, atol=1e-6)
+            x = zeros.newton(f, 3, x1=5, tol=1e-6)  # secant, x0 and x1
+            assert_allclose(f(x), 0, atol=1e-6)
+            x = zeros.newton(f, 3, fprime=f_1, tol=1e-6)   # newton
+            assert_allclose(f(x), 0, atol=1e-6)
+            x = zeros.newton(f, 3, fprime=f_1, fprime2=f_2, tol=1e-6)  # halley
+            assert_allclose(f(x), 0, atol=1e-6)
+
+    def test_newton_by_name(self):
+        r"""Invoke newton through root_scalar()"""
+        for f, f_1, f_2 in [(f1, f1_1, f1_2), (f2, f2_1, f2_2)]:
+            r = root_scalar(f, method='newton', x0=3, fprime=f_1, xtol=1e-6)
+            assert_allclose(f(r.root), 0, atol=1e-6)
+
+    def test_secant_by_name(self):
+        r"""Invoke secant through root_scalar()"""
+        for f, f_1, f_2 in [(f1, f1_1, f1_2), (f2, f2_1, f2_2)]:
+            r = root_scalar(f, method='secant', x0=3, x1=2, xtol=1e-6)
+            assert_allclose(f(r.root), 0, atol=1e-6)
+            r = root_scalar(f, method='secant', x0=3, x1=5, xtol=1e-6)
+            assert_allclose(f(r.root), 0, atol=1e-6)
+
+    def test_halley_by_name(self):
+        r"""Invoke halley through root_scalar()"""
+        for f, f_1, f_2 in [(f1, f1_1, f1_2), (f2, f2_1, f2_2)]:
+            r = root_scalar(f, method='halley', x0=3,
+                            fprime=f_1, fprime2=f_2, xtol=1e-6)
+            assert_allclose(f(r.root), 0, atol=1e-6)
+
+    def test_root_scalar_fail(self):
+        with pytest.raises(ValueError):
+            root_scalar(f1, method='secant', x0=3, xtol=1e-6)  # no x1
+        with pytest.raises(ValueError):
+            root_scalar(f1, method='newton', x0=3, xtol=1e-6)  # no fprime
+        with pytest.raises(ValueError):
+            root_scalar(f1, method='halley', fprime=f1_1, x0=3, xtol=1e-6)  # no fprime2
+        with pytest.raises(ValueError):
+            root_scalar(f1, method='halley', fprime2=f1_2, x0=3, xtol=1e-6)  # no fprime
+
+    def test_array_newton(self):
+        """test newton with array"""
+
+        def f1(x, *a):
+            b = a[0] + x * a[3]
+            return a[1] - a[2] * (np.exp(b / a[5]) - 1.0) - b / a[4] - x
+
+        def f1_1(x, *a):
+            b = a[3] / a[5]
+            return -a[2] * np.exp(a[0] / a[5] + x * b) * b - a[3] / a[4] - 1
+
+        def f1_2(x, *a):
+            b = a[3] / a[5]
+            return -a[2] * np.exp(a[0] / a[5] + x * b) * b**2
+
+        a0 = np.array([
+            5.32725221, 5.48673747, 5.49539973,
+            5.36387202, 4.80237316, 1.43764452,
+            5.23063958, 5.46094772, 5.50512718,
+            5.42046290
+        ])
+        a1 = (np.sin(range(10)) + 1.0) * 7.0
+        args = (a0, a1, 1e-09, 0.004, 10, 0.27456)
+        x0 = [7.0] * 10
+        x = zeros.newton(f1, x0, f1_1, args)
+        x_expected = (
+            6.17264965, 11.7702805, 12.2219954,
+            7.11017681, 1.18151293, 0.143707955,
+            4.31928228, 10.5419107, 12.7552490,
+            8.91225749
+        )
+        assert_allclose(x, x_expected)
+        # test halley's
+        x = zeros.newton(f1, x0, f1_1, args, fprime2=f1_2)
+        assert_allclose(x, x_expected)
+        # test secant
+        x = zeros.newton(f1, x0, args=args)
+        assert_allclose(x, x_expected)
+
+    def test_array_newton_complex(self):
+        def f(x):
+            return x + 1+1j
+
+        def fprime(x):
+            return 1.0
+
+        t = np.full(4, 1j)
+        x = zeros.newton(f, t, fprime=fprime)
+        assert_allclose(f(x), 0.)
+
+        # should work even if x0 is not complex
+        t = np.ones(4)
+        x = zeros.newton(f, t, fprime=fprime)
+        assert_allclose(f(x), 0.)
+
+        x = zeros.newton(f, t)
+        assert_allclose(f(x), 0.)
+
+    def test_array_secant_active_zero_der(self):
+        """test secant doesn't continue to iterate zero derivatives"""
+        x = zeros.newton(lambda x, *a: x*x - a[0], x0=[4.123, 5],
+                         args=[np.array([17, 25])])
+        assert_allclose(x, (4.123105625617661, 5.0))
+
+    def test_array_newton_integers(self):
+        # test secant with float
+        x = zeros.newton(lambda y, z: z - y ** 2, [4.0] * 2,
+                         args=([15.0, 17.0],))
+        assert_allclose(x, (3.872983346207417, 4.123105625617661))
+        # test integer becomes float
+        x = zeros.newton(lambda y, z: z - y ** 2, [4] * 2, args=([15, 17],))
+        assert_allclose(x, (3.872983346207417, 4.123105625617661))
+
+    def test_array_newton_zero_der_failures(self):
+        # test derivative zero warning
+        assert_warns(RuntimeWarning, zeros.newton,
+                     lambda y: y**2 - 2, [0., 0.], lambda y: 2 * y)
+        # test failures and zero_der
+        with pytest.warns(RuntimeWarning):
+            results = zeros.newton(lambda y: y**2 - 2, [0., 0.],
+                                   lambda y: 2*y, full_output=True)
+            assert_allclose(results.root, 0)
+            assert results.zero_der.all()
+            assert not results.converged.any()
+
+    def test_newton_combined(self):
+        f1 = lambda x: x**2 - 2*x - 1
+        f1_1 = lambda x: 2*x - 2
+        f1_2 = lambda x: 2.0 + 0*x
+
+        def f1_and_p_and_pp(x):
+            return x**2 - 2*x-1, 2*x-2, 2.0
+
+        sol0 = root_scalar(f1, method='newton', x0=3, fprime=f1_1)
+        sol = root_scalar(f1_and_p_and_pp, method='newton', x0=3, fprime=True)
+        assert_allclose(sol0.root, sol.root, atol=1e-8)
+        assert_equal(2*sol.function_calls, sol0.function_calls)
+
+        sol0 = root_scalar(f1, method='halley', x0=3, fprime=f1_1, fprime2=f1_2)
+        sol = root_scalar(f1_and_p_and_pp, method='halley', x0=3, fprime2=True)
+        assert_allclose(sol0.root, sol.root, atol=1e-8)
+        assert_equal(3*sol.function_calls, sol0.function_calls)
+
+    def test_newton_full_output(self):
+        # Test the full_output capability, both when converging and not.
+        # Use simple polynomials, to avoid hitting platform dependencies
+        # (e.g., exp & trig) in number of iterations
+
+        x0 = 3
+        expected_counts = [(6, 7), (5, 10), (3, 9)]
+
+        for derivs in range(3):
+            kwargs = {'tol': 1e-6, 'full_output': True, }
+            for k, v in [['fprime', self.f1_1], ['fprime2', self.f1_2]][:derivs]:
+                kwargs[k] = v
+
+            x, r = zeros.newton(self.f1, x0, disp=False, **kwargs)
+            assert_(r.converged)
+            assert_equal(x, r.root)
+            assert_equal((r.iterations, r.function_calls), expected_counts[derivs])
+            if derivs == 0:
+                assert(r.function_calls <= r.iterations + 1)
+            else:
+                assert_equal(r.function_calls, (derivs + 1) * r.iterations)
+
+            # Now repeat, allowing one fewer iteration to force convergence failure
+            iters = r.iterations - 1
+            x, r = zeros.newton(self.f1, x0, maxiter=iters, disp=False, **kwargs)
+            assert_(not r.converged)
+            assert_equal(x, r.root)
+            assert_equal(r.iterations, iters)
+
+            if derivs == 1:
+                # Check that the correct Exception is raised and
+                # validate the start of the message.
+                with pytest.raises(
+                    RuntimeError,
+                    match='Failed to converge after %d iterations, value is .*' % (iters)):
+                    x, r = zeros.newton(self.f1, x0, maxiter=iters, disp=True, **kwargs)
+
+    def test_deriv_zero_warning(self):
+        func = lambda x: x**2 - 2.0
+        dfunc = lambda x: 2*x
+        assert_warns(RuntimeWarning, zeros.newton, func, 0.0, dfunc, disp=False)
+        with pytest.raises(RuntimeError, match='Derivative was zero'):
+            zeros.newton(func, 0.0, dfunc)
+
+    def test_newton_does_not_modify_x0(self):
+        # https://github.com/scipy/scipy/issues/9964
+        x0 = np.array([0.1, 3])
+        x0_copy = x0.copy()  # Copy to test for equality.
+        newton(np.sin, x0, np.cos)
+        assert_array_equal(x0, x0_copy)
+
+    def test_maxiter_int_check(self):
+        for method in [zeros.bisect, zeros.newton, zeros.ridder, zeros.brentq,
+                       zeros.brenth, zeros.toms748]:
+            with pytest.raises(TypeError,
+                    match="'float' object cannot be interpreted as an integer"):
+                method(f1, 0.0, 1.0, maxiter=72.45)
+
+
+def test_gh_5555():
+    root = 0.1
+
+    def f(x):
+        return x - root
+
+    methods = [zeros.bisect, zeros.ridder]
+    xtol = rtol = TOL
+    for method in methods:
+        res = method(f, -1e8, 1e7, xtol=xtol, rtol=rtol)
+        assert_allclose(root, res, atol=xtol, rtol=rtol,
+                        err_msg='method %s' % method.__name__)
+
+
+def test_gh_5557():
+    # Show that without the changes in 5557 brentq and brenth might
+    # only achieve a tolerance of 2*(xtol + rtol*|res|).
+
+    # f linearly interpolates (0, -0.1), (0.5, -0.1), and (1,
+    # 0.4). The important parts are that |f(0)| < |f(1)| (so that
+    # brent takes 0 as the initial guess), |f(0)| < atol (so that
+    # brent accepts 0 as the root), and that the exact root of f lies
+    # more than atol away from 0 (so that brent doesn't achieve the
+    # desired tolerance).
+    def f(x):
+        if x < 0.5:
+            return -0.1
+        else:
+            return x - 0.6
+
+    atol = 0.51
+    rtol = 4 * _FLOAT_EPS
+    methods = [zeros.brentq, zeros.brenth]
+    for method in methods:
+        res = method(f, 0, 1, xtol=atol, rtol=rtol)
+        assert_allclose(0.6, res, atol=atol, rtol=rtol)
+
+
+class TestRootResults:
+    def test_repr(self):
+        r = zeros.RootResults(root=1.0,
+                              iterations=44,
+                              function_calls=46,
+                              flag=0)
+        expected_repr = ("      converged: True\n           flag: 'converged'"
+                         "\n function_calls: 46\n     iterations: 44\n"
+                         "           root: 1.0")
+        assert_equal(repr(r), expected_repr)
+
+
+def test_complex_halley():
+    """Test Halley's works with complex roots"""
+    def f(x, *a):
+        return a[0] * x**2 + a[1] * x + a[2]
+
+    def f_1(x, *a):
+        return 2 * a[0] * x + a[1]
+
+    def f_2(x, *a):
+        retval = 2 * a[0]
+        try:
+            size = len(x)
+        except TypeError:
+            return retval
+        else:
+            return [retval] * size
+
+    z = complex(1.0, 2.0)
+    coeffs = (2.0, 3.0, 4.0)
+    y = zeros.newton(f, z, args=coeffs, fprime=f_1, fprime2=f_2, tol=1e-6)
+    # (-0.75000000000000078+1.1989578808281789j)
+    assert_allclose(f(y, *coeffs), 0, atol=1e-6)
+    z = [z] * 10
+    coeffs = (2.0, 3.0, 4.0)
+    y = zeros.newton(f, z, args=coeffs, fprime=f_1, fprime2=f_2, tol=1e-6)
+    assert_allclose(f(y, *coeffs), 0, atol=1e-6)
+
+
+def test_zero_der_nz_dp():
+    """Test secant method with a non-zero dp, but an infinite newton step"""
+    # pick a symmetrical functions and choose a point on the side that with dx
+    # makes a secant that is a flat line with zero slope, EG: f = (x - 100)**2,
+    # which has a root at x = 100 and is symmetrical around the line x = 100
+    # we have to pick a really big number so that it is consistently true
+    # now find a point on each side so that the secant has a zero slope
+    dx = np.finfo(float).eps ** 0.33
+    # 100 - p0 = p1 - 100 = p0 * (1 + dx) + dx - 100
+    # -> 200 = p0 * (2 + dx) + dx
+    p0 = (200.0 - dx) / (2.0 + dx)
+    with suppress_warnings() as sup:
+        sup.filter(RuntimeWarning, "RMS of")
+        x = zeros.newton(lambda y: (y - 100.0)**2, x0=[p0] * 10)
+    assert_allclose(x, [100] * 10)
+    # test scalar cases too
+    p0 = (2.0 - 1e-4) / (2.0 + 1e-4)
+    with suppress_warnings() as sup:
+        sup.filter(RuntimeWarning, "Tolerance of")
+        x = zeros.newton(lambda y: (y - 1.0) ** 2, x0=p0, disp=False)
+    assert_allclose(x, 1)
+    with pytest.raises(RuntimeError, match='Tolerance of'):
+        x = zeros.newton(lambda y: (y - 1.0) ** 2, x0=p0, disp=True)
+    p0 = (-2.0 + 1e-4) / (2.0 + 1e-4)
+    with suppress_warnings() as sup:
+        sup.filter(RuntimeWarning, "Tolerance of")
+        x = zeros.newton(lambda y: (y + 1.0) ** 2, x0=p0, disp=False)
+    assert_allclose(x, -1)
+    with pytest.raises(RuntimeError, match='Tolerance of'):
+        x = zeros.newton(lambda y: (y + 1.0) ** 2, x0=p0, disp=True)
+
+
+def test_array_newton_failures():
+    """Test that array newton fails as expected"""
+    # p = 0.68  # [MPa]
+    # dp = -0.068 * 1e6  # [Pa]
+    # T = 323  # [K]
+    diameter = 0.10  # [m]
+    # L = 100  # [m]
+    roughness = 0.00015  # [m]
+    rho = 988.1  # [kg/m**3]
+    mu = 5.4790e-04  # [Pa*s]
+    u = 2.488  # [m/s]
+    reynolds_number = rho * u * diameter / mu  # Reynolds number
+
+    def colebrook_eqn(darcy_friction, re, dia):
+        return (1 / np.sqrt(darcy_friction) +
+                2 * np.log10(roughness / 3.7 / dia +
+                             2.51 / re / np.sqrt(darcy_friction)))
+
+    # only some failures
+    with pytest.warns(RuntimeWarning):
+        result = zeros.newton(
+            colebrook_eqn, x0=[0.01, 0.2, 0.02223, 0.3], maxiter=2,
+            args=[reynolds_number, diameter], full_output=True
+        )
+        assert not result.converged.all()
+    # they all fail
+    with pytest.raises(RuntimeError):
+        result = zeros.newton(
+            colebrook_eqn, x0=[0.01] * 2, maxiter=2,
+            args=[reynolds_number, diameter], full_output=True
+        )
+
+
+# this test should **not** raise a RuntimeWarning
+def test_gh8904_zeroder_at_root_fails():
+    """Test that Newton or Halley don't warn if zero derivative at root"""
+
+    # a function that has a zero derivative at it's root
+    def f_zeroder_root(x):
+        return x**3 - x**2
+
+    # should work with secant
+    r = zeros.newton(f_zeroder_root, x0=0)
+    assert_allclose(r, 0, atol=zeros._xtol, rtol=zeros._rtol)
+    # test again with array
+    r = zeros.newton(f_zeroder_root, x0=[0]*10)
+    assert_allclose(r, 0, atol=zeros._xtol, rtol=zeros._rtol)
+
+    # 1st derivative
+    def fder(x):
+        return 3 * x**2 - 2 * x
+
+    # 2nd derivative
+    def fder2(x):
+        return 6*x - 2
+
+    # should work with newton and halley
+    r = zeros.newton(f_zeroder_root, x0=0, fprime=fder)
+    assert_allclose(r, 0, atol=zeros._xtol, rtol=zeros._rtol)
+    r = zeros.newton(f_zeroder_root, x0=0, fprime=fder,
+                     fprime2=fder2)
+    assert_allclose(r, 0, atol=zeros._xtol, rtol=zeros._rtol)
+    # test again with array
+    r = zeros.newton(f_zeroder_root, x0=[0]*10, fprime=fder)
+    assert_allclose(r, 0, atol=zeros._xtol, rtol=zeros._rtol)
+    r = zeros.newton(f_zeroder_root, x0=[0]*10, fprime=fder,
+                     fprime2=fder2)
+    assert_allclose(r, 0, atol=zeros._xtol, rtol=zeros._rtol)
+
+    # also test that if a root is found we do not raise RuntimeWarning even if
+    # the derivative is zero, EG: at x = 0.5, then fval = -0.125 and
+    # fder = -0.25 so the next guess is 0.5 - (-0.125/-0.5) = 0 which is the
+    # root, but if the solver continued with that guess, then it will calculate
+    # a zero derivative, so it should return the root w/o RuntimeWarning
+    r = zeros.newton(f_zeroder_root, x0=0.5, fprime=fder)
+    assert_allclose(r, 0, atol=zeros._xtol, rtol=zeros._rtol)
+    # test again with array
+    r = zeros.newton(f_zeroder_root, x0=[0.5]*10, fprime=fder)
+    assert_allclose(r, 0, atol=zeros._xtol, rtol=zeros._rtol)
+    # doesn't apply to halley
+
+
+def test_gh_8881():
+    r"""Test that Halley's method realizes that the 2nd order adjustment
+    is too big and drops off to the 1st order adjustment."""
+    n = 9
+
+    def f(x):
+        return power(x, 1.0/n) - power(n, 1.0/n)
+
+    def fp(x):
+        return power(x, (1.0-n)/n)/n
+
+    def fpp(x):
+        return power(x, (1.0-2*n)/n) * (1.0/n) * (1.0-n)/n
+
+    x0 = 0.1
+    # The root is at x=9.
+    # The function has positive slope, x0 < root.
+    # Newton succeeds in 8 iterations
+    rt, r = newton(f, x0, fprime=fp, full_output=True)
+    assert(r.converged)
+    # Before the Issue 8881/PR 8882, halley would send x in the wrong direction.
+    # Check that it now succeeds.
+    rt, r = newton(f, x0, fprime=fp, fprime2=fpp, full_output=True)
+    assert(r.converged)
+
+
+def test_gh_9608_preserve_array_shape():
+    """
+    Test that shape is preserved for array inputs even if fprime or fprime2 is
+    scalar
+    """
+    def f(x):
+        return x**2
+
+    def fp(x):
+        return 2 * x
+
+    def fpp(x):
+        return 2
+
+    x0 = np.array([-2], dtype=np.float32)
+    rt, r = newton(f, x0, fprime=fp, fprime2=fpp, full_output=True)
+    assert(r.converged)
+
+    x0_array = np.array([-2, -3], dtype=np.float32)
+    # This next invocation should fail
+    with pytest.raises(IndexError):
+        result = zeros.newton(
+            f, x0_array, fprime=fp, fprime2=fpp, full_output=True
+        )
+
+    def fpp_array(x):
+        return np.full(np.shape(x), 2, dtype=np.float32)
+
+    result = zeros.newton(
+        f, x0_array, fprime=fp, fprime2=fpp_array, full_output=True
+    )
+    assert result.converged.all()
+
+
+@pytest.mark.parametrize(
+    "maximum_iterations,flag_expected",
+    [(10, zeros.CONVERR), (100, zeros.CONVERGED)])
+def test_gh9254_flag_if_maxiter_exceeded(maximum_iterations, flag_expected):
+    """
+    Test that if the maximum iterations is exceeded that the flag is not
+    converged.
+    """
+    result = zeros.brentq(
+        lambda x: ((1.2*x - 2.3)*x + 3.4)*x - 4.5,
+        -30, 30, (), 1e-6, 1e-6, maximum_iterations,
+        full_output=True, disp=False)
+    assert result[1].flag == flag_expected
+    if flag_expected == zeros.CONVERR:
+        # didn't converge because exceeded maximum iterations
+        assert result[1].iterations == maximum_iterations
+    elif flag_expected == zeros.CONVERGED:
+        # converged before maximum iterations
+        assert result[1].iterations < maximum_iterations
+
+
+def test_gh9551_raise_error_if_disp_true():
+    """Test that if disp is true then zero derivative raises RuntimeError"""
+
+    def f(x):
+        return x*x + 1
+
+    def f_p(x):
+        return 2*x
+
+    assert_warns(RuntimeWarning, zeros.newton, f, 1.0, f_p, disp=False)
+    with pytest.raises(
+            RuntimeError,
+            match=r'^Derivative was zero\. Failed to converge after \d+ iterations, value is [+-]?\d*\.\d+\.$'):
+        zeros.newton(f, 1.0, f_p)
+    root = zeros.newton(f, complex(10.0, 10.0), f_p)
+    assert_allclose(root, complex(0.0, 1.0))