Uploaded Test files

This commit is contained in:
Batuhan Berk Başoğlu 2020-11-12 11:05:57 -05:00
parent f584ad9d97
commit 2e81cb7d99
16627 changed files with 2065359 additions and 102444 deletions

View file

@ -0,0 +1,299 @@
import pytest
import numpy as np
from numpy.testing import assert_allclose
from numpy.testing import assert_array_equal
from sklearn.compose import make_column_transformer
from sklearn.datasets import make_classification
from sklearn.exceptions import NotFittedError
from sklearn.linear_model import LogisticRegression
from sklearn.pipeline import make_pipeline
from sklearn.preprocessing import StandardScaler
from sklearn.svm import SVC, SVR
from sklearn.metrics import confusion_matrix
from sklearn.metrics import plot_confusion_matrix
from sklearn.metrics import ConfusionMatrixDisplay
# TODO: Remove when https://github.com/numpy/numpy/issues/14397 is resolved
pytestmark = pytest.mark.filterwarnings(
"ignore:In future, it will be an error for 'np.bool_':DeprecationWarning:"
"matplotlib.*")
@pytest.fixture(scope="module")
def n_classes():
return 5
@pytest.fixture(scope="module")
def data(n_classes):
X, y = make_classification(n_samples=100, n_informative=5,
n_classes=n_classes, random_state=0)
return X, y
@pytest.fixture(scope="module")
def fitted_clf(data):
return SVC(kernel='linear', C=0.01).fit(*data)
@pytest.fixture(scope="module")
def y_pred(data, fitted_clf):
X, _ = data
return fitted_clf.predict(X)
def test_error_on_regressor(pyplot, data):
X, y = data
est = SVR().fit(X, y)
msg = "plot_confusion_matrix only supports classifiers"
with pytest.raises(ValueError, match=msg):
plot_confusion_matrix(est, X, y)
def test_error_on_invalid_option(pyplot, fitted_clf, data):
X, y = data
msg = (r"normalize must be one of \{'true', 'pred', 'all', "
r"None\}")
with pytest.raises(ValueError, match=msg):
plot_confusion_matrix(fitted_clf, X, y, normalize='invalid')
@pytest.mark.parametrize("with_labels", [True, False])
@pytest.mark.parametrize("with_display_labels", [True, False])
def test_plot_confusion_matrix_custom_labels(pyplot, data, y_pred, fitted_clf,
n_classes, with_labels,
with_display_labels):
X, y = data
ax = pyplot.gca()
labels = [2, 1, 0, 3, 4] if with_labels else None
display_labels = ['b', 'd', 'a', 'e', 'f'] if with_display_labels else None
cm = confusion_matrix(y, y_pred, labels=labels)
disp = plot_confusion_matrix(fitted_clf, X, y,
ax=ax, display_labels=display_labels,
labels=labels)
assert_allclose(disp.confusion_matrix, cm)
if with_display_labels:
expected_display_labels = display_labels
elif with_labels:
expected_display_labels = labels
else:
expected_display_labels = list(range(n_classes))
expected_display_labels_str = [str(name)
for name in expected_display_labels]
x_ticks = [tick.get_text() for tick in disp.ax_.get_xticklabels()]
y_ticks = [tick.get_text() for tick in disp.ax_.get_yticklabels()]
assert_array_equal(disp.display_labels, expected_display_labels)
assert_array_equal(x_ticks, expected_display_labels_str)
assert_array_equal(y_ticks, expected_display_labels_str)
@pytest.mark.parametrize("normalize", ['true', 'pred', 'all', None])
@pytest.mark.parametrize("include_values", [True, False])
def test_plot_confusion_matrix(pyplot, data, y_pred, n_classes, fitted_clf,
normalize, include_values):
X, y = data
ax = pyplot.gca()
cmap = 'plasma'
cm = confusion_matrix(y, y_pred)
disp = plot_confusion_matrix(fitted_clf, X, y,
normalize=normalize,
cmap=cmap, ax=ax,
include_values=include_values)
assert disp.ax_ == ax
if normalize == 'true':
cm = cm / cm.sum(axis=1, keepdims=True)
elif normalize == 'pred':
cm = cm / cm.sum(axis=0, keepdims=True)
elif normalize == 'all':
cm = cm / cm.sum()
assert_allclose(disp.confusion_matrix, cm)
import matplotlib as mpl
assert isinstance(disp.im_, mpl.image.AxesImage)
assert disp.im_.get_cmap().name == cmap
assert isinstance(disp.ax_, pyplot.Axes)
assert isinstance(disp.figure_, pyplot.Figure)
assert disp.ax_.get_ylabel() == "True label"
assert disp.ax_.get_xlabel() == "Predicted label"
x_ticks = [tick.get_text() for tick in disp.ax_.get_xticklabels()]
y_ticks = [tick.get_text() for tick in disp.ax_.get_yticklabels()]
expected_display_labels = list(range(n_classes))
expected_display_labels_str = [str(name)
for name in expected_display_labels]
assert_array_equal(disp.display_labels, expected_display_labels)
assert_array_equal(x_ticks, expected_display_labels_str)
assert_array_equal(y_ticks, expected_display_labels_str)
image_data = disp.im_.get_array().data
assert_allclose(image_data, cm)
if include_values:
assert disp.text_.shape == (n_classes, n_classes)
fmt = '.2g'
expected_text = np.array([format(v, fmt) for v in cm.ravel(order="C")])
text_text = np.array([
t.get_text() for t in disp.text_.ravel(order="C")])
assert_array_equal(expected_text, text_text)
else:
assert disp.text_ is None
def test_confusion_matrix_display(pyplot, data, fitted_clf, y_pred, n_classes):
X, y = data
cm = confusion_matrix(y, y_pred)
disp = plot_confusion_matrix(fitted_clf, X, y, normalize=None,
include_values=True, cmap='viridis',
xticks_rotation=45.0)
assert_allclose(disp.confusion_matrix, cm)
assert disp.text_.shape == (n_classes, n_classes)
rotations = [tick.get_rotation() for tick in disp.ax_.get_xticklabels()]
assert_allclose(rotations, 45.0)
image_data = disp.im_.get_array().data
assert_allclose(image_data, cm)
disp.plot(cmap='plasma')
assert disp.im_.get_cmap().name == 'plasma'
disp.plot(include_values=False)
assert disp.text_ is None
disp.plot(xticks_rotation=90.0)
rotations = [tick.get_rotation() for tick in disp.ax_.get_xticklabels()]
assert_allclose(rotations, 90.0)
disp.plot(values_format='e')
expected_text = np.array([format(v, 'e') for v in cm.ravel(order="C")])
text_text = np.array([
t.get_text() for t in disp.text_.ravel(order="C")])
assert_array_equal(expected_text, text_text)
def test_confusion_matrix_contrast(pyplot):
# make sure text color is appropriate depending on background
cm = np.eye(2) / 2
disp = ConfusionMatrixDisplay(cm, display_labels=[0, 1])
disp.plot(cmap=pyplot.cm.gray)
# diagonal text is black
assert_allclose(disp.text_[0, 0].get_color(), [0.0, 0.0, 0.0, 1.0])
assert_allclose(disp.text_[1, 1].get_color(), [0.0, 0.0, 0.0, 1.0])
# off-diagonal text is white
assert_allclose(disp.text_[0, 1].get_color(), [1.0, 1.0, 1.0, 1.0])
assert_allclose(disp.text_[1, 0].get_color(), [1.0, 1.0, 1.0, 1.0])
disp.plot(cmap=pyplot.cm.gray_r)
# diagonal text is white
assert_allclose(disp.text_[0, 1].get_color(), [0.0, 0.0, 0.0, 1.0])
assert_allclose(disp.text_[1, 0].get_color(), [0.0, 0.0, 0.0, 1.0])
# off-diagonal text is black
assert_allclose(disp.text_[0, 0].get_color(), [1.0, 1.0, 1.0, 1.0])
assert_allclose(disp.text_[1, 1].get_color(), [1.0, 1.0, 1.0, 1.0])
# Regression test for #15920
cm = np.array([[19, 34], [32, 58]])
disp = ConfusionMatrixDisplay(cm, display_labels=[0, 1])
disp.plot(cmap=pyplot.cm.Blues)
min_color = pyplot.cm.Blues(0)
max_color = pyplot.cm.Blues(255)
assert_allclose(disp.text_[0, 0].get_color(), max_color)
assert_allclose(disp.text_[0, 1].get_color(), max_color)
assert_allclose(disp.text_[1, 0].get_color(), max_color)
assert_allclose(disp.text_[1, 1].get_color(), min_color)
@pytest.mark.parametrize(
"clf", [LogisticRegression(),
make_pipeline(StandardScaler(), LogisticRegression()),
make_pipeline(make_column_transformer((StandardScaler(), [0, 1])),
LogisticRegression())])
def test_confusion_matrix_pipeline(pyplot, clf, data, n_classes):
X, y = data
with pytest.raises(NotFittedError):
plot_confusion_matrix(clf, X, y)
clf.fit(X, y)
y_pred = clf.predict(X)
disp = plot_confusion_matrix(clf, X, y)
cm = confusion_matrix(y, y_pred)
assert_allclose(disp.confusion_matrix, cm)
assert disp.text_.shape == (n_classes, n_classes)
@pytest.mark.parametrize("values_format", ['e', 'n'])
def test_confusion_matrix_text_format(pyplot, data, y_pred, n_classes,
fitted_clf, values_format):
# Make sure plot text is formatted with 'values_format'.
X, y = data
cm = confusion_matrix(y, y_pred)
disp = plot_confusion_matrix(fitted_clf, X, y,
include_values=True,
values_format=values_format)
assert disp.text_.shape == (n_classes, n_classes)
expected_text = np.array([format(v, values_format)
for v in cm.ravel()])
text_text = np.array([
t.get_text() for t in disp.text_.ravel()])
assert_array_equal(expected_text, text_text)
def test_confusion_matrix_standard_format(pyplot):
cm = np.array([[10000000, 0], [123456, 12345678]])
plotted_text = ConfusionMatrixDisplay(
cm, display_labels=[False, True]).plot().text_
# Values should be shown as whole numbers 'd',
# except the first number which should be shown as 1e+07 (longer length)
# and the last number will be shown as 1.2e+07 (longer length)
test = [t.get_text() for t in plotted_text.ravel()]
assert test == ['1e+07', '0', '123456', '1.2e+07']
cm = np.array([[0.1, 10], [100, 0.525]])
plotted_text = ConfusionMatrixDisplay(
cm, display_labels=[False, True]).plot().text_
# Values should now formatted as '.2g', since there's a float in
# Values are have two dec places max, (e.g 100 becomes 1e+02)
test = [t.get_text() for t in plotted_text.ravel()]
assert test == ['0.1', '10', '1e+02', '0.53']
@pytest.mark.parametrize("display_labels, expected_labels", [
(None, ["0", "1"]),
(["cat", "dog"], ["cat", "dog"]),
])
def test_default_labels(pyplot, display_labels, expected_labels):
cm = np.array([[10, 0], [12, 120]])
disp = ConfusionMatrixDisplay(cm, display_labels=display_labels).plot()
x_ticks = [tick.get_text() for tick in disp.ax_.get_xticklabels()]
y_ticks = [tick.get_text() for tick in disp.ax_.get_yticklabels()]
assert_array_equal(x_ticks, expected_labels)
assert_array_equal(y_ticks, expected_labels)

View file

@ -0,0 +1,192 @@
import pytest
import numpy as np
from numpy.testing import assert_allclose
from sklearn.base import BaseEstimator, ClassifierMixin
from sklearn.metrics import plot_precision_recall_curve
from sklearn.metrics import PrecisionRecallDisplay
from sklearn.metrics import average_precision_score
from sklearn.metrics import precision_recall_curve
from sklearn.datasets import make_classification
from sklearn.datasets import load_breast_cancer
from sklearn.tree import DecisionTreeClassifier, DecisionTreeRegressor
from sklearn.linear_model import LogisticRegression
from sklearn.exceptions import NotFittedError
from sklearn.pipeline import make_pipeline
from sklearn.preprocessing import StandardScaler
from sklearn.compose import make_column_transformer
# TODO: Remove when https://github.com/numpy/numpy/issues/14397 is resolved
pytestmark = pytest.mark.filterwarnings(
"ignore:In future, it will be an error for 'np.bool_':DeprecationWarning:"
"matplotlib.*")
def test_errors(pyplot):
X, y_multiclass = make_classification(n_classes=3, n_samples=50,
n_informative=3,
random_state=0)
y_binary = y_multiclass == 0
# Unfitted classifer
binary_clf = DecisionTreeClassifier()
with pytest.raises(NotFittedError):
plot_precision_recall_curve(binary_clf, X, y_binary)
binary_clf.fit(X, y_binary)
multi_clf = DecisionTreeClassifier().fit(X, y_multiclass)
# Fitted multiclass classifier with binary data
msg = "DecisionTreeClassifier should be a binary classifier"
with pytest.raises(ValueError, match=msg):
plot_precision_recall_curve(multi_clf, X, y_binary)
reg = DecisionTreeRegressor().fit(X, y_multiclass)
msg = "DecisionTreeRegressor should be a binary classifier"
with pytest.raises(ValueError, match=msg):
plot_precision_recall_curve(reg, X, y_binary)
@pytest.mark.parametrize(
"response_method, msg",
[("predict_proba", "response method predict_proba is not defined in "
"MyClassifier"),
("decision_function", "response method decision_function is not defined "
"in MyClassifier"),
("auto", "response method decision_function or predict_proba is not "
"defined in MyClassifier"),
("bad_method", "response_method must be 'predict_proba', "
"'decision_function' or 'auto'")])
def test_error_bad_response(pyplot, response_method, msg):
X, y = make_classification(n_classes=2, n_samples=50, random_state=0)
class MyClassifier(BaseEstimator, ClassifierMixin):
def fit(self, X, y):
self.fitted_ = True
self.classes_ = [0, 1]
return self
clf = MyClassifier().fit(X, y)
with pytest.raises(ValueError, match=msg):
plot_precision_recall_curve(clf, X, y, response_method=response_method)
@pytest.mark.parametrize("response_method",
["predict_proba", "decision_function"])
@pytest.mark.parametrize("with_sample_weight", [True, False])
def test_plot_precision_recall(pyplot, response_method, with_sample_weight):
X, y = make_classification(n_classes=2, n_samples=50, random_state=0)
lr = LogisticRegression().fit(X, y)
if with_sample_weight:
rng = np.random.RandomState(42)
sample_weight = rng.randint(0, 4, size=X.shape[0])
else:
sample_weight = None
disp = plot_precision_recall_curve(lr, X, y, alpha=0.8,
response_method=response_method,
sample_weight=sample_weight)
y_score = getattr(lr, response_method)(X)
if response_method == 'predict_proba':
y_score = y_score[:, 1]
prec, recall, _ = precision_recall_curve(y, y_score,
sample_weight=sample_weight)
avg_prec = average_precision_score(y, y_score, sample_weight=sample_weight)
assert_allclose(disp.precision, prec)
assert_allclose(disp.recall, recall)
assert disp.average_precision == pytest.approx(avg_prec)
assert disp.estimator_name == "LogisticRegression"
# cannot fail thanks to pyplot fixture
import matplotlib as mpl # noqa
assert isinstance(disp.line_, mpl.lines.Line2D)
assert disp.line_.get_alpha() == 0.8
assert isinstance(disp.ax_, mpl.axes.Axes)
assert isinstance(disp.figure_, mpl.figure.Figure)
expected_label = "LogisticRegression (AP = {:0.2f})".format(avg_prec)
assert disp.line_.get_label() == expected_label
assert disp.ax_.get_xlabel() == "Recall"
assert disp.ax_.get_ylabel() == "Precision"
# draw again with another label
disp.plot(name="MySpecialEstimator")
expected_label = "MySpecialEstimator (AP = {:0.2f})".format(avg_prec)
assert disp.line_.get_label() == expected_label
@pytest.mark.parametrize(
"clf", [make_pipeline(StandardScaler(), LogisticRegression()),
make_pipeline(make_column_transformer((StandardScaler(), [0, 1])),
LogisticRegression())])
def test_precision_recall_curve_pipeline(pyplot, clf):
X, y = make_classification(n_classes=2, n_samples=50, random_state=0)
with pytest.raises(NotFittedError):
plot_precision_recall_curve(clf, X, y)
clf.fit(X, y)
disp = plot_precision_recall_curve(clf, X, y)
assert disp.estimator_name == clf.__class__.__name__
def test_precision_recall_curve_string_labels(pyplot):
# regression test #15738
cancer = load_breast_cancer()
X = cancer.data
y = cancer.target_names[cancer.target]
lr = make_pipeline(StandardScaler(), LogisticRegression())
lr.fit(X, y)
for klass in cancer.target_names:
assert klass in lr.classes_
disp = plot_precision_recall_curve(lr, X, y)
y_pred = lr.predict_proba(X)[:, 1]
avg_prec = average_precision_score(y, y_pred,
pos_label=lr.classes_[1])
assert disp.average_precision == pytest.approx(avg_prec)
assert disp.estimator_name == lr.__class__.__name__
def test_plot_precision_recall_curve_estimator_name_multiple_calls(pyplot):
# non-regression test checking that the `name` used when calling
# `plot_roc_curve` is used as well when calling `disp.plot()`
X, y = make_classification(n_classes=2, n_samples=50, random_state=0)
clf_name = "my hand-crafted name"
clf = LogisticRegression().fit(X, y)
disp = plot_precision_recall_curve(clf, X, y, name=clf_name)
assert disp.estimator_name == clf_name
pyplot.close("all")
disp.plot()
assert clf_name in disp.line_.get_label()
pyplot.close("all")
clf_name = "another_name"
disp.plot(name=clf_name)
assert clf_name in disp.line_.get_label()
@pytest.mark.parametrize(
"average_precision, estimator_name, expected_label",
[
(0.9, None, "AP = 0.90"),
(None, "my_est", "my_est"),
(0.8, "my_est2", "my_est2 (AP = 0.80)"),
]
)
def test_default_labels(pyplot, average_precision, estimator_name,
expected_label):
prec = np.array([1, 0.5, 0])
recall = np.array([0, 0.5, 1])
disp = PrecisionRecallDisplay(prec, recall,
average_precision=average_precision,
estimator_name=estimator_name)
disp.plot()
assert disp.line_.get_label() == expected_label

View file

@ -0,0 +1,170 @@
import pytest
from numpy.testing import assert_allclose
import numpy as np
from sklearn.tree import DecisionTreeClassifier
from sklearn.metrics import plot_roc_curve
from sklearn.metrics import RocCurveDisplay
from sklearn.datasets import load_iris
from sklearn.linear_model import LogisticRegression
from sklearn.metrics import roc_curve, auc
from sklearn.base import ClassifierMixin
from sklearn.exceptions import NotFittedError
from sklearn.pipeline import make_pipeline
from sklearn.preprocessing import StandardScaler
from sklearn.compose import make_column_transformer
# TODO: Remove when https://github.com/numpy/numpy/issues/14397 is resolved
pytestmark = pytest.mark.filterwarnings(
"ignore:In future, it will be an error for 'np.bool_':DeprecationWarning:"
"matplotlib.*")
@pytest.fixture(scope="module")
def data():
return load_iris(return_X_y=True)
@pytest.fixture(scope="module")
def data_binary(data):
X, y = data
return X[y < 2], y[y < 2]
def test_plot_roc_curve_error_non_binary(pyplot, data):
X, y = data
clf = DecisionTreeClassifier()
clf.fit(X, y)
msg = "DecisionTreeClassifier should be a binary classifier"
with pytest.raises(ValueError, match=msg):
plot_roc_curve(clf, X, y)
@pytest.mark.parametrize(
"response_method, msg",
[("predict_proba", "response method predict_proba is not defined in "
"MyClassifier"),
("decision_function", "response method decision_function is not defined "
"in MyClassifier"),
("auto", "response method decision_function or predict_proba is not "
"defined in MyClassifier"),
("bad_method", "response_method must be 'predict_proba', "
"'decision_function' or 'auto'")])
def test_plot_roc_curve_error_no_response(pyplot, data_binary, response_method,
msg):
X, y = data_binary
class MyClassifier(ClassifierMixin):
def fit(self, X, y):
self.classes_ = [0, 1]
return self
clf = MyClassifier().fit(X, y)
with pytest.raises(ValueError, match=msg):
plot_roc_curve(clf, X, y, response_method=response_method)
@pytest.mark.parametrize("response_method",
["predict_proba", "decision_function"])
@pytest.mark.parametrize("with_sample_weight", [True, False])
@pytest.mark.parametrize("drop_intermediate", [True, False])
@pytest.mark.parametrize("with_strings", [True, False])
def test_plot_roc_curve(pyplot, response_method, data_binary,
with_sample_weight, drop_intermediate,
with_strings):
X, y = data_binary
pos_label = None
if with_strings:
y = np.array(["c", "b"])[y]
pos_label = "c"
if with_sample_weight:
rng = np.random.RandomState(42)
sample_weight = rng.randint(1, 4, size=(X.shape[0]))
else:
sample_weight = None
lr = LogisticRegression()
lr.fit(X, y)
viz = plot_roc_curve(lr, X, y, alpha=0.8, sample_weight=sample_weight,
drop_intermediate=drop_intermediate)
y_pred = getattr(lr, response_method)(X)
if y_pred.ndim == 2:
y_pred = y_pred[:, 1]
fpr, tpr, _ = roc_curve(y, y_pred, sample_weight=sample_weight,
drop_intermediate=drop_intermediate,
pos_label=pos_label)
assert_allclose(viz.roc_auc, auc(fpr, tpr))
assert_allclose(viz.fpr, fpr)
assert_allclose(viz.tpr, tpr)
assert viz.estimator_name == "LogisticRegression"
# cannot fail thanks to pyplot fixture
import matplotlib as mpl # noqal
assert isinstance(viz.line_, mpl.lines.Line2D)
assert viz.line_.get_alpha() == 0.8
assert isinstance(viz.ax_, mpl.axes.Axes)
assert isinstance(viz.figure_, mpl.figure.Figure)
expected_label = "LogisticRegression (AUC = {:0.2f})".format(viz.roc_auc)
assert viz.line_.get_label() == expected_label
assert viz.ax_.get_ylabel() == "True Positive Rate"
assert viz.ax_.get_xlabel() == "False Positive Rate"
@pytest.mark.parametrize(
"clf", [LogisticRegression(),
make_pipeline(StandardScaler(), LogisticRegression()),
make_pipeline(make_column_transformer((StandardScaler(), [0, 1])),
LogisticRegression())])
def test_roc_curve_not_fitted_errors(pyplot, data_binary, clf):
X, y = data_binary
with pytest.raises(NotFittedError):
plot_roc_curve(clf, X, y)
clf.fit(X, y)
disp = plot_roc_curve(clf, X, y)
assert clf.__class__.__name__ in disp.line_.get_label()
assert disp.estimator_name == clf.__class__.__name__
def test_plot_roc_curve_estimator_name_multiple_calls(pyplot, data_binary):
# non-regression test checking that the `name` used when calling
# `plot_roc_curve` is used as well when calling `disp.plot()`
X, y = data_binary
clf_name = "my hand-crafted name"
clf = LogisticRegression().fit(X, y)
disp = plot_roc_curve(clf, X, y, name=clf_name)
assert disp.estimator_name == clf_name
pyplot.close("all")
disp.plot()
assert clf_name in disp.line_.get_label()
pyplot.close("all")
clf_name = "another_name"
disp.plot(name=clf_name)
assert clf_name in disp.line_.get_label()
@pytest.mark.parametrize(
"roc_auc, estimator_name, expected_label",
[
(0.9, None, "AUC = 0.90"),
(None, "my_est", "my_est"),
(0.8, "my_est2", "my_est2 (AUC = 0.80)")
]
)
def test_default_labels(pyplot, roc_auc, estimator_name,
expected_label):
fpr = np.array([0, 0.5, 1])
tpr = np.array([0, 0.5, 1])
disp = RocCurveDisplay(fpr=fpr, tpr=tpr, roc_auc=roc_auc,
estimator_name=estimator_name).plot()
assert disp.line_.get_label() == expected_label