354 lines
13 KiB
Python
354 lines
13 KiB
Python
|
import time
|
||
|
|
from datetime import date
|
||
|
|
|
||
|
|
import numpy as np
|
||
|
|
from numpy.testing import (
|
||
|
|
assert_, assert_equal, assert_allclose, assert_raises,
|
||
|
|
)
|
||
|
|
from numpy.lib._iotools import (
|
||
|
|
LineSplitter, NameValidator, StringConverter,
|
||
|
|
has_nested_fields, easy_dtype, flatten_dtype
|
||
|
|
)
|
||
|
|
|
||
|
|
|
||
|
|
class TestLineSplitter:
|
||
|
|
"Tests the LineSplitter class."
|
||
|
|
|
||
|
|
def test_no_delimiter(self):
|
||
|
|
"Test LineSplitter w/o delimiter"
|
||
|
|
strg = " 1 2 3 4 5 # test"
|
||
|
|
test = LineSplitter()(strg)
|
||
|
|
assert_equal(test, ['1', '2', '3', '4', '5'])
|
||
|
|
test = LineSplitter('')(strg)
|
||
|
|
assert_equal(test, ['1', '2', '3', '4', '5'])
|
||
|
|
|
||
|
|
def test_space_delimiter(self):
|
||
|
|
"Test space delimiter"
|
||
|
|
strg = " 1 2 3 4 5 # test"
|
||
|
|
test = LineSplitter(' ')(strg)
|
||
|
|
assert_equal(test, ['1', '2', '3', '4', '', '5'])
|
||
|
|
test = LineSplitter(' ')(strg)
|
||
|
|
assert_equal(test, ['1 2 3 4', '5'])
|
||
|
|
|
||
|
|
def test_tab_delimiter(self):
|
||
|
|
"Test tab delimiter"
|
||
|
|
strg = " 1\t 2\t 3\t 4\t 5 6"
|
||
|
|
test = LineSplitter('\t')(strg)
|
||
|
|
assert_equal(test, ['1', '2', '3', '4', '5 6'])
|
||
|
|
strg = " 1 2\t 3 4\t 5 6"
|
||
|
|
test = LineSplitter('\t')(strg)
|
||
|
|
assert_equal(test, ['1 2', '3 4', '5 6'])
|
||
|
|
|
||
|
|
def test_other_delimiter(self):
|
||
|
|
"Test LineSplitter on delimiter"
|
||
|
|
strg = "1,2,3,4,,5"
|
||
|
|
test = LineSplitter(',')(strg)
|
||
|
|
assert_equal(test, ['1', '2', '3', '4', '', '5'])
|
||
|
|
#
|
||
|
|
strg = " 1,2,3,4,,5 # test"
|
||
|
|
test = LineSplitter(',')(strg)
|
||
|
|
assert_equal(test, ['1', '2', '3', '4', '', '5'])
|
||
|
|
|
||
|
|
# gh-11028 bytes comment/delimiters should get encoded
|
||
|
|
strg = b" 1,2,3,4,,5 % test"
|
||
|
|
test = LineSplitter(delimiter=b',', comments=b'%')(strg)
|
||
|
|
assert_equal(test, ['1', '2', '3', '4', '', '5'])
|
||
|
|
|
||
|
|
def test_constant_fixed_width(self):
|
||
|
|
"Test LineSplitter w/ fixed-width fields"
|
||
|
|
strg = " 1 2 3 4 5 # test"
|
||
|
|
test = LineSplitter(3)(strg)
|
||
|
|
assert_equal(test, ['1', '2', '3', '4', '', '5', ''])
|
||
|
|
#
|
||
|
|
strg = " 1 3 4 5 6# test"
|
||
|
|
test = LineSplitter(20)(strg)
|
||
|
|
assert_equal(test, ['1 3 4 5 6'])
|
||
|
|
#
|
||
|
|
strg = " 1 3 4 5 6# test"
|
||
|
|
test = LineSplitter(30)(strg)
|
||
|
|
assert_equal(test, ['1 3 4 5 6'])
|
||
|
|
|
||
|
|
def test_variable_fixed_width(self):
|
||
|
|
strg = " 1 3 4 5 6# test"
|
||
|
|
test = LineSplitter((3, 6, 6, 3))(strg)
|
||
|
|
assert_equal(test, ['1', '3', '4 5', '6'])
|
||
|
|
#
|
||
|
|
strg = " 1 3 4 5 6# test"
|
||
|
|
test = LineSplitter((6, 6, 9))(strg)
|
||
|
|
assert_equal(test, ['1', '3 4', '5 6'])
|
||
|
|
|
||
|
|
# -----------------------------------------------------------------------------
|
||
|
|
|
||
|
|
|
||
|
|
class TestNameValidator:
|
||
|
|
|
||
|
|
def test_case_sensitivity(self):
|
||
|
|
"Test case sensitivity"
|
||
|
|
names = ['A', 'a', 'b', 'c']
|
||
|
|
test = NameValidator().validate(names)
|
||
|
|
assert_equal(test, ['A', 'a', 'b', 'c'])
|
||
|
|
test = NameValidator(case_sensitive=False).validate(names)
|
||
|
|
assert_equal(test, ['A', 'A_1', 'B', 'C'])
|
||
|
|
test = NameValidator(case_sensitive='upper').validate(names)
|
||
|
|
assert_equal(test, ['A', 'A_1', 'B', 'C'])
|
||
|
|
test = NameValidator(case_sensitive='lower').validate(names)
|
||
|
|
assert_equal(test, ['a', 'a_1', 'b', 'c'])
|
||
|
|
|
||
|
|
# check exceptions
|
||
|
|
assert_raises(ValueError, NameValidator, case_sensitive='foobar')
|
||
|
|
|
||
|
|
def test_excludelist(self):
|
||
|
|
"Test excludelist"
|
||
|
|
names = ['dates', 'data', 'Other Data', 'mask']
|
||
|
|
validator = NameValidator(excludelist=['dates', 'data', 'mask'])
|
||
|
|
test = validator.validate(names)
|
||
|
|
assert_equal(test, ['dates_', 'data_', 'Other_Data', 'mask_'])
|
||
|
|
|
||
|
|
def test_missing_names(self):
|
||
|
|
"Test validate missing names"
|
||
|
|
namelist = ('a', 'b', 'c')
|
||
|
|
validator = NameValidator()
|
||
|
|
assert_equal(validator(namelist), ['a', 'b', 'c'])
|
||
|
|
namelist = ('', 'b', 'c')
|
||
|
|
assert_equal(validator(namelist), ['f0', 'b', 'c'])
|
||
|
|
namelist = ('a', 'b', '')
|
||
|
|
assert_equal(validator(namelist), ['a', 'b', 'f0'])
|
||
|
|
namelist = ('', 'f0', '')
|
||
|
|
assert_equal(validator(namelist), ['f1', 'f0', 'f2'])
|
||
|
|
|
||
|
|
def test_validate_nb_names(self):
|
||
|
|
"Test validate nb names"
|
||
|
|
namelist = ('a', 'b', 'c')
|
||
|
|
validator = NameValidator()
|
||
|
|
assert_equal(validator(namelist, nbfields=1), ('a',))
|
||
|
|
assert_equal(validator(namelist, nbfields=5, defaultfmt="g%i"),
|
||
|
|
['a', 'b', 'c', 'g0', 'g1'])
|
||
|
|
|
||
|
|
def test_validate_wo_names(self):
|
||
|
|
"Test validate no names"
|
||
|
|
namelist = None
|
||
|
|
validator = NameValidator()
|
||
|
|
assert_(validator(namelist) is None)
|
||
|
|
assert_equal(validator(namelist, nbfields=3), ['f0', 'f1', 'f2'])
|
||
|
|
|
||
|
|
# -----------------------------------------------------------------------------
|
||
|
|
|
||
|
|
|
||
|
|
def _bytes_to_date(s):
|
||
|
|
return date(*time.strptime(s, "%Y-%m-%d")[:3])
|
||
|
|
|
||
|
|
|
||
|
|
class TestStringConverter:
|
||
|
|
"Test StringConverter"
|
||
|
|
|
||
|
|
def test_creation(self):
|
||
|
|
"Test creation of a StringConverter"
|
||
|
|
converter = StringConverter(int, -99999)
|
||
|
|
assert_equal(converter._status, 1)
|
||
|
|
assert_equal(converter.default, -99999)
|
||
|
|
|
||
|
|
def test_upgrade(self):
|
||
|
|
"Tests the upgrade method."
|
||
|
|
|
||
|
|
converter = StringConverter()
|
||
|
|
assert_equal(converter._status, 0)
|
||
|
|
|
||
|
|
# test int
|
||
|
|
assert_equal(converter.upgrade('0'), 0)
|
||
|
|
assert_equal(converter._status, 1)
|
||
|
|
|
||
|
|
# On systems where long defaults to 32-bit, the statuses will be
|
||
|
|
# offset by one, so we check for this here.
|
||
|
|
import numpy.core.numeric as nx
|
||
|
|
status_offset = int(nx.dtype(nx.int_).itemsize < nx.dtype(nx.int64).itemsize)
|
||
|
|
|
||
|
|
# test int > 2**32
|
||
|
|
assert_equal(converter.upgrade('17179869184'), 17179869184)
|
||
|
|
assert_equal(converter._status, 1 + status_offset)
|
||
|
|
|
||
|
|
# test float
|
||
|
|
assert_allclose(converter.upgrade('0.'), 0.0)
|
||
|
|
assert_equal(converter._status, 2 + status_offset)
|
||
|
|
|
||
|
|
# test complex
|
||
|
|
assert_equal(converter.upgrade('0j'), complex('0j'))
|
||
|
|
assert_equal(converter._status, 3 + status_offset)
|
||
|
|
|
||
|
|
# test str
|
||
|
|
# note that the longdouble type has been skipped, so the
|
||
|
|
# _status increases by 2. Everything should succeed with
|
||
|
|
# unicode conversion (8).
|
||
|
|
for s in ['a', b'a']:
|
||
|
|
res = converter.upgrade(s)
|
||
|
|
assert_(type(res) is str)
|
||
|
|
assert_equal(res, 'a')
|
||
|
|
assert_equal(converter._status, 8 + status_offset)
|
||
|
|
|
||
|
|
def test_missing(self):
|
||
|
|
"Tests the use of missing values."
|
||
|
|
converter = StringConverter(missing_values=('missing',
|
||
|
|
'missed'))
|
||
|
|
converter.upgrade('0')
|
||
|
|
assert_equal(converter('0'), 0)
|
||
|
|
assert_equal(converter(''), converter.default)
|
||
|
|
assert_equal(converter('missing'), converter.default)
|
||
|
|
assert_equal(converter('missed'), converter.default)
|
||
|
|
try:
|
||
|
|
converter('miss')
|
||
|
|
except ValueError:
|
||
|
|
pass
|
||
|
|
|
||
|
|
def test_upgrademapper(self):
|
||
|
|
"Tests updatemapper"
|
||
|
|
dateparser = _bytes_to_date
|
||
|
|
_original_mapper = StringConverter._mapper[:]
|
||
|
|
try:
|
||
|
|
StringConverter.upgrade_mapper(dateparser, date(2000, 1, 1))
|
||
|
|
convert = StringConverter(dateparser, date(2000, 1, 1))
|
||
|
|
test = convert('2001-01-01')
|
||
|
|
assert_equal(test, date(2001, 1, 1))
|
||
|
|
test = convert('2009-01-01')
|
||
|
|
assert_equal(test, date(2009, 1, 1))
|
||
|
|
test = convert('')
|
||
|
|
assert_equal(test, date(2000, 1, 1))
|
||
|
|
finally:
|
||
|
|
StringConverter._mapper = _original_mapper
|
||
|
|
|
||
|
|
def test_string_to_object(self):
|
||
|
|
"Make sure that string-to-object functions are properly recognized"
|
||
|
|
old_mapper = StringConverter._mapper[:] # copy of list
|
||
|
|
conv = StringConverter(_bytes_to_date)
|
||
|
|
assert_equal(conv._mapper, old_mapper)
|
||
|
|
assert_(hasattr(conv, 'default'))
|
||
|
|
|
||
|
|
def test_keep_default(self):
|
||
|
|
"Make sure we don't lose an explicit default"
|
||
|
|
converter = StringConverter(None, missing_values='',
|
||
|
|
default=-999)
|
||
|
|
converter.upgrade('3.14159265')
|
||
|
|
assert_equal(converter.default, -999)
|
||
|
|
assert_equal(converter.type, np.dtype(float))
|
||
|
|
#
|
||
|
|
converter = StringConverter(
|
||
|
|
None, missing_values='', default=0)
|
||
|
|
converter.upgrade('3.14159265')
|
||
|
|
assert_equal(converter.default, 0)
|
||
|
|
assert_equal(converter.type, np.dtype(float))
|
||
|
|
|
||
|
|
def test_keep_default_zero(self):
|
||
|
|
"Check that we don't lose a default of 0"
|
||
|
|
converter = StringConverter(int, default=0,
|
||
|
|
missing_values="N/A")
|
||
|
|
assert_equal(converter.default, 0)
|
||
|
|
|
||
|
|
def test_keep_missing_values(self):
|
||
|
|
"Check that we're not losing missing values"
|
||
|
|
converter = StringConverter(int, default=0,
|
||
|
|
missing_values="N/A")
|
||
|
|
assert_equal(
|
||
|
|
converter.missing_values, {'', 'N/A'})
|
||
|
|
|
||
|
|
def test_int64_dtype(self):
|
||
|
|
"Check that int64 integer types can be specified"
|
||
|
|
converter = StringConverter(np.int64, default=0)
|
||
|
|
val = "-9223372036854775807"
|
||
|
|
assert_(converter(val) == -9223372036854775807)
|
||
|
|
val = "9223372036854775807"
|
||
|
|
assert_(converter(val) == 9223372036854775807)
|
||
|
|
|
||
|
|
def test_uint64_dtype(self):
|
||
|
|
"Check that uint64 integer types can be specified"
|
||
|
|
converter = StringConverter(np.uint64, default=0)
|
||
|
|
val = "9223372043271415339"
|
||
|
|
assert_(converter(val) == 9223372043271415339)
|
||
|
|
|
||
|
|
|
||
|
|
class TestMiscFunctions:
|
||
|
|
|
||
|
|
def test_has_nested_dtype(self):
|
||
|
|
"Test has_nested_dtype"
|
||
|
|
ndtype = np.dtype(float)
|
||
|
|
assert_equal(has_nested_fields(ndtype), False)
|
||
|
|
ndtype = np.dtype([('A', '|S3'), ('B', float)])
|
||
|
|
assert_equal(has_nested_fields(ndtype), False)
|
||
|
|
ndtype = np.dtype([('A', int), ('B', [('BA', float), ('BB', '|S1')])])
|
||
|
|
assert_equal(has_nested_fields(ndtype), True)
|
||
|
|
|
||
|
|
def test_easy_dtype(self):
|
||
|
|
"Test ndtype on dtypes"
|
||
|
|
# Simple case
|
||
|
|
ndtype = float
|
||
|
|
assert_equal(easy_dtype(ndtype), np.dtype(float))
|
||
|
|
# As string w/o names
|
||
|
|
ndtype = "i4, f8"
|
||
|
|
assert_equal(easy_dtype(ndtype),
|
||
|
|
np.dtype([('f0', "i4"), ('f1', "f8")]))
|
||
|
|
# As string w/o names but different default format
|
||
|
|
assert_equal(easy_dtype(ndtype, defaultfmt="field_%03i"),
|
||
|
|
np.dtype([('field_000', "i4"), ('field_001', "f8")]))
|
||
|
|
# As string w/ names
|
||
|
|
ndtype = "i4, f8"
|
||
|
|
assert_equal(easy_dtype(ndtype, names="a, b"),
|
||
|
|
np.dtype([('a', "i4"), ('b', "f8")]))
|
||
|
|
# As string w/ names (too many)
|
||
|
|
ndtype = "i4, f8"
|
||
|
|
assert_equal(easy_dtype(ndtype, names="a, b, c"),
|
||
|
|
np.dtype([('a', "i4"), ('b', "f8")]))
|
||
|
|
# As string w/ names (not enough)
|
||
|
|
ndtype = "i4, f8"
|
||
|
|
assert_equal(easy_dtype(ndtype, names=", b"),
|
||
|
|
np.dtype([('f0', "i4"), ('b', "f8")]))
|
||
|
|
# ... (with different default format)
|
||
|
|
assert_equal(easy_dtype(ndtype, names="a", defaultfmt="f%02i"),
|
||
|
|
np.dtype([('a', "i4"), ('f00', "f8")]))
|
||
|
|
# As list of tuples w/o names
|
||
|
|
ndtype = [('A', int), ('B', float)]
|
||
|
|
assert_equal(easy_dtype(ndtype), np.dtype([('A', int), ('B', float)]))
|
||
|
|
# As list of tuples w/ names
|
||
|
|
assert_equal(easy_dtype(ndtype, names="a,b"),
|
||
|
|
np.dtype([('a', int), ('b', float)]))
|
||
|
|
# As list of tuples w/ not enough names
|
||
|
|
assert_equal(easy_dtype(ndtype, names="a"),
|
||
|
|
np.dtype([('a', int), ('f0', float)]))
|
||
|
|
# As list of tuples w/ too many names
|
||
|
|
assert_equal(easy_dtype(ndtype, names="a,b,c"),
|
||
|
|
np.dtype([('a', int), ('b', float)]))
|
||
|
|
# As list of types w/o names
|
||
|
|
ndtype = (int, float, float)
|
||
|
|
assert_equal(easy_dtype(ndtype),
|
||
|
|
np.dtype([('f0', int), ('f1', float), ('f2', float)]))
|
||
|
|
# As list of types w names
|
||
|
|
ndtype = (int, float, float)
|
||
|
|
assert_equal(easy_dtype(ndtype, names="a, b, c"),
|
||
|
|
np.dtype([('a', int), ('b', float), ('c', float)]))
|
||
|
|
# As simple dtype w/ names
|
||
|
|
ndtype = np.dtype(float)
|
||
|
|
assert_equal(easy_dtype(ndtype, names="a, b, c"),
|
||
|
|
np.dtype([(_, float) for _ in ('a', 'b', 'c')]))
|
||
|
|
# As simple dtype w/o names (but multiple fields)
|
||
|
|
ndtype = np.dtype(float)
|
||
|
|
assert_equal(
|
||
|
|
easy_dtype(ndtype, names=['', '', ''], defaultfmt="f%02i"),
|
||
|
|
np.dtype([(_, float) for _ in ('f00', 'f01', 'f02')]))
|
||
|
|
|
||
|
|
def test_flatten_dtype(self):
|
||
|
|
"Testing flatten_dtype"
|
||
|
|
# Standard dtype
|
||
|
|
dt = np.dtype([("a", "f8"), ("b", "f8")])
|
||
|
|
dt_flat = flatten_dtype(dt)
|
||
|
|
assert_equal(dt_flat, [float, float])
|
||
|
|
# Recursive dtype
|
||
|
|
dt = np.dtype([("a", [("aa", '|S1'), ("ab", '|S2')]), ("b", int)])
|
||
|
|
dt_flat = flatten_dtype(dt)
|
||
|
|
assert_equal(dt_flat, [np.dtype('|S1'), np.dtype('|S2'), int])
|
||
|
|
# dtype with shaped fields
|
||
|
|
dt = np.dtype([("a", (float, 2)), ("b", (int, 3))])
|
||
|
|
dt_flat = flatten_dtype(dt)
|
||
|
|
assert_equal(dt_flat, [float, int])
|
||
|
|
dt_flat = flatten_dtype(dt, True)
|
||
|
|
assert_equal(dt_flat, [float] * 2 + [int] * 3)
|
||
|
|
# dtype w/ titles
|
||
|
|
dt = np.dtype([(("a", "A"), "f8"), (("b", "B"), "f8")])
|
||
|
|
dt_flat = flatten_dtype(dt)
|
||
|
|
assert_equal(dt_flat, [float, float])
|