Skip to content

Commit 421a366

Browse files
charrischarris
authored
Merge pull request #32557 from charris/backport-32508-32358
BUG: Reset an unrepresentable fill_value on a dtype change (#32508)
2 parents 2e4a107 + 2be536a commit 421a366

3 files changed

Lines changed: 68 additions & 9 deletions

File tree

‎doc/release/upcoming_changes/32508.change.rst‎

Lines changed: 7 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,7 @@
1+
A ``MaskedArray`` fill_value that cannot be represented in a new dtype is now
2+
reset to the default for that dtype in more cases. Previously only casts that
3+
raised were detected. A floating point fill_value that overflows an integer
4+
dtype fails through the floating point error state instead, and the resulting
5+
out-of-range value was kept together with a ``RuntimeWarning``. Reading
6+
``arr.fill_value`` before a dtype change was enough to reach this, since
7+
reading it stores the default fill_value on the array.

‎numpy/ma/core.py‎

Lines changed: 6 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -2320,7 +2320,7 @@ def masked_object(x, value, copy=True, shrink=True):
23202320
else:
23212321
condition = umath.equal(np.asarray(x), value)
23222322
mask = nomask
2323-
mask = mask_or(mask, make_mask(condition, shrink=shrink))
2323+
mask = mask_or(mask, make_mask(condition, shrink=shrink), shrink=shrink)
23242324
return masked_array(x, mask=mask, copy=copy, fill_value=value)
23252325

23262326

@@ -3039,9 +3039,12 @@ def _update_from(self, obj):
30393039
_optinfo.update(getattr(obj, '__dict__', {}))
30403040
_fill_value = getattr(obj, '_fill_value', None)
30413041
if _fill_value is not None and getattr(obj, 'dtype', None) != self.dtype:
3042+
# _check_fill_value does not raise when a float overflows an
3043+
# integer dtype; that failure only shows up as an FP error.
30423044
try:
3043-
_fill_value = _check_fill_value(_fill_value, self.dtype)
3044-
except (TypeError, ValueError, OverflowError):
3045+
with np.errstate(invalid='raise'):
3046+
_fill_value = _check_fill_value(_fill_value, self.dtype)
3047+
except (TypeError, ValueError, OverflowError, FloatingPointError):
30453048
_fill_value = None
30463049
_dict = {'_fill_value': _fill_value,
30473050
'_hardmask': getattr(obj, '_hardmask', False),

‎numpy/ma/tests/test_core.py‎

Lines changed: 55 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -49,6 +49,7 @@
4949
array,
5050
asarray,
5151
choose,
52+
common_fill_value,
5253
concatenate,
5354
conjugate,
5455
cos,
@@ -89,6 +90,7 @@
8990
masked_less,
9091
masked_less_equal,
9192
masked_not_equal,
93+
masked_object,
9294
masked_outside,
9395
masked_print_option,
9496
masked_values,
@@ -2691,6 +2693,37 @@ def test_fillvalue_bytes_or_str(self):
26912693
assert_equal(a["f0"].fill_value, default_fill_value(b"spam"))
26922694
assert_equal(a["f1"].fill_value, default_fill_value("eggs"))
26932695

2696+
def test_common_fill_value(self):
2697+
# Test with matching fill value, across different dtypes and shapes.
2698+
a = array([1, 2, 3], dtype=int, fill_value=10)
2699+
b = array([[4, 5], [6, 7]], dtype=float, fill_value=10)
2700+
assert_equal(common_fill_value(a, b), 10)
2701+
2702+
# Test with non-matching fill value.
2703+
b.fill_value = -10
2704+
assert common_fill_value(a, b) is None
2705+
2706+
@pytest.mark.skipif(IS_WASM, reason="fp errors don't work in wasm")
2707+
def test_fillvalue_reset_on_lossy_float_cast(self):
2708+
# gh-28255
2709+
a = arange(9.0)
2710+
untouched = np.ones_like(a, dtype="int64")
2711+
a.fill_value # materialise the default fill_value
2712+
with warnings.catch_warnings():
2713+
warnings.simplefilter("error")
2714+
touched = np.ones_like(a, dtype="int64")
2715+
assert_equal(touched.fill_value, untouched.fill_value)
2716+
assert_equal(touched.fill_value, default_fill_value(touched.dtype))
2717+
2718+
def test_fillvalue_kept_on_exact_float_cast(self):
2719+
a = array([1.0, 2.0], mask=[0, 1], fill_value=5.0)
2720+
assert_equal(np.ones_like(a, dtype="int64").fill_value, 5)
2721+
2722+
def test_fillvalue_setter_still_raises(self):
2723+
a = array([1, 2], dtype="int64")
2724+
with pytest.raises(TypeError):
2725+
a.fill_value = 1e20
2726+
26942727

26952728
class TestUfuncs:
26962729
# Test class for the application of ufuncs on MaskedArrays.
@@ -4792,10 +4825,12 @@ def test_where(self):
47924825
tmp[(xm <= 2).filled(True)] = True
47934826
assert_equal(d._mask, tmp)
47944827

4795-
with np.errstate(invalid="warn"):
4796-
# The fill value is 1e20, it cannot be converted to `int`:
4797-
with pytest.warns(RuntimeWarning, match="invalid value"):
4798-
ixm = xm.astype(int)
4828+
# The fill value is 1e20, it cannot be converted to `int`, so the
4829+
# cast falls back to the default fill_value (gh-28255):
4830+
with warnings.catch_warnings():
4831+
warnings.simplefilter("error")
4832+
ixm = xm.astype(int)
4833+
assert_equal(ixm.fill_value, default_fill_value(ixm.dtype))
47994834
d = where(ixm > 2, ixm, masked)
48004835
assert_equal(d, [-9, -9, -9, -9, -9, 4, -9, -9, 10, -9, -9, 3])
48014836
assert_equal(d.dtype, ixm.dtype)
@@ -5747,8 +5782,22 @@ def test_attributes_readonly(self):
57475782

57485783

57495784
class TestMaskedWhereAliases:
5750-
5751-
# TODO: Test masked_object, masked_equal, ...
5785+
def test_masked_object(self):
5786+
food = np.array(['green_eggs', 'ham'], dtype=object)
5787+
res = masked_object(food, 'green_eggs')
5788+
assert_equal(res.mask, [True, False])
5789+
assert_(res[0] is masked)
5790+
assert_equal(res.fill_value, 'green_eggs')
5791+
5792+
res = masked_object(food, 'cheese')
5793+
assert_(res.mask is nomask)
5794+
5795+
res = masked_object(food, 'cheese', shrink=False)
5796+
assert_equal(res.mask, [False, False])
5797+
5798+
xm = array(['a', 'b', 'c'], mask=[1, 0, 0], dtype=object)
5799+
res = masked_object(xm, 'c')
5800+
assert_equal(res.mask, [True, False, True])
57525801

57535802
def test_masked_values(self):
57545803
res = masked_values(np.array([-32768.0]), np.int16(-32768))

0 commit comments

Comments
 (0)