import ctypes import gc import operator import pickle import random import sys import types from itertools import permutations from typing import Any import hypothesis import pytest from hypothesis.extra import numpy as hynp import numpy as np import numpy.dtypes from numpy._core._multiarray_tests import create_custom_field_dtype from numpy._core._rational_tests import rational from numpy.testing import ( HAS_REFCOUNT, IS_PYSTON, IS_WASM, assert_, assert_array_equal, assert_equal, assert_raises, ) def assert_dtype_equal(a, b): assert_equal(a, b) assert_equal(hash(a), hash(b), "two equivalent types do not hash to the same value !") def assert_dtype_not_equal(a, b): assert_(a != b) assert_(hash(a) != hash(b), "two different types hash to the same value !") class TestBuiltin: @pytest.mark.parametrize('t', [int, float, complex, np.int32, str, object]) def test_run(self, t): """Only test hash runs at all.""" dt = np.dtype(t) hash(dt) @pytest.mark.parametrize('t', [int, float]) def test_dtype(self, t): # Make sure equivalent byte order char hash the same (e.g. < and = on # little endian) dt = np.dtype(t) dt2 = dt.newbyteorder("<") dt3 = dt.newbyteorder(">") if dt == dt2: assert_(dt.byteorder != dt2.byteorder, "bogus test") assert_dtype_equal(dt, dt2) else: assert_(dt.byteorder != dt3.byteorder, "bogus test") assert_dtype_equal(dt, dt3) def test_equivalent_dtype_hashing(self): # Make sure equivalent dtypes with different type num hash equal uintp = np.dtype(np.uintp) if uintp.itemsize == 4: left = uintp right = np.dtype(np.uint32) else: left = uintp right = np.dtype(np.ulonglong) assert_(left == right) assert_(hash(left) == hash(right)) def test_invalid_types(self): # Make sure invalid type strings raise an error assert_raises(TypeError, np.dtype, 'O3') assert_raises(TypeError, np.dtype, 'O5') assert_raises(TypeError, np.dtype, 'O7') assert_raises(TypeError, np.dtype, 'b3') assert_raises(TypeError, np.dtype, 'h4') assert_raises(TypeError, np.dtype, 'I5') assert_raises(TypeError, np.dtype, 'e3') assert_raises(TypeError, np.dtype, 'f5') if np.dtype('g').itemsize == 8 or np.dtype('g').itemsize == 16: assert_raises(TypeError, np.dtype, 'g12') elif np.dtype('g').itemsize == 12: assert_raises(TypeError, np.dtype, 'g16') if np.dtype('l').itemsize == 8: assert_raises(TypeError, np.dtype, 'l4') assert_raises(TypeError, np.dtype, 'L4') else: assert_raises(TypeError, np.dtype, 'l8') assert_raises(TypeError, np.dtype, 'L8') if np.dtype('q').itemsize == 8: assert_raises(TypeError, np.dtype, 'q4') assert_raises(TypeError, np.dtype, 'Q4') else: assert_raises(TypeError, np.dtype, 'q8') assert_raises(TypeError, np.dtype, 'Q8') # Make sure negative-sized dtype raises an error assert_raises(TypeError, np.dtype, 'S-1') assert_raises(TypeError, np.dtype, 'U-1') assert_raises(TypeError, np.dtype, 'V-1') def test_richcompare_invalid_dtype_equality(self): # Make sure objects that cannot be converted to valid # dtypes results in False/True when compared to valid dtypes. # Here 7 cannot be converted to dtype. No exceptions should be raised assert not np.dtype(np.int32) == 7, "dtype richcompare failed for ==" assert np.dtype(np.int32) != 7, "dtype richcompare failed for !=" @pytest.mark.parametrize( 'operation', [operator.le, operator.lt, operator.ge, operator.gt]) def test_richcompare_invalid_dtype_comparison(self, operation): # Make sure TypeError is raised for comparison operators # for invalid dtypes. Here 7 is an invalid dtype. with pytest.raises(TypeError): operation(np.dtype(np.int32), 7) @pytest.mark.parametrize("dtype", ['Bool', 'Bytes0', 'Complex32', 'Complex64', 'Datetime64', 'Float16', 'Float32', 'Float64', 'Int8', 'Int16', 'Int32', 'Int64', 'Object0', 'Str0', 'Timedelta64', 'UInt8', 'UInt16', 'Uint32', 'UInt32', 'Uint64', 'UInt64', 'Void0', "Float128", "Complex128"]) def test_numeric_style_types_are_invalid(self, dtype): with assert_raises(TypeError): np.dtype(dtype) def test_expired_dtypes_with_bad_bytesize(self): match: str = r".*removed in NumPy 2.0.*" with pytest.raises(TypeError, match=match): np.dtype("int0") with pytest.raises(TypeError, match=match): np.dtype("uint0") with pytest.raises(TypeError, match=match): np.dtype("bool8") with pytest.raises(TypeError, match=match): np.dtype("bytes0") with pytest.raises(TypeError, match=match): np.dtype("str0") with pytest.raises(TypeError, match=match): np.dtype("object0") with pytest.raises(TypeError, match=match): np.dtype("void0") @pytest.mark.parametrize( 'value', ['m8', 'M8', 'datetime64', 'timedelta64', 'i4, (2,3)f8, f4', 'S3, 3u8, (3,4)S10', '>f', '= (3, 12), reason="Python 3.12 has immortal refcounts, this test will no longer " "work. See gh-23986" ) @pytest.mark.skipif(not HAS_REFCOUNT, reason="Python lacks refcounts") class TestStructuredObjectRefcounting: """These tests cover various uses of complicated structured types which include objects and thus require reference counting. """ @pytest.mark.parametrize(['dt', 'pat', 'count', 'singleton'], iter_struct_object_dtypes()) @pytest.mark.parametrize(["creation_func", "creation_obj"], [ pytest.param(np.empty, None, # None is probably used for too many things marks=pytest.mark.skip("unreliable due to python's behaviour")), (np.ones, 1), (np.zeros, 0)]) def test_structured_object_create_delete(self, dt, pat, count, singleton, creation_func, creation_obj): """Structured object reference counting in creation and deletion""" # The test assumes that 0, 1, and None are singletons. gc.collect() before = sys.getrefcount(creation_obj) arr = creation_func(3, dt) now = sys.getrefcount(creation_obj) assert now - before == count * 3 del arr now = sys.getrefcount(creation_obj) assert now == before @pytest.mark.parametrize(['dt', 'pat', 'count', 'singleton'], iter_struct_object_dtypes()) def test_structured_object_item_setting(self, dt, pat, count, singleton): """Structured object reference counting for simple item setting""" one = 1 gc.collect() before = sys.getrefcount(singleton) arr = np.array([pat] * 3, dt) assert sys.getrefcount(singleton) - before == count * 3 # Fill with `1` and check that it was replaced correctly: before2 = sys.getrefcount(one) arr[...] = one after2 = sys.getrefcount(one) assert after2 - before2 == count * 3 del arr gc.collect() assert sys.getrefcount(one) == before2 assert sys.getrefcount(singleton) == before @pytest.mark.parametrize(['dt', 'pat', 'count', 'singleton'], iter_struct_object_dtypes()) @pytest.mark.parametrize( ['shape', 'index', 'items_changed'], [((3,), ([0, 2],), 2), ((3, 2), ([0, 2], slice(None)), 4), ((3, 2), ([0, 2], [1]), 2), ((3,), ([True, False, True]), 2)]) def test_structured_object_indexing(self, shape, index, items_changed, dt, pat, count, singleton): """Structured object reference counting for advanced indexing.""" # Use two small negative values (should be singletons, but less likely # to run into race-conditions). This failed in some threaded envs # When using 0 and 1. If it fails again, should remove all explicit # checks, and rely on `pytest-leaks` reference count checker only. val0 = -4 val1 = -5 arr = np.full(shape, val0, dt) gc.collect() before_val0 = sys.getrefcount(val0) before_val1 = sys.getrefcount(val1) # Test item getting: part = arr[index] after_val0 = sys.getrefcount(val0) assert after_val0 - before_val0 == count * items_changed del part # Test item setting: arr[index] = val1 gc.collect() after_val0 = sys.getrefcount(val0) after_val1 = sys.getrefcount(val1) assert before_val0 - after_val0 == count * items_changed assert after_val1 - before_val1 == count * items_changed @pytest.mark.parametrize(['dt', 'pat', 'count', 'singleton'], iter_struct_object_dtypes()) def test_structured_object_take_and_repeat(self, dt, pat, count, singleton): """Structured object reference counting for specialized functions. The older functions such as take and repeat use different code paths then item setting (when writing this). """ indices = [0, 1] arr = np.array([pat] * 3, dt) gc.collect() before = sys.getrefcount(singleton) res = arr.take(indices) after = sys.getrefcount(singleton) assert after - before == count * 2 new = res.repeat(10) gc.collect() after_repeat = sys.getrefcount(singleton) assert after_repeat - after == count * 2 * 10 class TestStructuredDtypeSparseFields: """Tests subarray fields which contain sparse dtypes so that not all memory is used by the dtype work. Such dtype's should leave the underlying memory unchanged. """ dtype = np.dtype([('a', {'names': ['aa', 'ab'], 'formats': ['f', 'f'], 'offsets': [0, 4]}, (2, 3))]) sparse_dtype = np.dtype([('a', {'names': ['ab'], 'formats': ['f'], 'offsets': [4]}, (2, 3))]) def test_sparse_field_assignment(self): arr = np.zeros(3, self.dtype) sparse_arr = arr.view(self.sparse_dtype) sparse_arr[...] = np.finfo(np.float32).max # dtype is reduced when accessing the field, so shape is (3, 2, 3): assert_array_equal(arr["a"]["aa"], np.zeros((3, 2, 3))) def test_sparse_field_assignment_fancy(self): # Fancy assignment goes to the copyswap function for complex types: arr = np.zeros(3, self.dtype) sparse_arr = arr.view(self.sparse_dtype) sparse_arr[[0, 1, 2]] = np.finfo(np.float32).max # dtype is reduced when accessing the field, so shape is (3, 2, 3): assert_array_equal(arr["a"]["aa"], np.zeros((3, 2, 3))) class TestMonsterType: """Test deeply nested subtypes.""" def test1(self): simple1 = np.dtype({'names': ['r', 'b'], 'formats': ['u1', 'u1'], 'titles': ['Red pixel', 'Blue pixel']}) a = np.dtype([('yo', int), ('ye', simple1), ('yi', np.dtype((int, (3, 2))))]) b = np.dtype([('yo', int), ('ye', simple1), ('yi', np.dtype((int, (3, 2))))]) assert_dtype_equal(a, b) c = np.dtype([('yo', int), ('ye', simple1), ('yi', np.dtype((a, (3, 2))))]) d = np.dtype([('yo', int), ('ye', simple1), ('yi', np.dtype((a, (3, 2))))]) assert_dtype_equal(c, d) @pytest.mark.skipif(IS_PYSTON, reason="Pyston disables recursion checking") @pytest.mark.skipif(IS_WASM, reason="Pyodide/WASM has limited stack size") def test_list_recursion(self): l = [] l.append(('f', l)) with pytest.raises(RecursionError): np.dtype(l) @pytest.mark.skipif(IS_PYSTON, reason="Pyston disables recursion checking") @pytest.mark.skipif(IS_WASM, reason="Pyodide/WASM has limited stack size") def test_tuple_recursion(self): d = np.int32 for i in range(100000): d = (d, (1,)) with pytest.raises(RecursionError): np.dtype(d) @pytest.mark.skipif(IS_PYSTON, reason="Pyston disables recursion checking") @pytest.mark.skipif(IS_WASM, reason="Pyodide/WASM has limited stack size") def test_dict_recursion(self): d = {"names": ['self'], "formats": [None], "offsets": [0]} d['formats'][0] = d with pytest.raises(RecursionError): np.dtype(d) class TestMetadata: def test_no_metadata(self): d = np.dtype(int) assert_(d.metadata is None) def test_metadata_takes_dict(self): d = np.dtype(int, metadata={'datum': 1}) assert_(d.metadata == {'datum': 1}) def test_metadata_rejects_nondict(self): assert_raises(TypeError, np.dtype, int, metadata='datum') assert_raises(TypeError, np.dtype, int, metadata=1) assert_raises(TypeError, np.dtype, int, metadata=None) def test_nested_metadata(self): d = np.dtype([('a', np.dtype(int, metadata={'datum': 1}))]) assert_(d['a'].metadata == {'datum': 1}) def test_base_metadata_copied(self): d = np.dtype((np.void, np.dtype('i4,i4', metadata={'datum': 1}))) assert_(d.metadata == {'datum': 1}) class TestString: def test_complex_dtype_str(self): dt = np.dtype([('top', [('tiles', ('>f4', (64, 64)), (1,)), ('rtile', '>f4', (64, 36))], (3,)), ('bottom', [('bleft', ('>f4', (8, 64)), (1,)), ('bright', '>f4', (8, 36))])]) assert_equal(str(dt), "[('top', [('tiles', ('>f4', (64, 64)), (1,)), " "('rtile', '>f4', (64, 36))], (3,)), " "('bottom', [('bleft', ('>f4', (8, 64)), (1,)), " "('bright', '>f4', (8, 36))])]") # If the sticky aligned flag is set to True, it makes the # str() function use a dict representation with an 'aligned' flag dt = np.dtype([('top', [('tiles', ('>f4', (64, 64)), (1,)), ('rtile', '>f4', (64, 36))], (3,)), ('bottom', [('bleft', ('>f4', (8, 64)), (1,)), ('bright', '>f4', (8, 36))])], align=True) assert_equal(str(dt), "{'names': ['top', 'bottom']," " 'formats': [([('tiles', ('>f4', (64, 64)), (1,)), " "('rtile', '>f4', (64, 36))], (3,)), " "[('bleft', ('>f4', (8, 64)), (1,)), " "('bright', '>f4', (8, 36))]]," " 'offsets': [0, 76800]," " 'itemsize': 80000," " 'aligned': True}") with np.printoptions(legacy='1.21'): assert_equal(str(dt), "{'names':['top','bottom'], " "'formats':[([('tiles', ('>f4', (64, 64)), (1,)), " "('rtile', '>f4', (64, 36))], (3,))," "[('bleft', ('>f4', (8, 64)), (1,)), " "('bright', '>f4', (8, 36))]], " "'offsets':[0,76800], " "'itemsize':80000, " "'aligned':True}") assert_equal(np.dtype(eval(str(dt))), dt) dt = np.dtype({'names': ['r', 'g', 'b'], 'formats': ['u1', 'u1', 'u1'], 'offsets': [0, 1, 2], 'titles': ['Red pixel', 'Green pixel', 'Blue pixel']}) assert_equal(str(dt), "[(('Red pixel', 'r'), 'u1'), " "(('Green pixel', 'g'), 'u1'), " "(('Blue pixel', 'b'), 'u1')]") dt = np.dtype({'names': ['rgba', 'r', 'g', 'b'], 'formats': ['f4', (64, 64)), (1,)), ('rtile', '>f4', (64, 36))], (3,)), ('bottom', [('bleft', ('>f4', (8, 64)), (1,)), ('bright', '>f4', (8, 36))])]) assert_equal(repr(dt), "dtype([('top', [('tiles', ('>f4', (64, 64)), (1,)), " "('rtile', '>f4', (64, 36))], (3,)), " "('bottom', [('bleft', ('>f4', (8, 64)), (1,)), " "('bright', '>f4', (8, 36))])])") dt = np.dtype({'names': ['r', 'g', 'b'], 'formats': ['u1', 'u1', 'u1'], 'offsets': [0, 1, 2], 'titles': ['Red pixel', 'Green pixel', 'Blue pixel']}, align=True) assert_equal(repr(dt), "dtype([(('Red pixel', 'r'), 'u1'), " "(('Green pixel', 'g'), 'u1'), " "(('Blue pixel', 'b'), 'u1')], align=True)") def test_repr_structured_not_packed(self): dt = np.dtype({'names': ['rgba', 'r', 'g', 'b'], 'formats': ['i4") assert np.result_type(dt).isnative assert np.result_type(dt).num == dt.num # dtype with empty space: struct_dt = np.dtype(">i4,i1,f4', (2, 1)), ('b', 'u4')]) self.check(BigEndStruct, expected) def test_little_endian_structure_packed(self): class LittleEndStruct(ctypes.LittleEndianStructure): _fields_ = [ ('one', ctypes.c_uint8), ('two', ctypes.c_uint32) ] _pack_ = 1 expected = np.dtype([('one', 'u1'), ('two', 'B'), ('b', '>H') ], align=True) self.check(PaddedStruct, expected) def test_simple_endian_types(self): self.check(ctypes.c_uint16.__ctype_le__, np.dtype('u2')) self.check(ctypes.c_uint8.__ctype_le__, np.dtype('u1')) self.check(ctypes.c_uint8.__ctype_be__, np.dtype('u1')) all_types = set(np.typecodes['All']) all_pairs = permutations(all_types, 2) @pytest.mark.parametrize("pair", all_pairs) def test_pairs(self, pair): """ Check that np.dtype('x,y') matches [np.dtype('x'), np.dtype('y')] Example: np.dtype('d,I') -> dtype([('f0', ' None: alias = np.dtype[Any] assert isinstance(alias, types.GenericAlias) assert alias.__origin__ is np.dtype @pytest.mark.parametrize("code", np.typecodes["All"]) def test_dtype_subclass(self, code: str) -> None: cls = type(np.dtype(code)) alias = cls[Any] assert isinstance(alias, types.GenericAlias) assert alias.__origin__ is cls @pytest.mark.parametrize("arg_len", range(4)) def test_subscript_tuple(self, arg_len: int) -> None: arg_tup = (Any,) * arg_len if arg_len == 1: assert np.dtype[arg_tup] else: with pytest.raises(TypeError): np.dtype[arg_tup] def test_subscript_scalar(self) -> None: assert np.dtype[Any] def test_result_type_integers_and_unitless_timedelta64(): # Regression test for gh-20077. The following call of `result_type` # would cause a seg. fault. td = np.timedelta64(4) result = np.result_type(0, td) assert_dtype_equal(result, td.dtype) def test_creating_dtype_with_dtype_class_errors(): # Regression test for #25031, calling `np.dtype` with itself segfaulted. with pytest.raises(TypeError, match="Cannot convert np.dtype into a"): np.array(np.ones(10), dtype=np.dtype)