diff --git a/Lib/test/test_ordered_dict.py b/Lib/test/test_ordered_dict.py index 6bafb8be307ce4..1a2b5a16919df0 100644 --- a/Lib/test/test_ordered_dict.py +++ b/Lib/test/test_ordered_dict.py @@ -676,7 +676,7 @@ def test_sizeof_exact(self): size = support.calcobjsize check = self.check_sizeof - basicsize = size('nQ2P' + '3PnPn2P') + calcsize('2nP2n') + basicsize = size('nQ2P' + '3Pn2P') + calcsize('2nP2nI0P') entrysize = calcsize('n2P') p = calcsize('P') diff --git a/Lib/test/test_sys.py b/Lib/test/test_sys.py index 6933b41353bd56..bf9cc114a2bd65 100644 --- a/Lib/test/test_sys.py +++ b/Lib/test/test_sys.py @@ -965,9 +965,9 @@ def inner(): # method-wrapper (descriptor object) check({}.__iter__, size('2P')) # dict - check({}, size('nQ2P') + calcsize('2nP2n') + 8 + (8*2//3)*calcsize('n2P')) + check({}, size('nQ2P') + calcsize('2nP2nI0P') + 8 + (8*2//3)*calcsize('n2P')) longdict = {1:1, 2:2, 3:3, 4:4, 5:5, 6:6, 7:7, 8:8} - check(longdict, size('nQ2P') + calcsize('2nP2n') + 16 + (16*2//3)*calcsize('n2P')) + check(longdict, size('nQ2P') + calcsize('2nP2nI0P') + 16 + (16*2//3)*calcsize('n2P')) # dictionary-keyview check({}.keys(), size('P')) # dictionary-valueview @@ -1128,13 +1128,13 @@ def delx(self): del self.__x '4P') class newstyleclass(object): pass # Separate block for PyDictKeysObject with 8 keys and 5 entries - check(newstyleclass, s + calcsize("2nP2n0P") + 8 + 5*calcsize("n2P")) + check(newstyleclass, s + calcsize("2nP2nI0P") + 8 + 5*calcsize("n2P")) # dict with shared keys check(newstyleclass().__dict__, size('nQ2P') + 5*self.P) o = newstyleclass() o.a = o.b = o.c = o.d = o.e = o.f = o.g = o.h = 1 # Separate block for PyDictKeysObject with 16 keys and 10 entries - check(newstyleclass, s + calcsize("2nP2n0P") + 16 + 10*calcsize("n2P")) + check(newstyleclass, s + calcsize("2nP2nI0P") + 16 + 10*calcsize("n2P")) # dict with shared keys check(newstyleclass().__dict__, size('nQ2P') + 10*self.P) # unicode diff --git a/Objects/dict-common.h b/Objects/dict-common.h index 3e524686b4400d..f53bd25b9221c6 100644 --- a/Objects/dict-common.h +++ b/Objects/dict-common.h @@ -46,6 +46,9 @@ struct _dictkeysobject { /* Number of used entries in dk_entries. */ Py_ssize_t dk_nentries; + /* Whether OrderedDict's cache is synchronized with dict table */ + unsigned int dk_clean; + /* Actual hash table of dk_size entries. It holds indices in dk_entries, or DKIX_EMPTY(-1) or DKIX_DUMMY(-2). diff --git a/Objects/dictobject.c b/Objects/dictobject.c index b20b85c909e333..cfdffc0139b9d1 100644 --- a/Objects/dictobject.c +++ b/Objects/dictobject.c @@ -417,11 +417,12 @@ dk_set_index(PyDictKeysObject *keys, Py_ssize_t i, Py_ssize_t ix) * (which cannot fail and thus can do no allocation). */ static PyDictKeysObject empty_keys_struct = { - 1, /* dk_refcnt */ - 1, /* dk_size */ - lookdict_split, /* dk_lookup */ - 0, /* dk_usable (immutable) */ - 0, /* dk_nentries */ + .dk_refcnt = 1, + .dk_size = 1, + .dk_lookup = lookdict_split, + .dk_usable = 0, + .dk_nentries = 0, + .dk_clean = 0, .dk_indices = { .as_1 = {DKIX_EMPTY, DKIX_EMPTY, DKIX_EMPTY, DKIX_EMPTY, DKIX_EMPTY, DKIX_EMPTY, DKIX_EMPTY, DKIX_EMPTY}}, }; @@ -544,6 +545,7 @@ static PyDictKeysObject *new_keys_object(Py_ssize_t size) dk->dk_usable = usable; dk->dk_lookup = lookdict_unicode_nodummy; dk->dk_nentries = 0; + dk->dk_clean = 0; memset(&dk->dk_indices.as_1[0], 0xff, es * size); memset(DK_ENTRIES(dk), 0, sizeof(PyDictKeyEntry) * usable); return dk; @@ -1078,27 +1080,24 @@ dictresize(PyDictObject *mp, Py_ssize_t minsize) } oldkeys = mp->ma_keys; - - /* NOTE: Current odict checks mp->ma_keys to detect resize happen. - * So we can't reuse oldkeys even if oldkeys->dk_size == newsize. - * TODO: Try reusing oldkeys when reimplement odict. - */ - - /* Allocate a new table. */ - mp->ma_keys = new_keys_object(newsize); - if (mp->ma_keys == NULL) { - mp->ma_keys = oldkeys; - return -1; + oldvalues = mp->ma_values; + if (newsize != oldkeys->dk_size || oldvalues != NULL) { + /* Allocate a new table. */ + mp->ma_keys = new_keys_object(newsize); + if (mp->ma_keys == NULL) { + mp->ma_keys = oldkeys; + return -1; + } + // New table must be large enough. + assert(mp->ma_keys->dk_usable >= mp->ma_used); + if (oldkeys->dk_lookup == lookdict) + mp->ma_keys->dk_lookup = lookdict; } - // New table must be large enough. - assert(mp->ma_keys->dk_usable >= mp->ma_used); - if (oldkeys->dk_lookup == lookdict) - mp->ma_keys->dk_lookup = lookdict; + mp->ma_keys->dk_clean = 0; numentries = mp->ma_used; oldentries = DK_ENTRIES(oldkeys); newentries = DK_ENTRIES(mp->ma_keys); - oldvalues = mp->ma_values; if (oldvalues != NULL) { /* Convert split table into new combined table. * We must incref keys; we can transfer values. @@ -1122,7 +1121,8 @@ dictresize(PyDictObject *mp, Py_ssize_t minsize) } else { // combined table. if (oldkeys->dk_nentries == numentries) { - memcpy(newentries, oldentries, numentries * sizeof(PyDictKeyEntry)); + /* Source and destination can overlap if reuse an old table. */ + memmove(newentries, oldentries, numentries * sizeof(PyDictKeyEntry)); } else { PyDictKeyEntry *ep = oldentries; @@ -1133,14 +1133,24 @@ dictresize(PyDictObject *mp, Py_ssize_t minsize) } } - assert(oldkeys->dk_lookup != lookdict_split); - assert(oldkeys->dk_refcnt == 1); - if (oldkeys->dk_size == PyDict_MINSIZE && - numfreekeys < PyDict_MAXFREELIST) { - DK_DEBUG_DECREF keys_free_list[numfreekeys++] = oldkeys; + if (oldkeys != mp->ma_keys) { + assert(oldkeys->dk_lookup != lookdict_split); + assert(oldkeys->dk_refcnt == 1); + if (oldkeys->dk_size == PyDict_MINSIZE && + numfreekeys < PyDict_MAXFREELIST) + { + DK_DEBUG_DECREF keys_free_list[numfreekeys++] = oldkeys; + } + else { + DK_DEBUG_DECREF PyObject_FREE(oldkeys); + } } else { - DK_DEBUG_DECREF PyObject_FREE(oldkeys); + oldkeys->dk_usable = USABLE_FRACTION(newsize); + memset(&oldkeys->dk_indices.as_1[0], 0xff, + DK_IXSIZE(oldkeys) * newsize); + memset(newentries + numentries, 0, + sizeof(PyDictKeyEntry) * (oldkeys->dk_usable - numentries)); } } diff --git a/Objects/odictobject.c b/Objects/odictobject.c index 218aa2c5d9ce03..993121b9957df9 100644 --- a/Objects/odictobject.c +++ b/Objects/odictobject.c @@ -490,12 +490,9 @@ struct _odictobject { PyDictObject od_dict; /* the underlying dict */ _ODictNode *od_first; /* first node in the linked list, if any */ _ODictNode *od_last; /* last node in the linked list, if any */ - /* od_fast_nodes, od_fast_nodes_size and od_resize_sentinel are managed - * by _odict_resize(). - * Note that we rely on implementation details of dict for both. */ - _ODictNode **od_fast_nodes; /* hash table that mirrors the dict table */ - Py_ssize_t od_fast_nodes_size; - void *od_resize_sentinel; /* changes if odict should be resized */ + /* od_fast_nodes is managed by _odict_resize(). + * Note that we rely on implementation details of dict for it. */ + _ODictNode **od_fast_nodes; /* table that mirrors the dict table */ size_t od_state; /* incremented whenever the LL changes */ PyObject *od_inst_dict; /* OrderedDict().__dict__ */ @@ -587,8 +584,7 @@ _odict_resize(PyODictObject *od) { /* Replace the old fast nodes table. */ _odict_free_fast_nodes(od); od->od_fast_nodes = fast_nodes; - od->od_fast_nodes_size = size; - od->od_resize_sentinel = ((PyDictObject *)od)->ma_keys; + ((PyDictObject *)od)->ma_keys->dk_clean = 1; return 0; } @@ -596,14 +592,8 @@ _odict_resize(PyODictObject *od) { static Py_ssize_t _odict_get_index(PyODictObject *od, PyObject *key, Py_hash_t hash) { - PyDictKeysObject *keys; - - assert(key != NULL); - keys = ((PyDictObject *)od)->ma_keys; - /* Ensure od_fast_nodes and dk_entries are in sync. */ - if (od->od_resize_sentinel != keys || - od->od_fast_nodes_size != keys->dk_size) { + if (!((PyDictObject *)od)->ma_keys->dk_clean) { int resize_res = _odict_resize(od); if (resize_res < 0) return -1;