Merge branch 'master' into enh-masked-2d

Author: jbrockmendel

.github/workflows/ci.yml

@@ -169,6 +169,7 @@ jobs:
         pytest pandas/tests/indexing/multiindex/test_setitem.py::TestMultiIndexSetItem::test_frame_setitem_multi_column
 
         pytest pandas/tests/api/
+        pytest pandas/tests/apply/
         pytest pandas/tests/arrays/
         pytest pandas/tests/base/
         pytest pandas/tests/computation/

.pre-commit-config.yaml

@@ -2,7 +2,7 @@ minimum_pre_commit_version: 2.9.2
 exclude: ^LICENSES/|\.(html|csv|svg)$
 repos:
 -   repo: https://github.com/MarcoGorelli/absolufy-imports
-    rev: v0.2.1
+    rev: v0.3.0
     hooks:
     -   id: absolufy-imports
         files: ^pandas/
@@ -33,7 +33,7 @@ repos:
         exclude: ^pandas/_libs/src/(klib|headers)/
         args: [--quiet, '--extensions=c,h', '--headers=h', --recursive, '--filter=-readability/casting,-runtime/int,-build/include_subdir']
 -   repo: https://gitlab.com/pycqa/flake8
-    rev: 3.8.4
+    rev: 3.9.0
     hooks:
     -   id: flake8
         additional_dependencies: [flake8-comprehensions>=3.1.0]

Makefile

@@ -1,4 +1,4 @@
-.PHONY : develop build clean clean_pyc doc lint-diff black
+.PHONY : develop build clean clean_pyc doc lint-diff black test-scripts
 
 all: develop
 
@@ -25,3 +25,6 @@ doc:
 	cd doc; \
 	python make.py clean; \
 	python make.py html
+
+test-scripts:
+	pytest scripts

doc/source/user_guide/window.rst

@@ -168,7 +168,7 @@ parameter:
 =============  ====================
 Value          Behavior
 =============  ====================
-``right'``     close right endpoint
+``'right'``     close right endpoint
 ``'left'``     close left endpoint
 ``'both'``     close both endpoints
 ``'neither'``  open endpoints
@@ -214,7 +214,7 @@ ending indices of the windows. Additionally, ``num_values``, ``min_periods``, ``
 and will automatically be passed to ``get_window_bounds`` and the defined method must
 always accept these arguments.
 
-For example, if we have the following :class:``DataFrame``:
+For example, if we have the following :class:`DataFrame`
 
 .. ipython:: python
 
@@ -370,8 +370,8 @@ two :class:`Series` or any combination of :class:`DataFrame`/:class:`Series` or
   with the passed Series, thus returning a DataFrame.
 * :class:`DataFrame`/:class:`DataFrame`: by default compute the statistic for matching column
   names, returning a DataFrame. If the keyword argument ``pairwise=True`` is
-  passed then computes the statistic for each pair of columns, returning a
-  ``MultiIndexed DataFrame`` whose ``index`` are the dates in question (see :ref:`the next section
+  passed then computes the statistic for each pair of columns, returning a :class:`DataFrame` with a
+  :class:`MultiIndex` whose values are the dates in question (see :ref:`the next section
   <window.corr_pairwise>`).
 
 For example:

doc/source/whatsnew/v1.3.0.rst

@@ -225,6 +225,41 @@ In pandas 1.3.0, ``df`` continues to share data with ``values``
    np.shares_memory(df["A"], values)
 
 
+.. _whatsnew_130.notable_bug_fixes.setitem_never_inplace:
+
+Never Operate Inplace When Setting ``frame[keys] = values``
+^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
+
+When setting multiple columns using ``frame[keys] = values`` new arrays will
+replace pre-existing arrays for these keys, which will *not* be over-written
+(:issue:`39510`).  As a result, the columns will retain the dtype(s) of ``values``,
+never casting to the dtypes of the existing arrays.
+
+.. ipython:: python
+
+   df = pd.DataFrame(range(3), columns=["A"], dtype="float64")
+   df[["A"]] = 5
+
+In the old behavior, ``5`` was cast to ``float64`` and inserted into the existing
+array backing ``df``:
+
+*pandas 1.2.x*
+
+.. code-block:: ipython
+
+   In [1]: df.dtypes
+   Out[1]:
+   A    float64
+
+In the new behavior, we get a new array, and retain an integer-dtyped ``5``:
+
+*pandas 1.3.0*
+
+.. ipython:: python
+
+   df.dtypes
+
+
 .. _whatsnew_130.notable_bug_fixes.setitem_with_bool_casting:
 
 Consistent Casting With Setting Into Boolean Series
@@ -300,7 +335,7 @@ Optional libraries below the lowest tested version may still work, but are not c
 +=================+=================+=========+
 | beautifulsoup4  | 4.6.0           |         |
 +-----------------+-----------------+---------+
-| fastparquet     | 0.3.2           |         |
+| fastparquet     | 0.4.0           |    X    |
 +-----------------+-----------------+---------+
 | fsspec          | 0.7.4           |         |
 +-----------------+-----------------+---------+
@@ -627,6 +662,7 @@ Other
 - Bug in :class:`Styler` where multiple elements in CSS-selectors were not correctly added to ``table_styles`` (:issue:`39942`)
 - Bug in :meth:`DataFrame.equals`, :meth:`Series.equals`, :meth:`Index.equals` with object-dtype containing ``np.datetime64("NaT")`` or ``np.timedelta64("NaT")`` (:issue:`39650`)
 - Bug in :func:`pandas.util.show_versions` where console JSON output was not proper JSON (:issue:`39701`)
+- Bug in :meth:`DataFrame.convert_dtypes` incorrectly raised ValueError when called on an empty DataFrame (:issue:`40393`)
 
 
 .. ---------------------------------------------------------------------------

pandas/_libs/index.pyx

@@ -1,5 +1,7 @@
 import warnings
 
+cimport cython
+
 import numpy as np
 
 cimport numpy as cnp
@@ -47,6 +49,7 @@ cdef inline bint is_definitely_invalid_key(object val):
 _SIZE_CUTOFF = 1_000_000
 
 
+@cython.freelist(32)
 cdef class IndexEngine:
 
     cdef readonly:

pandas/_libs/missing.pyx

@@ -104,6 +104,7 @@ cpdef bint checknull(object val):
      - np.datetime64 representation of NaT
      - np.timedelta64 representation of NaT
      - NA
+     - Decimal("NaN")
 
     Parameters
     ----------
@@ -143,6 +144,8 @@ cpdef bint checknull_old(object val):
      - NaT
      - np.datetime64 representation of NaT
      - np.timedelta64 representation of NaT
+     - NA
+     - Decimal("NaN")
 
     Parameters
     ----------
@@ -175,6 +178,8 @@ cpdef ndarray[uint8_t] isnaobj(ndarray arr):
      - NaT
      - np.datetime64 representation of NaT
      - np.timedelta64 representation of NaT
+     - NA
+     - Decimal("NaN")
 
     Parameters
     ----------
@@ -211,6 +216,7 @@ def isnaobj_old(arr: ndarray) -> ndarray:
      - NEGINF
      - NaT
      - NA
+     - Decimal("NaN")
 
     Parameters
     ----------
@@ -249,6 +255,8 @@ def isnaobj2d(arr: ndarray) -> ndarray:
      - NaT
      - np.datetime64 representation of NaT
      - np.timedelta64 representation of NaT
+     - NA
+     - Decimal("NaN")
 
     Parameters
     ----------
@@ -293,6 +301,8 @@ def isnaobj2d_old(arr: ndarray) -> ndarray:
      - NaT
      - np.datetime64 representation of NaT
      - np.timedelta64 representation of NaT
+     - NA
+     - Decimal("NaN")
 
     Parameters
     ----------

pandas/_libs/parsers.pyx

@@ -284,21 +284,10 @@ cdef extern from "parser/tokenizer.h":
 
 
 cdef extern from "parser/io.h":
-    void *new_mmap(char *fname)
-    int del_mmap(void *src)
-    void* buffer_mmap_bytes(void *source, size_t nbytes,
-                            size_t *bytes_read, int *status)
-
-    void *new_file_source(char *fname, size_t buffer_size) except NULL
-
     void *new_rd_source(object obj) except NULL
 
-    int del_file_source(void *src)
     int del_rd_source(void *src)
 
-    void* buffer_file_bytes(void *source, size_t nbytes,
-                            size_t *bytes_read, int *status)
-
     void* buffer_rd_bytes(void *source, size_t nbytes,
                           size_t *bytes_read, int *status, const char *encoding_errors)