Merge branch 'master' into enh-masked-2d

Author: jbrockmendel

doc/source/getting_started/install.rst

@@ -255,47 +255,52 @@ For example, :func:`pandas.read_hdf` requires the ``pytables`` package, while
 optional dependency is not installed, pandas will raise an ``ImportError`` when
 the method requiring that dependency is called.
 
+Visualization
+^^^^^^^^^^^^^
+
 ========================= ================== =============================================================
 Dependency                Minimum Version    Notes
 ========================= ================== =============================================================
-BeautifulSoup4            4.6.0              HTML parser for read_html (see :ref:`note <optional_html>`)
+matplotlib                2.2.3              Plotting library
 Jinja2                    2.10               Conditional formatting with DataFrame.style
-PyQt4                                        Clipboard I/O
-PyQt5                                        Clipboard I/O
-PyTables                  3.5.1              HDF5-based reading / writing
-SQLAlchemy                1.3.0              SQL support for databases other than sqlite
+tabulate                  0.8.7              Printing in Markdown-friendly format (see `tabulate`_)
+========================= ================== =============================================================
+
+Computation
+^^^^^^^^^^^
+
+========================= ================== =============================================================
+Dependency                Minimum Version    Notes
+========================= ================== =============================================================
 SciPy                     1.12.0             Miscellaneous statistical functions
-xlsxwriter                1.0.2              Excel writing
-blosc                     1.17.0             Compression for HDF5
-fsspec                    0.7.4              Handling files aside from local and HTTP
-fastparquet               0.4.0              Parquet reading / writing
-gcsfs                     0.6.0              Google Cloud Storage access
-html5lib                  1.0.1              HTML parser for read_html (see :ref:`note <optional_html>`)
-lxml                      4.3.0              HTML parser for read_html (see :ref:`note <optional_html>`)
-matplotlib                2.2.3              Visualization
 numba                     0.46.0             Alternative execution engine for rolling operations
+                                             (see :ref:`Enhancing Performance <enhancingperf.numba>`)
+xarray                    0.12.3             pandas-like API for N-dimensional data
+========================= ================== =============================================================
+
+Excel files
+^^^^^^^^^^^
+
+========================= ================== =============================================================
+Dependency                Minimum Version    Notes
+========================= ================== =============================================================
+xlrd                      1.2.0              Reading Excel
+xlwt                      1.3.0              Writing Excel
+xlsxwriter                1.0.2              Writing Excel
 openpyxl                  3.0.0              Reading / writing for xlsx files
-pandas-gbq                0.12.0             Google Big Query access
-psycopg2                  2.7                PostgreSQL engine for sqlalchemy
-pyarrow                   0.15.0             Parquet, ORC, and feather reading / writing
-pymysql                   0.8.1              MySQL engine for sqlalchemy
-pyreadstat                                   SPSS files (.sav) reading
 pyxlsb                    1.0.6              Reading for xlsb files
-qtpy                                         Clipboard I/O
-s3fs                      0.4.0              Amazon S3 access
-tabulate                  0.8.7              Printing in Markdown-friendly format (see `tabulate`_)
-xarray                    0.12.3             pandas-like API for N-dimensional data
-xclip                                        Clipboard I/O on linux
-xlrd                      1.2.0              Excel reading
-xlwt                      1.3.0              Excel writing
-xsel                                         Clipboard I/O on linux
-zlib                                         Compression for HDF5
 ========================= ================== =============================================================
 
-.. _optional_html:
+HTML
+^^^^
 
-Optional dependencies for parsing HTML
-^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
+========================= ================== =============================================================
+Dependency                Minimum Version    Notes
+========================= ================== =============================================================
+BeautifulSoup4            4.6.0              HTML parser for read_html
+html5lib                  1.0.1              HTML parser for read_html
+lxml                      4.3.0              HTML parser for read_html
+========================= ================== =============================================================
 
 One of the following combinations of libraries is needed to use the
 top-level :func:`~pandas.read_html` function:
@@ -320,3 +325,52 @@ top-level :func:`~pandas.read_html` function:
 .. _BeautifulSoup4: https://www.crummy.com/software/BeautifulSoup
 .. _lxml: https://lxml.de
 .. _tabulate: https://github.com/astanin/python-tabulate
+
+SQL databases
+^^^^^^^^^^^^^
+
+========================= ================== =============================================================
+Dependency                Minimum Version    Notes
+========================= ================== =============================================================
+SQLAlchemy                1.3.0              SQL support for databases other than sqlite
+psycopg2                  2.7                PostgreSQL engine for sqlalchemy
+pymysql                   0.8.1              MySQL engine for sqlalchemy
+========================= ================== =============================================================
+
+Other data sources
+^^^^^^^^^^^^^^^^^^
+
+========================= ================== =============================================================
+Dependency                Minimum Version    Notes
+========================= ================== =============================================================
+PyTables                  3.5.1              HDF5-based reading / writing
+blosc                     1.17.0             Compression for HDF5
+zlib                                         Compression for HDF5
+fastparquet               0.4.0              Parquet reading / writing
+pyarrow                   0.15.0             Parquet, ORC, and feather reading / writing
+pyreadstat                                   SPSS files (.sav) reading
+========================= ================== =============================================================
+
+Access data in the cloud
+^^^^^^^^^^^^^^^^^^^^^^^^
+
+========================= ================== =============================================================
+Dependency                Minimum Version    Notes
+========================= ================== =============================================================
+fsspec                    0.7.4              Handling files aside from simple local and HTTP
+gcsfs                     0.6.0              Google Cloud Storage access
+pandas-gbq                0.12.0             Google Big Query access
+s3fs                      0.4.0              Amazon S3 access
+========================= ================== =============================================================
+
+Clipboard
+^^^^^^^^^
+
+========================= ================== =============================================================
+Dependency                Minimum Version    Notes
+========================= ================== =============================================================
+PyQt4/PyQt5                                  Clipboard I/O
+qtpy                                         Clipboard I/O
+xclip                                        Clipboard I/O on linux
+xsel                                         Clipboard I/O on linux
+========================= ================== =============================================================

doc/source/user_guide/cookbook.rst

@@ -1410,7 +1410,7 @@ Often it's useful to obtain the lower (or upper) triangular form of a correlatio
 
     corr_mat.where(mask)
 
-The ``method`` argument within ``DataFrame.corr`` can accept a callable in addition to the named correlation types.  Here we compute the ``distance correlation <https://en.wikipedia.org/wiki/Distance_correlation>``__ matrix for a ``DataFrame`` object.
+The ``method`` argument within ``DataFrame.corr`` can accept a callable in addition to the named correlation types.  Here we compute the `distance correlation <https://en.wikipedia.org/wiki/Distance_correlation>`__ matrix for a ``DataFrame`` object.
 
 .. ipython:: python
 

doc/source/user_guide/style.ipynb

@@ -1154,7 +1154,7 @@
    "metadata": {},
    "outputs": [],
    "source": [
-    "from IPython.html import widgets\n",
+    "from ipywidgets import widgets\n",
     "@widgets.interact\n",
     "def f(h_neg=(0, 359, 1), h_pos=(0, 359), s=(0., 99.9), l=(0., 99.9)):\n",
     "    return df.style.background_gradient(\n",

doc/source/whatsnew/v1.2.3.rst

@@ -16,7 +16,7 @@ Fixed regressions
 ~~~~~~~~~~~~~~~~~
 
 - Fixed regression in :meth:`~DataFrame.to_excel` raising ``KeyError`` when giving duplicate columns with ``columns`` attribute (:issue:`39695`)
--
+- Fixed regression in :meth:`DataFrame.__setitem__` not aligning :class:`DataFrame` on right-hand side for boolean indexer (:issue:`39931`)
 
 .. ---------------------------------------------------------------------------
 

doc/source/whatsnew/v1.3.0.rst

@@ -482,6 +482,7 @@ Other
 - Bug in :class:`Styler` where rendered HTML was missing a column class identifier for certain header cells (:issue:`39716`)
 - Bug in :meth:`Styler.background_gradient` where text-color was not determined correctly (:issue:`39888`)
 - Bug in :meth:`DataFrame.equals`, :meth:`Series.equals`, :meth:`Index.equals` with object-dtype containing ``np.datetime64("NaT")`` or ``np.timedelta64("NaT")`` (:issue:`39650`)
+- Bug in :func:`pandas.util.show_versions` where console JSON output was not proper JSON (:issue:`39701`)
 
 
 .. ---------------------------------------------------------------------------

pandas/core/frame.py

@@ -3264,6 +3264,9 @@ def _setitem_array(self, key, value):
             key = check_bool_indexer(self.index, key)
             indexer = key.nonzero()[0]
             self._check_setitem_copy()
+            if isinstance(value, DataFrame):
+                # GH#39931 reindex since iloc does not align
+                value = value.reindex(self.index.take(indexer))
             self.iloc[indexer] = value
         else:
             if isinstance(value, DataFrame):

pandas/core/internals/blocks.py

@@ -17,7 +17,6 @@
 
 from pandas._libs import (
     Interval,
-    NaT,
     Period,
     Timestamp,
     algos as libalgos,
@@ -45,6 +44,7 @@
     maybe_downcast_numeric,
     maybe_downcast_to_dtype,
     maybe_upcast,
+    sanitize_to_nanoseconds,
     soft_convert_objects,
 )
 from pandas.core.dtypes.common import (
@@ -72,6 +72,7 @@
 from pandas.core.dtypes.missing import (
     is_valid_na_for_dtype,
     isna,
+    na_value_for_dtype,
 )
 
 import pandas.core.algorithms as algos
@@ -95,11 +96,13 @@
     DatetimeArray,
     ExtensionArray,
     PandasArray,
-    TimedeltaArray,
 )
 from pandas.core.base import PandasObject
 import pandas.core.common as com
-from pandas.core.construction import extract_array
+from pandas.core.construction import (
+    ensure_wrapped_if_datetimelike,
+    extract_array,
+)
 from pandas.core.indexers import (
     check_setitem_lengths,
     is_empty_indexer,
@@ -2095,8 +2098,6 @@ class DatetimeLikeBlockMixin(NDArrayBackedExtensionBlock):
 
     is_numeric = False
     _can_hold_na = True
-    _dtype: np.dtype
-    _holder: Type[Union[DatetimeArray, TimedeltaArray]]
 
     @classmethod
     def _maybe_coerce_values(cls, values):
@@ -2112,25 +2113,26 @@ def _maybe_coerce_values(cls, values):
         Returns
         -------
         values : ndarray[datetime64ns/timedelta64ns]
-
-        Overridden by DatetimeTZBlock.
         """
-        if values.dtype != cls._dtype:
-            # non-nano we will convert to nano
-            if values.dtype.kind != cls._dtype.kind:
-                # caller is responsible for ensuring td64/dt64 dtype
-                raise TypeError(values.dtype)  # pragma: no cover
-
-            values = cls._holder._from_sequence(values)._data
-
-        if isinstance(values, cls._holder):
+        values = extract_array(values, extract_numpy=True)
+        if isinstance(values, np.ndarray):
+            values = sanitize_to_nanoseconds(values)
+        elif isinstance(values.dtype, np.dtype):
+            # i.e. not datetime64tz
             values = values._data
 
-        assert isinstance(values, np.ndarray), type(values)
         return values
 
     def array_values(self):
-        return self._holder._simple_new(self.values)
+        return ensure_wrapped_if_datetimelike(self.values)
+
+    @property
+    def _holder(self):
+        return type(self.array_values())
+
+    @property
+    def fill_value(self):
+        return na_value_for_dtype(self.dtype)
 
     def to_native_types(self, na_rep="NaT", **kwargs):
         """ convert to our native types format """
@@ -2142,9 +2144,6 @@ def to_native_types(self, na_rep="NaT", **kwargs):
 
 class DatetimeBlock(DatetimeLikeBlockMixin):
     __slots__ = ()
-    fill_value = np.datetime64("NaT", "ns")
-    _dtype = fill_value.dtype
-    _holder = DatetimeArray
 
     def set_inplace(self, locs, values):
         """
@@ -2165,42 +2164,16 @@ class DatetimeTZBlock(ExtensionBlock, DatetimeBlock):
     _can_hold_na = True
     is_numeric = False
 
-    _holder = DatetimeArray
-
     internal_values = Block.internal_values
     _can_hold_element = DatetimeBlock._can_hold_element
     to_native_types = DatetimeBlock.to_native_types
     diff = DatetimeBlock.diff
-    fill_value = NaT
     where = DatetimeBlock.where
     putmask = DatetimeLikeBlockMixin.putmask
     fillna = DatetimeLikeBlockMixin.fillna
 
     array_values = ExtensionBlock.array_values
 
-    @classmethod
-    def _maybe_coerce_values(cls, values):
-        """
-        Input validation for values passed to __init__. Ensure that
-        we have datetime64TZ, coercing if necessary.
-
-        Parameters
-        ----------
-        values : array-like
-            Must be convertible to datetime64
-
-        Returns
-        -------
-        values : DatetimeArray
-        """
-        if not isinstance(values, cls._holder):
-            values = cls._holder(values)
-
-        if values.tz is None:
-            raise ValueError("cannot create a DatetimeTZBlock without a tz")
-
-        return values
-
     @property
     def is_view(self) -> bool:
         """ return a boolean if I am possibly a view """
@@ -2216,9 +2189,6 @@ def external_values(self):
 
 class TimeDeltaBlock(DatetimeLikeBlockMixin):
     __slots__ = ()
-    _holder = TimedeltaArray
-    fill_value = np.timedelta64("NaT", "ns")
-    _dtype = fill_value.dtype
 
 
 class ObjectBlock(Block):

pandas/io/clipboard/__init__.py

@@ -66,7 +66,7 @@
 EXCEPT_MSG = """
     Pyperclip could not find a copy/paste mechanism for your system.
     For more information, please visit
-    https://pyperclip.readthedocs.io/en/latest/introduction.html#not-implemented-error
+    https://pyperclip.readthedocs.io/en/latest/#not-implemented-error
     """
 
 ENCODING = "utf-8"

pandas/tests/frame/indexing/test_setitem.py

@@ -575,3 +575,12 @@ def test_setitem_boolean_mask(self, mask_type, float_frame):
         expected = df.copy()
         expected.values[np.array(mask)] = np.nan
         tm.assert_frame_equal(result, expected)
+
+    @pytest.mark.parametrize("indexer", [tm.setitem, tm.loc])
+    def test_setitem_boolean_mask_aligning(self, indexer):
+        # GH#39931
+        df = DataFrame({"a": [1, 4, 2, 3], "b": [5, 6, 7, 8]})
+        expected = df.copy()
+        mask = df["a"] >= 3
+        indexer(df)[mask] = indexer(df)[mask].sort_values("a")
+        tm.assert_frame_equal(df, expected)

pandas/tests/frame/methods/test_astype.py

@@ -648,3 +648,32 @@ def test_astype_bytes(self):
         # GH#39474
         result = DataFrame(["foo", "bar", "baz"]).astype(bytes)
         assert result.dtypes[0] == np.dtype("S3")
+
+
+class TestAstypeCategorical:
+    def test_astype_from_categorical3(self):
+        df = DataFrame({"cats": [1, 2, 3, 4, 5, 6], "vals": [1, 2, 3, 4, 5, 6]})
+        cats = Categorical([1, 2, 3, 4, 5, 6])
+        exp_df = DataFrame({"cats": cats, "vals": [1, 2, 3, 4, 5, 6]})
+        df["cats"] = df["cats"].astype("category")
+        tm.assert_frame_equal(exp_df, df)
+
+    def test_astype_from_categorical4(self):
+        df = DataFrame(
+            {"cats": ["a", "b", "b", "a", "a", "d"], "vals": [1, 2, 3, 4, 5, 6]}
+        )
+        cats = Categorical(["a", "b", "b", "a", "a", "d"])
+        exp_df = DataFrame({"cats": cats, "vals": [1, 2, 3, 4, 5, 6]})
+        df["cats"] = df["cats"].astype("category")
+        tm.assert_frame_equal(exp_df, df)
+
+    def test_categorical_astype_to_int(self, any_int_or_nullable_int_dtype):
+        # GH#39402
+
+        df = DataFrame(data={"col1": pd.array([2.0, 1.0, 3.0])})
+        df.col1 = df.col1.astype("category")
+        df.col1 = df.col1.astype(any_int_or_nullable_int_dtype)
+        expected = DataFrame(
+            {"col1": pd.array([2, 1, 3], dtype=any_int_or_nullable_int_dtype)}
+        )
+        tm.assert_frame_equal(df, expected)