[SPARK-35176][PYTHON] Standardize input validation error type

python/docs/source/migration_guide/index.rst

@@ -25,6 +25,7 @@ This page describes the migration guide specific to PySpark.
 .. toctree::
    :maxdepth: 2
 
+   pyspark_3.1_to_3.2
    pyspark_2.4_to_3.0
    pyspark_2.3_to_2.4
    pyspark_2.3.0_to_2.3.1_above

python/docs/source/migration_guide/pyspark_3.1_to_3.2.rst

@@ -0,0 +1,23 @@
+..  Licensed to the Apache Software Foundation (ASF) under one
+    or more contributor license agreements.  See the NOTICE file
+    distributed with this work for additional information
+    regarding copyright ownership.  The ASF licenses this file
+    to you under the Apache License, Version 2.0 (the
+    "License"); you may not use this file except in compliance
+    with the License.  You may obtain a copy of the License at
+
+..    http://www.apache.org/licenses/LICENSE-2.0
+
+..  Unless required by applicable law or agreed to in writing,
+    software distributed under the License is distributed on an
+    "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+    KIND, either express or implied.  See the License for the
+    specific language governing permissions and limitations
+    under the License.
+
+
+=================================
+Upgrading from PySpark 3.1 to 3.2
+=================================
+
+* In Spark 3.2, the PySpark methods from sql, ml, spark_on_pandas modules raise the ``TypeError`` instead of ``ValueError`` when are applied to an param of inappropriate type.

python/pyspark/ml/base.py

@@ -160,8 +160,8 @@ def fit(self, dataset, params=None):
             else:
                 return self._fit(dataset)
         else:
-            raise ValueError("Params must be either a param map or a list/tuple of param maps, "
-                             "but got %s." % type(params))
+            raise TypeError("Params must be either a param map or a list/tuple of param maps, "
+                            "but got %s." % type(params))
 
 
 @inherit_doc
@@ -216,7 +216,7 @@ def transform(self, dataset, params=None):
             else:
                 return self._transform(dataset)
         else:
-            raise ValueError("Params must be a param map but got %s." % type(params))
+            raise TypeError("Params must be a param map but got %s." % type(params))
 
 
 @inherit_doc

python/pyspark/ml/classification.py

@@ -759,7 +759,7 @@ def evaluate(self, dataset):
             Test dataset to evaluate model on.
         """
         if not isinstance(dataset, DataFrame):
-            raise ValueError("dataset must be a DataFrame but got %s." % type(dataset))
+            raise TypeError("dataset must be a DataFrame but got %s." % type(dataset))
         java_lsvc_summary = self._call_java("evaluate", dataset)
         return LinearSVCSummary(java_lsvc_summary)
 
@@ -1263,7 +1263,7 @@ def evaluate(self, dataset):
             Test dataset to evaluate model on.
         """
         if not isinstance(dataset, DataFrame):
-            raise ValueError("dataset must be a DataFrame but got %s." % type(dataset))
+            raise TypeError("dataset must be a DataFrame but got %s." % type(dataset))
         java_blr_summary = self._call_java("evaluate", dataset)
         if self.numClasses <= 2:
             return BinaryLogisticRegressionSummary(java_blr_summary)
@@ -1869,7 +1869,7 @@ def evaluate(self, dataset):
             Test dataset to evaluate model on.
         """
         if not isinstance(dataset, DataFrame):
-            raise ValueError("dataset must be a DataFrame but got %s." % type(dataset))
+            raise TypeError("dataset must be a DataFrame but got %s." % type(dataset))
         java_rf_summary = self._call_java("evaluate", dataset)
         if self.numClasses <= 2:
             return BinaryRandomForestClassificationSummary(java_rf_summary)
@@ -2722,7 +2722,7 @@ def evaluate(self, dataset):
             Test dataset to evaluate model on.
         """
         if not isinstance(dataset, DataFrame):
-            raise ValueError("dataset must be a DataFrame but got %s." % type(dataset))
+            raise TypeError("dataset must be a DataFrame but got %s." % type(dataset))
         java_mlp_summary = self._call_java("evaluate", dataset)
         return MultilayerPerceptronClassificationSummary(java_mlp_summary)
 
@@ -3521,7 +3521,7 @@ def evaluate(self, dataset):
             Test dataset to evaluate model on.
         """
         if not isinstance(dataset, DataFrame):
-            raise ValueError("dataset must be a DataFrame but got %s." % type(dataset))
+            raise TypeError("dataset must be a DataFrame but got %s." % type(dataset))
         java_fm_summary = self._call_java("evaluate", dataset)
         return FMClassificationSummary(java_fm_summary)
 

python/pyspark/ml/evaluation.py

@@ -83,7 +83,7 @@ def evaluate(self, dataset, params=None):
             else:
                 return self._evaluate(dataset)
         else:
-            raise ValueError("Params must be a param map but got %s." % type(params))
+            raise TypeError("Params must be a param map but got %s." % type(params))
 
     @since("1.5.0")
     def isLargerBetter(self):

python/pyspark/ml/param/__init__.py

@@ -435,7 +435,7 @@ def _resolveParam(self, param):
         elif isinstance(param, str):
             return self.getParam(param)
         else:
-            raise ValueError("Cannot resolve %r as a param." % param)
+            raise TypeError("Cannot resolve %r as a param." % param)
 
     def _testOwnParam(self, param_parent, param_name):
         """

python/pyspark/ml/regression.py

@@ -371,7 +371,7 @@ def evaluate(self, dataset):
             instance of :py:class:`pyspark.sql.DataFrame`
         """
         if not isinstance(dataset, DataFrame):
-            raise ValueError("dataset must be a DataFrame but got %s." % type(dataset))
+            raise TypeError("dataset must be a DataFrame but got %s." % type(dataset))
         java_lr_summary = self._call_java("evaluate", dataset)
         return LinearRegressionSummary(java_lr_summary)
 
@@ -2294,7 +2294,7 @@ def evaluate(self, dataset):
             instance of :py:class:`pyspark.sql.DataFrame`
         """
         if not isinstance(dataset, DataFrame):
-            raise ValueError("dataset must be a DataFrame but got %s." % type(dataset))
+            raise TypeError("dataset must be a DataFrame but got %s." % type(dataset))
         java_glr_summary = self._call_java("evaluate", dataset)
         return GeneralizedLinearRegressionSummary(java_glr_summary)
 

python/pyspark/ml/tests/test_base.py

@@ -19,7 +19,15 @@
 
 from pyspark.sql.types import DoubleType, IntegerType
 from pyspark.testing.mlutils import MockDataset, MockEstimator, MockUnaryTransformer, \
-    SparkSessionTestCase
+    MockTransformer, SparkSessionTestCase
+
+
+class TransformerTests(unittest.TestCase):
+
+    def test_transform_invalid_type(self):
+        transformer = MockTransformer()
+        data = MockDataset()
+        self.assertRaises(TypeError, transformer.transform, data, "")
 
 
 class UnaryTransformerTests(SparkSessionTestCase):
@@ -52,13 +60,18 @@ def test_unary_transformer_transform(self):
 
 
 class EstimatorTest(unittest.TestCase):
+    def setUp(self):
+        self.estimator = MockEstimator()
+        self.data = MockDataset()
+
+    def test_fit_invalid_params(self):
+        invalid_type_parms = ""
+        self.assertRaises(TypeError, self.estimator.fit, self.data, invalid_type_parms)
 
     def testDefaultFitMultiple(self):
         N = 4
-        data = MockDataset()
-        estimator = MockEstimator()
-        params = [{estimator.fake: i} for i in range(N)]
-        modelIter = estimator.fitMultiple(data, params)
+        params = [{self.estimator.fake: i} for i in range(N)]
+        modelIter = self.estimator.fitMultiple(self.data, params)
         indexList = []
         for index, model in modelIter:
             self.assertEqual(model.getFake(), index)

python/pyspark/ml/tests/test_evaluation.py

@@ -27,6 +27,12 @@
 
 class EvaluatorTests(SparkSessionTestCase):
 
+    def test_evaluate_invalid_type(self):
+        evaluator = RegressionEvaluator(metricName="r2")
+        df = self.spark.createDataFrame([Row(label=1.0, prediction=1.1)])
+        invalid_type = ""
+        self.assertRaises(TypeError, evaluator.evaluate, df, invalid_type)
+
     def test_java_params(self):
         """
         This tests a bug fixed by SPARK-18274 which causes multiple copies

python/pyspark/ml/tests/test_param.py

@@ -30,6 +30,7 @@
 from pyspark.ml.linalg import DenseVector, SparseVector, Vectors
 from pyspark.ml.param import Param, Params, TypeConverters
 from pyspark.ml.param.shared import HasInputCol, HasMaxIter, HasSeed
+from pyspark.ml.regression import LinearRegressionModel, GeneralizedLinearRegressionModel
 from pyspark.ml.wrapper import JavaParams
 from pyspark.testing.mlutils import check_params, PySparkTestCase, SparkSessionTestCase
 
@@ -197,6 +198,10 @@ def test_resolveparam(self):
         self.assertEqual(testParams._resolveParam(u"maxIter"), testParams.maxIter)
         self.assertRaises(AttributeError, lambda: testParams._resolveParam(u"아"))
 
+        # Invalid type
+        invalid_type = 1
+        self.assertRaises(TypeError, testParams._resolveParam, invalid_type)
+
     def test_params(self):
         testParams = TestParams()
         maxIter = testParams.maxIter
@@ -332,6 +337,16 @@ def test_default_params_transferred(self):
         self.assertFalse(binarizer.isSet(binarizer.outputCol))
         self.assertEqual(result[0][0], 1.0)
 
+    def test_lr_evaluate_invaild_type(self):
+        lr = LinearRegressionModel()
+        invalid_type = ""
+        self.assertRaises(TypeError, lr.evaluate, invalid_type)
+
+    def test_glr_evaluate_invaild_type(self):
+        glr = GeneralizedLinearRegressionModel()
+        invalid_type = ""
+        self.assertRaises(TypeError, glr.evaluate, invalid_type)
+
 
 class DefaultValuesTests(PySparkTestCase):
     """

python/pyspark/mllib/linalg/distributed.py

@@ -465,8 +465,7 @@ def multiply(self, matrix):
         [DenseVector([2.0, 3.0]), DenseVector([6.0, 11.0])]
         """
         if not isinstance(matrix, DenseMatrix):
-            raise ValueError("Only multiplication with DenseMatrix "
-                             "is supported.")
+            raise TypeError("Only multiplication with DenseMatrix is supported.")
         j_model = self._java_matrix_wrapper.call("multiply", matrix)
         return RowMatrix(j_model)
 
@@ -854,8 +853,7 @@ def multiply(self, matrix):
         [IndexedRow(0, [2.0,3.0]), IndexedRow(1, [6.0,11.0])]
         """
         if not isinstance(matrix, DenseMatrix):
-            raise ValueError("Only multiplication with DenseMatrix "
-                             "is supported.")
+            raise TypeError("Only multiplication with DenseMatrix is supported.")
         return IndexedRowMatrix(self._java_matrix_wrapper.call("multiply", matrix))
 
 

python/pyspark/mllib/tests/test_linalg.py

@@ -26,7 +26,7 @@
     Vector, SparseVector, DenseVector, VectorUDT, _convert_to_vector,
     DenseMatrix, SparseMatrix, Vectors, Matrices, MatrixUDT
 )
-from pyspark.mllib.linalg.distributed import RowMatrix, IndexedRowMatrix
+from pyspark.mllib.linalg.distributed import RowMatrix, IndexedRowMatrix, IndexedRow
 from pyspark.mllib.regression import LabeledPoint
 from pyspark.sql import Row
 from pyspark.testing.mllibutils import MLlibTestCase
@@ -452,6 +452,17 @@ def test_indexed_row_matrix_from_dataframe(self):
         with self.assertRaises(IllegalArgumentException):
             IndexedRowMatrix(df.drop("_1"))
 
+    def test_row_matrix_invalid_type(self):
+        rows = self.sc.parallelize([[1, 2, 3], [4, 5, 6]])
+        invalid_type = ""
+        matrix = RowMatrix(rows)
+        self.assertRaises(TypeError, matrix.multiply, invalid_type)
+
+        irows = self.sc.parallelize([IndexedRow(0, [1, 2, 3]),
+                                     IndexedRow(1, [4, 5, 6])])
+        imatrix = IndexedRowMatrix(irows)
+        self.assertRaises(TypeError, imatrix.multiply, invalid_type)
+
 
 class MatrixUDTTests(MLlibTestCase):
 

python/pyspark/pandas/base.py

@@ -1498,7 +1498,7 @@ def shift(self, periods=1, fill_value=None) -> Union["Series", "Index"]:
 
     def _shift(self, periods, fill_value, *, part_cols=()):
         if not isinstance(periods, int):
-            raise ValueError("periods should be an int; however, got [%s]" % type(periods).__name__)
+            raise TypeError("periods should be an int; however, got [%s]" % type(periods).__name__)
 
         col = self.spark.column
         window = (
@@ -1828,7 +1828,7 @@ def take(self, indices) -> Union["Series", "Index"]:
                    )
         """
         if not is_list_like(indices) or isinstance(indices, (dict, set)):
-            raise ValueError("`indices` must be a list-like except dict or set")
+            raise TypeError("`indices` must be a list-like except dict or set")
         if isinstance(self, ps.Series):
             return cast(ps.Series, self.iloc[indices])
         else:

python/pyspark/pandas/config.py

@@ -70,7 +70,7 @@ class Option:
     >>> option.validate('abc')  # doctest: +NORMALIZE_WHITESPACE
     Traceback (most recent call last):
       ...
-    ValueError: The value for option 'option.name' was <class 'str'>;
+    TypeError: The value for option 'option.name' was <class 'str'>;
     however, expected types are [(<class 'float'>, <class 'int'>)].
 
     >>> option.validate(-1.1)
@@ -101,7 +101,7 @@ def validate(self, v: Any) -> None:
         Validate the given value and throw an exception with related information such as key.
         """
         if not isinstance(v, self.types):
-            raise ValueError(
+            raise TypeError(
                 "The value for option '%s' was %s; however, expected types are "
                 "[%s]." % (self.key, type(v), str(self.types))
             )