[SPARK-40789][PYTHON][TESTS] Separate tests under pyspark.sql.tests

.github/labeler.yml

@@ -84,12 +84,12 @@ SPARK SHELL:
   - "repl/**/*"
   - "bin/spark-shell*"
 SQL:
-#- any: ["**/sql/**/*", "!python/pyspark/sql/avro/**/*", "!python/pyspark/sql/streaming/**/*", "!python/pyspark/sql/tests/test_streaming.py"]
+#- any: ["**/sql/**/*", "!python/pyspark/sql/avro/**/*", "!python/pyspark/sql/streaming/**/*", "!python/pyspark/sql/tests/streaming/test_streaming.py"]
   - "**/sql/**/*"
   - "common/unsafe/**/*"
   #- "!python/pyspark/sql/avro/**/*"
   #- "!python/pyspark/sql/streaming/**/*"
-  #- "!python/pyspark/sql/tests/test_streaming.py"
+  #- "!python/pyspark/sql/tests/streaming/test_streaming.py"
   - "bin/spark-sql*"
   - "bin/beeline*"
   - "sbin/*thriftserver*.sh"
@@ -125,7 +125,7 @@ STRUCTURED STREAMING:
   - "**/sql/**/streaming/**/*"
   - "connector/kafka-0-10-sql/**/*"
   - "python/pyspark/sql/streaming/**/*"
-  - "python/pyspark/sql/tests/test_streaming.py"
+  - "python/pyspark/sql/tests/streaming/test_streaming.py"
   - "**/*streaming.R"
 PYTHON:
   - "bin/pyspark*"
@@ -156,5 +156,5 @@ CONNECT:
   - "**/sql/sparkconnect/**/*"
   - "python/pyspark/sql/**/connect/**/*"
 PROTOBUF:
- - "connector/protobuf/**/*"
- - "python/pyspark/sql/protobuf/**/*"
+  - "connector/protobuf/**/*"
+  - "python/pyspark/sql/protobuf/**/*"

dev/sparktestsupport/modules.py

@@ -461,22 +461,22 @@ def __hash__(self):
         "pyspark.sql.tests.test_datasources",
         "pyspark.sql.tests.test_functions",
         "pyspark.sql.tests.test_group",
-        "pyspark.sql.tests.test_pandas_cogrouped_map",
-        "pyspark.sql.tests.test_pandas_grouped_map",
-        "pyspark.sql.tests.test_pandas_grouped_map_with_state",
-        "pyspark.sql.tests.test_pandas_map",
+        "pyspark.sql.tests.pandas.test_pandas_cogrouped_map",
+        "pyspark.sql.tests.pandas.test_pandas_grouped_map",
+        "pyspark.sql.tests.pandas.test_pandas_grouped_map_with_state",
+        "pyspark.sql.tests.pandas.test_pandas_map",
         "pyspark.sql.tests.test_arrow_map",
-        "pyspark.sql.tests.test_pandas_udf",
-        "pyspark.sql.tests.test_pandas_udf_grouped_agg",
-        "pyspark.sql.tests.test_pandas_udf_scalar",
-        "pyspark.sql.tests.test_pandas_udf_typehints",
-        "pyspark.sql.tests.test_pandas_udf_typehints_with_future_annotations",
-        "pyspark.sql.tests.test_pandas_udf_window",
+        "pyspark.sql.tests.pandas.test_pandas_udf",
+        "pyspark.sql.tests.pandas.test_pandas_udf_grouped_agg",
+        "pyspark.sql.tests.pandas.test_pandas_udf_scalar",
+        "pyspark.sql.tests.pandas.test_pandas_udf_typehints",
+        "pyspark.sql.tests.pandas.test_pandas_udf_typehints_with_future_annotations",
+        "pyspark.sql.tests.pandas.test_pandas_udf_window",
         "pyspark.sql.tests.test_readwriter",
         "pyspark.sql.tests.test_serde",
         "pyspark.sql.tests.test_session",
-        "pyspark.sql.tests.test_streaming",
-        "pyspark.sql.tests.test_streaming_listener",
+        "pyspark.sql.tests.streaming.test_streaming",
+        "pyspark.sql.tests.streaming.test_streaming_listener",
         "pyspark.sql.tests.test_types",
         "pyspark.sql.tests.test_udf",
         "pyspark.sql.tests.test_udf_profiler",
@@ -492,10 +492,10 @@ def __hash__(self):
         # doctests
         # No doctests yet.
         # unittests
-        "pyspark.sql.tests.test_connect_column_expressions",
-        "pyspark.sql.tests.test_connect_plan_only",
-        "pyspark.sql.tests.test_connect_select_ops",
-        "pyspark.sql.tests.test_connect_basic",
+        "pyspark.sql.tests.connect.test_connect_column_expressions",
+        "pyspark.sql.tests.connect.test_connect_plan_only",
+        "pyspark.sql.tests.connect.test_connect_select_ops",
+        "pyspark.sql.tests.connect.test_connect_basic",
     ],
     excluded_python_implementations=[
         "PyPy"  # Skip these tests under PyPy since they require numpy, pandas, and pyarrow and

python/pyspark/sql/connect/README.md

@@ -46,6 +46,6 @@ To use the release version of Spark Connect:
 ## Run Tests
 
 ```bash
-./python/run-tests --testnames 'pyspark.sql.tests.test_connect_basic'
+./python/run-tests --testnames 'pyspark.sql.tests.connect.test_connect_basic'
 ```
 

python/pyspark/sql/tests/connect/__init__.py

@@ -0,0 +1,16 @@
+#
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements.  See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License.  You may obtain a copy of the License at
+#
+#    http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#

python/pyspark/sql/tests/test_connect_basic.py → python/pyspark/sql/tests/connect/test_connect_basic.py

@@ -81,7 +81,7 @@ def test_simple_explain_string(self):
 
 
 if __name__ == "__main__":
-    from pyspark.sql.tests.test_connect_basic import *  # noqa: F401
+    from pyspark.sql.tests.connect.test_connect_basic import *  # noqa: F401
 
     try:
         import xmlrunner  # type: ignore

python/pyspark/sql/tests/test_connect_column_expressions.py → python/pyspark/sql/tests/connect/test_connect_column_expressions.py

@@ -54,7 +54,7 @@ def test_column_literals(self):
 
 if __name__ == "__main__":
     import unittest
-    from pyspark.sql.tests.test_connect_column_expressions import *  # noqa: F401
+    from pyspark.sql.tests.connect.test_connect_column_expressions import *  # noqa: F401
 
     try:
         import xmlrunner  # type: ignore

python/pyspark/sql/tests/test_connect_plan_only.py → python/pyspark/sql/tests/connect/test_connect_plan_only.py

@@ -64,7 +64,7 @@ def read_table(x):
 
 
 if __name__ == "__main__":
-    from pyspark.sql.tests.test_connect_plan_only import *  # noqa: F401
+    from pyspark.sql.tests.connect.test_connect_plan_only import *  # noqa: F401
 
     try:
         import xmlrunner  # type: ignore

python/pyspark/sql/tests/test_connect_select_ops.py → python/pyspark/sql/tests/connect/test_connect_select_ops.py

@@ -29,7 +29,7 @@ def test_select_with_literal(self):
 
 if __name__ == "__main__":
     import unittest
-    from pyspark.sql.tests.test_connect_select_ops import *  # noqa: F401
+    from pyspark.sql.tests.connect.test_connect_select_ops import *  # noqa: F401
 
     try:
         import xmlrunner  # type: ignore

python/pyspark/sql/tests/pandas/__init__.py

@@ -0,0 +1,16 @@
+#
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements.  See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License.  You may obtain a copy of the License at
+#
+#    http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#

python/pyspark/sql/tests/test_pandas_cogrouped_map.py → python/pyspark/sql/tests/pandas/test_pandas_cogrouped_map.py

@@ -404,7 +404,7 @@ def merge_pandas(lft, rgt):
 
 
 if __name__ == "__main__":
-    from pyspark.sql.tests.test_pandas_cogrouped_map import *  # noqa: F401
+    from pyspark.sql.tests.pandas.test_pandas_cogrouped_map import *  # noqa: F401
 
     try:
         import xmlrunner  # type: ignore[import]

python/pyspark/sql/tests/test_pandas_grouped_map.py → python/pyspark/sql/tests/pandas/test_pandas_grouped_map.py

@@ -741,7 +741,7 @@ def my_pandas_udf(pdf):
 
 
 if __name__ == "__main__":
-    from pyspark.sql.tests.test_pandas_grouped_map import *  # noqa: F401
+    from pyspark.sql.tests.pandas.test_pandas_grouped_map import *  # noqa: F401
 
     try:
         import xmlrunner  # type: ignore[import]

python/pyspark/sql/tests/test_pandas_grouped_map_with_state.py → python/pyspark/sql/tests/pandas/test_pandas_grouped_map_with_state.py

@@ -237,7 +237,7 @@ def assert_test():
 
 
 if __name__ == "__main__":
-    from pyspark.sql.tests.test_pandas_grouped_map_with_state import *  # noqa: F401
+    from pyspark.sql.tests.pandas.test_pandas_grouped_map_with_state import *  # noqa: F401
 
     try:
         import xmlrunner  # type: ignore[import]

python/pyspark/sql/tests/test_pandas_map.py → python/pyspark/sql/tests/pandas/test_pandas_map.py

@@ -204,7 +204,7 @@ def func(iterator):
 
 
 if __name__ == "__main__":
-    from pyspark.sql.tests.test_pandas_map import *  # noqa: F401
+    from pyspark.sql.tests.pandas.test_pandas_map import *  # noqa: F401
 
     try:
         import xmlrunner  # type: ignore[import]

python/pyspark/sql/tests/test_pandas_udf.py → python/pyspark/sql/tests/pandas/test_pandas_udf.py

@@ -293,7 +293,7 @@ def noop(s: pd.Series) -> pd.Series:
 
 
 if __name__ == "__main__":
-    from pyspark.sql.tests.test_pandas_udf import *  # noqa: F401
+    from pyspark.sql.tests.pandas.test_pandas_udf import *  # noqa: F401
 
     try:
         import xmlrunner  # type: ignore[import]

python/pyspark/sql/tests/test_pandas_udf_grouped_agg.py → python/pyspark/sql/tests/pandas/test_pandas_udf_grouped_agg.py

@@ -548,7 +548,7 @@ def mean(x):
 
 
 if __name__ == "__main__":
-    from pyspark.sql.tests.test_pandas_udf_grouped_agg import *  # noqa: F401
+    from pyspark.sql.tests.pandas.test_pandas_udf_grouped_agg import *  # noqa: F401
 
     try:
         import xmlrunner  # type: ignore[import]

python/pyspark/sql/tests/test_pandas_udf_scalar.py → python/pyspark/sql/tests/pandas/test_pandas_udf_scalar.py

@@ -1330,7 +1330,7 @@ def udf(x):
 
 
 if __name__ == "__main__":
-    from pyspark.sql.tests.test_pandas_udf_scalar import *  # noqa: F401
+    from pyspark.sql.tests.pandas.test_pandas_udf_scalar import *  # noqa: F401
 
     try:
         import xmlrunner  # type: ignore[import]

python/pyspark/sql/tests/test_pandas_udf_typehints.py → python/pyspark/sql/tests/pandas/test_pandas_udf_typehints.py

@@ -357,7 +357,7 @@ def func(col: "Union[pd.Series, pd.DataFrame]", *, col2: "pd.DataFrame") -> "pd.
 
 
 if __name__ == "__main__":
-    from pyspark.sql.tests.test_pandas_udf_typehints import *  # noqa: #401
+    from pyspark.sql.tests.pandas.test_pandas_udf_typehints import *  # noqa: #401
 
     try:
         import xmlrunner  # type: ignore[import]

python/pyspark/sql/tests/test_pandas_udf_typehints_with_future_annotations.py → python/pyspark/sql/tests/pandas/test_pandas_udf_typehints_with_future_annotations.py

@@ -364,7 +364,7 @@ def func(col: "Union[pd.Series, pd.DataFrame]", *, col2: "pd.DataFrame") -> "pd.
 
 
 if __name__ == "__main__":
-    from pyspark.sql.tests.test_pandas_udf_typehints_with_future_annotations import *  # noqa: #401
+    from pyspark.sql.tests.pandas.test_pandas_udf_typehints_with_future_annotations import *  # noqa: #401
 
     try:
         import xmlrunner  # type: ignore[import]

python/pyspark/sql/tests/test_pandas_udf_window.py → python/pyspark/sql/tests/pandas/test_pandas_udf_window.py

@@ -395,7 +395,7 @@ def test_bounded_mixed(self):
 
 
 if __name__ == "__main__":
-    from pyspark.sql.tests.test_pandas_udf_window import *  # noqa: F401
+    from pyspark.sql.tests.pandas.test_pandas_udf_window import *  # noqa: F401
 
     try:
         import xmlrunner  # type: ignore[import]

python/pyspark/sql/tests/streaming/__init__.py

@@ -0,0 +1,16 @@
+#
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements.  See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License.  You may obtain a copy of the License at
+#
+#    http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#

python/pyspark/sql/tests/test_streaming.py → python/pyspark/sql/tests/streaming/test_streaming.py

@@ -629,7 +629,7 @@ def test_streaming_write_to_table(self):
 
 if __name__ == "__main__":
     import unittest
-    from pyspark.sql.tests.test_streaming import *  # noqa: F401
+    from pyspark.sql.tests.streaming.test_streaming import *  # noqa: F401
 
     try:
         import xmlrunner  # type: ignore[import]

python/pyspark/sql/tests/test_streaming_listener.py → python/pyspark/sql/tests/streaming/test_streaming_listener.py

@@ -296,7 +296,7 @@ def onQueryTerminated(self, event):
 if __name__ == "__main__":
     import unittest
 
-    from pyspark.sql.tests.test_streaming_listener import *  # noqa: F401
+    from pyspark.sql.tests.streaming.test_streaming_listener import *  # noqa: F401
 
     try:
         import xmlrunner  # type: ignore[import]