[SPARK-53602][PYTHON] Profile dump improvement and profiler doc fix

### What changes were proposed in this pull request?
- Avoid race condition when creating directory of profile dump
- Fix profiler docs

### Why are the changes needed?
Race-safe dump and better docs.

### Does this PR introduce _any_ user-facing change?
No.

### How was this patch tested?
Existing tests

### Was this patch authored or co-authored using generative AI tooling?
No

Closes #52360 from xinrong-meng/profile_impr.

Authored-by: Xinrong Meng <[email protected]>
Signed-off-by: Hyukjin Kwon <[email protected]>

Author: xinrong-meng

python/pyspark/sql/profiler.py

@@ -194,8 +194,7 @@ def dump(id: int) -> None:
             s = stats.get(id)
 
             if s is not None:
-                if not os.path.exists(path):
-                    os.makedirs(path)
+                os.makedirs(path, exist_ok=True)
                 p = os.path.join(path, f"udf_{id}_perf.pstats")
                 s.dump_stats(p)
 
@@ -231,8 +230,7 @@ def dump(id: int) -> None:
             cm = code_map.get(id)
 
             if cm is not None:
-                if not os.path.exists(path):
-                    os.makedirs(path)
+                os.makedirs(path, exist_ok=True)
                 p = os.path.join(path, f"udf_{id}_memory.txt")
 
                 with open(p, "w+") as f:
@@ -316,7 +314,7 @@ class Profile:
     """User-facing profile API. This instance can be accessed by
     :attr:`spark.profile`.
 
-    .. versionadded: 4.0.0
+    .. versionadded:: 4.0.0
     """
 
     def __init__(self, profiler_collector: ProfilerCollector):
@@ -421,7 +419,8 @@ def render(
         id : int
             The UDF ID whose profiling results should be rendered.
         type : str, optional
-            The profiler type to clear results for, which can be either "perf" or "memory".
+            The profiler type to render results for, which can be either "perf" or "memory".
+            If not specified, defaults to "perf".
         renderer : str or callable, optional
             The renderer to use. If not specified, the default renderer will be "flameprof"
             for the "perf" profiler, which returns an :class:`IPython.display.HTML` object in

python/pyspark/sql/session.py

@@ -950,13 +950,13 @@ def dataSource(self) -> "DataSourceRegistration":
 
     @property
     def profile(self) -> Profile:
-        """Returns a :class:`Profile` for performance/memory profiling.
+        """Returns a :class:`pyspark.sql.profile.Profile` for performance/memory profiling.
 
         .. versionadded:: 4.0.0
 
         Returns
         -------
-        :class:`Profile`
+        :class:`pyspark.sql.profile.Profile`
 
         Notes
         -----