Try and unify path used for shell script file, add a README.md file f…

…or Python and use it for long_description in PyPi

bin/pyspark

@@ -61,12 +61,7 @@ export PYTHONPATH="${SPARK_HOME}/python/lib/py4j-0.10.4-src.zip:$PYTHONPATH"
 
 # Load the PySpark shell.py script when ./pyspark is used interactively:
 export OLD_PYTHONSTARTUP="$PYTHONSTARTUP"
-# Check if we are pip installed or otherwise
-if [ -d ${SPARK_HOME}/python/pyspark ]; then
-  export PYTHONSTARTUP="${SPARK_HOME}/python/pyspark/shell.py"
-else
-    export PYTHONSTARTUP="${SPARK_HOME}/shell.py"
-fi
+export PYTHONSTARTUP="${SPARK_HOME}/python/pyspark/shell.py"
 
 # For pyspark tests
 if [[ -n "$SPARK_TESTING" ]]; then

bin/pyspark2.cmd

@@ -34,10 +34,6 @@ set PYTHONPATH=%SPARK_HOME%\python\lib\py4j-0.10.4-src.zip;%PYTHONPATH%
 
 set OLD_PYTHONSTARTUP=%PYTHONSTARTUP%
 rem Check if we are pip installed or not
-IF EXIST %SPARK_HOME%\python\pyspark\shell.py (
 set PYTHONSTARTUP=%SPARK_HOME%\python\pyspark\shell.py
-) ELSE (
-set PYTHONSTARTUP=%SPARK_HOME%\shell.py
-)
 
 call "%SPARK_HOME%\bin\spark-submit2.cmd" pyspark-shell-main --name "PySparkShell" %*

python/MANIFEST.in

@@ -2,3 +2,4 @@ recursive-include deps/jars *.jar
 recursive-include deps/bin *
 recursive-include deps/examples *.py
 recursive-include lib *.zip
+inlcude README.md

python/setup.py

@@ -15,6 +15,9 @@
 EXAMPLES_TARGET = "%s/examples" % TEMP_PATH
 
 # Check and see if we are under the spark path in which case we need to build the symlink farm.
+# This is important because we only want to build the symlink farm while under Spark otherwise we
+# want to use the symlink farm. And if the symlink farm exists under while under Spark (e.g. a
+# partially built sdist) we should error and have the user sort it out.
 in_spark = os.path.isfile("../core/src/main/scala/org/apache/spark/SparkContext.scala")
 if (in_spark):
     # Construct links for setup
@@ -26,13 +29,29 @@
 
 try:
     if (in_spark):
+        # Construct the symlink farm
         os.symlink(JARS_PATH, JARS_TARGET)
         os.symlink(SCRIPTS_PATH, SCRIPTS_TARGET)
         os.symlink(EXAMPLES_PATH, EXAMPLES_TARGET)
+        # Parse the README markdown file into rst for PyPi
+        try:
+            import pypandoc
+            long_description = pypandoc.convert('README.md', 'rst')
+        except ImportError:
+            print("Could not import pypandoc - required to package PySpark", file=sys.stderr)
+            long_description = "!!!!! missing pandoc do not upload to PyPi !!!!"
     else:
         # We add find_spark_home.py to the bin directory we install so that pip installed PySpark
         # will search for SPARK_HOME with Python.
+        # We only do this copy when we aren't inside of Spark (e.g. the packaging tool has copied
+        # all the files into a temp directory) since otherwise the copy would go into the symlinked
+        # directory.
         copyfile("pyspark/find_spark_home.py", SCRIPTS_TARGET + "/find_spark_home.py")
+        # We copy the shell script to be under pyspark/python/pyspark so that the launcher scripts
+        # find it where expected. The rest of the files aren't copied because they are accessed
+        # using Python imports instead which will be resolved correctly.
+        os.makedirs("pyspark/python/pyspark")
+        copyfile("pyspark/shell.py", "pyspark/python/pyspark/shell.py")
 
     if not os.path.isdir(SCRIPTS_TARGET):
         print("For packaging reasons you must first create a source dist and install that source dist.", file=sys.stderr)
@@ -46,6 +65,7 @@
         name='pyspark',
         version=VERSION,
         description='Apache Spark Python API',
+        long_description=long_description,
         author='Spark Developers',
         author_email='[email protected]',
         url='https://github.com/apache/spark/tree/master/python',
@@ -73,6 +93,7 @@
         scripts=scripts,
         license='http://www.apache.org/licenses/LICENSE-2.0',
         install_requires=['py4j==0.10.4'],
+        setup_requires=['pypandoc'],
         extras_require={
             'ml': ['numpy>=1.7'],
             'mllib': ['numpy<=1.7'],