Check if pip installed when finding the shell file

bin/pyspark

@@ -61,7 +61,12 @@ export PYTHONPATH="${SPARK_HOME}/python/lib/py4j-0.10.3-src.zip:$PYTHONPATH"
 
 # Load the PySpark shell.py script when ./pyspark is used interactively:
 export OLD_PYTHONSTARTUP="$PYTHONSTARTUP"
-export PYTHONSTARTUP="${SPARK_HOME}/python/pyspark/shell.py"
+# Check if we are pip installed or otherwise
+if [ -d ${SPARK_HOME}/python/pyspark ]; then
+  export PYTHONSTARTUP="${SPARK_HOME}/python/pyspark/shell.py"
+else
+    export PYTHONSTARTUP="${SPARK_HOME}/shell.py"
+fi
 
 # For pyspark tests
 if [[ -n "$SPARK_TESTING" ]]; then

bin/pyspark2.cmd

@@ -33,6 +33,11 @@ set PYTHONPATH=%SPARK_HOME%\python;%PYTHONPATH%
 set PYTHONPATH=%SPARK_HOME%\python\lib\py4j-0.10.3-src.zip;%PYTHONPATH%
 
 set OLD_PYTHONSTARTUP=%PYTHONSTARTUP%
+rem Check if we are pip installed or not
+IF EXIST %SPARK_HOME%\python\pyspark\shell.py (
 set PYTHONSTARTUP=%SPARK_HOME%\python\pyspark\shell.py
+) ELSE (
+set PYTHONSTARTUP=%SPARK_HOME%\shell.py
+)
 
 call "%SPARK_HOME%\bin\spark-submit2.cmd" pyspark-shell-main --name "PySparkShell" %*