Skip to content
Closed
Changes from 1 commit
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
Prev Previous commit
Next Next commit
bin/pyspark for Windows
This is tested on Windows 7. The only thing that does not work is
having perverse app names with escaped quotes (e.g. "foo \" jam").
App names with spaces are fine.

Other than that, the behavior on Windows is the same on OSX.
  • Loading branch information
andrewor14 committed May 17, 2014
commit 01066fa6ce4a61097c75f3611c38a87d16db9060
21 changes: 18 additions & 3 deletions bin/pyspark2.cmd
Original file line number Diff line number Diff line change
Expand Up @@ -31,7 +31,7 @@ set FOUND_JAR=0
for %%d in ("%FWDIR%assembly\target\scala-%SCALA_VERSION%\spark-assembly*hadoop*.jar") do (
set FOUND_JAR=1
)
if "%FOUND_JAR%"=="0" (
if [%FOUND_JAR%] == [0] (
echo Failed to find Spark assembly JAR.
echo You need to build Spark with sbt\sbt assembly before running this program.
goto exit
Expand All @@ -42,15 +42,30 @@ rem Load environment variables from conf\spark-env.cmd, if it exists
if exist "%FWDIR%conf\spark-env.cmd" call "%FWDIR%conf\spark-env.cmd"

rem Figure out which Python to use.
if "x%PYSPARK_PYTHON%"=="x" set PYSPARK_PYTHON=python
if [%PYSPARK_PYTHON%] == [] set PYSPARK_PYTHON=python

set PYTHONPATH=%FWDIR%python;%PYTHONPATH%
set PYTHONPATH=%FWDIR%python\lib\py4j-0.8.1-src.zip;%PYTHONPATH%

set OLD_PYTHONSTARTUP=%PYTHONSTARTUP%
set PYTHONSTARTUP=%FWDIR%python\pyspark\shell.py
set PYSPARK_SUBMIT_ARGS=%*

echo Running %PYSPARK_PYTHON% with PYTHONPATH=%PYTHONPATH%

"%PYSPARK_PYTHON%" %*
rem Check whether the argument is a file
for /f %%i in ('echo %1^| findstr /R "\.py"') do (
set PYTHON_FILE=%%i
)

if [%PYTHON_FILE%] == [] (
%PYSPARK_PYTHON%
) else (
echo.
echo WARNING: Running python applications through ./bin/pyspark.cmd is deprecated as of Spark 1.0.
echo Use ./bin/spark-submit ^<python file^>
echo.
"%FWDIR%\bin\spark-submit.cmd" %PYSPARK_SUBMIT_ARGS%
)

:exit