Skip to content

[SPARK-31153][BUILD] Cleanup several failures in lint-python #27910

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Closed
nchammas wants to merge 7 commits into from
Closed

[SPARK-31153][BUILD] Cleanup several failures in lint-python #27910

Changes from all commits
Commits
Show all changes
7 commits
Select commit Hold shift + click to select a range
26b01bf
add function for doing minimum version check
nchammas Mar 14, 2020
a150f27
use consistent VERSION naming + don't suppress errors!
nchammas Mar 14, 2020
51488cf
check expected versions using new function
nchammas Mar 14, 2020
abe157e
use vendored packaging lib
nchammas Mar 14, 2020
9b543b0
restore flake8 version extraction
nchammas Mar 14, 2020
02184d3
typo -> EXPECTED_PYDOCSTYLE
nchammas Mar 14, 2020
5ad9396
quote consistently
nchammas Mar 14, 2020
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
35 changes: 19 additions & 16 deletions dev/lint-python
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -29,6 +29,17 @@ SPHINX_BUILD="sphinx-build"

PYTHON_EXECUTABLE="python3"

function satisfies_min_version {
local provided_version="$1"
local expected_version="$2"
echo "$(
"$PYTHON_EXECUTABLE" << EOM
from setuptools.extern.packaging import version
print(version.parse('$provided_version') >= version.parse('$expected_version'))
EOM
)"
}

function compile_python_test {
local COMPILE_STATUS=
local COMPILE_REPORT=
Expand Down Expand Up @@ -58,7 +69,7 @@ function pycodestyle_test {
local PYCODESTYLE_STATUS=
local PYCODESTYLE_REPORT=
local RUN_LOCAL_PYCODESTYLE=
local VERSION=
local PYCODESTYLE_VERSION=
local EXPECTED_PYCODESTYLE=
local PYCODESTYLE_SCRIPT_PATH="$SPARK_ROOT_DIR/dev/pycodestyle-$MINIMUM_PYCODESTYLE.py"
local PYCODESTYLE_SCRIPT_REMOTE_PATH="https://raw.githubusercontent.com/PyCQA/pycodestyle/$MINIMUM_PYCODESTYLE/pycodestyle.py"
Expand All @@ -71,11 +82,8 @@ function pycodestyle_test {
# check for locally installed pycodestyle & version
RUN_LOCAL_PYCODESTYLE="False"
if hash "$PYCODESTYLE_BUILD" 2> /dev/null; then
VERSION=$( $PYCODESTYLE_BUILD --version 2> /dev/null)
EXPECTED_PYCODESTYLE=$( ("$PYTHON_EXECUTABLE" -c 'from distutils.version import LooseVersion;
Copy link
Member

@HyukjinKwon HyukjinKwon Mar 15, 2020

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

I think it was mistakenly re-indented at 42c4838#diff-ccd847a0316575dde31bd89786bbe1f2. cc @shaneknapp FYI

print(LooseVersion("""'${VERSION[0]}'""") >= LooseVersion("""'$MINIMUM_PYCODESTYLE'"""))')\
2> /dev/null)

PYCODESTYLE_VERSION="$($PYCODESTYLE_BUILD --version)"
EXPECTED_PYCODESTYLE="$(satisfies_min_version $PYCODESTYLE_VERSION $MINIMUM_PYCODESTYLE)"
if [ "$EXPECTED_PYCODESTYLE" == "True" ]; then
RUN_LOCAL_PYCODESTYLE="True"
fi
Expand Down Expand Up @@ -119,7 +127,6 @@ function pycodestyle_test {

function flake8_test {
local FLAKE8_VERSION=
local VERSION=
local EXPECTED_FLAKE8=
local FLAKE8_REPORT=
local FLAKE8_STATUS=
Expand All @@ -130,11 +137,9 @@ function flake8_test {
exit 1
fi

FLAKE8_VERSION="$($FLAKE8_BUILD --version 2> /dev/null)"
VERSION=($FLAKE8_VERSION)
EXPECTED_FLAKE8=$( ("$PYTHON_EXECUTABLE" -c 'from distutils.version import LooseVersion;
print(LooseVersion("""'${VERSION[0]}'""") >= LooseVersion("""'$MINIMUM_FLAKE8'"""))') \
2> /dev/null)
_FLAKE8_VERSION=($($FLAKE8_BUILD --version))
FLAKE8_VERSION="${_FLAKE8_VERSION[0]}"
EXPECTED_FLAKE8="$(satisfies_min_version $FLAKE8_VERSION $MINIMUM_FLAKE8)"

if [[ "$EXPECTED_FLAKE8" == "False" ]]; then
echo "\
Expand Down Expand Up @@ -176,10 +181,8 @@ function pydocstyle_test {
return
fi

PYDOCSTYLE_VERSION="$($PYDOCSTYLEBUILD --version 2> /dev/null)"
EXPECTED_PYDOCSTYLE=$("$PYTHON_EXECUTABLE" -c 'from distutils.version import LooseVersion; \
print(LooseVersion("""'$PYDOCSTYLE_VERSION'""") >= LooseVersion("""'$MINIMUM_PYDOCSTYLE'"""))' \
2> /dev/null)
PYDOCSTYLE_VERSION="$($PYDOCSTYLE_BUILD --version)"
EXPECTED_PYDOCSTYLE="$(satisfies_min_version $PYDOCSTYLE_VERSION $MINIMUM_PYDOCSTYLE)"

if [[ "$EXPECTED_PYDOCSTYLE" == "False" ]]; then
echo "\
Expand Down