From 3e217a3c73ce825e15ef5c9ea06c2759292bb287 Mon Sep 17 00:00:00 2001 From: mozga-intel Date: Fri, 9 Jul 2021 12:25:25 +0200 Subject: [PATCH 01/10] [Flake8] This pull-request contains pre-commit hook. There is used flake8 tool to perform additional formatting and semantic checking of code. --- .flake8 | 33 +++++++++++++++++++++++++++++++++ requirements-flake8.txt | 5 +++++ tools/git-pre-commit | 3 +++ tools/lint/flake8_hook.py | 28 ++++++++++++++++++++++++++++ 4 files changed, 69 insertions(+) create mode 100644 .flake8 create mode 100644 requirements-flake8.txt create mode 100644 tools/lint/flake8_hook.py diff --git a/.flake8 b/.flake8 new file mode 100644 index 000000000000..3e7d6511a515 --- /dev/null +++ b/.flake8 @@ -0,0 +1,33 @@ +[flake8] +select = B,C,E,F,P,T4,W,B9 +max-line-length = 120 +ignore = + E203,E305,E402,E501,E721,E741,F405,F821,F841,F999,W503,W504,C408,E302,W291,E303,F403 + + EXE001, + + B007,B008, E722 + + C400,C401,C402,C403,C404,C405,C407,C411,C413,C414,C415 +optional-ascii-coding = True +exclude = + ./.git, + ./.github, + ./3rdparty, + ./benchmark, + ./cd, + ./ci, + ./cmake, + ./config, + ./contrib, + ./cpp-package, + ./docker, + ./docs, + ./example, + ./include, + ./licences, + ./plugin, + ./src, + ./tools, + ./build, + *.pyi diff --git a/requirements-flake8.txt b/requirements-flake8.txt new file mode 100644 index 000000000000..eee31f4d7eb3 --- /dev/null +++ b/requirements-flake8.txt @@ -0,0 +1,5 @@ +flake8==3.8.2 +flake8-pyi==20.5.0 +flake8-bugbear==20.1.4 +flake8-comprehensions==3.3.0 +flake8-executable==2.0.4 \ No newline at end of file diff --git a/tools/git-pre-commit b/tools/git-pre-commit index 2d697889af50..d877f6ece929 100644 --- a/tools/git-pre-commit +++ b/tools/git-pre-commit @@ -20,3 +20,6 @@ set -e echo "Running pre-commit clang-format" tools/lint/git-clang-format HEAD~ --force + +echo "Running pre-commit flake8 checker" +python tools/lint/flake8_hook.py \ No newline at end of file diff --git a/tools/lint/flake8_hook.py b/tools/lint/flake8_hook.py new file mode 100644 index 000000000000..4de0e74c6d65 --- /dev/null +++ b/tools/lint/flake8_hook.py @@ -0,0 +1,28 @@ +#!/usr/bin/env python3 + +import sys + +from flake8.main import git + +if __name__ == '__main__': + sys.exit( + git.hook( + # (optional): + # any value > 0 enables complexity checking with mccabe + complexity=0, + + # (optional): + # if True, this returns the total number of error which will cause + # the hook to fail + strict=True, + + # (optional): + # a comma-separated list of errors and warnings to ignore + # ignore= + + # (optional): + # allows for the instances where you don't add the the files to the + # index before running a commit, e.g git commit -a + lazy=git.config_for('lazy'), + ) + ) From 3bbbd47a415e06559f59706d58adfda5087a102c Mon Sep 17 00:00:00 2001 From: mozga-intel Date: Fri, 9 Jul 2021 16:07:13 +0200 Subject: [PATCH 02/10] FLake8 hook python licence --- tools/lint/flake8_hook.py | 16 ++++++++++++++++ 1 file changed, 16 insertions(+) diff --git a/tools/lint/flake8_hook.py b/tools/lint/flake8_hook.py index 4de0e74c6d65..49e2f65f9a41 100644 --- a/tools/lint/flake8_hook.py +++ b/tools/lint/flake8_hook.py @@ -1,3 +1,19 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. #!/usr/bin/env python3 import sys From ccf3b2e2ac2f7ac92d47d9238c7e72182c5893fd Mon Sep 17 00:00:00 2001 From: mozga-intel Date: Fri, 9 Jul 2021 16:31:17 +0200 Subject: [PATCH 03/10] Added licence to .flake8 and git-pre-commit files --- .flake8 | 26 ++++++++++++++++++-------- 1 file changed, 18 insertions(+), 8 deletions(-) diff --git a/.flake8 b/.flake8 index 3e7d6511a515..a1a4693b2a4f 100644 --- a/.flake8 +++ b/.flake8 @@ -1,14 +1,24 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + [flake8] select = B,C,E,F,P,T4,W,B9 max-line-length = 120 -ignore = - E203,E305,E402,E501,E721,E741,F405,F821,F841,F999,W503,W504,C408,E302,W291,E303,F403 - - EXE001, - - B007,B008, E722 - - C400,C401,C402,C403,C404,C405,C407,C411,C413,C414,C415 +ignore = E203,E305 optional-ascii-coding = True exclude = ./.git, From 66fcf8058feb25c22bf0f54584a96665723d59da Mon Sep 17 00:00:00 2001 From: mozga-intel Date: Fri, 30 Jul 2021 14:27:10 +0200 Subject: [PATCH 04/10] Flake8 - CI runner --- ci/docker/runtime_functions.sh | 18 ++++++++++++++++++ 1 file changed, 18 insertions(+) diff --git a/ci/docker/runtime_functions.sh b/ci/docker/runtime_functions.sh index 84b56013c052..5668f86a0347 100755 --- a/ci/docker/runtime_functions.sh +++ b/ci/docker/runtime_functions.sh @@ -704,6 +704,7 @@ sanity_check() { sanity_clang sanity_license sanity_python + sanity_python_flake8 sanity_cpp } @@ -712,6 +713,23 @@ sanity_license() { tools/license_header.py check } +sanity_python_flake8() { + set -ex + + # Install dependencies + pip install -r requirements-flake8.txt --user + flake8 --version + + # Run flake8 + flake8 | tee "${GITHUB_WORKSPACE}"/flake8-output.txt + + if [ -s "${GITHUB_WORKSPACE}"/flake8-output.txt ]; then + echo 'Please fix the above Flake8 warnings.' + exit 1 + fi + exit 0 +} + sanity_cpp() { set -ex 3rdparty/dmlc-core/scripts/lint.py mxnet cpp include src plugin cpp-package tests --exclude_path src/operator/contrib/ctc_include include/onednn From 44d5821e988e79187b0562fff0503371e70db752 Mon Sep 17 00:00:00 2001 From: mozga-intel Date: Fri, 30 Jul 2021 14:33:01 +0200 Subject: [PATCH 05/10] Flake8: version --- ci/docker/runtime_functions.sh | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/ci/docker/runtime_functions.sh b/ci/docker/runtime_functions.sh index 5668f86a0347..a97cbb26dc5b 100755 --- a/ci/docker/runtime_functions.sh +++ b/ci/docker/runtime_functions.sh @@ -718,10 +718,9 @@ sanity_python_flake8() { # Install dependencies pip install -r requirements-flake8.txt --user - flake8 --version # Run flake8 - flake8 | tee "${GITHUB_WORKSPACE}"/flake8-output.txt + flake8-3.8.2 | tee "${GITHUB_WORKSPACE}"/flake8-output.txt if [ -s "${GITHUB_WORKSPACE}"/flake8-output.txt ]; then echo 'Please fix the above Flake8 warnings.' From 557e944426afea1ed1f6f0bf162cad52c3d9cda7 Mon Sep 17 00:00:00 2001 From: mozga-intel Date: Mon, 2 Aug 2021 11:34:08 +0200 Subject: [PATCH 06/10] Flake8 file create in PR space --- ci/docker/runtime_functions.sh | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/ci/docker/runtime_functions.sh b/ci/docker/runtime_functions.sh index a97cbb26dc5b..2bf0fdf3edbd 100755 --- a/ci/docker/runtime_functions.sh +++ b/ci/docker/runtime_functions.sh @@ -718,11 +718,11 @@ sanity_python_flake8() { # Install dependencies pip install -r requirements-flake8.txt --user - + # Run flake8 - flake8-3.8.2 | tee "${GITHUB_WORKSPACE}"/flake8-output.txt + flake8-3.8.2 | tee flake8-output.txt - if [ -s "${GITHUB_WORKSPACE}"/flake8-output.txt ]; then + if [ -s flake8-output.txt ]; then echo 'Please fix the above Flake8 warnings.' exit 1 fi From 1db028e7e0a0c9afb18548acfe9278c89d987dcd Mon Sep 17 00:00:00 2001 From: mozga-intel Date: Wed, 4 Aug 2021 09:36:44 +0200 Subject: [PATCH 07/10] Change flake8 runner name --- ci/docker/runtime_functions.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ci/docker/runtime_functions.sh b/ci/docker/runtime_functions.sh index 2bf0fdf3edbd..049701de3b54 100755 --- a/ci/docker/runtime_functions.sh +++ b/ci/docker/runtime_functions.sh @@ -720,7 +720,7 @@ sanity_python_flake8() { pip install -r requirements-flake8.txt --user # Run flake8 - flake8-3.8.2 | tee flake8-output.txt + flake8 | tee flake8-output.txt if [ -s flake8-output.txt ]; then echo 'Please fix the above Flake8 warnings.' From cc78dc3549ec8a002bf90989b57953f2b17d1007 Mon Sep 17 00:00:00 2001 From: mozga-intel Date: Wed, 4 Aug 2021 11:06:31 +0200 Subject: [PATCH 08/10] Print a list of pip3 packages --- ci/docker/runtime_functions.sh | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/ci/docker/runtime_functions.sh b/ci/docker/runtime_functions.sh index 049701de3b54..5b5c49716477 100755 --- a/ci/docker/runtime_functions.sh +++ b/ci/docker/runtime_functions.sh @@ -717,8 +717,8 @@ sanity_python_flake8() { set -ex # Install dependencies - pip install -r requirements-flake8.txt --user - + pip3 install -r requirements-flake8.txt --user + echo `(pip3 list | grep flake)` # Run flake8 flake8 | tee flake8-output.txt From c4e7e3c383cd660df52c901c176be2e4c6cac083 Mon Sep 17 00:00:00 2001 From: mozga-intel Date: Wed, 4 Aug 2021 11:14:28 +0200 Subject: [PATCH 09/10] Print a list of pip3 packages --- ci/docker/runtime_functions.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ci/docker/runtime_functions.sh b/ci/docker/runtime_functions.sh index 5b5c49716477..05906e572044 100755 --- a/ci/docker/runtime_functions.sh +++ b/ci/docker/runtime_functions.sh @@ -718,7 +718,7 @@ sanity_python_flake8() { # Install dependencies pip3 install -r requirements-flake8.txt --user - echo `(pip3 list | grep flake)` + pip3 list | grep flake # Run flake8 flake8 | tee flake8-output.txt From 08af07da38e65960af18519ad612d232b0f68ec5 Mon Sep 17 00:00:00 2001 From: mozga-intel Date: Wed, 4 Aug 2021 11:36:24 +0200 Subject: [PATCH 10/10] Python executor -m has been added to run flake8 --- ci/docker/runtime_functions.sh | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/ci/docker/runtime_functions.sh b/ci/docker/runtime_functions.sh index 05906e572044..b83a14e504b8 100755 --- a/ci/docker/runtime_functions.sh +++ b/ci/docker/runtime_functions.sh @@ -718,9 +718,9 @@ sanity_python_flake8() { # Install dependencies pip3 install -r requirements-flake8.txt --user - pip3 list | grep flake + # Run flake8 - flake8 | tee flake8-output.txt + python3 -m flake8 | tee flake8-output.txt if [ -s flake8-output.txt ]; then echo 'Please fix the above Flake8 warnings.'