Skip to content

Commit e4b4205

Browse files
ssusiessusie
authored
Adding pylint checks and code_style.sh to fix lint issues. Also adding files modified by code_style.sh (#114)
* add pylint files * change files with code_style.sh
1 parent 0c2700c commit e4b4205

190 files changed

Lines changed: 69118 additions & 69524 deletions

File tree

Some content is hidden

Large Commits have some content hidden by default. Use the searchbox below for content that may be hidden.

‎.github/workflows/CPUTests.yml‎

Lines changed: 46 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,46 @@
1+
name: Linter
2+
3+
on:
4+
push:
5+
branches:
6+
- '**'
7+
8+
jobs:
9+
cpu:
10+
name: "CPU tests"
11+
runs-on: ${{ matrix.os }}
12+
strategy:
13+
matrix:
14+
os: [ubuntu-20.04]
15+
python-version: ['3.10']
16+
steps:
17+
- uses: actions/checkout@v3
18+
- name: Set up Python ${{ matrix.python-version }}
19+
uses: actions/setup-python@v4
20+
with:
21+
python-version: ${{ matrix.python-version }}
22+
- name: Install Dependencies
23+
run: |
24+
python -m pip install --upgrade pip
25+
pip install pylint pyink pytype==2024.2.27
26+
# - name: Typecheck the code with pytype
27+
# run: |
28+
# pytype --jobs auto --disable import-error src/maxdiffusion/
29+
- name: Analysing the code with pylint in maxdiffusion/
30+
run: |
31+
pylint --fail-under=7 src/maxdiffusion/ && \
32+
echo 'maxdiffusion PyLint check successful' || { echo \
33+
'PyLint check has failed. Please run bash code_style.sh to fix issues'; exit 20; }
34+
- name: Analysing the code with pylint in end_to_end/
35+
run: |
36+
pylint --fail-under=7 end_to_end/ && \
37+
echo 'PyLint check on pend_to_end/ is successful' || { echo \
38+
'PyLint check has failed. Please run bash code_style.sh to fix issues'; exit 20; }
39+
- name: Analysing the code with pyink in maxdiffusion/
40+
run: |
41+
pyink src/maxdiffusion --check --diff --color --pyink-indentation=2 --line-length=125
42+
- name: Analysing the code with pyink in end_to_end/
43+
run: |
44+
pyink end_to_end --check --diff --color --pyink-indentation=2 --line-length=125
45+
46+

‎code_style.sh‎

Lines changed: 40 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,40 @@
1+
# Copyright 2023 Google LLC
2+
#
3+
# Licensed under the Apache License, Version 2.0 (the "License");
4+
# you may not use this file except in compliance with the License.
5+
# You may obtain a copy of the License at
6+
#
7+
# http://www.apache.org/licenses/LICENSE-2.0
8+
#
9+
# Unless required by applicable law or agreed to in writing, software
10+
# distributed under the License is distributed on an "AS IS" BASIS,
11+
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12+
# See the License for the specific language governing permissions and
13+
# limitations under the License.
14+
15+
# Clean up Python codes using Pylint & Pyink
16+
# Googlers: please run `sudo apt install pipx; pipx install pylint --force; pipx install pyink==23.10.0` in advance
17+
18+
set -e # Exit immediately if any command fails
19+
20+
FOLDERS_TO_FORMAT=("src/maxdiffusion" "end_to_end/tpu")
21+
LINE_LENGTH=$(grep -E "^max-line-length=" pylintrc | cut -d '=' -f 2)
22+
23+
# Check for --check flag
24+
CHECK_ONLY_PYINK_FLAGS=""
25+
if [[ "$1" == "--check" ]]; then
26+
CHECK_ONLY_PYINK_FLAGS="--check --diff --color"
27+
fi
28+
29+
for folder in "${FOLDERS_TO_FORMAT[@]}"
30+
do
31+
pyink "$folder" ${CHECK_ONLY_PYINK_FLAGS} --pyink-indentation=2 --line-length=${LINE_LENGTH}
32+
done
33+
34+
for folder in "${FOLDERS_TO_FORMAT[@]}"
35+
do
36+
# pylint doesn't change files, only reports errors.
37+
pylint "./$folder"
38+
done
39+
40+
echo "Successfully clean up all codes."

‎end_to_end/tpu/eval_assert.py‎

Lines changed: 7 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -20,9 +20,10 @@
2020
from typing import Sequence
2121
import json
2222

23+
2324
def get_last_n_data(metrics_file, target, n=10):
2425
last_n_data = []
25-
with open(metrics_file, 'r', encoding='utf8') as file:
26+
with open(metrics_file, "r", encoding="utf8") as file:
2627
lines = file.readlines()
2728
for line in lines[::-1]:
2829
metrics = json.loads(line)
@@ -35,24 +36,24 @@ def get_last_n_data(metrics_file, target, n=10):
3536

3637
def test_final_loss(metrics_file, target_loss):
3738
target_loss = float(target_loss)
38-
with open(metrics_file, 'r', encoding='utf8') as _:
39+
with open(metrics_file, "r", encoding="utf8") as _:
3940
use_last_n_data = 10
40-
last_n_data = get_last_n_data(metrics_file, 'learning/loss', use_last_n_data)
41+
last_n_data = get_last_n_data(metrics_file, "learning/loss", use_last_n_data)
4142
avg_last_n_data = sum(last_n_data) / len(last_n_data)
4243
print(f"Mean of last {len(last_n_data)} losses is {avg_last_n_data}")
4344
print(f"Target loss is {target_loss}")
4445
assert avg_last_n_data < target_loss
45-
print('Final loss test passed.')
46+
print("Final loss test passed.")
4647

4748

4849
def main(argv: Sequence[str]) -> None:
4950

5051
_, test_scenario, *test_vars = argv
5152

52-
if test_scenario == 'final_loss':
53+
if test_scenario == "final_loss":
5354
test_final_loss(*test_vars)
5455
else:
55-
raise ValueError(f"Unrecognized test_scenario {test_scenario}")
56+
raise ValueError(f"Unrecognized test_scenario {test_scenario}")
5657

5758

5859
if __name__ == "__main__":

0 commit comments

Comments
 (0)