Merge pull request #1118 from mathbunnyru/asalikhov/py_codestyle

Fix python codestyle

.flake8

+[flake8]
+ignore = W605,W503,W504,H306,H238,H301,H202
+max-line-length = 120
+per-file-ignores =
+    test/test_packages.py:E501

.pre-commit-config.yaml

@@ -15,6 +15,14 @@ repos:
     rev: 2.0.0
     hooks:
       - id: bashate
+  - repo: https://gitlab.com/pycqa/flake8
+    rev: 3.8.3
+    hooks:
+      - id: flake8
+  - repo: https://github.com/pre-commit/mirrors-autopep8
+    rev: v1.5.4
+    hooks:
+      - id: autopep8
   - repo: local
     hooks:
       - id: hadolint

all-spark-notebook/test/test_spark_notebooks.py

@@ -21,7 +21,10 @@ def test_nbconvert(container, test_file):
     output_dir = "/tmp"
     timeout_ms = 600
     LOGGER.info(f"Test that {test_file} notebook can be executed ...")
-    command = f"jupyter nbconvert --to markdown --ExecutePreprocessor.timeout={timeout_ms} --output-dir {output_dir} --execute {cont_data_dir}/{test_file}.ipynb"
+    command = "jupyter nbconvert --to markdown " + \
+        f"--ExecutePreprocessor.timeout={timeout_ms} " + \
+        f"--output-dir {output_dir} " + \
+        f"--execute {cont_data_dir}/{test_file}.ipynb"
     c = container.run(
         volumes={host_data_dir: {"bind": cont_data_dir, "mode": "ro"}},
         tty=True,

base-notebook/jupyter_notebook_config.py

@@ -7,7 +7,7 @@ import os
 import errno
 import stat
 
-c = get_config()
+c = get_config()  # noqa: F821
 c.NotebookApp.ip = '0.0.0.0'
 c.NotebookApp.port = 8888
 c.NotebookApp.open_browser = False
@@ -52,4 +52,4 @@ distinguished_name = req_distinguished_name
 # Change default umask for all subprocesses of the notebook server if set in
 # the environment
 if 'NB_UMASK' in os.environ:
-    os.umask(int(os.environ['NB_UMASK'], 8))
\ No newline at end of file
+    os.umask(int(os.environ['NB_UMASK'], 8))

base-notebook/test/test_container_options.py

@@ -72,26 +72,28 @@ def test_nb_user_change(container):
     running_container = container.run(
         tty=True,
         user="root",
-        environment=[f"NB_USER={nb_user}",
-                      "CHOWN_HOME=yes"],
+        environment=[
+            f"NB_USER={nb_user}",
+            "CHOWN_HOME=yes"
+        ],
         working_dir=f"/home/{nb_user}",
         command=['start.sh', 'bash', '-c', 'sleep infinity']
     )
-    
-    # Give the chown time to complete. Use sleep, not wait, because the 
+
+    # Give the chown time to complete. Use sleep, not wait, because the
     # container sleeps forever.
     time.sleep(10)
     LOGGER.info(f"Checking if the user is changed to {nb_user} by the start script ...")
     output = running_container.logs(stdout=True).decode("utf-8")
-    assert f"Set username to: {nb_user}" in output, f"User is not changed to {nb_user}" 
-    
+    assert f"Set username to: {nb_user}" in output, f"User is not changed to {nb_user}"
+
     LOGGER.info(f"Checking {nb_user} id ...")
     command = "id"
     expected_output = f"uid=1000({nb_user}) gid=100(users) groups=100(users)"
     cmd = running_container.exec_run(command, user=nb_user)
     output = cmd.output.decode("utf-8").strip("\n")
     assert output == expected_output, f"Bad user {output}, expected {expected_output}"
-    
+
     LOGGER.info(f"Checking if {nb_user} owns his home folder ...")
     command = f'stat -c "%U %G" /home/{nb_user}/'
     expected_output = f"{nb_user} users"
@@ -105,10 +107,11 @@ def test_chown_extra(container):
     c = container.run(
         tty=True,
         user='root',
-        environment=['NB_UID=1010',
-                     'NB_GID=101',
-                     'CHOWN_EXTRA=/opt/conda',
-                     'CHOWN_EXTRA_OPTS=-R',
+        environment=[
+            'NB_UID=1010',
+            'NB_GID=101',
+            'CHOWN_EXTRA=/opt/conda',
+            'CHOWN_EXTRA_OPTS=-R'
         ],
         command=['start.sh', 'bash', '-c', 'stat -c \'%n:%u:%g\' /opt/conda/LICENSE.txt']
     )
@@ -118,13 +121,14 @@ def test_chown_extra(container):
 
 
 def test_chown_home(container):
-    """Container should change the NB_USER home directory owner and 
+    """Container should change the NB_USER home directory owner and
     group to the current value of NB_UID and NB_GID."""
     c = container.run(
         tty=True,
         user='root',
-        environment=['CHOWN_HOME=yes',
-                     'CHOWN_HOME_OPTS=-R',
+        environment=[
+            'CHOWN_HOME=yes',
+            'CHOWN_HOME_OPTS=-R'
         ],
         command=['start.sh', 'bash', '-c', 'chown root:root /home/jovyan && ls -alsh /home']
     )

base-notebook/test/test_pandoc.py

@@ -3,8 +3,6 @@
 
 import logging
 
-import pytest
-
 LOGGER = logging.getLogger(__name__)
 
 
@@ -17,4 +15,3 @@ def test_pandoc(container):
     logs = c.logs(stdout=True).decode("utf-8")
     LOGGER.debug(logs)
     assert "<p><strong>BOLD</strong></p>" in logs
-

conftest.py

@@ -13,6 +13,7 @@ from requests.adapters import HTTPAdapter
 
 LOGGER = logging.getLogger(__name__)
 
+
 @pytest.fixture(scope='session')
 def http_client():
     """Requests session with retries and backoff."""
@@ -48,6 +49,7 @@ class TrackedContainer(object):
     **kwargs: dict, optional
         Default keyword arguments to pass to docker.DockerClient.containers.run
     """
+
     def __init__(self, docker_client, image_name, **kwargs):
         self.container = None
         self.docker_client = docker_client
@@ -78,7 +80,7 @@ class TrackedContainer(object):
         LOGGER.info(f"Running {self.image_name} with args {all_kwargs} ...")
         self.container = self.docker_client.containers.run(self.image_name, **all_kwargs)
         return self.container
-    
+
     def remove(self):
         """Kills and removes the tracked docker container."""
         if self.container:

datascience-notebook/test/test_julia.py

@@ -2,18 +2,16 @@
 # Distributed under the terms of the Modified BSD License.
 import logging
 
-import pytest
-
 LOGGER = logging.getLogger(__name__)
 
 
 def test_julia(container):
     """Basic julia test"""
-    LOGGER.info(f"Test that julia is correctly installed ...")
+    LOGGER.info("Test that julia is correctly installed ...")
     running_container = container.run(
         tty=True, command=["start.sh", "bash", "-c", "sleep infinity"]
     )
-    command = f"julia --version"
+    command = "julia --version"
     cmd = running_container.exec_run(command)
     output = cmd.output.decode("utf-8")
     assert cmd.exit_code == 0, f"Command {command} failed {output}"

docs/conf.py

@@ -21,11 +21,6 @@
 # import sys
 # sys.path.insert(0, os.path.abspath('.'))
 
-# For conversion from markdown to html
-import recommonmark.parser
-from recommonmark.transform import AutoStructify
-
-
 # -- General configuration ------------------------------------------------
 
 # If your documentation needs a minimal Sphinx version, state it here.
@@ -199,4 +194,4 @@ linkcheck_anchors = False
 # -- Translation ----------------------------------------------------------
 
 gettext_uuid = True
-locale_dirs = ['locale/']
\ No newline at end of file
+locale_dirs = ['locale/']

pyspark-notebook/test/test_spark.py

 # Copyright (c) Jupyter Development Team.
 # Distributed under the terms of the Modified BSD License.
-import time
 import logging
 
-import pytest
 
 LOGGER = logging.getLogger(__name__)
 
+
 def test_spark_shell(container):
     """Checking if Spark (spark-shell) is running properly"""
     c = container.run(
@@ -18,6 +17,7 @@ def test_spark_shell(container):
     LOGGER.debug(logs)
     assert 'res0: Int = 2' in logs
 
+
 def test_pyspark(container):
     """PySpark should be in the Python path"""
     c = container.run(

scipy-notebook/test/data/matplotlib_1.py

@@ -3,7 +3,6 @@
 
 # Optional test with [Matplotlib Jupyter Integration](https://github.com/matplotlib/ipympl)
 # %matplotlib widget
-import matplotlib
 import matplotlib.pyplot as plt
 import numpy as np
 import os
@@ -21,4 +20,4 @@ ax.grid()
 # Note that the test can be run headless by checking if an image is produced
 file_path = os.path.join("/tmp", "test.png")
 fig.savefig(file_path)
-print(f"File {file_path} saved")
\ No newline at end of file
+print(f"File {file_path} saved")

scipy-notebook/test/test_pandas.py

@@ -8,17 +8,23 @@ LOGGER = logging.getLogger(__name__)
 
 
 @pytest.mark.parametrize(
-    "name,command",
+    "name,command_list",
     [
         (
             "Sum series",
-            "import pandas as pd; import numpy as np; np.random.seed(0); print(pd.Series(np.random.randint(0, 7, size=10)).sum())",
+            [
+                "import pandas as pd",
+                "import numpy as np",
+                "np.random.seed(0)",
+                "print(pd.Series(np.random.randint(0, 7, size=10)).sum())"
+            ]
         ),
     ],
 )
-def test_pandas(container, name, command):
+def test_pandas(container, name, command_list):
     """Basic pandas tests"""
     LOGGER.info(f"Testing pandas: {name} ...")
+    command = ';'.join(command_list)
     c = container.run(tty=True, command=["start.sh", "python", "-c", command])
     rv = c.wait(timeout=30)
     assert rv == 0 or rv["StatusCode"] == 0, f"Command {command} failed"

test/helpers.py

@@ -64,7 +64,7 @@ class CondaPackageHelper:
     def installed_packages(self):
         """Return the installed packages"""
         if self.installed is None:
-            LOGGER.info(f"Grabing the list of installed packages ...")
+            LOGGER.info("Grabing the list of installed packages ...")
             self.installed = CondaPackageHelper._packages_from_json(
                 self._execute_command(CondaPackageHelper._conda_export_command())
             )
@@ -73,7 +73,7 @@ class CondaPackageHelper:
     def specified_packages(self):
         """Return the specifications (i.e. packages installation requested)"""
         if self.specs is None:
-            LOGGER.info(f"Grabing the list of specifications ...")
+            LOGGER.info("Grabing the list of specifications ...")
             self.specs = CondaPackageHelper._packages_from_json(
                 self._execute_command(CondaPackageHelper._conda_export_command(True))
             )
@@ -87,11 +87,11 @@ class CondaPackageHelper:
     @staticmethod
     def _packages_from_json(env_export):
         """Extract packages and versions from the lines returned by the list of specifications"""
-        #dependencies = filter(lambda x:  isinstance(x, str), json.loads(env_export).get("dependencies"))
+        # dependencies = filter(lambda x:  isinstance(x, str), json.loads(env_export).get("dependencies"))
         dependencies = json.loads(env_export).get("dependencies")
         # Filtering packages installed through pip in this case it's a dict {'pip': ['toree==0.3.0']}
         # Since we only manage packages installed through conda here
-        dependencies = filter(lambda x:  isinstance(x, str), dependencies)
+        dependencies = filter(lambda x: isinstance(x, str), dependencies)
         packages_dict = dict()
         for split in map(lambda x: x.split("=", 1), dependencies):
             # default values
@@ -112,7 +112,7 @@ class CondaPackageHelper:
         """Return the available packages"""
         if self.available is None:
             LOGGER.info(
-                f"Grabing the list of available packages (can take a while) ..."
+                "Grabing the list of available packages (can take a while) ..."
             )
             # Keeping command line output since `conda search --outdated --json` is way too long ...
             self.available = CondaPackageHelper._extract_available(
@@ -135,7 +135,7 @@ class CondaPackageHelper:
         installed = self.installed_packages()
         available = self.available_packages()
         self.comparison = list()
-        for pkg, inst_vs in self.installed.items():
+        for pkg, inst_vs in installed.items():
             if not specifications_only or pkg in specs:
                 avail_vs = sorted(
                     list(available[pkg]), key=CondaPackageHelper.semantic_cmp
@@ -158,7 +158,8 @@ class CondaPackageHelper:
         """Manage semantic versioning for comparison"""
 
         def mysplit(string):
-            version_substrs = lambda x: re.findall(r"([A-z]+|\d+)", x)
+            def version_substrs(x):
+                return re.findall(r"([A-z]+|\d+)", x)
             return list(chain(map(version_substrs, string.split("."))))
 
         def str_ord(string):

test/test_packages.py

@@ -131,7 +131,7 @@ def _import_packages(package_helper, filtered_packages, check_function, max_fail
     Note: using a list of packages instead of a fixture for the list of packages since pytest prevents use of multiple yields
     """
     failures = {}
-    LOGGER.info(f"Testing the import of packages ...")
+    LOGGER.info("Testing the import of packages ...")
     for package in filtered_packages:
         LOGGER.info(f"Trying to import {package}")
         try: