Merge pull request #289 from tekktrik/dev/use-ci-check

Convert pylint check to CI check

Author: kattni

.github/workflows/test.yml

@@ -53,6 +53,7 @@ jobs:
         ADABOT_EMAIL: ${{ secrets.ADABOT_EMAIL }}
         ADABOT_GITHUB_USER: ${{ secrets.ADABOT_GITHUB_USER }}
         ADABOT_GITHUB_ACCESS_TOKEN: ${{ secrets.ADABOT_GITHUB_ACCESS_TOKEN }}
+        RTD_TOKEN: ${{ secrets.RTD_TOKEN }}
         REDIS_PORT: ${{ job.services.redis.ports[6379] }}
       run: |
         python3 -u -m pytest

adabot/circuitpython_libraries.py

@@ -95,7 +95,7 @@ def run_library_checks(validators, kw_args, error_depth):
     pylint_info = pypi.get("/pypi/pylint/json")
     if pylint_info and pylint_info.ok:
         latest_pylint = pylint_info.json()["info"]["version"]
-    logger.info("Latest pylint is: %s", latest_pylint)
+    # logger.info("Latest pylint is: %s", latest_pylint)
 
     repos = common_funcs.list_repos(
         include_repos=tuple(blinka_repos)

adabot/lib/circuitpython_library_validators.py

@@ -8,25 +8,15 @@
 errors, across the entire CirtuitPython library ecosystem."""
 
 import datetime
-from io import StringIO
-import json
 import os
 import logging
-import pathlib
 import re
 import time
-from tempfile import TemporaryDirectory
 
 from packaging.version import parse as pkg_version_parse
 
-from pylint import lint
-from pylint.reporters import JSONReporter
-
 import requests
 
-import sh
-from sh.contrib import git
-
 import yaml
 import parse
 
@@ -38,18 +28,6 @@
 GH_INTERFACE = pygithub.Github(os.environ["ADABOT_GITHUB_ACCESS_TOKEN"])
 
 
-class CapturedJsonReporter(JSONReporter):
-    """Helper class to stringify PyLint JSON reports."""
-
-    def __init__(self):
-        self._stringio = StringIO()
-        super().__init__(self._stringio)
-
-    def get_result(self):
-        """The current value."""
-        return self._stringio.getvalue()
-
-
 # Define constants for error strings to make checking against them more robust:
 ERROR_README_DOWNLOAD_FAILED = "Failed to download README"
 ERROR_README_IMAGE_MISSING_ALT = "README image missing alt text"
@@ -61,6 +39,7 @@ def get_result(self):
     "README CI badge needs to be changed to GitHub Actions"
 )
 ERROR_PYFILE_DOWNLOAD_FAILED = "Failed to download .py code file"
+ERROR_TOMLFILE_DOWNLOAD_FAILED = "Failed to download .toml file"
 ERROR_PYFILE_MISSING_STRUCT = (
     ".py file contains reference to import ustruct"
     " without reference to import struct.  See issue "
@@ -99,9 +78,12 @@ def get_result(self):
 ERROR_MISSING_CODE_OF_CONDUCT = "Missing CODE_OF_CONDUCT.md"
 ERROR_MISSING_README_RST = "Missing README.rst"
 ERROR_MISSING_READTHEDOCS = "Missing readthedocs.yaml"
-ERROR_MISSING_PYPROJECT_TOML = "For pypi compatibility, missing pyproject.toml"
+ERROR_MISSING_PYPROJECT_TOML = "For PyPI compatibility, missing pyproject.toml"
 ERROR_MISSING_PRE_COMMIT_CONFIG = "Missing .pre-commit-config.yaml"
-ERROR_MISSING_REQUIREMENTS_TXT = "For pypi compatibility, missing requirements.txt"
+ERROR_MISSING_REQUIREMENTS_TXT = "For PyPI compatibility, missing requirements.txt"
+ERROR_MISSING_OPTIONAL_REQUIREMENTS_TXT = (
+    "For PyPI compatibility, missing optional_requirements.txt"
+)
 ERROR_MISSING_BLINKA = (
     "For pypi compatibility, missing Adafruit-Blinka in requirements.txt"
 )
@@ -114,7 +96,7 @@ def get_result(self):
 ERROR_ONLY_ALLOW_MERGES = "Only allow merges, disallow rebase and squash"
 ERROR_RTD_SUBPROJECT_MISSING = "ReadTheDocs missing as a subproject on CircuitPython"
 ERROR_RTD_ADABOT_MISSING = "ReadTheDocs project missing adabot as owner"
-ERROR_RTD_FAILED_TO_LOAD_BUILD_STATUS = "Failed to load build status"
+ERROR_RTD_FAILED_TO_LOAD_BUILD_STATUS = "Failed to load RTD build status"
 ERROR_RTD_SUBPROJECT_FAILED = "Failed to list CircuitPython subprojects on ReadTheDocs"
 ERROR_RTD_OUTPUT_HAS_WARNINGS = "ReadTheDocs latest build has warnings and/or errors"
 ERROR_GITHUB_NO_RELEASE = "Library repository has no releases"
@@ -145,7 +127,7 @@ def get_result(self):
     "Missing or incorrect pre-commit version in .pre-commit-config.yaml"
 )
 ERROR_PYLINT_VERSION = "Missing or incorrect pylint version in .pre-commit-config.yaml"
-ERROR_PYLINT_FAILED_LINTING = "Failed PyLint checks"
+ERROR_CI_BUILD = "Failed CI build"
 ERROR_NEW_REPO_IN_WORK = "New repo(s) currently in work, and unreleased"
 
 # Temp category for GitHub Actions migration.
@@ -308,26 +290,6 @@ def validate_repo_state(self, repo):
             errors.append(ERROR_ONLY_ALLOW_MERGES)
         return errors
 
-    def validate_actions_state(self, repo):
-        """Validate if the most recent GitHub Actions run on the default branch
-        has passed.
-        Just returns a message stating that the most recent run failed.
-        """
-
-        if not (
-            repo["owner"]["login"] == "adafruit"
-            and repo["name"].startswith("Adafruit_CircuitPython")
-        ):
-            return []
-
-        try:
-            repo_obj = GH_INTERFACE.get_repo("Adafruit/" + repo["full_name"])
-            workflow = repo_obj.get_workflow("build.yml")
-            workflow_runs = workflow.get_runs(branch="main")
-            return [] if workflow_runs[0].conclusion else [ERROR_GITHUB_FAILING_ACTIONS]
-        except pygithub.GithubException:
-            return [ERROR_UNABLE_PULL_REPO_DETAILS]
-
     # pylint: disable=too-many-locals,too-many-return-statements,too-many-branches
     def validate_release_state(self, repo):
         """Validate if a repo 1) has a release, and 2) if there have been commits
@@ -577,17 +539,14 @@ def _validate_pre_commit_config_yaml(self, file_info):
         return errors
 
     def _validate_pyproject_toml(self, file_info):
-        """Check prproject.toml for pypi compatibility"""
+        """Check pyproject.toml for pypi compatibility"""
         download_url = file_info["download_url"]
         contents = requests.get(download_url, timeout=30)
         if not contents.ok:
-            return [ERROR_PYFILE_DOWNLOAD_FAILED]
-
-        errors = []
-
-        return errors
+            return [ERROR_TOMLFILE_DOWNLOAD_FAILED]
+        return []
 
-    def _validate_requirements_txt(self, repo, file_info):
+    def _validate_requirements_txt(self, repo, file_info, check_blinka=True):
         """Check requirements.txt for pypi compatibility"""
         download_url = file_info["download_url"]
         contents = requests.get(download_url, timeout=30)
@@ -598,7 +557,11 @@ def _validate_requirements_txt(self, repo, file_info):
         lines = contents.text.split("\n")
         blinka_lines = [l for l in lines if re.match(r"[\s]*Adafruit-Blinka[\s]*", l)]
 
-        if not blinka_lines and repo["name"] not in LIBRARIES_DONT_NEED_BLINKA:
+        if (
+            not blinka_lines
+            and repo["name"] not in LIBRARIES_DONT_NEED_BLINKA
+            and check_blinka
+        ):
             errors.append(ERROR_MISSING_BLINKA)
         return errors
 
@@ -733,6 +696,13 @@ def validate_contents(self, repo):
                 errors.extend(self._validate_requirements_txt(repo, file_info))
             else:
                 errors.append(ERROR_MISSING_REQUIREMENTS_TXT)
+            if "optional_requirements.txt" in files:
+                file_info = content_list[files.index("optional_requirements.txt")]
+                errors.extend(
+                    self._validate_requirements_txt(repo, file_info, check_blinka=False)
+                )
+            else:
+                errors.append(ERROR_MISSING_OPTIONAL_REQUIREMENTS_TXT)
 
         # Check for an examples folder.
         dirs = [
@@ -1167,71 +1137,38 @@ def validate_labels(self, repo):
 
         return errors
 
-    def validate_passes_linting(self, repo):
-        """Clones the repo and runs pylint on the Python files"""
+    def validate_actions_state(self, repo):
+        """Validate if the most recent GitHub Actions run on the default branch
+        has passed.
+        Just returns a message stating that the most recent run failed.
+        """
+
         if not repo["name"].startswith("Adafruit_CircuitPython"):
             return []
 
-        ignored_py_files = ["conf.py"]
-
-        desination_type = TemporaryDirectory
-        if self.keep_repos:
-            desination_type = pathlib.Path("repos").absolute
-
-        with desination_type() as tempdir:
-            repo_dir = pathlib.Path(tempdir) / repo["name"]
+        while True:
             try:
-                if not repo_dir.exists():
-                    git.clone("--depth=1", repo["clone_url"], repo_dir)
-            except sh.ErrorReturnCode as err:
-                self.output_file_data.append(
-                    f"Failed to clone repo for linting: {repo['full_name']}\n {err.stderr}"
-                )
-                return [ERROR_OUTPUT_HANDLER]
-
-            if self.keep_repos and (repo_dir / ".pylint-ok").exists():
-                return []
-
-            for file in repo_dir.rglob("*.py"):
-                if file.name in ignored_py_files or str(file.parent).endswith(
-                    "examples"
-                ):
-                    continue
-
-                pylint_args = [str(file)]
-                if (repo_dir / ".pylintrc").exists():
-                    pylint_args += [f"--rcfile={str(repo_dir / '.pylintrc')}"]
-
-                reporter = CapturedJsonReporter()
+                lib_repo = GH_INTERFACE.get_repo(repo["full_name"])
 
-                logging.debug("Running pylint on %s", file)
+                if lib_repo.archived:
+                    return []
 
-                lint.Run(pylint_args, reporter=reporter, exit=False)
-                pylint_stderr = ""
-                pylint_stdout = reporter.get_result()
-
-                if pylint_stderr:
-                    self.output_file_data.append(
-                        f"PyLint error ({repo['name']}): '{pylint_stderr}'"
-                    )
-                    return [ERROR_OUTPUT_HANDLER]
+                arg_dict = {"branch": lib_repo.default_branch}
 
                 try:
-                    pylint_result = json.loads(pylint_stdout)
-                except json.JSONDecodeError as json_err:
-                    self.output_file_data.append(
-                        f"PyLint output JSONDecodeError: {json_err.msg}"
-                    )
-                    return [ERROR_OUTPUT_HANDLER]
-
-                if pylint_result:
-                    return [ERROR_PYLINT_FAILED_LINTING]
-
-            if self.keep_repos:
-                with open(repo_dir / ".pylint-ok", "w") as pylint_ok:
-                    pylint_ok.write("".join(pylint_result))
-
-        return []
+                    workflow = lib_repo.get_workflow("build.yml")
+                    workflow_runs = workflow.get_runs(**arg_dict)
+                except pygithub.GithubException:  # This can probably be tightened later
+                    # No workflows or runs yet
+                    return []
+                if not workflow_runs[0].conclusion:
+                    return [ERROR_CI_BUILD]
+                return []
+            except pygithub.RateLimitExceededException:
+                core_rate_limit_reset = GH_INTERFACE.get_rate_limit().core.reset
+                sleep_time = core_rate_limit_reset - datetime.datetime.now()
+                logging.warning("Rate Limit will reset at: %s", core_rate_limit_reset)
+                time.sleep(sleep_time.seconds)
 
     def validate_default_branch(self, repo):
         """Makes sure that the default branch is main"""

requirements.txt

@@ -4,7 +4,7 @@
 
 black==22.3.0
 packaging==20.3
-pylint
+pylint==2.11.1
 pytest
 pyyaml==5.4.1
 redis==2.10.6