diff --git a/.readthedocs.yml b/.readthedocs.yml index 994c28d4..0eca8070 100644 --- a/.readthedocs.yml +++ b/.readthedocs.yml @@ -8,7 +8,7 @@ sphinx: configuration: docs/conf.py python: - version: 3.6 + version: 3.8 install: - method: pip path: . diff --git a/HISTORY b/HISTORY index 16aa1df3..b2818d57 100644 --- a/HISTORY +++ b/HISTORY @@ -1,3 +1,17 @@ +4.9.0 +===== + +Changes +-------- + +- Redesign of the init process. Mostly moved into the `pyapp.init` module to allow + for init of applications that are not using the CliApplication mechanism. The init + methods also provide an easy location for initialisation of testing. + +- Creation of the `pytest-pyapp` package, a PyTest plugin with helper methods and + fixtures for simplifying the testing of pyApp applications. + + 4.8.2 ===== diff --git a/README.rst b/README.rst index 0627b6c5..9a8eee0b 100644 --- a/README.rst +++ b/README.rst @@ -2,17 +2,13 @@ pyApp - A python application framework ###################################### -*Let us handle the boring stuff!* +*Let pyApp handle the boring stuff!* +---------+------------------------------------------------------------------------------------------------------------+ | Docs | .. image:: https://readthedocs.org/projects/pyapp/badge/?version=latest | | | :target: https://docs.pyapp.info/ | | | :alt: ReadTheDocs | +---------+------------------------------------------------------------------------------------------------------------+ -| Build | .. image:: https://api.dependabot.com/badges/status?host=github&repo=pyapp-org/pyapp | -| | :target: https://dependabot.com | -| | :alt: Dependabot Status | -+---------+------------------------------------------------------------------------------------------------------------+ | Quality | .. image:: https://sonarcloud.io/api/project_badges/measure?project=pyapp-org_pyapp&metric=sqale_rating | | | :target: https://sonarcloud.io/dashboard?id=pyapp-org_pyapp | | | :alt: Maintainability | @@ -38,35 +34,39 @@ pyApp - A python application framework | | :target: https://pypi.io/pypi/pyapp/ | +---------+------------------------------------------------------------------------------------------------------------+ -pyApp takes care of the boring boilerplate code for building a CLI, managing -settings and much more so you can focus on your application logic. +pyApp takes care of the boilerplate code for building an modern application with a CLI, managing +settings and much more so you can focus on the business logic. + So what does pyApp handle? ========================== -- **Configuration** - Loading, merging your settings from different sources +- Configuration - Loading, merging your settings from different sources + Python modules + File and HTTP(S) endpoints for JSON and YAML files. -- **Instance Factories** - Configuration of plugins, database connections, or just +- Instance Factories - Configuration of plugins, database connections, or just implementations of an ``ABC``. Leveraging settings to make setup of your application easy and reduce coupling. -- **Dependency Injection** - Easy to use dependency injection without complicated setup. +- Dependency Injection - Easy to use dependency injection without complicated setup. + +- Feature Flags - Simple methods to enable and disable features in your application + at runtime. -- **Checks** - A framework for checking settings are correct and environment is +- Checks - A framework for checking settings are correct and environment is operating correctly (your ops team will love you)? -- **Extensions** - Extend the basic framework with extensions. Provides deterministic +- Extensions - Extend the basic framework with extensions. Provides deterministic startup, extension of the CLI and the ability to register checks and extension specific default settings. -- **Application** - Provides a extensible and simple CLI interface for running +- User Interface - Provides a extensible and simple CLI interface for running commands (including async), comes with built-in commands to execute check, setting and extension reports. -- **Logging** - Initialise and apply sane logging defaults. +- Logging - Initialise and apply sane logging defaults. - Highly tested and ready for production use. diff --git a/docs/conf.py b/docs/conf.py index e62bb885..1bb7c7f4 100644 --- a/docs/conf.py +++ b/docs/conf.py @@ -35,9 +35,9 @@ master_doc = "index" # General information about the project. -project = u"pyApp" -copyright = u"2017, Tim Savage" -author = u"Tim Savage" +project = "pyApp" +copyright = "2017, Tim Savage" +author = "Tim Savage" # The version info for the project you're documenting, acts as replacement for # |version| and |release|, also used in various other places throughout the @@ -208,7 +208,7 @@ # (source start file, target name, title, # author, documentclass [howto, manual, or own class]). latex_documents = [ - (master_doc, "PyApp.tex", u"pyApp Documentation", u"Tim Savage", "manual") + (master_doc, "PyApp.tex", "pyApp Documentation", "Tim Savage", "manual") ] # The name of an image file (relative to this directory) to place at the top of @@ -236,7 +236,7 @@ # One entry per manual page. List of tuples # (source start file, name, description, authors, manual section). -man_pages = [(master_doc, "pyapp", u"pyApp Documentation", [author], 1)] +man_pages = [(master_doc, "pyapp", "pyApp Documentation", [author], 1)] # If true, show URL addresses after external links. # man_show_urls = False @@ -251,7 +251,7 @@ ( master_doc, "pyApp", - u"pyApp Documentation", + "pyApp Documentation", author, "pyApp", "One line description of project.", diff --git a/docs/developers.rst b/docs/developers.rst index 07ffaf1e..aeedc8cf 100644 --- a/docs/developers.rst +++ b/docs/developers.rst @@ -1,27 +1,29 @@ Developers ========== -We welcome contributions to the pyApp project (and sub projects). +Contributions to the pyApp project (and sub projects) are welcome. -To get our PR accepted quickly please ensure the following requirements are +To get a PR accepted quickly please ensure the following requirements are met: - Install the `pre-commit `_ hooks to - ensure you code is formatted by `black `_. + ensure: -- Ensure your code has unit test coverage (using pyTest). Unittests should be - designed to be as fast as possible. + - All code is formatted by `black `_ + - Code passes PyLint checks (this is part of the automated build) -- Ensure your code passes the pyLint checks (this is part of the automated build). +- Ensure code has unit test coverage (using pyTest). Unittests should be + designed to be as fast as possible. -- Update the docs with the details if required. +- Documentation has been updated to reflect the change - The API matters, ensure any features provide a nice API for end users. +The core pyApp package is intended to be light and primarily made up of plumbing +code. To add support for a particular service or server a new pyApp extension is +the way to achieve this. -The core pyApp package is intended to be light and mainly made up of plumbing -code. If you want to add support for a particular service or server an extension -is the way to do this. +See the `Developing an Extension`_ section of the extensions doc for guidance on +building a new extension. -See the *Developing an Extension* section of the extensions doc for guidance on -building your own extension. +.. _Developing an Extension: diff --git a/docs/extensions.rst b/docs/extensions.rst index 8cfec1e8..6113dffc 100644 --- a/docs/extensions.rst +++ b/docs/extensions.rst @@ -2,93 +2,41 @@ Extensions ########## - -Available Extensions -==================== - -pyApp Developed ---------------- - -🔌 SQLAlchemy - `pyapp.sqlalchemy`_ - -🔌 Redis - `pyapp.redis`_ - -In Beta -~~~~~~~ - -🐛 Rollbar - `pyapp.rollbar`_ - -📧 AIO SMTPlib - `pyapp.aiosmtplib`_ Extension for aiosmtplib - -☁ Boto3 - `pyapp.boto3`_ - -☁ AIOBotocore - `pyapp.aiobotocore`_ - -📨 Messaging - `pyapp.messaging`_ - Extension to provide abstract interfaces for Message Queues. - -- 📨 AWS Messaging - `pyapp.messaging-aws`_ - Messaging extension for AWS (SQS/SNS) - -In development -~~~~~~~~~~~~~~ - -📧 SMTP - `pyapp.SMTP`_ - -📨 Aio-Pika - `pyapp.aiopika`_ - Messaging extension for pika (RabbitMQ/AMQP) - -🔌 PySpark - `pyapp.pyspark`_ - Extension for PySpark - -🔎 Elastic Search - `pyapp.elasticsearch`_ - Extension for Elasticsearch - -Coming soon ------------ - -📨 AMQP Messaging - Messaging extension for AMQP (RabbitMQ) - -.. _pyapp.sqlalchemy: https://www.github.com/pyapp-org/pyapp.sqlalchemy -.. _pyapp.redis: https://www.github.com/pyapp-org/pyapp.redis -.. _pyapp.aiobotocore: https://www.github.com/pyapp-org/pyapp.aiobotocore -.. _pyapp.SMTP: https://www.github.com/pyapp-org/pyapp.SMTP -.. _pyapp.boto3: https://www.github.com/pyapp-org/pyapp.boto3 -.. _pyapp.rollbar: https://www.github.com/pyapp-org/pyapp.rollbar -.. _pyapp.aiosmtplib: https://www.github.com/pyapp-org/pyapp.aiosmtplib -.. _pyapp.messaging: https://www.github.com/pyapp-org/pyapp-messaging -.. _pyapp.messaging-aws: https://www.github.com/pyapp-org/pyapp-messaging-aws -.. _pyapp.aiopika: https://www.github.com/pyapp-org/pyapp.aiopika -.. _pyapp.pyspark: https://www.github.com/pyapp-org/pyapp.pyspark -.. _pyapp.elasticsearch: https://www.github.com/pyapp-org/pyapp.elasticsearch - -.. note:: - The development status of these projects may have changed from when this - documentation was generated, see the repository (or PyPi) of the extension - package for up to date status. - - Developing an Extension ======================= -An extension is a standard Python package that exports a known entry point that -pyApp uses to identify extensions. This entry point will reference a class with -known attributes that pyApp recognises. +A pyApp extension is a standard Python package that exports an entry point that +pyApp utilises to load/activate the code. The entry point will reference a class +with attributes that pyApp recognises. A Basic Project --------------- -An extensions consists of a standard Python project structure eg:: +The structure of an extension is similar to any other Python package. + +With Setuptools:: ├┬ my_extension │└ __init__.py ├ README.rst - ├ pyproject.toml ├ setup.cfg └ setup.py +With Poetry:: + + ├┬ my_extension + │└ __init__.py + ├ README.rst + ├ pyproject.toml + ├ poetry.lock + -The contents of which are: +The contents of each file: ``my_extension/__init__.py`` The package init file, this file contains the extension entry point. While a - package must container an Extension class every attribute on the class is optional. + package must contain an Extension class every attribute on the class is optional. .. code-block:: python @@ -114,11 +62,13 @@ The contents of which are: .. tip:: - A gotcha when building extensions is attempting to access settings to early - this is the reason for the ``ready`` event on the Extension class. Once ready - has been called settings are setup and ready for use. + One gotcha when building extensions is attempting to access settings too early + before they have been loaded by pyApp, this is the use of the ``ready`` event + on the Extension class. The ``ready`` method will be called once all initialisation + activities have been completed and settings etc are ready for use. + -``README.rst`` +``README.rst`` or ``README.md`` While not strictly necessary a README document is *highly recommended* and is included in the package as the long description. @@ -211,3 +161,63 @@ Using poetry [tool.poetry.plugins."pyapp.extensions"] "my-extension" = "my_extension:Extension" + + +Available Extensions +==================== + +pyApp Developed +--------------- + +🔌 SQLAlchemy - `pyapp.sqlalchemy`_ + +🔌 Redis - `pyapp.redis`_ + +In Beta +~~~~~~~ + +🐛 Rollbar - `pyapp.rollbar`_ + +📧 AIO SMTPlib - `pyapp.aiosmtplib`_ Extension for aiosmtplib + +☁ Boto3 - `pyapp.boto3`_ + +☁ AIOBotocore - `pyapp.aiobotocore`_ + +📨 Messaging - `pyapp.messaging`_ - Extension to provide abstract interfaces for Message Queues. + +- 📨 AWS Messaging - `pyapp.messaging-aws`_ - Messaging extension for AWS (SQS/SNS) + +In development +~~~~~~~~~~~~~~ + +📧 SMTP - `pyapp.SMTP`_ + +📨 Aio-Pika - `pyapp.aiopika`_ - Messaging extension for pika (RabbitMQ/AMQP) + +🔌 PySpark - `pyapp.pyspark`_ - Extension for PySpark + +🔎 Elastic Search - `pyapp.elasticsearch`_ - Extension for Elasticsearch + +Coming soon +----------- + +📨 AMQP Messaging - Messaging extension for AMQP (RabbitMQ) + +.. _pyapp.sqlalchemy: https://www.github.com/pyapp-org/pyapp.sqlalchemy +.. _pyapp.redis: https://www.github.com/pyapp-org/pyapp.redis +.. _pyapp.aiobotocore: https://www.github.com/pyapp-org/pyapp.aiobotocore +.. _pyapp.SMTP: https://www.github.com/pyapp-org/pyapp.SMTP +.. _pyapp.boto3: https://www.github.com/pyapp-org/pyapp.boto3 +.. _pyapp.rollbar: https://www.github.com/pyapp-org/pyapp.rollbar +.. _pyapp.aiosmtplib: https://www.github.com/pyapp-org/pyapp.aiosmtplib +.. _pyapp.messaging: https://www.github.com/pyapp-org/pyapp-messaging +.. _pyapp.messaging-aws: https://www.github.com/pyapp-org/pyapp-messaging-aws +.. _pyapp.aiopika: https://www.github.com/pyapp-org/pyapp.aiopika +.. _pyapp.pyspark: https://www.github.com/pyapp-org/pyapp.pyspark +.. _pyapp.elasticsearch: https://www.github.com/pyapp-org/pyapp.elasticsearch + +.. note:: + The development status of these projects may have changed from when this + documentation was generated, see the repository (or PyPi) of the extension + package for up to date status. diff --git a/docs/index.rst b/docs/index.rst index d61b703b..5e625ec2 100644 --- a/docs/index.rst +++ b/docs/index.rst @@ -1,26 +1,13 @@ -.. PyApp documentation master file, created by - sphinx-quickstart on Thu Jan 12 12:26:34 2017. - You can adapt this file completely to your liking, but it should at least - contain the root `toctree` directive. - - - Welcome to PyApp's documentation! ================================= -*Let us handle the boring stuff!* - -As of pyApp 4.0, Python < 3.6 is no longer supported. +*Let pyApp handle the boring stuff!* +---------+------------------------------------------------------------------------------------------------------------+ | Docs | .. image:: https://readthedocs.org/projects/pyapp/badge/?version=latest | | | :target: https://docs.pyapp.info/ | | | :alt: ReadTheDocs | +---------+------------------------------------------------------------------------------------------------------------+ -| Build | .. image:: https://api.dependabot.com/badges/status?host=github&repo=pyapp-org/pyapp | -| | :target: https://dependabot.com | -| | :alt: Dependabot Status | -+---------+------------------------------------------------------------------------------------------------------------+ | Quality | .. image:: https://sonarcloud.io/api/project_badges/measure?project=pyapp-org_pyapp&metric=sqale_rating | | | :target: https://sonarcloud.io/dashboard?id=pyapp-org_pyapp | | | :alt: Maintainability | @@ -46,12 +33,12 @@ As of pyApp 4.0, Python < 3.6 is no longer supported. | | :target: https://pypi.io/pypi/pyapp/ | +---------+------------------------------------------------------------------------------------------------------------+ -pyApp takes care of the boring boilerplate code for building a CLI, managing -settings and much more so you can focus on your business logic. +pyApp takes care of the boilerplate code for building an modern application with a CLI, managing +settings and much more so you can focus on the business logic. -So what do we handle? -===================== +So what does pyApp handle? +========================== - Configuration - Loading, merging your settings from different sources @@ -74,7 +61,7 @@ So what do we handle? startup, extension of the CLI and the ability to register checks and extension specific default settings. -- Application - Provides a extensible and simple CLI interface for running +- User Interface - Provides a extensible and simple CLI interface for running commands (including async), comes with built-in commands to execute check, setting and extension reports. @@ -98,11 +85,12 @@ Table of Contents :maxdepth: 2 getting-started + topics/index sys-admin-playbook - recipes/index - reference/index extensions developers + reference/index + recipes/index change-history Indices and tables diff --git a/docs/sys-admin-playbook.rst b/docs/sys-admin-playbook.rst index 6f3457b7..d2a9eb0d 100644 --- a/docs/sys-admin-playbook.rst +++ b/docs/sys-admin-playbook.rst @@ -1,12 +1,12 @@ System Admin Playbook ##################### -While pyApp offers a lot of features to help developers be more productive, it -also provides a suite of tools for Sysadmins to help identify application issues -and simplify operation of an application developed with pyApp. +While pyApp offers a lot of features to help developers be more productive, there +is also a suite of tools for Sys-Admins to help identify application issues and +simplify operation of an application developed with pyApp. .. tip:: - pyApp provides a CLI that provides a ``--help`` option at most locations, this + pyApp generates a CLI that provides a ``--help`` option at most locations, this is a great way to find out more about a command or what commands are available. All builtin commands include help using this method. diff --git a/docs/topics/checks.rst b/docs/topics/checks.rst new file mode 100644 index 00000000..3a132a90 --- /dev/null +++ b/docs/topics/checks.rst @@ -0,0 +1,3 @@ +###### +Checks +###### diff --git a/docs/topics/dependency-injection.rst b/docs/topics/dependency-injection.rst new file mode 100644 index 00000000..580c5d1c --- /dev/null +++ b/docs/topics/dependency-injection.rst @@ -0,0 +1,3 @@ +#################### +Dependency Injection +#################### diff --git a/docs/topics/deployment.rst b/docs/topics/deployment.rst new file mode 100644 index 00000000..14367e31 --- /dev/null +++ b/docs/topics/deployment.rst @@ -0,0 +1,3 @@ +########## +Deployment +########## diff --git a/docs/topics/events.rst b/docs/topics/events.rst new file mode 100644 index 00000000..5f16bd57 --- /dev/null +++ b/docs/topics/events.rst @@ -0,0 +1,3 @@ +###### +Events +###### diff --git a/docs/topics/extensions.rst b/docs/topics/extensions.rst new file mode 100644 index 00000000..3c3ec73b --- /dev/null +++ b/docs/topics/extensions.rst @@ -0,0 +1,3 @@ +########## +Extensions +########## diff --git a/docs/topics/feature-flags.rst b/docs/topics/feature-flags.rst new file mode 100644 index 00000000..108e5254 --- /dev/null +++ b/docs/topics/feature-flags.rst @@ -0,0 +1,3 @@ +############# +Feature Flags +############# diff --git a/docs/topics/index.rst b/docs/topics/index.rst new file mode 100644 index 00000000..767484c5 --- /dev/null +++ b/docs/topics/index.rst @@ -0,0 +1,18 @@ +###### +Topics +###### + +Contents: + +.. toctree:: + :maxdepth: 2 + + settings + dependency-injection + feature-flags + events + initialisation + checks + testing + extensions + deployment diff --git a/docs/topics/initialisation.rst b/docs/topics/initialisation.rst new file mode 100644 index 00000000..cc10977c --- /dev/null +++ b/docs/topics/initialisation.rst @@ -0,0 +1,3 @@ +############## +Initialisation +############## diff --git a/docs/topics/settings.rst b/docs/topics/settings.rst new file mode 100644 index 00000000..da135055 --- /dev/null +++ b/docs/topics/settings.rst @@ -0,0 +1,53 @@ +######## +Settings +######## + +Settings are a key part of the pyApp framework and are utilised to control many +aspects of the framework. This section goes into how settings are compiled and +some of the ways to use them to created more configurable applications that can +more easily adapt to change. + + +What are settings? +================== + +Default Settings +---------------- + +Both an application and each extension that is loaded provide a set of default settings. +Default settings can be thought of as both the definition of available settings and +the source of the initial settings value. + +Runtime Settings +---------------- + +These are the settings supplied to the application when it is started. These can be +specified as either a command line flag or via and environment variable. Typically +runtime settings are environment specific eg settings for development, staging or +production environments. + +.. tip:: The default CLI includes the ``--settings`` option that is used to + specify runtime settings at startup. + +How Settings are Compiled? +========================== + +Settings are collected for a number of sources when an application starts up. +These settings are then combined into the settings collection to produce the final +complete set of settings. + +Settings are applied in the following order: + +- Default settings for each extension (that supplies settings) + +- Default settings for the application + +- Runtime settings + + +Changing Settings at Runtime +============================ + + +Common Pattens +============== diff --git a/docs/topics/testing.rst b/docs/topics/testing.rst new file mode 100644 index 00000000..653b0b76 --- /dev/null +++ b/docs/topics/testing.rst @@ -0,0 +1,5 @@ +####### +Testing +####### + + diff --git a/noxfile.py b/noxfile.py index 2c0b47bd..a2b0201e 100644 --- a/noxfile.py +++ b/noxfile.py @@ -8,6 +8,7 @@ def tests(session: Session): with TemporaryDirectory() as tmpdir: session.install("poetry") + session.run("poetry", "build") session.run( "poetry", "export", @@ -15,5 +16,5 @@ def tests(session: Session): "--format=requirements.txt", f"--output={tmpdir}/requirements.txt", ) - session.install(f"-r{tmpdir}/requirements.txt") + session.install(f"-r{tmpdir}/requirements.txt dist/pytest_pyapp*") session.run("pytest") diff --git a/pyproject.toml b/pyproject.toml index acf89ece..37aa3145 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -4,7 +4,7 @@ build-backend = "rtd_poetry" [tool.poetry] name = "pyapp" -version = "4.8.2" +version = "4.9.0" description = "A Python application framework - Let us handle the boring stuff!" authors = ["Tim Savage "] license = "BSD-3-Clause" diff --git a/src/pyapp/__init__.py b/src/pyapp/__init__.py index 143bcbf4..b1155c6f 100644 --- a/src/pyapp/__init__.py +++ b/src/pyapp/__init__.py @@ -6,11 +6,11 @@ *Let us handle the boring stuff!* """ -import logging +import logging as _log from pyapp.versioning import get_installed_version # Configure a default null handler for logging. -logging.getLogger(__name__).addHandler(logging.NullHandler()) +_log.getLogger(__name__).addHandler(_log.NullHandler()) __author__ = "Tim Savage " diff --git a/src/pyapp/app/__init__.py b/src/pyapp/app/__init__.py index 4ca00eb8..58eeedbd 100644 --- a/src/pyapp/app/__init__.py +++ b/src/pyapp/app/__init__.py @@ -141,7 +141,6 @@ def my_command( import os import sys from argparse import ArgumentParser -from argparse import FileType from argparse import Namespace as CommandOptions from typing import Optional from typing import Sequence @@ -149,16 +148,14 @@ def my_command( import argcomplete import colorama -from . import init_logger from .. import conf -from .. import extensions -from .. import feature_flags +from .. import init from ..app import builtin_handlers -from ..injection import register_factory +from ..utils.compatibility import deprecated from ..utils.inspect import import_root_module from .argument_actions import * from .arguments import * -from .logging_formatter import ColourFormatter + logger = logging.getLogger(__name__) @@ -189,24 +186,17 @@ class CliApplication(CommandGroup): """ - default_log_handler = logging.StreamHandler(sys.stderr) + default_log_handler = init.DEFAULT_LOG_HANDLER """ Log handler applied by default to root logger. """ - default_log_formatter = logging.Formatter( - "%(asctime)s | %(levelname)s | %(name)s | %(message)s" - ) + default_log_formatter = init.DEFAULT_LOG_FORMATTER """ Log formatter applied by default to root logger handler. """ - default_color_log_formatter = ColourFormatter( - f"{colorama.Fore.YELLOW}%(asctime)s{colorama.Fore.RESET} " - f"%(clevelname)s " - f"{colorama.Fore.LIGHTBLUE_EX}%(name)s{colorama.Fore.RESET} " - f"%(message)s" - ) + default_color_log_formatter = init.DEFAULT_COLOR_LOG_FORMATTER """ Log formatter applied by default to root logger handler. """ @@ -270,8 +260,10 @@ def __init__( self.env_loglevel_key = env_loglevel_key # Configure Logging as early as possible - self._init_logger = init_logger.InitHandler(self.default_log_handler) - self.pre_configure_logging() + init.pre_configure_logging( + log_handler=self.default_log_handler, + log_formatter=self.default_log_formatter, + ) self._init_parser() self.register_builtin_handlers() @@ -395,64 +387,53 @@ def register_builtin_handlers(self): for additional_handler in self.additional_handlers: additional_handler(self) - def pre_configure_logging(self): + @deprecated("This method is no longer used, call pyapp.init.pre_configure_logging") + def pre_configure_logging(self): # pragma: no cover """ - Set some default logging so settings are logged. - - The main logging configuration is in settings leaving us with a chicken - and egg situation. - + Deprecated please use :func:`pyapp.init.pre_configure_logging`. Will be removed in pyApp 5 """ - self.default_log_handler.formatter = self.default_log_formatter - - # Apply handler to root logger - logging.root.setLevel(logging.DEBUG) - logging.root.handlers = [self._init_logger] + return init.pre_configure_logging( + log_handler=self.default_log_handler, + log_formatter=self.default_log_formatter, + ) - @staticmethod - def register_factories(): + @deprecated("This method is no longer used, call pyapp.init.register_factories") + def register_factories(self): # pragma: no cover """ - Register any abstract interface factories. + Deprecated please use :func:`pyapp.init.register_factories`. Will be removed in pyApp 5 """ - # pylint: disable=import-outside-toplevel - from asyncio import AbstractEventLoop, get_event_loop - - register_factory(AbstractEventLoop, get_event_loop) + init.register_factories() - def load_extensions(self): + @deprecated("This method is no longer used, call pyapp.init.load_extensions") + def load_extensions(self): # pragma: no cover """ - Load/Configure extensions. + Deprecated please use :func:`pyapp.init.load_extensions`. Will be removed in pyApp 5 """ - entry_points = extensions.ExtensionEntryPoints(self.ext_allow_list) - extensions.registry.load_from(entry_points.extensions()) - extensions.registry.register_commands(self) + init.load_extensions(self.ext_allow_list, command_group=self) - def configure_settings(self, opts: CommandOptions): + @deprecated("This method is no longer used, call pyapp.init.configure_settings") + def configure_settings(self, opts: CommandOptions): # pragma: no cover """ - Configure settings container. + Deprecated please use :func:`pyapp.init.configure_settings`. Will be removed in pyApp 5 """ - application_settings = list(extensions.registry.default_settings) - if self.application_settings: - application_settings.append(self.application_settings) - - conf.settings.configure( - application_settings, opts.settings, env_settings_key=self.env_settings_key + init.configure_settings( + self.application_settings, + opts.settings, + env_settings_key=self.env_settings_key, ) - @staticmethod - def configure_feature_flags(opts: CommandOptions): + @deprecated( + "This method is no longer used, call pyapp.init.configure_feature_flags" + ) + def configure_feature_flags(self, opts: CommandOptions): # pragma: no cover """ - Configure feature flags cache. + Deprecated please use :func:`pyapp.init.configure_feature_flags`. Will be removed in pyApp 5 """ - if opts.enable_feature_flags: - for flag in opts.enable_feature_flags: - feature_flags.DEFAULT.set(flag, True) - - if opts.disable_feature_flags: - for flag in opts.disable_feature_flags: - feature_flags.DEFAULT.set(flag, False) + init.configure_feature_flags( + opts.enable_feature_flags, opts.disable_feature_flags + ) - def get_log_formatter(self, log_color) -> logging.Formatter: + def get_log_formatter(self, log_colour) -> logging.Formatter: """ Get log formatter """ @@ -460,45 +441,28 @@ def get_log_formatter(self, log_color) -> logging.Formatter: # Auto-detect colour mode if ( - log_color is None + log_colour is None and isinstance(log_handler, logging.StreamHandler) and hasattr(log_handler.stream, "isatty") ): - log_color = log_handler.stream.isatty() + log_colour = log_handler.stream.isatty() # Enable colour if specified. - if log_color: + if log_colour: return self.default_color_log_formatter return self.default_log_formatter - def configure_logging(self, opts: CommandOptions): + @deprecated("This method is no longer used, call pyapp.init.configure_logging") + def configure_logging(self, opts: CommandOptions): # pragma: no cover """ - Configure the logging framework. + Deprecated please use :func:`pyapp.init.configure_logging`. Will be removed in pyApp 5 """ - # Prevent duplicate runs - if hasattr(self, "_init_logger"): - self.default_log_handler.formatter = self.get_log_formatter(opts.log_color) - - if conf.settings.LOGGING: - logger.info("Applying logging configuration.") - - # Replace root handler with the default handler - logging.root.handlers.pop(0) - logging.root.handlers.append(self.default_log_handler) - - if conf.settings.LOGGING: - # Set a default version if not supplied by settings - dict_config = conf.settings.LOGGING.copy() - dict_config.setdefault("version", 1) - logging.config.dictConfig(dict_config) - - # Configure root log level - logging.root.setLevel(opts.log_level) - - # Replay initial entries and remove - self._init_logger.replay() - del self._init_logger + init.configure_logging( + opts.log_level, + log_handler=self.default_log_handler, + log_formatter=self.get_log_formatter(opts.log_color), + ) def checks_on_startup(self, opts: CommandOptions): """ @@ -545,10 +509,12 @@ def dispatch(self, args: Sequence[str] = None) -> None: """ Dispatch command to registered handler. """ + colorama.init() + # Initialisation phase _set_running_application(self) - self.register_factories() - self.load_extensions() + init.register_factories() + init.load_extensions(self.ext_allow_list, command_group=self) # Parse arguments phase argcomplete.autocomplete(self.parser) @@ -559,17 +525,26 @@ def dispatch(self, args: Sequence[str] = None) -> None: logger.info("Starting %s", self.application_summary) # Load settings and configure logger - self.configure_settings(opts) - self.configure_feature_flags(opts) - self.configure_logging(opts) + init.configure_settings( + self.application_settings, + opts.settings, + env_settings_key=self.env_settings_key, + ) + init.configure_feature_flags( + opts.enable_feature_flags, + opts.disable_feature_flags, + ) + init.configure_logging( + opts.log_level, + log_handler=self.default_log_handler, + log_formatter=self.get_log_formatter(opts.log_color), + ) handler_name = getattr(opts, ":handler", None) if handler_name != "checks": self.checks_on_startup(opts) - else: - self.configure_settings(opts) - extensions.registry.ready() + init.startup_completed() # Dispatch to handler. try: @@ -589,7 +564,7 @@ def dispatch(self, args: Sequence[str] = None) -> None: sys.exit(exit_code) finally: - self.logging_shutdown() + init.shutdown_completed() CURRENT_APP: Optional[CliApplication] = None diff --git a/src/pyapp/app/init_logger.py b/src/pyapp/app/init_logger.py index 54167f85..c1d50755 100644 --- a/src/pyapp/app/init_logger.py +++ b/src/pyapp/app/init_logger.py @@ -1,43 +1,6 @@ """ Logger used in the initial setup. """ -import logging -from typing import List +from pyapp.logging import InitHandler - -class InitHandler(logging.Handler): - """ - Handler that provides initial logging and captures logging up to a certain - level, it is then replayed once logging has been initialised. - """ - - def __init__(self, handler: logging.Handler, pass_through_level=logging.WARNING): - super().__init__(logging.DEBUG) - self.handler = handler - self.pass_through_level = pass_through_level - self._store: List[logging.LogRecord] = [] - - def handle(self, record: logging.LogRecord) -> None: - """ - Handle record - """ - self._store.append(record) - if record.levelno >= self.pass_through_level: - super().handle(record) - - def replay(self): - """ - Replay stored log records - """ - - for record in self._store: - logging.getLogger(record.name).handle(record) - self._store.clear() - - def emit(self, record: logging.LogRecord) -> None: - """ - Emit the record - """ - - # Pass to initial handler - self.handler.emit(record) +__all__ = ("InitHandler",) diff --git a/src/pyapp/app/logging_formatter.py b/src/pyapp/app/logging_formatter.py index 71f850b6..cbf07ba3 100644 --- a/src/pyapp/app/logging_formatter.py +++ b/src/pyapp/app/logging_formatter.py @@ -5,42 +5,8 @@ Custom formatter for logging messages. """ -import logging - -import colorama - -RESET_ALL = colorama.Style.RESET_ALL - - -class ColourFormatter(logging.Formatter): - """ - Formatter that adds colourised versions of log levels - - Extends LogRecord with: - - %(clevelno)s Numeric logging level for the message (DEBUG, INFO, - WARNING, ERROR, CRITICAL) with ANSI terminal colours - applied. - %(clevelname)s Text logging level for the message ("DEBUG", "INFO", - "WARNING", "ERROR", "CRITICAL") with ANSI terminal - colours applied. - """ - - COLOURS = { - logging.CRITICAL: colorama.Fore.RED + colorama.Style.BRIGHT, - logging.ERROR: colorama.Fore.RED, - logging.WARNING: colorama.Fore.BLUE, - logging.INFO: colorama.Fore.GREEN, - logging.DEBUG: colorama.Fore.LIGHTBLACK_EX, - logging.NOTSET: colorama.Fore.WHITE, - } - - def formatMessage(self, record: logging.LogRecord): - color = self.COLOURS[record.levelno] - record.clevelname = f"{color}{record.levelname}{RESET_ALL}" - record.clevelno = f"{color}{record.levelno}{RESET_ALL}" - return super().formatMessage(record) - +import pyapp.logging # For "English" variants =oP -ColorFormatter = ColourFormatter +ColourFormatter = pyapp.logging.ColourFormatter +ColorFormatter = pyapp.logging.ColourFormatter diff --git a/src/pyapp/extensions/registry.py b/src/pyapp/extensions/registry.py index 81ef7af4..7765de6b 100644 --- a/src/pyapp/extensions/registry.py +++ b/src/pyapp/extensions/registry.py @@ -14,9 +14,11 @@ import pkg_resources from pyapp.app.arguments import CommandGroup +from pyapp.utils.compatibility import deprecated_argument __all__ = ("registry", "ExtensionEntryPoints", "ExtensionDetail") + ENTRY_POINTS = "pyapp.extensions" @@ -72,19 +74,32 @@ class ExtensionEntryPoints: Identifies and loads extensions. """ - def __init__(self, white_list: Sequence[str] = None): - self.white_list = white_list + __slots__ = ("allow_list",) + + def __init__( + self, + allow_list: Sequence[str] = None, + *, + white_list: Sequence[str] = None, + ): + if white_list: + deprecated_argument("white_list", "has been replaced by `allow_list`") + allow_list = ( + (set(allow_list) | set(white_list)) if allow_list else white_list + ) + + self.allow_list = tuple(allow_list) if allow_list else None def _entry_points(self) -> Iterator[pkg_resources.EntryPoint]: """ Iterator of filtered extension entry points """ - white_list = self.white_list + allow_list = self.allow_list for entry_point in pkg_resources.iter_entry_points(ENTRY_POINTS): - if white_list is None or entry_point.name in white_list: + if allow_list is None or entry_point.name in allow_list: yield entry_point - def extensions(self, load: bool = True) -> Iterator[object]: + def extensions(self, load: bool = True) -> Iterator[ExtensionDetail]: """ Iterator of loaded extensions. """ diff --git a/src/pyapp/init.py b/src/pyapp/init.py new file mode 100644 index 00000000..1c3be03c --- /dev/null +++ b/src/pyapp/init.py @@ -0,0 +1,210 @@ +""" +Initialisation +~~~~~~~~~~~~~~ + +Initialisation of the framework + +""" +import logging +import sys +from typing import Optional +from typing import Sequence +from typing import Union + +import colorama +from pyapp import conf +from pyapp import extensions +from pyapp import feature_flags +from pyapp.conf import DEFAULT_ENV_KEY +from pyapp.injection import register_factory +from pyapp.logging import ColourFormatter +from pyapp.logging import InitHandler + +log = logging.getLogger(__name__) + + +DEFAULT_LOG_HANDLER = logging.StreamHandler(sys.stderr) +""" +Log handler applied by default to root logger. +""" + +DEFAULT_LOG_FORMATTER = logging.Formatter( + "%(asctime)s | %(levelname)s | %(name)s | %(message)s" +) +""" +Log formatter applied by default to root logger handler. +""" + +DEFAULT_COLOR_LOG_FORMATTER = ColourFormatter( + f"{colorama.Fore.YELLOW}%(asctime)s{colorama.Fore.RESET} " + f"%(clevelname)s " + f"{colorama.Fore.LIGHTBLUE_EX}%(name)s{colorama.Fore.RESET} " + f"%(message)s" +) +""" +Log formatter applied by default to root logger handler. +""" + +# Marker variable to ensure logging is not configured twice +_INIT_LOGGER: Optional[InitHandler] = None + + +def pre_configure_logging( + *, + log_handler: logging.Handler = DEFAULT_LOG_HANDLER, + log_formatter: logging.Formatter = DEFAULT_LOG_FORMATTER, +): + """ + Set some default logging so settings are logged. + + The main logging configuration is in settings leaving us with a chicken + and egg situation. + + """ + global _INIT_LOGGER # pylint: disable=global-statement + + # Configure formatter and setup init handler + log_handler.formatter = log_formatter + _INIT_LOGGER = InitHandler(log_handler) + + # Apply handler to root logger + logging.root.setLevel(logging.DEBUG) + logging.root.handlers = [_INIT_LOGGER] + + +def register_factories(): + """ + Register any abstract interface factories. + """ + # pylint: disable=import-outside-toplevel + from asyncio import AbstractEventLoop, get_event_loop + + register_factory(AbstractEventLoop, get_event_loop) + + +def load_extensions( + extension_allow_list: Sequence[str] = None, + *, + command_group=None, +): + """ + Load/Configure extensions. + """ + entry_points = extensions.ExtensionEntryPoints(extension_allow_list) + extensions.registry.load_from(entry_points.extensions()) + + if command_group: + extensions.registry.register_commands(command_group) + + +def configure_settings( + default_settings: Union[str, Sequence[str]], + runtime_settings: str = None, + *, + env_settings_key: str = DEFAULT_ENV_KEY, +): + """ + Configure settings container. + """ + application_settings = list(extensions.registry.default_settings) + if default_settings: + application_settings.append(default_settings) + + conf.settings.configure( + application_settings, runtime_settings, env_settings_key=env_settings_key + ) + + +def configure_feature_flags( + enable_feature_flags: Sequence[str] = None, + disable_feature_flags: Sequence[str] = None, +): + """ + Configure feature flags cache. + """ + if enable_feature_flags: + for flag in enable_feature_flags: + feature_flags.DEFAULT.set(flag, True) + + if disable_feature_flags: + for flag in disable_feature_flags: + feature_flags.DEFAULT.set(flag, False) + + +def configure_logging( + log_level=logging.INFO, + *, + log_handler: logging.Handler = DEFAULT_LOG_HANDLER, + log_formatter: logging.Formatter = DEFAULT_LOG_FORMATTER, +): + """ + Configure the logging framework. + """ + global _INIT_LOGGER # pylint: disable=global-statement + + # Prevent duplicate runs + if _INIT_LOGGER: + log_handler.formatter = log_formatter + + if conf.settings.LOGGING: + log.info("Applying logging configuration.") + + # Replace root handler with the default handler + logging.root.handlers.pop(0) + logging.root.handlers.append(log_handler) + + if conf.settings.LOGGING: + # Set a default version if not supplied by settings + dict_config = conf.settings.LOGGING.copy() + dict_config.setdefault("version", 1) + logging.config.dictConfig(dict_config) + + # Configure root log level + logging.root.setLevel(log_level) + + # Replay initial entries and remove + _INIT_LOGGER.replay() + _INIT_LOGGER = None + + +def startup_completed(): + """ + Startup has completed, perform required operations + """ + extensions.registry.ready() + + +def shutdown_completed(): + """ + Shutdown has completed, perform required operations + """ + logging.shutdown() + + +def initialise( + default_settings: Union[str, Sequence[str]], + log_level=logging.INFO, + *, + log_handler: logging.Handler = DEFAULT_LOG_HANDLER, + log_formatter: logging.Formatter = DEFAULT_LOG_FORMATTER, + extension_allow_list: Sequence[str] = None, + runtime_settings: str = None, + env_settings_key: str = DEFAULT_ENV_KEY, + enable_feature_flags: Sequence[str] = None, + disable_feature_flags: Sequence[str] = None, +): + """ + Initialise a pyApp application outside a CLIApplication instance. + + Useful if you have a script or want to integrate pyApp into an existing application. + """ + + pre_configure_logging(log_handler=log_handler, log_formatter=log_formatter) + register_factories() + load_extensions(extension_allow_list=extension_allow_list) + configure_settings( + default_settings, runtime_settings, env_settings_key=env_settings_key + ) + configure_feature_flags(enable_feature_flags, disable_feature_flags) + configure_logging(log_level, log_handler=log_handler, log_formatter=log_formatter) + startup_completed() diff --git a/src/pyapp/logging.py b/src/pyapp/logging.py new file mode 100644 index 00000000..26eb0cb2 --- /dev/null +++ b/src/pyapp/logging.py @@ -0,0 +1,82 @@ +""" +Logger used in the initial setup. +""" +import logging +from typing import List + +import colorama + + +class InitHandler(logging.Handler): + """ + Handler that provides initial logging and captures logging up to a certain + level, it is then replayed once logging has been initialised. + """ + + def __init__(self, handler: logging.Handler, pass_through_level=logging.WARNING): + super().__init__(logging.DEBUG) + self.handler = handler + self.pass_through_level = pass_through_level + self._store: List[logging.LogRecord] = [] + + def handle(self, record: logging.LogRecord) -> None: + """ + Handle record + """ + self._store.append(record) + if record.levelno >= self.pass_through_level: + super().handle(record) + + def replay(self): + """ + Replay stored log records + """ + + for record in self._store: + logging.getLogger(record.name).handle(record) + self._store.clear() + + def emit(self, record: logging.LogRecord) -> None: + """ + Emit the record + """ + + # Pass to initial handler + self.handler.emit(record) + + +RESET_ALL = colorama.Style.RESET_ALL + + +class ColourFormatter(logging.Formatter): + """ + Formatter that adds colourised versions of log levels + + Extends LogRecord with: + + %(clevelno)s Numeric logging level for the message (DEBUG, INFO, + WARNING, ERROR, CRITICAL) with ANSI terminal colours + applied. + %(clevelname)s Text logging level for the message ("DEBUG", "INFO", + "WARNING", "ERROR", "CRITICAL") with ANSI terminal + colours applied. + """ + + COLOURS = { + logging.CRITICAL: colorama.Fore.RED + colorama.Style.BRIGHT, + logging.ERROR: colorama.Fore.RED, + logging.WARNING: colorama.Fore.BLUE, + logging.INFO: colorama.Fore.GREEN, + logging.DEBUG: colorama.Fore.LIGHTBLACK_EX, + logging.NOTSET: colorama.Fore.WHITE, + } + + def formatMessage(self, record: logging.LogRecord): + color = self.COLOURS[record.levelno] + record.clevelname = f"{color}{record.levelname}{RESET_ALL}" + record.clevelno = f"{color}{record.levelno}{RESET_ALL}" + return super().formatMessage(record) + + +# For "English" variants =oP +ColorFormatter = ColourFormatter diff --git a/src/pyapp/utils/compatibility.py b/src/pyapp/utils/compatibility.py index b02af6fd..fbb4a0c0 100644 --- a/src/pyapp/utils/compatibility.py +++ b/src/pyapp/utils/compatibility.py @@ -8,9 +8,13 @@ import functools import inspect import warnings +from typing import Type -def deprecated(message: str, category: Warning = DeprecationWarning): +def deprecated( + message: str, + category: Type[Warning] = DeprecationWarning, +): """ Decorator for marking classes/functions as being deprecated and are to be removed in the future. @@ -37,7 +41,7 @@ def init_wrapper(*args, **kwargs): @functools.wraps(obj) def func_wrapper(*args, **kwargs): warnings.warn( - "{obj.__name__} is deprecated and scheduled for removal. {message}", + f"{obj.__name__} is deprecated and scheduled for removal. {message}", category=category, ) return obj(*args, **kwargs) @@ -45,3 +49,22 @@ def func_wrapper(*args, **kwargs): return func_wrapper return decorator + + +def deprecated_argument( + argument: str, + message: str, + category: Type[Warning] = DeprecationWarning, +): + """ + Method to marking an argument as deprecated + + :param argument: Name of argument + :param message: Message provided. + :param category: Category of warning. + + """ + warnings.warn( + f"Argument {argument!r} is deprecated and scheduled for removal. {message}", + category=category, + )