Update config (#1903)

* New aproach for uploading sanic app config.

* Update config.rst

Co-authored-by: tigerthelion <bjt.thompson@gmail.com>
Co-authored-by: Adam Hopkins <admhpkns@gmail.com>

docs/sanic/config.rst

@@ -12,9 +12,9 @@ Sanic holds the configuration in the `config` attribute of the application objec
 
     app = Sanic('myapp')
     app.config.DB_NAME = 'appdb'
-    app.config.DB_USER = 'appuser'
+    app.config['DB_USER'] = 'appuser'
 
-Since the config object actually is a dictionary, you can use its `update` method in order to set several values at once:
+Since the config object has a type that inherits from dictionary, you can use its ``update`` method in order to set several values at once:
 
 .. code-block:: python
 
@@ -47,9 +47,90 @@ Then the above variable would be `MYAPP_REQUEST_TIMEOUT`. If you want to disable
 
     app = Sanic(__name__, load_env=False)   
 
+From file, dict, or any object (having __dict__ attribute).
+~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
+
+You can store app configurations in: (1) a Python file, (2) a dictionary, or (3) in some other type of custom object.
+
+In order to load configuration from ove of those, you can use ``app.upload_config()``.
+
+**1) From file**
+
+
+Let's say you have ``my_config.py`` file that looks like this:
+
+.. code-block:: python
+
+    # my_config.py
+    A = 1
+    B = 2
+
+Loading config from this file is as easy as:
+
+.. code-block:: python
+
+    app.update_config("/path/to/my_config.py")
+ 
+You can also use environment variables in the path name here.
+
+Let's say you have an environment variable like this:
+
+.. code-block:: shell
+
+    $ export my_path="/path/to"
+    
+Then you can use it like this:
+
+.. code-block:: python
+
+    app.update_config("${my_path}/my_config.py")
+
+.. note::
+
+    Just remember that you have to provide environment variables in the format ${environment_variable} and that $environment_variable is not expanded (is treated as "plain" text).
+
+**2) From dict**
+
+You can also set your app config by providing a ``dict``:
+
+.. code-block:: python
+
+    d = {"A": 1, "B": 2}
+    
+    app.update_config(d)
+ 
+**3) From _any_ object**
+
+App config can be taken from an object. Internally, it uses ``__dict__`` to retrieve keys and values.
+
+For example, pass the class:
+
+.. code-block:: python
+
+    class C:
+        A = 1
+        B = 2
+        
+    app.update_config(C)
+
+or, it can be instantiated:
+
+.. code-block:: python
+
+    c = C()
+    
+    app.update_config(c)
+    
+- From an object (having __dict__ attribute)
+
+
 From an Object
 ~~~~~~~~~~~~~~
 
+.. note::
+
+     Deprecated, will be removed in version 21.3.
+
 If there are a lot of configuration values and they have sensible defaults it might be helpful to put them into a module:
 
 .. code-block:: python
@@ -71,6 +152,10 @@ You could use a class or any other object as well.
 From a File
 ~~~~~~~~~~~
 
+.. note::
+
+     Deprecated, will be removed in version 21.3.
+
 Usually you will want to load configuration from a file that is not part of the distributed application. You can load configuration from a file using `from_pyfile(/path/to/config_file)`. However, that requires the program to know the path to the config file. So instead you can specify the location of the config file in an environment variable and tell Sanic to use that to find the config file:
 
 .. code-block:: python

sanic/app.py

@@ -1452,3 +1452,13 @@ class Sanic:
         self.asgi = True
         asgi_app = await ASGIApp.create(self, scope, receive, send)
         await asgi_app()
+
+    # -------------------------------------------------------------------- #
+    # Configuration
+    # -------------------------------------------------------------------- #
+    def update_config(self, config: Union[bytes, str, dict, Any]):
+        """Update app.config.
+
+        Please refer to config.py::Config.update_config for documentation."""
+
+        self.config.update_config(config)

sanic/config.py

@@ -1,8 +1,15 @@
-import os
+from os import environ
-import types
+from typing import Any, Union
 
-from sanic.exceptions import PyFileError
+# NOTE(tomaszdrozdz): remove in version: 21.3
-from sanic.helpers import import_string
+# We replace from_envvar(), from_object(), from_pyfile() config object methods
+# with one simpler update_config() method.
+# We also replace "loading module from file code" in from_pyfile()
+# in a favour of load_module_from_file_location().
+# Please see pull request: 1903
+# and issue: 1895
+from .deprecated import from_envvar, from_object, from_pyfile  # noqa
+from .utils import load_module_from_file_location, str_to_bool
 
 
 SANIC_PREFIX = "SANIC_"
@@ -59,76 +66,23 @@ class Config(dict):
     def __setattr__(self, attr, value):
         self[attr] = value
 
-    def from_envvar(self, variable_name):
+    # NOTE(tomaszdrozdz): remove in version: 21.3
-        """Load a configuration from an environment variable pointing to
+    # We replace from_envvar(), from_object(), from_pyfile() config object
-        a configuration file.
+    # methods with one simpler update_config() method.
-
+    # We also replace "loading module from file code" in from_pyfile()
-        :param variable_name: name of the environment variable
+    # in a favour of load_module_from_file_location().
-        :return: bool. ``True`` if able to load config, ``False`` otherwise.
+    # Please see pull request: 1903
-        """
+    # and issue: 1895
-        config_file = os.environ.get(variable_name)
+    from_envvar = from_envvar
-        if not config_file:
+    from_pyfile = from_pyfile
-            raise RuntimeError(
+    from_object = from_object
-                "The environment variable %r is not set and "
-                "thus configuration could not be loaded." % variable_name
-            )
-        return self.from_pyfile(config_file)
-
-    def from_pyfile(self, filename):
-        """Update the values in the config from a Python file.
-        Only the uppercase variables in that module are stored in the config.
-
-        :param filename: an absolute path to the config file
-        """
-        module = types.ModuleType("config")
-        module.__file__ = filename
-        try:
-            with open(filename) as config_file:
-                exec(  # nosec
-                    compile(config_file.read(), filename, "exec"),
-                    module.__dict__,
-                )
-        except IOError as e:
-            e.strerror = "Unable to load configuration file (%s)" % e.strerror
-            raise
-        except Exception as e:
-            raise PyFileError(filename) from e
-
-        self.from_object(module)
-        return True
-
-    def from_object(self, obj):
-        """Update the values from the given object.
-        Objects are usually either modules or classes.
-
-        Just the uppercase variables in that object are stored in the config.
-        Example usage::
-
-            from yourapplication import default_config
-            app.config.from_object(default_config)
-
-            or also:
-            app.config.from_object('myproject.config.MyConfigClass')
-
-        You should not use this function to load the actual configuration but
-        rather configuration defaults. The actual config should be loaded
-        with :meth:`from_pyfile` and ideally from a location not within the
-        package because the package might be installed system wide.
-
-        :param obj: an object holding the configuration
-        """
-        if isinstance(obj, str):
-            obj = import_string(obj)
-        for key in dir(obj):
-            if key.isupper():
-                self[key] = getattr(obj, key)
 
     def load_environment_vars(self, prefix=SANIC_PREFIX):
         """
         Looks for prefixed environment variables and applies
         them to the configuration if present.
         """
-        for k, v in os.environ.items():
+        for k, v in environ.items():
             if k.startswith(prefix):
                 _, config_key = k.split(prefix, 1)
                 try:
@@ -138,23 +92,47 @@ class Config(dict):
                         self[config_key] = float(v)
                     except ValueError:
                         try:
-                            self[config_key] = strtobool(v)
+                            self[config_key] = str_to_bool(v)
                         except ValueError:
                             self[config_key] = v
 
+    def update_config(self, config: Union[bytes, str, dict, Any]):
+        """Update app.config.
 
-def strtobool(val):
+        Note:: only upper case settings are considered.
-    """
-    This function was borrowed from distutils.utils. While distutils
-    is part of stdlib, it feels odd to use distutils in main application code.
 
-    The function was modified to walk its talk and actually return bool
+        You can upload app config by providing path to py file
-    and not int.
+        holding settings.
-    """
+
-    val = val.lower()
+            # /some/py/file
-    if val in ("y", "yes", "t", "true", "on", "1"):
+            A = 1
-        return True
+            B = 2
-    elif val in ("n", "no", "f", "false", "off", "0"):
+
-        return False
+        config.update_config("${some}/py/file")
-    else:
+
-        raise ValueError("invalid truth value %r" % (val,))
+        Yes you can put environment variable here, but they must be provided
+        in format: ${some_env_var}, and mark that $some_env_var is treated
+        as plain string.
+
+        You can upload app config by providing dict holding settings.
+
+            d = {"A": 1, "B": 2}
+            config.update_config(d)
+
+        You can upload app config by providing any object holding settings,
+        but in such case config.__dict__ will be used as dict holding settings.
+
+            class C:
+                A = 1
+                B = 2
+            config.update_config(C)"""
+
+        if isinstance(config, (bytes, str)):
+            config = load_module_from_file_location(location=config)
+
+        if not isinstance(config, dict):
+            config = config.__dict__
+
+        config = dict(filter(lambda i: i[0].isupper(), config.items()))
+
+        self.update(config)

sanic/deprecated.py

@@ -0,0 +1,106 @@
+# NOTE(tomaszdrozdz): remove in version: 21.3
+# We replace from_envvar(), from_object(), from_pyfile() config object methods
+# with one simpler update_config() method.
+# We also replace "loading module from file code" in from_pyfile()
+# in a favour of load_module_from_file_location().
+# Please see pull request: 1903
+# and issue: 1895
+import types
+
+from os import environ
+from typing import Any
+from warnings import warn
+
+from sanic.exceptions import PyFileError
+from sanic.helpers import import_string
+
+
+def from_envvar(self, variable_name: str) -> bool:
+    """Load a configuration from an environment variable pointing to
+    a configuration file.
+
+    :param variable_name: name of the environment variable
+    :return: bool. ``True`` if able to load config, ``False`` otherwise.
+    """
+
+    warn(
+        "Using `from_envvar` method is deprecated and will be removed in "
+        "v21.3, use `app.update_config` method instead.",
+        DeprecationWarning,
+        stacklevel=2,
+    )
+
+    config_file = environ.get(variable_name)
+    if not config_file:
+        raise RuntimeError(
+            f"The environment variable {variable_name} is not set and "
+            f"thus configuration could not be loaded."
+        )
+    return self.from_pyfile(config_file)
+
+
+def from_pyfile(self, filename: str) -> bool:
+    """Update the values in the config from a Python file.
+    Only the uppercase variables in that module are stored in the config.
+
+    :param filename: an absolute path to the config file
+    """
+
+    warn(
+        "Using `from_pyfile` method is deprecated and will be removed in "
+        "v21.3, use `app.update_config` method instead.",
+        DeprecationWarning,
+        stacklevel=2,
+    )
+
+    module = types.ModuleType("config")
+    module.__file__ = filename
+    try:
+        with open(filename) as config_file:
+            exec(  # nosec
+                compile(config_file.read(), filename, "exec"),
+                module.__dict__,
+            )
+    except IOError as e:
+        e.strerror = "Unable to load configuration file (e.strerror)"
+        raise
+    except Exception as e:
+        raise PyFileError(filename) from e
+
+    self.from_object(module)
+    return True
+
+
+def from_object(self, obj: Any) -> None:
+    """Update the values from the given object.
+    Objects are usually either modules or classes.
+
+    Just the uppercase variables in that object are stored in the config.
+    Example usage::
+
+        from yourapplication import default_config
+        app.config.from_object(default_config)
+
+        or also:
+        app.config.from_object('myproject.config.MyConfigClass')
+
+    You should not use this function to load the actual configuration but
+    rather configuration defaults. The actual config should be loaded
+    with :meth:`from_pyfile` and ideally from a location not within the
+    package because the package might be installed system wide.
+
+    :param obj: an object holding the configuration
+    """
+
+    warn(
+        "Using `from_object` method is deprecated and will be removed in "
+        "v21.3, use `app.update_config` method instead.",
+        DeprecationWarning,
+        stacklevel=2,
+    )
+
+    if isinstance(obj, str):
+        obj = import_string(obj)
+    for key in dir(obj):
+        if key.isupper():
+            self[key] = getattr(obj, key)

sanic/exceptions.py

@@ -169,6 +169,10 @@ class Unauthorized(SanicException):
             }
 
 
+class LoadFileException(SanicException):
+    pass
+
+
 def abort(status_code, message=None):
     """
     Raise an exception based on SanicException. Returns the HTTP response

sanic/utils.py

@@ -0,0 +1,99 @@
+from importlib.util import module_from_spec, spec_from_file_location
+from os import environ as os_environ
+from re import findall as re_findall
+from typing import Union
+
+from .exceptions import LoadFileException
+
+
+def str_to_bool(val: str) -> bool:
+    """Takes string and tries to turn it into bool as human would do.
+
+    If val is in case insensitive (
+        "y", "yes", "yep", "yup", "t",
+        "true", "on", "enable", "enabled", "1"
+    ) returns True.
+    If val is in case insensitive (
+        "n", "no", "f", "false", "off", "disable", "disabled", "0"
+    ) returns False.
+    Else Raise ValueError."""
+
+    val = val.lower()
+    if val in {
+        "y",
+        "yes",
+        "yep",
+        "yup",
+        "t",
+        "true",
+        "on",
+        "enable",
+        "enabled",
+        "1",
+    }:
+        return True
+    elif val in {"n", "no", "f", "false", "off", "disable", "disabled", "0"}:
+        return False
+    else:
+        raise ValueError(f"Invalid truth value {val}")
+
+
+def load_module_from_file_location(
+    location: Union[bytes, str], encoding: str = "utf8", *args, **kwargs
+):
+    """Returns loaded module provided as a file path.
+
+    :param args:
+        Coresponds to importlib.util.spec_from_file_location location
+        parameters,but with this differences:
+        - It has to be of a string or bytes type.
+        - You can also use here environment variables
+          in format ${some_env_var}.
+          Mark that $some_env_var will not be resolved as environment variable.
+    :encoding:
+        If location parameter is of a bytes type, then use this encoding
+        to decode it into string.
+    :param args:
+        Coresponds to the rest of importlib.util.spec_from_file_location
+        parameters.
+    :param kwargs:
+        Coresponds to the rest of importlib.util.spec_from_file_location
+        parameters.
+
+    For example You can:
+
+        some_module = load_module_from_file_location(
+            "some_module_name",
+            "/some/path/${some_env_var}"
+        )
+    """
+
+    # 1) Parse location.
+    if isinstance(location, bytes):
+        location = location.decode(encoding)
+
+    # A) Check if location contains any environment variables
+    #    in format ${some_env_var}.
+    env_vars_in_location = set(re_findall(r"\${(.+?)}", location))
+
+    # B) Check these variables exists in environment.
+    not_defined_env_vars = env_vars_in_location.difference(os_environ.keys())
+    if not_defined_env_vars:
+        raise LoadFileException(
+            "The following environment variables are not set: "
+            f"{', '.join(not_defined_env_vars)}"
+        )
+
+    # C) Substitute them in location.
+    for env_var in env_vars_in_location:
+        location = location.replace("${" + env_var + "}", os_environ[env_var])
+
+    # 2) Load and return module.
+    name = location.split("/")[-1].split(".")[
+        0
+    ]  # get just the file name without path and .py extension
+    _mod_spec = spec_from_file_location(name, location, *args, **kwargs)
+    module = module_from_spec(_mod_spec)
+    _mod_spec.loader.exec_module(module)  # type: ignore
+
+    return module

tests/static/app_test_config.py

@@ -0,0 +1 @@
+TEST_SETTING_VALUE = 1

tests/test_load_module_from_file_location.py

@@ -0,0 +1,35 @@
+from pathlib import Path
+from types import ModuleType
+
+import pytest
+
+from sanic.exceptions import LoadFileException
+from sanic.utils import load_module_from_file_location
+
+
+@pytest.fixture
+def loaded_module_from_file_location():
+    return load_module_from_file_location(
+        str(Path(__file__).parent / "static/app_test_config.py")
+    )
+
+
+@pytest.mark.dependency(name="test_load_module_from_file_location")
+def test_load_module_from_file_location(loaded_module_from_file_location):
+    assert isinstance(loaded_module_from_file_location, ModuleType)
+
+
+@pytest.mark.dependency(depends=["test_load_module_from_file_location"])
+def test_loaded_module_from_file_location_name(
+    loaded_module_from_file_location,
+):
+    assert loaded_module_from_file_location.__name__ == "app_test_config"
+
+
+def test_load_module_from_file_location_with_non_existing_env_variable():
+    with pytest.raises(
+        LoadFileException,
+        match="The following environment variables are not set: MuuMilk",
+    ):
+
+        load_module_from_file_location("${MuuMilk}")

tests/test_update_config.py

@@ -0,0 +1,36 @@
+from pathlib import Path
+
+import pytest
+
+
+_test_setting_as_dict = {"TEST_SETTING_VALUE": 1}
+_test_setting_as_class = type("C", (), {"TEST_SETTING_VALUE": 1})
+_test_setting_as_module = str(
+    Path(__file__).parent / "static/app_test_config.py"
+)
+
+
+@pytest.mark.parametrize(
+    "conf_object",
+    [
+        _test_setting_as_dict,
+        _test_setting_as_class,
+        pytest.param(
+            _test_setting_as_module,
+            marks=pytest.mark.dependency(
+                depends=["test_load_module_from_file_location"],
+                scope="session",
+            ),
+        ),
+    ],
+    ids=["from_dict", "from_class", "from_file"],
+)
+def test_update(app, conf_object):
+    app.update_config(conf_object)
+    assert app.config["TEST_SETTING_VALUE"] == 1
+
+
+def test_update_from_lowercase_key(app):
+    d = {"test_setting_value": 1}
+    app.update_config(d)
+    assert "test_setting_value" not in app.config

tox.ini

@@ -12,12 +12,13 @@ deps =
     pytest-cov
     pytest-sanic
     pytest-sugar
+    pytest-benchmark
+    pytest-dependency
     httpcore==0.3.0
     httpx==0.15.4
     chardet<=2.3.0
     beautifulsoup4
     gunicorn
-    pytest-benchmark
     uvicorn
     websockets>=8.1,<9.0
 commands =