[platform] Install ltchiptool in separate virtual environment (#166)

* [platform] Install ltchiptool in separate virtual environment

* [platform] Fix f-string syntax, set LibreTiny path in ltchiptool

* [platform] Fix venv site-packages path

* [platform] Fix installing pip without ensurepip

* [platform] Install binary dependencies only

builder/main.py

@@ -17,6 +17,13 @@ env: Environment = DefaultEnvironment()
 platform: PlatformBase = env.PioPlatform()
 board: PlatformBoardConfig = env.BoardConfig()
 
+python_deps = {
+    "ltchiptool": ">=4.5.1,<5.0",
+}
+env.SConscript("python-venv.py", exports="env")
+env.ConfigurePythonVenv()
+env.InstallPythonDependencies(python_deps)
+
 # Utilities
 env.SConscript("utils/config.py", exports="env")
 env.SConscript("utils/cores.py", exports="env")
@@ -24,7 +31,7 @@ env.SConscript("utils/env.py", exports="env")
 env.SConscript("utils/flash.py", exports="env")
 env.SConscript("utils/libs-external.py", exports="env")
 env.SConscript("utils/libs-queue.py", exports="env")
-env.SConscript("utils/ltchiptool.py", exports="env")
+env.SConscript("utils/ltchiptool-util.py", exports="env")
 
 # Firmware name
 if env.get("PROGNAME", "program") == "program":

builder/python-venv.py

@@ -0,0 +1,122 @@
+# Copyright (c) Kuba Szczodrzyński 2023-09-07.
+
+import json
+import site
+import subprocess
+import sys
+from pathlib import Path
+
+import semantic_version
+from platformio.compat import IS_WINDOWS
+from platformio.package.version import pepver_to_semver
+from platformio.platform.base import PlatformBase
+from SCons.Script import DefaultEnvironment, Environment
+
+env: Environment = DefaultEnvironment()
+platform: PlatformBase = env.PioPlatform()
+
+# code borrowed and modified from espressif32/builder/frameworks/espidf.py
+
+
+def env_configure_python_venv(env: Environment):
+    venv_path = Path(env.subst("${PROJECT_CORE_DIR}"), "penv", ".libretiny")
+
+    pip_path = venv_path.joinpath(
+        "Scripts" if IS_WINDOWS else "bin",
+        "pip" + (".exe" if IS_WINDOWS else ""),
+    )
+    python_path = venv_path.joinpath(
+        "Scripts" if IS_WINDOWS else "bin",
+        "python" + (".exe" if IS_WINDOWS else ""),
+    )
+    site_path = venv_path.joinpath(
+        "Lib" if IS_WINDOWS else "lib",
+        "." if IS_WINDOWS else f"python{sys.version_info[0]}.{sys.version_info[1]}",
+        "site-packages",
+    )
+
+    if not pip_path.is_file():
+        # Use the built-in PlatformIO Python to create a standalone virtual env
+        result = env.Execute(
+            env.VerboseAction(
+                f'"$PYTHONEXE" -m venv --clear "{venv_path.absolute()}"',
+                "LibreTiny: Creating a virtual environment for Python dependencies",
+            )
+        )
+        if not python_path.is_file():
+            # Creating the venv failed
+            raise RuntimeError(
+                f"Failed to create virtual environment. Error code {result}"
+            )
+        if not pip_path.is_file():
+            # Creating the venv succeeded but pip didn't get installed
+            # (i.e. Debian/Ubuntu without ensurepip)
+            print(
+                "LibreTiny: Failed to install pip, running get-pip.py", file=sys.stderr
+            )
+            import requests
+
+            with requests.get("https://bootstrap.pypa.io/get-pip.py") as r:
+                p = subprocess.Popen(
+                    args=str(python_path.absolute()),
+                    stdin=subprocess.PIPE,
+                )
+                p.communicate(r.content)
+                p.wait()
+
+    assert (
+        pip_path.is_file()
+    ), f"Error: Missing the pip binary in virtual environment `{pip_path.absolute()}`"
+    assert (
+        python_path.is_file()
+    ), f"Error: Missing Python executable file `{python_path.absolute()}`"
+    assert (
+        site_path.is_dir()
+    ), f"Error: Missing site-packages directory `{site_path.absolute()}`"
+
+    env.Replace(LTPYTHONEXE=python_path.absolute(), LTPYTHONENV=venv_path.absolute())
+    site.addsitedir(str(site_path.absolute()))
+
+
+def env_install_python_dependencies(env: Environment, dependencies: dict):
+    try:
+        pip_output = subprocess.check_output(
+            [
+                env.subst("${LTPYTHONEXE}"),
+                "-m",
+                "pip",
+                "list",
+                "--format=json",
+                "--disable-pip-version-check",
+            ]
+        )
+        pip_data = json.loads(pip_output)
+        packages = {p["name"]: pepver_to_semver(p["version"]) for p in pip_data}
+    except:
+        print(
+            "LibreTiny: Warning! Couldn't extract the list of installed Python packages"
+        )
+        packages = {}
+
+    to_install = []
+    for name, spec in dependencies.items():
+        install_spec = f'"{name}{dependencies[name]}"'
+        if name not in packages:
+            to_install.append(install_spec)
+        elif spec:
+            version_spec = semantic_version.Spec(spec)
+            if not version_spec.match(packages[name]):
+                to_install.append(install_spec)
+
+    if to_install:
+        env.Execute(
+            env.VerboseAction(
+                '"${LTPYTHONEXE}" -m pip install --prefer-binary -U '
+                + " ".join(to_install),
+                "LibreTiny: Installing Python dependencies",
+            )
+        )
+
+
+env.AddMethod(env_configure_python_venv, "ConfigurePythonVenv")
+env.AddMethod(env_install_python_dependencies, "InstallPythonDependencies")

builder/utils/env.py

@@ -8,6 +8,7 @@ from subprocess import PIPE, Popen
 from typing import Dict
 
 from ltchiptool import Family, get_version
+from ltchiptool.util.lvm import LVM
 from ltchiptool.util.misc import sizeof
 from platformio.platform.base import PlatformBase
 from platformio.platform.board import PlatformBoardConfig
@@ -77,7 +78,7 @@ def env_configure(
         # ltchiptool config:
         # -r    output raw log messages
         # -i 1  indent log messages
-        LTCHIPTOOL='"${PYTHONEXE}" -m ltchiptool -r -i 1',
+        LTCHIPTOOL='"${LTPYTHONEXE}" -m ltchiptool -r -i 1 -L "${LT_DIR}"',
         # Fix for link2bin to get tmpfile name in argv
         LINKCOM="${LINK} ${LINKARGS}",
         LINKARGS="${TEMPFILE('-o $TARGET $LINKFLAGS $__RPATH $SOURCES $_LIBDIRFLAGS $_LIBFLAGS', '$LINKCOMSTR')}",
@@ -87,6 +88,8 @@ def env_configure(
     )
     # Store family parameters as environment variables
     env.Replace(**dict(family))
+    # Set platform directory in ltchiptool (for use in this process only)
+    LVM.add_path(platform.get_dir())
     return family
 
 

platform.py

@@ -1,12 +1,12 @@
 # Copyright (c) Kuba Szczodrzyński 2022-04-20.
 
-import importlib
 import json
 import os
 import platform
+import site
 import sys
 from os.path import dirname
-from subprocess import Popen
+from pathlib import Path
 from typing import Dict, List
 
 import click
@@ -15,73 +15,8 @@ from platformio.debug.exception import DebugInvalidOptionsError
 from platformio.package.meta import PackageItem
 from platformio.platform.base import PlatformBase
 from platformio.platform.board import PlatformBoardConfig
-from semantic_version import SimpleSpec, Version
 
-LTCHIPTOOL_VERSION = "^4.2.3"
+site.addsitedir(Path(__file__).absolute().parent.joinpath("tools"))
-
-
-# Install & import tools
-def check_ltchiptool(install: bool):
-    if install:
-        # update ltchiptool to a supported version
-        print("Installing/updating ltchiptool")
-        p = Popen(
-            [
-                sys.executable,
-                "-m",
-                "pip",
-                "install",
-                "-U",
-                "--force-reinstall",
-                f"ltchiptool >= {LTCHIPTOOL_VERSION[1:]}, < 5.0",
-            ],
-        )
-        p.wait()
-
-        # unload all modules from the old version
-        for name, module in list(sorted(sys.modules.items())):
-            if not name.startswith("ltchiptool"):
-                continue
-            del sys.modules[name]
-            del module
-
-    # try to import it
-    ltchiptool = importlib.import_module("ltchiptool")
-
-    # check if the version is known
-    version = Version.coerce(ltchiptool.get_version()).truncate()
-    if version in SimpleSpec(LTCHIPTOOL_VERSION):
-        return
-    if not install:
-        raise ImportError(f"Version incompatible: {version}")
-
-
-def try_check_ltchiptool():
-    install_modes = [False, True]
-    exception = None
-    for install in install_modes:
-        try:
-            check_ltchiptool(install)
-            return
-        except (ImportError, AttributeError) as ex:
-            exception = ex
-    print(
-        "!!! Installing ltchiptool failed, or version outdated. "
-        "Please install ltchiptool manually using pip. "
-        f"Cannot continue. {type(exception).name}: {exception}"
-    )
-    raise exception
-
-
-try_check_ltchiptool()
-import ltchiptool
-
-# Remove current dir so it doesn't conflict with PIO
-if dirname(__file__) in sys.path:
-    sys.path.remove(dirname(__file__))
-
-# Let ltchiptool know about LT's location
-ltchiptool.lt_set_path(dirname(__file__))
 
 
 def get_os_specifiers():
@@ -119,6 +54,12 @@ class LibretinyPlatform(PlatformBase):
         super().__init__(manifest_path)
         self.custom_opts = {}
         self.versions = {}
+        self.verbose = (
+            "-v" in sys.argv
+            or "--verbose" in sys.argv
+            or "PIOVERBOSE=1" in sys.argv
+            or os.environ.get("PIOVERBOSE", "0") == "1"
+        )
 
     def print(self, *args, **kwargs):
         if not self.verbose:
@@ -137,11 +78,8 @@ class LibretinyPlatform(PlatformBase):
         return spec
 
     def configure_default_packages(self, options: dict, targets: List[str]):
-        from ltchiptool.util.dict import RecursiveDict
+        from libretiny import RecursiveDict
 
-        self.verbose = (
-            "-v" in sys.argv or "--verbose" in sys.argv or "PIOVERBOSE=1" in sys.argv
-        )
         self.print(f"configure_default_packages(targets={targets})")
 
         pioframework = options.get("pioframework") or ["base"]
@@ -298,19 +236,19 @@ class LibretinyPlatform(PlatformBase):
         return result
 
     def update_board(self, board: PlatformBoardConfig):
+        from libretiny import Board, Family, merge_dicts
+
         if "_base" in board:
-            board._manifest = ltchiptool.Board.get_data(board._manifest)
+            board._manifest = Board.get_data(board._manifest)
             board._manifest.pop("_base")
 
         if self.custom("board"):
-            from ltchiptool.util.dict import merge_dicts
-
             with open(self.custom("board"), "r") as f:
                 custom_board = json.load(f)
             board._manifest = merge_dicts(board._manifest, custom_board)
 
         family = board.get("build.family")
-        family = ltchiptool.Family.get(short_name=family)
+        family = Family.get(short_name=family)
         # add "frameworks" key with the default "base"
         board.manifest["frameworks"] = ["base"]
         # add "arduino" framework if supported

tools/libretiny/__init__.py

@@ -0,0 +1,14 @@
+# Copyright (c) Kuba Szczodrzyński 2023-09-07.
+
+from .board import Board
+from .dict import RecursiveDict, merge_dicts
+from .family import Family
+
+# TODO refactor and remove all this from here
+
+__all__ = [
+    "Board",
+    "Family",
+    "RecursiveDict",
+    "merge_dicts",
+]

tools/libretiny/board.py

@@ -0,0 +1,34 @@
+# Copyright (c) Kuba Szczodrzyński 2022-07-29.
+
+from typing import Union
+
+from genericpath import isfile
+
+from .dict import merge_dicts
+from .fileio import readjson
+from .lvm import lvm_load_json
+
+
+class Board:
+    @staticmethod
+    def get_data(board: Union[str, dict]) -> dict:
+        if not isinstance(board, dict):
+            if isfile(board):
+                board = readjson(board)
+                if not board:
+                    raise FileNotFoundError(f"Board not found: {board}")
+            else:
+                source = board
+                board = lvm_load_json(f"boards/{board}.json")
+                board["source"] = source
+        if "_base" in board:
+            base = board["_base"]
+            if not isinstance(base, list):
+                base = [base]
+            result = {}
+            for base_name in base:
+                board_base = lvm_load_json(f"boards/_base/{base_name}.json")
+                merge_dicts(result, board_base)
+            merge_dicts(result, board)
+            board = result
+        return board

tools/libretiny/dict.py

@@ -0,0 +1,65 @@
+# Copyright (c) Kuba Szczodrzyński 2022-07-29.
+
+from .obj import get, has, pop, set_
+
+
+class RecursiveDict(dict):
+    def __init__(self, data: dict = None):
+        if data:
+            data = {
+                key: RecursiveDict(value) if isinstance(value, dict) else value
+                for key, value in data.items()
+            }
+            super().__init__(data)
+        else:
+            super().__init__()
+
+    def __getitem__(self, key):
+        if "." not in key:
+            return super().get(key, None)
+        return get(self, key)
+
+    def __setitem__(self, key, value):
+        if "." not in key:
+            return super().__setitem__(key, value)
+        set_(self, key, value, newtype=RecursiveDict)
+
+    def __delitem__(self, key):
+        if "." not in key:
+            return super().pop(key, None)
+        return pop(self, key)
+
+    def __contains__(self, key) -> bool:
+        if "." not in key:
+            return super().__contains__(key)
+        return has(self, key)
+
+    def get(self, key, default=None):
+        if "." not in key:
+            return super().get(key, default)
+        return get(self, key) or default
+
+    def pop(self, key, default=None):
+        if "." not in key:
+            return super().pop(key, default)
+        return pop(self, key, default)
+
+
+def merge_dicts(d1, d2):
+    # force RecursiveDict instances to be treated as regular dicts
+    d1_type = dict if isinstance(d1, RecursiveDict) else type(d1)
+    d2_type = dict if isinstance(d2, RecursiveDict) else type(d2)
+    if d1 is not None and d1_type != d2_type:
+        raise TypeError(f"d1 and d2 are of different types: {type(d1)} vs {type(d2)}")
+    if isinstance(d2, list):
+        if d1 is None:
+            d1 = []
+        d1.extend(merge_dicts(None, item) for item in d2)
+    elif isinstance(d2, dict):
+        if d1 is None:
+            d1 = {}
+        for key in d2:
+            d1[key] = merge_dicts(d1.get(key, None), d2[key])
+    else:
+        d1 = d2
+    return d1

tools/libretiny/family.py

@@ -0,0 +1,97 @@
+# Copyright (c) Kuba Szczodrzyński 2022-06-02.
+
+from dataclasses import dataclass, field
+from typing import List, Optional, Union
+
+from .lvm import lvm_load_json, lvm_path
+
+LT_FAMILIES: List["Family"] = []
+
+
+@dataclass
+class Family:
+    name: str
+    parent: Union["Family", None]
+    code: str
+    description: str
+    id: Optional[int] = None
+    short_name: Optional[str] = None
+    package: Optional[str] = None
+    mcus: List[str] = field(default_factory=lambda: [])
+    children: List["Family"] = field(default_factory=lambda: [])
+
+    # noinspection PyTypeChecker
+    def __post_init__(self):
+        if self.id:
+            self.id = int(self.id, 16)
+        self.mcus = set(self.mcus)
+
+    @classmethod
+    def get_all(cls) -> List["Family"]:
+        global LT_FAMILIES
+        if LT_FAMILIES:
+            return LT_FAMILIES
+        families = lvm_load_json("families.json")
+        LT_FAMILIES = [
+            cls(name=k, **v) for k, v in families.items() if isinstance(v, dict)
+        ]
+        # attach parents and children to all families
+        for family in LT_FAMILIES:
+            if family.parent is None:
+                continue
+            try:
+                parent = next(f for f in LT_FAMILIES if f.name == family.parent)
+            except StopIteration:
+                raise ValueError(
+                    f"Family parent '{family.parent}' of '{family.name}' doesn't exist"
+                )
+            family.parent = parent
+            parent.children.append(family)
+        return LT_FAMILIES
+
+    @classmethod
+    def get(
+        cls,
+        any: str = None,
+        id: Union[str, int] = None,
+        short_name: str = None,
+        name: str = None,
+        code: str = None,
+        description: str = None,
+    ) -> "Family":
+        if any:
+            id = any
+            short_name = any
+            name = any
+            code = any
+            description = any
+        if id and isinstance(id, str) and id.startswith("0x"):
+            id = int(id, 16)
+        for family in cls.get_all():
+            if id and family.id == id:
+                return family
+            if short_name and family.short_name == short_name.upper():
+                return family
+            if name and family.name == name.lower():
+                return family
+            if code and family.code == code.lower():
+                return family
+            if description and family.description == description:
+                return family
+        if any:
+            raise ValueError(f"Family not found - {any}")
+        items = [hex(id) if id else None, short_name, name, code, description]
+        text = ", ".join(filter(None, items))
+        raise ValueError(f"Family not found - {text}")
+
+    @property
+    def has_arduino_core(self) -> bool:
+        if lvm_path().joinpath("cores", self.name, "arduino").is_dir():
+            return True
+        if self.parent:
+            return self.parent.has_arduino_core
+        return False
+
+    @property
+    def target_package(self) -> Optional[str]:
+        return self.package or self.parent and self.parent.target_package

tools/libretiny/fileio.py

@@ -0,0 +1,17 @@
+# Copyright (c) Kuba Szczodrzyński 2022-06-10.
+
+import json
+from json import JSONDecodeError
+from os.path import isfile
+from typing import Optional, Union
+
+
+def readjson(file: str) -> Optional[Union[dict, list]]:
+    """Read a JSON file into a dict or list."""
+    if not isfile(file):
+        return None
+    with open(file, "r", encoding="utf-8") as f:
+        try:
+            return json.load(f)
+        except JSONDecodeError:
+            return None

tools/libretiny/lvm.py

@@ -0,0 +1,19 @@
+#  Copyright (c) Kuba Szczodrzyński 2023-3-18.
+
+import json
+from pathlib import Path
+from typing import Dict, Union
+
+json_cache: Dict[str, Union[list, dict]] = {}
+libretiny_path = Path(__file__).parents[2]
+
+
+def lvm_load_json(path: str) -> Union[list, dict]:
+    if path not in json_cache:
+        with libretiny_path.joinpath(path).open("rb") as f:
+            json_cache[path] = json.load(f)
+    return json_cache[path]
+
+
+def lvm_path() -> Path:
+    return libretiny_path

tools/libretiny/obj.py

@@ -0,0 +1,62 @@
+# Copyright (c) Kuba Szczodrzyński 2022-06-02.
+
+from enum import Enum
+from typing import Type
+
+# The following helpers force using base dict class' methods.
+# Because RecursiveDict uses these helpers, this prevents it
+# from running into endless nested loops.
+
+
+def get(data: dict, path: str):
+    if not isinstance(data, dict) or not path:
+        return None
+    if dict.__contains__(data, path):
+        return dict.get(data, path, None)
+    key, _, path = path.partition(".")
+    return get(dict.get(data, key, None), path)
+
+
+def pop(data: dict, path: str, default=None):
+    if not isinstance(data, dict) or not path:
+        return default
+    if dict.__contains__(data, path):
+        return dict.pop(data, path, default)
+    key, _, path = path.partition(".")
+    return pop(dict.get(data, key, None), path, default)
+
+
+def has(data: dict, path: str) -> bool:
+    if not isinstance(data, dict) or not path:
+        return False
+    if dict.__contains__(data, path):
+        return True
+    key, _, path = path.partition(".")
+    return has(dict.get(data, key, None), path)
+
+
+def set_(data: dict, path: str, value, newtype=dict):
+    if not isinstance(data, dict) or not path:
+        return
+    # can't use __contains__ here, as we're setting,
+    # so it's obvious 'data' doesn't have the item
+    if "." not in path:
+        dict.__setitem__(data, path, value)
+    else:
+        key, _, path = path.partition(".")
+        # allow creation of parent objects
+        if key in data:
+            sub_data = dict.__getitem__(data, key)
+        else:
+            sub_data = newtype()
+            dict.__setitem__(data, key, sub_data)
+        set_(sub_data, path, value)
+
+
+def str2enum(cls: Type[Enum], key: str):
+    if not key:
+        return None
+    try:
+        return next(e for e in cls if e.name.lower() == key.lower())
+    except StopIteration:
+        return None