Add script to export API Ids as JSON. (#53)

Author: microbit-robert

.gitignore

@@ -1,5 +1,6 @@
 /typeshed.json
 /typeshed.*.json
+/api-ids.json
 /node_modules/
 *.rst
 /crowdin/

package.json

@@ -4,8 +4,9 @@
   "description": "Temporary home for micro:bit MicroPython stubs.",
   "scripts": {
     "test": "pyright -p test-pyrightconfig.json",
-    "i18n:typeshed-to-crowdin": "python3 scripts/crowdin-convert.py typeshed-to-crowdin",
-    "i18n:crowdin-to-typeshed": "python3 scripts/crowdin-convert.py crowdin-to-typeshed"
+    "i18n:typeshed-to-crowdin": "python3 scripts/crowdin_convert.py typeshed-to-crowdin",
+    "i18n:crowdin-to-typeshed": "python3 scripts/crowdin_convert.py crowdin-to-typeshed",
+    "export-api-ids": "python3 scripts/export_api_ids.py"
   },
   "repository": {
     "type": "git",

scripts/common.py

@@ -0,0 +1,117 @@
+"""
+    Functions, types and variables
+    shared with various scripts.
+    Including:
+        crowdin_convert.py
+        export_api_ids.py
+"""
+
+from dataclasses import dataclass
+import os
+import ast
+from typing import Any
+from typing import Optional
+
+DIR = os.path.dirname(__file__)
+
+
+@dataclass
+class TypeshedFile:
+    file_path: str
+    module_name: str
+    python_file: bool
+
+
+def get_source(file_path):
+    with open(file_path, "r", encoding="utf-8") as file:
+        return file.read()
+
+
+def module_name_for_path(file_path: str):
+    """Hacky determination of the module name."""
+    name = os.path.basename(file_path)
+    in_microbit_package = os.path.basename(os.path.dirname(file_path)) == "microbit"
+    if in_microbit_package:
+        if name == "__init__.pyi":
+            return "microbit"
+        return ".".join(["microbit", os.path.splitext(name)[0]])
+    return os.path.splitext(name)[0]
+
+
+def get_stub_files() -> list[TypeshedFile]:
+    top = os.path.join(DIR, "..", "lang/en/typeshed/stdlib")
+    files_to_process: list[TypeshedFile] = []
+    for root, dirs, files in os.walk(top):
+        for name in files:
+            file_path = os.path.join(root, name)
+            # Skip audio stubs file that imports from microbit audio
+            # (so we don't include its docstring)
+            if (
+                os.path.basename(os.path.dirname(file_path)) != "microbit"
+                and name == "audio.pyi"
+            ):
+                continue
+            if name.endswith(".pyi"):
+                files_to_process.append(
+                    TypeshedFile(
+                        file_path=file_path,
+                        module_name=module_name_for_path(file_path),
+                        python_file=True,
+                    )
+                )
+            else:
+                files_to_process.append(
+                    TypeshedFile(
+                        file_path=file_path,
+                        module_name="",
+                        python_file=False,
+                    )
+                )
+    return sorted(files_to_process, key=lambda x: x.file_path)
+
+
+class DocStringVisitor(ast.NodeVisitor):
+    def __init__(self, module_name):
+        self.module_name = module_name
+        self.key = []
+        self.used_keys = set()
+        self.preceding: Optional[str] = None
+
+    def visit_Module(self, node: ast.Module) -> Any:
+        name = self.module_name
+        self.handle_docstring(node, name)
+
+        self.key.append(name)
+        self.generic_visit(node)
+        self.key.pop()
+
+    def visit_ClassDef(self, node):
+        name = node.name
+        self.handle_docstring(node, name)
+
+        self.key.append(name)
+        self.generic_visit(node)
+        self.key.pop()
+
+    def visit_FunctionDef(self, node: ast.FunctionDef) -> Any:
+        self.preceding = None
+        self.handle_docstring(node, node.name)
+
+    def visit_AnnAssign(self, node: ast.AnnAssign) -> Any:
+        self.preceding = node.target.id  # type: ignore
+
+    def visit_Assign(self, node: ast.Assign) -> Any:
+        if len(node.targets) != 1:
+            raise AssertionError()
+        self.preceding = node.targets[0].id  # type: ignore
+
+    def visit_Expr(self, node: ast.Expr) -> Any:
+        if self.preceding:
+            self.handle_docstring(node, self.preceding)
+
+    def generic_visit(self, node: ast.AST) -> Any:
+        self.preceding = None
+        return super().generic_visit(node)
+
+    def handle_docstring(self, node: ast.AST, name: str) -> None:
+        raise NotImplementedError()

scripts/crowdin-convert.py renamed to scripts/crowdin_convert.py

@@ -7,19 +7,14 @@
 """
 
 import ast
-from dataclasses import dataclass
 import os
 import json
 import re
 import sys
-
-from typing import Any
-
-from typing import Optional
+from common import TypeshedFile, get_stub_files, DIR, get_source, DocStringVisitor
 
 
 NODE_TYPES_WITH_DOCSTRINGS = (ast.FunctionDef, ast.Module, ast.ClassDef)
-DIR = os.path.dirname(__file__)
 EN_JSON_PATH = os.path.join(DIR, "../crowdin/api.en.json")
 TRANSLATED_JSON_DIR = os.path.join(DIR, "../crowdin/translated")
 
@@ -35,55 +30,6 @@ def typeshed_to_crowdin():
     save_docstrings_as_json(data)
 
 
-@dataclass
-class TypeshedFile:
-    file_path: str
-    module_name: str
-    python_file: bool
-
-
-def get_stub_files() -> list[TypeshedFile]:
-    top = os.path.join(DIR, "..", "lang/en/typeshed/stdlib")
-    files_to_process: list[TypeshedFile] = []
-    for root, dirs, files in os.walk(top):
-        for name in files:
-            file_path = os.path.join(root, name)
-            # Skip audio stubs file that imports from microbit audio (so we don't include its docstring)
-            if (
-                os.path.basename(os.path.dirname(file_path)) != "microbit"
-                and name == "audio.pyi"
-            ):
-                continue
-            if name.endswith(".pyi"):
-                files_to_process.append(
-                    TypeshedFile(
-                        file_path=file_path,
-                        module_name=module_name_for_path(file_path),
-                        python_file=True,
-                    )
-                )
-            else:
-                files_to_process.append(
-                    TypeshedFile(
-                        file_path=file_path,
-                        module_name="",
-                        python_file=False,
-                    )
-                )
-    return sorted(files_to_process, key=lambda x: x.file_path)
-
-
-def module_name_for_path(file_path: str):
-    """Hacky determination of the module name used as a translation key."""
-    name = os.path.basename(file_path)
-    in_microbit_package = os.path.basename(os.path.dirname(file_path)) == "microbit"
-    if in_microbit_package:
-        if name == "__init__.pyi":
-            return "microbit"
-        return ".".join(["microbit", os.path.splitext(name)[0]])
-    return os.path.splitext(name)[0]
-
-
 # Translation key to dict with message/description fields.
 TranslationJSON = dict[str, dict[str, str]]
 
@@ -101,9 +47,14 @@ def handle_docstring(self, node: ast.AST, name: str) -> None:
             key_root = ".".join([*self.key, name])
             key = key_root
             suffix = 1
-            while key in self.used_keys:
-                key = f"{key_root}-{suffix}"
-                suffix += 1
+            if isinstance(node, ast.FunctionDef):  # ctx.id
+                for decorator in node.decorator_list:
+                    if hasattr(decorator, "id"):
+                        if decorator.id == "overload":
+                            key = f"{key}-{suffix}"
+                            while key in self.used_keys:
+                                suffix += 1
+                                key = f"{key_root}-{suffix}"
             self.used_keys.add(key)
             self.data.update(get_entries(node, name, key))
 
@@ -112,11 +63,6 @@ def handle_docstring(self, node: ast.AST, name: str) -> None:
     return collector.data
 
 
-def get_source(file_path):
-    with open(file_path, "r", encoding="utf-8") as file:
-        return file.read()
-
-
 def pretty_api_name(name):
     return name.replace("_", " ").strip().lower()
 
@@ -458,53 +404,6 @@ def maybe_dir(maybe_path):
         os.mkdir(maybe_path)
 
 
-class DocStringVisitor(ast.NodeVisitor):
-    def __init__(self, module_name):
-        self.module_name = module_name
-        self.key = []
-        self.used_keys = set()
-        self.preceding: Optional[str] = None
-
-    def visit_Module(self, node: ast.Module) -> Any:
-        name = self.module_name
-        self.handle_docstring(node, name)
-
-        self.key.append(name)
-        self.generic_visit(node)
-        self.key.pop()
-
-    def visit_ClassDef(self, node):
-        name = node.name
-        self.handle_docstring(node, name)
-
-        self.key.append(name)
-        self.generic_visit(node)
-        self.key.pop()
-
-    def visit_FunctionDef(self, node: ast.FunctionDef) -> Any:
-        self.preceding = None
-        self.handle_docstring(node, node.name)
-
-    def visit_AnnAssign(self, node: ast.AnnAssign) -> Any:
-        self.preceding = node.target.id  # type: ignore
-
-    def visit_Assign(self, node: ast.Assign) -> Any:
-        if len(node.targets) != 1:
-            raise AssertionError()
-        self.preceding = node.targets[0].id  # type: ignore
-
-    def visit_Expr(self, node: ast.Expr) -> Any:
-        if self.preceding:
-            self.handle_docstring(node, self.preceding)
-
-    def generic_visit(self, node: ast.AST) -> Any:
-        self.preceding = None
-        return super().generic_visit(node)
-
-    def handle_docstring(self, node: ast.AST, name: str) -> None:
-        raise NotImplementedError()
-
-
 if __name__ == "__main__":
     operation = sys.argv[1]
     if operation == "typeshed-to-crowdin":

scripts/export_api_ids.py

@@ -0,0 +1,95 @@
+"""
+    Creates api-ids.json file which
+    contains all API calls that correspond
+    to documentation shown in the
+    Python Editor sidebar.
+"""
+
+import ast
+import os
+import json
+from common import (
+    get_stub_files,
+    DIR,
+    TypeshedFile,
+    get_source,
+    DocStringVisitor,
+)
+
+modules = [
+    "builtins",
+    "gc",
+    "log",
+    "machine",
+    "math",
+    "microbit",
+    "micropython",
+    "music",
+    "neopixel",
+    "os",
+    "radio",
+    "random",
+    "speech",
+    "struct",
+    "sys",
+    "time",
+]
+
+
+def export_api_ids():
+    data_list = []
+    files_to_process = get_stub_files()
+    for ts_file in files_to_process:
+        if ts_file.python_file:
+            data_list = data_list + get_api_ids(ts_file)
+    data_list.sort()
+    data = {"apiIds": data_list}
+    save_api_ids(data)
+    pass
+
+
+def save_api_ids(data):
+    with open(os.path.join(DIR, "../", "api-ids.json"), "w") as file:
+        file.write(json.dumps(data, indent=2))
+
+
+def checkModuleRequired(module_name):
+    if module_name in modules:
+        return True
+    if "microbit" in module_name:
+        return True
+    return False
+
+
+def get_api_ids(ts_file: TypeshedFile):
+    source = get_source(ts_file.file_path)
+    tree = ast.parse(source)
+
+    class DocStringCollector(DocStringVisitor):
+        def __init__(self):
+            super().__init__(ts_file.module_name)
+            self.data: list[str] = []
+
+        def handle_docstring(self, node: ast.AST, name: str) -> None:
+            key_root = ".".join([*self.key, name])
+            key = key_root
+            suffix = 1
+            if isinstance(node, ast.FunctionDef):  # ctx.id
+                for decorator in node.decorator_list:
+                    if hasattr(decorator, "id"):
+                        if decorator.id == "overload":
+                            key = f"{key}-{suffix}"
+                            while key in self.used_keys:
+                                suffix += 1
+                                key = f"{key_root}-{suffix}"
+            self.used_keys.add(key)
+            if checkModuleRequired(ts_file.module_name):
+                self.data.append(key)
+
+    collector = DocStringCollector()
+    collector.visit(tree)
+    return collector.data
+
+
+if __name__ == "__main__":
+    export_api_ids()