use new empack (#5)

* using empack 3.0.0
* removed emsdk
* disabled node 
* removed dead code

Author: DerThorsten

.github/workflows/main.yaml

@@ -19,8 +19,6 @@ jobs:
 
     strategy:
       fail-fast: false
-      matrix:
-        emsdk_ver: ["3.1.2"]
 
     steps:
       - uses: actions/checkout@v2
@@ -33,17 +31,12 @@ jobs:
         with:
           environment-file: dev-env.yml
           environment-name: dev-env
-          micromamba-version: "0.22.0"
+          micromamba-version: "1.4.1"
 
       - name: Install Playwright
         run: |
           playwright install
 
-      - name: Setup emsdk
-        run: |
-          micromamba activate dev-env
-          emsdk install ${{matrix.emsdk_ver}}
-
       - name: Install pyjs-code-runner
         run: |
           micromamba activate dev-env
@@ -52,9 +45,6 @@ jobs:
       - name: Run Tests 
         run: |
           micromamba activate dev-env
-          emsdk activate ${{matrix.emsdk_ver}}
-          source $CONDA_EMSDK_DIR/emsdk_env.sh
-
           pytest -s
 
 
@@ -64,8 +54,6 @@ jobs:
 
     strategy:
       fail-fast: false
-      matrix:
-        emsdk_ver: ["3.1.2"]
 
     steps:
       - uses: actions/checkout@v2
@@ -78,22 +66,17 @@ jobs:
         with:
           environment-file: dev-env-pip.yml
           environment-name: dev-env
-          micromamba-version: "0.22.0"
+          micromamba-version: "1.4.1"
 
-      
       - name: Install pyjs-code-runner
         run: |
           micromamba activate dev-env
-          emsdk install ${{matrix.emsdk_ver}}
-
           python -m pip install . 
           playwright install
         
       - name: Run Tests 
         run: |
           micromamba activate dev-env
-          emsdk activate ${{matrix.emsdk_ver}}
-          source $CONDA_EMSDK_DIR/emsdk_env.sh
 
           pytest -s
 

README.md

@@ -20,9 +20,9 @@ in browser environment is a complex process with a lot of (complicated) steps.
 Here we assume a file `main.py` located at `~/foo/bar/main.py`.
 
 ```bash
-# run with node
+# run in browser-main-thread backend 
 pyjs_code_runner run script                                                                \
-    node                                                                                   \
+    browser-main                                                                           \
     --conda-env     ~/micromamba/envs/my_env         `# the emscripten-forge env`          \
                                                      `# in which to run the code`          \
                                                                                            \
@@ -37,25 +37,10 @@ pyjs_code_runner run script
                                                                                            \
     --async-main                                     `# should a top-level async`          \
                                                      `# function named main be called`     \
-
-```
-
-
-```bash
-# run in browser-main-thread backend 
-# in a headless fashion
-pyjs_code_runner run script \
-    browser-main \
-    --conda-env     ~/micromamba/envs/my_env \
-    --mount         ~/foo/bar:/home/web_user/fubar \
-    --script        main.py \
-    --work-dir      /tests \
-    --async-main           \
     --headless 
 
 ```
 
-
 ```bash
 # run in browser-worker-thread backend 
 # in a headless fashion

dev-env.yml

@@ -7,7 +7,7 @@ dependencies:
   - pytest
   - typer
   - appdirs
-  - empack >=2.0.0
+  - empack >=3,<4
   - microsoft::playwright
   - emsdk
   - rich

pyjs_code_runner/backend/backend.py

@@ -3,7 +3,7 @@
 
 
 class BackendFamilyType(str, Enum):
-    browser = "browsef"
+    browser = "browser"
     node = "node"
 
 
@@ -61,9 +61,8 @@ def ensure_playwright_imports():
 
 def get_backend_cls(backend_type):
     if backend_type == BackendType.node:
-        from .node.node import NodeBackend
+        raise RuntimeError("the node backend is currently disabled as its not (yet) working with empack>=3.0.0")
 
-        return NodeBackend
 
     elif backend_type == BackendType.browser_main:
         ensure_playwright_imports()

pyjs_code_runner/backend/browser_main/browser_main.py

@@ -58,17 +58,7 @@ async def playwright_run_in_main_thread(self, page_url):
 
             async def handle_console(msg):
                 txt = str(msg)
-                if (
-                    txt.startswith(
-                        "warning: Browser does not support creating object URLs"
-                    )
-                    or txt.startswith("Failed to load resource:")
-                    or txt.startswith("Could not find platform dependent libraries")
-                    or txt.startswith("Consider setting $PYTHONHOME")
-                ):
-                    pass
-                else:
-                    print(txt)
+                print(txt)
 
             page.on("console", handle_console)
 

pyjs_code_runner/backend/browser_worker/browser_worker.py

@@ -101,9 +101,6 @@ async def handle_console(msg):
                     txt.startswith(
                         "warning: Browser does not support creating object URLs"
                     )
-                    or txt.startswith("Failed to load resource:")
-                    or txt.startswith("Could not find platform dependent libraries")
-                    or txt.startswith("Consider setting $PYTHONHOME")
                 ):
                     pass
                 else:

pyjs_code_runner/cli/run.py

@@ -15,19 +15,13 @@
 from ..get_file_filter import get_file_filter
 from ..constants import EMSCRIPTEN_HOME
 
-# packaging
 run_app = typer.Typer()
 app.add_typer(run_app, name="run")
 
-
-# packaging
 script_app = typer.Typer()
 run_app.add_typer(script_app, name="script")
 
 
-# constants
-
-
 def parse_mounts(mounts):
     wrong_patter_exception = lambda m: exit_with_err(
         f"wrong pattern, must be: <to_mount>:<mount_path> but is `{m}`"
@@ -54,10 +48,15 @@ def parse_mounts(mounts):
 
 conda_env_option = require_option(
     *make_names("conda-env"),
-    help="name of the conda environment in which to run the code",
+    help="host location of the conda environment in which to run the code",
 )
 script_option = require_option(
-    *make_names("script"), help="path of script INSIDE VIRTUAL FILESYSTEM to run"
+    *make_names("script"), help="path of script inside the virtual fileystem to run"
+)
+relocate_prefix_option =  typer.Option(
+    "/",
+    *make_names("relocate-prefix"),
+    help="location of the conda environment in the virtual file",
 )
 async_main_option = typer.Option(
     False,
@@ -123,6 +122,7 @@ def browser_main(
     script: Path = script_option,
     async_main: bool = async_main_option,
     mounts: List[str] = mounts_option,
+    relocate_prefix: Optional[Path] = relocate_prefix_option,
     work_dir: Optional[Path] = work_dir_option,
     pkg_file_filter: Optional[List[Path]] = pkg_file_filter_option,
     pyjs_dir: Optional[Path] = pyjs_dir_option,
@@ -136,6 +136,7 @@ def browser_main(
     run_script(
         backend_type=BackendType.browser_main,
         conda_env=conda_env,
+        relocate_prefix=relocate_prefix,
         script=script,
         async_main=async_main,
         mounts=mounts,
@@ -155,6 +156,7 @@ def browser_worker(
     script: Path = script_option,
     async_main: bool = async_main_option,
     mounts: List[str] = mounts_option,
+    relocate_prefix: Optional[Path] = relocate_prefix_option,
     work_dir: Optional[Path] = work_dir_option,
     pkg_file_filter: List[Path] = pkg_file_filter_option,
     pyjs_dir: Optional[Path] = pyjs_dir_option,
@@ -168,6 +170,7 @@ def browser_worker(
     run_script(
         backend_type=BackendType.browser_worker,
         conda_env=conda_env,
+        relocate_prefix=relocate_prefix,
         script=script,
         async_main=async_main,
         mounts=mounts,
@@ -194,6 +197,7 @@ def node(
     script: Path = script_option,
     async_main: bool = async_main_option,
     mounts: List[str] = mounts_option,
+    relocate_prefix: Optional[Path] = relocate_prefix_option,
     work_dir: Optional[Path] = work_dir_option,
     pkg_file_filter: List[Path] = pkg_file_filter_option,
     pyjs_dir: Optional[Path] = pyjs_dir_option,
@@ -205,6 +209,7 @@ def node(
     run_script(
         backend_type=BackendType.node,
         conda_env=conda_env,
+        relocate_prefix=relocate_prefix,
         script=script,
         async_main=async_main,
         mounts=mounts,
@@ -221,6 +226,7 @@ def node(
 def run_script(
     backend_type,
     conda_env,
+    relocate_prefix,
     script,
     async_main,
     mounts,
@@ -244,6 +250,7 @@ def run_script(
 
     run(
         conda_env=conda_env,
+        relocate_prefix=relocate_prefix,
         backend_type=backend_type,
         script=script,
         async_main=async_main,

pyjs_code_runner/get_file_filter.py

@@ -7,15 +7,7 @@
 
 def get_file_filter(pkg_file_filter, cache_dir):
     if pkg_file_filter is None or len(pkg_file_filter) == 0:
-
-        # empack now supports passing None
         return None
-        # path_in_cache = Path(cache_dir) / "empack_config.yaml"
-        # if not path_in_cache.exists():
-
-        #     urllib.request.urlretrieve(EMPACK_FILE_FILTER_URL, path_in_cache)
-        # return pkg_file_filter_from_yaml(path_in_cache)
-
     else:
         print(pkg_file_filter)
         return pkg_file_filter_from_yaml(*pkg_file_filter)

pyjs_code_runner/js/utils.js

@@ -1,33 +1,51 @@
+async function fetchMount(pyjs, mount){
+    let url = `./${mount}`;
+    let filename = `/mount_tarballs/${mount}`;
+    let byte_array = await pyjs._fetch_byte_array(url, filename);
+    pyjs.FS.writeFile(filename, byte_array);
+    pyjs._untar_from_python(filename, "/");
+}
+
+async function fetchMounts(pyjs) {
+    let response = await fetch("./mounts.json");
+    if (!response.ok) {
+        throw new Error(`HTTP error while fetching ./mounts.json! status: ${response.status}`);
+    }
+    let mounts = await response.json();
+    pyjs.FS.mkdir("/mount_tarballs");
+    await Promise.all(mounts.map(mount => fetchMount(pyjs, mount)));
+    
+}
+
 async function make_pyjs(print, error) {
-    var pyjs = await createModule({print:print,error:print})
-    var EmscriptenForgeModule = pyjs
-    globalThis.EmscriptenForgeModule = pyjs
-    globalThis.pyjs = pyjs
+    var pyjs = await createModule({ print: print, error: print })
+    
+   await pyjs.bootstrap_from_empack_packed_environment(
+        `./empack_env_meta.json`, /* packages_json_url */
+        ".",               /* package_tarballs_root_url */
+        false              /* verbose */
+    );
 
-    const { default: importPackages }  = await import("./packed_env.js")
-    await importPackages();
+    await fetchMounts(pyjs);
+    //await Promise.all([promise_env, promise_mount]);
 
-    const { default: importMounts }  = await import("./packed_mounts.js")
-    await importMounts();
-        
-    await pyjs.init()
+    //await pyjs.init()
 
+    globalThis.pyjs = pyjs
     return pyjs
 }
 
 globalThis.make_pyjs = make_pyjs
 
-
-
 function eval_main_script(pyjs, workdir, filename) {
-    try{
+    try {
         pyjs.exec("import os;from os.path import exists")
         pyjs.exec(`os.chdir("${workdir}")`)
         pyjs.eval_file(filename);
         return 0;
     }
-    catch(e){
-        console.error("error while evaluating main file:",e)
+    catch (e) {
+        console.error("error while evaluating main file:", e)
         return 1;
     }
     return 0
@@ -56,36 +74,25 @@ async def main_runner():
 asyncio.ensure_future(main_runner())
     `)
 
-    while(true)
-    {
+    while (true) {
         await new Promise(resolve => setTimeout(resolve, 100));
         const _async_done_ = pyjs.eval("_async_done_[0]")
-        if(_async_done_)
-        {
+        if (_async_done_) {
             break;
         }
     }
     return pyjs.eval("_ret_code[0]")
-                
+
 }
 globalThis.run_async_python_main = run_async_python_main
 
-// export { make_pyjs, run_async_python_main, eval_main_script };
-
 
-if (typeof exports === 'object' && typeof module === 'object'){
-    console.log("A")
-  module.exports = make_pyjs;
+if (typeof exports === 'object' && typeof module === 'object') {
+    module.exports = make_pyjs;
 }
-else if (typeof define === 'function' && define['amd']){
-    console.log("B")
-  define([], function() { return make_pyjs; });
+else if (typeof define === 'function' && define['amd']) {
+    define([], function () { return make_pyjs; });
 }
-else if (typeof exports === 'object'){
-    console.log("C")
-  exports["make_pyjs"] = make_pyjs;
+else if (typeof exports === 'object') {
+    exports["make_pyjs"] = make_pyjs;
 }
-// else{
-//     console.log("D")
-//     export { make_pyjs, run_async_python_main, eval_main_script };
-// }