diff --git a/.github/workflows/make-and-test.yml b/.github/workflows/make-and-test.yml index 088354e702f..a51af692607 100644 --- a/.github/workflows/make-and-test.yml +++ b/.github/workflows/make-and-test.yml @@ -39,7 +39,7 @@ jobs: detached: true - uses: actions/checkout@v4 - name: Install python3 requests - run: sudo apt-get install python3-requests + run: sudo apt-get install python3-requests python3-yapf - name: Check doc links run: cd src/scripts && python3 check_doc_urls.py || sleep 5 || python3 check_doc_urls.py @@ -91,19 +91,15 @@ jobs: # cache: "pnpm" # cache-dependency-path: "src/packages/pnpm-lock.yaml" - - name: Download and install Valkey - run: | - VALKEY_VERSION=8.1.2 - curl -LOq https://download.valkey.io/releases/valkey-${VALKEY_VERSION}-jammy-x86_64.tar.gz - tar -xzf valkey-${VALKEY_VERSION}-jammy-x86_64.tar.gz - sudo cp valkey-${VALKEY_VERSION}-jammy-x86_64/bin/valkey-server /usr/local/bin/ + - name: Install btrfs-progs and bup for @cocalc/file-server + run: sudo apt-get update && sudo apt-get install -y btrfs-progs bup - name: Set up Python venv and Jupyter kernel run: | python3 -m pip install --upgrade pip virtualenv python3 -m virtualenv venv source venv/bin/activate - pip install ipykernel + pip install ipykernel yapf python -m ipykernel install --prefix=./jupyter-local --name python3-local --display-name "Python 3 (Local)" @@ -128,30 +124,30 @@ jobs: name: "test-results-node-${{ matrix.node-version }}-pg-${{ matrix.pg-version }}" path: 'src/packages/*/junit.xml' - report: - runs-on: ubuntu-latest +# report: +# runs-on: ubuntu-latest - needs: [test] +# needs: [test] - if: ${{ !cancelled() }} +# if: ${{ !cancelled() }} - steps: - - name: Checkout code - uses: actions/checkout@v4 +# steps: +# - name: Checkout code +# uses: actions/checkout@v4 - - name: Download all test artifacts - uses: actions/download-artifact@v4 - with: - pattern: "test-results-*" - merge-multiple: true - path: test-results/ +# - name: Download all test artifacts +# uses: actions/download-artifact@v4 +# with: +# pattern: "test-results-*" +# merge-multiple: true +# path: test-results/ - - name: Test Report - uses: dorny/test-reporter@v2 - with: - name: CoCalc Jest Tests - path: 'test-results/**/junit.xml' - reporter: jest-junit - use-actions-summary: 'true' - fail-on-error: false +# - name: Test Report +# uses: dorny/test-reporter@v2 +# with: +# name: CoCalc Jest Tests +# path: 'test-results/**/junit.xml' +# reporter: jest-junit +# use-actions-summary: 'true' +# fail-on-error: false diff --git a/.gitignore b/.gitignore index b03ea399673..261ec4f771e 100644 --- a/.gitignore +++ b/.gitignore @@ -164,8 +164,24 @@ src/.claude/settings.local.json # test reports by jest-junit junit.xml + + +sea-prep.blob +cocalc +cocalc-lite.tar.* *.egg-info .python-version +src/packages/lite/sea/cocalc*gz +src/packages/lite/sea/cocalc*xz +src/packages/lite/sea/cocalc*zip +src/packages/lite/sea/cocalc*gnu + +g # autogenerated docs **/cocalc-api/site/** +*.pkg +*.zip + +src/packages/lite/build +src/packages/project-runner/build \ No newline at end of file diff --git a/docs/architecture.md b/docs/architecture.md new file mode 100644 index 00000000000..45d3701c584 --- /dev/null +++ b/docs/architecture.md @@ -0,0 +1,236 @@ +# CoCalc2 Architecture Overview (Draft) + +> This is a working draft meant to capture the current design in one place. + +--- + +## Goals & Non‑Goals + +**Goals** + +- Fast, durable, multi‑tenant project storage with clear quotas. +- Predictable save from project runner VMs to the central file server \(no “I did work but it can’t be saved”\). +- Efficient storage via transparent compression; simple mental model for users. +- Rolling snapshots for user self‑service restore. Separate quota for snapshots, which users mostly don't worry about. + +**Non‑Goals** + +- Per‑user UID separation on runners \(we rely on containerization and subvolume quotas instead\). +- Snapshots on runner VMs \(server owns snapshot history; runners are ephemeral\). + +--- + +## High‑Level Components + +1. **Central File Server** \(single large Btrfs filesystem\) + - One Btrfs **subvolume per project** \(live working set\). + - Compression enabled \(e.g., `zstd`\). + - **Qgroups/Quotas** enabled for hard limits. + - **Rolling snapshots** per project for user restore. + - Named **user created snapshots**. + +2. **Project Runner VMs** \(many; fast local SSD\) + - Also Btrfs with compression and **per‑project subvolumes**. + - Hard quotas sized slightly below the server quota to maintain save‑back headroom. + - No persistent snapshots \(might use short‑lived read only snapshots for atomic rsync of rootfs\). + +3. **Sync Layer** + - **Mutagen**: near real‑time sync for user home files. + - **rsync**: periodic sync for the container rootfs upper overlay. + +4. **Web UI & Services** + - Surfaced usage and limits \(live and snapshots\), snapshot browser/restore, warnings. + +--- + +## Storage Model & Quotas + +### Per‑Project Subvolume (File Server) + +- Each project lives at `/mnt/project-` as its **own subvolume**. +- **Compression** is enabled at the filesystem level; **quotas are enforced** _**after compression**_. +- Two distinct quota budget buckets: + - **Live quota**: applies to the live subvolume. + - **Snapshots quota**: applies to the aggregate of _all_ snapshots for that project. +- Quota for snapshots will be a simple function \(probably 2x\) of the live quota. + +### Qgroups Structure + +- Btrfs assigns each subvolume an implicit qgroup `0/`. +- We create an **aggregate qgroup** `1/` for that project’s snapshots. +- We apply limits: + - **Live**: limit `0/` \(or the path directly\) to, say, **10 GiB**. + - **Snapshots**: limit `1/` to, say, **20 GiB** total across all snapshots. +- On snapshot creation, we assign the snapshot’s `0/` **into** `1/`. +- Using the **live subvolume ID as the aggregate id** avoids external ID bookkeeping. + +### Runner VM Quotas + +- Each runner has a **per‑project subvolume** with **quota set to ~85–90%** of the server’s live quota. +- Rationale: keeps **headroom** so save‑back to the server succeeds even if compression ratios differ. + +### User‑Facing Explanation (docs‑ready blurb) + +> **Storage quota is measured after compression.** Your project has a quota that measures the actual space consumed on disk. If your data compresses well, the sum of file sizes you see in the editor may exceed your quota and still fit. Snapshots have a separate quota \(twice the project quota\) that limits how much historical data is retained. + +--- + +## Snapshots + +- **Where**: server only, per project \(no long‑term snapshots on runners\). +- **How**: periodic RO snapshots \(e.g., 15 minute/hourly/daily/weekly retention\). +- **Budget**: snapshots all share the project’s **snapshot quota** \(`1/` limit\). When the budget is exceeded, the snapshot retention policy prunes oldest automatic snapshots until under budget. Explicit user created named snapshots are not automatically deleted. +- **Self‑service**: UI lets users browse/restore from snapshots; command line restore via rsync is also supported. + +> **Note**: Runner nodes may take a **short‑lived RO snapshot** strictly for consistent `rsync` (copy‑on‑write point‑in‑time view), then delete it immediately after sync completes. This does not change policy: history lives on the server. + +--- + +## Data Flow + +1. **Active work on runner** + - User edits files in their per‑project subvolume on a runner. + - **Mutagen** streams home‑dir changes to the server nearly immediately. In case of file change conflicts the central file server always wins. + - **rsync** pushes the rootfs overlay periodically \(e.g., every minute\) from a transient snapshot for consistency. + +2. **File Server receives changes** + - Writes land in the project’s live subvolume, bounded by the live quota. + - Periodic snapshots capture history and consume from the snapshots quota. + +3. **Restore** + - Users restore individual files or directories from snapshots via UI or CLI. + +--- + +## Operational Procedures + +The following is roughly what the actual Javascript code in `packages/file-server` does. + +### One‑Time Setup (per filesystem) + +```bash +# Enable quotas once +sudo btrfs quota enable /mnt/fs +# Optional after bulk ops or enabling late +sudo btrfs quota rescan -w /mnt/fs +``` + +### Create a New Project (Server) + +```bash +# Live subvolume +sudo btrfs subvolume create /mnt/project-$PROJECT_ID + +# Set live quota (example: 10 GiB) +sudo btrfs qgroup limit 10G /mnt/project-$PROJECT_ID + +# Snapshot aggregate group uses the live subvolume ID +LIVEID=$(sudo btrfs subvolume show /mnt/project-$PROJECT_ID | awk '/Subvolume ID:/ {print $3}') + +# Create and limit the snapshots group +sudo btrfs qgroup create 1/$LIVEID /mnt/ +sudo btrfs qgroup limit 20G 1/$LIVEID /mnt/ # example snapshots budget +``` + +### Snapshot Creation (Server) + +```bash +# Create RO snapshot +TS=$(date -u +%Y%m%dT%H%M%SZ) +SNAP=/mnt/project-$PROJECT_ID/.snapshots/$TS +sudo btrfs subvolume snapshot -r /mnt/project-$PROJECT_ID "$SNAP" + +# Assign snapshot to the project’s snapshot group +SNAPID=$(sudo btrfs subvolume show "$SNAP" | awk '/ID:/ {print $2}') +LIVEID=$(sudo btrfs subvolume show /mnt/project-$PROJECT_ID | awk '/ID:/ {print $2}') +sudo btrfs qgroup assign 0/$SNAPID 1/$LIVEID /mnt +``` + +### Runner Subvolume & Quota + +```bash +# Create per‑project subvolume on runner +sudo btrfs subvolume create /runnerfs/project-$PROJECT_ID + +# Set runner quota to ~90% of server limit (example: 9 GiB) +sudo btrfs qgroup limit 9G /runnerfs/project-$PROJECT_ID +``` + +### Rsync from Runner \(optional transient snapshot\) + +```bash +# (TODO) +P=/runnerfs/projects/$PROJ +TS=$(date -u +%Y%m%dT%H%M%SZ) +rsync -aHAX --delete ... file-server:/mnt/projects-$PROJECT_ID/.local/overlay/... +``` + +### Inspecting Usage + +```bash +# Qgroup usage (referenced/exclusive, human‑readable) +sudo btrfs qgroup show -reF /mnt | less + +# Filesystem space by class (useful with compression) +sudo btrfs filesystem df /mnt +``` + +--- + +## Policies & Safety + +- **Hard quotas**: enforced by the kernel via qgroups \(both server and runner\). When a project exceeds its quota, writes fail with ENOSPC scoped to that subvolume. +- **Headroom on runners**: prevents the common failure mode where work done on a runner can’t be saved back to the server due to tighter server limits or different compression ratios. +- **User guidance**: expose a `~/scratch` directory \(separate subvolume and policy\) for large temporary files not intended for sync—reduces quota pressure on the live budget. +- **Performance knobs**: `compress=zstd[:3]`, `ssd`, `discard=async`. Consider `autodefrag` only for heavy small‑random‑write workloads. Set `chattr +C` sparingly on paths needing no‑CoW \(trades off checksumming\). +- **Dedup** on runners: optional **bees** on runners to reduce local SSD usage; measure CPU/IO overhead under realistic load. Use reflink copy\-on\-write when possible \(e.g., cloning projects\). +- **Dedup** on file server: optional **bees** to reduce disk usage. Also extensively use copy\-on\-write, e.g., when copying files between projects. + +--- + +## Failure Modes & Mitigations + +- **Runner quota exceeded** → user sees ENOSPC early; save‑back fails fast and visibly. UI should warn near 80–90%. +- **Server live quota exceeded** → incoming syncs fail; UI callouts \+ guidance to delete files or increase quota. +- **Snapshot budget exceeded** → retention pruner deletes oldest snapshots until under budget. +- **Qgroup counter drift** \(rare, after crashes/bulk ops\) → `btrfs quota rescan -w` to reconcile. +- **Filesystem nearly full** → monitor `btrfs filesystem df`; alert admins before metadata pools are pressured. + +--- + +## Observability (What to Monitor) + +- Live and snapshots usage per project (qgroup referenced/exclusive). +- Runner vs server usage deltas (to detect pathological compression differences). +- Snapshot creation latency; pruner actions count. +- Error rates from mutagen/rsync; ENOSPC events; quota rescans. + +--- + +## FAQ (User‑Facing) + +**Q: My files add up to more than my quota, but I’m not blocked. Why?** +A: Quotas measure space **after compression**. If your data compresses well, you can store more than the sum of uncompressed file sizes. + +**Q: Do snapshots count against my main quota?** +A: No. Snapshots have a **separate budget which is twice your main quota**. When that fills, older snapshots are pruned automatically. + +**Q: What happens if I hit the quota while working?** +A: New writes fail with “out of space.” Delete data or request a higher quota, then try again. + +**Q: Can I keep big temporary outputs?** +A: Use `~/scratch` \(limited retention and a separate quota\). Only the project’s live area is synced and counted against your main quota. + +--- + +## Appendix: Rationale for Design Choices + +- **Per‑project subvolumes** enable kernel‑level quotas, small blast radius, and fast deletion. +- **Server‑side snapshots only** simplify reasoning about history, save SSD cycles on runners, and reduce operational complexity. +- **Aggregate snapshot qgroup** provides a single dial for “how much history a project can accumulate.” +- **Runner quotas < server quotas** provide a simple, robust guardrail against save‑back failures due to compression variance. + +--- + +_End of draft._ + diff --git a/src/compute/.npmrc b/src/compute/.npmrc deleted file mode 100644 index 86c81f8e463..00000000000 --- a/src/compute/.npmrc +++ /dev/null @@ -1,8 +0,0 @@ -# We have to hoist @ant-design/cssinjs since it is CRUCIAL that -# the version we import in next/pages/_document.tsx is exactly the -# same was what antd itself is using. I can't think of any other -# way except for hoisting to ensure this, and of course antd doesn't -# expose the module. - -public-hoist-pattern[]=*@ant-design/cssinjs* -git-checks=false diff --git a/src/compute/README.md b/src/compute/README.md deleted file mode 100644 index d4af54cba69..00000000000 --- a/src/compute/README.md +++ /dev/null @@ -1,10 +0,0 @@ -# Compute PNPM Workspace Packages - -This is the code for external compute that gets added to cocalc from outside. - -It's in a different pnpm workspace directory since: - -- it's not needed internally as part of cocalc -- some of the websocketfs functionality is difficult to build -- we want this to be very lightweight - diff --git a/src/compute/api-client b/src/compute/api-client deleted file mode 120000 index 9d5ef76f278..00000000000 --- a/src/compute/api-client +++ /dev/null @@ -1 +0,0 @@ -../packages/api-client \ No newline at end of file diff --git a/src/compute/backend b/src/compute/backend deleted file mode 120000 index 3496879647b..00000000000 --- a/src/compute/backend +++ /dev/null @@ -1 +0,0 @@ -../packages/backend \ No newline at end of file diff --git a/src/compute/comm b/src/compute/comm deleted file mode 120000 index 77de50e6e6c..00000000000 --- a/src/compute/comm +++ /dev/null @@ -1 +0,0 @@ -../packages/comm/ \ No newline at end of file diff --git a/src/compute/compute/README.md b/src/compute/compute/README.md deleted file mode 100644 index 7683c4c1d6f..00000000000 --- a/src/compute/compute/README.md +++ /dev/null @@ -1,111 +0,0 @@ -# @cocalc/compute - -NOTE: a lot of this is out of date with the NATS, then Conat rewrites. In most cases now a compute server is exactly the same as a project. It just has a compute_server_id that is positive instead of 0. That's it. - -## Goal - -The minimal goal of this package is to connect from a nodejs process to a cocalc project, open a Jupyter notebook sync session, and provide the output. I.e., instead of the project itself running a kernel and providing output, the kernel will be provided by whatever client is running this `@cocalc/compute` package! - -Concern: I want this package to remain lightweight if at all possible, so it's fast to install and uses little space. Also, we eventually plan to run a variant of it in a web browser, which is another reason to keep it small. On the other hand, to offer a really useful Jupyter kernel environment, this will probably be part of a big Docker container or something. - -This is used by [cocalc\-compute\-docker](https://github.com/sagemathinc/cocalc-compute-docker). - -## Build - -```sh -pnpm install -pnpm build -``` - -## Examples of where to run this - -- A powerful computer \(possibly with a GPU?\) and a Jupyter kernel installed -- A web browser providing Python code evaluation via WebAssembly. - - You point a web browser on some random powerful compute you have at cocalc - - You \(and collabs\) can then use this power from cocalc on your laptop. -- A web browser with WebGPU [providing PyTorch](https://praeclarum.org/2023/05/19/webgpu-torch.html) \(say\). - -## The File System - -The file system from the project will get mounted via [WebSocketFS](https://github.com/sagemathinc/websocketfs). This will initially only be for FUSE, but later could also use WASI in the browser. - -## Status - -This is currently an unfinished work in progress. We will focus mostly on the powerful Linux host for @cocalc/compute first, since it's also what we need to make cocalc vastly more useful to people. - -We are also focusing initially on a single Jupyter notebook. However, this could also be useful for terminals and many other things. - -## Try It Out - -Define the following three environment variables: - -```sh -export API_KEY="sk-gEWEutsR9tK9q2Dd000002" -export PROJECT_ID="34ce85cd-b4ad-4786-a8f0-67fa9c729b4f" -export IPYNB_PATH="Untitled.ipynb" -``` - -- `API_KEY` -- You make this in project settings. It is specific to the project you want to connect to on https://cocalc.com: -- `PROJECT_ID` -- The project id is in the URL or project settings -- `IPYNB_PATH` -- The IPYNB_PATH is the path of a Jupyter notebook. You should have that notebook open in your browser. - -After setting the above variables, you can FUSE WebSocketFS mount the -home directory of the project and switch to using your compute for -that kernel as follows: - -```sh -cd /cocalc/src/packages/compute -node ./bin/kernel.js -``` - -### Tweaks - -Do this if you want to see VERY verbose logs: - -```sh -export DEBUG=* -export DEBUG_CONSOLE=yes -``` - -If you're using a different server, these could be relevant: - -```sh -export BASE_PATH="/" -export API_BASE_PATH="/" -export API_SERVER="https://cocalc.com" -``` - -E.g., for local dev these might be - -```sh -export BASE_PATH='/ab3c2e56-32c4-4fa5-a3ee-6fd980d10fbf/port/5000' -export API_SERVER='http://localhost:5000' -export API_BASE_PATH='/ab3c2e56-32c4-4fa5-a3ee-6fd980d10fbf/port/5000' -``` - -### Mounting just the project home directory - -Mount the project's HOME directory at /tmp/project by -running this code in nodejs after setting all of the above environment variables. - -```js -await require("@cocalc/compute").mountProject({ - project_id: process.env.PROJECT_ID, - path: "/tmp/project", -}); -0; -``` - -### Jupyter - -You should open the notebook Untitled.ipynb on [cocalc.com](http://cocalc.com). -Then set all the above env variables in another terminal and run the following code in node.js. **Running of that Jupyter notebook will then switch to your local machine.** - -```js -await require("@cocalc/compute").jupyter({ - project_id: process.env.PROJECT_ID, - path: "Untitled.ipynb", - cwd: "/tmp/project", -}); -0; -``` diff --git a/src/compute/compute/bin/start.js b/src/compute/compute/bin/start.js deleted file mode 100755 index b1505819c2e..00000000000 --- a/src/compute/compute/bin/start.js +++ /dev/null @@ -1,93 +0,0 @@ -#!/usr/bin/env node - -/* -This is to be place in /cocalc/src/packages/compute/ and run there. -Actually, it just needs @cocalc/compute to be require-able. -*/ - -process.env.BASE_PATH = process.env.BASE_PATH ?? "/"; -process.env.API_SERVER = process.env.API_SERVER ?? "https://cocalc.com"; -process.env.API_BASE_PATH = process.env.API_BASE_PATH ?? "/"; - -const { mountProject, jupyter } = require("@cocalc/compute"); - -const PROJECT_HOME = "/home/user"; - -async function main() { - let unmount = null; - let kernel = null; - let term = null; - const exitHandler = async () => { - console.log("cleaning up..."); - process.removeListener("exit", exitHandler); - process.removeListener("SIGINT", exitHandler); - process.removeListener("SIGTERM", exitHandler); - await unmount?.(); - process.exit(); - }; - - process.on("exit", exitHandler); - process.on("SIGINT", exitHandler); - process.on("SIGTERM", exitHandler); - - const { apiKey } = require("@cocalc/backend/data"); - try { - if (!process.env.PROJECT_ID) { - throw Error("You must set the PROJECT_ID environment variable"); - } - - if (!apiKey) { - throw Error("You must set the API_KEY environment variable"); - } - } catch (err) { - const help = () => { - console.log(err.message); - console.log( - "See https://github.com/sagemathinc/cocalc-compute-docker#readme", - ); - }; - help(); - setInterval(help, 5000); - return; - } - - console.log("Mounting project", process.env.PROJECT_ID, "at", PROJECT_HOME); - try { - unmount = await mountProject({ - project_id: process.env.PROJECT_ID, - path: PROJECT_HOME, - }); - - if (process.env.IPYNB_PATH) { - console.log("Connecting to", process.env.IPYNB_PATH); - kernel = await jupyter({ - project_id: process.env.PROJECT_ID, - path: process.env.IPYNB_PATH, - cwd: PROJECT_HOME, - }); - } - } catch (err) { - console.log("something went wrong ", err); - exitHandler(); - } - - const info = () => { - console.log("Success!"); - - if (process.env.IPYNB_PATH) { - console.log( - `Your notebook ${process.env.IPYNB_PATH} should be running in this container.`, - ); - console.log( - ` ${process.env.API_SERVER}/projects/${process.env.PROJECT_ID}/files/${process.env.IPYNB_PATH}`, - ); - } - - console.log(`Your home directory is mounted at ${PROJECT_HOME}`); - console.log("\nPress Control+C to exit."); - }; - - info(); -} - -main(); diff --git a/src/compute/compute/dev/1-websocketfs.sh b/src/compute/compute/dev/1-websocketfs.sh deleted file mode 100755 index b5cc97cbdc6..00000000000 --- a/src/compute/compute/dev/1-websocketfs.sh +++ /dev/null @@ -1,14 +0,0 @@ -#!/usr/bin/env bash - -set -v - -. env.sh - -fusermount -uz $UNIONFS_LOWER 2>/dev/null || true - - -export PROJECT_HOME=$UNIONFS_LOWER -unset UNIONFS_LOWER -unset UNIONFS_UPPER - -node ./start-filesystem.js diff --git a/src/compute/compute/dev/2-syncfs.sh b/src/compute/compute/dev/2-syncfs.sh deleted file mode 100755 index 6f95122153f..00000000000 --- a/src/compute/compute/dev/2-syncfs.sh +++ /dev/null @@ -1,9 +0,0 @@ -#!/usr/bin/env bash - -set -v - -. env.sh - -fusermount -uz $UNIONFS_UPPER 2>/dev/null || true - -node ./start-filesystem.js diff --git a/src/compute/compute/dev/3-compute.sh b/src/compute/compute/dev/3-compute.sh deleted file mode 100755 index c4e28e2228a..00000000000 --- a/src/compute/compute/dev/3-compute.sh +++ /dev/null @@ -1,7 +0,0 @@ -#!/usr/bin/env bash - -set -v - -. env.sh - -node ./start-compute.js diff --git a/src/compute/compute/dev/4-startup-script.sh b/src/compute/compute/dev/4-startup-script.sh deleted file mode 100755 index e9f4edee70c..00000000000 --- a/src/compute/compute/dev/4-startup-script.sh +++ /dev/null @@ -1,62 +0,0 @@ -#!/bin/bash - -# This is a stripped down startup script for testng mainly the state reporting. -# To really do dev with a simulated compute server, make sure to run -# ./filesystem.sh -# ./syncfs.sh -# .compute.sh -# in turn in three separate terminals, after correctly figuring out the contents of the -# conf directory, based, e.g., on the on prem run script. The api_server can be -# especially tricky to untangle when doing dev. - -# NOTE: this doesn't work with the new openapi validation, so you have to do -# -# export COCALC_DISABLE_API_VALIDATION=yes -# -# before running your hub dev server to disable that. - -set -v - -. env.sh - -function setState { - id=$COMPUTE_SERVER_ID - name=$1 - state=${2:-'ready'} - extra=${3:-''} - timeout=${4:-0} - progress=${5:-100} - project_id=$PROJECT_ID - - echo "$name is $state" - PAYLOAD="{\"id\":$id,\"name\":\"$name\",\"state\":\"$state\",\"extra\":\"$extra\",\"timeout\":$timeout,\"progress\":$progress,\"project_id\":\"$project_id\"}" - echo $PAYLOAD - curl -sk -u $API_KEY: -H 'Content-Type: application/json' -d $PAYLOAD $API_SERVER/api/v2/compute/set-detailed-state -} - - -setState state running - -sleep 0.1 -setState install configure '' 60 10 - - -setState install install-docker '' 120 20 -sleep 0.1 -setState install install-nodejs 60 50 -sleep 0.1 -setState install install-cocalc '' 60 70 -sleep 0.1 -setState install install-user '' 60 80 -sleep 0.1 -setState install ready '' 0 100 - -setState vm start '' 60 60 -sleep 0.1 - -while true; do - setState compute ready '' 35 100 - setState filesystem-sync ready '' 35 100 - setState vm ready '' 35 100 - sleep 30 -done diff --git a/src/compute/compute/dev/README.md b/src/compute/compute/dev/README.md deleted file mode 100644 index 560147a228a..00000000000 --- a/src/compute/compute/dev/README.md +++ /dev/null @@ -1,49 +0,0 @@ -The scripts here are helpful for developing the compute\-server manager, which is defined in this package. - -1. Create the directory /tmp/user and make sure you can read it. Maybe even mount it from the target project. - -2. The conf/ directory here has the same files as on /cocalc/conf in an actual compute\-server, except: - - replace api_server by something like `http://127.0.0.1:5000/6659c2e3-ff5e-4bb4-9a43-8830aa951282/port/5000`, where the port is what you're using for your dev server and the project id is of your dev server. The point is that we're going to connect directly without going through some external server. - - api_key: the one from an actual server will get deleted when you turn that server off, so make a different project level api key. - - Type `tar xvf conf.tar` to get a template for the conf directory. - You will need to change the contents of all the files you get, as - mentioned above! Also, regarding the api_server, be especially careful - about ipv4 versus ipv6, e.g., use 127.0.0.1 instead of localhost to - nail down the protocol. - -This is potentially confusing, and when developing this it was 10x worse... Maybe you'll be confused for 2 hours instead of 2 days. - -3. Run each of the following four shell scripts in different terminals, in order. - -```sh -1-websocketfs.sh -2-syncfs.sh -3-compute.sh -4-startup-script.sh -``` - -However, a bunch of things are likely to go wrong. - -**Problem:** Regarding the id of the compute server in the file [conf/compute\_server\_id](./conf/compute_server_id), create a self\-hosted compute server in the project on your dev server, then find the record in the postgresql database by querying the `compute_servers` table, and copy the id field from that. Note that the displayed id in the UI starts from 1 for each project, but `compute_server_id` must be the id in the database. - -**Problem:** Get the [conf/api_key](./conf/api_key) by clicking start on the self\-hosted compute server, inspect the URL, and copy it from there. If you stop the server explicitly, then the api key is deleted from the project, so you need to make it again. - -**Problem:** The scripts `1-websocketfs.sh` and `2-syncfs.sh` will definitely fail if support for FUSE isn't enabled for normal users where you are working! Test bindfs locally. - -**Problem:** For `2-syncfs.sh`, you must also install unionfs\-fuse via `sudo apt install unionfs-fuse,` since the cocalc package @cocalc/sync\-fs assumes unionfs\-fuse is installed. - -**Problem:** You need to do the following so that you can fully test the scratch functionality \(see [conf/exclude_from_sync](./conf/exclude_from_sync)\): - -```sh -sudo mkdir -p /data/scratch && sudo chown -R `whoami`:`whoami` /data -``` - -Once you get the 4 scripts above to run, the net result is basically the same as using a compute server, but you can run it all locally, and development and debugging is ~~massively easier~~ possible! Without something like this, development is impossible, and even figuring out what configuration goes where could cost me days of confusion \(even though I wrote it all!\). It's complicated. - -For debugging set the DEBUG env variable to different things according to the debug npm module. E.g., - -```sh -DEBUG_CONSOLE=yes DEBUG=* ./2-syncfs.sh -``` - diff --git a/src/compute/compute/dev/conf.tar b/src/compute/compute/dev/conf.tar deleted file mode 100644 index 850237833b3..00000000000 Binary files a/src/compute/compute/dev/conf.tar and /dev/null differ diff --git a/src/compute/compute/dev/env.sh b/src/compute/compute/dev/env.sh deleted file mode 100755 index 2dbad3327ac..00000000000 --- a/src/compute/compute/dev/env.sh +++ /dev/null @@ -1,33 +0,0 @@ -#!/usr/bin/env bash - -unset NATS_SERVER -unset COCALC_NATS_SERVER -unset COCALC_PROJECT_ID -unset COCALC_USERNAME - -export API_KEY=`cat conf/api_key` -export API_SERVER=`cat conf/api_server` -export PROJECT_ID=`cat conf/project_id` -export COMPUTE_SERVER_ID=`cat conf/compute_server_id` -export HOSTNAME=`cat conf/hostname` -export UNIONFS_UPPER=/tmp/upper5002 -export UNIONFS_LOWER=/tmp/lower5002 -export PROJECT_HOME=/tmp/home5002 -export READ_TRACKING_FILE=/tmp/reads5002 -export METADATA_FILE=$UNIONFS_LOWER/.compute-servers/$COMPUTE_SERVER_ID/meta/meta.lz4 -export EXCLUDE_FROM_SYNC=`cat conf/exclude_from_sync` - -echo API_KEY=$API_KEY -echo API_SERVER=$API_SERVER -echo PROJECT_ID=$PROJECT_ID -echo COMPUTE_SERVER_ID=$COMPUTE_SERVER_ID -echo HOSTNAME=$HOSTNAME -echo UNIONFS_UPPER=$UNIONFS_UPPER -echo UNIONFS_LOWER=$UNIONFS_LOWER -echo PROJECT_HOME=$PROJECT_HOME -echo READ_TRACKING_FILE=$READ_TRACKING_FILE -echo METADATA_FILE=$METADATA_FILE - -mkdir -p $UNIONFS_UPPER -mkdir -p $UNIONFS_LOWER -mkdir -p $PROJECT_HOME diff --git a/src/compute/compute/dev/start-compute.js b/src/compute/compute/dev/start-compute.js deleted file mode 100644 index 7fae8fbbbd9..00000000000 --- a/src/compute/compute/dev/start-compute.js +++ /dev/null @@ -1,36 +0,0 @@ -#!/usr/bin/env node - -process.env.API_SERVER = process.env.API_SERVER ?? "https://cocalc.com"; -console.log("API_SERVER=", process.env.API_SERVER); -const PROJECT_HOME = process.env.PROJECT_HOME ?? "/tmp/home"; -const project_id = process.env.PROJECT_ID; -process.env.COCALC_PROJECT_ID = project_id; -process.env.COCALC_USERNAME = project_id.replace(/-/g, ""); -process.env.HOME = PROJECT_HOME; - -const { manager } = require("../dist/lib"); - -async function main() { - const exitHandler = async () => { - console.log("cleaning up..."); - process.removeListener("exit", exitHandler); - process.removeListener("SIGINT", exitHandler); - process.removeListener("SIGTERM", exitHandler); - process.exit(); - }; - - process.on("exit", exitHandler); - process.on("SIGINT", exitHandler); - process.on("SIGTERM", exitHandler); - - const M = manager({ - waitHomeFilesystemType: - process.env.UNIONFS_UPPER && process.env.UNIONFS_LOWER - ? "fuse.unionfs-fuse" - : "fuse", - }); - exports.manager = M; - await M.init(); -} - -main(); diff --git a/src/compute/compute/dev/start-filesystem.js b/src/compute/compute/dev/start-filesystem.js deleted file mode 100644 index 499e8b9089f..00000000000 --- a/src/compute/compute/dev/start-filesystem.js +++ /dev/null @@ -1,81 +0,0 @@ -#!/usr/bin/env node - -process.env.API_SERVER = process.env.API_SERVER ?? "https://cocalc.com"; -console.log("API_SERVER=", process.env.API_SERVER); -const PROJECT_HOME = process.env.PROJECT_HOME ?? "/tmp/home"; -const project_id = process.env.PROJECT_ID; -process.env.COCALC_PROJECT_ID = project_id; -process.env.COCALC_USERNAME = project_id.replace(/-/g, ""); -process.env.HOME = PROJECT_HOME; - -console.log("API_SERVER=", process.env.API_SERVER); - -const { mountProject } = require("../dist/lib"); - -const EXCLUDE_FROM_SYNC = process.env.EXCLUDE_FROM_SYNC ?? ""; - -async function main() { - let unmount = null; - let kernel = null; - let term = null; - const exitHandler = async () => { - console.log("cleaning up..."); - process.removeListener("exit", exitHandler); - process.removeListener("SIGINT", exitHandler); - process.removeListener("SIGTERM", exitHandler); - await unmount?.(); - process.exit(); - }; - - process.on("exit", exitHandler); - process.on("SIGINT", exitHandler); - process.on("SIGTERM", exitHandler); - - const { apiKey } = require("@cocalc/backend/data"); - let unionfs; - if (process.env.UNIONFS_UPPER && process.env.UNIONFS_LOWER) { - unionfs = { - lower: process.env.UNIONFS_LOWER, - upper: process.env.UNIONFS_UPPER, - waitLowerFilesystemType: "fuse", - }; - } else { - unionfs = undefined; - } - - console.log("Mounting project", process.env.PROJECT_ID, "at", PROJECT_HOME); - const exclude = [".*"].concat( - EXCLUDE_FROM_SYNC ? EXCLUDE_FROM_SYNC.split("|") : [], - ); - console.log("exclude = ", exclude); - try { - exports.fs = await mountProject({ - project_id: process.env.PROJECT_ID, - path: PROJECT_HOME, - // NOTE: allowOther is disabled by default on Ubuntu and we do not need it. - options: { mountOptions: { allowOther: false, nonEmpty: true } }, - unionfs, - readTrackingFile: process.env.READ_TRACKING_FILE, - exclude, - metadataFile: process.env.METADATA_FILE, - syncIntervalMin: 60 * 5, - syncIntervalMax: 60 * 15, - cacheTimeout: 0, // websocketfs -- critical to not use its cache, which is very painful for cocalc, e.g., when making new files. - }); - unmount = exports.fs.unmount; - } catch (err) { - console.trace("something went wrong ", err); - exitHandler(); - return; - } - - const info = () => { - console.log("Success!"); - console.log(`Home directory is mounted at ${PROJECT_HOME}`); - console.log("\nPress Control+C to exit."); - }; - - info(); -} - -main(); diff --git a/src/compute/compute/lib/file-server.ts b/src/compute/compute/lib/file-server.ts deleted file mode 100644 index c4d61a80d97..00000000000 --- a/src/compute/compute/lib/file-server.ts +++ /dev/null @@ -1,47 +0,0 @@ -/* -Simple file server to manage sync for one specific file. - - -*/ - -import SyncClient from "@cocalc/sync-client"; -import debug from "debug"; - -const log = debug("cocalc:compute:file-server"); - -export async function fileServer({ client, path }) { - const x = new FileServer(path); - await x.init(client); - return x; -} - -class FileServer { - private path: string; - private syncdoc; - - constructor(path: string) { - this.path = path; - this.log("constructor"); - } - - init = async (client: SyncClient) => { - this.log("init: open_existing_sync_document"); - this.syncdoc = await client.sync_client.open_existing_sync_document({ - project_id: client.project_id, - path: this.path, - }); - }; - - private log = (...args) => { - log(`FileServer("${this.path}")`, ...args); - }; - - close = async () => { - if (this.syncdoc == null) { - return; - } - this.syncdoc.close(); - delete this.syncdoc; - this.log("close: done"); - }; -} diff --git a/src/compute/compute/lib/filesystem.ts b/src/compute/compute/lib/filesystem.ts deleted file mode 100644 index 972bab36a2b..00000000000 --- a/src/compute/compute/lib/filesystem.ts +++ /dev/null @@ -1,281 +0,0 @@ -/* -Mount a remote CoCalc project's file system locally over a websocket using FUSE. - - await require('.').mount({remote:'wss://cocalc.com/10f0e544-313c-4efe-8718-2142ac97ad11/raw/.smc/websocketfs',path:process.env.HOME + '/dev2', connectOptions:{perMessageDeflate: false, headers: {Cookie: require('cookie').serialize('api_key', 'sk-at7ALcGBKMbzq7Vc00000P')}}}) - - -*/ - -import { apiKey } from "@cocalc/backend/data"; -import { mount } from "websocketfs"; -import getLogger from "@cocalc/backend/logger"; -import { project } from "@cocalc/api-client"; -import { serialize } from "cookie"; -import { API_COOKIE_NAME } from "@cocalc/backend/auth/cookie-names"; -import syncFS from "@cocalc/sync-fs"; -import { - pingProjectUntilSuccess, - waitUntilFilesystemIsOfType, - getProjectWebsocketUrl, -} from "./util"; -import { apiCall } from "@cocalc/api-client"; -import sendFiles from "./send-files"; -import getFiles from "./get-files"; -// ensure that the conat client is initialized so that syncfs can connect properly. -import "@cocalc/project/conat"; - -const logger = getLogger("compute:filesystem"); - -interface Options { - // which project -- defaults to process.env.PROJECT_ID, which must be given if this isn't - project_id?: string; - // path to mount at -- defaults to '/home/user' - path?: string; - // these options are passed on to the websocketfs mount command - options?; - // options used for unionfs caching, which is used only if these two directories are set. - // They should be empty directories that exists and user has write access to, and they - // - lower = where websocketfs is mounted - // - upper = local directory used for caching. - unionfs?: { - upper: string; - lower: string; - // If true, doesn't do anything until the type of the file system that lower is - // mounted on is of this type, e.g., "fuse". This is done *INSTEAD OF* just - // trying to mount that file system. Why? because in docker we hit a deadlock - // when trying to do both in the same process (?), which I can't solve -- maybe - // a bug in node. In any case, separating the unionfs into a separate container - // is nice anyways. - waitLowerFilesystemType?: string; - }; - compute_server_id?: number; - cacheTimeout?: number; - syncIntervalMin?: number; - syncIntervalMax?: number; - exclude?: string[]; - readTrackingFile?: string; - metadataFile?: string; -} - -export async function mountProject({ - project_id = process.env.PROJECT_ID, - path = "/home/user", // where to mount the project's HOME directory - unionfs, - options, - compute_server_id = parseInt(process.env.COMPUTE_SERVER_ID ?? "0"), - cacheTimeout, - syncIntervalMin, - syncIntervalMax, - exclude = [], - readTrackingFile, - metadataFile, -}: Options = {}) { - const log = (...args) => logger.debug(path, ...args); - const reportState = async ( - type: "cache" | "network" | "filesystem", - opts: { state; extra?; timeout?; progress? }, - ) => { - log("reportState", { type, opts }); - try { - await apiCall("v2/compute/set-detailed-state", { - id: compute_server_id, - name: type == "filesystem" ? "filesystem" : `filesystem-${type}`, - ...opts, - }); - } catch (err) { - log("reportState: WARNING -- ", err); - } - }; - log(); - try { - if (!compute_server_id) { - throw Error("set the compute_server_id or process.env.COMPUTE_SERVER_ID"); - } - if (!project_id) { - throw Error("project_id or process.env.PROJECT_ID must be given"); - } - if (!apiKey) { - throw Error("api key must be set (e.g., set API_KEY env variable)"); - } - - // Ping to start project so it's possible to mount. - await pingProjectUntilSuccess(project_id); - - const remote = getProjectWebsocketUrl(project_id) + "/websocketfs"; - log("connecting to ", remote); - const headers = { Cookie: serialize(API_COOKIE_NAME, apiKey) }; - // SECURITY: DO NOT log headers and connectOptions, obviously! - - let homeMountPoint; - if (unionfs == null) { - homeMountPoint = path; - } else { - homeMountPoint = unionfs.lower; - if (!unionfs.lower || !unionfs.upper) { - throw Error( - "if unionfs is specified, both lower and upper must be set", - ); - } - } - - let unmount; - let pingInterval: null | ReturnType = null; - if (unionfs?.waitLowerFilesystemType) { - // we just wait for it to get mounted in some other way - unmount = null; - reportState("cache", { - state: "waiting", - timeout: 120, - progress: 30, - }); - await waitUntilFilesystemIsOfType( - unionfs.lower, - unionfs?.waitLowerFilesystemType, - ); - } else { - // we mount it ourselves. - reportState("network", { - state: "mounting", - timeout: 120, - progress: 30, - }); - - const websocketfsMountOptions = { - remote, - path: homeMountPoint, - ...options, - connectOptions: { - perMessageDeflate: false, - headers, - ...options.connectOptions, - }, - mountOptions: { - ...options.mountOptions, - allowOther: true, // this is critical to allow for fast bind mounts of scratch etc. as root. - nonEmpty: true, - }, - cacheTimeout, - hidePath: "/.unionfs", - // timeout = only track files that were read this recently - // update = update read tracking file this frequently - // modified = ignore any file modified with this many seconds (at least); - // also ignores any file not in the stat cache. - readTrackingFile: readTrackingFile, - readTrackingExclude: exclude, - // metadata file - metadataFile, - }; - - log("websocketfs -- mount options", websocketfsMountOptions); - - try { - ({ unmount } = await mount(websocketfsMountOptions)); - } catch (err) { - log("failed trying to mount -- ", err); - log( - "try again without allowOther, since some versions of FUSE do not support this option", - ); - websocketfsMountOptions.mountOptions.allowOther = false; - ({ unmount } = await mount(websocketfsMountOptions)); - - // This worked so the problem is allow_other. - throw Error( - "fusermount: option allow_other only allowed if 'user_allow_other' is set in /etc/fuse.conf\n\n\nFix this:\n\n sudo sed -i 's/#user_allow_other/user_allow_other/g' /etc/fuse.conf\n\n\n", - ); - } - - pingInterval = setInterval(async () => { - try { - await project.ping({ project_id }); - log("ping project -- SUCCESS"); - } catch (err) { - log(`ping project -- ERROR '${err}'`); - } - }, 30000); - reportState("network", { state: "ready", progress: 100 }); - } - - let syncfs; - if (unionfs != null) { - if (/\s/.test(unionfs.lower) || /\s/.test(unionfs.upper)) { - throw Error("paths cannot contain whitespace"); - } - - syncfs = syncFS({ - role: "compute_server", - lower: unionfs.lower, - upper: unionfs.upper, - mount: path, - project_id, - compute_server_id, - syncIntervalMin, - syncIntervalMax, - exclude, - readTrackingFile, - tar: { - send: async ({ - createArgs, - extractArgs, - HOME = unionfs.upper, - }: { - createArgs: string[]; - extractArgs: string[]; - HOME?: string; - }) => - await sendFiles({ - createArgs, - extractArgs, - project_id, - HOME, - }), - get: async ({ - createArgs, - extractArgs, - HOME = unionfs.upper, - }: { - createArgs: string[]; - extractArgs: string[]; - HOME?: string; - }) => - await getFiles({ - createArgs, - extractArgs, - project_id, - HOME, - }), - }, - }); - await syncfs.init(); - reportState("cache", { state: "ready", progress: 100 }); - } else { - syncfs = null; - } - - reportState("filesystem", { state: "ready", progress: 100 }); - - return { - syncfs, - unmount: async () => { - if (pingInterval) { - clearInterval(pingInterval); - pingInterval = null; - } - if (syncfs != null) { - await syncfs.close(); - } - if (unmount != null) { - logger.debug("unmount"); - unmount(); - } - }, - }; - } catch (err) { - const e = `${err}`; - reportState(unionfs != null ? "cache" : "network", { - state: "error", - extra: e, - }); - log(e); - throw err; - } -} diff --git a/src/compute/compute/lib/get-files.ts b/src/compute/compute/lib/get-files.ts deleted file mode 100644 index 8462aaf945c..00000000000 --- a/src/compute/compute/lib/get-files.ts +++ /dev/null @@ -1,70 +0,0 @@ -/* -Get files from the project over a websocket. - -We don't call this "receive" because it's isn't just passively waiting to -receive something. -*/ - -import { apiKey } from "@cocalc/backend/data"; -import { API_COOKIE_NAME } from "@cocalc/backend/auth/cookie-names"; -import { getProjectWebsocketUrl } from "./util"; -import { join } from "path"; -import recvFilesWS from "@cocalc/sync-fs/lib/recv-files"; -import getLogger from "@cocalc/backend/logger"; -import { serialize } from "cookie"; -import WebSocket from "ws"; -import { callback } from "awaiting"; - -const logger = getLogger("compute:recv-files"); - -interface Options { - project_id: string; - // used to make the tarball - createArgs: string[]; - // used when extracting the tarball - extractArgs: string[]; - // our local HOME here - HOME?: string; -} - -export default async function getFiles({ - project_id, - createArgs, - extractArgs, - HOME = process.env.HOME, -}: Options) { - logger.debug("getFiles:", { project_id, createArgs, extractArgs, HOME }); - await callback(doGetFiles, project_id, createArgs, extractArgs, HOME); - logger.debug("getFiles: done!"); -} - -function doGetFiles(project_id: string, createArgs, extractArgs, HOME, cb) { - const remote = join(getProjectWebsocketUrl(project_id), "sync-fs", "send"); - logger.debug("connecting to ", remote); - const headers = { Cookie: serialize(API_COOKIE_NAME, apiKey) }; - const ws = new WebSocket(remote, { headers }); - - ws.on("open", () => { - logger.debug("connected to ", remote); - // tell it how/what to send us files - ws.send(JSON.stringify(createArgs)); - // receive the files - recvFilesWS({ ws, HOME, args: extractArgs }); - }); - - ws.on("close", () => { - cb?.(); - cb = undefined; - }); - - ws.on("end", () => { - cb?.(); - cb = undefined; - }); - - ws.on("error", (err) => { - logger.debug("ERROR -- ", err); - cb?.(err); - cb = undefined; - }); -} diff --git a/src/compute/compute/lib/index.ts b/src/compute/compute/lib/index.ts deleted file mode 100644 index 46900d38b1b..00000000000 --- a/src/compute/compute/lib/index.ts +++ /dev/null @@ -1,4 +0,0 @@ -export { mountProject } from "./filesystem"; -export { tasks } from "./tasks"; -export { jupyter } from "./jupyter"; -export { manager } from "./manager"; diff --git a/src/compute/compute/lib/jupyter.ts b/src/compute/compute/lib/jupyter.ts deleted file mode 100644 index 3387c54989d..00000000000 --- a/src/compute/compute/lib/jupyter.ts +++ /dev/null @@ -1,165 +0,0 @@ -/* -Connect from this nodejs process to a remote cocalc project over a websocket and -provide a remote Jupyter backend session. -*/ - -import SyncClient from "@cocalc/sync-client"; -import { meta_file } from "@cocalc/util/misc"; -import { initJupyterRedux } from "@cocalc/jupyter/kernel"; -import { redux } from "@cocalc/jupyter/redux/app"; -import debug from "debug"; -import { once } from "@cocalc/util/async-utils"; -import { COMPUTE_THRESH_MS } from "@cocalc/util/compute/manager"; -import { SYNCDB_OPTIONS } from "@cocalc/jupyter/redux/sync"; -import { reuseInFlight } from "@cocalc/util/reuse-in-flight"; - -const log = debug("cocalc:compute:jupyter"); - -export function jupyter({ client, path }) { - return new RemoteJupyter({ client, path }); -} - -class RemoteJupyter { - private client: SyncClient; - private websocket; - private path: string; - private syncdb; - private actions?; - private store?; - private interval; - - constructor({ client, path }: { client: SyncClient; path: string }) { - log("creating remote jupyter session"); - this.client = client; - this.path = path; - this.log("constructor"); - const syncdb_path = meta_file(path, "jupyter2"); - - this.syncdb = client.sync_client.sync_db({ - ...SYNCDB_OPTIONS, - project_id: client.project_id, - path: syncdb_path, - }); - this.registerWithProject(); - this.initRedux(); - } - - private log = (...args) => { - log(`RemoteJupyter("${this.path}")`, ...args); - }; - - close = async () => { - if (this.syncdb == null) { - return; - } - this.log("close"); - clearInterval(this.interval); - - this.log("save_asap"); - await this.actions.save_asap(); - this.log("halt kernel"); - await this.actions.halt(); - - const { syncdb } = this; - delete this.syncdb; - syncdb.removeAllListeners("message"); - - // Stop listening for messages, since as we start to close - // things before, handling messages would lead to a crash. - // clear our cursor, so project immediately knows that we disconnected. - this.log("close: closing actions..."); - // we have to explicitly disable save here, since things are just - // too complicated to properly do the close with a save after - // we already started doing the close. - this.actions.close({ noSave: true }); - this.log("close: actions closed"); - delete this.actions; - delete this.store; - this.log("close: clearing cursors..."); - await syncdb.setCursorLocsNoThrottle([]); - await syncdb.close(); - this.log("close: done"); - }; - - // On reconnect, this registerWithProject can in some cases get - // called a bunch of times at once, so the reuseInFlight is - // very important. Otherwise, we end over a long time with - // many disconnect and reconnects, with eventually more and - // more attempts to register, and the process crashes and runs - // out of memory. - private registerWithProject = reuseInFlight(async () => { - if (this.syncdb == null) { - return; - } - this.log("registerWithProject"); - this.websocket = await this.client.project_client.websocket( - this.client.project_id, - ); - this.log("registerWithProject: got websocket"); - if (this.syncdb.get_state() == "init") { - await once(this.syncdb, "ready"); - } - this.log("registerWithProject: syncdb ready"); - // Register to handle websocket api requests from frontend - // clients to the project jupyter instance. - try { - await this.syncdb.sendMessageToProject({ - event: "register-to-handle-api", - }); - } catch (err) { - this.log("WARNING: failed to register -- ", err); - return; - } - this.log("registerWithProject: sent register-to-handle-api"); - - // Periodically update cursor to indicate that we would like - // to handle code evaluation, i.e., we are the cell running. - if (this.interval) { - clearInterval(this.interval); - delete this.interval; - } - const registerAsCellRunner = async () => { - this.log("registerAsCellRunner"); - if (this.syncdb == null) { - return; - } - this.syncdb.registerAsComputeServer(); - // we also continually also register to handle messages, just in case - // the above didn't get through (e.g., right when restarting project). - await this.syncdb.sendMessageToProject({ - event: "register-to-handle-api", - }); - }; - this.interval = setInterval(registerAsCellRunner, COMPUTE_THRESH_MS / 2); - registerAsCellRunner(); - - // remove it first, in case it was already installed: - this.websocket.removeListener("state", this.handleWebsocketStateChange); - this.websocket.on("state", this.handleWebsocketStateChange); - }); - - private handleWebsocketStateChange = (state) => { - if (state == "offline") { - // no point in registering with server while offline - clearInterval(this.interval); - delete this.interval; - } else if (state == "online") { - this.log("websocket online"); - this.registerWithProject(); - } - }; - - private initRedux = async () => { - this.log("initializing jupyter redux..."); - await initJupyterRedux(this.syncdb, this.client); - const { project_id } = this.client; - const { path } = this; - this.actions = redux.getEditorActions(project_id, path); - if (this.actions.is_closed()) { - throw Error( - `initRedux -- actions can't be closed already (path="${path}")`, - ); - } - this.store = redux.getEditorStore(project_id, path); - }; -} diff --git a/src/compute/compute/lib/listings.ts b/src/compute/compute/lib/listings.ts deleted file mode 100644 index d7181edd373..00000000000 --- a/src/compute/compute/lib/listings.ts +++ /dev/null @@ -1,63 +0,0 @@ -/* -Manage listings table from the perspective of this compute server. -*/ - -import { registerListingsTable } from "@cocalc/sync/listings"; -import getLogger from "@cocalc/backend/logger"; -import { existsSync } from "fs"; -import getListing0 from "@cocalc/backend/get-listing"; -import { Watcher } from "@cocalc/backend/path-watcher"; - -const logger = getLogger("compute:listings"); - -export async function initListings({ - client, - project_id, - compute_server_id, - home, -}: { - client; - project_id: string; - compute_server_id: number; - home: string; -}) { - logger.debug("initListings", { project_id, compute_server_id }); - - const table = await client.synctable_project( - project_id, - { - listings: [ - { - project_id, - compute_server_id, - path: null, - listing: null, - time: null, - interest: null, - missing: null, - error: null, - deleted: null, - }, - ], - }, - [], - ); - - const getListing = async (path: string, hidden: boolean) => { - return await getListing0(path, hidden, home); - }; - - registerListingsTable({ - table, - project_id, - compute_server_id, - getListing, - createWatcher: (path: string, debounce: number) => - new Watcher(path, { debounce }), - onDeletePath: (path) => { - logger.debug("onDeletePath -- TODO:", { path }); - }, - existsSync, - getLogger, - }); -} diff --git a/src/compute/compute/lib/manager.ts b/src/compute/compute/lib/manager.ts deleted file mode 100644 index 3a6ff4ad451..00000000000 --- a/src/compute/compute/lib/manager.ts +++ /dev/null @@ -1,138 +0,0 @@ -/* -The manager does the following: - -- Waits until the filesystem is mounted -- Then connects to NATS in the same was as a project, but with compute_server_id positive. - -*/ - -import debug from "debug"; -import startProjectServers from "@cocalc/project/conat"; -import { pingProjectUntilSuccess, waitUntilFilesystemIsOfType } from "./util"; -import { apiCall, project } from "@cocalc/api-client"; - -const logger = debug("cocalc:compute:manager"); - -const STATUS_INTERVAL_MS = 20 * 1000; - -interface Options { - waitHomeFilesystemType?: string; -} - -process.on("exit", () => { - console.log("manager has exited"); -}); - -const STARS = - "\nBUG ****************************************************************************\n"; -process.on("uncaughtException", (err) => { - console.trace(err); - console.error(STARS); - console.error(`uncaughtException: ${err}`); - console.error(err.stack); - console.error(STARS); -}); -process.on("unhandledRejection", (err) => { - console.trace(err); - console.error(STARS); - console.error(typeof err); - console.error(`unhandledRejection: ${err}`); - console.error((err as any)?.stack); - console.error(STARS); -}); - -export function manager(opts: Options) { - return new Manager(opts); -} - -class Manager { - private state: "new" | "init" | "ready" = "new"; - private project_id: string; - private home: string; - private waitHomeFilesystemType?: string; - private compute_server_id: number; - - constructor({ waitHomeFilesystemType }: Options) { - this.waitHomeFilesystemType = waitHomeFilesystemType; - // default so that any windows that any user apps in terminal or jupyter run will - // automatically just work in xpra's X11 desktop.... if they happen to be running it. - process.env.DISPLAY = ":0"; - if (!process.env.COMPUTE_SERVER_ID) { - throw Error("env variable COMPUTE_SERVER_ID must be set"); - } - this.compute_server_id = parseInt(process.env.COMPUTE_SERVER_ID); - if (!process.env.HOME) { - throw Error("HOME must be set"); - } - this.home = process.env.HOME; - } - - init = async () => { - if (this.state != "new") { - throw Error("init can only be run once"); - } - this.log("initialize the Manager"); - this.state = "init"; - // Ping to start the project and ensure there is a hub connection to it. - await pingProjectUntilSuccess(this.project_id); - // wait for home directory file system to be mounted: - if (this.waitHomeFilesystemType) { - this.reportComponentState({ - state: "waiting", - extra: `for ${this.home} to mount`, - timeout: 60, - progress: 15, - }); - await waitUntilFilesystemIsOfType(this.home, this.waitHomeFilesystemType); - } - - await startProjectServers(); - - this.state = "ready"; - this.reportComponentState({ - state: "ready", - progress: 100, - timeout: Math.ceil(STATUS_INTERVAL_MS / 1000 + 3), - }); - setInterval(this.reportStatus, STATUS_INTERVAL_MS); - }; - - private log = (func, ...args) => { - logger(`Manager.${func}`, ...args); - }; - - private reportComponentState = async (opts: { - state; - extra?; - timeout?; - progress?; - }) => { - this.log("reportState", opts); - try { - await apiCall("v2/compute/set-detailed-state", { - id: this.compute_server_id, - name: "compute", - ...opts, - }); - } catch (err) { - this.log("reportState: WARNING -- ", err); - } - }; - - private reportStatus = async () => { - this.log("reportStatus"); - // Ping to start the project and ensure there is a hub connection to it. - try { - await project.ping({ project_id: this.project_id }); - this.log("ping project -- SUCCESS"); - } catch (err) { - this.log(`ping project -- ERROR '${err}'`); - return; - } - this.reportComponentState({ - state: "ready", - progress: 100, - timeout: STATUS_INTERVAL_MS + 3, - }); - }; -} diff --git a/src/compute/compute/lib/send-files.ts b/src/compute/compute/lib/send-files.ts deleted file mode 100644 index 235b75857c5..00000000000 --- a/src/compute/compute/lib/send-files.ts +++ /dev/null @@ -1,66 +0,0 @@ -/* -Sending files to the project over a websocket. -*/ - -import { apiKey } from "@cocalc/backend/data"; -import { API_COOKIE_NAME } from "@cocalc/backend/auth/cookie-names"; -import { getProjectWebsocketUrl } from "./util"; -import { join } from "path"; -import sendFilesWS from "@cocalc/sync-fs/lib/send-files"; -import getLogger from "@cocalc/backend/logger"; -import { serialize } from "cookie"; -import WebSocket from "ws"; -import { callback } from "awaiting"; - -const logger = getLogger("compute:send-files"); - -interface Options { - project_id: string; - // These are the args to tar after "c" and not involving compression, e.g., this - // would send files listed in /tmp/files.txt: - // createArgs = ["--no-recursion", "--verbatim-files-from", "--files-from", "/tmp/files.txt"] - // You must give something here so we know what to send. - // used to make the tarball - createArgs: string[]; - // used when extracting the tarball - extractArgs: string[]; - // HOME directory for purposes of creating tarball - HOME?: string; -} - -export default async function sendFiles({ - project_id, - createArgs, - extractArgs, - HOME = process.env.HOME, -}: Options) { - await callback(doSendFiles, project_id, createArgs, extractArgs, HOME); -} - -function doSendFiles( - project_id: string, - createArgs: string[], - extractArgs: string[], - HOME, - cb, -) { - const remote = join(getProjectWebsocketUrl(project_id), "sync-fs", "recv"); - logger.debug("connecting to ", remote); - const headers = { Cookie: serialize(API_COOKIE_NAME, apiKey) }; - const ws = new WebSocket(remote, { headers }); - ws.on("open", () => { - logger.debug("connected to ", remote); - // tell it how to receive our files: - logger.debug("sending extractArgs = ", extractArgs); - ws.send(JSON.stringify(extractArgs)); - // send them - sendFilesWS({ ws, args: createArgs, HOME }); - }); - ws.on("close", () => { - cb?.(); - }); - ws.on("error", (err) => { - cb(err); - cb = undefined; - }); -} diff --git a/src/compute/compute/lib/tasks.ts b/src/compute/compute/lib/tasks.ts deleted file mode 100644 index e6e08158521..00000000000 --- a/src/compute/compute/lib/tasks.ts +++ /dev/null @@ -1,19 +0,0 @@ -import SyncClient from "@cocalc/sync-client"; -import { SyncDB } from "@cocalc/sync/editor/db"; - -export function tasks({ - project_id, - path, -}: { - project_id: string; - path: string; -}): SyncDB { - const c = new SyncClient({ project_id, role: "compute_server" }); - const s = c.sync_client.sync_db({ - project_id, - path, - primary_keys: ["task_id"], - string_cols: ["desc"], - }); - return s; -} diff --git a/src/compute/compute/lib/util.ts b/src/compute/compute/lib/util.ts deleted file mode 100644 index e15fa836b97..00000000000 --- a/src/compute/compute/lib/util.ts +++ /dev/null @@ -1,65 +0,0 @@ -import { promisify } from "util"; -import { delay } from "awaiting"; -import getLogger from "@cocalc/backend/logger"; -import { apiServer } from "@cocalc/backend/data"; -import { join } from "path"; -import { project } from "@cocalc/api-client"; - -const logger = getLogger("compute:util"); -const exec = promisify(require("child_process").exec); - -export async function getFilesystemType(path: string): Promise { - try { - const { stdout } = await exec(`df -T ${path} | awk 'NR==2 {print \$2}'`); - return stdout.trim(); - } catch (error) { - logger.error(`getFilesystemType -- WARNING -- exec error: ${error}`); - return null; - } -} - -export async function waitUntilFilesystemIsOfType(path: string, type: string) { - let d = 500; - while (true) { - const cur = await getFilesystemType(path); - if (cur == type) { - return; - } - logger.debug( - `getFilesystemType: '${path}' of type '${cur}'. Waiting for type '${type}'...`, - ); - await delay(d); - d = Math.min(3000, d * 1.3); - } -} - -export function getProjectWebsocketUrl(project_id: string) { - let protocol, host; - if (apiServer.startsWith("https://")) { - protocol = "wss://"; - host = apiServer.slice("https://".length); - } else if (apiServer.startsWith("http://")) { - protocol = "ws://"; - host = apiServer.slice("http://".length); - } else { - throw Error("API_SERVER must start with http:// or https://"); - } - const remote = `${protocol}${host}/${join(project_id, "raw/.smc")}`; - return remote; -} - -export async function pingProjectUntilSuccess(project_id: string) { - let d = 2000; - while (true) { - try { - await project.ping({ project_id }); - return; - } catch (err) { - logger.debug( - `pingProjectUntilSuccess: '${project_id}' failed (${err}). Will try again in ${d}ms...`, - ); - } - await delay(d); - d = Math.min(7000, d * 1.2); - } -} diff --git a/src/compute/compute/package.json b/src/compute/compute/package.json deleted file mode 100644 index a3f53d90447..00000000000 --- a/src/compute/compute/package.json +++ /dev/null @@ -1,65 +0,0 @@ -{ - "name": "@cocalc/compute", - "version": "0.1.6", - "description": "CoCalc remote compute provider -- connects to project and provides remote compute", - "exports": { - ".": "./dist/lib/index.js" - }, - "scripts": { - "preinstall": "npx only-allow pnpm", - "clean": "rm -rf dist node_modules", - "make": "pnpm install && pnpm build", - "build": "../../packages/node_modules/.bin/tsc", - "tsc": "../../packages/node_modules/.bin/tsc --watch --pretty --preserveWatchOutput" - }, - "bin": { - "cocalc-compute-start": "./bin/start.js" - }, - "files": [ - "dist/**", - "bin/**", - "README.md", - "package.json" - ], - "author": "SageMath, Inc.", - "keywords": [ - "cocalc", - "jupyter" - ], - "license": "SEE LICENSE.md", - "dependencies": { - "@cocalc/api-client": "workspace:*", - "@cocalc/backend": "workspace:*", - "@cocalc/compute": "link:", - "@cocalc/conat": "workspace:*", - "@cocalc/jupyter": "workspace:*", - "@cocalc/project": "workspace:*", - "@cocalc/sync": "workspace:*", - "@cocalc/sync-client": "workspace:*", - "@cocalc/sync-fs": "workspace:*", - "@cocalc/util": "workspace:*", - "@types/ws": "^8.18.1", - "awaiting": "^3.0.0", - "cookie": "^1.0.0", - "debug": "^4.4.0", - "websocketfs": "^0.17.6", - "ws": "^8.18.0" - }, - "homepage": "https://github.com/sagemathinc/cocalc/tree/master/src/packages/compute", - "repository": { - "type": "git", - "url": "https://github.com/sagemathinc/cocalc" - }, - "devDependencies": { - "@types/cookie": "^0.6.0", - "@types/node": "^18.16.14", - "typescript": "^5.9.2" - }, - "pnpm-comment": "There is a WRONG warning during install saying this onlyBuiltDependencies won't be used because it is in this file, but this is the ONLY place that works. We do also put it there.", - "pnpm": { - "onlyBuiltDependencies": [ - "websocketfs", - "@cocalc/fuse-native" - ] - } -} diff --git a/src/compute/compute/tsconfig.json b/src/compute/compute/tsconfig.json deleted file mode 100644 index 7d9f01f99f0..00000000000 --- a/src/compute/compute/tsconfig.json +++ /dev/null @@ -1,18 +0,0 @@ -{ - "extends": "../tsconfig.json", - "compilerOptions": { - "rootDir": "./", - "outDir": "dist" - }, - "exclude": ["node_modules", "dist", "test", "dev", "dev2"], - "references": [ - { "path": "../api-client" }, - { "path": "../backend" }, - { "path": "../jupyter" }, - { "path": "../project" }, - { "path": "../sync" }, - { "path": "../sync-client" }, - { "path": "../sync-fs" }, - { "path": "../util" } - ] -} diff --git a/src/compute/conat b/src/compute/conat deleted file mode 120000 index 877aba11481..00000000000 --- a/src/compute/conat +++ /dev/null @@ -1 +0,0 @@ -../packages/conat \ No newline at end of file diff --git a/src/compute/jupyter b/src/compute/jupyter deleted file mode 120000 index 2f9fd54817a..00000000000 --- a/src/compute/jupyter +++ /dev/null @@ -1 +0,0 @@ -../packages/jupyter \ No newline at end of file diff --git a/src/compute/pnpm-lock.yaml b/src/compute/pnpm-lock.yaml deleted file mode 100644 index 66ffcd0b98d..00000000000 --- a/src/compute/pnpm-lock.yaml +++ /dev/null @@ -1,382 +0,0 @@ -lockfileVersion: '9.0' - -settings: - autoInstallPeers: true - excludeLinksFromLockfile: false - -importers: - - compute: - dependencies: - '@cocalc/api-client': - specifier: workspace:* - version: link:../api-client - '@cocalc/backend': - specifier: workspace:* - version: link:../backend - '@cocalc/compute': - specifier: 'link:' - version: 'link:' - '@cocalc/conat': - specifier: workspace:* - version: link:../conat - '@cocalc/jupyter': - specifier: workspace:* - version: link:../jupyter - '@cocalc/project': - specifier: workspace:* - version: link:../project - '@cocalc/sync': - specifier: workspace:* - version: link:../sync - '@cocalc/sync-client': - specifier: workspace:* - version: link:../sync-client - '@cocalc/sync-fs': - specifier: workspace:* - version: link:../sync-fs - '@cocalc/util': - specifier: workspace:* - version: link:../util - '@types/ws': - specifier: ^8.18.1 - version: 8.18.1 - awaiting: - specifier: ^3.0.0 - version: 3.0.0 - cookie: - specifier: ^1.0.0 - version: 1.0.0 - debug: - specifier: ^4.4.0 - version: 4.4.0 - websocketfs: - specifier: ^0.17.6 - version: 0.17.6 - ws: - specifier: ^8.18.0 - version: 8.18.0 - devDependencies: - '@types/cookie': - specifier: ^0.6.0 - version: 0.6.0 - '@types/node': - specifier: ^18.16.14 - version: 18.16.14 - typescript: - specifier: ^5.7.3 - version: 5.7.3 - -packages: - - '@antoniomuso/lz4-napi-android-arm-eabi@2.8.0': - resolution: {integrity: sha512-j1AF1SXOpgiEUxF74cOQLPTwAB9hvR4TqoHJR/ClzP1iWrWZCm6P6tNLy1P1KSn1x3ATGyiTHJqn1V5GvimJvQ==} - engines: {node: '>= 10'} - cpu: [arm] - os: [android] - - '@antoniomuso/lz4-napi-android-arm64@2.8.0': - resolution: {integrity: sha512-lWWPa65eDfhopwFx77bzEUClMHC0WBmy1mOb6LaLRBuT8FaBIscwanb93Zmz1VSbrHTg3sgW3qbGYCEz/WP0Lg==} - engines: {node: '>= 10'} - cpu: [arm64] - os: [android] - - '@antoniomuso/lz4-napi-darwin-arm64@2.8.0': - resolution: {integrity: sha512-vJPGIBBwQKfYW8bp/WZBvSuNxiTHAe5RcQY44If9/wzo+uSNbbglCJOBSJurvRHEVFPREBWlVtXhTVP4qCxZdQ==} - engines: {node: '>= 10'} - cpu: [arm64] - os: [darwin] - - '@antoniomuso/lz4-napi-darwin-x64@2.8.0': - resolution: {integrity: sha512-DK+jjKKmQtZT80UC08z2Uz08G3QJyZijL8BjyxsMHNXEPHp+bYbWWM6esecadwbl61zrHIiiHx4m7WTLaC1ijA==} - engines: {node: '>= 10'} - cpu: [x64] - os: [darwin] - - '@antoniomuso/lz4-napi-freebsd-x64@2.8.0': - resolution: {integrity: sha512-4etI4CH+maAMfhRoSn2vGPcPOVA5ASFkeowfOL0BxKGN6QrFtRwzACGl6f/61tcmdDV7mypaw///6VXL3o2ksQ==} - engines: {node: '>= 10'} - cpu: [x64] - os: [freebsd] - - '@antoniomuso/lz4-napi-linux-arm-gnueabihf@2.8.0': - resolution: {integrity: sha512-PwWXPXCEmN/PYpqiKaxxbqLhPFtS05YYaFjmcZ55PXSpb/Okq/aO93+JCUuB5+zCMeKBfv3SGDaxN62moEGr5w==} - engines: {node: '>= 10'} - cpu: [arm] - os: [linux] - - '@antoniomuso/lz4-napi-linux-arm64-gnu@2.8.0': - resolution: {integrity: sha512-ky5Av50tnYLZTMcCFK2r/yQng7PRvdTS6J3FMiFsaMQbOgieKa0Ll61MMIHWrixhJSHW0oOFbsWIcr06xO7Gdg==} - engines: {node: '>= 10'} - cpu: [arm64] - os: [linux] - - '@antoniomuso/lz4-napi-linux-arm64-musl@2.8.0': - resolution: {integrity: sha512-lc4HCcmbvvXggoeb3CeGVhjBQrcaoGsIUUkWVaWdBOJTggjLCj67S7xEcqwFCeJQlGT7Ak36CLTbAbKWDRMMDA==} - engines: {node: '>= 10'} - cpu: [arm64] - os: [linux] - - '@antoniomuso/lz4-napi-linux-x64-gnu@2.8.0': - resolution: {integrity: sha512-iyaES5Gm3Bhxu+G9CzUlGNWCSzquyoKGne1T7kxeN1FnQa4vSVbX+QvN/3NueSNhoErqy3Eh3cdkoSyPj9fJZg==} - engines: {node: '>= 10'} - cpu: [x64] - os: [linux] - - '@antoniomuso/lz4-napi-linux-x64-musl@2.8.0': - resolution: {integrity: sha512-/EmFdbvT2LPcxDGalqvQXmSplGNmNySseWN7XBG2Z7AYq6kEobPZOU1+9z1YjmrR2LDQRO0wBZNXYGHRdKuMjg==} - engines: {node: '>= 10'} - cpu: [x64] - os: [linux] - - '@antoniomuso/lz4-napi-win32-arm64-msvc@2.8.0': - resolution: {integrity: sha512-m4ZgbxnUeNJDC861k+qc+zFjbYATgIx7xevyTavoAr04Apv9rBadrk9VmzUyqjjqTEpyI9OEyoFsIqb8hyCZ6w==} - engines: {node: '>= 10'} - cpu: [arm64] - os: [win32] - - '@antoniomuso/lz4-napi-win32-ia32-msvc@2.8.0': - resolution: {integrity: sha512-uMOirYEGNdg2GYD2wpRg6C5tv/D3S17XHifjELoYGmTdE0p31/yhcjN7o/rRNze5aEoo72eEjwQAruZi4SYPgQ==} - engines: {node: '>= 10'} - cpu: [ia32] - os: [win32] - - '@antoniomuso/lz4-napi-win32-x64-msvc@2.8.0': - resolution: {integrity: sha512-Vm1jrS59bB9tRBIuuDff6m6wTIrfl212FadFprI3QZpiErlSO0FKutTmv+DcMJ3sEkfXpMWR98D5cVBmFihW/g==} - engines: {node: '>= 10'} - cpu: [x64] - os: [win32] - - '@cocalc/fuse-native@2.4.2': - resolution: {integrity: sha512-l5YrVahdBTnrX/4jQplOSXqkrSMM5jXh5vC/OWbkImzInTbc7ymgZ/GBoYM6pzyoxjCLDTIauJSsmEssx4Nfxg==} - hasBin: true - - '@isaacs/ttlcache@1.4.1': - resolution: {integrity: sha512-RQgQ4uQ+pLbqXfOmieB91ejmLwvSgv9nLx6sT6sD83s7umBypgg+OIBOBbEUiJXrfpnp9j0mRhYYdzp9uqq3lA==} - engines: {node: '>=12'} - - '@napi-rs/triples@1.2.0': - resolution: {integrity: sha512-HAPjR3bnCsdXBsATpDIP5WCrw0JcACwhhrwIAQhiR46n+jm+a2F8kBsfseAuWtSyQ+H3Yebt2k43B5dy+04yMA==} - - '@node-rs/helper@1.6.0': - resolution: {integrity: sha512-2OTh/tokcLA1qom1zuCJm2gQzaZljCCbtX1YCrwRVd/toz7KxaDRFeLTAPwhs8m9hWgzrBn5rShRm6IaZofCPw==} - - '@types/cookie@0.6.0': - resolution: {integrity: sha512-4Kh9a6B2bQciAhf7FSuMRRkUWecJgJu9nPnx3yzpsfXX/c50REIqpHY4C82bXP90qrLtXtkDxTZosYO3UpOwlA==} - - '@types/node@18.16.14': - resolution: {integrity: sha512-+ImzUB3mw2c5ISJUq0punjDilUQ5GnUim0ZRvchHIWJmOC0G+p0kzhXBqj6cDjK0QdPFwzrHWgrJp3RPvCG5qg==} - - '@types/ws@8.18.1': - resolution: {integrity: sha512-ThVF6DCVhA8kUGy+aazFQ4kXQ7E1Ty7A3ypFOe0IcJV8O/M511G99AW24irKrW56Wt44yG9+ij8FaqoBGkuBXg==} - - awaiting@3.0.0: - resolution: {integrity: sha512-19i4G7Hjxj9idgMlAM0BTRII8HfvsOdlr4D9cf3Dm1MZhvcKjBpzY8AMNEyIKyi+L9TIK15xZatmdcPG003yww==} - engines: {node: '>=7.6.x'} - - binarysearch@1.0.1: - resolution: {integrity: sha512-FqhwdeXh1ZSAS/YpJ6lD9+SMf8JodCibe7c51Z9L1zAjHKUDTBisQgdmpfaL+m1qHvwAHnSLR8d9UHc79Hr34g==} - - cookie@0.5.0: - resolution: {integrity: sha512-YZ3GUyn/o8gfKJlnlX7g7xq4gyO6OSuhGPKaaGssGB2qgDUS0gPgtTvoyZLTt9Ab6dC4hfc9dV5arkvc/OCmrw==} - engines: {node: '>= 0.6'} - - cookie@1.0.0: - resolution: {integrity: sha512-bsSztFoaR8bw9MlFCrTHzc1wOKCUKOBsbgFdoDilZDkETAOOjKSqV7L+EQLbTaylwvZasd9vM4MGKotJaUfSpA==} - engines: {node: '>=18'} - - debug@4.4.0: - resolution: {integrity: sha512-6WTZ/IxCY/T6BALoZHaE4ctp9xm+Z5kY/pzYaCHRFeyVhojxlrm+46y68HA6hr0TcwEssoxNiDEUJQjfPZ/RYA==} - engines: {node: '>=6.0'} - peerDependencies: - supports-color: '*' - peerDependenciesMeta: - supports-color: - optional: true - - inherits@2.0.4: - resolution: {integrity: sha512-k/vGaX4/Yla3WzyMCvTQOXYeIHvqOKtnqBduzTHpzpQZzAskKMhZ2K+EnBiSM9zGSoIFeMpXKxa4dYeZIQqewQ==} - - lz4-napi@2.8.0: - resolution: {integrity: sha512-mvJVO3AlJCr5EACl1JxHUA3aMej77bjo53JXyNZXeB1HQLjWntjJSXUYLUwyUuB8NT7GkyPdUbzNYi0UDcdUJg==} - engines: {node: '>= 10'} - - ms@2.1.3: - resolution: {integrity: sha512-6FlzubTLZG3J2a/NVCAleEhjzq5oxgHyaCU9yYXvcLsvoVaHJq/s5xXI6/XXP6tz7R9xAOtHnSO/tXtF3WRTlA==} - - nanoresource@1.3.0: - resolution: {integrity: sha512-OI5dswqipmlYfyL3k/YMm7mbERlh4Bd1KuKdMHpeoVD1iVxqxaTMKleB4qaA2mbQZ6/zMNSxCXv9M9P/YbqTuQ==} - - napi-macros@2.2.2: - resolution: {integrity: sha512-hmEVtAGYzVQpCKdbQea4skABsdXW4RUh5t5mJ2zzqowJS2OyXZTU1KhDVFhx+NlWZ4ap9mqR9TcDO3LTTttd+g==} - - node-gyp-build@4.8.4: - resolution: {integrity: sha512-LA4ZjwlnUblHVgq0oBF3Jl/6h/Nvs5fzBLwdEF4nuxnFdsfajde4WfxtJr3CaiH+F6ewcIB/q4jQ4UzPyid+CQ==} - hasBin: true - - port-get@1.0.4: - resolution: {integrity: sha512-B8RcNfc8Ld+7C31DPaKIQz2aO9dqIs+4sUjhxJ2TSjEaidwyxu05WBbm08FJe+qkVvLiQqPbEAfNw1rB7JbjtA==} - - typescript@5.7.3: - resolution: {integrity: sha512-84MVSjMEHP+FQRPy3pX9sTVV/INIex71s9TL2Gm5FG/WG1SqXeKyZ0k7/blY/4FdOzI12CBy1vGc4og/eus0fw==} - engines: {node: '>=14.17'} - hasBin: true - - websocket-sftp@0.8.4: - resolution: {integrity: sha512-2HaXu1ytAso3qDk6TsW5PtkYJUWdY5SdXVPavu8wyTETHlNUUVb0eFXmIGsfupuZc94p7lCquNdTw4Zo6ITepg==} - engines: {node: '>=0.16.0'} - - websocketfs@0.17.6: - resolution: {integrity: sha512-ncffrhje7jOIGjKXKFDnUb8/qScp4aacCehdltURLMCYPagYq4jjpCCpaZ40iYHNYPx+FfCB4tvZi53kTOopyQ==} - engines: {node: '>=0.16.0'} - hasBin: true - - ws@8.18.0: - resolution: {integrity: sha512-8VbfWfHLbbwu3+N6OKsOMpBdT4kXPDDB9cJk2bJ6mh9ucxdlnNvH1e+roYkKmN9Nxw2yjz7VzeO9oOz2zJ04Pw==} - engines: {node: '>=10.0.0'} - peerDependencies: - bufferutil: ^4.0.1 - utf-8-validate: '>=5.0.2' - peerDependenciesMeta: - bufferutil: - optional: true - utf-8-validate: - optional: true - -snapshots: - - '@antoniomuso/lz4-napi-android-arm-eabi@2.8.0': - optional: true - - '@antoniomuso/lz4-napi-android-arm64@2.8.0': - optional: true - - '@antoniomuso/lz4-napi-darwin-arm64@2.8.0': - optional: true - - '@antoniomuso/lz4-napi-darwin-x64@2.8.0': - optional: true - - '@antoniomuso/lz4-napi-freebsd-x64@2.8.0': - optional: true - - '@antoniomuso/lz4-napi-linux-arm-gnueabihf@2.8.0': - optional: true - - '@antoniomuso/lz4-napi-linux-arm64-gnu@2.8.0': - optional: true - - '@antoniomuso/lz4-napi-linux-arm64-musl@2.8.0': - optional: true - - '@antoniomuso/lz4-napi-linux-x64-gnu@2.8.0': - optional: true - - '@antoniomuso/lz4-napi-linux-x64-musl@2.8.0': - optional: true - - '@antoniomuso/lz4-napi-win32-arm64-msvc@2.8.0': - optional: true - - '@antoniomuso/lz4-napi-win32-ia32-msvc@2.8.0': - optional: true - - '@antoniomuso/lz4-napi-win32-x64-msvc@2.8.0': - optional: true - - '@cocalc/fuse-native@2.4.2': - dependencies: - nanoresource: 1.3.0 - napi-macros: 2.2.2 - node-gyp-build: 4.8.4 - - '@isaacs/ttlcache@1.4.1': {} - - '@napi-rs/triples@1.2.0': {} - - '@node-rs/helper@1.6.0': - dependencies: - '@napi-rs/triples': 1.2.0 - - '@types/cookie@0.6.0': {} - - '@types/node@18.16.14': {} - - '@types/ws@8.18.1': - dependencies: - '@types/node': 18.16.14 - - awaiting@3.0.0: {} - - binarysearch@1.0.1: {} - - cookie@0.5.0: {} - - cookie@1.0.0: {} - - debug@4.4.0: - dependencies: - ms: 2.1.3 - - inherits@2.0.4: {} - - lz4-napi@2.8.0: - dependencies: - '@node-rs/helper': 1.6.0 - optionalDependencies: - '@antoniomuso/lz4-napi-android-arm-eabi': 2.8.0 - '@antoniomuso/lz4-napi-android-arm64': 2.8.0 - '@antoniomuso/lz4-napi-darwin-arm64': 2.8.0 - '@antoniomuso/lz4-napi-darwin-x64': 2.8.0 - '@antoniomuso/lz4-napi-freebsd-x64': 2.8.0 - '@antoniomuso/lz4-napi-linux-arm-gnueabihf': 2.8.0 - '@antoniomuso/lz4-napi-linux-arm64-gnu': 2.8.0 - '@antoniomuso/lz4-napi-linux-arm64-musl': 2.8.0 - '@antoniomuso/lz4-napi-linux-x64-gnu': 2.8.0 - '@antoniomuso/lz4-napi-linux-x64-musl': 2.8.0 - '@antoniomuso/lz4-napi-win32-arm64-msvc': 2.8.0 - '@antoniomuso/lz4-napi-win32-ia32-msvc': 2.8.0 - '@antoniomuso/lz4-napi-win32-x64-msvc': 2.8.0 - - ms@2.1.3: {} - - nanoresource@1.3.0: - dependencies: - inherits: 2.0.4 - - napi-macros@2.2.2: {} - - node-gyp-build@4.8.4: {} - - port-get@1.0.4: {} - - typescript@5.7.3: {} - - websocket-sftp@0.8.4: - dependencies: - awaiting: 3.0.0 - debug: 4.4.0 - port-get: 1.0.4 - ws: 8.18.0 - transitivePeerDependencies: - - bufferutil - - supports-color - - utf-8-validate - - websocketfs@0.17.6: - dependencies: - '@cocalc/fuse-native': 2.4.2 - '@isaacs/ttlcache': 1.4.1 - awaiting: 3.0.0 - binarysearch: 1.0.1 - cookie: 0.5.0 - debug: 4.4.0 - lz4-napi: 2.8.0 - port-get: 1.0.4 - websocket-sftp: 0.8.4 - transitivePeerDependencies: - - bufferutil - - supports-color - - utf-8-validate - - ws@8.18.0: {} diff --git a/src/compute/pnpm-workspace.yaml b/src/compute/pnpm-workspace.yaml deleted file mode 100644 index 5cfb73b0fd8..00000000000 --- a/src/compute/pnpm-workspace.yaml +++ /dev/null @@ -1,20 +0,0 @@ -packages: - - api-client - - backend - - comm - - jupyter - - conat - - project - - sync - - sync-fs - - sync-client - - util - - terminal - - compute - - conat - -onlyBuiltDependencies: - - '@cocalc/fuse-native' - - websocket-sftp - - websocketfs - diff --git a/src/compute/project b/src/compute/project deleted file mode 120000 index bf7fea7ae0a..00000000000 --- a/src/compute/project +++ /dev/null @@ -1 +0,0 @@ -../packages/project \ No newline at end of file diff --git a/src/compute/sync b/src/compute/sync deleted file mode 120000 index 54ede9a1c29..00000000000 --- a/src/compute/sync +++ /dev/null @@ -1 +0,0 @@ -../packages/sync \ No newline at end of file diff --git a/src/compute/sync-client b/src/compute/sync-client deleted file mode 120000 index f52dae26a07..00000000000 --- a/src/compute/sync-client +++ /dev/null @@ -1 +0,0 @@ -../packages/sync-client \ No newline at end of file diff --git a/src/compute/sync-fs b/src/compute/sync-fs deleted file mode 120000 index 1cab5ee5267..00000000000 --- a/src/compute/sync-fs +++ /dev/null @@ -1 +0,0 @@ -../packages/sync-fs \ No newline at end of file diff --git a/src/compute/terminal b/src/compute/terminal deleted file mode 120000 index 0d945645464..00000000000 --- a/src/compute/terminal +++ /dev/null @@ -1 +0,0 @@ -../packages/terminal \ No newline at end of file diff --git a/src/compute/tsconfig.json b/src/compute/tsconfig.json deleted file mode 120000 index aa3db7833f5..00000000000 --- a/src/compute/tsconfig.json +++ /dev/null @@ -1 +0,0 @@ -../packages/tsconfig.json \ No newline at end of file diff --git a/src/compute/util b/src/compute/util deleted file mode 120000 index c79f1420186..00000000000 --- a/src/compute/util +++ /dev/null @@ -1 +0,0 @@ -../packages/util \ No newline at end of file diff --git a/src/package.json b/src/package.json index 103695369bf..64e59f7cb81 100644 --- a/src/package.json +++ b/src/package.json @@ -6,8 +6,10 @@ "make": "pnpm run build", "make-dev": "pnpm run build-dev", "build": "./workspaces.py install && ./workspaces.py build && pnpm python-api", + "tsc-all": "./workspaces.py tsc --parallel", "build-dev": "./workspaces.py install && ./workspaces.py build --dev && pnpm python-api", - "clean": "rm -rf packages/node_modules && ./workspaces.py clean && cd compute/compute && pnpm clean ", + "build-lite-dev": "./workspaces.py install && ./workspaces.py build --dev --exclude=hub,server,next,file-server", + "clean": "rm -rf packages/node_modules && ./workspaces.py clean ", "hub": "cd packages/hub && npm run hub-project-dev-nobuild", "hub-prod": "cd packages/hub && npm run hub-project-prod-nobuild", "rspack": "cd packages/static && pnpm watch", @@ -18,14 +20,14 @@ "version-check": "pip3 install typing_extensions mypy || pip3 install --break-system-packages typing_extensions mypy && ./workspaces.py version-check && mypy scripts/check_npm_packages.py", "test-parallel": "unset DEBUG && pnpm run version-check && cd packages && pnpm run -r --parallel test", "test": "unset DEBUG && pnpm run depcheck && pnpm run version-check && ./workspaces.py test", - "test-github-ci": "unset DEBUG && pnpm run depcheck && pnpm run version-check && ./workspaces.py test --exclude=jupyter,file-server --retries=1", - "depcheck": "cd packages && pnpm run -r --parallel depcheck", + "test-github-ci": "unset DEBUG && pnpm run depcheck && pnpm run version-check && ./workspaces.py test --test-github-ci --exclude=jupyter --retries=1", + "depcheck": "cd packages && pnpm run -r --workspace-concurrency=4 depcheck", "prettier-all": "cd packages/", "local-ci": "./scripts/ci.sh", "conat-connections": "cd packages/backend && pnpm conat-connections", "conat-watch": "cd packages/backend && pnpm conat-watch", "conat-inventory": "cd packages/backend && pnpm conat-inventory", - "python-api": "cd python/cocalc-api && make all" + "python-api": "cd python/cocalc-api && PATH=$HOME/.local/bin:$PATH make all" }, "repository": { "type": "git", diff --git a/src/packages/.npmrc b/src/packages/.npmrc index 86c81f8e463..8377bc43bcb 100644 --- a/src/packages/.npmrc +++ b/src/packages/.npmrc @@ -6,3 +6,8 @@ public-hoist-pattern[]=*@ant-design/cssinjs* git-checks=false + +# When working with electron it mutates binaries in packages, and this +# makes this only impact the copy of cocalc where that happens, NOT +# all copies on your computer. +package-import-method=clone diff --git a/src/packages/api-client/package.json b/src/packages/api-client/package.json index b5d16040ec2..d2592e4da89 100644 --- a/src/packages/api-client/package.json +++ b/src/packages/api-client/package.json @@ -9,14 +9,21 @@ "tsc": "../node_modules/.bin/tsc --watch --pretty --preserveWatchOutput", "depcheck": "pnpx depcheck --ignores @cocalc/api-client " }, - "files": ["dist/**", "bin/**", "README.md", "package.json"], + "files": [ + "dist/**", + "bin/**", + "README.md", + "package.json" + ], "author": "SageMath, Inc.", - "keywords": ["cocalc", "jupyter"], + "keywords": [ + "cocalc", + "jupyter" + ], "license": "SEE LICENSE.md", "dependencies": { "@cocalc/api-client": "workspace:*", - "@cocalc/backend": "workspace:*", - "@cocalc/util": "workspace:*" + "@cocalc/backend": "workspace:*" }, "homepage": "https://github.com/sagemathinc/cocalc/tree/master/src/packages/compute", "repository": { diff --git a/src/packages/api-client/src/project.ts b/src/packages/api-client/src/project.ts index e05f0abf978..29ae86efbec 100644 --- a/src/packages/api-client/src/project.ts +++ b/src/packages/api-client/src/project.ts @@ -4,8 +4,6 @@ API key should be enough to allow them. */ import { apiCall } from "./call"; -import { siteUrl } from "./urls"; -import { type JupyterApiOptions } from "@cocalc/util/jupyter/api-types"; // Starts a project running. export async function start(opts: { project_id: string }) { @@ -22,91 +20,3 @@ export async function stop(opts: { project_id: string }) { export async function touch(opts: { project_id: string }) { return await apiCall("v2/projects/touch", opts); } - -// There's a TCP connection from some hub to the project, which has -// a limited api, which callProject exposes. This includes: -// mesg={event:'ping'}, etc. -// See src/packages/server/projects/connection/call.ts -// We use this to implement some functions below. -async function callProject(opts: { project_id: string; mesg: object }) { - return await apiCall("v2/projects/call", opts); -} - -export async function ping(opts: { project_id: string }) { - return await callProject({ ...opts, mesg: { event: "ping" } }); -} - -export async function exec(opts: { - project_id: string; - path?: string; - command?: string; - args?: string[]; - timeout?: number; // in seconds; default 10 - aggregate?: any; - max_output?: number; - bash?: boolean; - err_on_exit?: boolean; // default true -}) { - return await callProject({ - project_id: opts.project_id, - mesg: { event: "project_exec", ...opts }, - }); -} - -// Returns URL of the file or directory, which you can -// download (from the postgres blob store). It gets autodeleted. -// There is a limit of about 10MB for this. -// For text use readTextFileToProject -export async function readFile(opts: { - project_id: string; - path: string; // file or directory - archive?: "tar" | "tar.bz2" | "tar.gz" | "zip" | "7z"; - ttlSeconds?: number; -}): Promise { - const { archive, data_uuid } = await callProject({ - project_id: opts.project_id, - mesg: { event: "read_file_from_project", ...opts }, - }); - return siteUrl( - `blobs/${opts.path}${archive ? `.${archive}` : ""}?uuid=${data_uuid}`, - ); -} - -// export async function writeFileToProject(opts: { -// project_id: string; -// path: string; // file or directory -// archive?: "tar" | "tar.bz2" | "tar.gz" | "zip" | "7z"; -// ttlSeconds?: number; -// }): Promise { - -// } - -export async function writeTextFile(opts: { - project_id: string; - path: string; - content: string; -}): Promise { - await callProject({ - project_id: opts.project_id, - mesg: { event: "write_text_file_to_project", ...opts }, - }); -} - -export async function readTextFile(opts: { - project_id: string; - path: string; -}): Promise { - return await callProject({ - project_id: opts.project_id, - mesg: { event: "read_text_file_from_project", ...opts }, - }); -} - -export async function jupyterExec(opts: JupyterApiOptions): Promise { - return ( - await callProject({ - project_id: opts.project_id, - mesg: { event: "jupyter_execute", ...opts }, - }) - ).output; -} diff --git a/src/packages/backend/bin/install.sh b/src/packages/backend/bin/install.sh new file mode 100755 index 00000000000..16c8b00d73d --- /dev/null +++ b/src/packages/backend/bin/install.sh @@ -0,0 +1,8 @@ +#!/usr/bin/env bash +set -ev + +cp "`pwd`/dist/bin/open.js" node_modules/.bin/open +chmod +x node_modules/.bin/open + + + diff --git a/src/packages/backend/bin/open.ts b/src/packages/backend/bin/open.ts new file mode 100644 index 00000000000..f6c0f69bc7e --- /dev/null +++ b/src/packages/backend/bin/open.ts @@ -0,0 +1,225 @@ +#!/usr/bin/env node +/** + * write an "open" event to $COCALC_CONTROL_DIR for the CoCalc spool server + * + * Requirements + * - Zero dependencies + * - Durable write: tmp -> fsync(file) -> rename -> fsync(dir) + * - Preserve shell view of CWD via $PWD + * - Map abs paths outside $HOME to "$HOME/.smc/root/..." + * - Create parents + touch missing files + * - Ignore non-expanded globs when path does not exist + * - Message schema: { event: "open", paths: [{ file|directory: string }] } + */ + +import fs from "node:fs"; +import fsp, { FileHandle } from "node:fs/promises"; +import path from "node:path"; + +const MAX_FILES = 15 as const; +const ROOT_SYMLINK = ".smc/root" as const; // relative to $HOME + +function usage(): void { + console.error(`Usage: ${path.basename(process.argv[1])} [path names] ...`); +} + +function hasGlobChars(p: string): boolean { + return /[\*\?\{]/.test(p); +} + +async function pathExists(p: string): Promise { + try { + await fsp.lstat(p); + return true; + } catch (err: unknown) { + const e = err as NodeJS.ErrnoException; + if (e && e.code === "ENOENT") return false; + throw err; + } +} + +async function ensureParentsAndMaybeTouch(p: string): Promise { + const dir = path.dirname(p); + try { + await fsp.mkdir(dir, { recursive: true, mode: 0o755 }); + } catch (e: unknown) { + const err = e as NodeJS.ErrnoException; + if (!err || err.code !== "EEXIST") throw err; + } + if (!p.endsWith("/")) { + try { + const fh = await fsp.open( + p, + fs.constants.O_CREAT | fs.constants.O_EXCL | fs.constants.O_WRONLY, + 0o644, + ); + await fh.close(); + } catch (e: unknown) { + const err = e as NodeJS.ErrnoException; + if (!err || err.code !== "EEXIST") throw err; + } + } +} + +function resolveAbsPreservePWD(p: string): string { + if (path.isAbsolute(p)) return p; + const pwd = process.env.PWD; + if (pwd && path.isAbsolute(pwd)) return path.join(pwd, p); + return path.resolve(p); +} + +async function classifyPath(absPath: string): Promise<"file" | "directory"> { + const st = await fsp.lstat(absPath); + return st.isDirectory() ? "directory" : "file"; +} + +function mapToHomeView(absPath: string, home: string): string { + const sep = path.sep; + if (absPath.startsWith(home + sep)) { + return absPath.slice(home.length + 1); + } + return path.join(ROOT_SYMLINK, absPath); +} + +function hrtimeNs(): bigint { + const [s, ns] = process.hrtime(); + return BigInt(s) * 1_000_000_000n + BigInt(ns); +} + +function randomHex8(): string { + return Math.floor(Math.random() * 0xffffffff) + .toString(16) + .padStart(8, "0"); +} + +async function fsyncDir(dirPath: string): Promise { + let fh: FileHandle | undefined; + try { + // O_DIRECTORY may not exist on some platforms; types allow number + fh = await fsp.open( + dirPath, + fs.constants.O_RDONLY | (fs.constants as any).O_DIRECTORY, + ); + } catch { + try { + fh = await fsp.open(dirPath, fs.constants.O_RDONLY); + } catch { + return; + } + } + try { + await fh.sync(); + } catch { + // best-effort + } finally { + try { + await fh.close(); + } catch {} + } +} + +interface PathMsgFile { + file: string; +} +interface PathMsgDir { + directory: string; +} + +type PathMsg = PathMsgFile | PathMsgDir; + +interface OpenMessage { + event: "open"; + paths: PathMsg[]; +} + +async function writeDurable( + dir: string, + baseName: string, + data: Buffer, +): Promise { + await fsp.mkdir(dir, { recursive: true, mode: 0o755 }); + const tmp = path.join(dir, "." + baseName + ".tmp"); + const dst = path.join(dir, baseName); + + const fh = await fsp.open( + tmp, + fs.constants.O_CREAT | fs.constants.O_EXCL | fs.constants.O_WRONLY, + 0o600, + ); + try { + await fh.writeFile(data); + await fh.sync(); + } finally { + await fh.close(); + } + await fsp.rename(tmp, dst); + await fsyncDir(dir); +} + +async function main(): Promise { + const args = process.argv.slice(2); + if (args.length === 0) { + usage(); + process.exit(1); + } + + const home = process.env.HOME; + if (!home) { + console.error("HOME not set"); + process.exit(2); + } + + const controlDir = process.env.COCALC_CONTROL_DIR; + if (!controlDir) { + console.error("COCALC_CONTROL_DIR not set"); + process.exit(2); + } + + const trimmed = args.map((s) => s.trim()).filter(Boolean); + let inputs = trimmed.slice(0, MAX_FILES); + if (trimmed.length > MAX_FILES) { + console.error(`You may open at most ${MAX_FILES} items; truncating.`); + } + + const out: PathMsg[] = []; + + for (const p of inputs) { + const exists = await pathExists(p); + if (!exists && hasGlobChars(p)) { + console.error(`no match for '${p}', so not creating`); + continue; + } + if (!exists) { + await ensureParentsAndMaybeTouch(p); + } + + const abs = resolveAbsPreservePWD(p); + const kind = await classifyPath(abs); + const name = mapToHomeView(abs, home); + if (kind === "directory") out.push({ directory: name }); + else out.push({ file: name }); + } + + if (out.length === 0) return; + + const message: OpenMessage = { event: "open", paths: out }; + const json = Buffer.from(JSON.stringify(message) + "\n", "utf8"); + const base = `${hrtimeNs()}-${process.pid}-${randomHex8()}.json`; + + try { + await writeDurable(controlDir, base, json); + } catch (e: unknown) { + const err = e as Error; + console.error( + "failed to write control message:", + err?.message ?? String(e), + ); + process.exit(3); + } +} + +main().catch((e: unknown) => { + const err = e as Error; + console.error(err?.stack ?? err?.message ?? String(e)); + process.exit(1); +}); diff --git a/src/packages/backend/conat/files/local-path.ts b/src/packages/backend/conat/files/local-path.ts new file mode 100644 index 00000000000..e392c2a8d79 --- /dev/null +++ b/src/packages/backend/conat/files/local-path.ts @@ -0,0 +1,90 @@ +import { fsServer, DEFAULT_FILE_SERVICE } from "@cocalc/conat/files/fs"; +import { SandboxedFilesystem } from "@cocalc/backend/sandbox"; +import { isValidUUID } from "@cocalc/util/misc"; +import { mkdir } from "fs/promises"; +import { join } from "path"; +import { type Client } from "@cocalc/conat/core/client"; +import { conat } from "@cocalc/backend/conat/conat"; +import { client as createFileClient } from "@cocalc/conat/files/file-server"; +import getLogger from "@cocalc/backend/logger"; + +const logger = getLogger("conat:files:local-path"); + +export async function localPathFileserver({ + path, + service = DEFAULT_FILE_SERVICE, + client, + project_id, + unsafeMode, +}: { + service?: string; + client?: Client; + // if project_id is specified, only serve this one project_id + project_id?: string; + + // - if path is given, serve projects from `${path}/${project_id}`, except in 1-project mode (when project_id is given above), + // in which case we just server the project from path directly. + // - if path not given, connect to the file-server service on the conat network. + path?: string; + unsafeMode?: boolean; +} = {}) { + logger.debug("localPathFileserver", { + service, + project_id, + unsafeMode, + path, + }); + client ??= conat(); + logger.debug("localPathFileserver: got client"); + + const getPath = async (project_id2: string) => { + if (project_id != null && project_id != project_id2) { + throw Error(`only serves ${project_id}`); + } + if (path != null) { + if (project_id != null) { + // in 1-project mode just server directly from path + return path; + } + const p = join(path, project_id2); + try { + await mkdir(p); + } catch {} + return p; + } else { + const fsclient = createFileClient({ client }); + return (await fsclient.mount({ project_id: project_id2 })).path; + } + }; + + logger.debug("creating fsServer...", { service }); + const server = await fsServer({ + service, + client, + project_id, + fs: async (subject: string) => { + const project_id = getProjectId(subject); + return new SandboxedFilesystem(await getPath(project_id), { + unsafeMode, + host: project_id, + }); + }, + }); + logger.debug("created fsServer...", { service }); + return { server, client, path, service, close: () => server.close() }; +} + +function getProjectId(subject: string) { + const v = subject.split("."); + if (v.length != 2) { + throw Error("subject must have 2 segments"); + } + if (!v[1].startsWith("project-")) { + throw Error("second segment of subject must start with 'project-'"); + } + const project_id = v[1].slice("project-".length); + if (!isValidUUID(project_id)) { + throw Error("not a valid project id"); + } + return project_id; +} diff --git a/src/packages/backend/conat/files/test/listing.test.ts b/src/packages/backend/conat/files/test/listing.test.ts new file mode 100644 index 00000000000..4f223a3f59e --- /dev/null +++ b/src/packages/backend/conat/files/test/listing.test.ts @@ -0,0 +1,93 @@ +import { SandboxedFilesystem } from "@cocalc/backend/sandbox"; +import { mkdtemp, rm } from "node:fs/promises"; +import { tmpdir } from "node:os"; +import { join } from "path"; +import { randomId } from "@cocalc/conat/names"; +import listing from "@cocalc/conat/files/listing"; + +let tmp; +beforeAll(async () => { + tmp = await mkdtemp(join(tmpdir(), `cocalc-${randomId()}0`)); +}); + +afterAll(async () => { + try { + await rm(tmp, { force: true, recursive: true }); + } catch {} +}); + +describe("creating a listing monitor starting with an empty directory", () => { + let fs, dir; + it("creates sandboxed filesystem", async () => { + fs = new SandboxedFilesystem(tmp); + dir = await listing({ path: "", fs }); + }); + + it("initial listing is empty", () => { + expect(Object.keys(dir.files)).toEqual([]); + }); + + let iter; + it("create a file and get an update", async () => { + iter = dir.iter(); + await fs.writeFile("a.txt", "hello"); + let { value } = await iter.next(); + expect(value).toEqual({ + mtime: value.mtime, + name: "a.txt", + size: value.size, + }); + // it's possible that the file isn't written completely above. + if (value.size != 5) { + ({ value } = await iter.next()); + } + const stat = await fs.stat("a.txt"); + expect(stat.mtimeMs).toEqual(value.mtime); + expect(dir.files["a.txt"]).toEqual({ mtime: value.mtime, size: 5 }); + }); + + it("modify the file and get two updates -- one when it starts and another when done", async () => { + await fs.appendFile("a.txt", " there"); + const { value } = await iter.next(); + expect(value).toEqual({ mtime: value.mtime, name: "a.txt", size: 5 }); + const { value: value2 } = await iter.next(); + expect(value2).toEqual({ mtime: value2.mtime, name: "a.txt", size: 11 }); + const stat = await fs.stat("a.txt"); + expect(stat.mtimeMs).toEqual(value2.mtime); + expect(dir.files["a.txt"]).toEqual({ mtime: value2.mtime, size: 11 }); + }); + + it("create another monitor starting with the now nonempty directory", async () => { + const dir2 = await listing({ path: "", fs }); + expect(Object.keys(dir2.files!)).toEqual(["a.txt"]); + expect(dir.files["a.txt"].mtime).toBeCloseTo(dir2.files!["a.txt"].mtime); + dir2.close(); + }); + + const count = 500; + it(`creates ${count} files and see they are found`, async () => { + const n = Object.keys(dir.files).length; + + for (let i = 0; i < count; i++) { + await fs.writeFile(`${i}`, ""); + } + const values: string[] = []; + while (true) { + const { value } = await iter.next(); + if (value == "a.txt") { + continue; + } + values.push(value); + if (value.name == `${count - 1}`) { + break; + } + } + expect(new Set(values).size).toEqual(count); + + expect(Object.keys(dir.files).length).toEqual(n + count); + }); + + it("cleans up", () => { + dir.close(); + }); +}); diff --git a/src/packages/backend/conat/files/test/local-path.test.ts b/src/packages/backend/conat/files/test/local-path.test.ts new file mode 100644 index 00000000000..9092d0ee460 --- /dev/null +++ b/src/packages/backend/conat/files/test/local-path.test.ts @@ -0,0 +1,477 @@ +import { link, readFile, symlink, writeFile } from "node:fs/promises"; +import { join } from "path"; +import { fsClient } from "@cocalc/conat/files/fs"; +import { randomId } from "@cocalc/conat/names"; +import { before, after } from "@cocalc/backend/conat/test/setup"; +import { uuid } from "@cocalc/util/misc"; +import { + createPathFileserver, + cleanupFileservers, +} from "@cocalc/backend/conat/files/test/util"; +import { TextDecoder } from "node:util"; + +beforeAll(before); + +describe("use all the standard api functions of fs", () => { + let server; + it("creates the simple fileserver service", async () => { + server = await createPathFileserver(); + }); + + const project_id = uuid(); + let fs; + it("create a client", () => { + fs = fsClient({ subject: `${server.service}.project-${project_id}` }); + }); + + it("appendFile works", async () => { + await fs.writeFile("a", ""); + await fs.appendFile("a", "foo"); + expect(await fs.readFile("a", "utf8")).toEqual("foo"); + }); + + it("chmod works", async () => { + await fs.writeFile("b", "hi"); + await fs.chmod("b", 0o755); + const s = await fs.stat("b"); + expect(s.mode.toString(8)).toBe("100755"); + }); + + it("constants work", async () => { + const constants = await fs.constants(); + expect(constants.O_RDONLY).toBe(0); + expect(constants.O_WRONLY).toBe(1); + expect(constants.O_RDWR).toBe(2); + }); + + it("copyFile works", async () => { + await fs.writeFile("c", "hello"); + await fs.copyFile("c", "d.txt"); + expect(await fs.readFile("d.txt", "utf8")).toEqual("hello"); + }); + + it("cp works on a directory", async () => { + await fs.mkdir("folder"); + await fs.writeFile("folder/a.txt", "hello"); + await fs.cp("folder", "folder2", { recursive: true }); + expect(await fs.readFile("folder2/a.txt", "utf8")).toEqual("hello"); + }); + + it("exists works", async () => { + expect(await fs.exists("does-not-exist")).toBe(false); + await fs.writeFile("does-exist", ""); + expect(await fs.exists("does-exist")).toBe(true); + }); + + it("creating a hard link works", async () => { + await fs.writeFile("source", "the source"); + await fs.link("source", "target"); + expect(await fs.readFile("target", "utf8")).toEqual("the source"); + // hard link, not symlink + expect(await fs.realpath("target")).toBe("target"); + + await fs.appendFile("source", " and more"); + expect(await fs.readFile("target", "utf8")).toEqual("the source and more"); + }); + + it("mkdir works", async () => { + await fs.mkdir("xyz"); + const s = await fs.stat("xyz"); + expect(s.isDirectory()).toBe(true); + expect(s.isFile()).toBe(false); + }); + + it("readFile works", async () => { + await fs.writeFile("a", Buffer.from([1, 2, 3])); + const s = await fs.readFile("a"); + expect(s).toEqual(Buffer.from([1, 2, 3])); + + await fs.writeFile("b.txt", "conat"); + const t = await fs.readFile("b.txt", "utf8"); + expect(t).toEqual("conat"); + }); + + it("the full error message structure is preserved exactly as in the nodejs library", async () => { + const path = randomId(); + try { + await fs.readFile(path); + } catch (err) { + expect(err.message).toEqual( + `ENOENT: no such file or directory, open '${path}'`, + ); + expect(err.message).toContain(path); + expect(err.code).toEqual("ENOENT"); + expect(err.errno).toEqual(-2); + expect(err.path).toEqual(path); + expect(err.syscall).toEqual("open"); + } + }); + + let fire; + it("readdir works", async () => { + await fs.mkdir("dirtest"); + for (let i = 0; i < 5; i++) { + await fs.writeFile(`dirtest/${i}`, `${i}`); + } + fire = "🔥.txt"; + await fs.writeFile(join("dirtest", fire), "this is ️‍🔥!"); + const v = await fs.readdir("dirtest"); + expect(v).toEqual(["0", "1", "2", "3", "4", fire]); + }); + + it("readdir with the withFileTypes option", async () => { + const path = "readdir-1"; + await fs.mkdir(path); + expect(await fs.readdir(path, { withFileTypes: true })).toEqual([]); + await fs.writeFile(join(path, "a.txt"), ""); + + { + const v = await fs.readdir(path, { withFileTypes: true }); + expect(v.map(({ name }) => name)).toEqual(["a.txt"]); + expect(v.map((x) => x.isFile())).toEqual([true]); + } + { + await fs.mkdir(join(path, "co")); + const v = await fs.readdir(path, { withFileTypes: true }); + expect(v.map(({ name }) => name)).toEqual(["a.txt", "co"]); + expect(v.map((x) => x.isFile())).toEqual([true, false]); + expect(v.map((x) => x.isDirectory())).toEqual([false, true]); + } + + { + await fs.symlink(join(path, "a.txt"), join(path, "link")); + const v = await fs.readdir(path, { withFileTypes: true }); + expect(v.map(({ name }) => name)).toEqual(["a.txt", "co", "link"]); + expect(v[2].isSymbolicLink()).toEqual(true); + } + }); + + it("readdir with the recursive option", async () => { + const path = "readdir-2"; + await fs.mkdir(path); + expect(await fs.readdir(path, { recursive: true })).toEqual([]); + await fs.mkdir(join(path, "subdir")); + await fs.writeFile(join(path, "subdir", "b.txt"), "x"); + const v = await fs.readdir(path, { recursive: true }); + expect(v).toEqual(["subdir", "subdir/b.txt"]); + + // and withFileTypes + const w = await fs.readdir(path, { recursive: true, withFileTypes: true }); + expect(w.map(({ name }) => name)).toEqual(["subdir", "b.txt"]); + expect(w[0]).toEqual( + expect.objectContaining({ + name: "subdir", + parentPath: path, + }), + ); + expect(w[0].isDirectory()).toBe(true); + expect(w[1]).toEqual( + expect.objectContaining({ + name: "b.txt", + parentPath: join(path, "subdir"), + }), + ); + expect(w[1].isFile()).toBe(true); + expect(await fs.readFile(join(w[1].parentPath, w[1].name), "utf8")).toEqual( + "x", + ); + }); + + it("readdir works with non-utf8 filenames in the path", async () => { + // this test uses internal implementation details (kind of crappy) + const path = "readdir-3"; + await fs.mkdir(path); + const fullPath = join(server.path, project_id, path); + + const buf = Buffer.from([0xff, 0xfe, 0xfd]); + expect(() => { + const decoder = new TextDecoder("utf-8", { fatal: true }); + decoder.decode(buf); + }).toThrow("not valid"); + const c = process.cwd(); + process.chdir(fullPath); + await writeFile(buf, "hi"); + process.chdir(c); + const w = await fs.readdir(path, { encoding: "buffer" }); + expect(w[0]).toEqual(buf); + }); + + it("use the find command instead of readdir", async () => { + const { stdout } = await fs.find("dirtest", { + options: ["-maxdepth", "1", "-mindepth", "1", "-printf", "%f\n"], + }); + const v = stdout.toString().trim().split("\n"); + // output of find is NOT in alphabetical order: + expect(new Set(v)).toEqual(new Set(["0", "1", "2", "3", "4", fire])); + }); + + it("realpath works", async () => { + await fs.writeFile("file0", "file0"); + await fs.symlink("file0", "file1"); + expect(await fs.readFile("file1", "utf8")).toBe("file0"); + const r = await fs.realpath("file1"); + expect(r).toBe("file0"); + + await fs.writeFile("file2", "file2"); + await fs.link("file2", "file3"); + expect(await fs.readFile("file3", "utf8")).toBe("file2"); + const r3 = await fs.realpath("file3"); + expect(r3).toBe("file3"); + }); + + it("rename a file", async () => { + await fs.writeFile("bella", "poo"); + await fs.rename("bella", "bells"); + expect(await fs.readFile("bells", "utf8")).toBe("poo"); + await fs.mkdir("x"); + await fs.rename("bells", "x/belltown"); + }); + + it("rm a file", async () => { + await fs.writeFile("bella-to-rm", "poo"); + await fs.rm("bella-to-rm"); + expect(await fs.exists("bella-to-rm")).toBe(false); + }); + + it("rm a directory", async () => { + await fs.mkdir("rm-dir"); + expect(async () => { + await fs.rm("rm-dir"); + }).rejects.toThrow("Path is a directory"); + await fs.rm("rm-dir", { recursive: true }); + expect(await fs.exists("rm-dir")).toBe(false); + }); + + it("rm a nonempty directory", async () => { + await fs.mkdir("rm-dir2"); + await fs.writeFile("rm-dir2/a", "a"); + await fs.rm("rm-dir2", { recursive: true }); + expect(await fs.exists("rm-dir2")).toBe(false); + }); + + it("rmdir empty directory", async () => { + await fs.mkdir("rm-dir3"); + await fs.rmdir("rm-dir3"); + expect(await fs.exists("rm-dir3")).toBe(false); + }); + + it("stat not existing path", async () => { + expect(async () => { + await fs.stat(randomId()); + }).rejects.toThrow("no such file or directory"); + }); + + it("stat a file", async () => { + await fs.writeFile("abc.txt", "hi"); + const stat = await fs.stat("abc.txt"); + expect(stat.size).toBe(2); + expect(stat.isFile()).toBe(true); + expect(stat.isSymbolicLink()).toBe(false); + expect(stat.isDirectory()).toBe(false); + expect(stat.isBlockDevice()).toBe(false); + expect(stat.isCharacterDevice()).toBe(false); + expect(stat.isSymbolicLink()).toBe(false); + expect(stat.isFIFO()).toBe(false); + expect(stat.isSocket()).toBe(false); + }); + + it("stat a directory", async () => { + await fs.mkdir("my-stat-dir"); + const stat = await fs.stat("my-stat-dir"); + expect(stat.isFile()).toBe(false); + expect(stat.isSymbolicLink()).toBe(false); + expect(stat.isDirectory()).toBe(true); + expect(stat.isBlockDevice()).toBe(false); + expect(stat.isCharacterDevice()).toBe(false); + expect(stat.isSymbolicLink()).toBe(false); + expect(stat.isFIFO()).toBe(false); + expect(stat.isSocket()).toBe(false); + }); + + it("stat a symlink", async () => { + await fs.writeFile("sl2", "the source"); + await fs.symlink("sl2", "target-sl2"); + const stat = await fs.stat("target-sl2"); + // this is how stat works! + expect(stat.isFile()).toBe(true); + expect(stat.isSymbolicLink()).toBe(false); + // so use lstat + const lstat = await fs.lstat("target-sl2"); + expect(lstat.isFile()).toBe(false); + expect(lstat.isSymbolicLink()).toBe(true); + }); + + it("truncate a file", async () => { + await fs.writeFile("t", ""); + await fs.truncate("t", 10); + const s = await fs.stat("t"); + expect(s.size).toBe(10); + }); + + it("delete a file with unlink", async () => { + await fs.writeFile("to-unlink", ""); + await fs.unlink("to-unlink"); + expect(await fs.exists("to-unlink")).toBe(false); + }); + + it("sets times of a file", async () => { + await fs.writeFile("my-times", ""); + const atime = Date.now() - 100_000; + const mtime = Date.now() - 10_000_000; + // NOTE: fs.utimes in nodejs takes *seconds*, not ms, hence + // dividing by 1000 here: + await fs.utimes("my-times", atime / 1000, mtime / 1000); + const s = await fs.stat("my-times"); + expect(s.atimeMs).toBeCloseTo(atime); + expect(s.mtimeMs).toBeCloseTo(mtime); + expect(s.atime.valueOf()).toBeCloseTo(atime); + expect(s.mtime.valueOf()).toBeCloseTo(mtime); + }); + + it("creating a symlink works (as does using lstat)", async () => { + await fs.writeFile("source1", "the source"); + await fs.symlink("source1", "target1"); + expect(await fs.readFile("target1", "utf8")).toEqual("the source"); + // symlink, not hard + expect(await fs.realpath("target1")).toBe("source1"); + await fs.appendFile("source1", " and more"); + expect(await fs.readFile("target1", "utf8")).toEqual("the source and more"); + const stats = await fs.stat("target1"); + expect(stats.isSymbolicLink()).toBe(false); + + const lstats = await fs.lstat("target1"); + expect(lstats.isSymbolicLink()).toBe(true); + + const stats0 = await fs.stat("source1"); + expect(stats0.isSymbolicLink()).toBe(false); + }); + + it("watch a file", async () => { + await fs.writeFile("a.txt", "hi"); + const w = await fs.watch("a.txt"); + await fs.appendFile("a.txt", " there"); + expect(await w.next()).toEqual({ + done: false, + value: { eventType: "change", filename: "a.txt" }, + }); + }); + + it("watch a directory", async () => { + const FOLDER = randomId(); + await fs.mkdir(FOLDER); + const w = await fs.watch(FOLDER); + + await fs.writeFile(join(FOLDER, "x"), "hi"); + expect(await w.next()).toEqual({ + done: false, + value: { eventType: "rename", filename: "x" }, + }); + expect(await w.next()).toEqual({ + done: false, + value: { eventType: "change", filename: "x" }, + }); + + await fs.appendFile(join(FOLDER, "x"), "xxx"); + expect(await w.next()).toEqual({ + done: false, + value: { eventType: "change", filename: "x" }, + }); + + await fs.writeFile(join(FOLDER, "z"), "there"); + expect(await w.next()).toEqual({ + done: false, + value: { eventType: "rename", filename: "z" }, + }); + expect(await w.next()).toEqual({ + done: false, + value: { eventType: "change", filename: "z" }, + }); + + // this is correct -- from the node docs "On most platforms, 'rename' is emitted whenever a filename appears or disappears in the directory." + await fs.unlink(join(FOLDER, "z")); + expect(await w.next()).toEqual({ + done: false, + value: { eventType: "rename", filename: "z" }, + }); + }); +}); + +describe("SECURITY CHECKS: dangerous symlinks can't be followed", () => { + let server; + let tempDir; + it("creates the simple fileserver service", async () => { + server = await createPathFileserver(); + tempDir = server.path; + }); + + const project_id = uuid(); + const project_id2 = uuid(); + let fs, fs2; + it("create two clients", () => { + fs = fsClient({ subject: `${server.service}.project-${project_id}` }); + fs2 = fsClient({ subject: `${server.service}.project-${project_id2}` }); + }); + + it("create a secret in one", async () => { + await fs.writeFile("password", "s3cr3t"); + await fs2.writeFile("a", "init"); + }); + + // This is setup bypassing security and is part of our threat model, due to users + // having full access internally to their sandbox fs. + it("directly create a dangerous file that is a symlink outside of the sandbox -- this should work", async () => { + await symlink( + join(tempDir, project_id, "password"), + join(tempDir, project_id2, "danger"), + ); + const s = await readFile(join(tempDir, project_id2, "danger"), "utf8"); + expect(s).toBe("s3cr3t"); + }); + + it("fails to read the symlink content via the api", async () => { + await expect(async () => { + await fs2.readFile("danger", "utf8"); + }).rejects.toThrow("outside of sandbox"); + }); + + it("directly create a dangerous relative symlink ", async () => { + await symlink( + join("..", project_id, "password"), + join(tempDir, project_id2, "danger2"), + ); + const s = await readFile(join(tempDir, project_id2, "danger2"), "utf8"); + expect(s).toBe("s3cr3t"); + }); + + it("fails to read the relative symlink content via the api", async () => { + await expect(async () => { + await fs2.readFile("danger2", "utf8"); + }).rejects.toThrow("outside of sandbox"); + }); + + // This is not a vulnerability, because there's no way for the user + // to create a hard link like this from within an nfs mount (say) + // of their own folder. + it("directly create a hard link", async () => { + await link( + join(tempDir, project_id, "password"), + join(tempDir, project_id2, "danger3"), + ); + const s = await readFile(join(tempDir, project_id2, "danger3"), "utf8"); + expect(s).toBe("s3cr3t"); + }); + + it("a hardlink *can* get outside the sandbox", async () => { + const s = await fs2.readFile("danger3", "utf8"); + expect(s).toBe("s3cr3t"); + }); + + it("closes the server", () => { + server.close(); + }); +}); + +afterAll(async () => { + await after(); + await cleanupFileservers(); +}); diff --git a/src/packages/backend/conat/files/test/util.ts b/src/packages/backend/conat/files/test/util.ts new file mode 100644 index 00000000000..b50a114c5da --- /dev/null +++ b/src/packages/backend/conat/files/test/util.ts @@ -0,0 +1,30 @@ +import { localPathFileserver } from "../local-path"; +import { mkdtemp, rm } from "node:fs/promises"; +import { tmpdir } from "node:os"; +import { join } from "path"; +import { client } from "@cocalc/backend/conat/test/setup"; +import { randomId } from "@cocalc/conat/names"; + +const tempDirs: string[] = []; +const servers: any[] = []; +export async function createPathFileserver({ + service = `fs-${randomId()}`, +}: { service?: string } = {}) { + const tempDir = await mkdtemp(join(tmpdir(), `cocalc-${randomId()}0`)); + tempDirs.push(tempDir); + const server = await localPathFileserver({ client, service, path: tempDir }); + servers.push(server); + return server; +} + +// clean up any +export async function cleanupFileservers() { + for (const server of servers) { + server.close(); + } + for (const tempDir of tempDirs) { + try { + await rm(tempDir, { force: true, recursive: true }); + } catch {} + } +} diff --git a/src/packages/backend/conat/files/test/watch.test.ts b/src/packages/backend/conat/files/test/watch.test.ts new file mode 100644 index 00000000000..dbe9dbc0e12 --- /dev/null +++ b/src/packages/backend/conat/files/test/watch.test.ts @@ -0,0 +1,75 @@ +import { before, after, client, wait } from "@cocalc/backend/conat/test/setup"; +import { watchServer, watchClient } from "@cocalc/conat/files/watch"; +import { SandboxedFilesystem } from "@cocalc/backend/sandbox"; +import { mkdtemp, rm } from "node:fs/promises"; +import { tmpdir } from "node:os"; +import { join } from "path"; +import { randomId } from "@cocalc/conat/names"; + +let tmp; +beforeAll(async () => { + await before(); + tmp = await mkdtemp(join(tmpdir(), `cocalc-${randomId()}0`)); +}); +afterAll(async () => { + await after(); + try { + await rm(tmp, { force: true, recursive: true }); + } catch {} +}); + +describe("basic core of the async path watch functionality", () => { + let fs; + it("creates sandboxed filesystem", () => { + fs = new SandboxedFilesystem(tmp); + }); + + let server; + it("create watch server", () => { + server = watchServer({ client, subject: "foo", watch: fs.watch }); + }); + + it("create a file", async () => { + await fs.writeFile("a.txt", "hi"); + }); + + let w; + it("create a watcher client for 'a.txt'", async () => { + w = await watchClient({ client, subject: "foo", path: "a.txt", fs }); + }); + + it("observe watch works", async () => { + await fs.appendFile("a.txt", "foo"); + expect(await w.next()).toEqual({ + done: false, + value: { eventType: "change", filename: "a.txt" }, + }); + + await fs.appendFile("a.txt", "bar"); + expect(await w.next()).toEqual({ + done: false, + value: { eventType: "change", filename: "a.txt" }, + }); + }); + + it("close the watcher client frees up a server socket", async () => { + expect(Object.keys(server.sockets).length).toEqual(1); + w.close(); + await wait({ until: () => Object.keys(server.sockets).length == 0 }); + expect(Object.keys(server.sockets).length).toEqual(0); + }); + + it("trying to watch file that does not exist throws error", async () => { + await expect(async () => { + await watchClient({ client, subject: "foo", path: "b.txt", fs }); + }).rejects.toThrow( + "Error: ENOENT: no such file or directory, watch 'b.txt'", + ); + + try { + await watchClient({ client, subject: "foo", path: "b.txt", fs }); + } catch (err) { + expect(err.code).toEqual("ENOENT"); + } + }); +}); diff --git a/src/packages/backend/conat/sync.ts b/src/packages/backend/conat/sync.ts index 3bccd54978a..50963ac6960 100644 --- a/src/packages/backend/conat/sync.ts +++ b/src/packages/backend/conat/sync.ts @@ -7,7 +7,6 @@ import { dkv as createDKV, type DKV, type DKVOptions } from "@cocalc/conat/sync/ import { dko as createDKO, type DKO } from "@cocalc/conat/sync/dko"; import { akv as createAKV, type AKV } from "@cocalc/conat/sync/akv"; import { astream as createAStream, type AStream } from "@cocalc/conat/sync/astream"; -import { createOpenFiles, type OpenFiles } from "@cocalc/conat/sync/open-files"; export { inventory } from "@cocalc/conat/sync/inventory"; import "./index"; @@ -35,6 +34,3 @@ export async function dko(opts: DKVOptions): Promise> { return await createDKO(opts); } -export async function openFiles(project_id: string, opts?): Promise { - return await createOpenFiles({ project_id, ...opts }); -} diff --git a/src/packages/backend/conat/test/core/core-stream-break.test.ts b/src/packages/backend/conat/test/core/core-stream-break.test.ts index e0fdaeecd1c..7b0cb49828b 100644 --- a/src/packages/backend/conat/test/core/core-stream-break.test.ts +++ b/src/packages/backend/conat/test/core/core-stream-break.test.ts @@ -70,7 +70,7 @@ describe("stop persist server, create a client, create an ephemeral core-stream, try { await stream.publish("y", { timeout: 100 }); } catch (err) { - expect(`${err}`).toContain("timeout"); + expect(`${err}`).toContain("timed out"); } }); diff --git a/src/packages/backend/conat/test/files/file-server.test.ts b/src/packages/backend/conat/test/files/file-server.test.ts new file mode 100644 index 00000000000..22eb7ea750b --- /dev/null +++ b/src/packages/backend/conat/test/files/file-server.test.ts @@ -0,0 +1,178 @@ +import { before, after, connect } from "@cocalc/backend/conat/test/setup"; +import { + server as createFileServer, + client as createFileClient, +} from "@cocalc/conat/files/file-server"; +import { uuid } from "@cocalc/util/misc"; +import { type SnapshotCounts } from "@cocalc/util/consts/snapshots"; + +beforeAll(before); + +const Stub = (async () => {}) as any; + +describe("create basic mocked file server and test it out", () => { + let client1, client2; + it("create two clients", () => { + client1 = connect(); + client2 = connect(); + }); + + const volumes = new Set(); + const quotaSize: { [project_id: string]: number } = {}; + it("create file server", async () => { + await createFileServer({ + client: client1, + mount: async ({ project_id }): Promise<{ path: string }> => { + volumes.add(project_id); + return { path: `/mnt/${project_id}` }; + }, + + // create project_id as an exact lightweight clone of src_project_id + clone: async (opts: { + project_id: string; + src_project_id: string; + }): Promise => { + volumes.add(opts.project_id); + }, + + getUsage: async (_opts: { + project_id: string; + }): Promise<{ + size: number; + used: number; + free: number; + }> => { + return { size: 0, used: 0, free: 0 }; + }, + + getQuota: async (_opts: { + project_id: string; + }): Promise<{ + size: number; + used: number; + }> => { + return { size: quotaSize[project_id] ?? 0, used: 0 }; + }, + + setQuota: async ({ + project_id, + size, + }: { + project_id: string; + size: number | string; + }): Promise => { + quotaSize[project_id] = typeof size == "string" ? parseInt(size) : size; + }, + + cp: async (_opts: { + // the src paths are relative to the src volume + src: { project_id: string; path: string | string[] }; + // the dest path is relative to the dest volume + dest: { project_id: string; path: string }; + options?; + }): Promise => {}, + + createBackup: async (_opts: { + project_id: string; + }): Promise<{ time: Date; id: string }> => { + return { time: new Date(), id: "0" }; + }, + + restoreBackup: async (_opts: { + project_id: string; + id: string; + path?: string; + dest?: string; + }): Promise => {}, + + deleteBackup: async (_opts: { + project_id: string; + id: string; + }): Promise => {}, + + updateBackups: async (_opts: { + project_id: string; + counts?: Partial; + limit?: number; + }): Promise => {}, + + getBackups: async (_opts: { + project_id: string; + }): Promise< + { + id: string; + time: Date; + }[] + > => { + return []; + }, + + getBackupFiles: async (_opts: { + project_id: string; + id: string; + }): Promise => { + return []; + }, + + createSnapshot: async (_opts: { + project_id: string; + name?: string; + limit?: number; + }): Promise => {}, + + deleteSnapshot: async (_opts: { + project_id: string; + name: string; + }): Promise => {}, + + updateSnapshots: async (_opts: { + project_id: string; + counts?: Partial; + limit?: number; + }): Promise => {}, + + allSnapshotUsage: Stub, + + createSync: Stub, + getAllSyncs: Stub, + getSync: Stub, + syncCommand: Stub, + }); + }); + + let project_id; + it("make a client and test the file server", async () => { + project_id = uuid(); + const fileClient = createFileClient({ + client: client2, + }); + const { path } = await fileClient.mount({ project_id }); + expect(path).toEqual(`/mnt/${project_id}`); + expect(volumes.has(project_id)); + + expect(await fileClient.getUsage({ project_id })).toEqual({ + size: 0, + used: 0, + free: 0, + }); + + expect(await fileClient.getQuota({ project_id })).toEqual({ + size: 0, + used: 0, + }); + + await fileClient.setQuota({ project_id, size: 10 }); + + expect(await fileClient.getQuota({ project_id })).toEqual({ + size: 10, + used: 0, + }); + + await fileClient.cp({ + src: { project_id, path: "x" }, + dest: { project_id, path: "y" }, + }); + }); +}); + +afterAll(after); diff --git a/src/packages/backend/conat/test/llm.test.ts b/src/packages/backend/conat/test/llm.test.ts index 9ac69af07fb..ab0eeb8ece8 100644 --- a/src/packages/backend/conat/test/llm.test.ts +++ b/src/packages/backend/conat/test/llm.test.ts @@ -18,7 +18,7 @@ beforeAll(before); describe("create an llm server, client, and stub evaluator, and run an evaluation", () => { // define trivial evaluate - const OUTPUT = "Thanks for asing about "; + const OUTPUT = "Thanks for asking about "; async function evaluate({ input, stream }) { stream(OUTPUT); stream(input); @@ -52,7 +52,7 @@ describe("create an llm server, client, and stub evaluator, and run an evaluatio describe("test an evaluate that throws an error half way through", () => { // define trivial evaluate - const OUTPUT = "Thanks for asing about "; + const OUTPUT = "Thanks for asking about "; const ERROR = "I give up"; async function evaluate({ stream }) { stream(OUTPUT); diff --git a/src/packages/backend/conat/test/project/jupyter/run-code.test.ts b/src/packages/backend/conat/test/project/jupyter/run-code.test.ts new file mode 100644 index 00000000000..344255efdd3 --- /dev/null +++ b/src/packages/backend/conat/test/project/jupyter/run-code.test.ts @@ -0,0 +1,287 @@ +/* + +DEVELOPMENT: + +pnpm test `pwd`/run-code.test.ts + +*/ + +import { + before, + after, + connect, + delay, + wait, +} from "@cocalc/backend/conat/test/setup"; +import { + jupyterClient, + jupyterServer, +} from "@cocalc/conat/project/jupyter/run-code"; +import { uuid } from "@cocalc/util/misc"; + +// it's really 100+, but tests fails if less than this. +const MIN_EVALS_PER_SECOND = 10; + +beforeAll(before); + +async function getKernelStatus(_opts: { path: string }) { + return { backend_state: "off" as "off", kernel_state: "idle" as "idle" }; +} + +describe("create very simple mocked jupyter runner and test evaluating code", () => { + let client1, client2; + it("create two clients", async () => { + client1 = connect(); + client2 = connect(); + }); + + let server; + const project_id = uuid(); + it("create jupyter code run server", () => { + // running code with this just results in two responses: the path and the cells + async function run({ path, cells }) { + async function* runner() { + yield { path, id: "0" }; + yield { cells, id: "0" }; + } + return runner(); + } + + server = jupyterServer({ + client: client1, + project_id, + run, + getKernelStatus, + }); + }); + + let client; + const path = "a.ipynb"; + const cells = [{ id: "a", input: "2+3" }]; + it("create a jupyter client, then run some code", async () => { + client = jupyterClient({ path, project_id, client: client2 }); + const iter = await client.run(cells); + const v: any[] = []; + for await (const output of iter) { + v.push(...output); + } + expect(v).toEqual([ + { path, id: "0" }, + { cells, id: "0" }, + ]); + }); + + it("start iterating over the output after waiting", async () => { + // this is the same as the previous test, except we insert a + // delay from when we create the iterator, and when we start + // reading values out of it. This is important to test, because + // it was broken in my first implementation, and is a common mistake + // when implementing async iterators. + client.verbose = true; + const iter = await client.run(cells); + const v: any[] = []; + await delay(500); + for await (const output of iter) { + v.push(...output); + } + expect(v).toEqual([ + { path, id: "0" }, + { cells, id: "0" }, + ]); + }); + + const count = 100; + it(`run ${count} evaluations to ensure that the speed is reasonable (and also everything is kept properly ordered, etc.)`, async () => { + const start = Date.now(); + for (let i = 0; i < count; i++) { + const v: any[] = []; + const cells = [{ id: `${i}`, input: `${i} + ${i}` }]; + for await (const output of await client.run(cells)) { + v.push(...output); + } + expect(v).toEqual([ + { path, id: "0" }, + { cells, id: "0" }, + ]); + } + const evalsPerSecond = Math.floor((1000 * count) / (Date.now() - start)); + if (process.env.BENCH) { + console.log({ evalsPerSecond }); + } + expect(evalsPerSecond).toBeGreaterThan(MIN_EVALS_PER_SECOND); + }); + + it("cleans up", () => { + server.close(); + client.close(); + }); +}); + +describe("create simple mocked jupyter runner that does actually eval an expression", () => { + let client1, client2; + it("create two clients", async () => { + client1 = connect(); + client2 = connect(); + }); + + let server; + const project_id = uuid(); + const compute_server_id = 3; + it("create jupyter code run server", () => { + // running code with this just results in two responses: the path and the cells + async function run({ cells }) { + async function* runner() { + for (const { id, input } of cells) { + yield { id, output: eval(input) }; + } + } + return runner(); + } + + server = jupyterServer({ + client: client1, + project_id, + run, + compute_server_id, + getKernelStatus, + }); + }); + + let client; + const path = "b.ipynb"; + const cells = [ + { id: "a", input: "2+3" }, + { id: "b", input: "3**5" }, + ]; + it("create a jupyter client, then run some code", async () => { + client = jupyterClient({ + path, + project_id, + client: client2, + compute_server_id, + }); + const iter = await client.run(cells); + const v: any[] = []; + for await (const output of iter) { + v.push(...output); + } + expect(v).toEqual([ + { id: "a", output: 5 }, + { id: "b", output: 243 }, + ]); + }); + + it("run code that FAILS and see error is visible to client properly", async () => { + const iter = await client.run([ + { id: "a", input: "2+3" }, + { id: "b", input: "2+invalid" }, + ]); + try { + for await (const _ of iter) { + } + } catch (err) { + expect(`${err}`).toContain("ReferenceError: invalid is not defined"); + } + }); + + it("cleans up", () => { + server.close(); + client.close(); + }); +}); + +describe("create mocked jupyter runner that does failover to backend output management when client disconnects", () => { + let client1, client2; + it("create two clients", async () => { + client1 = connect(); + client2 = connect(); + }); + + const path = "b.ipynb"; + const cells = [ + { id: "a", input: "10*(2+3)" }, + { id: "b", input: "100" }, + ]; + let server; + const project_id = uuid(); + let handler: any = null; + + it("create jupyter code run server that also takes as long as the output to run", () => { + async function run({ cells }) { + async function* runner() { + for (const { id, input } of cells) { + const output = eval(input); + await delay(output); + yield { id, output }; + } + } + return runner(); + } + + class OutputHandler { + messages: any[] = []; + + constructor(public cells) {} + + process = (mesg: any) => { + this.messages.push(mesg); + }; + done = () => { + this.messages.push({ done: true }); + }; + } + + function outputHandler({ path: path0, cells }) { + if (path0 != path) { + throw Error(`path must be ${path}`); + } + handler = new OutputHandler(cells); + return handler; + } + + server = jupyterServer({ + client: client1, + project_id, + run, + outputHandler, + getKernelStatus, + }); + }); + + let client; + it("create a jupyter client, then run some code (doesn't use output handler)", async () => { + client = jupyterClient({ + path, + project_id, + client: client2, + }); + const iter = await client.run(cells); + const v: any[] = []; + for await (const output of iter) { + v.push(output); + } + expect(v).toEqual([[{ id: "a", output: 50 }], [{ id: "b", output: 100 }]]); + }); + + it("starts code running then closes the client, which causes output to have to be placed in the handler instead.", async () => { + await client.run(cells); + client.close(); + await wait({ + until: () => { + return handler.messages.length >= 3; + }, + }); + expect(handler.messages).toEqual([ + { id: "a", output: 50 }, + { id: "b", output: 100 }, + { done: true }, + ]); + }); + + it("cleans up", () => { + server.close(); + client.close(); + }); +}); + +afterAll(after); diff --git a/src/packages/backend/conat/test/project/runner/load-balancer-2.test.ts b/src/packages/backend/conat/test/project/runner/load-balancer-2.test.ts new file mode 100644 index 00000000000..f568760c0b8 --- /dev/null +++ b/src/packages/backend/conat/test/project/runner/load-balancer-2.test.ts @@ -0,0 +1,102 @@ +/* + +DEVELOPMENT: + + +pnpm test `pwd`/load-balancer-2.test.ts + + +*/ + +import { before, after, connect } from "@cocalc/backend/conat/test/setup"; +import { server as projectRunnerServer } from "@cocalc/conat/project/runner/run"; +import { + server as lbServer, + client as lbClient, +} from "@cocalc/conat/project/runner/load-balancer"; +import { uuid } from "@cocalc/util/misc"; + +beforeAll(before); + +describe("create runner and load balancer with getConfig function", () => { + let client1, client2; + it("create two clients", () => { + client1 = connect(); + client2 = connect(); + }); + + const running: { [project_id: string]: any } = {}; + const projectState: { [project_id: string]: any } = {}; + + it("create project runner server and load balancer with getConfig and setState functions", async () => { + await projectRunnerServer({ + id: "0", + client: client1, + start: async ({ project_id, config }) => { + running[project_id] = { ...config }; + }, + stop: async ({ project_id }) => { + if (project_id) { + delete running[project_id]; + } else { + Object.keys(running).forEach( + (project_id) => delete running[project_id], + ); + } + }, + status: async ({ project_id }) => { + return running[project_id] != null + ? { state: "running" } + : { state: "opened" }; + }, + localPath: async ({ project_id }) => { + return { home: `/tmp/${project_id}` }; + }, + move: async () => {}, + save: async () => {}, + }); + await lbServer({ + client: client1, + getConfig: async ({ project_id }) => { + return { name: project_id }; + }, + setState: async ({ project_id, state }) => { + projectState[project_id] = state; + }, + }); + }); + + it("make a client for the load balancer, and test the runner via the load balancer", async () => { + const project_id = uuid(); + const lbc = lbClient({ + subject: `project.${project_id}.run`, + client: client2, + }); + await lbc.start(); + + expect(projectState).toEqual({ [project_id]: "running" }); + expect(running[project_id]).toEqual({ name: project_id }); + + expect(await lbc.status()).toEqual({ + server: "0", + state: "running", + }); + + const lbc2 = lbClient({ + subject: `project.${uuid()}.run`, + client: client2, + }); + expect(await lbc2.status()).toEqual({ + server: "0", + state: "opened", + }); + + await lbc.stop(); + expect(await lbc.status()).toEqual({ + server: "0", + state: "opened", + }); + }); +}); + +afterAll(after); diff --git a/src/packages/backend/conat/test/project/runner/load-balancer.test.ts b/src/packages/backend/conat/test/project/runner/load-balancer.test.ts new file mode 100644 index 00000000000..1392cb628b0 --- /dev/null +++ b/src/packages/backend/conat/test/project/runner/load-balancer.test.ts @@ -0,0 +1,112 @@ +/* + +DEVELOPMENT: + +pnpm test `pwd`/load-balancer.test.ts + +*/ + +import { before, after, connect } from "@cocalc/backend/conat/test/setup"; +import { + server as projectRunnerServer, + client as projectRunnerClient, +} from "@cocalc/conat/project/runner/run"; +import { + server as lbServer, + client as lbClient, +} from "@cocalc/conat/project/runner/load-balancer"; +import { uuid } from "@cocalc/util/misc"; + +beforeAll(before); + +describe("create basic mocked project runner service and test", () => { + let client1, client2; + it("create two clients", () => { + client1 = connect(); + client2 = connect(); + }); + + it("create project runner server", async () => { + const running = new Set(); + await projectRunnerServer({ + id: "0", + client: client1, + start: async ({ project_id }) => { + running.add(project_id); + }, + stop: async ({ project_id }) => { + if (project_id) { + running.delete(project_id); + } else { + running.clear(); + } + }, + status: async ({ project_id }) => { + return running.has(project_id) + ? { state: "running" } + : { state: "opened" }; + }, + localPath: async ({ project_id }) => { + return { home: `/tmp/${project_id}` }; + }, + move: async () => {}, + save: async () => {}, + }); + }); + + it("make a client and test the server", async () => { + const project_id = uuid(); + const runClient = projectRunnerClient({ + subject: "project-runner.0", + client: client2, + }); + await runClient.start({ project_id }); + expect(await runClient.status({ project_id })).toEqual({ + server: "0", + state: "running", + }); + expect(await runClient.status({ project_id: uuid() })).toEqual({ + server: "0", + state: "opened", + }); + await runClient.stop({ project_id }); + expect(await runClient.status({ project_id })).toEqual({ + server: "0", + state: "opened", + }); + }); + + it("make a load balancer", async () => { + await lbServer({ client: client1 }); + }); + + it("make a client for the load balancer, and test the runner via the load balancer", async () => { + const project_id = uuid(); + const lbc = lbClient({ + subject: `project.${project_id}.run`, + client: client2, + }); + await lbc.start(); + expect(await lbc.status()).toEqual({ + server: "0", + state: "running", + }); + + const lbc2 = lbClient({ + subject: `project.${uuid()}.run`, + client: client2, + }); + expect(await lbc2.status()).toEqual({ + server: "0", + state: "opened", + }); + + await lbc.stop(); + expect(await lbc.status()).toEqual({ + server: "0", + state: "opened", + }); + }); +}); + +afterAll(after); diff --git a/src/packages/backend/conat/test/project/runner/run.test.ts b/src/packages/backend/conat/test/project/runner/run.test.ts new file mode 100644 index 00000000000..789320a593a --- /dev/null +++ b/src/packages/backend/conat/test/project/runner/run.test.ts @@ -0,0 +1,122 @@ +/* + +DEVELOPMENT: + +pnpm test `pwd`/run-code.test.ts + +*/ + +import { before, after, connect, wait } from "@cocalc/backend/conat/test/setup"; +import { + server as projectRunnerServer, + client as projectRunnerClient, +} from "@cocalc/conat/project/runner/run"; +import { uuid } from "@cocalc/util/misc"; +import state from "@cocalc/conat/project/runner/state"; + +beforeAll(before); + +describe("create basic mocked project runner service and test", () => { + let client1, client2; + it("create two clients", () => { + client1 = connect(); + client2 = connect(); + }); + + it("create project runner server", async () => { + const running = new Set(); + await projectRunnerServer({ + id: "0", + client: client1, + start: async ({ project_id }) => { + running.add(project_id); + }, + stop: async ({ project_id }) => { + if (project_id) { + running.delete(project_id); + } else { + running.clear(); + } + }, + status: async ({ project_id }) => + running.has(project_id) ? { state: "running" } : { state: "opened" }, + localPath: async ({ project_id }) => { + return { home: `/tmp/${project_id}` }; + }, + move: async () => {}, + save: async () => {}, + }); + }); + + let project_id; + it("make a client and test the server", async () => { + project_id = uuid(); + const runClient = projectRunnerClient({ + subject: "project-runner.0", + client: client2, + }); + await runClient.start({ project_id }); + expect(await runClient.status({ project_id })).toEqual({ + state: "running", + server: "0", + }); + expect(await runClient.status({ project_id: uuid() })).toEqual({ + state: "opened", + server: "0", + }); + await runClient.stop({ project_id }); + expect(await runClient.status({ project_id })).toEqual({ + state: "opened", + server: "0", + }); + }); + + it("get the status of the runner", async () => { + const { projects, runners } = await state({ client: client2 }); + expect(runners.getAll()).toEqual({ "0": { time: runners.get("0")?.time } }); + await wait({ until: () => projects.get(project_id)?.state == "opened" }); + expect(projects.get(project_id)).toEqual({ state: "opened", server: "0" }); + }); + + it("add another runner and observe it appears", async () => { + const running = new Set(); + await projectRunnerServer({ + id: "1", + client: client1, + start: async ({ project_id }) => { + running.add(project_id); + }, + stop: async ({ project_id }) => { + if (project_id) { + running.delete(project_id); + } else { + running.clear(); + } + }, + status: async ({ project_id }) => + running.has(project_id) ? { state: "running" } : { state: "opened" }, + localPath: async ({ project_id }) => { + return { home: `/tmp/${project_id}` }; + }, + move: async () => {}, + save: async () => {}, + }); + + const { runners } = await state({ client: client2 }); + await wait({ + until: () => runners.get("1") != null, + }); + }); + + it("run a projects on server 1", async () => { + const runClient = projectRunnerClient({ + subject: "project-runner.1", + client: client2, + }); + const project_id = uuid(); + const x = await runClient.start({ project_id }); + expect(x.server).toEqual("1"); + }); +}); + +afterAll(after); diff --git a/src/packages/backend/conat/test/setup.ts b/src/packages/backend/conat/test/setup.ts index 151eff2227a..e36ba0ee05f 100644 --- a/src/packages/backend/conat/test/setup.ts +++ b/src/packages/backend/conat/test/setup.ts @@ -21,6 +21,7 @@ import { once } from "@cocalc/util/async-utils"; import { until } from "@cocalc/util/async-utils"; import { randomId } from "@cocalc/conat/names"; import { isEqual } from "lodash"; +import { setConatServer } from "@cocalc/backend/data"; export { wait, delay, once }; @@ -150,6 +151,7 @@ export async function before( } server = await createServer(); + setConatServer(server.address()); client = connect(); persistServer = createPersistServer({ client }); setConatClient({ @@ -299,7 +301,15 @@ export async function waitForConsistentState( export async function after() { persistServer?.close(); - await rm(tempDir, { force: true, recursive: true }); + while (true) { + try { + await rm(tempDir, { force: true, recursive: true }); + break; + } catch (err) { + console.log(err); + await delay(1000); + } + } try { server?.close(); } catch {} diff --git a/src/packages/backend/conat/test/socket/basic.test.ts b/src/packages/backend/conat/test/socket/basic.test.ts index 1ae6e364b2f..36e4467b2ac 100644 --- a/src/packages/backend/conat/test/socket/basic.test.ts +++ b/src/packages/backend/conat/test/socket/basic.test.ts @@ -168,7 +168,7 @@ describe("create a client first and write more messages than the queue size resu }); it("wait for client to drain; then we can now send another message without an error", async () => { - await client.waitUntilDrain(); + await client.drain(); client.write("foo"); }); diff --git a/src/packages/backend/conat/test/sync-doc/conflict.test.ts b/src/packages/backend/conat/test/sync-doc/conflict.test.ts new file mode 100644 index 00000000000..739b33da7e6 --- /dev/null +++ b/src/packages/backend/conat/test/sync-doc/conflict.test.ts @@ -0,0 +1,277 @@ +/* +Illustrate and test behavior when there is conflict. + +TODO: we must get noAutosave to fully work so we can make +the tests of conflicts, etc., better. + +E.g, the test below WILL RANDOMLY FAIL right now due to autosave randomness... +*/ + +import { + before, + after, + uuid, + connect, + server, + once, + delay, + waitUntilSynced, +} from "./setup"; +import { split } from "@cocalc/util/misc"; + +beforeAll(before); +afterAll(after); + +const GAP_DELAY = 50; + +describe("synchronized editing with branching and merging", () => { + const project_id = uuid(); + let s1, s2, client1, client2; + + it("creates two clients", async () => { + client1 = connect(); + client2 = connect(); + s1 = client1.sync.string({ + project_id, + path: "a.txt", + service: server.service, + noAutosave: true, + }); + await once(s1, "ready"); + + s2 = client2.sync.string({ + project_id, + path: "a.txt", + service: server.service, + noAutosave: true, + }); + await once(s2, "ready"); + expect(s1.to_str()).toBe(""); + expect(s2.to_str()).toBe(""); + expect(s1 === s2).toBe(false); + }); + + it("both clients set the first version independently and inconsistently", async () => { + s2.from_str("y"); + s1.from_str("x"); + s1.commit(); + // delay so s2's time is always bigger than s1's so our unit test + // is well defined + await delay(GAP_DELAY); + s2.commit(); + await s1.save(); + await s2.save(); + }); + + it("wait until both clients see two heads", async () => { + await waitUntilSynced([s1, s2]); + const heads1 = s1.patch_list.getHeads(); + const heads2 = s2.patch_list.getHeads(); + expect(heads1.length).toBe(2); + expect(heads2.length).toBe(2); + expect(heads1).toEqual(heads2); + }); + + it("get the current value, which is a merge", () => { + const v1 = s1.to_str(); + const v2 = s2.to_str(); + expect(v1).toEqual("xy"); + expect(v2).toEqual("xy"); + }); + + it("commit current value and see that there is a new single head that both share, thus resolving the merge in this way", async () => { + s1.commit(); + await s1.save(); + await waitUntilSynced([s1, s2]); + const heads1 = s1.patch_list.getHeads(); + const heads2 = s2.patch_list.getHeads(); + expect(heads1.length).toBe(1); + expect(heads2.length).toBe(1); + expect(heads1).toEqual(heads2); + }); + + it("set values inconsistently again and explicitly resolve the merge conflict in a way that is different than the default", async () => { + s1.from_str("xy1"); + s1.commit(); + await delay(GAP_DELAY); + s2.from_str("xy2"); + s2.commit(); + await s1.save(); + await s2.save(); + + await waitUntilSynced([s1, s2]); + expect(s1.to_str()).toEqual("xy12"); + expect(s2.to_str()).toEqual("xy12"); + + // resolve the conflict in our own way + s1.from_str("xy3"); + s1.commit(); + await s1.save(); + await waitUntilSynced([s1, s2]); + + // everybody has this state now + expect(s1.to_str()).toEqual("xy3"); + expect(s2.to_str()).toEqual("xy3"); + }); +}); + +describe("do the example in the blog post 'Lies I was Told About Collaborative Editing, Part 1: Algorithms for offline editing' -- https://www.moment.dev/blog/lies-i-was-told-pt-1", () => { + const project_id = uuid(); + let client1, client2; + + async function getInitialState(path: string) { + client1 ??= connect(); + client2 ??= connect(); + client1 + .fs({ project_id, service: server.service }) + .writeFile(path, "The Color of Pomegranates"); + const alice = client1.sync.string({ + project_id, + path, + service: server.service, + noAutosave: true, + }); + await once(alice, "ready"); + await alice.save(); + + const bob = client2.sync.string({ + project_id, + path, + service: server.service, + noAutosave: true, + }); + await once(bob, "ready"); + await bob.save(); + await waitUntilSynced([bob, alice]); + + return { alice, bob }; + } + + let alice, bob; + it("creates two clients", async () => { + ({ alice, bob } = await getInitialState("first.txt")); + expect(alice.to_str()).toEqual("The Color of Pomegranates"); + expect(bob.to_str()).toEqual("The Color of Pomegranates"); + }); + + it("Bob changes the spelling of Color to the British Colour and unaware Alice deletes all of the text.", async () => { + bob.from_str("The Colour of Pomegranates"); + bob.commit(); + alice.from_str(""); + alice.commit(); + }); + + it("Both come back online -- the resolution is the empty (with either order above) string because the **best effort** application of inserting the u (with context) to either is a no-op.", async () => { + await bob.save(); + await alice.save(); + await waitUntilSynced([bob, alice]); + expect(alice.to_str()).toEqual(""); + expect(bob.to_str()).toEqual(""); + }); + + it("the important thing about the cocalc approach is that a consistent history is saved, so everybody knows precisely what happened. **I.e., the fact that at one point Bob adding a British u is not lost to either party!**", () => { + const v = alice.versions(); + const x = v.map((t) => alice.version(t).to_str()); + expect(new Set(x)).toEqual( + new Set(["The Color of Pomegranates", "The Colour of Pomegranates", ""]), + ); + + const w = alice.versions(); + const y = w.map((t) => bob.version(t).to_str()); + expect(y).toEqual(x); + }); + + it("reset -- create alicea and bob again", async () => { + ({ alice, bob } = await getInitialState("second.txt")); + }); + + // opposite order this time + it("Bob changes the spelling of Color to the British Colour and unaware Alice deletes all of the text.", async () => { + alice.from_str(""); + alice.commit(); + bob.from_str("The Colour of Pomegranates"); + bob.commit(); + }); + + it("both empty again", async () => { + await bob.save(); + await alice.save(); + await waitUntilSynced([bob, alice]); + expect(alice.to_str()).toEqual(""); + expect(bob.to_str()).toEqual(""); + }); + + it("There are two heads; either client can resolve the merge conflict.", async () => { + expect(alice.patch_list.getHeads().length).toBe(2); + expect(bob.patch_list.getHeads().length).toBe(2); + bob.from_str("The Colour of Pomegranates"); + bob.commit(); + await bob.save(); + + await waitUntilSynced([bob, alice]); + expect(alice.to_str()).toEqual("The Colour of Pomegranates"); + expect(bob.to_str()).toEqual("The Colour of Pomegranates"); + }); +}); + +const numHeads = 15; +describe(`create editing conflict with ${numHeads} heads`, () => { + const project_id = uuid(); + let docs: any[] = [], + clients: any[] = []; + + it(`create ${numHeads} clients`, async () => { + const v: any[] = []; + for (let i = 0; i < numHeads; i++) { + const client = connect(); + clients.push(client); + const doc = client.sync.string({ + project_id, + path: "a.txt", + service: server.service, + noAutosave: true, + }); + docs.push(doc); + v.push(once(doc, "ready")); + } + await Promise.all(v); + }); + + it("every client writes a different value all at once", async () => { + for (let i = 0; i < numHeads; i++) { + docs[i].from_str(`${i} `); + docs[i].commit(); + docs[i].save(); + } + await waitUntilSynced(docs); + const heads = docs[0].patch_list.getHeads(); + expect(heads.length).toBe(docs.length); + }); + + it("merge -- order is random, but value is consistent", async () => { + const value = docs[0].to_str(); + let v = new Set(); + for (let i = 0; i < numHeads; i++) { + v.add(`${i}`); + expect(docs[i].to_str()).toEqual(value); + } + const t = new Set(split(docs[0].to_str())); + expect(t).toEqual(v); + }); + + it(`resolve the merge conflict -- all ${numHeads} clients then see the resolution`, async () => { + let r = ""; + for (let i = 0; i < numHeads; i++) { + r += `${i} `; + } + docs[0].from_str(r); + docs[0].commit(); + await docs[0].save(); + + await waitUntilSynced(docs); + for (let i = 0; i < numHeads; i++) { + expect(docs[i].to_str()).toEqual(r); + } + // docs[0].show_history(); + }); +}); diff --git a/src/packages/backend/conat/test/sync-doc/delete.test.ts b/src/packages/backend/conat/test/sync-doc/delete.test.ts new file mode 100644 index 00000000000..7802b6578f4 --- /dev/null +++ b/src/packages/backend/conat/test/sync-doc/delete.test.ts @@ -0,0 +1,115 @@ +import { before, after, uuid, connect, server, once, delay } from "./setup"; + +beforeAll(before); +afterAll(after); + +describe("deleting a file that is open as a syncdoc", () => { + const project_id = uuid(); + const path = "a.txt"; + let client1, client2, s1, s2, fs; + const deletedThreshold = 50; // make test faster + const watchRecreateWait = 100; + const readLockTimeout = 250; + + it("creates two clients editing 'a.txt'", async () => { + client1 = connect(); + client2 = connect(); + fs = client1.fs({ project_id, service: server.service }); + await fs.writeFile(path, "my existing file"); + s1 = client1.sync.string({ + project_id, + path, + fs, + deletedThreshold, + watchRecreateWait, + readLockTimeout, + }); + + await once(s1, "ready"); + + s2 = client2.sync.string({ + project_id, + path, + service: server.service, + deletedThreshold, + watchRecreateWait, + readLockTimeout, + }); + await once(s2, "ready"); + }); + + it(`delete 'a.txt' from disk and both clients emit 'deleted' event in about ${deletedThreshold}ms`, async () => { + const start = Date.now(); + const d1 = once(s1, "deleted"); + const d2 = once(s2, "deleted"); + await fs.unlink("a.txt"); + await d1; + await d2; + expect(Date.now() - start).toBeLessThan(deletedThreshold + 1000); + }); + + it("clients still work (clients can ignore 'deleted' if they want)", async () => { + expect(s1.isClosed()).toBe(false); + expect(s2.isClosed()).toBe(false); + s1.from_str("back"); + const w1 = once(s1, "watching"); + const w2 = once(s2, "watching"); + await s1.save_to_disk(); + await w1; + await w2; + + // note: we lock for a moment after write to avoid a race condition + // with multiple clientss editing. + try { + await fs.readFile("a.txt", "utf8"); + } catch (err) { + expect(`${err}`).toContain("locked"); + } + await delay(readLockTimeout * 3); + expect(await fs.readFile("a.txt", "utf8")).toEqual("back"); + }); + + it(`deleting 'a.txt' again -- still triggers deleted events`, async () => { + const start = Date.now(); + const d1 = once(s1, "deleted"); + const d2 = once(s2, "deleted"); + await fs.unlink("a.txt"); + await d1; + await d2; + expect(Date.now() - start).toBeLessThan(deletedThreshold + 1000); + }); +}); + +describe("deleting a file then recreate it quickly does NOT trigger a 'deleted' event", () => { + const project_id = uuid(); + const path = "a.txt"; + let client1, s1, fs; + const deletedThreshold = 250; + + it("creates two clients editing 'a.txt'", async () => { + client1 = connect(); + fs = client1.fs({ project_id, service: server.service }); + await fs.writeFile(path, "my existing file"); + s1 = client1.sync.string({ + project_id, + path, + fs, + service: server.service, + deletedThreshold, + }); + + await once(s1, "ready"); + }); + + it(`delete 'a.txt' from disk and both clients emit 'deleted' event in about ${deletedThreshold}ms`, async () => { + let c1 = 0; + s1.once("deleted", () => { + c1++; + }); + await fs.unlink("a.txt"); + await delay(deletedThreshold - 100); + await fs.writeFile(path, "I'm back!"); + await delay(deletedThreshold); + expect(c1).toBe(0); + }); +}); diff --git a/src/packages/backend/conat/test/sync-doc/no-autosave.test.ts b/src/packages/backend/conat/test/sync-doc/no-autosave.test.ts new file mode 100644 index 00000000000..032526f381a --- /dev/null +++ b/src/packages/backend/conat/test/sync-doc/no-autosave.test.ts @@ -0,0 +1,88 @@ +import { + before, + after, + uuid, + connect, + server, + once, + delay, + waitUntilSynced, +} from "./setup"; + +beforeAll(before); +afterAll(after); + +describe("confirm noAutosave works", () => { + const project_id = uuid(); + const path = "a.txt"; + let client1, client2, s1, s2; + + it("creates two clients with noAutosave enabled", async () => { + client1 = connect(); + client2 = connect(); + await client1 + .fs({ project_id, service: server.service }) + .writeFile(path, ""); + s1 = client1.sync.string({ + project_id, + path, + service: server.service, + noAutosave: true, + }); + + await once(s1, "ready"); + + s2 = client2.sync.string({ + project_id, + path, + service: server.service, + noAutosave: true, + }); + await once(s2, "ready"); + expect(s1.noAutosave).toEqual(true); + expect(s2.noAutosave).toEqual(true); + }); + + const howLong = 750; + it(`write a change to s1 and commit it, but observe s2 does not see it even after ${howLong}ms (which should be plenty of time)`, async () => { + s1.from_str("new-ver"); + s1.commit(); + + expect(s2.to_str()).toEqual(""); + await delay(howLong); + expect(s2.to_str()).toEqual(""); + }); + + it("explicitly save and see s2 does get the change", async () => { + await s1.save(); + await waitUntilSynced([s1, s2]); + expect(s2.to_str()).toEqual("new-ver"); + }); + + it("make a change resulting in two heads", async () => { + s2.from_str("new-ver-1"); + s2.commit(); + // no background saving happening: + await delay(100); + s1.from_str("new-ver-2"); + s1.commit(); + await Promise.all([s1.save(), s2.save()]); + }); + + it("there are two heads and value is merged", async () => { + await waitUntilSynced([s1, s2]); + expect(s1.to_str()).toEqual("new-ver-1-2"); + expect(s2.to_str()).toEqual("new-ver-1-2"); + expect(s1.patch_list.getHeads().length).toBe(2); + expect(s2.patch_list.getHeads().length).toBe(2); + }); + + it("string state info matches", async () => { + const a1 = s1.syncstring_table_get_one().toJS(); + const a2 = s2.syncstring_table_get_one().toJS(); + expect(a1).toEqual(a2); + expect(new Set(a1.users)).toEqual( + new Set([s1.client.client_id(), s2.client.client_id()]), + ); + }); +}); diff --git a/src/packages/backend/conat/test/sync-doc/no-disk.test.ts b/src/packages/backend/conat/test/sync-doc/no-disk.test.ts new file mode 100644 index 00000000000..67bdb5d3801 --- /dev/null +++ b/src/packages/backend/conat/test/sync-doc/no-disk.test.ts @@ -0,0 +1,42 @@ + + +import { before, after, uuid, connect, server, once, delay } from "./setup"; + +beforeAll(before); +afterAll(after); + +describe("loading/saving syncstring to disk and setting values", () => { + let s; + const project_id = uuid(); + let client; + + it("creates the client", () => { + client = connect(); + }); + + it("create syncstring -- we still have to give a filename to define the 'location'", async () => { + s = client.sync.string({ + project_id, + path: "new.txt", + service: server.service, + noSaveToDisk: true, + }); + await once(s, "ready"); + expect(s.to_str()).toBe(""); + expect(s.versions().length).toBe(0); + s.from_str("foo"); + await s.save_to_disk(); + }); + + let fs; + it("get fs access and observe new.txt is NOT created on disk", async () => { + fs = client.fs({ project_id, service: server.service }); + expect(await fs.exists("new.txt")).toBe(false); + }); + + it("writing to new.txt has no impact on the non-fs syncstring", async () => { + await fs.writeFile("new.txt", "hello"); + await delay(200); + expect(s.to_str()).toEqual("foo"); + }); +}); diff --git a/src/packages/backend/conat/test/sync-doc/setup.ts b/src/packages/backend/conat/test/sync-doc/setup.ts new file mode 100644 index 00000000000..b6ce38042d2 --- /dev/null +++ b/src/packages/backend/conat/test/sync-doc/setup.ts @@ -0,0 +1,43 @@ +import { + before as before0, + after as after0, + client as client0, + wait, +} from "@cocalc/backend/conat/test/setup"; +export { connect, wait, once, delay } from "@cocalc/backend/conat/test/setup"; +import { + createPathFileserver, + cleanupFileservers, +} from "@cocalc/backend/conat/files/test/util"; +export { uuid } from "@cocalc/util/misc"; + +export { client0 as client }; + +export let server, fs; + +export async function before() { + await before0(); + server = await createPathFileserver(); +} + +export async function after() { + await cleanupFileservers(); + await after0(); +} + +// wait until the state of several syncdocs all have same heads- they may have multiple +// heads, but they all have the same heads +export async function waitUntilSynced(syncdocs: any[]) { + await wait({ + until: () => { + const X = new Set(); + for (const s of syncdocs) { + X.add(JSON.stringify(s.patch_list.getHeads()?.sort())); + if (X.size > 1) { + return false; + } + } + return true; + }, + }); +} diff --git a/src/packages/backend/conat/test/sync-doc/syncdb.test.ts b/src/packages/backend/conat/test/sync-doc/syncdb.test.ts new file mode 100644 index 00000000000..24e2706f52f --- /dev/null +++ b/src/packages/backend/conat/test/sync-doc/syncdb.test.ts @@ -0,0 +1,99 @@ +import { + before, + after, + uuid, + wait, + connect, + server, + once, +} from "./setup"; + +beforeAll(before); +afterAll(after); + +describe("loading/saving syncstring to disk and setting values", () => { + let s; + const project_id = uuid(); + let client; + + it("creates a client", () => { + client = connect(); + }); + + it("a syncdb associated to a file that does not exist on disk is initialized to empty", async () => { + s = client.sync.db({ + project_id, + path: "new.syncdb", + service: server.service, + primary_keys: ["name"], + }); + await once(s, "ready"); + expect(s.to_str()).toBe(""); + expect(s.versions().length).toBe(0); + }); + + it("store a record", async () => { + s.set({ name: "cocalc", value: 10 }); + expect(s.to_str()).toBe('{"name":"cocalc","value":10}'); + const t = s.get_one({ name: "cocalc" }).toJS(); + expect(t).toEqual({ name: "cocalc", value: 10 }); + await s.commit(); + await s.save(); + // [ ] TODO: this save to disk definitely should NOT be needed + await s.save_to_disk(); + }); + + let client2, s2; + it("connect another client", async () => { + client2 = connect(); + // [ ] loading this resets the state if we do not save above. + s2 = client2.sync.db({ + project_id, + path: "new.syncdb", + service: server.service, + primary_keys: ["name"], + }); + await once(s2, "ready"); + expect(s2).not.toBe(s); + expect(s2.to_str()).toBe('{"name":"cocalc","value":10}'); + const t = s2.get_one({ name: "cocalc" }).toJS(); + expect(t).toEqual({ name: "cocalc", value: 10 }); + + s2.set({ name: "conat", date: new Date() }); + s2.commit(); + await s2.save(); + }); + + it("verifies the change on s2 is seen by s (and also that Date objects do NOT work)", async () => { + await wait({ until: () => s.get_one({ name: "conat" }) != null }); + const t = s.get_one({ name: "conat" }).toJS(); + expect(t).toEqual({ name: "conat", date: t.date }); + // They don't work because we're storing syncdb's in jsonl format, + // so json is used. We should have a new format called + // msgpackl and start using that. + expect(t.date instanceof Date).toBe(false); + }); + + const count = 1000; + it(`store ${count} records`, async () => { + const before = s.get().size; + for (let i = 0; i < count; i++) { + s.set({ name: i }); + } + s.commit(); + await s.save(); + expect(s.get().size).toBe(count + before); + }); + + it("confirm file saves to disk with many lines", async () => { + await s.save_to_disk(); + const v = (await s.fs.readFile("new.syncdb", "utf8")).split("\n"); + expect(v.length).toBe(s.get().size); + }); + + it("verifies lookups are not too slow (there is an index)", () => { + for (let i = 0; i < count; i++) { + expect(s.get_one({ name: i }).get("name")).toEqual(i); + } + }); +}); diff --git a/src/packages/backend/conat/test/sync-doc/syncstring-bench.test.ts b/src/packages/backend/conat/test/sync-doc/syncstring-bench.test.ts new file mode 100644 index 00000000000..2e5a64ef2a5 --- /dev/null +++ b/src/packages/backend/conat/test/sync-doc/syncstring-bench.test.ts @@ -0,0 +1,32 @@ +import { before, after, uuid, client, server, once } from "./setup"; + +beforeAll(before); +afterAll(after); + +const log = process.env.BENCH ? console.log : (..._args) => {}; + +describe("loading/saving syncstring to disk and setting values", () => { + let s; + const project_id = uuid(); + let fs; + it("time opening a syncstring for editing a file that already exists on disk", async () => { + fs = client.fs({ project_id, service: server.service }); + await fs.writeFile("a.txt", "hello"); + + const t0 = Date.now(); + await fs.readFile("a.txt", "utf8"); + log("lower bound: time to read file", Date.now() - t0, "ms"); + + const start = Date.now(); + s = client.sync.string({ + project_id, + path: "a.txt", + service: server.service, + }); + await once(s, "ready"); + const total = Date.now() - start; + log("actual time to open sync document", total); + + expect(s.to_str()).toBe("hello"); + }); +}); diff --git a/src/packages/backend/conat/test/sync-doc/syncstring.test.ts b/src/packages/backend/conat/test/sync-doc/syncstring.test.ts new file mode 100644 index 00000000000..35fea0662c1 --- /dev/null +++ b/src/packages/backend/conat/test/sync-doc/syncstring.test.ts @@ -0,0 +1,175 @@ +import { + before, + after, + uuid, + wait, + connect, + server, + once, + delay, +} from "./setup"; + +beforeAll(before); +afterAll(after); + +describe("create syncstring without setting fs, so saving to disk and loading from disk are a no-op", () => { + let s; + const project_id = uuid(); + let client; + + it("creates the client", () => { + client = connect(); + }); + + it("a syncstring associated to a file that does not exist on disk is initialized to the empty string", async () => { + s = client.sync.string({ + project_id, + path: "new.txt", + service: server.service, + }); + await once(s, "ready"); + expect(s.to_str()).toBe(""); + expect(s.versions().length).toBe(0); + s.close(); + }); + + let fs; + it("a syncstring for editing a file that already exists on disk is initialized to that file", async () => { + fs = client.fs({ project_id, service: server.service }); + await fs.writeFile("a.txt", "hello"); + s = client.sync.string({ + project_id, + path: "a.txt", + service: server.service, + }); + await once(s, "ready"); + expect(s.fs).not.toEqual(undefined); + }); + + it("initially it is 'hello'", () => { + expect(s.to_str()).toBe("hello"); + expect(s.versions().length).toBe(1); + }); + + it("set the value", () => { + s.from_str("test"); + expect(s.to_str()).toBe("test"); + expect(s.versions().length).toBe(1); + }); + + it("save value to disk", async () => { + await s.save_to_disk(); + const disk = await fs.readFile("a.txt", "utf8"); + expect(disk).toEqual("test"); + }); + + it("commit the value", () => { + s.commit(); + expect(s.versions().length).toBe(2); + }); + + it("change the value and commit a second time", () => { + s.from_str("bar"); + s.commit(); + expect(s.versions().length).toBe(3); + }); + + it("get first version", () => { + expect(s.version(s.versions()[0]).to_str()).toBe("hello"); + expect(s.version(s.versions()[1]).to_str()).toBe("test"); + }); +}); + +describe("synchronized editing with two copies of a syncstring", () => { + const project_id = uuid(); + let s1, s2, client1, client2; + + it("creates the fs client and two copies of a syncstring", async () => { + client1 = connect(); + client2 = connect(); + s1 = client1.sync.string({ + project_id, + path: "a.txt", + service: server.service, + }); + await once(s1, "ready"); + + s2 = client2.sync.string({ + project_id, + path: "a.txt", + service: server.service, + }); + await once(s2, "ready"); + expect(s1.to_str()).toBe(""); + expect(s2.to_str()).toBe(""); + expect(s1 === s2).toBe(false); + }); + + it("change one, commit and save, and see change reflected in the other", async () => { + s1.from_str("hello world"); + s1.commit(); + await s1.save(); + await wait({ + until: () => { + return s2.to_str() == "hello world"; + }, + }); + }); + + it("change second and see change reflected in first", async () => { + s2.from_str("hello world!"); + s2.commit(); + await s2.save(); + await wait({ until: () => s1.to_str() == "hello world!" }); + }); + + it("view the history from each", async () => { + expect(s1.versions().length).toEqual(2); + expect(s2.versions().length).toEqual(2); + + const v1: string[] = [], + v2: string[] = []; + s1.show_history({ log: (x) => v1.push(x) }); + s2.show_history({ log: (x) => v2.push(x) }); + expect(v1).toEqual(v2); + }); +}); + +describe.only("opening a new syncstring for a file that does NOT exist on disk does not emit a 'deleted' event", () => { + let s; + const project_id = uuid(); + let deleted = 0; + it("a syncstring associated to a file that does not exist on disk is initialized to the empty string", async () => { + const client = connect(); + s = client.sync.string({ + project_id, + path: "this-file-is-not-on-disk.txt", + service: server.service, + // very aggressive: + deletedThreshold: 50, + deletedCheckInterval: 50, + watchRecreateWait: 50, + }); + s.on("deleted", () => { + deleted++; + }); + await once(s, "ready"); + expect(s.to_str()).toBe(""); + expect(s.versions().length).toBe(0); + }); + + it("wait a bit and deleted is NOT emited", async () => { + await delay(500); + expect(deleted).toEqual(0); + }); + + it("change, save to disk, then delete file and get the deleted event", async () => { + s.from_str("foo"); + s.commit(); + await s.save_to_disk(); + await delay(500); + await s.fs.unlink(s.path); + await delay(500); + expect(deleted).toBe(1); + }); +}); diff --git a/src/packages/backend/conat/test/sync-doc/watch-file.test.ts b/src/packages/backend/conat/test/sync-doc/watch-file.test.ts new file mode 100644 index 00000000000..16066151d78 --- /dev/null +++ b/src/packages/backend/conat/test/sync-doc/watch-file.test.ts @@ -0,0 +1,194 @@ +import { + before, + after, + uuid, + connect, + server, + once, + wait, + delay, + waitUntilSynced, +} from "./setup"; + +beforeAll(before); +afterAll(after); + +const readLockTimeout = 100; +const watchDebounce = 50; + +describe("basic watching of file on disk happens automatically", () => { + const project_id = uuid(); + const path = "a.txt"; + let client, s, fs; + + it("creates client", async () => { + client = connect(); + fs = client.fs({ project_id, service: server.service }); + await fs.writeFile(path, "init"); + s = client.sync.string({ + project_id, + path, + service: server.service, + readLockTimeout, + watchDebounce, + }); + await once(s, "ready"); + expect(s.to_str()).toEqual("init"); + }); + + it("changes the file on disk and call readFile to immediately update", async () => { + await delay(1.5 * readLockTimeout); + await fs.writeFile(path, "modified"); + await s.readFile(); + expect(s.to_str()).toEqual("modified"); + }); + + it("change file on disk and it automatically updates with no explicit call needed", async () => { + await delay(2 * watchDebounce); + await fs.writeFile(path, "changed again!"); + await wait({ + until: () => { + return s.to_str() == "changed again!"; + }, + }); + }); + + it("change file on disk should not trigger a load from disk", async () => { + await delay(2 * watchDebounce); + const orig = s.readFileDebounced; + let c = 0; + s.readFileDebounced = () => { + c += 1; + }; + s.from_str("a different value"); + await s.save_to_disk(); + expect(c).toBe(0); + await delay(100); + expect(c).toBe(0); + s.readFileDebounced = orig; + // disable the ignore that happens as part of save_to_disk, + // or the tests below won't work + await s.fileWatcher?.ignore(0); + }); + + let client2, s2; + it("file watching also works if there are multiple clients, with only one handling the change", async () => { + client2 = connect(); + s2 = client2.sync.string({ + project_id, + path, + service: server.service, + readLockTimeout, + watchDebounce, + }); + await once(s2, "ready"); + let c = 0, + c2 = 0; + s.on("handle-file-change", () => c++); + s2.on("handle-file-change", () => c2++); + + await fs.writeFile(path, "version3"); + expect(await fs.readFile(path, "utf8")).toEqual("version3"); + await wait({ + until: () => { + return s2.to_str() == "version3" && s.to_str() == "version3"; + }, + }); + expect(s.to_str()).toEqual("version3"); + expect(s2.to_str()).toEqual("version3"); + expect(c + c2).toBe(1); + }); + + it("file watching must still work if either client is closed", async () => { + s.close(); + await delay(2 * watchDebounce); + await fs.writeFile(path, "version4"); + await wait({ + until: () => { + return s2.to_str() == "version4"; + }, + }); + expect(s2.to_str()).toEqual("version4"); + }); + + it("another change and test", async () => { + await delay(watchDebounce * 2); + await fs.writeFile(path, "version5"); + await wait({ + until: () => { + return s2.to_str() == "version5"; + }, + }); + expect(s2.to_str()).toEqual("version5"); + }); + + it("add a third client, close client2 and have file watching still work", async () => { + const client3 = connect(); + const s3 = client3.sync.string({ + project_id, + path, + service: server.service, + readLockTimeout, + watchDebounce, + }); + await once(s3, "ready"); + s2.close(); + await delay(watchDebounce * 2); + await fs.writeFile(path, "version6"); + + await wait({ + until: () => { + return s3.to_str() == "version6"; + }, + }); + expect(s3.to_str()).toEqual("version6"); + }); +}); + +describe("has unsaved changes", () => { + const project_id = uuid(); + let s1, s2, client1, client2; + + it("creates two clients and opens a new file (does not exist on disk yet)", async () => { + client1 = connect(); + client2 = connect(); + s1 = client1.sync.string({ + project_id, + path: "a.txt", + service: server.service, + }); + await once(s1, "ready"); + // definitely has unsaved changes, since it doesn't even exist + expect(s1.has_unsaved_changes()).toBe(true); + + s2 = client2.sync.string({ + project_id, + path: "a.txt", + service: server.service, + }); + await once(s2, "ready"); + expect(s1.to_str()).toBe(""); + expect(s2.to_str()).toBe(""); + expect(s1 === s2).toBe(false); + expect(s2.has_unsaved_changes()).toBe(true); + }); + + it("save empty file to disk -- now no unsaved changes", async () => { + await s1.save_to_disk(); + expect(s1.has_unsaved_changes()).toBe(false); + // but s2 doesn't know anything + expect(s2.has_unsaved_changes()).toBe(true); + }); + + it("make a change via s2 and save", async () => { + s2.from_str("i am s2"); + await s2.save_to_disk(); + expect(s2.has_unsaved_changes()).toBe(false); + }); + + it("as soon as s1 learns that there was a change to the file on disk, it doesn't know", async () => { + await waitUntilSynced([s1, s2]); + expect(s1.has_unsaved_changes()).toBe(true); + expect(s1.to_str()).toEqual("i am s2"); + }); +}); diff --git a/src/packages/backend/conat/test/sync/open-files.test.ts b/src/packages/backend/conat/test/sync/open-files.test.ts deleted file mode 100644 index 371ba6a4769..00000000000 --- a/src/packages/backend/conat/test/sync/open-files.test.ts +++ /dev/null @@ -1,132 +0,0 @@ -/* -Unit test basic functionality of the openFiles distributed key:value -store. Projects and compute servers use this to know what files -to open so they can fulfill their backend responsibilities: - - computation - - save to disk - - load from disk when file changes - -DEVELOPMENT: - -pnpm test ./open-files.test.ts - -*/ - -import { openFiles as createOpenFiles } from "@cocalc/backend/conat/sync"; -import { once } from "@cocalc/util/async-utils"; -import { delay } from "awaiting"; -import { before, after, wait } from "@cocalc/backend/conat/test/setup"; - -beforeAll(before); - -const project_id = "00000000-0000-4000-8000-000000000000"; -async function create() { - return await createOpenFiles(project_id, { noAutosave: true, noCache: true }); -} - -describe("create open file tracker and do some basic operations", () => { - let o1, o2; - let file1 = `${Math.random()}.txt`; - let file2 = `${Math.random()}.txt`; - - it("creates two open files trackers (tracking same project) and clear them", async () => { - o1 = await create(); - o2 = await create(); - // ensure caching disabled so our sync tests are real - expect(o1.getKv() === o2.getKv()).toBe(false); - o1.clear(); - await o1.save(); - expect(o1.hasUnsavedChanges()).toBe(false); - o2.clear(); - while (o2.hasUnsavedChanges() || o1.hasUnsavedChanges()) { - try { - // expected due to merge conflict and autosave being disabled. - await o2.save(); - } catch { - await delay(10); - } - } - }); - - it("confirm they are cleared", async () => { - expect(o1.getAll()).toEqual([]); - expect(o2.getAll()).toEqual([]); - }); - - it("touch file in one and observe change and timestamp getting assigned by server", async () => { - o1.touch(file1); - expect(o1.get(file1).time).toBeCloseTo(Date.now(), -3); - }); - - it("touches file in one and observes change by OTHER", async () => { - o1.touch(file2); - expect(o1.get(file2)?.path).toBe(file2); - expect(o2.get(file2)).toBe(undefined); - await o1.save(); - if (o2.get(file2) == null) { - await once(o2, "change", 250); - expect(o2.get(file2).path).toBe(file2); - expect(o2.get(file2).time == null).toBe(false); - } - }); - - it("get all in o2 sees both file1 and file2", async () => { - const v = o2.getAll(); - expect(v[0].path).toBe(file1); - expect(v[1].path).toBe(file2); - expect(v.length).toBe(2); - }); - - it("delete file1 and verify fact that it is deleted is sync'd", async () => { - o1.delete(file1); - expect(o1.get(file1)).toBe(undefined); - expect(o1.getAll().length).toBe(1); - await o1.save(); - - // verify file is gone in o2, at least after waiting (if necessary) - await wait({ - until: () => { - return o2.getAll().length == 1; - }, - }); - expect(o2.get(file1)).toBe(undefined); - // should be 1 due to file2 still being there: - expect(o2.getAll().length).toBe(1); - - // Also confirm file1 is gone in a newly opened one: - const o3 = await create(); - expect(o3.get(file1)).toBe(undefined); - // should be 1 due to file2 still being there, but not file1. - expect(o3.getAll().length).toBe(1); - o3.close(); - }); - - it("sets an error", async () => { - o2.setError(file2, Error("test error")); - expect(o2.get(file2).error.error).toBe("Error: test error"); - expect(typeof o2.get(file2).error.time == "number").toBe(true); - expect(Math.abs(Date.now() - o2.get(file2).error.time)).toBeLessThan(10000); - try { - // get a conflict due to above so resolve it... - await o2.save(); - } catch { - await o2.save(); - } - if (!o1.get(file2).error) { - await once(o1, "change", 250); - } - expect(o1.get(file2).error.error).toBe("Error: test error"); - }); - - it("clears the error", async () => { - o1.setError(file2); - expect(o1.get(file2).error).toBe(undefined); - await o1.save(); - if (o2.get(file2).error) { - await once(o2, "change", 250); - } - expect(o2.get(file2).error).toBe(undefined); - }); -}); - -afterAll(after); diff --git a/src/packages/backend/conat/test/util.ts b/src/packages/backend/conat/test/util.ts index a6a9adb4375..d3fa39ac63e 100644 --- a/src/packages/backend/conat/test/util.ts +++ b/src/packages/backend/conat/test/util.ts @@ -4,12 +4,14 @@ export async function wait({ until: f, start = 5, decay = 1.2, + min = 5, max = 300, timeout = 10000, }: { until: Function; start?: number; decay?: number; + min?: number; max?: number; timeout?: number; }) { @@ -25,7 +27,7 @@ export async function wait({ start, decay, max, - min: 5, + min, timeout, }, ); diff --git a/src/packages/backend/data.ts b/src/packages/backend/data.ts index a95c19dd7ff..df78d274537 100644 --- a/src/packages/backend/data.ts +++ b/src/packages/backend/data.ts @@ -30,16 +30,26 @@ import { existsSync, mkdirSync, readFileSync } from "fs"; import { isEmpty } from "lodash"; import basePath from "@cocalc/backend/base-path"; import port from "@cocalc/backend/port"; +import { FALLBACK_ACCOUNT_UUID } from "@cocalc/util/misc"; +// using old version of pkg-dir because of nextjs :-( +import { sync as packageDirectorySync } from "pkg-dir"; -function determineRootFromPath(): string { - const cur = __dirname; - const search = "/src/"; - const i = cur.lastIndexOf(search); - const root = resolve(cur.slice(0, i + search.length - 1)); +function determineRoot(): string { + const pd = packageDirectorySync(__dirname) ?? "/"; + const root = resolve(pd, "..", ".."); process.env.COCALC_ROOT = root; return root; } +// Path to where our special binaries are, e.g., working with the +// filesystem and also the "open" command. This is used, e.g., +// by cocalc-lite to know what to add to the PATH. +export const bin = join( + packageDirectorySync(__dirname) ?? "/", + "node_modules", + ".bin", +); + // Each field value in this interface is to be treated as though it originated from a raw // environment variable. These environment variables are used to configure CoCalc's SSL connection // to the database. @@ -169,7 +179,7 @@ export function sslConfigToPsqlEnv(config: SSLConfig): PsqlSSLEnvConfig { return psqlArgs; } -export const root: string = process.env.COCALC_ROOT ?? determineRootFromPath(); +export const root: string = process.env.COCALC_ROOT ?? determineRoot(); export const data: string = process.env.DATA ?? join(root, "data"); export const pguser: string = process.env.PGUSER ?? "smc"; export const pgdata: string = process.env.PGDATA ?? join(data, "postgres"); @@ -181,6 +191,23 @@ export const projects: string = process.env.PROJECTS ?? join(data, "projects", "[project_id]"); export const secrets: string = process.env.SECRETS ?? join(data, "secrets"); +export const account_id: string = + process.env.COCALC_ACCOUNT_ID ?? FALLBACK_ACCOUNT_UUID; + +// File server and project runner config: +export const rusticRepo: string = + process.env.COCALC_RUSTIC_REPO ?? join(data, "rustic"); +// If given, COCALC_FILE_SERVER_MOUNTPOINT must be the top level mountpoint +// of a btrfs filesystem that is dedicated to the central file server. +// If not specified, a sparse image file is created in data/btrfs, which +// is designed for *development* purposes (not production deployments). +// Similar remarks for COCALC_PROJECT_RUNNER_MOUNTPOINT, which must also +// be a dedicated btrfs mountpoint, which will be used by a project runner. +export const fileServerMountpoint: string | undefined = + process.env.COCALC_FILE_SERVER_MOUNTPOINT; +export const projectRunnerMountpoint: string | undefined = + process.env.COCALC_PROJECT_RUNNER_MOUNTPOINT; + // Where the sqlite database files used for sync are stored. // The idea is there is one very fast *ephemeral* directory // which is used for actively open sqlite database. Optionally, @@ -253,6 +280,10 @@ export let conatPersistCount = parseInt(process.env.CONAT_PERSIST_COUNT ?? "1"); // number of api servers (if configured to run) export let conatApiCount = parseInt(process.env.CONAT_API_COUNT ?? "1"); +export let projectRunnerCount = parseInt( + process.env.COCALC_PROJECT_RUNNER_COUNT ?? "1", +); + // if configured, will create a socketio cluster using // the cluster adapter, listening on the given port. export let conatClusterPort = parseInt(process.env.CONAT_CLUSTER_PORT ?? "0"); @@ -264,6 +295,17 @@ export let conatClusterHealthPort = parseInt( export const conatClusterName = process.env.CONAT_CLUSTER_NAME; +// SSH server -- {host, port}: +export const sshServer: { host: string; port: number } = (() => { + const [host, port = "2222"] = ( + process.env.COCALC_SSH_SERVER ?? "host.containers.internal" + ).split(":"); + return { + host: host ? host : "host.containers.internal", + port: parseInt(port), + }; +})(); + // API keys export let apiKey: string = process.env.API_KEY ?? ""; diff --git a/src/packages/backend/execute-code.test.ts b/src/packages/backend/execute-code.test.ts index 8cca0e920da..d2e2946e398 100644 --- a/src/packages/backend/execute-code.test.ts +++ b/src/packages/backend/execute-code.test.ts @@ -40,7 +40,7 @@ describe("tests involving bash mode", () => { it("reports missing executable in non-bash mode", async () => { try { await executeCode({ - command: "this_does_not_exist", + command: "/usr/bin/this_does_not_exist", args: ["nothing"], bash: false, }); @@ -52,7 +52,7 @@ describe("tests involving bash mode", () => { it("reports missing executable in non-bash mode when ignoring on exit", async () => { try { await executeCode({ - command: "this_does_not_exist", + command: "/usr/bin/this_does_not_exist", args: ["nothing"], err_on_exit: false, bash: false, @@ -268,7 +268,7 @@ describe("async", () => { // This test screws up running multiple tests in parallel. // ** HENCE SKIPPING THIS - enable it if you edit the executeCode code...** - it.skip("longer running async job", async () => { + it.skip("(BAD TEST) longer running async job", async () => { setMonitorIntervalSeconds(1); const c = await executeCode({ command: "sh", @@ -376,7 +376,7 @@ describe("await", () => { it("deal with unknown executables", async () => { const c = await executeCode({ - command: "random123unknown99", + command: "/usr/bin/random123unknown99", err_on_exit: false, async_call: true, }); diff --git a/src/packages/backend/execute-code.ts b/src/packages/backend/execute-code.ts index e72dcfc536b..73345783b9e 100644 --- a/src/packages/backend/execute-code.ts +++ b/src/packages/backend/execute-code.ts @@ -17,7 +17,6 @@ import { tmpdir } from "node:os"; import { join } from "node:path"; import { EventEmitter } from "node:stream"; import shellEscape from "shell-escape"; - import getLogger from "@cocalc/backend/logger"; import { envToInt } from "@cocalc/backend/misc/env-to-number"; import { aggregate } from "@cocalc/util/aggregate"; @@ -645,11 +644,11 @@ function doSpawn( if (opts.verbose && log.isEnabled("debug")) { log.debug( - "finished exec of", + "exec", opts.command, "took", - walltime(start_time), - "seconds", + Math.ceil(1000 * walltime(start_time)), + "milliseconds", ); log.debug({ stdout: trunc(stdout, 512), diff --git a/src/packages/backend/get-listing.ts b/src/packages/backend/get-listing.ts index 02bcdbc8355..90c66c3c8db 100644 --- a/src/packages/backend/get-listing.ts +++ b/src/packages/backend/get-listing.ts @@ -3,10 +3,12 @@ * License: MS-RSL – see LICENSE.md for details */ +// DEPRECATABLE? + /* This is used by backends to serve directory listings to clients: -{files:[..., {size:?,name:?,mtime:?,isdir:?}]} +{files:[..., {size:?,name:?,mtime:?,isDir:?}]} where mtime is integer SECONDS since epoch, size is in bytes, and isdir is only there if true. @@ -46,7 +48,7 @@ const getListing = reuseInFlight( if (!hidden && file.name[0] === ".") { continue; } - let entry: DirectoryListingEntry; + let entry: Partial; try { // I don't actually know if file.name can fail to be JSON-able with node.js -- is there // even a string in Node.js that cannot be dumped to JSON? With python @@ -61,14 +63,14 @@ const getListing = reuseInFlight( try { let stats: Stats; if (file.isSymbolicLink()) { - // Optimization: don't explicitly set issymlink if it is false - entry.issymlink = true; + // Optimization: don't explicitly set isSymLink if it is false + entry.isSymLink = true; } - if (entry.issymlink) { + if (entry.isSymLink) { // at least right now we only use this symlink stuff to display // information to the user in a listing, and nothing else. try { - entry.link_target = await readlink(dir + "/" + entry.name); + entry.linkTarget = await readlink(dir + "/" + entry.name); } catch (err) { // If we don't know the link target for some reason; just ignore this. } @@ -81,7 +83,7 @@ const getListing = reuseInFlight( } entry.mtime = stats.mtime.valueOf() / 1000; if (stats.isDirectory()) { - entry.isdir = true; + entry.isDir = true; const v = await readdir(dir + "/" + entry.name); if (hidden) { entry.size = v.length; @@ -100,7 +102,7 @@ const getListing = reuseInFlight( } catch (err) { entry.error = `${entry.error ? entry.error : ""}${err}`; } - files.push(entry); + files.push(entry as DirectoryListingEntry); } return files; }, diff --git a/src/packages/backend/logger.ts b/src/packages/backend/logger.ts index 6d1dcea6a3b..4b89029caad 100644 --- a/src/packages/backend/logger.ts +++ b/src/packages/backend/logger.ts @@ -71,8 +71,6 @@ function myFormat(...args): string { function defaultTransports(): { console?: boolean; file?: string } { if (process.env.SMC_TEST) { return {}; - } else if (process.env.COCALC_DOCKER) { - return { file: "/var/log/hub/log" }; } else if (process.env.NODE_ENV == "production") { return { console: true }; } else { diff --git a/src/packages/backend/misc.ts b/src/packages/backend/misc.ts index c52b14a34fd..2c567ca1b81 100644 --- a/src/packages/backend/misc.ts +++ b/src/packages/backend/misc.ts @@ -74,9 +74,13 @@ import { callback } from "awaiting"; import { randomBytes } from "crypto"; export async function secureRandomString(length: number): Promise { - return (await callback(randomBytes, length)).toString("base64"); + return (await callback(randomBytes, length + 2)) + .toString("base64") + .slice(0, -2); } export function secureRandomStringSync(length: number): string { - return randomBytes(length).toString("base64"); + return randomBytes(length + 2) + .toString("base64") + .slice(0, -2); } diff --git a/src/packages/backend/misc/async-server-listen.ts b/src/packages/backend/misc/async-server-listen.ts new file mode 100644 index 00000000000..f3a5dfa85c9 --- /dev/null +++ b/src/packages/backend/misc/async-server-listen.ts @@ -0,0 +1,35 @@ +import getLogger from "@cocalc/backend/logger"; +const logger = getLogger("server-listen"); + +export default async function listen({ + server, + port, + host, + desc, +}: { + // a node http server with a listen method and error event + server; + port: number; + host: string; + // used for log message + desc?: string; +}) { + await new Promise((resolve, reject) => { + const onError = (err) => { + logger.debug("ERROR starting server", desc, err); + reject(err); // e.g. EADDRINUSE + }; + server.once("error", onError); + + server.listen(port, host, () => { + const address = server.address(); + logger.debug(`Server ${desc ?? ""} listening`, { + address, + port, + host, + }); + server.removeListener("error", onError); + resolve(); + }); + }); +} diff --git a/src/packages/backend/package.json b/src/packages/backend/package.json index 7de45e05859..1f85488afa8 100644 --- a/src/packages/backend/package.json +++ b/src/packages/backend/package.json @@ -6,9 +6,13 @@ "./*": "./dist/*.js", "./database": "./dist/database/index.js", "./conat": "./dist/conat/index.js", + "./files/*": "./dist/files/*.js", + "./sandbox": "./dist/sandbox/index.js", + "./conat/sync/*": "./dist/conat/sync/*.js", "./server-settings": "./dist/server-settings/index.js", "./auth/*": "./dist/auth/*.js", - "./auth/tokens/*": "./dist/auth/tokens/*.js" + "./auth/tokens/*": "./dist/auth/tokens/*.js", + "./podman": "./dist/podman/index.js" }, "keywords": [ "utilities", @@ -17,11 +21,13 @@ "scripts": { "preinstall": "npx only-allow pnpm", "clean": "rm -rf dist node_modules", - "build": "pnpm exec tsc --build", + "install-sandbox-tools": "echo 'require(\"@cocalc/backend/sandbox/install\").install()' | node", + "install-all-tools": "echo 'require(\"@cocalc/backend/sandbox/install\").install(null,{optional:true})' | node", + "install-bin-tools": "./bin/install.sh", + "build": "pnpm exec tsc --build && pnpm install-sandbox-tools && pnpm install-bin-tools", "tsc": "pnpm exec tsc --watch --pretty --preserveWatchOutput", "test": "pnpm exec jest --forceExit", "test-conat": " pnpm exec jest --forceExit conat", - "testp": "pnpm exec jest --forceExit", "depcheck": "pnpx depcheck --ignores events", "prepublishOnly": "pnpm test", "conat-watch": "node ./bin/conat-watch.cjs", @@ -43,16 +49,16 @@ "@cocalc/backend": "workspace:*", "@cocalc/conat": "workspace:*", "@cocalc/util": "workspace:*", - "@types/debug": "^4.1.12", - "@types/jest": "^30.0.0", "awaiting": "^3.0.0", "better-sqlite3": "^12.4.1", - "chokidar": "^3.6.0", + "chokidar": "^4.0.3", "debug": "^4.4.0", - "fs-extra": "^11.2.0", + "fs-extra": "^11.3.1", "lodash": "^4.17.21", "lru-cache": "^7.18.3", + "micro-key-producer": "^0.8.1", "password-hash": "^1.2.2", + "pkg-dir": "^5.0.0", "prom-client": "^15.1.3", "rimraf": "^5.0.5", "shell-escape": "^0.2.0", @@ -65,6 +71,8 @@ }, "homepage": "https://github.com/sagemathinc/cocalc/tree/master/src/packages/backend", "devDependencies": { + "@types/debug": "^4.1.12", + "@types/jest": "^30.0.0", "@types/node": "^18.16.14" } } diff --git a/src/packages/backend/path-watcher.ts b/src/packages/backend/path-watcher.ts index ef9932eb859..76c03ba7385 100644 --- a/src/packages/backend/path-watcher.ts +++ b/src/packages/backend/path-watcher.ts @@ -32,8 +32,7 @@ we are only ever watching a relatively small number of directories with a long interval, so polling is not so bad. */ -import { watch, WatchOptions } from "chokidar"; -import { FSWatcher } from "fs"; +import { watch } from "chokidar"; import { join } from "path"; import { EventEmitter } from "events"; import { debounce } from "lodash"; @@ -57,7 +56,7 @@ const DEFAULT_POLL_MS = parseInt( process.env.COCALC_FS_WATCHER_POLL_INTERVAL_MS ?? "2000", ); -const ChokidarOpts: WatchOptions = { +const ChokidarOpts = { persistent: true, // otherwise won't work followSymlinks: false, // don't wander about disableGlobbing: true, // watch the path as it is, that's it @@ -77,8 +76,8 @@ const ChokidarOpts: WatchOptions = { export class Watcher extends EventEmitter { private path: string; private exists: boolean; - private watchContents?: FSWatcher; - private watchExistence?: FSWatcher; + private watchContents?; + private watchExistence?; private debounce_ms: number; private debouncedChange: any; private log: Function; diff --git a/src/packages/backend/podman/build-container.ts b/src/packages/backend/podman/build-container.ts new file mode 100644 index 00000000000..a13677114c7 --- /dev/null +++ b/src/packages/backend/podman/build-container.ts @@ -0,0 +1,78 @@ +import { execFile as execFile0 } from "node:child_process"; +import { promisify } from "node:util"; +import { cp, mkdtemp, rm, writeFile } from "node:fs/promises"; +import { basename, join } from "node:path"; +import { tmpdir } from "node:os"; +import { reuseInFlight } from "@cocalc/util/reuse-in-flight"; +import getLogger from "@cocalc/backend/logger"; + +const logger = getLogger("file-server:ssh:build-container"); + +const execFile = promisify(execFile0); + +const images = new Set(); + +async function hasImage(name: string): Promise { + if (images.has(name)) { + return true; + } + const { stdout } = await execFile("podman", [ + "image", + "list", + name, + "--format", + "json", + ]); + if (JSON.parse(stdout).length > 0) { + images.add(name); + logger.debug(`image ${name} now exists`); + return true; + } + return false; +} + +// builds image if it does not exist +export const build = reuseInFlight( + async ({ + Dockerfile, + name, + files, + fileContents, + }: { + Dockerfile: string; + name: string; + files?: string[]; + fileContents?: { [path: string]: string }; + }) => { + if (await hasImage(name)) { + return; + } + logger.debug("Building image", { Dockerfile, name }); + let path: string | undefined = undefined; + try { + path = await mkdtemp(join(tmpdir(), "-cocalc")); + logger.debug("Created temp dir:", path); + if (files != null) { + await Promise.all(files.map((x) => cp(x, join(path!, basename(x))))); + } + if (fileContents != null) { + const v: any[] = []; + for (const x in fileContents) { + v.push(writeFile(join(path, x), fileContents[x])); + } + await Promise.all(v); + } + await writeFile(join(path, "Dockerfile"), Dockerfile, "utf8"); + const { stderr } = await execFile("podman", ["build", "-t", name, "."], { + cwd: path, + }); + if (!(await hasImage(name))) { + throw Error(`failed to build image -- ${stderr}`); + } + } finally { + if (path) { + rm(path, { force: true, recursive: true }); + } + } + }, +); diff --git a/src/packages/backend/podman/build-podman.sh b/src/packages/backend/podman/build-podman.sh new file mode 100755 index 00000000000..2a591cfe548 --- /dev/null +++ b/src/packages/backend/podman/build-podman.sh @@ -0,0 +1,109 @@ +#!/usr/bin/env bash +set -ev + +# This is a script that will install deps and build a specific version of podman +# from source on Ubuntu and set it up properly integrated with the OS, and also +# you can easily switch between this and the official version. +# It works. I'm putting this here since I don't know where else to put it, +# and it'll likely be useful for building a container image for deployment +# of project runners on Kubernetes. I don't intend to depend on any latest +# features of podman, but it's good to have the many latest bugfixes as an option! + +# === Settings ================================================================ +PODMAN_VERSION="${PODMAN_VERSION:-v5.6.2}" +PREFIX="/opt/podman-${PODMAN_VERSION#v}" # /opt/podman-5.6.2 +MAKEFLAGS="${MAKEFLAGS:- -j$(nproc)}" +BUILDTAGS="${BUILDTAGS:-seccomp apparmor}" + +# === Pre-flight ============================================================== +if [[ $EUID -ne 0 ]]; then + echo "Please run as root: sudo $0" >&2 + exit 1 +fi + +# Basic build deps + runtime helpers from Ubuntu +apt-get update +DEBIAN_FRONTEND=noninteractive apt-get install -y \ + git make golang-go gcc g++ pkg-config \ + libseccomp-dev libapparmor-dev \ + uidmap iptables \ + conmon slirp4netns fuse-overlayfs \ + runc crun \ + bash-completion \ + libbtrfs-dev \ + libgpgme-dev libassuan-dev libgpg-error-dev \ + libsystemd-dev + +# Sanity: Go version (Podman 5.x works with Go >=1.20; Ubuntu 25.04 has 1.22) +go version + +# === Build Podman from source =============================================== +TMP="$(mktemp -d)" +cleanup() { rm -rf "$TMP"; } +trap cleanup EXIT + +cd "$TMP" +git clone --branch "$PODMAN_VERSION" --depth 1 https://github.com/containers/podman.git +cd podman + +# Build +echo "Building Podman ${PODMAN_VERSION} with BUILDTAGS='${BUILDTAGS}'..." +make ${MAKEFLAGS} BUILDTAGS="${BUILDTAGS}" + +# Stage install tree under $PREFIX +echo "Installing into ${PREFIX} ..." +install -d "${PREFIX}/bin" "${PREFIX}/share/bash-completion/completions" "${PREFIX}/share/zsh/site-functions" "${PREFIX}/share/fish/vendor_completions.d" + +# Install the podman binary only (use Ubuntu's helpers for the rest) +install -m 0755 bin/podman "${PREFIX}/bin/podman" + +# Shell completions +# Bash +if [[ -f completions/bash/podman ]]; then + install -m 0644 completions/bash/podman "${PREFIX}/share/bash-completion/completions/podman" +fi +# Zsh +if [[ -f completions/zsh/_podman ]]; then + install -m 0644 completions/zsh/_podman "${PREFIX}/share/zsh/site-functions/_podman" +fi +# Fish +if [[ -f completions/fish/podman.fish ]]; then + install -m 0644 completions/fish/podman.fish "${PREFIX}/share/fish/vendor_completions.d/podman.fish" +fi + +# Convenience symlink in /usr/local/bin +install -d /usr/local/bin +ln -sf "${PREFIX}/bin/podman" /usr/local/bin/podman-${PODMAN_VERSION#v} + +# === Alternatives switcher =================================================== +# Register distro podman (if not already registered) +if ! update-alternatives --query podman >/dev/null 2>&1; then + + # 1) Make a distinct target for the distro binary + mv /usr/bin/podman /usr/local/bin/podman-apt + + # 2) Reset any half-created “podman” group (ignore errors) + update-alternatives --remove-all podman 2>/dev/null || true + + # 3) Register both alternatives (note: link=/usr/bin/podman; targets are distinct files) + update-alternatives --install /usr/bin/podman podman /usr/local/bin/podman-apt 10 + update-alternatives --install /usr/bin/podman podman /usr/local/bin/podman-5.6.2 20 +fi + +# update-alternatives --config podman + +# === Show status ============================================================= +echo +echo "Installed ${PODMAN_VERSION} to ${PREFIX}" +echo "Added /usr/local/bin/podman-${PODMAN_VERSION#v} symlink." +echo +echo "Current alternatives:" +update-alternatives --display podman || true +echo +echo "To switch versions interactively:" +echo " sudo update-alternatives --config podman" +echo +echo "Active podman version:" +/usr/bin/podman --version || true +echo +echo "Done." diff --git a/src/packages/backend/podman/command.ts b/src/packages/backend/podman/command.ts new file mode 100644 index 00000000000..81a169056af --- /dev/null +++ b/src/packages/backend/podman/command.ts @@ -0,0 +1,23 @@ +import { executeCode } from "@cocalc/backend/execute-code"; +import getLogger from "@cocalc/backend/logger"; + +const logger = getLogger("podman"); + +// 30 minute timeout (?) +export default async function podman(args: string[], timeout = 30 * 60 * 1000) { + logger.debug("podman ", args.join(" ")); + try { + const x = await executeCode({ + verbose: false, + command: "podman", + args, + err_on_exit: true, + timeout, + }); + logger.debug("podman returned ", x); + return x; + } catch (err) { + logger.debug("podman run error: ", err); + throw err; + } +} diff --git a/src/packages/backend/podman/index.ts b/src/packages/backend/podman/index.ts new file mode 100644 index 00000000000..f48263aabe6 --- /dev/null +++ b/src/packages/backend/podman/index.ts @@ -0,0 +1,5 @@ +import mountArg from "./mount-arg"; +export { mountArg }; + +import podman from "./command"; +export { podman }; diff --git a/src/packages/backend/podman/memory.ts b/src/packages/backend/podman/memory.ts new file mode 100644 index 00000000000..77171e195b8 --- /dev/null +++ b/src/packages/backend/podman/memory.ts @@ -0,0 +1,63 @@ +import { readFile } from "node:fs/promises"; + +/** Total physical RAM in bytes (not counting swap). */ +let cached: { ram: number; swap: number } | null = null; +export async function getTotalMemoryBytes(): Promise<{ + ram: number; + swap: number; +}> { + if (cached != null) { + return cached; + } + const text = await readFile("/proc/meminfo", "utf8"); + let m = text.match(/^MemTotal:\s+(\d+)\s+(\w+)/m); + if (!m) { + throw new Error("MemTotal not found in /proc/meminfo."); + } + const value = parseInt(m[1], 10); + const unit = m[2].toLowerCase(); // typically "kB" + + const mult: Record = { + b: 1, + kb: 1024, + mb: 1024 ** 2, + gb: 1024 ** 3, + }; + if (mult[unit] == null) { + throw Error(`unknown unit ${unit}`); + } + const ram = value * mult[unit]; + + let swap; + m = text.match(/^SwapTotal:\s+(\d+)\s+(\w+)/m); + if (!m) { + swap = 0; + } else { + const value = parseInt(m[1], 10); + const unit = m[2].toLowerCase(); // typically "kB" + if (mult[unit] == null) { + throw Error(`unknown unit ${unit}`); + } + swap = value * mult[unit]; + } + + cached = { ram, swap }; + return cached; +} + +/* +See https://youtu.be/lgrdpUF-9-w?si=THwEEaC5VX8mGoEd&t=1278 + +The speaker recommends that for containers a formula for swap +based on the memory request. We are only using a memory limit, +so we use that instead. He suggests to compute the percent +of RAM the pod is guaranteed, then multiply that by the swap configured +on the machine, and use that for swap for the container. +*/ + +export async function getContainerSwapSizeMb( + memoryLimitBytes: number, +): Promise { + const { ram, swap } = await getTotalMemoryBytes(); + return Math.round(swap * (memoryLimitBytes / ram)); +} diff --git a/src/packages/backend/podman/mount-arg.ts b/src/packages/backend/podman/mount-arg.ts new file mode 100644 index 00000000000..3aefcad8196 --- /dev/null +++ b/src/packages/backend/podman/mount-arg.ts @@ -0,0 +1,21 @@ + +function escapeMountPath(p) { + return p + .replace(/\\/g, "\\\\") // literal backslashes + .replace(/,/g, "\\,") // commas separate K/V pairs + .replace(/=/g, "\\="); // equals separate keys from values +} + +export default function mountArg({ + source, + target, + readOnly = false, + options = "", +}: { + source: string; + target: string; + readOnly?: boolean; + options?: string; +}) { + return `--mount=type=bind,source=${escapeMountPath(source)},target=${escapeMountPath(target)},${readOnly ? "ro" : "rw"}${options ? "," + options : ""}`; +} diff --git a/src/packages/backend/process-stats.ts b/src/packages/backend/process-stats.ts index 731e44cf541..8835b6a2747 100644 --- a/src/packages/backend/process-stats.ts +++ b/src/packages/backend/process-stats.ts @@ -7,7 +7,7 @@ import { exec as cp_exec } from "node:child_process"; import { readFile, readdir, readlink } from "node:fs/promises"; import { join } from "node:path"; import { promisify } from "node:util"; - +import { uptime } from "node:os"; import { reuseInFlight } from "@cocalc/util/reuse-in-flight"; import { Cpu, @@ -23,6 +23,8 @@ const dbg = getLogger("process-stats").debug; const exec = promisify(cp_exec); +export const MIN_WARN_INTERVAL = 30_000; + /** * Return information about all processes (up to a limit or filter) in the environment, where this node.js process runs. * This has been refactored out of project/project-info/server.ts. @@ -152,13 +154,14 @@ export class ProcessStats { // measured in "ticks" since the machine started private async uptime(): Promise<[number, Date]> { // return uptime in secs - const out = await readFile("/proc/uptime", "utf8"); - const uptime = parseFloat(out.split(" ")[0]); - const boottime = new Date(new Date().getTime() - 1000 * uptime); - return [uptime, boottime]; + // macOS, Windows, etc.: seconds (integer) + const u = uptime(); + const boottime = new Date(Date.now() - u * 1000); + return [u, boottime]; } // this is where we gather information about all running processes + private lastWarn: number = 0; public async processes( timestamp?: number, ): Promise<{ procs: Processes; uptime: number; boottime: Date }> { @@ -178,7 +181,11 @@ export class ProcessStats { } // we avoid processing and sending too much data if (n > this.procLimit) { - dbg(`too many processes – limit of ${this.procLimit} reached!`); + // only log this once in while, otherwise it totally spams the logs + if (this.lastWarn <= Date.now() - MIN_WARN_INTERVAL) { + this.lastWarn = Date.now(); + dbg(`too many processes – limit of ${this.procLimit} reached!`); + } break; } else { n += 1; diff --git a/src/packages/backend/sandbox/cp.ts b/src/packages/backend/sandbox/cp.ts new file mode 100644 index 00000000000..bc3301afb19 --- /dev/null +++ b/src/packages/backend/sandbox/cp.ts @@ -0,0 +1,77 @@ +/* +Implement cp with same API as node, but using a spawned subprocess, because +Node's cp does NOT have reflink support, but we very much want it to get +the full benefit of btrfs's copy-on-write functionality. +*/ + +import exec from "./exec"; +import { type CopyOptions } from "@cocalc/conat/files/fs"; +export { type CopyOptions }; +import { exists } from "./install"; + +export default async function cp( + src: string[] | string, + dest: string, + options: CopyOptions = {}, +): Promise { + const args: string[] = []; + if (typeof src == "string") { + args.push("-T", src); + // Why the -T? When the src is string instead of an array, + // we maintain the + // cp semantics of ndoejs, where always dest is exactly what gets + // created/written, **not a file in dest**. E.g., if a is a directory, + // then doing + // cp('a','b',{recursive:true}) + // twice is idempotent, whereas with /usr/bin/cp without the -T option, + // then second call creates b/a. + } else { + args.push(...src); + } + args.push(dest); + const opts: string[] = []; + if (!options.dereference) { + opts.push("-d"); + } + if (!(options.force ?? true)) { + // according to node docs, when force=true: + // "overwrite existing file or directory" + // The -n (or --no-clobber) docs to cp: "do not overwrite an existing file", + // so I think force=false is same as --update. + if (options.errorOnExist) { + // If moreover errorOnExist is set, then node's cp will also throw an error + // with code "ERR_FS_CP_EEXIST" + // SystemError [ERR_FS_CP_EEXIST]: Target already exists + // /usr/bin/cp doesn't really have such an option, so we use exist directly. + if (await exists(dest)) { + const e = Error( + "SystemError [ERR_FS_CP_EEXIST]: Target already exists", + ); + // @ts-ignore + e.code = "ERR_FS_CP_EEXIST"; + throw e; + } + } + // silently does nothing if target exists + opts.push("-n"); + } + + if (options.preserveTimestamps) { + opts.push("-p"); + } + if (options.recursive) { + opts.push("-r"); + } + if (options.reflink) { + opts.push("--reflink=auto"); + } + + const { code, stderr } = await exec({ + cmd: "/usr/bin/cp", + safety: [...opts, ...args], + timeout: options.timeout, + }); + if (code) { + throw Error(stderr.toString()); + } +} diff --git a/src/packages/backend/sandbox/dust.test.ts b/src/packages/backend/sandbox/dust.test.ts new file mode 100644 index 00000000000..cd69a56951b --- /dev/null +++ b/src/packages/backend/sandbox/dust.test.ts @@ -0,0 +1,39 @@ +import dust from "./dust"; +import { mkdtemp, rm, writeFile } from "node:fs/promises"; +import { tmpdir } from "node:os"; +import { join } from "node:path"; + +let tempDir; +beforeAll(async () => { + tempDir = await mkdtemp(join(tmpdir(), "cocalc")); +}); +afterAll(async () => { + await rm(tempDir, { force: true, recursive: true }); +}); + +describe("dust works", () => { + it("directory starts empty - no results", async () => { + const { stdout, truncated } = await dust(tempDir, { options: ["-j"] }); + const s = JSON.parse(Buffer.from(stdout).toString()); + expect(s).toEqual({ children: [], name: tempDir, size: s.size }); + expect(truncated).toBe(false); + }); + + it("create a file and see it appears in the dust result", async () => { + await writeFile(join(tempDir, "a.txt"), "hello"); + const { stdout, truncated } = await dust(tempDir, { options: ["-j"] }); + const s = JSON.parse(Buffer.from(stdout).toString()); + expect(s).toEqual({ + size: s.size, + name: tempDir, + children: [ + { + size: s.children[0].size, + name: join(tempDir, "a.txt"), + children: [], + }, + ], + }); + expect(truncated).toBe(false); + }); +}); diff --git a/src/packages/backend/sandbox/dust.ts b/src/packages/backend/sandbox/dust.ts new file mode 100644 index 00000000000..1e5043ccc15 --- /dev/null +++ b/src/packages/backend/sandbox/dust.ts @@ -0,0 +1,121 @@ +import exec, { type ExecOutput, validate } from "./exec"; +import { type DustOptions } from "@cocalc/conat/files/fs"; +export { type DustOptions }; +import { dust as dustPath } from "./install"; + +export default async function dust( + path: string, + { options, darwin, linux, timeout, maxSize }: DustOptions = {}, +): Promise { + if (path == null) { + throw Error("path must be specified"); + } + + return await exec({ + cmd: dustPath, + cwd: path, + positionalArgs: [path], + options, + darwin, + linux, + maxSize, + timeout, + whitelist, + }); +} + +const whitelist = { + "-d": validate.int, + "--depth": validate.int, + + "-n": validate.int, + "--number-of-lines": validate.int, + + "-p": true, + "--full-paths": true, + + "-X": validate.str, + "--ignore-directory": validate.str, + + "-x": true, + "--limit-filesystem": true, + + "-s": true, + "--apparent-size": true, + + "-r": true, + "--reverse": true, + + "-c": true, + "--no-colors": true, + "-C": true, + "--force-colors": true, + + "-b": true, + "--no-percent-bars": true, + + "-B": true, + "--bars-on-right": true, + + "-z": validate.str, + "--min-size": validate.str, + + "-R": true, + "--screen-reader": true, + + "--skip-total": true, + + "-f": true, + "--filecount": true, + + "-i": true, + "--ignore-hidden": true, + + "-v": validate.str, + "--invert-filter": validate.str, + + "-e": validate.str, + "--filter": validate.str, + + "-t": validate.str, + "--file-types": validate.str, + + "-w": validate.int, + "--terminal-width": validate.int, + + "-P": true, + "--no-progress": true, + + "--print-errors": true, + + "-D": true, + "--only-dir": true, + + "-F": true, + "--only-file": true, + + "-o": validate.str, + "--output-format": validate.str, + + "-j": true, + "--output-json": true, + + "-M": validate.str, + "--mtime": validate.str, + + "-A": validate.str, + "--atime": validate.str, + + "-y": validate.str, + "--ctime": validate.str, + + "--collapse": validate.str, + + "-m": validate.set(["a", "c", "m"]), + "--filetime": validate.set(["a", "c", "m"]), + + "-h": true, + "--help": true, + "-V": true, + "--version": true, +} as const; diff --git a/src/packages/backend/sandbox/exec.test.ts b/src/packages/backend/sandbox/exec.test.ts new file mode 100644 index 00000000000..a3735a4b603 --- /dev/null +++ b/src/packages/backend/sandbox/exec.test.ts @@ -0,0 +1,30 @@ +/* +Test the exec command. +*/ + +import exec from "./exec"; +import { mkdtemp, rm, writeFile } from "node:fs/promises"; +import { tmpdir } from "node:os"; +import { join } from "node:path"; + +let tempDir; +beforeAll(async () => { + tempDir = await mkdtemp(join(tmpdir(), "cocalc")); +}); +afterAll(async () => { + await rm(tempDir, { force: true, recursive: true }); +}); + +describe("exec works", () => { + it(`create file and run ls command`, async () => { + await writeFile(join(tempDir, "a.txt"), "hello"); + const { stderr, stdout, truncated, code } = await exec({ + cmd: "ls", + cwd: tempDir, + }); + expect(code).toBe(0); + expect(truncated).toBe(false); + expect(stdout.toString()).toEqual("a.txt\n"); + expect(stderr.toString()).toEqual(""); + }); +}); diff --git a/src/packages/backend/sandbox/exec.ts b/src/packages/backend/sandbox/exec.ts new file mode 100644 index 00000000000..1b35d1db3e2 --- /dev/null +++ b/src/packages/backend/sandbox/exec.ts @@ -0,0 +1,245 @@ +import { execFile, spawn } from "node:child_process"; +import { arch } from "node:os"; +import { type ExecOutput } from "@cocalc/conat/files/fs"; +export { type ExecOutput }; +import getLogger from "@cocalc/backend/logger"; + +const logger = getLogger("files:sandbox:exec"); + +const DEFAULT_TIMEOUT = 3_000; +const DEFAULT_MAX_SIZE = 10_000_000; + +export interface Options { + // the path to the command + cmd: string; + // position args *before* any options; these are not sanitized + prefixArgs?: string[]; + // positional arguments; these are not sanitized, but are given after '--' for safety + positionalArgs?: string[]; + // whitelisted args flags; these are checked according to the whitelist specified below + options?: string[]; + // if given, use these options when os.arch()=='darwin' (i.e., macOS); these must match whitelist + darwin?: string[]; + // if given, use these options when os.arch()=='linux'; these must match whitelist + linux?: string[]; + // when total size of stdout and stderr hits this amount, command is terminated, and + // truncated is set. The total amount of output may thus be slightly larger than maxOutput + maxSize?: number; + // command is terminated after this many ms + timeout?: number; + // each command line option that is explicitly whitelisted + // should be a key in the following whitelist map. + // The value can be either: + // - true: in which case the option does not take a argument, or + // - a function: in which the option takes exactly one argument; the function should validate that argument + // and throw an error if the argument is not allowed. + whitelist?: { [option: string]: true | ValidateFunction }; + // where to launch command + cwd?: string; + + // options that are always included first for safety and need NOT match whitelist + safety?: string[]; + + // if nodejs is running as root and give this username, then cmd runs as this + // user instead. + username?: string; + + // by default the environment is EMPTY, which is usually what we want for fairly + // locked down execution. Use this to add something nontrivial to the default empty. + env?: { [name: string]: string }; +} + +type ValidateFunction = (value: string) => void; + +export default async function exec({ + cmd, + positionalArgs = [], + prefixArgs = [], + options = [], + linux = [], + darwin = [], + safety = [], + maxSize = DEFAULT_MAX_SIZE, + timeout = DEFAULT_TIMEOUT, + whitelist = {}, + cwd, + username, + env = {}, +}: Options): Promise { + if (arch() == "darwin") { + options = options.concat(darwin); + } else if (arch() == "linux") { + options = options.concat(linux); + } + options = safety.concat(parseAndValidateOptions(options, whitelist)); + const userId = username ? await getUserIds(username) : undefined; + + return new Promise((resolve, reject) => { + const stdoutChunks: Buffer[] = []; + const stderrChunks: Buffer[] = []; + let truncated = false; + let stdoutSize = 0; + let stderrSize = 0; + + let args = prefixArgs.concat(options); + if (positionalArgs.length > 0) { + args.push("--", ...positionalArgs); + } + + logger.debug(`cd ${cwd}; ${cmd} ${args.join(" ")}`); + + //console.log(`${cmd} ${args.join(" ")}`, { cwd, env }); + const child = spawn(cmd, args, { + stdio: ["ignore", "pipe", "pipe"], + // env as any because otherwise pnpm build with nextjs says " Property 'NODE_ENV' is + // missing in type '{ [name: string]: string; }' but required in type 'ProcessEnv'" + env: env as any, + cwd, + ...userId, + }); + + let timeoutHandle: NodeJS.Timeout | null = null; + + if (timeout > 0) { + timeoutHandle = setTimeout(() => { + truncated = true; + child.kill("SIGTERM"); + // Force kill after grace period + setTimeout(() => { + if (!child.killed) { + child.kill("SIGKILL"); + } + }, 1000); + }, timeout); + } + + child.stdout.on("data", (chunk: Buffer) => { + stdoutSize += chunk.length; + if (stdoutSize + stderrSize >= maxSize) { + truncated = true; + child.kill("SIGTERM"); + return; + } + stdoutChunks.push(chunk); + }); + + child.stderr.on("data", (chunk: Buffer) => { + stderrSize += chunk.length; + if (stdoutSize + stderrSize > maxSize) { + truncated = true; + child.kill("SIGTERM"); + return; + } + stderrChunks.push(chunk); + }); + + child.on("error", (err) => { + if (timeoutHandle) { + clearTimeout(timeoutHandle); + } + reject(err); + }); + + child.once("close", (code) => { + if (timeoutHandle) { + clearTimeout(timeoutHandle); + } + + resolve({ + stdout: Buffer.concat(stdoutChunks), + stderr: Buffer.concat(stderrChunks), + code, + truncated, + }); + }); + }); +} + +export function parseAndValidateOptions( + options: string[], + whitelist, +): string[] { + const validatedOptions: string[] = []; + let i = 0; + + while (i < options.length) { + const opt = options[i]; + + // Check if this is a safe option + const validate = whitelist[opt]; + if (!validate) { + throw new Error(`Disallowed option: ${opt}`); + } + validatedOptions.push(opt); + + // Handle options that take values + if (validate !== true) { + i++; + if (i >= options.length) { + throw new Error(`Option ${opt} requires a value`); + } + const value = String(options[i]); + validate(value); + // didn't throw, so good to go + validatedOptions.push(value); + } + i++; + } + return validatedOptions; +} + +export const validate = { + str: () => {}, + set: (allowed) => { + allowed = new Set(allowed); + return (value: string) => { + if (!allowed.includes(value)) { + throw Error("invalid value"); + } + }; + }, + int: (value: string) => { + const x = parseInt(value); + if (!isFinite(x)) { + throw Error("argument must be a number"); + } + }, + float: (value: string) => { + const x = parseFloat(value); + if (!isFinite(x)) { + throw Error("argument must be a number"); + } + }, +}; + +async function getUserIds( + username: string, +): Promise<{ uid: number; gid: number }> { + return Promise.all([ + new Promise((resolve, reject) => { + execFile("id", ["-u", username], (err, stdout) => { + if (err) return reject(err); + resolve(parseInt(stdout.trim(), 10)); + }); + }), + new Promise((resolve, reject) => { + execFile("id", ["-g", username], (err, stdout) => { + if (err) return reject(err); + resolve(parseInt(stdout.trim(), 10)); + }); + }), + ]).then(([uid, gid]) => ({ uid, gid })); +} + +// take the output of exec and convert stdout, stderr to strings. If code is nonzero, +// instead throw an error with message stderr. +export function parseOutput({ stdout, stderr, code, truncated }: ExecOutput) { + if (code) { + throw new Error(Buffer.from(stderr).toString()); + } + return { + stdout: Buffer.from(stdout).toString(), + stderr: Buffer.from(stderr).toString(), + truncated, + }; +} diff --git a/src/packages/backend/sandbox/fd.test.ts b/src/packages/backend/sandbox/fd.test.ts new file mode 100644 index 00000000000..f17132d9721 --- /dev/null +++ b/src/packages/backend/sandbox/fd.test.ts @@ -0,0 +1,27 @@ +import fd from "./fd"; +import { mkdtemp, rm, writeFile } from "node:fs/promises"; +import { tmpdir } from "node:os"; +import { join } from "node:path"; + +let tempDir; +beforeAll(async () => { + tempDir = await mkdtemp(join(tmpdir(), "cocalc")); +}); +afterAll(async () => { + await rm(tempDir, { force: true, recursive: true }); +}); + +describe("fd files", () => { + it("directory starts empty", async () => { + const { stdout, truncated } = await fd(tempDir); + expect(stdout.length).toBe(0); + expect(truncated).toBe(false); + }); + + it("create a file and see it appears via fd", async () => { + await writeFile(join(tempDir, "a.txt"), "hello"); + const { stdout, truncated } = await fd(tempDir); + expect(truncated).toBe(false); + expect(stdout.toString()).toEqual("a.txt\n"); + }); +}); diff --git a/src/packages/backend/sandbox/fd.ts b/src/packages/backend/sandbox/fd.ts new file mode 100644 index 00000000000..dfc92e8e1ea --- /dev/null +++ b/src/packages/backend/sandbox/fd.ts @@ -0,0 +1,119 @@ +import exec, { type ExecOutput, validate } from "./exec"; +import { type FdOptions } from "@cocalc/conat/files/fs"; +export { type FdOptions }; +import { fd as fdPath } from "./install"; + +export default async function fd( + path: string, + { options, darwin, linux, pattern, timeout, maxSize }: FdOptions = {}, +): Promise { + if (path == null) { + throw Error("path must be specified"); + } + + return await exec({ + cmd: fdPath, + cwd: path, + positionalArgs: pattern ? [pattern] : [], + options, + darwin, + linux, + safety: ["--no-follow"], + maxSize, + timeout, + whitelist, + }); +} + +const whitelist = { + "-H": true, + "--hidden": true, + + "-I": true, + "--no-ignore": true, + + "-u": true, + "--unrestricted": true, + + "--no-ignore-vcs": true, + "--no-require-git": true, + + "-s": true, + "--case-sensitive": true, + + "-i": true, + "--ignore-case": true, + + "-g": true, + "--glob": true, + + "--regex": true, + + "-F": true, + "--fixed-strings": true, + + "--and": validate.str, + + "-l": true, + "--list-details": true, + + "-p": true, + "--full-path": true, + + "-0": true, + "--print0": true, + + "--max-results": validate.int, + + "-1": true, + + "-q": true, + "--quite": true, + + "--show-errors": true, + + "--strip-cwd-prefix": validate.set(["never", "always", "auto"]), + + "--one-file-system": true, + "--mount": true, + "--xdev": true, + + "-h": true, + "--help": true, + + "-V": true, + "--version": true, + + "-d": validate.int, + "--max-depth": validate.int, + + "--min-depth": validate.int, + + "--exact-depth": validate.int, + + "--prune": true, + + "--type": validate.str, + + "-e": validate.str, + "--extension": validate.str, + + "-E": validate.str, + "--exclude": validate.str, + + "--ignore-file": validate.str, + + "-c": validate.set(["never", "always", "auto"]), + "--color": validate.set(["never", "always", "auto"]), + + "-S": validate.str, + "--size": validate.str, + + "--changed-within": validate.str, + "--changed-before": validate.str, + + "-o": validate.str, + "--owner": validate.str, + + "--format": validate.str, +} as const; diff --git a/src/packages/backend/sandbox/find.test.ts b/src/packages/backend/sandbox/find.test.ts new file mode 100644 index 00000000000..1ab6e0557c1 --- /dev/null +++ b/src/packages/backend/sandbox/find.test.ts @@ -0,0 +1,83 @@ +import find from "./find"; +import { mkdir, mkdtemp, rm, writeFile } from "node:fs/promises"; +import { tmpdir } from "node:os"; +import { join } from "node:path"; + + +let tempDir; +beforeAll(async () => { + tempDir = await mkdtemp(join(tmpdir(), "cocalc")); +}); +afterAll(async () => { + await rm(tempDir, { force: true, recursive: true }); +}); + + +jest.setTimeout(15000); +describe("find files", () => { + it("directory starts empty", async () => { + const { stdout, truncated } = await find(tempDir, { + options: ["-maxdepth", "1", "-mindepth", "1", "-printf", "%f\n"], + }); + expect(stdout.length).toBe(0); + expect(truncated).toBe(false); + }); + + it("create a file and see it appears in find", async () => { + await writeFile(join(tempDir, "a.txt"), "hello"); + const { stdout, truncated } = await find(tempDir, { + options: ["-maxdepth", "1", "-mindepth", "1", "-printf", "%f\n"], + }); + expect(truncated).toBe(false); + expect(stdout.toString()).toEqual("a.txt\n"); + }); + + it("find files matching a given pattern", async () => { + await writeFile(join(tempDir, "pattern"), ""); + await mkdir(join(tempDir, "blue")); + await writeFile(join(tempDir, "blue", "Patton"), ""); + const { stdout } = await find(tempDir, { + options: [ + "-maxdepth", + "1", + "-mindepth", + "1", + "-iname", + "patt*", + "-printf", + "%f\n", + ], + }); + const v = stdout.toString().trim().split("\n"); + expect(new Set(v)).toEqual(new Set(["pattern"])); + }); + + it("find file in a subdirectory too", async () => { + const { stdout } = await find(tempDir, { + options: ["-iname", "patt*", "-printf", "%P\n"], + }); + const w = stdout.toString().trim().split("\n"); + expect(new Set(w)).toEqual(new Set(["pattern", "blue/Patton"])); + }); + +// // this is NOT a great test, unfortunately. +// const count = 5000; +// it(`hopefully exceed the timeout by creating ${count} files`, async () => { +// for (let i = 0; i < count; i++) { +// await writeFile(join(tempDir, `${i}`), ""); +// } +// const t = Date.now(); +// const { stdout, truncated } = await find(tempDir, { +// options: ["-printf", "%f\n"], +// timeout: 0.002, +// }); + +// expect(truncated).toBe(true); +// expect(Date.now() - t).toBeGreaterThan(1); + +// const { stdout: stdout2 } = await find(tempDir, { +// options: ["-maxdepth", "1", "-mindepth", "1", "-printf", "%f\n"], +// }); +// expect(stdout2.length).toBeGreaterThan(stdout.length); +// }); +}); diff --git a/src/packages/backend/sandbox/find.ts b/src/packages/backend/sandbox/find.ts new file mode 100644 index 00000000000..8c1929ce705 --- /dev/null +++ b/src/packages/backend/sandbox/find.ts @@ -0,0 +1,117 @@ +/* + +NOTE: fd is a very fast parallel rust program for finding files matching +a pattern. It is complementary to find here though, because we mainly +use find to compute directory +listing info (e.g., file size, mtime, etc.), and fd does NOT do that; it can +exec ls, but that is slower than using find. So both find and fd are useful +for different tasks -- find is *better* for directory listings and fd is better +for finding filesnames in a directory tree that match a pattern. +*/ + +import type { FindOptions } from "@cocalc/conat/files/fs"; +export type { FindOptions }; +import exec, { type ExecOutput, validate } from "./exec"; + +export default async function find( + path: string, + { options, darwin, linux, timeout, maxSize }: FindOptions, +): Promise { + if (path == null) { + throw Error("path must be specified"); + } + return await exec({ + cmd: "find", + cwd: path, + prefixArgs: [path ? path : "."], + options, + darwin, + linux, + maxSize, + timeout, + whitelist, + safety: [], + }); +} + +const whitelist = { + // POSITIONAL OPTIONS + "-daystart": true, + "-regextype": validate.str, + "-warn": true, + "-nowarn": true, + + // GLOBAL OPTIONS + "-d": true, + "-depth": true, + "--help": true, + "-ignore_readdir_race": true, + "-maxdepth": validate.int, + "-mindepth": validate.int, + "-mount": true, + "-noignore_readdir_race": true, + "--version": true, + "-xdev": true, + + // TESTS + "-amin": validate.float, + "-anewer": validate.str, + "-atime": validate.float, + "-cmin": validate.float, + "-cnewer": validate.str, + "-ctime": validate.float, + "-empty": true, + "-executable": true, + "-fstype": validate.str, + "-gid": validate.int, + "-group": validate.str, + "-ilname": validate.str, + "-iname": validate.str, + "-inum": validate.int, + "-ipath": validate.str, + "-iregex": validate.str, + "-iwholename": validate.str, + "-links": validate.int, + "-lname": validate.str, + "-mmin": validate.int, + "-mtime": validate.int, + "-name": validate.str, + "-newer": validate.str, + "-newerXY": validate.str, + "-nogroup": true, + "-nouser": true, + "-path": validate.str, + "-perm": validate.str, + "-readable": true, + "-regex": validate.str, + "-samefile": validate.str, + "-size": validate.str, + "-true": true, + "-type": validate.str, + "-uid": validate.int, + "-used": validate.float, + "-user": validate.str, + "-wholename": validate.str, + "-writable": true, + "-xtype": validate.str, + "-context": validate.str, + + // ACTIONS: obviously many are not whitelisted! + "-ls": true, + "-print": true, + "-print0": true, + "-printf": validate.str, + "-prune": true, + "-quit": true, + + // OPERATORS + "(": true, + ")": true, + "!": true, + "-not": true, + "-a": true, + "-and": true, + "-o": true, + "-or": true, + ",": true, +} as const; diff --git a/src/packages/backend/sandbox/get-listing.ts b/src/packages/backend/sandbox/get-listing.ts new file mode 100644 index 00000000000..4d02bba1a1e --- /dev/null +++ b/src/packages/backend/sandbox/get-listing.ts @@ -0,0 +1,81 @@ +/* +This is needed on non-Linux instead of using find. We just have to do it directly. +*/ + +import { join } from "path"; +import * as fs from "fs/promises"; + +import type { + FileTypeLabel, + FileData, + Files, +} from "@cocalc/conat/files/listing"; +export type { Files }; + +export default async function getListing( + path: string, +): Promise<{ files: Files; truncated?: boolean }> { + const files: Files = {}; + + // Helper to map stats to your FileTypeLabel + const fileTypeLabel = (st: any): FileTypeLabel => { + if (typeof st?.type === "string") return st.type as FileTypeLabel; // honor fs.lstat extension if present + if (st.isSymbolicLink?.()) return "l"; + if (st.isDirectory?.()) return "d"; + if (st.isBlockDevice?.()) return "b"; + if (st.isCharacterDevice?.()) return "c"; + if (st.isSocket?.()) return "s"; + if (st.isFIFO?.()) return "p"; + return "f"; + }; + + // Prefer opendir (streaming, resilient to large dirs); fall back to readdir + let names: string[] = []; + const dir = await fs.opendir(path); + for await (const dirent of dir as AsyncIterable<{ name: string }>) { + // Skip "." and ".." if the implementation happens to surface them + if (dirent?.name === "." || dirent?.name === "..") continue; + names.push(dirent.name); + } + + // lstat (not stat!) each entry; resolve link target only for symlinks + // Do these in parallel. + await Promise.allSettled( + names.map(async (name) => { + const full = join(path, name); + try { + const st = await fs.lstat(full); + const type = fileTypeLabel(st); + const data: FileData = { + mtime: st.mtimeMs, // ms since epoch (matches your original parsing) + size: st.size, + type, + }; + + if (type === "l" || st.isSymbolicLink?.()) { + data.isSymLink = true; + try { + data.linkTarget = await fs.readlink(full); + } catch { + // ignore unreadable targets (permissions/race) + } + } + if (type === "d" || st.isDirectory?.()) { + data.isDir = true; + } + + files[name] = data; + } catch (err: any) { + // Handle races: entry could have been deleted between listing and lstat + if (err?.code === "ENOENT") { + // just skip + return; + } + // For other errors, mirror your existing behavior: warn and skip + console.warn("WARNING (getListing):", err); + } + }), + ); + + return { files, truncated: false }; +} diff --git a/src/packages/backend/sandbox/index.ts b/src/packages/backend/sandbox/index.ts new file mode 100644 index 00000000000..58e5bffb149 --- /dev/null +++ b/src/packages/backend/sandbox/index.ts @@ -0,0 +1,543 @@ +/* +Given a path to a folder on the filesystem, this provides +a wrapper class with an API similar to the fs/promises modules, +but which only allows access to files in that folder. +It's a bit simpler with return data that is always +serializable. + +Absolute and relative paths are considered as relative to the input folder path. + +REFERENCE: We don't use https://github.com/metarhia/sandboxed-fs, but did +look at the code. + +SECURITY: + +The following could be a big problem -- user somehow create or change path to +be a dangerous symlink *after* the realpath check below, but before we do an fs *read* +operation. If they did that, then we would end up reading the target of the +symlink. I.e., if they could somehow create the file *as an unsafe symlink* +right after we confirm that it does not exist and before we read from it. This +would only happen via something not involving this sandbox, e.g., the filesystem +mounted into a container some other way. + +In short, I'm worried about: + +1. Request to read a file named "link" which is just a normal file. We confirm this using realpath + in safeAbsPath. +2. Somehow delete "link" and replace it by a new file that is a symlink to "../{project_id}/.ssh/id_ed25519" +3. Read the file "link" and get the contents of "../{project_id}/.ssh/id_ed25519". + +The problem is that 1 and 3 happen microseconds apart as separate calls to the filesystem. + +**[ ] TODO -- NOT IMPLEMENTED YET: This is why we have to uses file descriptors!** + +1. User requests to read a file named "link" which is just a normal file. +2. We wet file descriptor fd for whatever "link" is. Then confirm this is OK using realpath in safeAbsPath. +3. user somehow deletes "link" and replace it by a new file that is a symlink to "../{project_id}/.ssh/id_ed25519" +4. We read from the file descriptor fd and get the contents of original "link" (or error). + +*/ + +import { + appendFile, + chmod, + cp, + constants, + copyFile, + link, + lstat, + readdir, + readFile, + readlink, + realpath, + rename, + rm, + rmdir, + mkdir, + stat, + symlink, + truncate, + writeFile, + unlink, + utimes, +} from "node:fs/promises"; +import { move } from "fs-extra"; +import { exists } from "@cocalc/backend/misc/async-utils-node"; +import { basename, dirname, join, resolve } from "path"; +import { replace_all } from "@cocalc/util/misc"; +import find, { type FindOptions } from "./find"; +import ripgrep, { type RipgrepOptions } from "./ripgrep"; +import fd, { type FdOptions } from "./fd"; +import dust, { type DustOptions } from "./dust"; +import rustic from "./rustic"; +import { type ExecOutput } from "./exec"; +import { rusticRepo } from "@cocalc/backend/data"; +import ouch, { type OuchOptions } from "./ouch"; +import cpExec from "./cp"; +import { type CopyOptions } from "@cocalc/conat/files/fs"; +export { type CopyOptions }; +import { ConatError } from "@cocalc/conat/core/client"; +import getListing, { type Files } from "./get-listing"; +import LRU from "lru-cache"; +import watch, { type WatchIterator, type WatchOptions } from "./watch"; +//import getLogger from "@cocalc/backend/logger"; + +//const logger = getLogger("sandbox:fs"); + +// max time code can run (in safe mode), e.g., for find, +// ripgrep, fd, and dust. +const MAX_TIMEOUT = 5_000; + +// Maximum amount of memory for the "last value on disk" data, which +// supports a much better "sync with file state on disk" algorithm. +const MAX_LAST_ON_DISK = 50_000_000; // 50 MB +const LAST_ON_DISK_TTL = 1000 * 60 * 5; // 5 minutes + +interface Options { + // unsafeMode -- if true, assume security model where user is running this + // themself, e.g., in a project, so no security is needed at all. + unsafeMode?: boolean; + // readonly -- only allow operations that don't change files + readonly?: boolean; + host?: string; + rusticRepo?: string; +} + +// If you add any methods below that are NOT for the public api +// be sure to exclude them here! +const INTERNAL_METHODS = new Set([ + "safeAbsPath", + "safeAbsPaths", + "constructor", + "path", + "unsafeMode", + "readonly", + "assertWritable", + "rusticRepo", + "host", + "readFileLock", + "_lockFile", + "lastOnDisk", +]); + +export class SandboxedFilesystem { + public readonly unsafeMode: boolean; + public readonly readonly: boolean; + public rusticRepo: string; + private host?: string; + private lastOnDisk = new LRU({ + maxSize: MAX_LAST_ON_DISK, + sizeCalculation: (value) => value.length+1, // must be positive! + ttl: LAST_ON_DISK_TTL, + }); + + constructor( + // path should be the path to a FOLDER on the filesystem (not a file) + public readonly path: string, + { + unsafeMode = false, + readonly = false, + host = "global", + rusticRepo: repo, + }: Options = {}, + ) { + this.unsafeMode = !!unsafeMode; + this.readonly = !!readonly; + this.host = host; + this.rusticRepo = repo ?? rusticRepo; + for (const f in this) { + if (INTERNAL_METHODS.has(f)) { + continue; + } + const orig = this[f]; + // @ts-ignore + this[f] = async (...args) => { + try { + // @ts-ignore + return await orig(...args); + } catch (err) { + if (err.path) { + err.path = err.path.slice(this.path.length + 1); + } + err.message = replace_all(err.message, this.path + "/", ""); + throw err; + } + }; + } + } + + private assertWritable = (path: string) => { + if (this.readonly) { + throw new SandboxError( + `EACCES: permission denied -- read only filesystem, open '${path}'`, + { errno: -13, code: "EACCES", syscall: "open", path }, + ); + } + }; + + safeAbsPaths = async (path: string[] | string): Promise => { + return await Promise.all( + (typeof path == "string" ? [path] : path).map(this.safeAbsPath), + ); + }; + + safeAbsPath = async (path: string): Promise => { + if (typeof path != "string") { + throw Error(`path must be a string but is of type ${typeof path}`); + } + // pathInSandbox is *definitely* a path in the sandbox: + const pathInSandbox = join(this.path, resolve("/", path)); + if (this.unsafeMode) { + // not secure -- just convenient. + return pathInSandbox; + } + // However, there is still one threat, which is that it could + // be a path to an existing link that goes out of the sandbox. So + // we resolve to the realpath: + try { + const p = await realpath(pathInSandbox); + if (p != this.path && !p.startsWith(this.path + "/")) { + throw Error( + `realpath of '${path}' resolves to a path outside of sandbox`, + ); + } + // don't return the result of calling realpath -- what's important is + // their path's realpath is in the sandbox. + return pathInSandbox; + } catch (err) { + if (err.code == "ENOENT") { + return pathInSandbox; + } else { + throw err; + } + } + }; + + appendFile = async (path: string, data: string | Buffer, encoding?) => { + this.assertWritable(path); + return await appendFile(await this.safeAbsPath(path), data, encoding); + }; + + chmod = async (path: string, mode: string | number) => { + this.assertWritable(path); + await chmod(await this.safeAbsPath(path), mode); + }; + + constants = async (): Promise<{ [key: string]: number }> => { + return constants; + }; + + copyFile = async (src: string, dest: string) => { + this.assertWritable(dest); + await copyFile(await this.safeAbsPath(src), await this.safeAbsPath(dest)); + }; + + cp = async (src: string | string[], dest: string, options?: CopyOptions) => { + this.assertWritable(dest); + dest = await this.safeAbsPath(dest); + + // ensure containing directory of destination exists -- node cp doesn't + // do this but for cocalc this is very convenient and saves some network + // round trips. + const destDir = dirname(dest); + if (destDir != this.path && !(await exists(destDir))) { + await mkdir(destDir, { recursive: true }); + } + + const v = await this.safeAbsPaths(src); + if (!options?.reflink) { + // can use node cp: + for (const path of v) { + if (typeof src == "string") { + await cp(path, dest, options); + } else { + // copying multiple files to a directory + await cp(path, join(dest, basename(path)), options); + } + } + } else { + // /usr/bin/cp. NOte that behavior depends on string versus string[], + // so we pass the absolute paths v in that way. + await cpExec( + typeof src == "string" ? v[0] : v, + dest, + capTimeout(options, MAX_TIMEOUT), + ); + } + }; + + exists = async (path: string) => { + return await exists(await this.safeAbsPath(path)); + }; + + find = async (path: string, options?: FindOptions): Promise => { + return await find( + await this.safeAbsPath(path), + capTimeout(options, MAX_TIMEOUT), + ); + }; + + getListing = async ( + path: string, + ): Promise<{ files: Files; truncated?: boolean }> => { + return await getListing(await this.safeAbsPath(path)); + }; + + // find files + fd = async (path: string, options?: FdOptions): Promise => { + return await fd( + await this.safeAbsPath(path), + capTimeout(options, MAX_TIMEOUT), + ); + }; + + // disk usage + dust = async (path: string, options?: DustOptions): Promise => { + return await dust( + await this.safeAbsPath(path), + // dust reasonably takes longer than the other commands and is used less, + // so for now we give it more breathing room. + capTimeout(options, 4 * MAX_TIMEOUT), + ); + }; + + // compression + ouch = async (args: string[], options?: OuchOptions): Promise => { + options = { ...options }; + if (options.cwd) { + options.cwd = await this.safeAbsPath(options.cwd); + } + return await ouch( + [args[0]].concat(await Promise.all(args.slice(1).map(this.safeAbsPath))), + capTimeout(options, 6 * MAX_TIMEOUT), + ); + }; + + // backups + rustic = async ( + args: string[], + { + timeout = 120_000, + maxSize = 10_000_000, // the json output can be quite large + cwd, + }: { timeout?: number; maxSize?: number; cwd?: string } = {}, + ): Promise => { + return await rustic(args, { + repo: this.rusticRepo, + safeAbsPath: this.safeAbsPath, + timeout, + maxSize, + host: this.host, + cwd, + }); + }; + + ripgrep = async ( + path: string, + pattern: string, + options?: RipgrepOptions, + ): Promise => { + return await ripgrep( + await this.safeAbsPath(path), + pattern, + capTimeout(options, MAX_TIMEOUT), + ); + }; + + // hard link + link = async (existingPath: string, newPath: string) => { + this.assertWritable(newPath); + return await link( + await this.safeAbsPath(existingPath), + await this.safeAbsPath(newPath), + ); + }; + + lstat = async (path: string) => { + return await lstat(await this.safeAbsPath(path)); + }; + + mkdir = async (path: string, options?) => { + this.assertWritable(path); + await mkdir(await this.safeAbsPath(path), options); + }; + + private readFileLock = new Set(); + readFile = async ( + path: string, + encoding?: any, + lock?: number, + ): Promise => { + const p = await this.safeAbsPath(path); + if (this.readFileLock.has(p)) { + throw new ConatError(`path is locked - ${p}`, { code: "LOCK" }); + } + if (lock) { + this._lockFile(p, lock); + } + + return await readFile(p, encoding); + }; + + lockFile = async (path: string, lock?: number) => { + const p = await this.safeAbsPath(path); + this._lockFile(p, lock); + }; + + private _lockFile = (path: string, lock?: number) => { + if (lock) { + this.readFileLock.add(path); + setTimeout(() => { + this.readFileLock.delete(path); + }, lock); + } else { + this.readFileLock.delete(path); + } + }; + + readdir = async (path: string, options?) => { + const x = (await readdir(await this.safeAbsPath(path), options)) as any[]; + if (options?.withFileTypes) { + // each entry in x has a name and parentPath field, which refers to the + // absolute paths to the directory that contains x or the target of x (if + // it is a link). This is an absolute path on the fileserver, which we try + // not to expose from the sandbox, hence we modify them all if possible. + for (const a of x) { + if (a.name.startsWith(this.path)) { + a.name = a.name.slice(this.path.length + 1); + } + if (a.parentPath.startsWith(this.path)) { + a.parentPath = a.parentPath.slice(this.path.length + 1); + } + } + } + + return x; + }; + + readlink = async (path: string): Promise => { + return await readlink(await this.safeAbsPath(path)); + }; + + realpath = async (path: string): Promise => { + const x = await realpath(await this.safeAbsPath(path)); + return x.slice(this.path.length + 1); + }; + + rename = async (oldPath: string, newPath: string) => { + this.assertWritable(newPath); + await rename( + await this.safeAbsPath(oldPath), + await this.safeAbsPath(newPath), + ); + }; + + move = async ( + src: string, + dest: string, + options?: { overwrite?: boolean }, + ) => { + this.assertWritable(dest); + await move( + await this.safeAbsPath(src), + await this.safeAbsPath(dest), + options, + ); + }; + + rm = async (path: string | string[], options?) => { + const v = await this.safeAbsPaths(path); + const f = async (path) => { + this.assertWritable(path); + await rm(path, options); + }; + await Promise.all(v.map(f)); + }; + + rmdir = async (path: string, options?) => { + this.assertWritable(path); + await rmdir(await this.safeAbsPath(path), options); + }; + + stat = async (path: string) => { + return await stat(await this.safeAbsPath(path)); + }; + + symlink = async (target: string, path: string) => { + this.assertWritable(path); + return await symlink( + await this.safeAbsPath(target), + await this.safeAbsPath(path), + ); + }; + + truncate = async (path: string, len?: number) => { + this.assertWritable(path); + await truncate(await this.safeAbsPath(path), len); + }; + + unlink = async (path: string) => { + this.assertWritable(path); + await unlink(await this.safeAbsPath(path)); + }; + + utimes = async ( + path: string, + atime: number | string | Date, + mtime: number | string | Date, + ) => { + this.assertWritable(path); + await utimes(await this.safeAbsPath(path), atime, mtime); + }; + + watch = async ( + path: string, + options: WatchOptions = {}, + ): Promise => { + return watch(await this.safeAbsPath(path), options, this.lastOnDisk); + }; + + writeFile = async ( + path: string, + data: string | Buffer, + saveLast?: boolean, + ) => { + this.assertWritable(path); + const p = await this.safeAbsPath(path); + if (saveLast && typeof data == "string") { + this.lastOnDisk.set(p, data); + } + return await writeFile(p, data); + }; +} + +export class SandboxError extends Error { + code: string; + errno: number; + syscall: string; + path: string; + constructor(mesg: string, { code, errno, syscall, path }) { + super(mesg); + this.code = code; + this.errno = errno; + this.syscall = syscall; + this.path = path; + } +} + +function capTimeout(options, max: number) { + if (options == null) { + return { timeout: max }; + } + + let timeout; + try { + timeout = parseFloat(options.timeout); + } catch { + return { ...options, timeout: max }; + } + if (!isFinite(timeout)) { + return { ...options, timeout: max }; + } + return { ...options, timeout: Math.min(timeout, max) }; +} diff --git a/src/packages/backend/sandbox/install.ts b/src/packages/backend/sandbox/install.ts new file mode 100644 index 00000000000..e119409ad1c --- /dev/null +++ b/src/packages/backend/sandbox/install.ts @@ -0,0 +1,519 @@ +/* +Download a ripgrep or fd binary for the operating system + +This supports x86_64/arm64 linux & macos + +This assumes tar is installed. + +NOTE: There are several npm modules that purport to install ripgrep. We do not use +https://www.npmjs.com/package/@vscode/ripgrep because it is not properly maintained, +e.g., + - security vulnerabilities: https://github.com/microsoft/ripgrep-prebuilt/issues/48 + - not updated to a major new release without a good reason: https://github.com/microsoft/ripgrep-prebuilt/issues/38 + +NOTE: there is a linux program "upx", which can be run on any of these binaries +(except ssh where it is already run), which makes them self-extracting executables. +The binaries become less than half their size, but startup time is typically +increased to about 100ms to do the decompression every time. We're not currently +using this, but it could be useful in some contexts, maybe. The main value in +these programs isn't that they are small, but that: + +- they are all statically linked, so run anywhere (e.g., in any container) +- they are fast (newer, in rust/go) often using parallelism well +*/ + +import { arch, platform } from "os"; +import { split } from "@cocalc/util/misc"; +import { execFileSync, execSync } from "child_process"; +import { executeCode } from "@cocalc/backend/execute-code"; +import { writeFile, stat, unlink, mkdir, chmod } from "fs/promises"; +import { join } from "path"; +// using old version of pkg-dir because of nextjs :-( +import { sync as packageDirectorySync } from "pkg-dir"; +import getLogger from "@cocalc/backend/logger"; + +const logger = getLogger("files:sandbox:install"); + +const pkgDir = packageDirectorySync(__dirname) ?? ""; + +const binPath = join(pkgDir, "node_modules", ".bin"); + +interface Spec { + nonFatal?: boolean; // true if failure to install is non-fatal + VERSION?: string; + BASE?: string; + binary?: string; + path: string; + stripComponents?: number; + pathInArchive?: () => string; + skip?: string[]; + script?: () => string; + platforms?: string[]; + fix?: string; + url?: () => string; + // if given, a bash shell line to run whose LAST output + // (split by whitespace) is the version + getVersion: string; +} + +export const SPEC = { + rg: { + // See https://github.com/BurntSushi/ripgrep/releases + VERSION: "14.1.1", + BASE: "https://github.com/BurntSushi/ripgrep/releases/download", + binary: "rg", + path: join(binPath, "rg"), + getVersion: "rg --version | head -n 1 | awk '{ print $2 }'", + url: () => + `${SPEC.rg.BASE}/${SPEC.rg.VERSION}/ripgrep-${SPEC.rg.VERSION}-${getOS()}.tar.gz`, + pathInArchive: () => + `ripgrep-${SPEC.rg.VERSION}-${getOS()}/${SPEC.rg.binary}`, + }, + fd: { + // See https://github.com/sharkdp/fd/releases + VERSION: "v10.2.0", + getVersion: `fd --version | awk '{print "v"$2}'`, + BASE: "https://github.com/sharkdp/fd/releases/download", + binary: "fd", + path: join(binPath, "fd"), + }, + dust: { + // See https://github.com/bootandy/dust/releases + VERSION: "v1.2.3", + getVersion: `dust --version | awk '{print "v"$2}'`, + BASE: "https://github.com/bootandy/dust/releases/download", + binary: "dust", + path: join(binPath, "dust"), + // github binaries exists for x86 mac only, which is dead - in homebrew. + platforms: ["linux"], + }, + ouch: { + // See https://github.com/ouch-org/ouch/releases + VERSION: "0.6.1", + getVersion: "ouch --version", + BASE: "https://github.com/ouch-org/ouch/releases/download", + binary: "ouch", + path: join(binPath, "ouch"), + // See https://github.com/ouch-org/ouch/issues/45; note that ouch is in home brew + // for this platform. + platforms: ["linux"], + url: () => { + const os = getOS(); + return `${SPEC.ouch.BASE}/${SPEC.ouch.VERSION}/ouch-${os}.tar.gz`; + }, + pathInArchive: () => `ouch-${getOS()}/${SPEC.ouch.binary}`, + }, + rustic: { + // See https://github.com/rustic-rs/rustic/releases + VERSION: "v0.10.0", + getVersion: "rustic --version", + BASE: "https://github.com/rustic-rs/rustic/releases/download", + binary: "rustic", + path: join(binPath, "rustic"), + stripComponents: 0, + pathInArchive: () => "rustic", + }, + // sshpiper -- used by the core + // See https://github.com/sagemathinc/sshpiper-binaries/releases + sshpiper: { + optional: true, + desc: "sshpiper reverse proxy for sshd", + path: join(binPath, "sshpiperd"), + // this is what --version outputs and is the sha hash of HEAD: + VERSION: "7fdd88982", + getVersion: "sshpiperd --version | awk '{print $4}' | cut -c 1-9", + script: () => { + // this is the actual version in our release page + const VERSION = "v1.5.0"; + const a = arch() == "x64" ? "amd64" : arch(); + return `curl -L https://github.com/sagemathinc/sshpiper-binaries/releases/download/${VERSION}/sshpiper-${VERSION}-${platform()}-${a}.tar.xz | tar -xJ -C "${binPath}" --strip-components=1`; + }, + url: () => { + const VERSION = SPEC.sshpiper.VERSION; + // https://github.com/sagemathinc/sshpiper-binaries/releases/download/v1.5.0/sshpiper-v1.5.0-darwin-amd64.tar.xz + return `sshpiper-${VERSION}-${arch() == "x64" ? "amd64" : arch()}.tar.xz`; + }, + BASE: "https://github.com/sagemathinc/sshpiper-binaries/releases", + }, + + // See https://github.com/sagemathinc/mutagen-open-source/releases + mutagen: { + // Mutagen seems to be critical for everything, so not optional. + // Below we also remove the windows agents and x86 darwin, since + // old macs are very rare now. + desc: "Fast file synchronization and network forwarding for remote development", + path: join(binPath, "mutagen"), + VERSION: "0.19.0-dev", + getVersion: "mutagen --version", + // for now + script: () => { + const VERSION = SPEC.mutagen.VERSION; + const a = arch() == "x64" ? "amd64" : arch(); + if (platform() == "darwin") { + return `curl -L https://github.com/sagemathinc/mutagen-open-source/releases/download/${VERSION}/mutagen_${platform()}_${a}_v${VERSION}.tar.gz | tar -zox -C ${binPath}`; + } else { + return `curl -L https://github.com/sagemathinc/mutagen-open-source/releases/download/${VERSION}/mutagen_${platform()}_${a}_v${VERSION}.tar.gz | tar -zox -C ${binPath} && cd ${binPath} && gunzip mutagen-agents.tar.gz && tar --delete -f mutagen-agents.tar darwin_amd64 windows_amd64 && gzip mutagen-agents.tar`; + } + }, + }, + + btm: { + // See https://github.com/ClementTsang/bottom/releases + VERSION: "0.11.1", + getVersion: "btm --version", + BASE: "https://github.com/ClementTsang/bottom/releases/download", + platforms: ["linux"], + binary: "btm", + script: () => { + const VERSION = SPEC.btm.VERSION; + const url = `${SPEC.btm.BASE}/${VERSION}/bottom_${getOS()}.tar.gz`; + return `curl -L ${url} | tar -xz -C ${binPath} btm`; + }, + path: join(binPath, "btm"), + }, + + dropbear: { + desc: "Dropbear Statically Linked SSH Server ", + platforms: ["linux"], + VERSION: "v2025.88", + getVersion: "dropbear -V", + path: join(binPath, "dropbear"), + // we grab just the dropbear binary out of the release; we don't + // need any of the others: + script: () => + `curl -L https://github.com/sagemathinc/dropbear/releases/download/main/dropbear-$(uname -m)-linux-musl.tar.xz | tar -xJ -C ${binPath} --strip-components=1 dropbear-$(uname -m)-linux-musl/dropbear`, + }, + /* + Locate the latest binaries are here: + https://github.com/sagemathinc/static-openssh-binaries/releases + E.g., the files look like + https://github.com/sagemathinc/static-openssh-binaries/releases/download/OpenSSH_9.9p2/openssh-static-x86_64-small-2025-10-02b.tar.gz + https://github.com/sagemathinc/static-openssh-binaries/releases/download/OpenSSH_9.9p2/openssh-static-aarch64-small-2025-10-02b.tar.gz + and they extract like this: +~# tar xvf openssh-static-x86_64-small-OpenSSH_9.9p2.tar.gz +openssh/ +openssh/sbin/ +openssh/sbin/sshd +openssh/etc/ +openssh/etc/sshd_config +openssh/bin/ +openssh/bin/ssh-add +openssh/bin/sftp +openssh/bin/ssh-keyscan +openssh/bin/ssh-keygen +openssh/bin/ssh-agent +openssh/bin/ssh +openssh/bin/scp +openssh/var/ +openssh/var/empty/ +openssh/libexec/ +openssh/libexec/ssh-keysign +openssh/libexec/sshd-session +openssh/libexec/sftp-server + +To build a new version figure out what version (say OpenSSH_9.9p2) +happens to be being built, then do + + git tag OpenSSH_9.9p2 + git push --tags + +to make a binary with that version + +--- +*/ + ssh: { + desc: "statically linked compressed openssh binaries: ssh, scp, ssh-keygen", + path: join(binPath, "ssh"), + platforms: ["linux"], + VERSION: "OpenSSH_9.9p2", + getVersion: "ssh -V 2>&1 | cut -f 1 -d ','", + script: () => + `curl -L https://github.com/sagemathinc/static-openssh-binaries/releases/download/${SPEC.ssh.VERSION}/openssh-static-$(uname -m)-small-${SPEC.ssh.VERSION}.tar.gz | tar -xz -C ${binPath} --strip-components=2 openssh/bin/ssh openssh/bin/ssh-keygen openssh/libexec/sftp-server`, + }, + + // See https://github.com/moparisthebest/static-curl/releases + // + // https://github.com/moparisthebest/static-curl/releases/download/v8.11.0/curl-amd64 + // https://github.com/moparisthebest/static-curl/releases/download/v8.11.0/curl-aarch64 + curl: { + desc: "statically linked curl", + path: join(binPath, "curl"), + platforms: ["linux"], + getVersion: "curl --version | head -n 1 | cut -f 2 -d ' '", + VERSION: "8.11.0", + script: () => + `curl -L https://github.com/moparisthebest/static-curl/releases/download/v${SPEC.curl.VERSION}/curl-${arch() == "x64" ? "amd64" : arch()} > ${join(binPath, "curl")} && chmod a+x ${join(binPath, "curl")}`, + }, + + // See https://github.com/sagemathinc/bees-binaries/releases + bees: { + desc: "Bees dedup binary for Ubuntu with minimal deps", + path: join(binPath, "bees"), + platforms: ["linux"], + VERSION: "2024-10-04a", + // https://github.com/sagemathinc/bees-binaries/releases/download/2024-10-04a/bees-2024-10-04a-x86_64-linux-glibc.tar.xz + script: () => { + const name = `bees-${SPEC.bees.VERSION}-${arch() == "x64" ? "x86_64" : arch()}-linux-glibc`; + return `curl -L https://github.com/sagemathinc/bees-binaries/releases/download/${SPEC.bees.VERSION}/${name}.tar.xz | tar -xJ -C ${binPath} --strip-components=2 ${name}/bin/bees`; + }, + }, +}; + +export const rg = SPEC.rg.path; +export const fd = SPEC.fd.path; +export const dust = SPEC.dust.path; +export const rustic = SPEC.rustic.path; +export const ouch = SPEC.ouch.path; +export const sshpiper = SPEC.sshpiper.path; +export const mutagen = SPEC.mutagen.path; +export const btm = SPEC.btm.path; +export const dropbear = SPEC.dropbear.path; +export const ssh = SPEC.ssh.path; +export const curl = SPEC.curl.path; + +type App = keyof typeof SPEC; + +// https://github.com/sharkdp/fd/releases/download/v10.2.0/fd-v10.2.0-x86_64-unknown-linux-musl.tar.gz +// https://github.com/BurntSushi/ripgrep/releases/download/14.1.1/ripgrep-14.1.1-x86_64-unknown-linux-musl.tar.gz + +export async function exists(path: string) { + try { + await stat(path); + return true; + } catch { + return false; + } +} + +export async function installedVersion(app: App): Promise { + const { path, getVersion } = SPEC[app] as Spec; + if (!(await exists(path))) { + return; + } + if (!getVersion) { + return; + } + try { + const { stdout, stderr } = await executeCode({ + verbose: true, + command: getVersion, + env: { ...process.env, PATH: binPath + ":/usr/bin:" + process.env.PATH }, + }); + const v = split(stdout + stderr) + .slice(-1)[0] + .trim(); + return v; + } catch (err) { + logger.debug("WARNING: issue getting version", { path, getVersion, err }); + } + return; +} + +export async function versions() { + const v: { [app: string]: string | undefined } = {}; + await Promise.all( + Object.keys(SPEC).map(async (app) => { + v[app] = await installedVersion(app as App); + }), + ); + return v; +} + +export async function alreadyInstalled(app: App) { + const { path, VERSION } = SPEC[app] as Spec; + if (!(await exists(path))) { + return false; + } + const v = await installedVersion(app); + if (v == null) { + // no version info + return true; + } + return v == VERSION; +} + +export async function install( + app?: App, + { optional }: { optional?: boolean } = {}, +) { + if (app == null) { + if (!(await exists(binPath))) { + await mkdir(binPath, { recursive: true }); + } + // @ts-ignore + await Promise.all( + Object.keys(SPEC) + .filter((x) => optional || !SPEC[x].optional) + .map((x) => install(x as App, { optional })), + ); + return; + } + + if (await alreadyInstalled(app)) { + return; + } + + const spec = SPEC[app] as Spec; + + if (spec.platforms != null && !spec.platforms?.includes(platform())) { + return; + } + + const { script } = spec; + try { + if (script != null) { + const s = script(); + console.log(s); + try { + execSync(s); + } catch (err) { + if (spec.fix) { + console.warn(`BUILD OF ${app} FAILED: Suggested fix -- ${spec.fix}`); + } + throw err; + } + if (!(await alreadyInstalled(app))) { + throw Error(`failed to install ${app}`); + } + return; + } + + if (!(await exists(binPath))) { + await mkdir(binPath, { recursive: true }); + } + + const url = getUrl(app); + if (!url) { + logger.debug("install: skipping ", app); + return; + } + logger.debug("install", { app, url }); + // - 1. Fetch the tarball from the github url (using the fetch library) + const response = await downloadFromGithub(url); + const tarballBuffer = Buffer.from(await response.arrayBuffer()); + + // - 2. Extract the file "rg" from the tarball to ${__dirname}/rg + // The tarball contains this one file "rg" at the top level, i.e., for + // ripgrep-14.1.1-x86_64-unknown-linux-musl.tar.gz + // we have "tar tvf ripgrep-14.1.1-x86_64-unknown-linux-musl.tar.gz" outputs + // ... + // ripgrep-14.1.1-x86_64-unknown-linux-musl/rg + + const { VERSION, binary, path, stripComponents = 1, pathInArchive } = spec; + + const archivePath = + pathInArchive?.() ?? `${app}-${VERSION}-${getOS()}/${binary}`; + + const tmpFile = join(__dirname, `${app}-${VERSION}.tar.gz`); + try { + await writeFile(tmpFile, tarballBuffer); + // sync is fine since this is run at *build time*. + execFileSync("tar", [ + "xzf", + tmpFile, + `--strip-components=${stripComponents}`, + `-C`, + binPath, + archivePath, + ]); + + // - 3. Make the file executable + await chmod(path, 0o755); + } finally { + try { + await unlink(tmpFile); + } catch {} + } + } catch (err) { + if (spec.nonFatal) { + console.log(`WARNING: unable to install ${app}`, err); + } else { + throw err; + } + } +} + +// Download from github, but aware of rate limits, the retry-after header, etc. +async function downloadFromGithub(url: string) { + const maxRetries = 10; + const baseDelay = 1000; // 1 second + + for (let attempt = 1; attempt <= maxRetries; attempt++) { + try { + const res = await fetch(url); + + if (res.status === 429) { + // Rate limit error + if (attempt === maxRetries) { + throw new Error("Rate limit exceeded after max retries"); + } + + const retryAfter = res.headers.get("retry-after"); + const delay = retryAfter + ? parseInt(retryAfter) * 1000 + : baseDelay * Math.pow(2, attempt - 1); // Exponential backoff + + console.log( + `Rate limited. Retrying in ${delay}ms (attempt ${attempt}/${maxRetries})`, + ); + await new Promise((resolve) => setTimeout(resolve, delay)); + continue; + } + + if (!res.ok) { + throw new Error(`HTTP ${res.status}: ${res.statusText}`); + } + return res; + } catch (error) { + if (attempt === maxRetries) { + throw error; + } + + const delay = baseDelay * Math.pow(2, attempt - 1); + console.log( + `Fetch ${url} failed. Retrying in ${delay}ms (attempt ${attempt}/${maxRetries})`, + ); + await new Promise((resolve) => setTimeout(resolve, delay)); + } + } + throw new Error("Should not reach here"); +} + +function getUrl(app: App) { + const spec = SPEC[app] as Spec; + if (spec.url != null) { + return spec.url(); + } + const { BASE, VERSION, skip } = spec; + const os = getOS(); + if (skip?.includes(os)) { + return ""; + } + // very common pattern with rust cli tools: + return `${BASE}/${VERSION}/${app}-${VERSION}-${os}.tar.gz`; +} + +function getOS() { + switch (platform()) { + case "linux": + switch (arch()) { + case "x64": + return "x86_64-unknown-linux-musl"; + case "arm64": + return "aarch64-unknown-linux-gnu"; + default: + throw Error(`unsupported arch '${arch()}'`); + } + case "darwin": + switch (arch()) { + case "x64": + return "x86_64-apple-darwin"; + case "arm64": + return "aarch64-apple-darwin"; + default: + throw Error(`unsupported arch '${arch()}'`); + } + default: + throw Error(`unsupported platform '${platform()}'`); + } +} diff --git a/src/packages/backend/sandbox/ouch.test.ts b/src/packages/backend/sandbox/ouch.test.ts new file mode 100644 index 00000000000..96ae332833e --- /dev/null +++ b/src/packages/backend/sandbox/ouch.test.ts @@ -0,0 +1,56 @@ +/* +Test the ouch compression api. +*/ + +import ouch from "./ouch"; +import { mkdtemp, mkdir, rm, readFile, writeFile } from "node:fs/promises"; +import { tmpdir } from "node:os"; +import { join } from "node:path"; +import { exists } from "@cocalc/backend/misc/async-utils-node"; + +let tempDir, options; +beforeAll(async () => { + tempDir = await mkdtemp(join(tmpdir(), "cocalc")); + options = { cwd: tempDir }; +}); +afterAll(async () => { + await rm(tempDir, { force: true, recursive: true }); +}); + +describe("ouch works on a little file", () => { + for (const ext of [ + "zip", + "7z", + "tar.gz", + "tar.xz", + "tar.bz", + "tar.bz2", + "tar.bz3", + "tar.lz4", + "tar.sz", + "tar.zst", + "tar.br", + ]) { + it(`create file and compress it up using ${ext}`, async () => { + await writeFile(join(tempDir, "a.txt"), "hello"); + const { truncated, code } = await ouch( + ["compress", "a.txt", `a.${ext}`], + options, + ); + expect(code).toBe(0); + expect(truncated).toBe(false); + expect(await exists(join(tempDir, `a.${ext}`))).toBe(true); + }); + + it(`extract ${ext} in subdirectory`, async () => { + await mkdir(join(tempDir, `target-${ext}`)); + const { code } = await ouch(["decompress", join(tempDir, `a.${ext}`)], { + cwd: join(tempDir, `target-${ext}`), + }); + expect(code).toBe(0); + expect( + (await readFile(join(tempDir, `target-${ext}`, "a.txt"))).toString(), + ).toEqual("hello"); + }); + } +}); diff --git a/src/packages/backend/sandbox/ouch.ts b/src/packages/backend/sandbox/ouch.ts new file mode 100644 index 00000000000..69754330b42 --- /dev/null +++ b/src/packages/backend/sandbox/ouch.ts @@ -0,0 +1,69 @@ +/* + +https://github.com/ouch-org/ouch + +ouch stands for Obvious Unified Compression Helper. + +The .tar.gz support in 'ouch' is excellent -- super fast and memory efficient, +since it is fully parallel. + +*/ + +import exec, { type ExecOutput, validate } from "./exec"; +import { type OuchOptions } from "@cocalc/conat/files/fs"; +export { type OuchOptions }; +import { ouch as ouchPath } from "./install"; + +export default async function ouch( + args: string[], + { timeout, options, cwd }: OuchOptions = {}, +): Promise { + const command = args[0]; + if (!commands.includes(command)) { + throw Error(`first argument must be one of ${commands.join(", ")}`); + } + + return await exec({ + cmd: ouchPath, + cwd, + positionalArgs: args.slice(1), + safety: [command, "-y", "-q"], + timeout, + options, + whitelist, + }); +} + +const commands = ["compress", "c", "decompress", "d", "list", "l", "ls"]; + +const whitelist = { + // general options, + "-H": true, + "--hidden": true, + g: true, + "--gitignore": true, + "-f": validate.str, + "--format": validate.str, + "-p": validate.str, + "--password": validate.str, + "-h": true, + "--help": true, + "-V": true, + "--version": true, + + // compression-specific options + // do NOT enable '-S, --follow-symlinks' as that could escape the sandbox! + // It's off by default. + + "-l": validate.str, + "--level": validate.str, + "--fast": true, + "--slow": true, + + // decompress specific options + "-d": validate.str, + "--dir": validate.str, + r: true, + "--remove": true, + "--no-smart-unpack": true, +} as const; diff --git a/src/packages/backend/sandbox/ripgrep.test.ts b/src/packages/backend/sandbox/ripgrep.test.ts new file mode 100644 index 00000000000..34ba5e0d675 --- /dev/null +++ b/src/packages/backend/sandbox/ripgrep.test.ts @@ -0,0 +1,27 @@ +import ripgrep from "./ripgrep"; +import { mkdtemp, rm, writeFile } from "node:fs/promises"; +import { tmpdir } from "node:os"; +import { join } from "node:path"; + +let tempDir; +beforeAll(async () => { + tempDir = await mkdtemp(join(tmpdir(), "cocalc")); +}); +afterAll(async () => { + await rm(tempDir, { force: true, recursive: true }); +}); + +describe("ripgrep files", () => { + it("directory starts empty - no results", async () => { + const { stdout, truncated } = await ripgrep(tempDir, ""); + expect(stdout.length).toBe(0); + expect(truncated).toBe(false); + }); + + it("create a file and see it appears in the rigrep result", async () => { + await writeFile(join(tempDir, "a.txt"), "hello"); + const { stdout, truncated } = await ripgrep(tempDir, "he"); + expect(truncated).toBe(false); + expect(stdout.toString()).toEqual("a.txt:hello\n"); + }); +}); diff --git a/src/packages/backend/sandbox/ripgrep.ts b/src/packages/backend/sandbox/ripgrep.ts new file mode 100644 index 00000000000..5e4f81a3259 --- /dev/null +++ b/src/packages/backend/sandbox/ripgrep.ts @@ -0,0 +1,237 @@ +import exec, { type ExecOutput, validate } from "./exec"; +import type { RipgrepOptions } from "@cocalc/conat/files/fs"; +export type { RipgrepOptions }; +import { rg as ripgrepPath } from "./install"; + +export default async function ripgrep( + path: string, + pattern: string, + { options, darwin, linux, timeout, maxSize }: RipgrepOptions = {}, +): Promise { + if (path == null) { + throw Error("path must be specified"); + } + if (pattern == null) { + throw Error("pattern must be specified"); + } + + return await exec({ + cmd: ripgrepPath, + cwd: path, + positionalArgs: [pattern], + options, + darwin, + linux, + maxSize, + timeout, + whitelist, + // if large memory usage is an issue, it might be caused by parallel interleaving; using + // -j1 below will prevent that, but will make ripgrep much slower (since not in parallel). + // See the ripgrep man page. + safety: ["--no-follow", "--block-buffered", "--no-config" /* "-j1"*/], + }); +} + +const whitelist = { + "-e": validate.str, + + "-s": true, + "--case-sensitive": true, + + "--crlf": true, + + "-E": validate.set([ + "utf-8", + "utf-16", + "utf-16le", + "utf-16be", + "ascii", + "latin-1", + ]), + "--encoding": validate.set([ + "utf-8", + "utf-16", + "utf-16le", + "utf-16be", + "ascii", + "latin-1", + ]), + + "--engine": validate.set(["default", "pcre2", "auto"]), + + "-F": true, + "--fixed-strings": true, + + "-i": true, + "--ignore-case": true, + + "-v": true, + "--invert-match": true, + + "-x": true, + "--line-regexp": true, + + "-m": validate.int, + "--max-count": validate.int, + + "-U": true, + "--multiline": true, + + "--multiline-dotall": true, + + "--no-unicode": true, + + "--null-data": true, + + "-P": true, + "--pcre2": true, + + "-S": true, + "--smart-case": true, + + "--stop-on-nonmatch": true, + + // this allows searching in binary files -- there is some danger of this + // using a lot more memory. Hence we do not allow it. + // "-a": true, + // "--text": true, + + "-w": true, + "--word-regexp": true, + + "--binary": true, + + "-g": validate.str, + "--glob": validate.str, + "--glob-case-insensitive": true, + + "-.": true, + "--hidden": true, + + "--iglob": validate.str, + + "--ignore-file-case-insensitive": true, + + "-d": validate.int, + "--max-depth": validate.int, + + "--max-filesize": validate.str, + + "--no-ignore": true, + "--no-ignore-dot": true, + "--no-ignore-exclude": true, + "--no-ignore-files": true, + "--no-ignore-global": true, + "--no-ignore-parent": true, + "--no-ignore-vcs": true, + "--no-require-git": true, + "--one-file-system": true, + + "-t": validate.str, + "--type": validate.str, + "-T": validate.str, + "--type-not": validate.str, + "--type-add": validate.str, + "--type-list": true, + "--type-clear": validate.str, + + "-u": true, + "--unrestricted": true, + + "-A": validate.int, + "--after-context": validate.int, + "-B": validate.int, + "--before-context": validate.int, + + "-b": true, + "--byte-offset": true, + + "--color": validate.set(["never", "auto", "always", "ansi"]), + "--colors": validate.str, + + "--column": true, + "-C": validate.int, + "--context": validate.int, + + "--context-separator": validate.str, + "--field-context-separator": validate.str, + "--field-match-separator": validate.str, + + "--heading": true, + "--no-heading": true, + + "-h": true, + "--help": true, + + "--include-zero": true, + + "-n": true, + "--line-number": true, + "-N": true, + "--no-line-number": true, + + "-M": validate.int, + "--max-columns": validate.int, + + "--max-columns-preview": validate.int, + + "-O": true, + "--null": true, + + "--passthru": true, + + "-p": true, + "--pretty": true, + + "-q": true, + "--quiet": true, + + // From the docs: "Neither this flag nor any other ripgrep flag will modify your files." + "-r": validate.str, + "--replace": validate.str, + + "--sort": validate.set(["none", "path", "modified", "accessed", "created"]), + "--sortr": validate.set(["none", "path", "modified", "accessed", "created"]), + + "--trim": true, + "--no-trim": true, + + "--vimgrep": true, + + "-H": true, + "--with-filename": true, + + "-I": true, + "--no-filename": true, + + "-c": true, + "--count": true, + + "--count-matches": true, + "-l": true, + "--files-with-matches": true, + "--files-without-match": true, + "--json": true, + + "--debug": true, + "--no-ignore-messages": true, + "--no-messages": true, + + "--stats": true, + + "--trace": true, + + "--files": true, + + "--generate": validate.set([ + "man", + "complete-bash", + "complete-zsh", + "complete-fish", + "complete-powershell", + ]), + + "--pcre2-version": true, + "-V": true, + "--version": true, +} as const; diff --git a/src/packages/backend/sandbox/rustic.test.ts b/src/packages/backend/sandbox/rustic.test.ts new file mode 100644 index 00000000000..0e9561c27ce --- /dev/null +++ b/src/packages/backend/sandbox/rustic.test.ts @@ -0,0 +1,80 @@ +/* +Test the rustic backup api. + +https://github.com/rustic-rs/rustic +*/ + +import rustic from "./rustic"; +import { mkdtemp, mkdir, rm, writeFile } from "node:fs/promises"; +import { tmpdir } from "node:os"; +import { join, resolve } from "node:path"; +import { parseOutput } from "./exec"; + +let tempDir, options, home; +beforeAll(async () => { + tempDir = await mkdtemp(join(tmpdir(), "cocalc")); + const repo = join(tempDir, "repo"); + home = join(tempDir, "home"); + await mkdir(home); + const safeAbsPath = (path: string) => join(home, resolve("/", path)); + options = { + host: "my-host", + repo, + safeAbsPath, + }; +}); +afterAll(async () => { + await rm(tempDir, { force: true, recursive: true }); +}); + +describe("rustic does something", () => { + it("there are initially no backups", async () => { + const { stdout, truncated } = await rustic( + ["snapshots", "--json"], + options, + ); + const s = JSON.parse(Buffer.from(stdout).toString()); + expect(s).toEqual([]); + expect(truncated).toBe(false); + }); + + it("create a file and back it up", async () => { + await writeFile(join(tempDir, "a.txt"), "hello"); + const { stdout, truncated } = await rustic( + ["backup", "--json", "a.txt"], + options, + ); + const s = JSON.parse(Buffer.from(stdout).toString()); + expect(s.paths).toEqual(["a.txt"]); + expect(truncated).toBe(false); + }); + + it("use a .toml file instead of explicitly passing in a repo", async () => { + await mkdir(join(home, "x")); + await writeFile( + join(home, "x/a.toml"), + ` +[repository] +repository = "${options.repo}" +password = "" +`, + ); + const options2 = { ...options, repo: join(home, "x/a.toml") }; + const { stdout } = parseOutput( + await rustic(["snapshots", "--json"], options2), + ); + const s = JSON.parse(stdout); + expect(s.length).toEqual(1); + expect(s[0][0].hostname).toEqual("my-host"); + }); + + // it("it appears in the snapshots list", async () => { + // const { stdout, truncated } = await rustic( + // ["snapshots", "--json"], + // options, + // ); + // const s = JSON.parse(Buffer.from(stdout).toString()); + // expect(s).toEqual([]); + // expect(truncated).toBe(false); + // }); +}); diff --git a/src/packages/backend/sandbox/rustic.ts b/src/packages/backend/sandbox/rustic.ts new file mode 100644 index 00000000000..eb4d2d7d5e7 --- /dev/null +++ b/src/packages/backend/sandbox/rustic.ts @@ -0,0 +1,368 @@ +/* +Whitelist: + +The idea is that + - the client can only work with snapshots with exactly the given host. + - any snapshots they create have that host + - snapshots are only of data in their sandbox + - snapshots can only be restored to their sandbox + +The subcommands with some whitelisted support are: + + - backup + - snapshots + - ls + - restore + - find + - forget + +The source options are relative paths and the command is run from the +root of the sandbox_path. + + rustic backup --host=sandbox_path [whitelisted options]... [source]... + + rustic snapshots --filter-host=... [whitelisted options]... + + +Here the snapshot id will be checked to have the right host before +the command is run. Destination is relative to sandbox_path. + + rustic restore [whitelisted options] + + +Dump is used for viewing a version of a file via timetravel: + + rustic dump + +Find is used for getting info about all versions of a file that are backed up: + + rustic find --filter-host=... + + rustic find --filter-host=... --glob='foo/x.txt' -h + + +Delete snapshots: + +- delete snapshot with specific id, which must have the specified host. + + rustic forget [id] + +- + + +*/ + +import exec, { + type ExecOutput, + parseAndValidateOptions, + validate, +} from "./exec"; +import { rustic as rusticPath } from "./install"; +import { exists } from "@cocalc/backend/misc/async-utils-node"; +import { join } from "path"; +import { rusticRepo } from "@cocalc/backend/data"; +import LRU from "lru-cache"; + +export interface RusticOptions { + repo?: string; + timeout?: number; + maxSize?: number; + safeAbsPath?: (path: string) => Promise; + host?: string; + cwd?: string; +} + +export default async function rustic( + args: string[], + options: RusticOptions, +): Promise { + const { + timeout, + maxSize, + repo = rusticRepo, + safeAbsPath, + host = "host", + } = options; + + let common; + if (repo.endsWith(".toml")) { + common = ["-P", repo.slice(0, -".toml".length)]; + } else { + common = ["--password", "", "-r", repo]; + } + + await ensureInitialized(repo); + const cwd = await safeAbsPath?.(options.cwd ?? ""); + + const run = async (sanitizedArgs: string[]) => { + return await exec({ + cmd: rusticPath, + cwd, + safety: [...common, args[0], ...sanitizedArgs], + maxSize, + timeout, + }); + }; + + switch (args[0]) { + case "init": { + if (safeAbsPath != null) { + throw Error("init not allowed"); + } + return await run([]); + } + case "backup": { + if (safeAbsPath == null || cwd == null) { + throw Error("safeAbsPath must be specified when making a backup"); + } + if (args.length == 1) { + throw Error("missing backup source"); + } + const source = (await safeAbsPath(args.slice(-1)[0])).slice(cwd.length); + const options = parseAndValidateOptions( + args.slice(1, -1), + whitelist.backup, + ); + + return await run([ + ...options, + "--no-scan", + "--host", + host, + "--", + source ? source : ".", + ]); + } + case "snapshots": { + const options = parseAndValidateOptions( + args.slice(1), + whitelist.snapshots, + ); + return await run([...options, "--filter-host", host]); + } + case "ls": { + if (args.length <= 1) { + throw Error("missing "); + } + const snapshot = args.slice(-1)[0]; // + await assertValidSnapshot({ snapshot, host, repo }); + const options = parseAndValidateOptions(args.slice(1, -1), whitelist.ls); + return await run([...options, snapshot]); + } + case "restore": { + if (args.length <= 2) { + throw Error("missing "); + } + if (safeAbsPath == null) { + throw Error("safeAbsPath must be specified when restoring"); + } + const snapshot = args.slice(-2)[0]; // + await assertValidSnapshot({ snapshot, host, repo }); + const destination = await safeAbsPath(args.slice(-1)[0]); // + const options = parseAndValidateOptions( + args.slice(1, -2), + whitelist.restore, + ); + return await run([...options, snapshot, destination]); + } + case "find": { + const options = parseAndValidateOptions(args.slice(1), whitelist.find); + return await run([...options, "--filter-host", host]); + } + case "forget": { + if (args.length == 2 && !args[1].startsWith("-")) { + // delete exactly id + const snapshot = args[1]; + await assertValidSnapshot({ snapshot, host, repo }); + return await run([snapshot]); + } + // delete several defined by rules. + const options = parseAndValidateOptions(args.slice(1), whitelist.forget); + return await run([...options, "--filter-host", host]); + } + default: + throw Error(`subcommand not allowed: ${args[0]}`); + } +} + +const whitelist = { + backup: { + "--label": validate.str, + "--tag": validate.str, + "--description": validate.str, + "--time": validate.str, + "--delete-after": validate.str, + "--as-path": validate.str, + "--with-atime": true, + "--ignore-devid": true, + "--json": true, + "--long": true, + "--quiet": true, + "-h": true, + "--help": true, + "--glob": validate.str, + "--iglob": validate.str, + "--git-ignore": true, + "--no-require-git": true, + "-x": true, + "--one-file-system": true, + "--exclude-larger-than": validate.str, + }, + snapshots: { + "-g": validate.str, + "--group-by": validate.str, + "--long": true, + "--json": true, + "--all": true, + "-h": true, + "--help": true, + "--filter-label": validate.str, + "--filter-paths": validate.str, + "--filter-paths-exact": validate.str, + "--filter-after": validate.str, + "--filter-before": validate.str, + "--filter-size": validate.str, + "--filter-size-added": validate.str, + "--filter-jq": validate.str, + }, + restore: { + "--delete": true, + "--verify-existing": true, + "--recursive": true, + "-h": true, + "--help": true, + "--glob": validate.str, + "--iglob": validate.str, + }, + ls: { + "-s": true, + "--summary": true, + "-l": true, + "--long": true, + "--json": true, + "--recursive": true, + "-h": true, + "--help": true, + "--glob": validate.str, + "--iglob": validate.str, + }, + find: { + "--glob": validate.str, + "--iglob": validate.str, + "--path": validate.str, + "-g": validate.str, + "--group-by": validate.str, + "--all": true, + "--show-misses": true, + "-h": true, + "--help": true, + "--filter-label": validate.str, + "--filter-paths": validate.str, + "--filter-paths-exact": validate.str, + "--filter-after": validate.str, + "--filter-before": validate.str, + "--filter-size": validate.str, + "--filter-size-added": validate.str, + "--filter-jq": validate.str, + }, + forget: { + "--json": true, + "-g": validate.str, + "--group-by": validate.str, + "-h": true, + "--help": true, + "--filter-label": validate.str, + "--filter-paths": validate.str, + "--filter-paths-exact": validate.str, + "--filter-after": validate.str, + "--filter-before": validate.str, + "--filter-size": validate.str, + "--filter-size-added": validate.str, + "--filter-jq": validate.str, + "--keep-tags": validate.str, + "--keep-id": validate.str, + "-l": validate.int, + "--keep-last": validate.int, + "-M": validate.int, + "--keep-minutely": validate.int, + "-H": validate.int, + "--keep-hourly": validate.int, + "-d": validate.int, + "--keep-daily": validate.int, + "-w": validate.int, + "--keep-weekly": validate.int, + "-m": validate.int, + "--keep-monthly": validate.int, + "--keep-quarter-yearly": validate.int, + "--keep-half-yearly": validate.int, + "-y": validate.int, + "--keep-yearly": validate.int, + "--keep-within": validate.str, + "--keep-within-minutely": validate.str, + "--keep-within-hourly": validate.str, + "--keep-within-daily": validate.str, + "--keep-within-weekly": validate.str, + "--keep-within-monthly": validate.str, + "--keep-within-quarter-yearly": validate.str, + "--keep-within-half-yearly": validate.str, + "--keep-within-yearly": validate.str, + "--keep-none": validate.str, + }, +} as const; + +async function ensureInitialized(repo: string) { + if (repo.endsWith(".toml")) { + // nothing to do + return; + } + const config = join(repo, "config"); + if (!(await exists(config))) { + await exec({ + cmd: rusticPath, + safety: ["--no-progress", "--password", "", "-r", repo, "init"], + }); + } +} + +async function assertValidSnapshot({ snapshot, host, repo }) { + const id = snapshot.split(":")[0]; + if (id == "latest") { + // possible race condition so do not allow + throw Error("latest is not allowed"); + } + const actualHost = await getHost({ id, repo }); + if (actualHost != host) { + throw Error( + `host for snapshot with id ${id} must be '${host}' but it is ${actualHost}`, + ); + } +} + +// we do not allow changing host so this is safe to cache. +const hostCache = new LRU({ + max: 10000, +}); + +export async function getHost(opts) { + if (hostCache.has(opts.id)) { + return hostCache.get(opts.id); + } + const info = await getSnapshot(opts); + const hostname = info[0][1][0]["hostname"]; + hostCache.set(opts.id, hostname); + return hostname; +} + +export async function getSnapshot({ + id, + repo = rusticRepo, +}: { + id: string; + repo?: string; +}) { + const { stdout } = await exec({ + cmd: rusticPath, + safety: ["--password", "", "-r", repo, "snapshots", "--json", id], + }); + return JSON.parse(stdout.toString()); +} diff --git a/src/packages/backend/sandbox/sandbox.test.ts b/src/packages/backend/sandbox/sandbox.test.ts new file mode 100644 index 00000000000..b4eb78961d2 --- /dev/null +++ b/src/packages/backend/sandbox/sandbox.test.ts @@ -0,0 +1,257 @@ +import { SandboxedFilesystem } from "@cocalc/backend/sandbox"; +import { + mkdtemp, + mkdir, + rm, + readFile, + symlink, + writeFile, +} from "node:fs/promises"; +import { tmpdir } from "node:os"; +import { join } from "path"; + +let tempDir; +beforeAll(async () => { + tempDir = await mkdtemp(join(tmpdir(), "cocalc")); +}); + +describe("test using the filesystem sandbox to do a few standard things", () => { + let fs; + it("creates and reads file", async () => { + await mkdir(join(tempDir, "test-1")); + fs = new SandboxedFilesystem(join(tempDir, "test-1")); + await fs.writeFile("a", "hi"); + const r = await fs.readFile("a", "utf8"); + expect(r).toEqual("hi"); + expect(fs.unsafeMode).toBe(false); + }); + + it("truncate file", async () => { + await fs.writeFile("b", "hello"); + await fs.truncate("b", 4); + const r = await fs.readFile("b", "utf8"); + expect(r).toEqual("hell"); + }); +}); + +describe("make various attempts to break out of the sandbox", () => { + let fs; + it("creates sandbox", async () => { + await mkdir(join(tempDir, "test-2")); + fs = new SandboxedFilesystem(join(tempDir, "test-2")); + await fs.writeFile("x", "hi"); + }); + + it("obvious first attempt to escape fails", async () => { + const v = await fs.readdir(".."); + expect(v).toEqual(["x"]); + }); + + it("obvious first attempt to escape fails", async () => { + const v = await fs.readdir("a/../.."); + expect(v).toEqual(["x"]); + }); + + it("another attempt", async () => { + await fs.copyFile("/x", "/tmp"); + const v = await fs.readdir("a/../.."); + expect(v).toEqual(["tmp", "x"]); + + const r = await fs.readFile("tmp", "utf8"); + expect(r).toEqual("hi"); + }); +}); + +describe("test watching a file and a folder in the sandbox", () => { + let fs; + it("creates sandbox", async () => { + await mkdir(join(tempDir, "test-watch")); + fs = new SandboxedFilesystem(join(tempDir, "test-watch")); + await fs.writeFile("x", "hi"); + }); + + it("watches the file x for changes", async () => { + await fs.writeFile("x", "hi"); + const w = await fs.watch("x"); + await fs.appendFile("x", " there"); + const x = await w.next(); + expect(x).toEqual({ + value: { eventType: "change", filename: "x" }, + done: false, + }); + w.end(); + }); + + it("the maxQueue parameter limits the number of queue events", async () => { + await fs.writeFile("x", "hi"); + const w = await fs.watch("x", { maxQueue: 2 }); + expect(w.queueSize()).toBe(0); + // make many changes + await fs.appendFile("x", "0"); + await fs.appendFile("x", "0"); + await fs.appendFile("x", "0"); + await fs.appendFile("x", "0"); + // there will only be 2 available: + expect(w.queueSize()).toBe(2); + const x0 = await w.next(); + expect(x0).toEqual({ + value: { eventType: "change", filename: "x" }, + done: false, + }); + const x1 = await w.next(); + expect(x1).toEqual({ + value: { eventType: "change", filename: "x" }, + done: false, + }); + // one more next would hang... + expect(w.queueSize()).toBe(0); + w.end(); + }); + + it("maxQueue with overflow throw", async () => { + await fs.writeFile("x", "hi"); + const w = await fs.watch("x", { maxQueue: 2, overflow: "throw" }); + await fs.appendFile("x", "0"); + await fs.appendFile("x", "0"); + await fs.appendFile("x", "0"); + expect(async () => { + await w.next(); + }).rejects.toThrow("maxQueue overflow"); + w.end(); + }); + + it("AbortController works", async () => { + const ac = new AbortController(); + const { signal } = ac; + await fs.writeFile("x", "hi"); + const w = await fs.watch("x", { signal }); + await fs.appendFile("x", "0"); + const e = await w.next(); + expect(e.done).toBe(false); + + // now abort + ac.abort(); + const { done } = await w.next(); + expect(done).toBe(true); + }); + + it("watches a directory", async () => { + await fs.mkdir("folder"); + const w = await fs.watch("folder"); + + await fs.writeFile("folder/x", "hi"); + expect(await w.next()).toEqual({ + done: false, + value: { eventType: "rename", filename: "x" }, + }); + expect(await w.next()).toEqual({ + done: false, + value: { eventType: "change", filename: "x" }, + }); + + await fs.appendFile("folder/x", "xxx"); + expect(await w.next()).toEqual({ + done: false, + value: { eventType: "change", filename: "x" }, + }); + + await fs.writeFile("folder/z", "there"); + expect(await w.next()).toEqual({ + done: false, + value: { eventType: "rename", filename: "z" }, + }); + expect(await w.next()).toEqual({ + done: false, + value: { eventType: "change", filename: "z" }, + }); + + // this is correct -- from the node docs "On most platforms, 'rename' is emitted whenever a filename appears or disappears in the directory." + await fs.unlink("folder/z"); + expect(await w.next()).toEqual({ + done: false, + value: { eventType: "rename", filename: "z" }, + }); + }); +}); + +describe("unsafe mode sandbox", () => { + let fs; + it("creates and reads file", async () => { + await mkdir(join(tempDir, "test-unsafe")); + fs = new SandboxedFilesystem(join(tempDir, "test-unsafe"), { + unsafeMode: true, + }); + expect(fs.unsafeMode).toBe(true); + await fs.writeFile("a", "hi"); + const r = await fs.readFile("a", "utf8"); + expect(r).toEqual("hi"); + }); + + it("directly create a dangerous file that is a symlink outside of the sandbox -- this should work", async () => { + await writeFile(join(tempDir, "password"), "s3cr3t"); + await symlink( + join(tempDir, "password"), + join(tempDir, "test-unsafe", "danger"), + ); + const s = await readFile(join(tempDir, "test-unsafe", "danger"), "utf8"); + expect(s).toBe("s3cr3t"); + }); + + it("can **UNSAFELY** read the symlink content via the api", async () => { + expect(await fs.readFile("danger", "utf8")).toBe("s3cr3t"); + }); +}); + +describe("safe mode sandbox", () => { + let fs; + it("creates and reads file", async () => { + await mkdir(join(tempDir, "test-safe")); + fs = new SandboxedFilesystem(join(tempDir, "test-safe"), { + unsafeMode: false, + }); + expect(fs.unsafeMode).toBe(false); + expect(fs.readonly).toBe(false); + await fs.writeFile("a", "hi"); + const r = await fs.readFile("a", "utf8"); + expect(r).toEqual("hi"); + }); + + it("directly create a dangerous file that is a symlink outside of the sandbox -- this should work", async () => { + await writeFile(join(tempDir, "password"), "s3cr3t"); + await symlink( + join(tempDir, "password"), + join(tempDir, "test-safe", "danger"), + ); + const s = await readFile(join(tempDir, "test-safe", "danger"), "utf8"); + expect(s).toBe("s3cr3t"); + }); + + it("cannot read the symlink content via the api", async () => { + await expect(async () => { + await fs.readFile("danger", "utf8"); + }).rejects.toThrow("outside of sandbox"); + }); +}); + +describe("read only sandbox", () => { + let fs; + it("creates and reads file", async () => { + await mkdir(join(tempDir, "test-ro")); + fs = new SandboxedFilesystem(join(tempDir, "test-ro"), { + readonly: true, + }); + expect(fs.readonly).toBe(true); + await expect(async () => { + await fs.writeFile("a", "hi"); + }).rejects.toThrow("permission denied -- read only filesystem"); + try { + await fs.writeFile("a", "hi"); + } catch (err) { + expect(err.code).toEqual("EACCES"); + } + }); +}); + +afterAll(async () => { + await rm(tempDir, { force: true, recursive: true }); +}); diff --git a/src/packages/backend/sandbox/watch.ts b/src/packages/backend/sandbox/watch.ts new file mode 100644 index 00000000000..9fc9cb400af --- /dev/null +++ b/src/packages/backend/sandbox/watch.ts @@ -0,0 +1,140 @@ +import { readFile } from "node:fs/promises"; +import { watch as chokidarWatch } from "chokidar"; +import { EventIterator } from "@cocalc/util/event-iterator"; +import { type WatchOptions, type ChangeEvent } from "@cocalc/conat/files/watch"; +import { EventEmitter } from "events"; +import { make_patch } from "@cocalc/util/patch"; +import LRU from "lru-cache"; + +export { type WatchOptions }; +export type WatchIterator = EventIterator; + +// do NOT use patch for tracking file changes if the file exceeds +// this size. The reason is mainly because computing diffs of +// large files on the server can take a long time! +const MAX_PATCH_FILE_SIZE = 1_000_000; + +const log = (...args) => console.log(...args); +//const log = (..._args) => {}; + +export default function watch( + path: string, + options: WatchOptions, + lastOnDisk: LRU, +): WatchIterator { + log("watch", { path, options }); + const watcher = new Watcher(path, options, lastOnDisk); + + const iter = new EventIterator(watcher, "change", { + maxQueue: options.maxQueue ?? 2048, + overflow: options.overflow, + map: (args) => args[0], + onEnd: () => { + //log("close ", path); + watcher.close(); + }, + }); + return iter; +} + +class Watcher extends EventEmitter { + private watcher: ReturnType; + private ready: boolean = false; + + constructor( + private path: string, + private options: WatchOptions, + private lastOnDisk: LRU, + ) { + super(); + this.watcher = chokidarWatch(path, { + depth: 0, + ignoreInitial: true, + followSymlinks: false, + alwaysStat: options.stats ?? false, + atomic: true, + usePolling: false, + awaitWriteFinish: { + stabilityThreshold: 150, + pollInterval: 70, + }, + }); + log("creating watcher ", path, options); + + this.watcher.once("ready", () => { + this.ready = true; + }); + this.watcher.on("all", async (...args) => { + const change = await this.handle(...args); + if (change !== undefined) { + this.emit("change", change); + } + }); + } + + handle = async (event, path, stats): Promise => { + if (!this.ready) { + return; + } + let filename = path.slice(this.path.length); + if (filename.startsWith("/")) { + filename = filename.slice(1); + } + const x: ChangeEvent = { event, filename }; + if (this.options.stats) { + x.stats = stats; + } + if (this.options.closeOnUnlink && path == this.path) { + this.emit("change", x); + this.close(); + return; + } + if (!this.options.patch) { + log(path, "patch option not set", this.options); + return x; + } + + const last = this.lastOnDisk.get(path); + if (last === undefined) { + log(path, "lastOnDisk not set"); + return x; + } + let cur; + try { + cur = await readFile(path, "utf8"); + } catch (err) { + log(path, "read error", err); + return x; + } + if (last == cur) { + log(path, "no change"); + // no change + return; + } + this.lastOnDisk.set(path, cur); + if ( + cur.length >= MAX_PATCH_FILE_SIZE || + last.length >= MAX_PATCH_FILE_SIZE + ) { + // just inform that there is a change + log(path, "patch -- file too big (cur)"); + return x; + } + // small enough to make a patch + log(path, "making a patch with ", last.length, cur.length); + const t = Date.now(); + x.patch = make_patch(last, cur); + log(path, "made patch", Date.now() - t, x.patch); + return x; + }; + + close() { + this.watcher.close(); + this.emit("close"); + this.removeAllListeners(); + // @ts-ignore + delete this.watcher; + // @ts-ignore + delete this.ready; + } +} diff --git a/src/packages/backend/spool-watcher.ts b/src/packages/backend/spool-watcher.ts new file mode 100644 index 00000000000..eb2747ff169 --- /dev/null +++ b/src/packages/backend/spool-watcher.ts @@ -0,0 +1,194 @@ +import fs from "fs"; +import fsp from "fs/promises"; +import path from "path"; + +export type Message = Record; +export type Handler = (msg: Message, filePath: string) => Promise | void; + +function isJsonFile(name: string): boolean { + return name.endsWith(".json") && !name.startsWith("."); +} + +async function ensureDir(dir: string) { + await fsp.mkdir(dir, { mode: 0o700, recursive: true }); + // Best-effort chmod (in case dir already existed): + try { + await fsp.chmod(dir, 0o700); + } catch {} +} + +export class SpoolWatcher { + private readonly dir: string; + private readonly handle: Handler; + private watching = false; + private watcher?: fs.FSWatcher; + private scanScheduled = false; + private inFlight = new Set(); // basenames + private closed = false; + + constructor(dir: string, handle: Handler) { + this.dir = path.resolve(dir); + this.handle = handle; + } + + async start(): Promise { + await ensureDir(this.dir); + this.watching = true; + + // Process any preexisting messages first. + await this.scanAndQueue(); + + // Start watch; always rescan on any event (robustness over cleverness). + this.watcher = fs.watch(this.dir, { persistent: true }, () => { + if (!this.scanScheduled) { + this.scanScheduled = true; + // microtask-ish debounce to coalesce bursts + setTimeout(() => { + this.scanScheduled = false; + void this.scanAndQueue(); + }, 10); + } + }); + + this.watcher.on("error", () => { + // If the dir vanished temporarily, try to recreate and resume. + // Otherwise, surface/log as needed for your server. + // You can plug your logger here. + // console.error("Spool watcher error:", err); + void this.recover(); + }); + + this.watcher.on("close", () => { + if (!this.closed) { + // Unexpected close; try to restart + void this.recover(); + } + }); + } + + async close(): Promise { + this.closed = true; + this.watching = false; + try { + this.watcher?.close(); + } catch {} + this.inFlight.clear(); + } + + // --- internals --- + + private async recover() { + if (this.closed) return; + try { + await ensureDir(this.dir); + await this.scanAndQueue(); + // restart watcher + try { + this.watcher?.close(); + } catch {} + this.watcher = fs.watch(this.dir, { persistent: true }, () => { + if (!this.scanScheduled) { + this.scanScheduled = true; + setTimeout(() => { + this.scanScheduled = false; + void this.scanAndQueue(); + }, 10); + } + }); + } catch { + // Delay and retry + setTimeout(() => void this.recover(), 250); + } + } + + private async scanAndQueue(): Promise { + if (!this.watching) return; + let names: string[]; + try { + names = await fsp.readdir(this.dir); + } catch { + // dir may not exist momentarily + return; + } + + // Filter valid message files and sort oldest-first (by filename). + const files = names + .filter(isJsonFile) + .sort((a, b) => (a < b ? -1 : a > b ? 1 : 0)); + + for (const base of files) { + if (this.inFlight.has(base)) continue; + this.inFlight.add(base); + // process sequentially to keep ordering predictable per directory scan + // (if you want concurrency, you can remove await, but keep inFlight logic) + await this.processOne(base).catch(() => { + /* handled inside */ + }); + this.inFlight.delete(base); + } + } + + private async processOne(base: string): Promise { + const full = path.join(this.dir, base); + + // LSTAT to guard against symlinks/devices. + let st: fs.Stats; + try { + st = await fsp.lstat(full); + } catch (e: any) { + // vanished between readdir and now + if (e?.code === "ENOENT") return; + throw e; + } + if (!st.isFile()) { + // Ignore non-regular (and symlinks) + await this.safeUnlink(full); + return; + } + + // Read file contents. Writer should have used atomic rename, so content is complete. + let data: string; + try { + data = await fsp.readFile(full, "utf8"); + } catch (e: any) { + if (e?.code === "ENOENT") return; // raced with unlink + // transient read error; try once more shortly + await this.sleep(10); + try { + data = await fsp.readFile(full, "utf8"); + } catch { + await this.safeUnlink(full); + return; + } + } + + // Parse NDJSON: one JSON object per line. + const lines = data + .split(/\r?\n/) + .map((s) => s.trim()) + .filter(Boolean); + for (const line of lines) { + try { + const msg = JSON.parse(line) as Message; + await this.handle(msg, full); + } catch (e) { + // Move bad file aside or just skip the bad line; here we skip the line. + // If you want, rename to .bad for postmortem: + // await this.safeRename(full, full + ".bad"); + } + } + + // Remove after successful processing to avoid replay. + await this.safeUnlink(full); + } + + private async safeUnlink(p: string) { + try { + await fsp.unlink(p); + } catch {} + } + + private sleep(ms: number) { + return new Promise((res) => setTimeout(res, ms)); + } +} diff --git a/src/packages/backend/ssh/authorized-keys.ts b/src/packages/backend/ssh/authorized-keys.ts new file mode 100644 index 00000000000..a8b8b69f4c5 --- /dev/null +++ b/src/packages/backend/ssh/authorized-keys.ts @@ -0,0 +1,130 @@ +/* +Given an array keys of strings that contain SSH public keys and a path to an +authorized_keys file, this function: + +- opens the authorized_keys file (treats missing file as empty) + +- for every key that is in keys but missing in authorized_keys, it adds it with the line +"# Added by CoCalc" on the previous line. + +- for every key that is in authorized_keys that ALSO has previous line "# Added by CoCalc" +but is not in the array keys, it deletes it from the file. + +- it leaves everything else unchanged and writes the file back to disk. +*/ + +import { mkdir, readFile, writeFile } from "node:fs/promises"; +import { dirname } from "node:path"; + +const MARKER = "# Added by CoCalc"; + +/** Normalize a key line for comparison (trim + collapse whitespace). */ +function normalizeKeyLine(s: string): string { + return s.trim().replace(/\s+/g, " "); +} + +export async function updateAuthorizedKeys({ + keys, + path, +}: { + keys: string[]; + path: string; +}): Promise { + // Normalize input keys and de-duplicate + const desiredKeys = Array.from(new Set(keys.map((k) => normalizeKeyLine(k)))); + + // Read file (treat missing file as empty) + let text: string; + try { + text = await readFile(path, "utf8"); + } catch (err: any) { + if (err && err.code === "ENOENT") { + text = ""; + } else { + throw err; + } + } + + // Split into lines without losing empty tail information + // We'll re-join with '\n' and ensure a trailing newline at the end. + const lines = text.length ? text.split(/\r?\n/) : []; + + // Build a set of existing normalized key lines (anywhere in the file) + const existingKeySet = new Set(); + for (const line of lines) { + const norm = normalizeKeyLine(line); + if (norm && !norm.startsWith("#")) { + existingKeySet.add(norm); + } + } + + // Pass 1: Remove MARKER+key blocks where the key is no longer desired. + // We walk through lines and build a new array. + const result: string[] = []; + for (let i = 0; i < lines.length; i++) { + const line = lines[i]; + + if (normalizeKeyLine(line) === MARKER && i + 1 < lines.length) { + const keyLine = lines[i + 1]; + const keyNorm = normalizeKeyLine(keyLine); + + // If the next line (key) is NOT in desiredKeys, skip both lines. + // Otherwise, keep both. + if (!desiredKeys.includes(keyNorm)) { + i++; // skip key line as well + continue; // drop this pair + } else { + // Keep the marker and the key + result.push(line); + result.push(keyLine); + i++; // we already handled the key line + continue; + } + } + + // Default: keep the line + result.push(line); + } + + // Recompute existing key set after deletions (so additions logic is accurate) + const existingAfterDeletion = new Set(); + for (const line of result) { + const norm = normalizeKeyLine(line); + if (norm && !norm.startsWith("#")) { + existingAfterDeletion.add(norm); + } + } + + // Pass 2: Append any desired keys that are missing. + const additions: string[] = []; + for (const key of desiredKeys) { + if (!existingAfterDeletion.has(key)) { + additions.push(MARKER, key); + existingAfterDeletion.add(key); // avoid double-adding if duplicates in input + } + } + + // If we’re adding, ensure there’s a separating blank line if the file + // already has content and doesn’t end with a blank line. + if (additions.length > 0) { + if ( + result.length > 0 && + normalizeKeyLine(result[result.length - 1]) !== "" + ) { + result.push(""); // add a blank line for readability + } + result.push(...additions); + } + + // Ensure the file ends with a single trailing newline + let out = result.join("\n"); + if (!out.endsWith("\n")) out += "\n"; + + if (out.trim().length == 0 && text.trim().length == 0) { + // if no keys, do not create file + return ""; + } + await mkdir(dirname(path), { recursive: true, mode: 0o700 }); + await writeFile(path, out, { encoding: "utf8", mode: 0o600 }); + return out; +} diff --git a/src/packages/backend/ssh/ssh-keys.ts b/src/packages/backend/ssh/ssh-keys.ts new file mode 100644 index 00000000000..77f711864e2 --- /dev/null +++ b/src/packages/backend/ssh/ssh-keys.ts @@ -0,0 +1,80 @@ +import ssh from "micro-key-producer/ssh.js"; +import { randomBytes } from "micro-key-producer/utils.js"; +import { mkdir, readFile, writeFile } from "node:fs/promises"; +import { dirname, join } from "path"; +import { existsSync } from "node:fs"; +import { type SshServer } from "@cocalc/conat/project/runner/types"; +import { SSH_IDENTITY_FILE } from "@cocalc/conat/project/runner/constants"; +import getLogger from "@cocalc/backend/logger"; + +const logger = getLogger("backend:ssh-keys"); + +function files(home = process.env.HOME) { + if (!home) { + throw Error("home must be specified"); + } + const privateFile = join(home, SSH_IDENTITY_FILE); + const publicFile = privateFile + ".pub"; + return { privateFile, publicFile }; +} + +export async function initSshKeys({ + home = process.env.HOME, + sshServers = [], +}: { home?: string; sshServers?: SshServer[] } = {}) { + const { privateFile, publicFile } = files(home); + if (!existsSync(privateFile)) { + logger.debug(`creating ${privateFile}`); + const seed = randomBytes(32); + const { privateKey, publicKey } = ssh(seed, "root"); + await mkdir(dirname(privateFile), { recursive: true, mode: 0o700 }); + await writeFile(privateFile, privateKey, { mode: 0o600 }); + await writeFile(publicFile, publicKey, { mode: 0o600 }); + } + + let cocalcHostConfig = ""; + for (const { name, host, port, user } of sshServers) { + // We need "UpdateHostKeys no" because otherwise we see + // client_global_hostkeys_prove_confirm: server gave bad signature for RSA key 0: incorrect signature" + // due to our sshpiperd proxy. + cocalcHostConfig += ` + +# Added by CoCalc +Host ${name} + User ${user} + HostName ${host} + Port ${port} + StrictHostKeyChecking no + UpdateHostKeys no + IdentityFile ~/${SSH_IDENTITY_FILE} +`; + } + + await mkdir(join(home!, ".ssh", ".cocalc"), { recursive: true, mode: 0o700 }); + const cocalcConfigPath = join(home!, ".ssh", ".cocalc", "config"); + await writeFile(cocalcConfigPath, cocalcHostConfig, { mode: 0o600 }); + + const configPath = join(home!, ".ssh", "config"); + let config; + try { + config = await readFile(configPath, "utf8"); + } catch { + config = ""; + } + if (!config.includes("Include .cocalc/config")) { + // put at front since only the first with a given name is used by ssh + logger.debug("updating .ssh/config to include custom host config"); + await writeFile(configPath, "Include .cocalc/config\n\n" + config, { + mode: 0o600, + }); + } +} +export async function sshPublicKey(): Promise { + const { publicFile } = files(); + try { + return await readFile(publicFile, "utf8"); + } catch { + await initSshKeys(); + return await readFile(publicFile, "utf8"); + } +} diff --git a/src/packages/conat/compute/manager.ts b/src/packages/conat/compute/manager.ts index 4f21918d80a..12d110ba4ae 100644 --- a/src/packages/conat/compute/manager.ts +++ b/src/packages/conat/compute/manager.ts @@ -5,7 +5,7 @@ is used to edit a given file. Access this in the browser for the project you have open: -> m = await cc.client.conat_client.computeServerManager({project_id:cc.current().project_id}) +m = cc.redux.getProjectActions(cc.current().project_id).computeServerManager */ diff --git a/src/packages/conat/core/client.ts b/src/packages/conat/core/client.ts index cfe8480a1ea..34615d676bc 100644 --- a/src/packages/conat/core/client.ts +++ b/src/packages/conat/core/client.ts @@ -228,7 +228,10 @@ import { EventEmitter } from "events"; import { isValidSubject, isValidSubjectWithoutWildcards, + ConatError, + headerToError, } from "@cocalc/conat/util"; +export { ConatError, headerToError }; import { reuseInFlight } from "@cocalc/util/reuse-in-flight"; import { once, until } from "@cocalc/util/async-utils"; import { delay } from "awaiting"; @@ -244,6 +247,17 @@ import { } from "@cocalc/conat/sync/dstream"; import { akv, type AKV } from "@cocalc/conat/sync/akv"; import { astream, type AStream } from "@cocalc/conat/sync/astream"; +import { + syncstring, + type SyncString, + type SyncStringOptions, +} from "@cocalc/conat/sync-doc/syncstring"; +import { + syncdb, + type SyncDB, + type SyncDBOptions, +} from "@cocalc/conat/sync-doc/syncdb"; +import { fsClient, fsSubject } from "@cocalc/conat/files/fs"; import TTL from "@isaacs/ttlcache"; import { ConatSocketServer, @@ -257,11 +271,13 @@ import { type ConatSyncTable, createSyncTable, } from "@cocalc/conat/sync/synctable"; +import mutagen from "@cocalc/conat/project/mutagen"; export const MAX_INTEREST_TIMEOUT = 90_000; const DEFAULT_WAIT_FOR_INTEREST_TIMEOUT = 30_000; +// WARNING: do NOT change MSGPACK_ENCODER_OPTIONS unless you know what you're doing! const MSGPACK_ENCODER_OPTIONS = { // ignoreUndefined is critical so database queries work properly, and // also we have a lot of api calls with tons of wasted undefined values. @@ -471,7 +487,7 @@ export class Client extends EventEmitter { if (!options.address) { if (!process.env.CONAT_SERVER) { throw Error( - "Must specificy address or set CONAT_SERVER environment variable", + "Must specify address or set CONAT_SERVER environment variable", ); } options = { ...options, address: process.env.CONAT_SERVER }; @@ -516,6 +532,9 @@ export class Client extends EventEmitter { } const firstTime = this.info == null; this.info = info; + if (firstTime) { + this.initInbox(); + } this.emit("info", info); setTimeout(this.syncSubscriptions, firstTime ? 3000 : 0); }); @@ -544,7 +563,6 @@ export class Client extends EventEmitter { this.disconnectAllSockets(); }); this.conn.io.connect(); - this.initInbox(); this.statsLoop(); } @@ -561,6 +579,14 @@ export class Client extends EventEmitter { setTimeout(() => this.conn.io.disconnect(), 1); }; + connect = () => { + this.conn.io.connect(); + }; + + isConnected = () => this.state == "connected"; + + isSignedIn = () => !!(this.info?.user && !this.info?.user?.error); + // this has NO timeout by default waitUntilSignedIn = reuseInFlight( async ({ timeout }: { timeout?: number } = {}) => { @@ -581,7 +607,7 @@ export class Client extends EventEmitter { this.state != "connected" || this.info?.user?.error ) { - throw Error("failed to sign in"); + throw Error(`failed to sign in - ${this.info?.user?.error}`); } }, ); @@ -629,7 +655,7 @@ export class Client extends EventEmitter { .emitWithAck("wait-for-interest", { subject, timeout }); return response; } catch (err) { - throw toConatError(err); + throw toConatError(err, { subject }); } }; @@ -683,6 +709,13 @@ export class Client extends EventEmitter { return this.inbox; }); + // if inboxPrefixHook is set, it will be called with the sign-in + // info, and what it retujrns will be used as the inboxPrefix, + // instead of using this.options.inboxPrefix. This is useful because + // the inbox prefix you might want to use could depend on your + // identity wrt a remote server (example: a project api key knows + // the project_id but the client might not). + public inboxPrefixHook?: (info: ServerInfo | undefined) => string | undefined; private initInbox = async () => { // For request/respond instead of setting up one // inbox *every time there is a request*, we setup a single @@ -697,7 +730,10 @@ export class Client extends EventEmitter { // multiple servers solving the race condition would slow everything down // due to having to wait for so many acknowledgements. Instead, we // remove all those problems by just using a single inbox subscription. - const inboxPrefix = this.options.inboxPrefix ?? INBOX_PREFIX; + const inboxPrefix = + this.inboxPrefixHook?.(this.info) ?? + this.options.inboxPrefix ?? + INBOX_PREFIX; if (!inboxPrefix.startsWith(INBOX_PREFIX)) { throw Error(`custom inboxPrefix must start with '${INBOX_PREFIX}'`); } @@ -721,7 +757,7 @@ export class Client extends EventEmitter { } return false; }, - { start: 1000, max: 15000 }, + { start: 3000, max: 30000 }, ); if (this.isClosed()) { return; @@ -974,7 +1010,7 @@ export class Client extends EventEmitter { }); } } catch (err) { - throw toConatError(err); + throw toConatError(err, { subject }); } if (response?.error) { throw new ConatError(response.error, { code: response.code }); @@ -1106,9 +1142,22 @@ export class Client extends EventEmitter { // good for services. await mesg.respond(result); } catch (err) { + let error = err.message; + if (!error) { + error = `${err}`.slice("Error: ".length); + } await mesg.respond(null, { - noThrow: true, // we're not catching this one - headers: { error: `${err}` }, + noThrow: true, // we're not catching this respond + headers: { + error, + error_attrs: { + code: err.code, + errno: err.errno, + path: err.path, + syscall: err.syscall, + subject: err.subject, + }, + }, }); } }; @@ -1126,18 +1175,18 @@ export class Client extends EventEmitter { call(subject: string, opts?: PublishOptions): T { const call = async (name: string, args: any[]) => { const resp = await this.request(subject, [name, args], opts); - if (resp.headers?.error) { - throw Error(`${resp.headers.error}`); - } else { - return resp.data; - } + return resp.data; }; return new Proxy( - {}, + { subject }, { - get: (_, name) => { - if (typeof name !== "string") { + get: (target, name) => { + const s = target[String(name)]; + if (s !== undefined) { + return s; + } + if (typeof name !== "string" || name == "then") { return undefined; } return async (...args) => await call(name, args); @@ -1157,7 +1206,7 @@ export class Client extends EventEmitter { for await (const resp of sub) { if (resp.headers?.error) { yield new ConatError(`${resp.headers.error}`, { - code: resp.headers.code, + code: resp.headers.code as string | number, }); } else { yield resp.data; @@ -1327,7 +1376,7 @@ export class Client extends EventEmitter { return response; } } catch (err) { - throw toConatError(err); + throw toConatError(err, { subject }); } } else { return await this.conn.emitWithAck("publish", v); @@ -1364,7 +1413,11 @@ export class Client extends EventEmitter { request = async ( subject: string, mesg: any, - { timeout = DEFAULT_REQUEST_TIMEOUT, ...options }: PublishOptions = {}, + { + timeout = DEFAULT_REQUEST_TIMEOUT, + ignoreErrorHeader, + ...options + }: PublishOptions & { ignoreErrorHeader?: boolean } = {}, ): Promise => { if (timeout <= 0) { throw Error("timeout must be positive"); @@ -1392,6 +1445,9 @@ export class Client extends EventEmitter { } for await (const resp of sub) { sub.stop(); + if (!ignoreErrorHeader && resp.headers?.error) { + throw headerToError(resp.headers); + } return resp; } sub.stop(); @@ -1454,6 +1510,17 @@ export class Client extends EventEmitter { return sub; }; + fs = (opts: { + project_id: string; + compute_server_id?: number; + service?: string; + }) => { + return fsClient({ + subject: fsSubject(opts), + client: this, + }); + }; + sync = { dkv: async (opts: DKVOptions): Promise> => await dkv({ ...opts, client: this }), @@ -1466,6 +1533,17 @@ export class Client extends EventEmitter { astream({ ...opts, client: this }), synctable: async (opts: SyncTableOptions): Promise => await createSyncTable({ ...opts, client: this }), + string: (opts: Omit, "fs">): SyncString => + syncstring({ ...opts, client: this }), + db: (opts: Omit, "fs">): SyncDB => + syncdb({ ...opts, client: this }), + mutagen: ({ + project_id, + compute_server_id = 0, + }: { + project_id: string; + compute_server_id?: number; + }) => mutagen({ client: this, project_id, compute_server_id }), }; socket = { @@ -1921,14 +1999,6 @@ export function messageData( export type Subscription = EventIterator; -export class ConatError extends Error { - code: string | number; - constructor(mesg: string, { code }) { - super(mesg); - this.code = code; - } -} - function isEmpty(obj: object): boolean { for (const _x in obj) { return false; @@ -1936,14 +2006,18 @@ function isEmpty(obj: object): boolean { return true; } -function toConatError(socketIoError) { +function toConatError(socketIoError, { subject }: { subject?: string } = {}) { // only errors are "disconnected" and a timeout const e = `${socketIoError}`; if (e.includes("disconnected")) { return e; } else { - return new ConatError(`timeout - ${e}`, { - code: 408, - }); + return new ConatError( + `timeout - ${e}${subject ? " subject:" + subject : ""}`, + { + code: 408, + subject, + }, + ); } } diff --git a/src/packages/conat/core/cluster.ts b/src/packages/conat/core/cluster.ts index fbaaedad0e5..c14785b2008 100644 --- a/src/packages/conat/core/cluster.ts +++ b/src/packages/conat/core/cluster.ts @@ -7,7 +7,6 @@ import { type StickyUpdate, } from "@cocalc/conat/core/server"; import type { DStream } from "@cocalc/conat/sync/dstream"; -import { once } from "@cocalc/util/async-utils"; import { server as createPersistServer } from "@cocalc/conat/persist/server"; import { getLogger } from "@cocalc/conat/client"; import { hash_string } from "@cocalc/util/misc"; @@ -165,7 +164,7 @@ class ClusterLink { if (Date.now() - start >= timeout) { throw Error("timeout"); } - await once(this.interest, "change"); + await this.interest.waitForChange(); if ((this.state as any) == "closed" || signal?.aborted) { return false; } diff --git a/src/packages/conat/core/patterns.ts b/src/packages/conat/core/patterns.ts index 79eada9e5e1..708df2afdef 100644 --- a/src/packages/conat/core/patterns.ts +++ b/src/packages/conat/core/patterns.ts @@ -2,6 +2,8 @@ import { isEqual } from "lodash"; import { getLogger } from "@cocalc/conat/client"; import { EventEmitter } from "events"; import { hash_string } from "@cocalc/util/misc"; +import { reuseInFlight } from "@cocalc/util/reuse-in-flight"; +import { once } from "@cocalc/util/async-utils"; type Index = { [pattern: string]: Index | string }; @@ -13,9 +15,14 @@ export class Patterns extends EventEmitter { constructor() { super(); - this.setMaxListeners(1000); + this.setMaxListeners(100); } + // wait until one single change event fires. Throws an error if this gets closed first. + waitForChange = reuseInFlight(async (timeout?) => { + await once(this, "change", timeout); + }); + close = () => { this.emit("closed"); this.patterns = {}; diff --git a/src/packages/conat/core/server.ts b/src/packages/conat/core/server.ts index dc87dcf1911..f2705621c07 100644 --- a/src/packages/conat/core/server.ts +++ b/src/packages/conat/core/server.ts @@ -53,7 +53,7 @@ import { import { Patterns } from "./patterns"; import { is_array } from "@cocalc/util/misc"; import { UsageMonitor } from "@cocalc/conat/monitor/usage"; -import { once, until } from "@cocalc/util/async-utils"; +import { until } from "@cocalc/util/async-utils"; import { clusterLink, type ClusterLink, @@ -638,6 +638,9 @@ export class ConatServer extends EventEmitter { // a subscriber before the socket is actually getting messages. await socket.join(room); await this.updateInterest({ op: "add", subject, room, queue }); + if (DEBUG) { + this.log("subscribe - succeeded ", { id: socket.id, subject, queue, room }); + } }; // get all interest in this subject across the cluster (NOT supercluster) @@ -1703,8 +1706,8 @@ export class ConatServer extends EventEmitter { throw Error("timeout"); } try { - // if signal is set only wait for the change for up to 1 second. - await once(this.interest, "change", signal != null ? 1000 : undefined); + // if signal is set, only wait for the change for up to 1 second. + await this.interest.waitForChange(signal != null ? 1000 : undefined); } catch { continue; } @@ -1739,7 +1742,7 @@ function socketSubjectRoom({ socket, subject }) { return JSON.stringify({ id: socket.id, subject }); } -export function randomChoice(v: Set): string { +export function randomChoice(v: Set): T { if (v.size == 0) { throw Error("v must have size at least 1"); } diff --git a/src/packages/hub/proxy/file-download.ts b/src/packages/conat/files/file-download.ts similarity index 60% rename from src/packages/hub/proxy/file-download.ts rename to src/packages/conat/files/file-download.ts index 992cbf96ddd..94037d033bc 100644 --- a/src/packages/hub/proxy/file-download.ts +++ b/src/packages/conat/files/file-download.ts @@ -1,28 +1,55 @@ -import { readFile as readProjectFile } from "@cocalc/conat/files/read"; +import { readFile } from "./read"; import { once } from "events"; import { path_split } from "@cocalc/util/misc"; +import { getLogger } from "@cocalc/conat/client"; +import { type Client as ConatClient } from "@cocalc/conat/core/client"; import mime from "mime-types"; -import getLogger from "../logger"; const DANGEROUS_CONTENT_TYPE = new Set(["image/svg+xml" /*, "text/html"*/]); -const logger = getLogger("hub:proxy:file-download"); +const logger = getLogger("conat:file-download"); // assumes request has already been authenticated! -export async function handleFileDownload(req, res, url, project_id) { - logger.debug("handling the request via conat file streaming", url); +export async function handleFileDownload({ + req, + res, + url, + allowUnsafe, + client, + // allow a long download time (1 hour), since files can be large and + // networks can be slow. + maxWait = 1000 * 60 * 60, +}: { + req; + res; + url?: string; + allowUnsafe?: boolean; + client?: ConatClient; + maxWait?: number; +}) { + url ??= req.url; + logger.debug("downloading file from project to browser", url); + if (!url) { + res.statusCode = 500; + res.end("Invalid URL"); + return; + } const i = url.indexOf("files/"); - const compute_server_id = req.query.id ?? 0; + const compute_server_id = parseInt(req.query.id ?? "0"); let j = url.lastIndexOf("?"); if (j == -1) { j = url.length; } const path = decodeURIComponent(url.slice(i + "files/".length, j)); + const project_id = url.split("/").slice(1)[0]; logger.debug("conat: get file", { project_id, path, compute_server_id, url }); const fileName = path_split(path).tail; const contentType = mime.lookup(fileName); - if (req.query.download != null || DANGEROUS_CONTENT_TYPE.has(contentType)) { + if ( + req.query.download != null || + (!allowUnsafe && DANGEROUS_CONTENT_TYPE.has(contentType)) + ) { const fileNameEncoded = encodeURIComponent(fileName) .replace(/['()]/g, escape) .replace(/\*/g, "%2A"); @@ -38,13 +65,12 @@ export async function handleFileDownload(req, res, url, project_id) { headersSent = true; }); try { - for await (const chunk of await readProjectFile({ + for await (const chunk of await readFile({ + client, project_id, compute_server_id, path, - // allow a long download time (1 hour), since files can be large and - // networks can be slow. - maxWait: 1000 * 60 * 60, + maxWait, })) { if (res.writableEnded || res.destroyed) { break; diff --git a/src/packages/conat/files/file-server.ts b/src/packages/conat/files/file-server.ts new file mode 100644 index 00000000000..b05243ba2a2 --- /dev/null +++ b/src/packages/conat/files/file-server.ts @@ -0,0 +1,188 @@ +/* +File server - manages where projects are stored. + +This is a conat service that runs directly on the btrfs file server. +Only admin processes (hubs) can talk directly to it, not normal users. +It handles: + +Core Functionality: + + - creating volume where a project's files are stored + - from scratch, and as a zero-cost clone of an existing project + - copy files between distinct volumes (with btrfs this is done via + highly efficient dedup'd cloning). + +Additional functionality: + - set a quota on project volume + - delete volume + - create snapshot + - update snapshots + - create backup + +*/ + +import { type Client } from "@cocalc/conat/core/client"; +import { conat } from "@cocalc/conat/client"; +import { type SnapshotCounts } from "@cocalc/util/consts/snapshots"; +import { type CopyOptions } from "./fs"; +export { type CopyOptions }; +import { type MutagenSyncSession } from "@cocalc/conat/project/mutagen/types"; + +const SUBJECT = "file-server"; + +export interface Sync { + // {volume-name}:path/into/volume + src: string; + dest: string; + + // if true, dest is kept as an exact copy of src + // and any changes to dest are immediately reverted; + // basically, dest acts as a read-only copy of src. + replica?: boolean; +} + +export interface Fileserver { + mount: (opts: { project_id: string }) => Promise<{ path: string }>; + + // create project_id as an exact lightweight clone of src_project_id + clone: (opts: { + project_id: string; + src_project_id: string; + }) => Promise; + + getUsage: (opts: { project_id: string }) => Promise<{ + size: number; + used: number; + free: number; + }>; + + getQuota: (opts: { project_id: string }) => Promise<{ + size: number; + used: number; + }>; + + setQuota: (opts: { + project_id: string; + size: number | string; + }) => Promise; + + cp: (opts: { + src: { project_id: string; path: string | string[] }; + dest: { project_id: string; path: string }; + options?: CopyOptions; + }) => Promise; + + ///////////// + // Sync + // Automated realtime bidirectional sync of files between a path + // in one project with a path in another project. + // It's bidirectional, but conflicts always resolve in favor + // of the source. + ///////////// + createSync: (sync: Sync & { ignores?: string[] }) => Promise; + // list all sync links with src or dest the given volume + getAllSyncs: (opts: { + name: string; + }) => Promise<(MutagenSyncSession & Sync)[]>; + getSync: (sync: Sync) => Promise; + syncCommand: ( + command: "flush" | "reset" | "pause" | "resume" | "terminate", + sync: Sync, + ) => Promise<{ stdout: string; stderr: string; exit_code: number }>; + + ///////////// + // BACKUPS + ///////////// + + // create new complete backup of the project; this first snapshots the + // project, makes a backup of the snapshot, then deletes the snapshot, so the + // backup is guranteed to be consistent. + createBackup: (opts: { + project_id: string; + limit?: number; + }) => Promise<{ time: Date; id: string }>; + // restore the given path in the backup to the given dest. The default + // path is '' (the whole project) and the default destination is the + // same as the path. + restoreBackup: (opts: { + project_id: string; + id: string; + path?: string; + dest?: string; + }) => Promise; + // delete the given backup + deleteBackup: (opts: { project_id: string; id: string }) => Promise; + // Return list of id's and timestamps of all backups of this project. + updateBackups: (opts: { + project_id: string; + counts?: Partial; + // global limit, same as with createBackup above; can prevent new backups from being + // made if counts are too large! + limit?: number; + }) => Promise; + getBackups: (opts: { project_id: string }) => Promise< + { + id: string; + time: Date; + }[] + >; + + // Return list of all files in the given backup. + // TODO: would be nice to filter path, since there could be millions of files (?). + getBackupFiles: (opts: { + project_id: string; + id: string; + }) => Promise; + + ///////////// + // SNAPSHOTS + ///////////// + createSnapshot: (opts: { + project_id: string; + name?: string; + // if given, throw error if there are already limit snapshots, i.e., this is a hard limit on + // the total number of snapshots (to avoid abuse/bugs). + limit?: number; + }) => Promise; + deleteSnapshot: (opts: { project_id: string; name: string }) => Promise; + updateSnapshots: (opts: { + project_id: string; + counts?: Partial; + // global limit, same as with createSnapshot above; can prevent new snapshots from being + // made if counts are too large! + limit?: number; + }) => Promise; + allSnapshotUsage: (opts: { project_id: string }) => Promise; +} + +export interface SnapshotUsage { + // name of this snapshot + name: string; + // amount of space used by this snapshot in bytes + used: number; + // amount of space that would be freed by deleting this snapshot + exclusive: number; + // total quota in bytes across all snapshot + quota: number; +} + +export interface Options extends Fileserver { + client?: Client; +} + +export async function server({ client, ...impl }: Options) { + client ??= conat(); + + const sub = await client.service(SUBJECT, impl); + + return { + close: () => { + sub.close(); + }, + }; +} + +export function client({ client }: { client?: Client } = {}): Fileserver { + client ??= conat(); + return client.call(SUBJECT); +} diff --git a/src/packages/conat/files/fs.ts b/src/packages/conat/files/fs.ts new file mode 100644 index 00000000000..ef54d4f652e --- /dev/null +++ b/src/packages/conat/files/fs.ts @@ -0,0 +1,621 @@ +/* +Tests are in + +packages/backend/conat/files/test/local-path.test.ts + +*/ + +import { type Client } from "@cocalc/conat/core/client"; +import { conat } from "@cocalc/conat/client"; +import { + watchServer, + watchClient, + type WatchIterator, +} from "@cocalc/conat/files/watch"; +import listing, { + type Listing, + type FileTypeLabel, + type Files, +} from "./listing"; +import { isValidUUID } from "@cocalc/util/misc"; +import { reuseInFlight } from "@cocalc/util/reuse-in-flight"; +import TTL from "@isaacs/ttlcache"; +import { getLogger } from "@cocalc/conat/client"; + +const logger = getLogger("files:fs"); + +export const DEFAULT_FILE_SERVICE = "fs"; + +export interface ExecOutput { + stdout: Buffer; + stderr: Buffer; + code: number | null; + // true if terminated early due to output size or time + truncated?: boolean; +} + +export interface RipgrepOptions { + options?: string[]; + darwin?: string[]; + linux?: string[]; + timeout?: number; + maxSize?: number; +} + +export interface FindOptions { + timeout?: number; + // all safe whitelisted options to the find command + options?: string[]; + darwin?: string[]; + linux?: string[]; + maxSize?: number; +} + +export interface FdOptions { + pattern?: string; + options?: string[]; + darwin?: string[]; + linux?: string[]; + timeout?: number; + maxSize?: number; +} + +export interface DustOptions { + options?: string[]; + darwin?: string[]; + linux?: string[]; + timeout?: number; + maxSize?: number; +} + +export interface OuchOptions { + cwd?: string; + options?: string[]; + timeout?: number; +} + +export const OUCH_FORMATS = [ + "zip", + "7z", + "tar.gz", + "tar.xz", + "tar.bz", + "tar.bz2", + "tar.bz3", + "tar.lz4", + "tar.sz", + "tar.zst", + "tar.br", +]; + +export interface CopyOptions { + dereference?: boolean; + errorOnExist?: boolean; + force?: boolean; + preserveTimestamps?: boolean; + recursive?: boolean; + verbatimSymlinks?: boolean; + // if true, will try to use copy-on-write - this spawns the operating system '/usr/bin/cp' command. + reflink?: boolean; + // when using /usr/bin/cp: + timeout?: number; +} + +export interface Filesystem { + appendFile: (path: string, data: string | Buffer, encoding?) => Promise; + chmod: (path: string, mode: string | number) => Promise; + constants: () => Promise<{ [key: string]: number }>; + copyFile: (src: string, dest: string) => Promise; + + cp: ( + // NOTE!: we also support any array of src's unlike node's cp; + // however, when src is an array, the target *must* be a directory and this works like + // /usr/bin/cp, where files are copied INTO that target. + // When src is a string, this is just normal node cp behavior. + src: string | string[], + dest: string, + options?: CopyOptions, + ) => Promise; + exists: (path: string) => Promise; + link: (existingPath: string, newPath: string) => Promise; + lstat: (path: string) => Promise; + mkdir: (path: string, options?) => Promise; + + // move from fs-extra -- https://github.com/jprichardson/node-fs-extra/blob/HEAD/docs/move.md + move: ( + src: string | string[], + dest: string, + options?: { overwrite?: boolean }, + ) => Promise; + + readFile: ( + path: string, + encoding?: string, + + // If lock is given then any other client that tries to read from this + // for lock ms after the lock is created will get an exception with code='LOCK'. + // This is an extension to node's fs.readFile that is very useful when + // initializing realtime sync clients. It makes it so we can have several + // clients all try to read at the same time, and exactly one wins. + lock?: number, + ) => Promise; + // lockFile is exactly like readFile with the lock parameter, but + // it lets you lock (or unlock) a file without actually reading it. + lockFile: (path: string, lock?: number) => Promise; + readdir(path: string, options?): Promise; + readdir(path: string, options: { withFileTypes?: false }): Promise; + readdir(path: string, options: { withFileTypes: true }): Promise; + readlink: (path: string) => Promise; + realpath: (path: string) => Promise; + rename: (oldPath: string, newPath: string) => Promise; + rm: (path: string | string[], options?) => Promise; + rmdir: (path: string, options?) => Promise; + stat: (path: string) => Promise; + symlink: (target: string, path: string) => Promise; + truncate: (path: string, len?: number) => Promise; + unlink: (path: string) => Promise; + utimes: ( + path: string, + atime: number | string | Date, + mtime: number | string | Date, + ) => Promise; + // for lock see docs for readFile above + writeFile: ( + path: string, + data: string | Buffer, + saveLast?: boolean, + ) => Promise; + // todo: typing + watch: (path: string, options?) => Promise; + + // compression + ouch: (args: string[], options?: OuchOptions) => Promise; + + // We add very little to the Filesystem api, but we have to add + // a sandboxed "find" command, since it is a 1-call way to get + // arbitrary directory listing info, which is just not possible + // with the fs API, but required in any serious application. + // find -P {path} -maxdepth 1 -mindepth 1 -printf {printf} + // For security reasons, this does not support all find arguments, + // and can only use limited resources. + find: (path: string, options?: FindOptions) => Promise; + getListing: (path: string) => Promise<{ files: Files; truncated?: boolean }>; + + // Convenience function that uses the find and stat support to + // provide all files in a directory by using tricky options to find, + // and ensuring they are used by stat in a consistent way for updates. + listing?: (path: string) => Promise; + + // fd is a rust rewrite of find that is extremely fast at finding + // files that match an expression. + fd: (path: string, options?: FdOptions) => Promise; + + // dust is an amazing disk space tool + dust: (path: string, options?: DustOptions) => Promise; + + // We add ripgrep, as a 1-call way to very efficiently search in files + // directly on whatever is serving files. + // For security reasons, this does not support all ripgrep arguments, + // and can only use limited resources. + ripgrep: ( + path: string, + pattern: string, + options?: RipgrepOptions, + ) => Promise; + + rustic: (args: string[]) => Promise; +} + +interface IDirent { + name: string; + parentPath: string; + path: string; + type?: number; +} + +const DIRENT_TYPES = { + 0: "UNKNOWN", + 1: "FILE", + 2: "DIR", + 3: "LINK", + 4: "FIFO", + 5: "SOCKET", + 6: "CHAR", + 7: "BLOCK", +}; + +class Dirent { + constructor( + public name: string, + public parentPath: string, + public path: string, + public type: number, + ) {} + isFile = () => DIRENT_TYPES[this.type] == "FILE"; + isDirectory = () => DIRENT_TYPES[this.type] == "DIR"; + isSymbolicLink = () => DIRENT_TYPES[this.type] == "LINK"; + isFIFO = () => DIRENT_TYPES[this.type] == "FIFO"; + isSocket = () => DIRENT_TYPES[this.type] == "SOCKET"; + isCharacterDevice = () => DIRENT_TYPES[this.type] == "CHAR"; + isBlockDevice = () => DIRENT_TYPES[this.type] == "BLOCK"; +} + +interface IStats { + dev: number; + ino: number; + mode: number; + nlink: number; + uid: number; + gid: number; + rdev: number; + size: number; + blksize: number; + blocks: number; + atimeMs: number; + mtimeMs: number; + ctimeMs: number; + birthtimeMs: number; + atime: Date; + mtime: Date; + ctime: Date; + birthtime: Date; +} + +export class Stats { + dev: number; + ino: number; + mode: number; + nlink: number; + uid: number; + gid: number; + rdev: number; + size: number; + blksize: number; + blocks: number; + atimeMs: number; + mtimeMs: number; + ctimeMs: number; + birthtimeMs: number; + atime: Date; + mtime: Date; + ctime: Date; + birthtime: Date; + + constructor(private constants: { [key: string]: number }) {} + + isSymbolicLink = () => + (this.mode & this.constants.S_IFMT) === this.constants.S_IFLNK; + + isFile = () => (this.mode & this.constants.S_IFMT) === this.constants.S_IFREG; + + isDirectory = () => + (this.mode & this.constants.S_IFMT) === this.constants.S_IFDIR; + + isBlockDevice = () => + (this.mode & this.constants.S_IFMT) === this.constants.S_IFBLK; + + isCharacterDevice = () => + (this.mode & this.constants.S_IFMT) === this.constants.S_IFCHR; + + isFIFO = () => (this.mode & this.constants.S_IFMT) === this.constants.S_IFIFO; + + isSocket = () => + (this.mode & this.constants.S_IFMT) === this.constants.S_IFSOCK; + + get type(): FileTypeLabel { + switch (this.mode & this.constants.S_IFMT) { + case this.constants.S_IFLNK: + return "l"; + case this.constants.S_IFREG: + return "f"; + case this.constants.S_IFDIR: + return "d"; + case this.constants.S_IFBLK: + return "b"; + case this.constants.S_IFCHR: + return "c"; + case this.constants.S_IFSOCK: + return "s"; + case this.constants.S_IFIFO: + return "p"; + } + return "f"; + } +} + +interface Options { + service: string; + client?: Client; + fs: (subject?: string) => Promise; + // project-id: if given, ONLY serve files for this one project, and the + // path must be the home of the project + // If not given, serves files for all projects. + project_id?: string; +} + +export async function fsServer({ + service, + fs: fs0, + client, + project_id, +}: Options) { + client ??= conat(); + const subject = project_id + ? `${service}.project-${project_id}` + : `${service}.*`; + + logger.debug("fsServer: ", { subject, service }); + + const watches: { [subject: string]: any } = {}; + + // It is extremely important to only have one copy of each + // Filesystem for each subject, since ths Filesystem does + // locking and coordination with clients. Hence this cache, + // given that fs(...) is called separately in all functions + // below. Any ttl cache is natural because this cache is used + // for locks, which are short lived. + const cache = new TTL({ ttl: 60 * 1000 * 60 }); + const fs = reuseInFlight(async (subject) => { + if (!cache.has(subject)) { + cache.set(subject, await fs0(subject)); + } + return cache.get(subject)!; + }); + + logger.debug("fsServer: starting subscription to ", subject); + const sub = await client.service(subject, { + async appendFile(path: string, data: string | Buffer, encoding?) { + await (await fs(this.subject)).appendFile(path, data, encoding); + }, + async chmod(path: string, mode: string | number) { + await (await fs(this.subject)).chmod(path, mode); + }, + async constants(): Promise<{ [key: string]: number }> { + return await (await fs(this.subject)).constants(); + }, + async copyFile(src: string, dest: string) { + await (await fs(this.subject)).copyFile(src, dest); + }, + async cp(src: string | string[], dest: string, options?) { + await (await fs(this.subject)).cp(src, dest, options); + }, + async dust(path: string, options?: DustOptions) { + return await (await fs(this.subject)).dust(path, options); + }, + async exists(path: string): Promise { + return await (await fs(this.subject)).exists(path); + }, + async fd(path: string, options?: FdOptions) { + return await (await fs(this.subject)).fd(path, options); + }, + async find(path: string, options?: FindOptions) { + return await (await fs(this.subject)).find(path, options); + }, + async getListing(path: string) { + return await (await fs(this.subject)).getListing(path); + }, + async link(existingPath: string, newPath: string) { + await (await fs(this.subject)).link(existingPath, newPath); + }, + async lstat(path: string): Promise { + return await (await fs(this.subject)).lstat(path); + }, + async mkdir(path: string, options?) { + await (await fs(this.subject)).mkdir(path, options); + }, + async ouch(args: string[], options?: OuchOptions) { + return await (await fs(this.subject)).ouch(args, options); + }, + async readFile(path: string, encoding?, lock?) { + return await (await fs(this.subject)).readFile(path, encoding, lock); + }, + async lockFile(path: string, lock?: number) { + return await (await fs(this.subject)).lockFile(path, lock); + }, + + async readdir(path: string, options?) { + const files = await (await fs(this.subject)).readdir(path, options); + if (!options?.withFileTypes) { + return files; + } + // Dirent - change the [Symbol(type)] field to something serializable so client can use this: + return files.map((x) => { + // @ts-ignore + return { ...x, type: x[Object.getOwnPropertySymbols(x)[0]] }; + }); + }, + async readlink(path: string) { + return await (await fs(this.subject)).readlink(path); + }, + async realpath(path: string) { + return await (await fs(this.subject)).realpath(path); + }, + async rename(oldPath: string, newPath: string) { + await (await fs(this.subject)).rename(oldPath, newPath); + }, + async move( + src: string | string[], + dest: string, + options?: { overwrite?: boolean }, + ) { + return await (await fs(this.subject)).move(src, dest, options); + }, + async ripgrep(path: string, pattern: string, options?: RipgrepOptions) { + return await (await fs(this.subject)).ripgrep(path, pattern, options); + }, + async rustic(args: string[]) { + return await (await fs(this.subject)).rustic(args); + }, + async rm(path: string | string[], options?) { + await (await fs(this.subject)).rm(path, options); + }, + async rmdir(path: string, options?) { + await (await fs(this.subject)).rmdir(path, options); + }, + async stat(path: string): Promise { + const s = await (await fs(this.subject)).stat(path); + return { + ...s, + // for some reason these times get corrupted on transport from the nodejs datastructure, + // so we make them standard Date objects. + atime: new Date(s.atime), + mtime: new Date(s.mtime), + ctime: new Date(s.ctime), + birthtime: new Date(s.birthtime), + }; + }, + async symlink(target: string, path: string) { + await (await fs(this.subject)).symlink(target, path); + }, + async truncate(path: string, len?: number) { + await (await fs(this.subject)).truncate(path, len); + }, + async unlink(path: string) { + await (await fs(this.subject)).unlink(path); + }, + async utimes( + path: string, + atime: number | string | Date, + mtime: number | string | Date, + ) { + await (await fs(this.subject)).utimes(path, atime, mtime); + }, + async writeFile(path: string, data: string | Buffer, saveLast?: boolean) { + await (await fs(this.subject)).writeFile(path, data, saveLast); + }, + // @ts-ignore + async watch() { + const subject = this.subject!; + if (watches[subject] != null) { + return; + } + const f = await fs(subject); + watches[subject] = watchServer({ + client, + subject: `watch-${subject}`, + watch: f.watch, + }); + }, + }); + logger.debug("fsServer: created subscription to ", subject); + + return { + close: () => { + for (const subject in watches) { + watches[subject].close(); + delete watches[subject]; + } + sub.close(); + }, + }; +} + +export type FilesystemClient = Omit, "lstat"> & { + listing: (path: string) => Promise; + stat: (path: string) => Promise; + lstat: (path: string) => Promise; +}; + +export function getService({ + compute_server_id, + service = DEFAULT_FILE_SERVICE, +}: { + compute_server_id?: number; + service?: string; +}) { + return compute_server_id ? `${service}/${compute_server_id}` : service; +} + +export function fsSubject({ + project_id, + compute_server_id = 0, + service = DEFAULT_FILE_SERVICE, +}: { + project_id: string; + compute_server_id?: number; + service?: string; +}) { + if (!isValidUUID(project_id)) { + throw Error(`project_id must be a valid uuid -- ${project_id}`); + } + if (typeof compute_server_id != "number") { + throw Error("compute_server_id must be a number"); + } + if (typeof service != "string") { + throw Error("service must be a string"); + } + return `${getService({ service, compute_server_id })}.project-${project_id}`; +} + +const DEFAULT_FS_CALL_TIMEOUT = 5 * 60_000; + +export function fsClient({ + client, + subject, + timeout = DEFAULT_FS_CALL_TIMEOUT, +}: { + client?: Client; + subject: string; + timeout?: number; +}): FilesystemClient { + client ??= conat(); + let call = client.call(subject, { timeout }); + + const readdir0 = call.readdir.bind(call); + call.readdir = async (path: string, options?) => { + const files = await readdir0(path, options); + if (options?.withFileTypes) { + return files.map((x) => new Dirent(x.name, x.parentPath, x.path, x.type)); + } else { + return files; + } + }; + + let constants: any = null; + const stat0 = call.stat.bind(call); + call.stat = async (path: string) => { + const s = await stat0(path); + constants = constants ?? (await call.constants()); + const stats = new Stats(constants); + for (const k in s) { + stats[k] = s[k]; + } + return stats; + }; + + const lstat0 = call.lstat.bind(call); + call.lstat = async (path: string) => { + const s = await lstat0(path); + constants = constants ?? (await call.constants()); + const stats = new Stats(constants); + for (const k in s) { + stats[k] = s[k]; + } + return stats; + }; + + const ensureWatchServerExists = call.watch.bind(call); + call.watch = async (path: string, options?) => { + if (!(await call.exists(path))) { + const err = new Error( + `ENOENT: no such file or directory, watch '${path}'`, + ); + // @ts-ignore + err.code = "ENOENT"; + throw err; + } + await ensureWatchServerExists(path, options); + return await watchClient({ + client, + subject: `watch-${subject}`, + path, + options, + fs: call, + }); + }; + call.listing = async (path: string) => { + return await listing({ fs: call, path }); + }; + + return call; +} diff --git a/src/packages/conat/files/listing.ts b/src/packages/conat/files/listing.ts new file mode 100644 index 00000000000..70edf708db1 --- /dev/null +++ b/src/packages/conat/files/listing.ts @@ -0,0 +1,158 @@ +/* +Directory Listing + +Tests in packages/backend/conat/files/test/listing.test.ts + + +*/ + +import { EventEmitter } from "events"; +import { join } from "path"; +import { type FilesystemClient } from "./fs"; +import { EventIterator } from "@cocalc/util/event-iterator"; + +export type FileTypeLabel = "f" | "d" | "l" | "b" | "c" | "s" | "p"; + +export const typeDescription = { + f: "regular file", + d: "directory", + l: "symlink", + b: "block device", + c: "character device", + s: "socket", + p: "fifo", +}; + +export interface FileData { + // last modification time as time since epoch in **milliseconds** (as is usual for javascript) + mtime: number; + size: number; + // isDir = mainly for backward compat: + isDir?: boolean; + // issymlink = mainly for backward compat: + isSymLink?: boolean; + linkTarget?: string; + // see typeDescription above. + type?: FileTypeLabel; +} + +export type Files = { [name: string]: FileData }; + +interface Options { + path: string; + fs: FilesystemClient; +} + +export default async function listing(opts: Options): Promise { + const listing = new Listing(opts); + await listing.init(); + return listing; +} + +export class Listing extends EventEmitter { + public files?: Files = {}; + public truncated?: boolean; + private watch?; + private iters: EventIterator[] = []; + constructor(public readonly opts: Options) { + super(); + } + + iter = () => { + const iter = new EventIterator(this, "change", { + map: (args) => { + return { name: args[0], ...args[1] }; + }, + }); + this.iters.push(iter); + return iter; + }; + + close = () => { + this.emit("closed"); + this.removeAllListeners(); + this.iters.map((iter) => iter.end()); + this.iters.length = 0; + this.watch?.close(); + delete this.files; + delete this.watch; + }; + + init = async () => { + const { fs, path } = this.opts; + // close on unlink is critical so that btrfs snapshots don't get locked when we try to delete them + this.watch = await fs.watch(path, { closeOnUnlink: true, stats: true }); + const { files, truncated } = await fs.getListing(path); + this.files = files; + this.truncated = truncated; + this.emit("ready"); + this.handleUpdates(); + }; + + private handleUpdates = async () => { + for await (const x of this.watch) { + if (this.files == null) { + return; + } + this.update(x); + } + }; + + private update = async ({ + filename, + event, + stats, + }: { + filename: string; + event; + stats; + }) => { + // console.log("update", { filename, event, stats }); + if (this.files == null) { + // closed or not initialized yet + return; + } + if (event.startsWith("unlink")) { + delete this.files[filename]; + } else { + try { + stats ??= await this.opts.fs.lstat(join(this.opts.path, filename)); + if (this.files == null) { + return; + } + const data: FileData = { + mtime: stats.mtimeMs, + size: stats.size, + type: stats.type, + }; + if (stats.isSymbolicLink()) { + // resolve target. + data.linkTarget = await this.opts.fs.readlink( + join(this.opts.path, filename), + ); + data.isSymLink = true; + } + if (stats.isDirectory()) { + data.isDir = true; + } + this.files[filename] = data; + } catch (err) { + if (this.files == null) { + return; + } + if (err.code == "ENOENT") { + // file deleted + delete this.files[filename]; + } else { + //if (!process.env.COCALC_TEST_MODE) { + console.warn("WARNING:", err); + // TODO: some other error -- e.g., network down or permissions, so we don't know anything. + // Should we retry (?). + //} + return; + } + } + } + this.emit("change", filename, this.files[filename]); + }; +} diff --git a/src/packages/conat/files/read.ts b/src/packages/conat/files/read.ts index 91ec0e20f1a..774cb7064e1 100644 --- a/src/packages/conat/files/read.ts +++ b/src/packages/conat/files/read.ts @@ -40,7 +40,10 @@ for await (const chunk of await a.readFile({project_id:'00847397-d6a8-4cb0-96a8- import { conat } from "@cocalc/conat/client"; import { projectSubject } from "@cocalc/conat/names"; -import { type Subscription } from "@cocalc/conat/core/client"; +import { + type Subscription, + type Client as ConatClient, +} from "@cocalc/conat/core/client"; import { delay } from "awaiting"; import { getLogger } from "@cocalc/conat/client"; @@ -68,8 +71,15 @@ function getSubject({ project_id, compute_server_id, name = "" }) { export async function createServer({ createReadStream, project_id, - compute_server_id, + compute_server_id = 0, name = "", + client = conat(), +}: { + createReadStream; + project_id: string; + compute_server_id?: number; + name?: string; + client?: ConatClient; }) { const subject = getSubject({ project_id, @@ -77,8 +87,7 @@ export async function createServer({ name, }); logger.debug("createServer", { subject }); - const cn = await conat(); - const sub = await cn.subscribe(subject); + const sub = await client.subscribe(subject); subs[subject] = sub; listen({ sub, createReadStream }); } @@ -161,9 +170,11 @@ export interface ReadFileOptions { path: string; name?: string; maxWait?: number; + client?: ConatClient; } export async function* readFile({ + client = conat(), project_id, compute_server_id = 0, path, @@ -171,7 +182,6 @@ export async function* readFile({ maxWait = 1000 * 60 * 10, // 10 minutes }: ReadFileOptions) { logger.debug("readFile", { project_id, compute_server_id, path }); - const cn = await conat(); const subject = getSubject({ project_id, compute_server_id, @@ -180,7 +190,7 @@ export async function* readFile({ const v: any = []; let seq = 0; let bytes = 0; - for await (const resp of await cn.requestMany( + for await (const resp of await client.requestMany( subject, { path }, { diff --git a/src/packages/conat/files/watch.ts b/src/packages/conat/files/watch.ts new file mode 100644 index 00000000000..145795ff2fd --- /dev/null +++ b/src/packages/conat/files/watch.ts @@ -0,0 +1,221 @@ +/* +Remotely proxying a fs.watch AsyncIterator over a Conat Socket. +*/ + +import { type Client as ConatClient } from "@cocalc/conat/core/client"; +import { + type ConatSocketServer, + type ServerSocket, +} from "@cocalc/conat/socket"; +import { EventIterator } from "@cocalc/util/event-iterator"; +import { getLogger } from "@cocalc/conat/client"; +import { type CompressedPatch } from "@cocalc/util/patch"; +import { Stats } from "./fs"; + +const logger = getLogger("conat:files:watch"); + +// (path:string, options:WatchOptions) => AsyncIterator +type AsyncWatchFunction = any; + +// This is NOT the nodejs watcher, but uses +// https://github.com/paulmillr/chokidar +// though we do not allow customization of many options. +// It basically works like the fs watcher without any options, +// and for a path recursively watches to a depth of "0", i.e., watches +// for changes to files in that folder, but no subfolders. +export interface WatchOptions { + maxQueue?: number; + overflow?: "ignore" | "throw"; + signal?; + + // if more than one client is actively watching the same path and has unique set, + // all but one should receive the extra field ignore:true in the update. + unique?: boolean; + + // if true, watcher will close if the path being watched is unlinked. + closeOnUnlink?: boolean; + + stats?: boolean; + + patch?: boolean; +} + +export function watchServer({ + client, + subject, + watch, +}: { + client: ConatClient; + subject: string; + watch: AsyncWatchFunction; +}) { + const server: ConatSocketServer = client.socket.listen(subject); + logger.debug("server: listening on ", { subject }); + + const unique: { [path: string]: ServerSocket[] } = {}; + const ignores: { [path: string]: { ignoreUntil: number }[] } = {}; + async function handleUnique({ mesg, socket, path, options, ignore }) { + let w: any = undefined; + + socket.once("closed", () => { + // when this socket closes, remove it from recipient list + unique[path] = unique[path]?.filter((x) => x.id != socket.id); + if (unique[path] != null && unique[path].length == 0) { + // nobody listening + w?.close(); + w = undefined; + delete unique[path]; + delete ignores[path]; + } + }); + + if (unique[path] == null) { + // set it up + unique[path] = [socket]; + ignores[path] = [ignore]; + w = await watch(path, options); + await mesg.respond(); + for await (const event of w) { + const now = Date.now(); + let ignore = false; + for (const { ignoreUntil } of ignores[path]) { + if (ignoreUntil > now) { + // every client is told to ignore this change, i.e., not load based on it happening + ignore = true; + break; + } + } + for (const s of unique[path]) { + if (s.state == "ready") { + if (ignore) { + s.write({ ...event, ignore: true }); + } else { + s.write(event); + ignore = true; + } + } + } + } + } else { + unique[path].push(socket); + ignores[path].push(ignore); + await mesg.respond(); + } + } + + async function handleNonUnique({ mesg, socket, path, options, ignore }) { + const w = await watch(path, options); + socket.once("closed", () => { + w.close(); + }); + await mesg.respond(); + for await (const event of w) { + if (ignore.ignoreUntil >= Date.now()) { + continue; + } + socket.write(event); + } + } + + server.on("connection", (socket: ServerSocket) => { + logger.debug("server: got new connection", { + id: socket.id, + subject: socket.subject, + }); + let initialized = false; + const ignore = { ignoreUntil: 0 }; + socket.on("request", async (mesg) => { + const data = mesg.data; + if (data.ignore != null) { + ignore.ignoreUntil = data.ignore > 0 ? Date.now() + data.ignore : 0; + await mesg.respond(null, { noThrow: true }); + return; + } + try { + if (initialized) { + throw Error("already initialized"); + } + initialized = true; + const { path, options } = data; + logger.debug("got request", { path, options }); + if (options?.unique) { + await handleUnique({ mesg, socket, path, options, ignore }); + } else { + await handleNonUnique({ mesg, socket, path, options, ignore }); + } + } catch (err) { + mesg.respondSync(null, { + headers: { error: `${err}`, code: err.code }, + }); + } + }); + }); + + return server; +} + +export type WatchIterator = EventIterator & { + ignore?: (ignore: number) => Promise; +}; + +export interface ChangeEvent { + event: "add" | "addDir" | "change" | "unlink" | "unlinkDir"; + filename: string; + ignore?: boolean; + patch?: CompressedPatch; + stats?; +} + +export async function watchClient({ + client, + subject, + path, + options, + fs, +}: { + client: ConatClient; + subject: string; + path: string; + options?: WatchOptions; + fs?; +}): Promise { + const socket = client.socket.connect(subject); + let constants = options?.stats ? await fs?.constants() : undefined; + const iter = new EventIterator( + socket, + "data", + { + map: (args) => { + if (args[0].stats && constants !== undefined) { + const s = args[0].stats; + const stats = new Stats(constants); + for (const k in s) { + stats[k] = s[k]; + } + args[0].stats = stats; + } + return args[0]; + }, + onEnd: () => { + socket.close(); + }, + }, + ); + socket.on("closed", () => { + iter.end(); + delete iter2.ignore; + }); + // tell it what to watch + await socket.request({ + path, + options, + }); + const iter2 = iter as WatchIterator; + + // ignore events for ignore ms. + iter2.ignore = async (ignore: number) => { + await socket.request({ ignore }); + }; + + return iter2; +} diff --git a/src/packages/conat/files/write.ts b/src/packages/conat/files/write.ts index 8ef3e07a778..040518e9932 100644 --- a/src/packages/conat/files/write.ts +++ b/src/packages/conat/files/write.ts @@ -71,7 +71,10 @@ import { readFile, } from "./read"; import { projectSubject } from "@cocalc/conat/names"; -import { type Subscription } from "@cocalc/conat/core/client"; +import { + type Subscription, + type Client as ConatClient, +} from "@cocalc/conat/core/client"; import { type Readable } from "node:stream"; import { getLogger } from "@cocalc/conat/client"; const logger = getLogger("conat:files:write"); @@ -96,10 +99,12 @@ export async function close({ project_id, compute_server_id }) { } export async function createServer({ + client = conat(), project_id, compute_server_id, createWriteStream, }: { + client?: ConatClient; project_id: string; compute_server_id: number; // createWriteStream returns a writeable stream @@ -113,8 +118,7 @@ export async function createServer({ if (sub != null) { return; } - const cn = await conat(); - sub = await cn.subscribe(subject); + sub = await client.subscribe(subject); subs[subject] = sub; listen({ sub, createWriteStream, project_id, compute_server_id }); } @@ -195,11 +199,19 @@ export interface WriteFileOptions { } export async function writeFile({ + client = conat(), project_id, compute_server_id = 0, path, stream, maxWait = 1000 * 60 * 10, // 10 minutes +}: { + client?: ConatClient; + project_id: string; + compute_server_id?: number; + path: string; + stream; + maxWait?: number; }): Promise<{ bytes: number; chunks: number }> { logger.debug("writeFile", { project_id, compute_server_id, path, maxWait }); const name = randomId(); @@ -215,8 +227,7 @@ export async function writeFile({ name, }); // tell compute server / project to start reading our file. - const cn = await conat(); - const resp = await cn.request( + const resp = await client.request( getWriteSubject({ project_id, compute_server_id }), { name, path, maxWait }, { timeout: maxWait }, diff --git a/src/packages/conat/hub/api/README.md b/src/packages/conat/hub/api/README.md new file mode 100644 index 00000000000..b52bf16f762 --- /dev/null +++ b/src/packages/conat/hub/api/README.md @@ -0,0 +1,15 @@ +This API gets called from various places: + +- a browser frontend (mostly): + see packages/frontend/conat/client.ts + +- a project + see packages/project/conat/hub.ts + +- the nextjs servers to make api/python work + +This API is *implemented* in two places: + +- the main hub itself in packages/server/conat/api + +- in lite a minimal version is implemented in packages/lite/hub/api.ts \ No newline at end of file diff --git a/src/packages/conat/hub/api/compute.ts b/src/packages/conat/hub/api/compute.ts new file mode 100644 index 00000000000..bc34b4333d5 --- /dev/null +++ b/src/packages/conat/hub/api/compute.ts @@ -0,0 +1,198 @@ +import { authFirstRequireAccount, authFirst } from "./util"; +import type { + Action, + Cloud, + ComputeServerTemplate, + ComputeServerUserInfo, + Configuration, + Images, + GoogleCloudImages, +} from "@cocalc/util/db-schema/compute-servers"; +import type { GoogleCloudData } from "@cocalc/util/compute/cloud/google-cloud/compute-cost"; +import type { HyperstackPriceData } from "@cocalc/util/compute/cloud/hyperstack/pricing"; +import type { + ConfigurationTemplate, + ConfigurationTemplates, +} from "@cocalc/util/compute/templates"; + +export const compute = { + createServer: authFirstRequireAccount, + computeServerAction: authFirstRequireAccount, + getServersById: authFirstRequireAccount, + getServers: authFirstRequireAccount, + getServerState: authFirstRequireAccount, + getSerialPortOutput: authFirstRequireAccount, + deleteServer: authFirstRequireAccount, + undeleteServer: authFirstRequireAccount, + isDnsAvailable: authFirstRequireAccount, + setServerColor: authFirstRequireAccount, + setServerTitle: authFirstRequireAccount, + setServerConfiguration: authFirstRequireAccount, + setTemplate: authFirstRequireAccount, + getTemplate: true, + getTemplates: authFirstRequireAccount, + setServerCloud: authFirstRequireAccount, + setServerOwner: authFirstRequireAccount, + getGoogleCloudPriceData: authFirstRequireAccount, + getHyperstackPriceData: authFirstRequireAccount, + getNetworkUsage: authFirstRequireAccount, + getApiKey: authFirstRequireAccount, + deleteApiKey: authFirstRequireAccount, + getLog: authFirstRequireAccount, + getTitle: authFirstRequireAccount, + setDetailedState: authFirstRequireAccount, + getImages: authFirst, + getGoogleCloudImages: authFirst, + setImageTested: authFirstRequireAccount, +}; + +export interface Compute { + // server lifecycle + createServer: (opts: { + account_id?: string; + project_id: string; + title?: string; + color?: string; + autorestart?: boolean; + cloud?: Cloud; + configuration?: Configuration; + notes?: string; + course_project_id?: string; + course_server_id?: number; + }) => Promise; + + computeServerAction: (opts: { + account_id?: string; + id: number; + action: Action; + }) => Promise; + + // Get servers across potentially different projects by their global unique id. + // Use the fields parameter to restrict to a much smaller subset of information + // about each server (e.g., just the state field). Caller must be a collaborator + // on each project containing the servers. + // If you give an id of a server that doesn't exist, it'll just be excluded in the result. + // Similarly, if you give a field that doesn't exist, it is excluded. + // The order of the returned servers and count probably will NOT match that in + // ids, so you should include 'id' in fields. + getServersById: (opts: { + account_id?: string; + ids: number[]; + fields?: Array; + }) => Promise[]>; + + getServers: (opts: { + account_id?: string; + id?: number; + project_id: string; + }) => Promise; + + getServerState: (opts: { + account_id?: string; + id: number; + }) => Promise; + getSerialPortOutput: (opts: { + account_id?: string; + id: number; + }) => Promise; + + deleteServer: (opts: { account_id?: string; id: number }) => Promise; + undeleteServer: (opts: { account_id?: string; id: number }) => Promise; + + isDnsAvailable: (opts: { + account_id?: string; + dns: string; + }) => Promise; + + // ownership & metadata + setServerColor: (opts: { + account_id?: string; + id: number; + color: string; + }) => Promise; + setServerTitle: (opts: { + account_id?: string; + id: number; + title: string; + }) => Promise; + setServerConfiguration: (opts: { + account_id?: string; + id: number; + configuration: Partial; + }) => Promise; + + setTemplate: (opts: { + account_id?: string; + id: number; + template: ComputeServerTemplate; + }) => Promise; + + getTemplate: (opts: { + account_id?: string; + id: number; + }) => Promise; + getTemplates: () => Promise; + + setServerCloud: (opts: { + account_id?: string; + id: number; + cloud: Cloud | string; + }) => Promise; + setServerOwner: (opts: { + account_id?: string; + id: number; + new_account_id: string; + }) => Promise; + + // pricing caches + getGoogleCloudPriceData: () => Promise; + getHyperstackPriceData: () => Promise; + + // usage & logs + getNetworkUsage: (opts: { + account_id?: string; + id: number; + start: Date; + end: Date; + }) => Promise<{ amount: number; cost: number }>; + + getApiKey: (opts: { account_id?: string; id: number }) => Promise; + deleteApiKey: (opts: { account_id?: string; id: number }) => Promise; + + getLog: (opts: { + account_id?: string; + id: number; + type: "activity" | "files"; + }) => Promise; + + getTitle: (opts: { account_id?: string; id: number }) => Promise<{ + title: string; + color: string; + project_specific_id: number; + }>; + + setDetailedState: (opts: { + account_id?: string; + project_id: string; + id: number; + name: string; + state?: string; + extra?: string; + timeout?: number; + progress?: number; + }) => Promise; + + getImages: (opts?: { + noCache?: boolean; + account_id?: string; + }) => Promise; + getGoogleCloudImages: (opts?: { + noCache?: boolean; + account_id?: string; + }) => Promise; + setImageTested: (opts: { + account_id?: string; + id: number; + tested: boolean; + }) => Promise; +} diff --git a/src/packages/conat/hub/api/file-sync.ts b/src/packages/conat/hub/api/file-sync.ts new file mode 100644 index 00000000000..f34f1b61f06 --- /dev/null +++ b/src/packages/conat/hub/api/file-sync.ts @@ -0,0 +1,50 @@ +/* +Synchronization of path between projects. + +This code manages bidirectional synchronization of one path in one project +with another path in a different project. + +Sync is done entirely within the fileserver, but then propagates to +running projects, of course. + +APPLICATIONS: + +- if you want somebody to edit one path but not have full access to the + contents of a project, make a new project for them, then sync that path + into the new project. + +- share common files between several projects (e.g., your ~/bin) and have them + update everywhere when you change one. + +- share data file or a software install with a group of users or projects +*/ + +import { authFirstRequireAccount } from "./util"; +import { type Sync } from "@cocalc/conat/files/file-server"; +import { type MutagenSyncSession } from "@cocalc/conat/project/mutagen/types"; + +export const fileSync = { + create: authFirstRequireAccount, + getAll: authFirstRequireAccount, + get: authFirstRequireAccount, + command: authFirstRequireAccount, +}; + +export interface FileSync { + create: ( + opts: Sync & { account_id: string; ignores?: string[] }, + ) => Promise; + get: ( + sync: Sync & { account_id: string }, + ) => Promise; + command: ( + sync: Sync & { + account_id: string; + command: "flush" | "reset" | "pause" | "resume" | "terminate"; + }, + ) => Promise<{ stdout: string; stderr: string; exit_code: number }>; + getAll: (opts: { + name: string; + account_id: string; + }) => Promise<(MutagenSyncSession & Sync)[]>; +} diff --git a/src/packages/conat/hub/api/index.ts b/src/packages/conat/hub/api/index.ts index 6aac0927719..5355e48c3b8 100644 --- a/src/packages/conat/hub/api/index.ts +++ b/src/packages/conat/hub/api/index.ts @@ -8,6 +8,8 @@ import { handleErrorMessage } from "@cocalc/conat/util"; import { type Sync, sync } from "./sync"; import { type Org, org } from "./org"; import { type Messages, messages } from "./messages"; +import { type Compute, compute } from "./compute"; +import { type FileSync, fileSync } from "./file-sync"; export interface HubApi { system: System; @@ -18,6 +20,8 @@ export interface HubApi { sync: Sync; org: Org; messages: Messages; + compute: Compute; + fileSync: FileSync; } const HubApiStructure = { @@ -29,6 +33,8 @@ const HubApiStructure = { sync, org, messages, + compute, + fileSync, } as const; export function transformArgs({ name, args, account_id, project_id }) { diff --git a/src/packages/conat/hub/api/projects.ts b/src/packages/conat/hub/api/projects.ts index bf42fd5d36d..76770cd63ba 100644 --- a/src/packages/conat/hub/api/projects.ts +++ b/src/packages/conat/hub/api/projects.ts @@ -1,6 +1,8 @@ -import { authFirstRequireAccount } from "./util"; +import { authFirstRequireAccount, authFirstRequireProject } from "./util"; import { type CreateProjectOptions } from "@cocalc/util/db-schema/projects"; -import { type UserCopyOptions } from "@cocalc/util/db-schema/projects"; +import { type SnapshotCounts } from "@cocalc/util/consts/snapshots"; +import { type CopyOptions } from "@cocalc/conat/files/fs"; +import { type SnapshotUsage } from "@cocalc/conat/files/file-server"; export const projects = { createProject: authFirstRequireAccount, @@ -10,8 +12,27 @@ export const projects = { inviteCollaborator: authFirstRequireAccount, inviteCollaboratorWithoutAccount: authFirstRequireAccount, setQuotas: authFirstRequireAccount, + + getDiskQuota: authFirstRequireAccount, + + createBackup: authFirstRequireAccount, + deleteBackup: authFirstRequireAccount, + restoreBackup: authFirstRequireAccount, + updateBackups: authFirstRequireAccount, + getBackups: authFirstRequireAccount, + getBackupFiles: authFirstRequireAccount, + getBackupQuota: authFirstRequireAccount, + + createSnapshot: authFirstRequireAccount, + deleteSnapshot: authFirstRequireAccount, + updateSnapshots: authFirstRequireAccount, + getSnapshotQuota: authFirstRequireAccount, + allSnapshotUsage: authFirstRequireAccount, + start: authFirstRequireAccount, stop: authFirstRequireAccount, + + getSshKeys: authFirstRequireProject, }; export type AddCollaborator = @@ -32,7 +53,11 @@ export interface Projects { // request to have conat permissions to project subjects. createProject: (opts: CreateProjectOptions) => Promise; - copyPathBetweenProjects: (opts: UserCopyOptions) => Promise; + copyPathBetweenProjects: (opts: { + src: { project_id: string; path: string | string[] }; + dest: { project_id: string; path: string }; + options?: CopyOptions; + }) => Promise; removeCollaborator: ({ account_id, @@ -87,6 +112,7 @@ export interface Projects { }; }) => Promise; + // for admins only! setQuotas: (opts: { account_id?: string; project_id: string; @@ -101,6 +127,94 @@ export interface Projects { always_running?: number; }) => Promise; + getDiskQuota: (opts: { + account_id?: string; + project_id: string; + }) => Promise<{ used: number; size: number }>; + + ///////////// + // BACKUPS + ///////////// + createBackup: (opts: { + account_id?: string; + project_id: string; + }) => Promise<{ time: Date; id: string }>; + + deleteBackup: (opts: { + account_id?: string; + project_id: string; + id: string; + }) => Promise; + + restoreBackup: (opts: { + account_id?: string; + project_id: string; + path?: string; + id: string; + }) => Promise; + + updateBackups: (opts: { + account_id?: string; + project_id: string; + counts?: Partial; + }) => Promise; + + getBackups: (opts: { account_id?: string; project_id: string }) => Promise< + { + id: string; + time: Date; + }[] + >; + + getBackupFiles: (opts: { + account_id?: string; + project_id: string; + }) => Promise; + + getBackupQuota: (opts: { + account_id?: string; + project_id: string; + }) => Promise<{ limit: number }>; + + ///////////// + // SNAPSHOTS + ///////////// + + createSnapshot: (opts: { + account_id?: string; + project_id: string; + name?: string; + }) => Promise; + + deleteSnapshot: (opts: { + account_id?: string; + project_id: string; + name: string; + }) => Promise; + + updateSnapshots: (opts: { + account_id?: string; + project_id: string; + counts?: Partial; + }) => Promise; + + getSnapshotQuota: (opts: { + account_id?: string; + project_id: string; + }) => Promise<{ limit: number }>; + + allSnapshotUsage: (opts: { project_id: string }) => Promise; + + ///////////// + // Project Control + ///////////// start: (opts: { account_id: string; project_id: string }) => Promise; stop: (opts: { account_id: string; project_id: string }) => Promise; + + // get a list if all public ssh authorized keys that apply to + // the given project. + // this is ALL global public keys for all collabs on the project, + // along with all project specific keys. This is called by the project + // on startup to configure itself. + getSshKeys: (opts?: { project_id?: string }) => Promise; } diff --git a/src/packages/conat/hub/api/system.ts b/src/packages/conat/hub/api/system.ts index ad8d961107d..6388fdf36e4 100644 --- a/src/packages/conat/hub/api/system.ts +++ b/src/packages/conat/hub/api/system.ts @@ -1,4 +1,4 @@ -import { noAuth, authFirst, requireAccount } from "./util"; +import { noAuth, authFirst, requireSignedIn } from "./util"; import type { Customize } from "@cocalc/util/db-schema/server-settings"; import type { ApiKey, @@ -17,7 +17,7 @@ export const system = { generateUserAuthToken: authFirst, revokeUserAuthToken: noAuth, userSearch: authFirst, - getNames: requireAccount, + getNames: requireSignedIn, adminResetPasswordLink: authFirst, sendEmailVerification: authFirst, deletePassport: authFirst, diff --git a/src/packages/conat/hub/api/util.ts b/src/packages/conat/hub/api/util.ts index ff7e8965140..2fa078963b9 100644 --- a/src/packages/conat/hub/api/util.ts +++ b/src/packages/conat/hub/api/util.ts @@ -12,10 +12,17 @@ export const authFirst = ({ args, account_id, project_id }) => { export const noAuth = ({ args }) => args; -// make no changes, except throw error if account_id not set (i.e., user not signed in) +// make no changes, except throw error if account_id not set (i.e., user not signed in with an account) export const requireAccount = ({ args, account_id }) => { if (!account_id) { - throw Error("user must be signed in"); + throw Error("user must be signed in with an account"); + } + return args; +}; + +export const requireSignedIn = ({ args, account_id, project_id }) => { + if (!account_id && !project_id) { + throw Error("must be signed in as account or project"); } return args; }; @@ -31,3 +38,13 @@ export const authFirstRequireAccount = async ({ args, account_id }) => { return args; }; +export const authFirstRequireProject = async ({ args, project_id }) => { + if (args[0] == null) { + args[0] = {} as any; + } + if (!project_id) { + throw Error("must be a project"); + } + args[0].project_id = project_id; + return args; +}; diff --git a/src/packages/conat/hub/call-hub.ts b/src/packages/conat/hub/call-hub.ts new file mode 100644 index 00000000000..b0ecbacc713 --- /dev/null +++ b/src/packages/conat/hub/call-hub.ts @@ -0,0 +1,38 @@ +import { type Client } from "@cocalc/conat/core/client"; +const DEFAULT_TIMEOUT = 15000; + +export default async function callHub({ + client, + account_id, + project_id, + name, + args = [], + timeout = DEFAULT_TIMEOUT, +}: { + client: Client; + account_id?: string; + project_id?: string; + name: string; + args?: any[]; + timeout?: number; +}) { + const subject = getSubject({ account_id, project_id }); + try { + const data = { name, args }; + const resp = await client.request(subject, data, { timeout }); + return resp.data; + } catch (err) { + err.message = `${err.message} - callHub: subject='${subject}', name='${name}', code='${err.code}' `; + throw err; + } +} + +function getSubject({ account_id, project_id }) { + if (account_id) { + return `hub.account.${account_id}.api`; + } else if (project_id) { + return `hub.project.${project_id}.api`; + } else { + throw Error("account_id or project_id must be specified"); + } +} diff --git a/src/packages/conat/hub/changefeeds/server.ts b/src/packages/conat/hub/changefeeds/server.ts index 1d685c6c82d..776b42c7fe5 100644 --- a/src/packages/conat/hub/changefeeds/server.ts +++ b/src/packages/conat/hub/changefeeds/server.ts @@ -145,7 +145,7 @@ export function changefeedServer({ err, ); try { - socket.write({ error: `${err}` }); + socket.write(err); } catch {} socket.close(); } diff --git a/src/packages/conat/llm/server.ts b/src/packages/conat/llm/server.ts index 77afd7d86d1..c13ef765d20 100644 --- a/src/packages/conat/llm/server.ts +++ b/src/packages/conat/llm/server.ts @@ -14,6 +14,9 @@ how paying for that would work. import { conat } from "@cocalc/conat/client"; import { isValidUUID } from "@cocalc/util/misc"; import type { Subscription } from "@cocalc/conat/core/client"; +import { getLogger } from "@cocalc/conat/client"; + +const logger = getLogger("conat:llm:server"); export const SUBJECT = process.env.COCALC_TEST_MODE ? "llm-test" : "llm"; @@ -61,7 +64,7 @@ export async function close() { if (sub == null) { return; } - sub.drain(); + sub.close(); sub = null; } @@ -77,24 +80,36 @@ async function listen(evaluate) { async function handleMessage(mesg, evaluate) { const options = mesg.data; - let seq = 0; - const respond = ({ text, error }: { text?: string; error?: string }) => { - mesg.respondSync({ text, error, seq }); + let seq = -1; + const respond = async ({ + text, + error, + }: { + text?: string; + error?: string; + }) => { seq += 1; + try { + await mesg.respond({ text, error, seq }); + } catch (err) { + logger.debug("WARNING: error sending response -- ", err); + end(); + } }; let done = false; - const end = () => { + const end = async () => { if (done) return; done = true; - // end response stream with null payload. - mesg.respondSync(null); + // end response stream with null payload -- send sync, or it could + // get sent before the responses above, which would cancel them out! + await mesg.respond(null, { noThrow: true }); }; - const stream = (text?) => { + const stream = async (text?) => { if (done) return; if (text != null) { - respond({ text }); + await respond({ text }); } else { end(); } diff --git a/src/packages/conat/package.json b/src/packages/conat/package.json index 3cc702d8468..f5f7704077c 100644 --- a/src/packages/conat/package.json +++ b/src/packages/conat/package.json @@ -5,14 +5,20 @@ "exports": { "./sync/*": "./dist/sync/*.js", "./llm/*": "./dist/llm/*.js", + "./files/*": "./dist/files/*.js", "./socket": "./dist/socket/index.js", "./socket/*": "./dist/socket/*.js", + "./sync-doc": "./dist/sync-doc/index.js", + "./sync-doc/*": "./dist/sync-doc/*.js", "./hub/changefeeds": "./dist/hub/changefeeds/index.js", "./hub/api": "./dist/hub/api/index.js", "./hub/api/*": "./dist/hub/api/*.js", "./compute/*": "./dist/compute/*.js", + "./sync-files": "./dist/sync-files/index.js", "./service": "./dist/service/index.js", "./project/api": "./dist/project/api/index.js", + "./project/mutagen": "./dist/project/mutagen/index.js", + "./project/terminal": "./dist/project/terminal/index.js", "./browser-api": "./dist/browser-api/index.js", "./*": "./dist/*.js" }, @@ -31,6 +37,7 @@ "dependencies": { "@cocalc/comm": "workspace:*", "@cocalc/conat": "workspace:*", + "@cocalc/sync": "workspace:*", "@cocalc/util": "workspace:*", "@isaacs/ttlcache": "^1.4.1", "@msgpack/msgpack": "^3.1.1", @@ -44,14 +51,14 @@ "js-base64": "^3.7.7", "json-stable-stringify": "^1.0.1", "lodash": "^4.17.21", + "mime-types": "^3.0.1", "socket.io": "^4.8.1", "socket.io-client": "^4.8.1", "utf-8-validate": "^6.0.5" }, "devDependencies": { "@types/better-sqlite3": "^7.6.13", - "@types/json-stable-stringify": "^1.0.32", - "@types/lodash": "^4.14.202", + "@types/lodash": "^4.17.20", "@types/node": "^18.16.14" }, "repository": { diff --git a/src/packages/conat/persist/auth.ts b/src/packages/conat/persist/auth.ts index abc28e8f760..d3064e54f32 100644 --- a/src/packages/conat/persist/auth.ts +++ b/src/packages/conat/persist/auth.ts @@ -49,13 +49,13 @@ export function assertHasWritePermission({ if (path.length > MAX_PATH_LENGTH) { throw new ConatError( `permission denied: path (of length ${path.length}) is too long (limit is '${MAX_PATH_LENGTH}' characters)`, - { code: 403 }, + { code: 403, subject }, ); } if (path.startsWith("/") || path.endsWith("/")) { throw new ConatError( `permission denied: path '${path}' must not start or end with '/'`, - { code: 403 }, + { code: 403, subject }, ); } const v = subject.split("."); @@ -79,10 +79,10 @@ export function assertHasWritePermission({ } else { throw new ConatError( `permission denied: subject '${subject}' does not grant write permission to path='${path}' since it is not under '${base}'`, - { code: 403 }, + { code: 403, subject }, ); } } } - throw new ConatError(`invalid subject: '${subject}'`, { code: 403 }); + throw new ConatError(`invalid subject: '${subject}'`, { code: 403, subject }); } diff --git a/src/packages/conat/persist/client.ts b/src/packages/conat/persist/client.ts index 3bfe76af64f..66bfd1c57ee 100644 --- a/src/packages/conat/persist/client.ts +++ b/src/packages/conat/persist/client.ts @@ -143,7 +143,7 @@ class PersistStreamClient extends EventEmitter { }); if (resp.headers?.error) { throw new ConatError(`${resp.headers?.error}`, { - code: resp.headers?.code, + code: resp.headers?.code as string | number, }); } if (this.changefeeds.length == 0 || this.state != "ready") { @@ -218,7 +218,7 @@ class PersistStreamClient extends EventEmitter { }); if (resp.headers?.error) { throw new ConatError(`${resp.headers?.error}`, { - code: resp.headers?.code, + code: resp.headers?.code as string | number, }); } // an iterator over any updates that are published. @@ -384,7 +384,9 @@ class PersistStreamClient extends EventEmitter { let seq = 0; // next expected seq number for the sub (not the data) for await (const { data, headers } of sub) { if (headers?.error) { - throw new ConatError(`${headers.error}`, { code: headers.code }); + throw new ConatError(`${headers.error}`, { + code: headers.code as string | number, + }); } if (data == null || this.socket.state == "closed") { // done diff --git a/src/packages/conat/persist/server.ts b/src/packages/conat/persist/server.ts index 5a270630e79..8ad17e4715a 100644 --- a/src/packages/conat/persist/server.ts +++ b/src/packages/conat/persist/server.ts @@ -59,7 +59,6 @@ import { type ConatSocketServer, type ServerSocket, } from "@cocalc/conat/socket"; -import { getLogger } from "@cocalc/conat/client"; import type { StoredMessage, PersistentStream, @@ -70,6 +69,7 @@ import { throttle } from "lodash"; import { type SetOptions } from "./client"; import { once } from "@cocalc/util/async-utils"; import { UsageMonitor } from "@cocalc/conat/monitor/usage"; +import { getLogger } from "@cocalc/conat/client"; const logger = getLogger("persist:server"); diff --git a/src/packages/conat/persist/storage.ts b/src/packages/conat/persist/storage.ts index 44caab17ad8..a5e90d3885d 100644 --- a/src/packages/conat/persist/storage.ts +++ b/src/packages/conat/persist/storage.ts @@ -491,10 +491,11 @@ export class PersistentStream extends EventEmitter { | StoredMessage | undefined => { let x; + const ttl = this.conf.allow_msg_ttl ? ", ttl" : ""; if (seq) { x = this.db .prepare( - "SELECT seq, key, time, compress, encoding, raw, headers FROM messages WHERE seq=?", + `SELECT seq, key, time, compress, encoding, raw, headers${ttl} FROM messages WHERE seq=?`, ) .get(seq); } else if (key != null) { @@ -502,7 +503,7 @@ export class PersistentStream extends EventEmitter { // row with a given key. Also there's a unique constraint. x = this.db .prepare( - "SELECT seq, key, time, compress, encoding, raw, headers FROM messages WHERE key=?", + `SELECT seq, key, time, compress, encoding, raw, headers${ttl} FROM messages WHERE key=?`, ) .get(key); } else { @@ -847,12 +848,22 @@ function dbToMessage( encoding: DataEncoding; raw: Buffer; headers?: string; + ttl?: number; } | undefined, ): StoredMessage | undefined { if (x === undefined) { return x; } + if (x.ttl && Date.now() - 1000 * x.time >= x.ttl) { + // the actual record will get cleared eventually from the + // database when enforceLimits is called. For now we + // just returned undefined. The check here makes it so + // ttl fully works as claimed, rather than "eventually", i.e., + // it can be used for a short-term lock, rather than just + // being something for saving space longterm. + return undefined; + } return { seq: x.seq, time: x.time * 1000, diff --git a/src/packages/conat/project/api/apps.ts b/src/packages/conat/project/api/apps.ts new file mode 100644 index 00000000000..4f206809cb9 --- /dev/null +++ b/src/packages/conat/project/api/apps.ts @@ -0,0 +1,34 @@ +export const apps = { + start: true, + stop: true, + status: true, +}; + +export interface Apps { + start: (name: string) => Promise<{ + state: "running" | "stopped"; + port: number; + url: string; + pid?: number; + stdout: Buffer; + stderr: Buffer; + spawnError?; + exit?: { code; signal? }; + }>; + + status: (name: string) => Promise< + | { + state: "running" | "stopped"; + port: number; + url: string; + pid?: number; + stdout: Buffer; + stderr: Buffer; + spawnError?; + exit?: { code; signal? }; + } + | { state: "stopped" } + >; + + stop: (name: string) => Promise; +} diff --git a/src/packages/conat/project/api/editor.ts b/src/packages/conat/project/api/editor.ts index cd70dc973dc..cf9984b9da5 100644 --- a/src/packages/conat/project/api/editor.ts +++ b/src/packages/conat/project/api/editor.ts @@ -1,17 +1,12 @@ -import type { NbconvertParams } from "@cocalc/util/jupyter/types"; -import type { RunNotebookOptions } from "@cocalc/util/jupyter/nbgrader-types"; import type { Options as FormatterOptions } from "@cocalc/util/code-formatter"; -import type { KernelSpec } from "@cocalc/util/jupyter/types"; export const editor = { - newFile: true, - jupyterStripNotebook: true, - jupyterNbconvert: true, - jupyterRunNotebook: true, - jupyterKernelLogo: true, - jupyterKernels: true, - formatterString: true, + formatString: true, + printSageWS: true, + sagewsStart: true, + sagewsStop: true, + createTerminalService: true, }; @@ -27,33 +22,16 @@ export interface CreateTerminalOptions { } export interface Editor { - // Create a new file with the given name, possibly aware of templates. - // This was cc-new-file in the old smc_pyutils python library. This - // is in editor, since it's meant to be for creating a file aware of the - // context of our editors. - newFile: (path: string) => Promise; - - jupyterStripNotebook: (path_ipynb: string) => Promise; - - jupyterNbconvert: (opts: NbconvertParams) => Promise; - - jupyterRunNotebook: (opts: RunNotebookOptions) => Promise; - - jupyterKernelLogo: ( - kernelName: string, - opts?: { noCache?: boolean }, - ) => Promise<{ filename: string; base64: string }>; - - jupyterKernels: (opts?: { noCache?: boolean }) => Promise; - - // returns a patch to transform str into formatted form. - formatterString: (opts: { + // returns formatted version of str. + formatString: (opts: { str: string; options: FormatterOptions; path?: string; // only used for CLANG }) => Promise; printSageWS: (opts) => Promise; + sagewsStart: (path_sagews: string) => Promise; + sagewsStop: (path_sagews: string) => Promise; createTerminalService: ( termPath: string, diff --git a/src/packages/conat/project/api/index.ts b/src/packages/conat/project/api/index.ts index 694229b2bcd..072cc3a3cf9 100644 --- a/src/packages/conat/project/api/index.ts +++ b/src/packages/conat/project/api/index.ts @@ -1,21 +1,34 @@ import { type System, system } from "./system"; import { type Editor, editor } from "./editor"; +import { type Jupyter, jupyter } from "./jupyter"; import { type Sync, sync } from "./sync"; +import { type Apps, apps } from "./apps"; import { handleErrorMessage } from "@cocalc/conat/util"; +export { projectApiClient } from "./project-client"; export interface ProjectApi { system: System; editor: Editor; + jupyter: Jupyter; sync: Sync; + apps: Apps; + isReady: () => Promise; + waitUntilReady: (opts?: { timeout?: number }) => Promise; } const ProjectApiStructure = { system, editor, + jupyter, sync, + apps, } as const; -export function initProjectApi(callProjectApi): ProjectApi { +export function initProjectApi({ + callProjectApi, + isReady, + waitUntilReady, +}): ProjectApi { const projectApi: any = {}; for (const group in ProjectApiStructure) { if (projectApi[group] == null) { @@ -31,5 +44,7 @@ export function initProjectApi(callProjectApi): ProjectApi { ); } } + projectApi.isReady = isReady; + projectApi.waitUntilReady = waitUntilReady; return projectApi as ProjectApi; } diff --git a/src/packages/conat/project/api/jupyter.ts b/src/packages/conat/project/api/jupyter.ts new file mode 100644 index 00000000000..2b289867385 --- /dev/null +++ b/src/packages/conat/project/api/jupyter.ts @@ -0,0 +1,82 @@ +import type { NbconvertParams } from "@cocalc/util/jupyter/types"; +import type { RunNotebookOptions } from "@cocalc/util/jupyter/nbgrader-types"; +import type { KernelSpec } from "@cocalc/util/jupyter/types"; +import { type ProjectJupyterApiOptions } from "@cocalc/util/jupyter/api-types"; + +export const jupyter = { + start: true, + stop: true, + stripNotebook: true, + nbconvert: true, + runNotebook: true, + kernelLogo: true, + kernels: true, + introspect: true, + complete: true, + signal: true, + getConnectionFile: true, + + sendCommMessageToKernel: true, + ipywidgetsGetBuffer: true, + + // jupyter stateless API + apiExecute: true, +}; + +// In the functions below path can be either the .ipynb or the .sage-jupyter2 path, and +// the correct backend kernel will get found/created automatically. +export interface Jupyter { + stripNotebook: (path_ipynb: string) => Promise; + + // path = the syncdb path (not *.ipynb) + start: (path: string) => Promise; + stop: (path: string) => Promise; + + nbconvert: (opts: NbconvertParams) => Promise; + + runNotebook: (opts: RunNotebookOptions) => Promise; + + kernelLogo: ( + kernelName: string, + opts?: { noCache?: boolean }, + ) => Promise<{ filename: string; base64: string }>; + + kernels: (opts?: { noCache?: boolean }) => Promise; + + introspect: (opts: { + path: string; + code: string; + cursor_pos: number; + detail_level: 0 | 1; + }) => Promise; + + complete: (opts: { + path: string; + code: string; + cursor_pos: number; + }) => Promise; + + getConnectionFile: (opts: { path: string }) => Promise; + + signal: (opts: { path: string; signal: string }) => Promise; + + apiExecute: (opts: ProjectJupyterApiOptions) => Promise; + + sendCommMessageToKernel: (opts: { + path: string; + msg: { + msg_id: string; + comm_id: string; + target_name: string; + data: any; + buffers64?: string[]; + buffers?: Buffer[]; + }; + }) => Promise; + + ipywidgetsGetBuffer: (opts: { + path: string; + model_id: string; + buffer_path: string | string[]; + }) => Promise<{ buffer64: string }>; +} diff --git a/src/packages/conat/project/api/project-client.ts b/src/packages/conat/project/api/project-client.ts new file mode 100644 index 00000000000..4bd74720c14 --- /dev/null +++ b/src/packages/conat/project/api/project-client.ts @@ -0,0 +1,70 @@ +/* +Create a client for the project's api. Anything that can publish to *.project-project_id... can use this. +*/ + +import { projectSubject } from "@cocalc/conat/names"; +import { type Client, connect } from "@cocalc/conat/core/client"; +import { isValidUUID } from "@cocalc/util/misc"; +import { type ProjectApi, initProjectApi } from "./index"; + +const DEFAULT_TIMEOUT = 15000; +const service = "api"; + +export function projectApiClient({ + project_id, + compute_server_id = 0, + client = connect(), + timeout = DEFAULT_TIMEOUT, +}: { + project_id: string; + compute_server_id?: number; + client?: Client; + timeout?: number; +}): ProjectApi { + if (!isValidUUID(project_id)) { + throw Error(`project_id = '${project_id}' must be a valid uuid`); + } + const subject = projectSubject({ project_id, compute_server_id, service }); + + const isReady = async () => { + return await client.interest(subject); + }; + + const waitUntilReady = async ({ timeout }: { timeout?: number } = {}) => { + await client.waitForInterest(subject, { timeout }); + }; + + const callProjectApi = async ({ name, args }) => { + return await callProject({ + client, + subject, + timeout, + name, + args, + }); + }; + return initProjectApi({ callProjectApi, isReady, waitUntilReady }); +} + +async function callProject({ + client, + subject, + name, + args = [], + timeout = DEFAULT_TIMEOUT, +}: { + client: Client; + subject: string; + name: string; + args: any[]; + timeout?: number; +}) { + const resp = await client.request( + subject, + { name, args }, + // we use waitForInterest because often the project hasn't + // quite fully started. + { timeout, waitForInterest: true }, + ); + return resp.data; +} diff --git a/src/packages/conat/project/api/sync.ts b/src/packages/conat/project/api/sync.ts index 8acbac37163..6918c2a47b5 100644 --- a/src/packages/conat/project/api/sync.ts +++ b/src/packages/conat/project/api/sync.ts @@ -5,8 +5,16 @@ export const sync = { // x11: true, // synctableChannel: true, // symmetricChannel: true, + + mutagen: true, }; +import { type ExecOutput } from "@cocalc/conat/files/fs"; + export interface Sync { close: (path: string) => Promise; + + // run mutagen with given args and return the output. There is no sandboxing, + // since this is running in the compute server (or maybe project). + mutagen: (args: string[]) => Promise; } diff --git a/src/packages/conat/project/api/system.ts b/src/packages/conat/project/api/system.ts index 3e780381c86..a66e4799826 100644 --- a/src/packages/conat/project/api/system.ts +++ b/src/packages/conat/project/api/system.ts @@ -7,7 +7,6 @@ import type { Configuration, ConfigurationAspect, } from "@cocalc/comm/project-configuration"; -import { type ProjectJupyterApiOptions } from "@cocalc/util/jupyter/api-types"; export const system = { terminate: true, @@ -15,7 +14,6 @@ export const system = { version: true, listing: true, - deleteFiles: true, moveFiles: true, renameFile: true, realpath: true, @@ -32,8 +30,13 @@ export const system = { signal: true, - // jupyter stateless API - jupyterExecute: true, + // named servers like jupyterlab, vscode, etc. + startNamedServer: true, + statusOfNamedServer: true, + + // ssh support + sshPublicKey: true, + updateSshKeys: true, }; export interface System { @@ -46,7 +49,6 @@ export interface System { path: string; hidden?: boolean; }) => Promise; - deleteFiles: (opts: { paths: string[] }) => Promise; moveFiles: (opts: { paths: string[]; dest: string }) => Promise; renameFile: (opts: { src: string; dest: string }) => Promise; realpath: (path: string) => Promise; @@ -73,5 +75,17 @@ export interface System { pid?: number; }) => Promise; - jupyterExecute: (opts: ProjectJupyterApiOptions) => Promise; + // return the ssh public key of this project/compute server. + // The project generates a public key on startup that is used + // internally for connecting to the file server, and this is that key. + // Basically this is a key that is used internally for communication + // within cocalc, so other services can trust the project. + // It can be changed without significant consequences (the file-server + // container gets restarted). + sshPublicKey: () => Promise; + + // calling updateSshKeys causes the project to ensure that + // ~/.ssh/authorized_keys contains all entries set + // in the database (in addition to whatever else might be there). + updateSshKeys: () => Promise; } diff --git a/src/packages/conat/project/jupyter/run-code.ts b/src/packages/conat/project/jupyter/run-code.ts new file mode 100644 index 00000000000..76fd787d86d --- /dev/null +++ b/src/packages/conat/project/jupyter/run-code.ts @@ -0,0 +1,397 @@ +/* +Tests are in + +packages/backend/conat/test/juypter/run-code.test.s + +*/ + +import { type Client as ConatClient } from "@cocalc/conat/core/client"; +import { + type ConatSocketServer, + type ServerSocket, +} from "@cocalc/conat/socket"; +import { EventIterator } from "@cocalc/util/event-iterator"; +import { getLogger } from "@cocalc/conat/client"; +import { Throttle } from "@cocalc/util/throttle"; +const MAX_MSGS_PER_SECOND = parseInt( + process.env.COCALC_JUPYTER_MAX_MSGS_PER_SECOND ?? "20", +); +const logger = getLogger("conat:project:jupyter:run-code"); + +function getSubject({ + project_id, + compute_server_id = 0, +}: { + project_id: string; + compute_server_id?: number; +}) { + return `jupyter.project-${project_id}.${compute_server_id}`; +} + +export interface InputCell { + id: string; + input: string; + output?: { [n: string]: OutputMessage | null } | null; + state?: "done" | "busy" | "run"; + exec_count?: number | null; + start?: number | null; + end?: number | null; + cell_type?: "code"; +} + +export interface OutputMessage { + // id = id of the cell + id: string; + // everything below is exactly from Jupyter + metadata?; + content?; + buffers?; + msg_type?: string; + done?: boolean; + more_output?: boolean; +} + +export interface RunOptions { + // syncdb path + path: string; + // array of input cells to run + cells: InputCell[]; + // if true do not halt running the cells, even if one fails with an error + noHalt?: boolean; + // the socket is used for raw_input, to communicate between the client + // that initiated the request and the server. + socket: ServerSocket; +} + +type JupyterCodeRunner = ( + opts: RunOptions, +) => Promise>; + +interface OutputHandler { + process: (mesg: OutputMessage) => void; + done: () => void; +} + +type CreateOutputHandler = (opts: { + path: string; + cells: InputCell[]; +}) => OutputHandler; + +export function jupyterServer({ + client, + project_id, + compute_server_id = 0, + // run takes a path and cells to run and returns an async iterator + // over the outputs. + run, + // outputHandler takes a path and returns an OutputHandler, which can be + // used to process the output and include it in the notebook. It is used + // as a fallback in case the client that initiated running cells is + // disconnected, so output won't be lost. + outputHandler, + getKernelStatus, +}: { + client: ConatClient; + project_id: string; + compute_server_id?: number; + run: JupyterCodeRunner; + outputHandler?: CreateOutputHandler; + getKernelStatus: (opts: { path: string }) => Promise<{ + backend_state: + | "failed" + | "off" + | "spawning" + | "starting" + | "running" + | "closed"; + kernel_state: "idle" | "busy" | "running"; + }>; +}) { + const subject = getSubject({ project_id, compute_server_id }); + const server: ConatSocketServer = client.socket.listen(subject, { + keepAlive: 5000, + keepAliveTimeout: 5000, + }); + logger.debug("server: listening on ", { subject }); + const moreOutput: { [path: string]: { [id: string]: any[] } } = {}; + + server.on("connection", (socket: ServerSocket) => { + logger.debug("server: got new connection", { + id: socket.id, + subject: socket.subject, + }); + + socket.on("request", async (mesg) => { + const { data } = mesg; + const { cmd, path } = data; + if (cmd == "more") { + logger.debug("more output ", { id: data.id }); + mesg.respondSync(moreOutput[path]?.[data.id]); + } else if (cmd == "get-kernel-status") { + mesg.respondSync(await getKernelStatus({ path })); + } else if (cmd == "run") { + const { cells, noHalt, limit } = data; + try { + mesg.respondSync(null); + if (moreOutput[path] == null) { + moreOutput[path] = {}; + } + await handleRequest({ + socket, + run, + outputHandler, + path, + cells, + noHalt, + limit, + moreOutput: moreOutput[path], + }); + } catch (err) { + logger.debug("server: failed to handle execute request -- ", err); + if (socket.state != "closed") { + try { + logger.debug("sending to client: ", { + headers: { error: `${err}` }, + }); + socket.write(null, { headers: { foo: "bar", error: `${err}` } }); + } catch (err) { + // an error trying to report an error shouldn't crash everything + logger.debug("WARNING: unable to send error to client", err); + } + } + } + } else { + const error = `Unknown command '${cmd}'`; + logger.debug(error); + mesg.respondSync(null, { headers: { error } }); + } + }); + + socket.on("closed", () => { + logger.debug("socket closed", { id: socket.id }); + }); + }); + + return server; +} + +async function handleRequest({ + socket, + run, + outputHandler, + path, + cells, + noHalt, + limit, + moreOutput, +}) { + const runner = await run({ path, cells, noHalt, socket }); + const output: OutputMessage[] = []; + for (const cell of cells) { + moreOutput[cell.id] = []; + } + logger.debug( + `handleRequest to evaluate ${cells.length} cells with limit=${limit} for path=${path}`, + ); + + const throttle = new Throttle(MAX_MSGS_PER_SECOND); + let unhandledClientWriteError: any = undefined; + throttle.on("data", async (mesgs) => { + try { + socket.write(mesgs); + } catch (err) { + if (err.code == "ENOBUFS") { + // wait for the over-filled socket to finish writing out data. + await socket.drain(); + socket.write(mesgs); + } else { + unhandledClientWriteError = err; + } + } + }); + + try { + let handler: OutputHandler | null = null; + let process: ((mesg: any) => void) | null = null; + + for await (const mesg of runner) { + if (socket.state == "closed") { + // client socket has closed -- the backend server must take over! + if (handler == null || process == null) { + logger.debug("socket closed -- server must handle output"); + if (outputHandler == null) { + throw Error("no output handler available"); + } + handler = outputHandler({ path, cells }); + if (handler == null) { + throw Error("bug -- outputHandler must return a handler"); + } + process = (mesg) => { + if (handler == null) return; + if (limit == null || output.length < limit) { + handler.process(mesg); + } else { + if (output.length == limit) { + handler.process({ id: mesg.id, more_output: true }); + moreOutput[mesg.id] = []; + } + moreOutput[mesg.id].push(mesg); + } + }; + + for (const prev of output) { + process(prev); + } + output.length = 0; + } + process(mesg); + } else { + if (unhandledClientWriteError) { + throw unhandledClientWriteError; + } + output.push(mesg); + if (limit == null || output.length < limit) { + throttle.write(mesg); + } else { + if (output.length == limit) { + throttle.write({ + id: mesg.id, + more_output: true, + }); + moreOutput[mesg.id] = []; + } + // save the more output + moreOutput[mesg.id].push(mesg); + } + } + } + // no errors happened, so close up and flush and + // remaining data immediately: + handler?.done(); + if (socket.state != "closed") { + throttle.flush(); + socket.write(null); + } + } finally { + throttle.close(); + } +} + +export class JupyterClient { + private iter?: EventIterator; + public readonly socket; + constructor( + private client: ConatClient, + private subject: string, + private path: string, + private stdin: (opts: { + id: string; + prompt: string; + password?: boolean; + }) => Promise, + ) { + this.socket = this.client.socket.connect(this.subject); + this.socket.once("close", () => this.iter?.end()); + this.socket.on("request", async (mesg) => { + const { data } = mesg; + try { + switch (data.type) { + case "stdin": + await mesg.respond(await this.stdin(data)); + return; + default: + console.warn(`Jupyter: got unknown message type '${data.type}'`); + await mesg.respond( + new Error(`unknown message type '${data.type}'`), + ); + } + } catch (err) { + console.warn("error responding to jupyter request", err); + } + }); + } + + close = () => { + try { + this.iter?.end(); + delete this.iter; + this.socket.close(); + } catch {} + }; + + moreOutput = async (id: string) => { + const { data } = await this.socket.request({ + cmd: "more", + path: this.path, + id, + }); + return data; + }; + + getKernelStatus = async () => { + const { data } = await this.socket.request({ + cmd: "get-kernel-status", + path: this.path, + }); + return data; + }; + + run = async ( + cells: InputCell[], + opts: { noHalt?: boolean; limit?: number } = {}, + ) => { + if (this.iter) { + // one evaluation at a time -- starting a new one ends the previous one. + // Each client browser has a separate instance of JupyterClient, so + // a properly implemented frontend client would never hit this. + this.iter.end(); + delete this.iter; + } + this.iter = new EventIterator(this.socket, "data", { + map: (args) => { + if (args[1]?.error) { + this.iter?.throw(Error(args[1].error)); + return; + } + if (args[0] == null) { + this.iter?.end(); + return; + } else { + return args[0]; + } + }, + }); + // get rid of any fields except id and input from the cells, since, e.g., + // if there is a lot of output in a cell, there is no need to send that to the backend. + const cells1 = cells.map(({ id, input }) => { + return { id, input }; + }); + await this.socket.request({ + cmd: "run", + ...opts, + path: this.path, + cells: cells1, + }); + return this.iter; + }; +} + +export function jupyterClient(opts: { + path: string; + project_id: string; + compute_server_id?: number; + client: ConatClient; + stdin?: (opts: { + id: string; + prompt: string; + password?: boolean; + }) => Promise; +}): JupyterClient { + const subject = getSubject(opts); + return new JupyterClient( + opts.client, + subject, + opts.path, + opts.stdin ?? (async () => "stdin not implemented"), + ); +} diff --git a/src/packages/conat/project/mutagen/forward.ts b/src/packages/conat/project/mutagen/forward.ts new file mode 100644 index 00000000000..22be7f79440 --- /dev/null +++ b/src/packages/conat/project/mutagen/forward.ts @@ -0,0 +1,137 @@ +import { type Client } from "@cocalc/conat/core/client"; +import { type ProjectApi, projectApiClient } from "@cocalc/conat/project/api"; +import { refCacheSync } from "@cocalc/util/refcache"; + +const DEFAULT_TIMEOUT = 1000 * 15; + +export interface Options { + project_id: string; + compute_server_id: number; + client: Client; +} + +export interface Selector { + name?: string; + sessions?: string[]; + labelSelector?: string; + all?: boolean; +} + +export class MutagenForward { + private api: ProjectApi; + + constructor(private opts: Options) { + this.api = projectApiClient({ ...this.opts, timeout: DEFAULT_TIMEOUT }); + } + + // We use mutagen to implement filesystem sync. We do not use it for + // any port forwarding here. + private mutagen = async ( + args: string[], + ): Promise<{ stdout: string; stderr: string }> => { + console.log("mutagen forward", args.join(" ")); + const { stdout, stderr, code } = await this.api.sync.mutagen([ + "forward", + ...args, + ]); + if (code) { + throw new Error(Buffer.from(stderr).toString()); + } + return { + stdout: Buffer.from(stdout).toString(), + stderr: Buffer.from(stderr).toString(), + }; + }; + + private mutagenAction = async ( + action: string, + opts: Selector, + extraArgs?: string[], + ) => { + const { sessions = [], labelSelector, all, name } = opts; + const args = [action, ...sessions]; + if (all) { + args.push("--all"); + } else if (labelSelector) { + args.push("--label-selector", labelSelector); + } else if (name) { + args.push(name); + } + if (extraArgs) { + args.push(...extraArgs); + } + return await this.mutagen(args); + }; + + close = () => {}; + + // create forward so any connection to source is actually sent on to destination. + create = async ( + source: string, + destination: string, + { + name, + label, + options, + }: { + name?: string; + label?: { [key: string]: string }; + options?: string[]; + } = {}, + ) => { + if (!source) { + throw Error("source must be specified"); + } + if (!destination) { + throw Error("destination must be specified"); + } + const args = ["create", source, destination]; + if (name) { + args.push("--name", name); + } + for (const key in label) { + args.push("--label", `${key}=${label[key]}`); + } + return await this.mutagen(args.concat(options ?? [])); + }; + + list = async ({ + labelSelector, + name, + }: { + name?: string; + // labelSelector uses exactly the same syntax as Kubernetes: + // https://kubernetes.io/docs/concepts/overview/working-with-objects/labels/#list-and-watch-filtering + labelSelector?: string; + } = {}) => { + const args = ["list", "-l", "--template='{{ json . }}'"]; + if (name) { + args.push(name); + } + if (labelSelector) { + args.push("--label-selector", labelSelector); + } + const { stdout } = await this.mutagen(args); + return JSON.parse(stdout.slice(1, -2)); + }; + + pause = async (opts: Selector) => await this.mutagenAction("pause", opts); + + resume = async (opts: Selector) => await this.mutagenAction("resume", opts); + + terminate = async (opts: Selector) => { + await this.mutagenAction("terminate", opts); + }; +} + +export const mutagenForward = refCacheSync< + Options & { noCache?: boolean }, + MutagenForward +>({ + name: "mutagen-forward", + createKey: ({ project_id, compute_server_id, client }: Options) => + JSON.stringify([project_id, compute_server_id, client.id]), + createObject: (opts: Options & { noCache?: boolean }) => { + return new MutagenForward(opts); + }, +}); diff --git a/src/packages/conat/project/mutagen/index.ts b/src/packages/conat/project/mutagen/index.ts new file mode 100644 index 00000000000..9d1a23b9f60 --- /dev/null +++ b/src/packages/conat/project/mutagen/index.ts @@ -0,0 +1,16 @@ +import { mutagenSync } from "./sync"; +import { mutagenForward } from "./forward"; +import { type Client } from "@cocalc/conat/core/client"; + +export default function mutagen({ + client, + project_id, + compute_server_id = 0, +}: { + client: Client; + project_id: string; + compute_server_id: number; +}) { + const opts = { client, project_id, compute_server_id }; + return { sync: mutagenSync(opts), forward: mutagenForward(opts) }; +} diff --git a/src/packages/conat/project/mutagen/sync.ts b/src/packages/conat/project/mutagen/sync.ts new file mode 100644 index 00000000000..02dc4694609 --- /dev/null +++ b/src/packages/conat/project/mutagen/sync.ts @@ -0,0 +1,177 @@ +import { type ProjectApi, projectApiClient } from "@cocalc/conat/project/api"; +import { refCacheSync } from "@cocalc/util/refcache"; +import { type Selector, type Options } from "./forward"; + +// a minutes -- some commands (e.g., flush) could take a long time. +const DEFAULT_TIMEOUT = 1000 * 60; + +export class MutagenSync { + private api: ProjectApi; + + constructor(private opts: Options) { + this.api = projectApiClient({ ...this.opts, timeout: DEFAULT_TIMEOUT }); + } + + // We use mutagen to implement filesystem sync. We do not use it for + // any port forwarding here. + private mutagen = async ( + args: string[], + ): Promise<{ stdout: string; stderr: string }> => { + console.log("mutagen sync", args.join(" ")); + const { stdout, stderr, code } = await this.api.sync.mutagen([ + "sync", + ...args, + ]); + if (code) { + throw new Error(Buffer.from(stderr).toString()); + } + return { + stdout: Buffer.from(stdout).toString(), + stderr: Buffer.from(stderr).toString(), + }; + }; + + private mutagenAction = async ( + action: string, + opts: Selector, + extraArgs?: string[], + ) => { + const { sessions = [], labelSelector, all, name } = opts; + const args = [action, ...sessions]; + if (all) { + args.push("--all"); + } else if (labelSelector) { + args.push("--label-selector", labelSelector); + } else if (name) { + args.push(name); + } + if (extraArgs) { + args.push(...extraArgs); + } + return await this.mutagen(args); + }; + + close = () => {}; + + // Sync path between us and path on the remote. Here remote + // is a connection with user = {project_id:...} that we get + // using a project specific api key. + create = async ( + alpha, + beta, + { + name, + + paused, + label = {}, + + ignore, + ignoreVcs, + noIgnoreVcs, + + symlinkMode = "posix-raw", // different default since usually what *WE* want. + maxFileSize, + options, + }: { + name?: string; + + paused?: boolean; + label?: { [key: string]: string }; + // resolve = + // - local -- all conflicts resolve to local + // - remote -- conflicts always resolve to remote + // - manual (default) -- conflicts must be manually resolved. + resolve?: "local" | "remote" | "manual"; + + ignore?: string[]; + ignoreVcs?: boolean; + noIgnoreVcs?: boolean; + + symlinkMode?: string; + maxFileSize?: string; + + options?: string[]; + } = {}, + ) => { + if (!alpha) { + throw Error("alpha must be specified"); + } + if (!beta) { + throw Error("beta must be specified"); + } + const args = [alpha, beta]; + if (symlinkMode) { + args.push("--symlink-mode", symlinkMode); + } + if (ignore) { + for (const x of ignore) { + args.push("--ignore", x); + } + } + if (ignoreVcs) { + args.push("--ignore-vcs"); + } + if (noIgnoreVcs) { + args.push("--no-ignore-vcs"); + } + if (maxFileSize) { + args.push("--max-staging-file-size", maxFileSize); + } + args.push("--no-global-configuration"); + args.push("--compression", "deflate"); + if (paused) { + args.push("--paused"); + } + if (name) { + args.push("--name", name); + } + for (const key in label) { + args.push("--label", `${key}=${label[key]}`); + } + return await this.mutagen(args.concat(options ?? [])); + }; + + list = async ({ + labelSelector, + }: { + // labelSelector uses exactly the same syntax as Kubernetes: + // https://kubernetes.io/docs/concepts/overview/working-with-objects/labels/#list-and-watch-filtering + labelSelector?: string; + } = {}) => { + const args = ["list", "-l", "--template='{{ json . }}'"]; + if (labelSelector) { + args.push("--label-selector", labelSelector); + } + const { stdout } = await this.mutagen(args); + return JSON.parse(stdout.slice(1, -2)); + }; + + flush = async (opts: Selector & { skipWait?: boolean }) => { + return await this.mutagenAction( + "flush", + opts, + opts.skipWait ? ["--skip-wait"] : undefined, + ); + }; + + pause = async (opts: Selector) => await this.mutagenAction("pause", opts); + + reset = async (opts: Selector) => await this.mutagenAction("reset", opts); + + resume = async (opts: Selector) => await this.mutagenAction("resume", opts); + + terminate = async (opts: Selector) => + await this.mutagenAction("terminate", opts); +} + +export const mutagenSync = refCacheSync< + Options & { noCache?: boolean }, + MutagenSync +>({ + name: "mutagen-sync", + createKey: ({ project_id, compute_server_id, client }: Options) => + JSON.stringify([project_id, compute_server_id, client.id]), + createObject: (opts: Options & { noCache?: boolean }) => { + return new MutagenSync(opts); + }, +}); diff --git a/src/packages/conat/project/mutagen/types.ts b/src/packages/conat/project/mutagen/types.ts new file mode 100644 index 00000000000..b251b301316 --- /dev/null +++ b/src/packages/conat/project/mutagen/types.ts @@ -0,0 +1,36 @@ +export interface MutagenSyncSession { + identifier: string; + version: number; + creationTime: string; // ISO timestamp + creatingVersion: string; + alpha: MutagenEndpoint; + beta: MutagenEndpoint; + mode: string; // e.g. "two-way-resolved" + ignore: Record; + symlink: Record; + watch: Record; + permissions: Record; + compression: Record; + name?: string; + labels?: Record; + paused: boolean; + status: string; // e.g. "watching", "staging", etc. + successfulCycles?: number; + lastError?: string; + lastErrorTime?: string; +} + +export interface MutagenEndpoint { + protocol: string; // e.g. "local", "docker", "ssh" + path: string; + ignore: Record; + symlink: Record; + watch: Record; + permissions: Record; + compression: Record; + connected: boolean; + scanned: boolean; + directories: number; + files: number; + totalFileSize: number; +} diff --git a/src/packages/conat/project/project-status.ts b/src/packages/conat/project/project-status.ts index f3a34d11d5b..923e676a2fe 100644 --- a/src/packages/conat/project/project-status.ts +++ b/src/packages/conat/project/project-status.ts @@ -20,11 +20,11 @@ function getSubject({ project_id, compute_server_id }) { // publishes status updates when they are emitted. export async function createPublisher({ + client = conat(), project_id, compute_server_id, projectStatusServer, }) { - const client = await conat(); const subject = getSubject({ project_id, compute_server_id }); logger.debug("publishing status updates on ", { subject }); projectStatusServer.on("status", (status) => { @@ -35,10 +35,10 @@ export async function createPublisher({ // async iterator over the status updates: export async function get({ + client = conat(), project_id, compute_server_id, }): Promise { - const client = await conat(); const subject = getSubject({ project_id, compute_server_id }); return await client.subscribe(subject); } diff --git a/src/packages/conat/project/runner/bootlog.ts b/src/packages/conat/project/runner/bootlog.ts new file mode 100644 index 00000000000..040fc94308e --- /dev/null +++ b/src/packages/conat/project/runner/bootlog.ts @@ -0,0 +1,131 @@ +/* +Logging boot up of a project. +*/ + +import { type Client } from "@cocalc/conat/core/client"; +import { type DStream } from "@cocalc/conat/sync/dstream"; +import { getLogger } from "@cocalc/conat/client"; +import { conat } from "@cocalc/conat/client"; + +const logger = getLogger("conat:project:runner:bootlog"); + +// one hour +const DEFAULT_TTL = 1000 * 60 * 60; + +export interface Event { + type: string; + progress?: number; + min?: number; + max?: number; // if given, normalize progress to be between 0 and max instead of 0 and 100. + error?; + desc?: string; + elapsed?: number; + speed?: string; + eta?: number; +} + +export interface Options extends Event { + project_id: string; + compute_server_id?: number; + client?: Client; + ttl?: number; +} + +function getName({ compute_server_id = 0 }: { compute_server_id: number }) { + return `bootlog.${compute_server_id}`; +} + +export async function resetBootlog({ + project_id, + compute_server_id = 0, + client = conat(), +}) { + const stream = client.sync.astream({ + project_id, + name: getName({ compute_server_id }), + }); + try { + await stream.delete({ all: true }); + } catch (err) { + logger.debug("ERROR reseting log", { + project_id, + compute_server_id, + err, + }); + } +} + +// publishing is not fatal +// should be a TTL cache ([ ] todo) +const start: { [key: string]: number } = {}; +export async function bootlog(opts: Options) { + const { + project_id, + compute_server_id = 0, + client = conat(), + ttl = DEFAULT_TTL, + min = 0, + max = 100, + ...event + } = opts; + const stream = client.sync.astream({ + project_id, + name: getName({ compute_server_id }), + }); + if (event.error != null && typeof event.error != "string") { + event.error = `${event.error}`; + } + const key = `${project_id}-${compute_server_id}-${event.type}`; + let progress; + if (event.progress != null) { + if (!event.progress) { + start[key] = Date.now(); + event.elapsed = 0; + } else if (start[key]) { + event.elapsed = Date.now() - start[key]; + } + progress = shiftProgress({ progress: event.progress, min, max }); + } else { + progress = undefined; + } + try { + await stream.publish({ ...event, progress }, { ttl }); + } catch (err) { + logger.debug("ERROR publishing to bootlog", { + project_id, + compute_server_id, + err, + }); + } +} + +export function shiftProgress({ + progress, + min = 0, + max = 100, +}: { + progress: T; + min?: number; + max?: number; +}): T extends number ? number : undefined { + if (progress == null) { + return undefined as any; + } + return Math.round(min + (progress / 100) * (max - min)) as any; +} + +// be sure to call .close() on the result when done +export async function get({ + project_id, + compute_server_id = 0, + client = conat(), +}: { + project_id: string; + compute_server_id?: number; + client?: Client; +}): Promise> { + return await client.sync.dstream({ + project_id, + name: getName({ compute_server_id }), + }); +} diff --git a/src/packages/conat/project/runner/constants.ts b/src/packages/conat/project/runner/constants.ts new file mode 100644 index 00000000000..83a2ae19c86 --- /dev/null +++ b/src/packages/conat/project/runner/constants.ts @@ -0,0 +1,55 @@ +import { join } from "path"; + +export const INTERNAL_SSH_CONFIG = ".ssh/.cocalc"; + +export const SSH_IDENTITY_FILE = join(INTERNAL_SSH_CONFIG, "id_ed25519"); + +export const FILE_SERVER_NAME = "file-server"; + +export const SSHD_CONFIG = join(INTERNAL_SSH_CONFIG, "sshd"); + +export const START_PROJECT_SSH = join(SSHD_CONFIG, "start-project-ssh.sh"); +export const START_PROJECT_FORWARDS = join( + SSHD_CONFIG, + "start-project-forwards.sh", +); + +export interface Ports { + "file-server": number; + sshd: number; + proxy: number; + web: number; +} + +// WARNING: if you change these ports than the mutagen port forwards setup +// in START_PROJECT_FORWARDS_SH of packages/project-runner/run/startup-scripts.ts +// for any existing project would break! And they will not be fixed unless +// one manually terminates them. So if there is some very good reason +// to change these, the start script also has to be changed to be more +// sophisticated and update existing assignments if they are wrong. BUT... +// don't just do that willy nilly, e.g., if you just terminate and recreate +// them it'll take 500ms at least on startup instead of 30ms, and dominate +// the project startup time! + +export const PORTS = { + // file-server = openssh sshd server running on same VM as + // file-server for close access to files. Runs + // in a locked down container. + "file-server": 2222, + // dropbear lightweight ssh server running in the project container + // directly, which users can ssh with full port forwarding and exactly + // standard ssh sematics (.ssh/authorized_keys|config|etc.), but + // runs in any container image. Forwarded to this container by mutagen + // (so reverse ssh). + sshd: 2200, + // very simple http proxy written in nodejs running in the project, which + // lets us proxy any webserver that supports base_url (e.g., juputerlab) + // or non-absolute URL's (e.g., vscode). This supports the same schema + // as in cocalc, so the base_url has to be of the form + // /{PROJECT_ID}/server/{PORT}/ or /{PROJECT_ID}/port/{PORT}/ + proxy: 8000, + // an arbitrary user-defined webserver, which will work without any base_url + // or other requirement. Served on wildcard subdomain at + // [project_id].your-domain.com + web: 8080, +} as Ports; diff --git a/src/packages/conat/project/runner/load-balancer.ts b/src/packages/conat/project/runner/load-balancer.ts new file mode 100644 index 00000000000..a139664521a --- /dev/null +++ b/src/packages/conat/project/runner/load-balancer.ts @@ -0,0 +1,266 @@ +/* +Service to load balance running cocalc projects across the runners. + +Tests are in + + - packages/backend/conat/test/project + +*/ + +import { type Client } from "@cocalc/conat/core/client"; +import { randomChoice } from "@cocalc/conat/core/server"; +import { conat } from "@cocalc/conat/client"; +import { client as projectRunnerClient, UPDATE_INTERVAL } from "./run"; +import state, { type ProjectStatus, type ProjectState } from "./state"; +import { delay } from "awaiting"; +import { getLogger } from "@cocalc/conat/client"; + +const logger = getLogger("conat:project:runner:load-balancer"); + +const MAX_STATUS_TRIES = 3; +const TIMEOUT = 30 * 60 * 1000; + +export interface Options { + subject?: string; + client?: Client; + setState?: (opts: { + project_id: string; + state: ProjectState; + }) => Promise; + getConfig?: ({ project_id }: { project_id: string }) => Promise; +} + +export interface API { + start: () => Promise; + stop: () => Promise; + status: () => Promise; + move: (opts?: { force?: boolean; server?: string }) => Promise; + save: (opts?: { + project_id: string; + rootfs?: boolean; + home?: boolean; + }) => Promise; +} + +export async function server({ + subject = "project.*.run", + client, + setState, + getConfig, +}: Options) { + client ??= conat(); + + // - [ ] get info about the runner's status (use a stream?) -- write that here. + // - [ ] connect to database to get quota for running a project -- via a function that is passed in + // - [ ] it will contact runner to run project -- write that here. + const { projects, runners } = await state({ client }); + + const getClient = async (project_id: string) => { + const cutoff = Date.now() - UPDATE_INTERVAL * 2.5; + + const cur = projects.get(project_id); + if (cur?.server) { + const { server } = cur; + const s = runners.get(server); + if ((s?.time ?? 0) > cutoff) { + return projectRunnerClient({ + client, + subject: `project-runner.${server}`, + timeout: TIMEOUT, + }); + } else { + throw Error(`project server '${server}' is not responding`); + } + } + const v = getActiveRunners(runners); + if (v.length == 0) { + throw Error("no project runners available -- try again later"); + } + + // Project assignment: just random for now: + const server = randomChoice(new Set(v)); + logger.debug("getClient -- assigning to ", { project_id, server }); + return projectRunnerClient({ + client, + subject: `project-runner.${server}`, + timeout: TIMEOUT, + }); + }; + + const getProjectId = (t: any) => { + const subject = t.subject as string; + const project_id = subject.split(".")[1]; + return project_id; + }; + + const setState1 = + setState == null + ? undefined + : async (opts: { project_id: string; state: ProjectState }) => { + if (setState == null) { + return; + } + try { + await setState(opts); + } catch (err) { + logger.debug(`WARNING: issue calling setState`, opts, err); + } + }; + + const sub = await client.service(subject, { + async move({ force, server }: { force?: boolean; server?: string } = {}) { + const project_id = getProjectId(this); + logger.debug("move", project_id); + const cur = projects.get(project_id); + if (cur == null || !cur.server) { + // it is not assigned to a server, so nothing to do. + return; + } + const setNewServer = async () => { + if (!server) { + const v = getActiveRunners(runners).filter( + (server) => server != cur?.server, + ); + server = v.length == 0 ? undefined : randomChoice(new Set(v)); + } + projects.set(project_id, { server, state: "opened" }); + await setState1?.({ project_id, state: "opened" }); + }; + + let runClient; + try { + runClient = await getClient(project_id); + } catch (err) { + if (!force) { + throw err; + } + } + try { + const status = await runClient.status({ project_id }); + if (status?.state == "opened") { + await setNewServer(); + return; + } + } catch (err) { + if (!force) { + throw err; + } + } + try { + await setState1?.({ project_id, state: "stopping" }); + await runClient.stop({ project_id, force }); + } catch (err) { + if (!force) { + await setState1?.({ project_id, state: "running" }); + throw err; + } + } + await setNewServer(); + }, + + async start() { + const project_id = getProjectId(this); + logger.debug("start", project_id); + const config = await getConfig?.({ project_id }); + const runClient = await getClient(project_id); + await setState1?.({ project_id, state: "starting" }); + await runClient.start({ project_id, config }); + await setState1?.({ project_id, state: "running" }); + }, + + async stop({ force }: { force?: boolean } = {}) { + const project_id = getProjectId(this); + logger.debug("stop", project_id); + const runClient = await getClient(project_id); + try { + await runClient.stop({ project_id, force }); + await setState1?.({ project_id, state: "opened" }); + } catch (err) { + if (err.code == 503) { + // the runner is no longer running, so obviously project isn't running there. + await setState1?.({ project_id, state: "opened" }); + } else { + // can't stop it (e.g., sync broken/disabled), so it is still running + await setState1?.({ project_id, state: "running" }); + } + throw err; + } + }, + + async save({ + rootfs = true, + home = true, + }: { + rootfs?: boolean; + home?: boolean; + } = {}) { + const project_id = getProjectId(this); + logger.debug("save", project_id); + const runClient = await getClient(project_id); + await runClient.save({ project_id, rootfs, home }); + }, + + async status() { + const project_id = getProjectId(this); + logger.debug("status", project_id); + const runClient = await getClient(project_id); + for (let i = 0; i < MAX_STATUS_TRIES; i++) { + try { + logger.debug("status", { project_id }); + const s = await runClient.status({ project_id }); + logger.debug("status: got ", s); + await setState1?.({ project_id, ...s }); + return s; + } catch (err) { + logger.debug("status: got err", err); + if (i < MAX_STATUS_TRIES - 1) { + logger.debug("status: waiting 3s and trying again..."); + await delay(3000); + continue; + } + if (err.code == 503) { + logger.debug( + "status: running is no longer running -- giving up on project", + ); + // the runner is no longer running, so obviously project isn't running there. + await setState1?.({ project_id, state: "opened" }); + } + logger.debug("status: reporting error"); + throw err; + } + } + logger.debug("status: bug"); + throw Error("bug"); + }, + }); + + return { + close: () => { + sub.close(); + }, + }; +} + +function getActiveRunners(runners): string[] { + const cutoff = Date.now() - UPDATE_INTERVAL * 2.5; + const k = runners.getAll(); + const v: string[] = []; + for (const server in k) { + if ((k[server].time ?? 0) <= cutoff) { + continue; + } + v.push(server); + } + return v; +} + +export function client({ + client, + subject, +}: { + client?: Client; + subject: string; +}): API { + client ??= conat(); + return client.call(subject); +} diff --git a/src/packages/conat/project/runner/run.ts b/src/packages/conat/project/runner/run.ts new file mode 100644 index 00000000000..a8527885236 --- /dev/null +++ b/src/packages/conat/project/runner/run.ts @@ -0,0 +1,241 @@ +/*f +Service to run a CoCalc project. + +Tests are in + + - packages/backend/conat/test/project + +*/ + +import { type Client } from "@cocalc/conat/core/client"; +import { conat } from "@cocalc/conat/client"; +import state, { type ProjectStatus } from "./state"; +import { until } from "@cocalc/util/async-utils"; +import type { + LocalPathFunction, + SshServersFunction, + Configuration, +} from "./types"; +import { getLogger } from "@cocalc/conat/client"; +import { isValidUUID } from "@cocalc/util/misc"; + +const logger = getLogger("conat:project:runner:run"); + +export const UPDATE_INTERVAL = 10_000; + +export interface Options { + // client -- Client for the Conat cluster. State of this project runner gets saved here, and it + // willl start a service listening here. + client?: Client; + // id -- the id of this project runner -- each project runner must have a different id, + // so the load balancer knows where to place projects and knows where a project is + // currently located. + id: string; + // start --- start the given project with the specified configuration. The configuration + // typically determines memory, disk spaces, the root filesystem image, etc. + start: (opts: { + project_id: string; + config?: Configuration; + localPath: LocalPathFunction; + sshServers?: SshServersFunction; + }) => Promise; + + // ensure rootfs and/or home are saved successfully to central file server + save: (opts: { + project_id: string; + // run save of rootfs -- default: true + rootfs?: boolean; + // run a mutagen sync flush of home -- default: true + home?: boolean; + }) => Promise; + + // ensure a specific project is not running on this runner, or + // if project_id not given, stop all projects + stop: (opts: { + project_id: string; + localPath: LocalPathFunction; + sshServers?: SshServersFunction; + force?: boolean; + }) => Promise; + + // get the status of a project here. + status: (opts: { + project_id: string; + localPath: LocalPathFunction; + sshServers?: SshServersFunction; + }) => Promise; + + move: (opts: { + project_id?: string; + force?: boolean; + server?: string; + }) => Promise; + + // local -- the absolute path on the filesystem where the home directory of this + // project is hosted. In case of a single server setup it could be the exact + // same path as the remote files and no sync is involved. + // Calling localPath may actually create the local path as a subvolume + // too (e.g,. as a btrfs volume). + localPath: LocalPathFunction; + + // sshServers -- when the project runs it connects over ssh to a server to expose + // ports and sync files. The sshServer function locates this server and provides + // the initial file sync and port forward configuration. + // - host, port - identifies the server from the point of view of the pod, e.g., + // use 'host.containers.internal' on podman on a single server, rather than + // 'localhost', for the 'pasta' network option. + // - user - the username the project should use to connect, which must identify + // the project somehow to the ssh server + // - sync - initial filesystem sync configuration, if needed. + // This is not needed on a single server deployment, but is very much needed + // when project run on a different machine than the file server. For a compute + // server it would be the list of directories to sync on startup. + // - forward - initial port forward configuration. + sshServers?: SshServersFunction; +} + +export interface API { + start: (opts?: { + project_id: string; + config?: Configuration; + }) => Promise; + stop: (opts?: { + project_id: string; + force?: boolean; + }) => Promise; + status: (opts?: { project_id: string }) => Promise; + move: (opts?: { force?: boolean }) => Promise; + save: (opts?: { + project_id?: string; + rootfs?: boolean; + home?: boolean; + }) => Promise; +} + +export async function server(options: Options) { + logger.debug(`Start project server ${options.id}`); + if (!options.id) { + throw Error("project server id MUST be specified"); + } + options.client ??= conat(); + + const { id, client, start, stop, status, save } = options; + const { projects, runners } = await state({ client }); + let running = true; + + runners.set(id, { time: Date.now() }); + until( + () => { + if (!running) { + return true; + } + runners.set(id, { time: Date.now() }); + return false; + }, + { min: UPDATE_INTERVAL, max: UPDATE_INTERVAL }, + ); + + const sub = await client.service(`project-runner.${id}`, { + async start(opts: { project_id: string; config?: Configuration }) { + logger.debug("start", opts.project_id); + projects.set(opts.project_id, { server: id, state: "starting" } as const); + await start({ + ...opts, + localPath: options.localPath, + sshServers: options.sshServers, + }); + const s = { server: id, state: "running" } as const; + projects.set(opts.project_id, s); + return s; + }, + + async stop(opts: { project_id: string; force?: boolean }) { + logger.debug("stop", opts); + projects.set(opts.project_id, { server: id, state: "stopping" } as const); + try { + await stop({ + ...opts, + localPath: options.localPath, + sshServers: options.sshServers, + }); + } catch (err) { + // couldn't stop it. + projects.set(opts.project_id, { + server: id, + state: "running", + } as const); + throw err; + } + const s = { server: id, state: "opened" } as const; + projects.set(opts.project_id, s); + return s; + }, + + async status(opts: { project_id: string }) { + logger.debug("status", opts.project_id); + const s = { + ...(await status({ + ...opts, + localPath: options.localPath, + sshServers: options.sshServers, + })), + server: id, + }; + projects.set(opts.project_id, { server: id, state: s.state } as const); + return s; + }, + + async move(_opts?: { force?: boolean }) { + // this is actually handled by the load balancer, since project runner + // might be down (as main motivation to move!) and archiving just + // involves stop and set something in projects state. + }, + + async save(opts: { + project_id: string; + rootfs?: boolean; + home?: boolean; + }): Promise { + await save(opts); + }, + }); + + return { + close: () => { + running = false; + runners.delete(id); + sub.close(); + }, + }; +} + +export interface BasicOptions { + client?: Client; + timeout?: number; + waitForInterest?: boolean; +} + +export function client({ + client, + project_id, + subject, + timeout, + waitForInterest = true, +}: + | (BasicOptions & { + project_id?: string; + subject: string; + }) + | (BasicOptions & { + project_id: string; + subject?: string; + })): API { + if (project_id && !isValidUUID(project_id)) { + throw Error(`invalid project_id ${project_id}`); + } + subject ??= `project.${project_id}.run`; + client ??= conat(); + // Note that the project_id field gets filled in automatically in the API + // because the project above is of the form project.{project_id}. + return client.call(subject, { waitForInterest, timeout }); +} diff --git a/src/packages/conat/project/runner/state.ts b/src/packages/conat/project/runner/state.ts new file mode 100644 index 00000000000..bea3dbe59c5 --- /dev/null +++ b/src/packages/conat/project/runner/state.ts @@ -0,0 +1,40 @@ +/* +The shared persistent state used by the load +balancer and all the project runners. + +From the backend package: + + client = require('@cocalc/backend/conat').conat(); a = require('@cocalc/conat/project/runner/state'); s = await a.default(client) +*/ + +import { dkv } from "@cocalc/conat/sync/dkv"; + +export interface RunnerStatus { + time: number; +} + +export type ProjectState = + | "opened" + | "starting" + | "running" + | "stopping"; + +export interface ProjectStatus { + server?: string; + state: ProjectState; + publicKey?: string; +} + +export default async function state({ client }) { + return { + projects: await dkv<{ server?: string; state: ProjectState }>({ + client, + name: "project-runner.projects", + }), + + runners: await dkv({ + client, + name: "project-runner.runners", + }), + }; +} diff --git a/src/packages/conat/project/runner/types.ts b/src/packages/conat/project/runner/types.ts new file mode 100644 index 00000000000..059a16dd527 --- /dev/null +++ b/src/packages/conat/project/runner/types.ts @@ -0,0 +1,45 @@ +export type LocalPathFunction = (opts: { + project_id: string; + // disk quota to set on the path (in bytes) + disk?: number; + // if set, create scratch space of this size in bytes and return path + // to it as scratch. + scratch?: number; +}) => Promise<{ home: string; scratch?: string }>; + +export interface SshServer { + name: string; + host: string; + port: number; + user: string; +} + +export type SshServersFunction = (opts: { + project_id: string; +}) => Promise; + +export interface Configuration { + // optional Docker image + image?: string; + // shared secret between project and hubs to enhance security (via defense in depth) + secret?: string; + // extra variables that get merged into the environment of the project. + env?: { [key: string]: string }; + // cpu priority: 1, 2 or 3, with 3 being highest + cpu?: number; + // memory limit in BYTES + memory?: number; + // swap -- enabled or not. The actual amount is a function of + // memory (above), RAM, and swap configuration on the runner itself -- see backend/podman/memory.ts + swap?: boolean; + // pid limit + pids?: number; + // disk size in bytes + disk?: number; + // if given, a /scratch is mounted in the container of this size in bytes + scratch?: number; + // if given create tmpfs ramdisk using this many bytes; if not given, + // but scratch is given, then /tmp is /scratch/tmp; if neither is + // given then tmp is part of the rootfs and is backed up (so NOT good). + tmp?: number; +} diff --git a/src/packages/conat/project/terminal/index.ts b/src/packages/conat/project/terminal/index.ts new file mode 100644 index 00000000000..e7f17fe0ae2 --- /dev/null +++ b/src/packages/conat/project/terminal/index.ts @@ -0,0 +1,411 @@ +/* +Terminal +*/ + +import { type Client as ConatClient } from "@cocalc/conat/core/client"; +import { + type ConatSocketServer, + type ServerSocket, +} from "@cocalc/conat/socket"; +import { getLogger } from "@cocalc/conat/client"; +import { ThrottleString } from "@cocalc/util/throttle"; +import { delay } from "awaiting"; +import { + createPtyWritable, + writeToWritablePty, + type Writable, +} from "./writable-pty"; +import { reuseInFlight } from "@cocalc/util/reuse-in-flight"; + +type State = "running" | "off"; + +const MAX_MSGS_PER_SECOND = parseInt( + process.env.COCALC_TERMINAL_MAX_MSGS_PER_SECOND ?? "24", +); + +const MAX_HISTORY_LENGTH = parseInt( + process.env.COCALC_TERMINAL_MAX_HISTORY_LENGTH ?? "1000000", +); + +const DEFAULT_SIZE_WAIT = 2000; + +import { EventEmitter } from "events"; + +const logger = getLogger("conat:project:terminal"); + +function getSubject({ + project_id, + compute_server_id = 0, +}: { + project_id: string; + compute_server_id?: number; +}) { + return `terminal.project-${project_id}.${compute_server_id}`; +} + +export interface Options { + cwd?: string; + env?: { [key: string]: string }; + // env0 is merged into existing environment, whereas env makes a new environment + env0?: { [key: string]: string }; + rows?: number; + cols?: number; + handleFlowControl?: boolean; + id?: string; + + // ms until throw error if backend doesn't respond + timeout?: number; +} + +const sessions: { [id: string]: any } = {}; +const history: { [id: string]: string } = {}; +const sizes: { [id: string]: { rows: number; cols: number }[] } = {}; + +export function terminalServer({ + client, + project_id, + compute_server_id = 0, + spawn, + cwd, + preHook, + postHook, +}: { + client: ConatClient; + project_id: string; + compute_server_id?: number; + // spawn a pseudo tty: + spawn: ( + command: string, + args?: string[], + options?: Options, + ) => Promise<{ pid: number }>; + // get the current working directory of the process with given pid + cwd?: (pid: number) => Promise; + preHook?: (opts: { + command: string; + args?: string[]; + options?: Options; + }) => Promise; + postHook?: (opts: { + command: string; + args?: string[]; + options?: Options; + pty; + }) => Promise; +}) { + const subject = getSubject({ project_id, compute_server_id }); + const server: ConatSocketServer = client.socket.listen(subject, { + keepAlive: 5000, + keepAliveTimeout: 5000, + }); + logger.debug("server: listening on ", { subject }); + + server.on("connection", (socket: ServerSocket) => { + logger.debug("server: got new connection", { + id: socket.id, + subject: socket.subject, + }); + + let sessionId: string | null = null; + let pty: any = null; + let wpty: Writable | null = null; + let buffer: string[] = []; + const setPty = (p) => { + pty = p; + wpty = p == null ? null : createPtyWritable(p); + buffer.length = 0; + }; + socket.on("data", (data) => { + buffer.push(data); + processBuffer(); + }); + + const processBuffer = reuseInFlight(async () => { + while (buffer.length > 0 && wpty != null) { + try { + const data = buffer.shift()!; + await writeToWritablePty(wpty, data); + } catch (err) { + logger.debug("server: writeToWritablePty", err); + return; + } + await delay(1); + } + }); + + const sendToClient = (data) => { + try { + socket.write(data); + } catch (err) { + if (err.code != "EPIPE") { + // epipe means socket is closed... + logger.debug("WARNING: error writing terminal data to socket", err); + } + } + }; + + const getClientSize = async () => { + if (!sessionId) { + return; + } + try { + const { data } = await socket.request({ cmd: "size" }); + if (data) { + sizes[sessionId] ??= []; + sizes[sessionId].push(data); + } + } catch {} + }; + + const broadcast = async (event, payload?) => { + try { + await socket.request({ cmd: "broadcast", event, payload }); + } catch {} + }; + + const removeListeners = () => { + if (pty == null) return; + pty.removeListener("data", sendToClient); + pty.removeListener("get-size", getClientSize); + pty.removeListener("broadcast", broadcast); + pty.emit("broadcast", "leave"); + }; + + socket.on("closed", removeListeners); + + const handleRequest = async ({ data }) => { + const { cmd } = data; + switch (cmd) { + case "destroy": + pty?.destroy(); + if (sessionId) { + delete sessions[sessionId]; + sessionId = null; + } + setPty(null); + return; + + case "env": + return process.env; + + case "cwd": + const pid = pty?.pid; + return pid ? cwd?.(pid) : undefined; + + case "state": + return (pty?.pid ? "running" : "off") as State; + + case "broadcast": + pty.emit("broadcast", data.event, data.payload); + return; + + case "sizes": + if (pty == null || !sessionId) { + return []; + } + sizes[sessionId] = []; + pty.emit("get-size"); + await delay(data.wait ?? DEFAULT_SIZE_WAIT); + return sizes[sessionId]; + + case "resize": + const { rows, cols } = data; + if (pty != null) { + pty.resize(cols, rows); + pty.emit("broadcast", "resize", { rows, cols }); + } + return; + + case "history": + return history[sessionId ?? ""]; + + case "spawn": + removeListeners(); + let { command, args, options = {} } = data; + const { id } = options ?? {}; + if (id) { + sessionId = id; + } + if (id && sessions[id] != null) { + setPty(sessions[id]); + } else { + if (preHook != null) { + const opts = { command, args, options }; + await preHook(opts); + ({ command, args, options } = opts); + } + if (options.env0 != null) { + options.env = { + ...(options.env ?? process.env), + ...options.env0, + }; + } + setPty(spawn(command, args, options)); + if (id) { + sessions[id] = pty; + history[id] = ""; + const maxLen = options?.maxHistoryLength ?? MAX_HISTORY_LENGTH; + pty.on("data", (data) => { + history[id] += data; + if (history[id].length > maxLen + 1000) { + history[id] = history[id].slice(-maxLen); + } + }); + } + await postHook?.({ command, args, options, pty }); + } + + const throttle = new ThrottleString(MAX_MSGS_PER_SECOND); + throttle.on("data", sendToClient); + pty.on("data", throttle.write); + + pty.once("exit", async () => { + setPty(null); + if (sessionId) { + delete sessions[sessionId]; + sessionId = null; + } + try { + await socket.request({ cmd: "exit" }); + } catch {} + }); + + pty.on("get-size", getClientSize); + pty.on("broadcast", broadcast); + + return { pid: pty.pid, history: history[id ?? ""] }; + + default: + throw Error(`unknown command '${cmd}'`); + } + }; + + socket.on("request", async (mesg) => { + try { + const resp = await handleRequest(mesg); + mesg.respondSync(resp ?? null); + } catch (err) { + logger.debug(err); + mesg.respondSync(err); + } + }); + + socket.on("closed", () => { + logger.debug("socket closed", { id: socket.id }); + }); + }); + + return server; +} + +export class TerminalClient extends EventEmitter { + public readonly socket; + public pid: number; + private getSize?: () => undefined | { rows: number; cols: number }; + + constructor({ + client, + subject, + getSize, + }: { + client: ConatClient; + subject: string; + getSize?: () => undefined | { rows: number; cols: number }; + }) { + super(); + this.getSize = getSize; + this.socket = client.socket.connect(subject); + + const handleRequest = ({ data }) => { + switch (data.cmd) { + case "size": + return this.getSize?.(); + case "broadcast": + this.emit(data.event, data.payload); + return; + case "exit": + this.emit("exit"); + return; + default: + throw new Error(`unknown message type '${data.type}'`); + } + }; + + this.socket.on("request", (mesg) => { + try { + const resp = handleRequest(mesg); + mesg.respondSync(resp ?? null); + } catch (err) { + console.warn(err); + mesg.respondSync(err); + } + }); + } + + close = () => { + this.removeAllListeners(); + try { + this.socket.close(); + } catch {} + }; + + spawn = async ( + command, + args?: string[], + options?: Options, + ): Promise => { + const { data } = await this.socket.request( + { + cmd: "spawn", + command, + args, + options, + }, + { timeout: options?.timeout }, + ); + // console.log("spawned terminal with pid", data.pid); + this.pid = data.pid; + return data.history; + }; + + destroy = async () => { + await this.socket.request({ cmd: "destroy" }); + }; + + history = async () => { + return (await this.socket.request({ cmd: "history" })).data; + }; + + env = async () => { + return (await this.socket.request({ cmd: "env" })).data; + }; + + cwd = async () => { + return (await this.socket.request({ cmd: "cwd" })).data; + }; + + state = async (): Promise => { + return (await this.socket.request({ cmd: "state" })).data; + }; + + resize = async ({ rows, cols }: { rows: number; cols: number }) => { + await this.socket.request({ cmd: "resize", rows, cols }); + }; + + sizes = async (wait?: number) => { + return (await this.socket.request({ cmd: "sizes", wait })).data; + }; + + broadcast = async (event: string, payload?) => { + await this.socket.request({ cmd: "broadcast", event, payload }); + }; +} + +export function terminalClient(opts: { + project_id: string; + compute_server_id?: number; + client: ConatClient; + getSize?: () => undefined | { rows: number; cols: number }; +}): TerminalClient { + return new TerminalClient({ ...opts, subject: getSubject(opts) }); +} diff --git a/src/packages/conat/project/terminal/writable-pty.ts b/src/packages/conat/project/terminal/writable-pty.ts new file mode 100644 index 00000000000..b76f35403a6 --- /dev/null +++ b/src/packages/conat/project/terminal/writable-pty.ts @@ -0,0 +1,65 @@ +import { Writable } from "stream"; +import { delay } from "awaiting"; + +export { type Writable }; + +export function createPtyWritable(pty): Writable { + return new Writable({ + write(chunk, _encoding, callback) { + try { + // Normalize: always pass a string to pty.write + const str = Buffer.isBuffer(chunk) ? chunk.toString("utf8") : chunk; + pty.write(str); + callback(); + } catch (err) { + callback(err); + } + }, + }); +} + +export async function writeToWritablePty( + writable: Writable, + data: string, + chunkSize = 1024, // 1 KB chunks by default +): Promise { + let offset = 0; + + while (offset < data.length) { + const chunk = data.slice(offset, offset + chunkSize); + offset += chunkSize; + + const ok = writable.write(chunk); + if (!ok) { + // Wait until PTY drains before writing the next chunk + await waitForDrain(writable); + } + await delay(1); + } +} + +function waitForDrain(stream: Writable): Promise { + return new Promise((resolve, reject) => { + function onDrain() { + cleanup(); + resolve(); + } + function onError(err) { + cleanup(); + reject(err); + } + function onClose() { + cleanup(); + reject(new Error("Stream closed before drain")); + } + function cleanup() { + stream.off("drain", onDrain); + stream.off("error", onError); + stream.off("close", onClose); + } + + stream.once("drain", onDrain); + stream.once("error", onError); + stream.once("close", onClose); + }); +} diff --git a/src/packages/conat/project/usage-info.ts b/src/packages/conat/project/usage-info.ts index 0b80f00fd61..8c9143d18f3 100644 --- a/src/packages/conat/project/usage-info.ts +++ b/src/packages/conat/project/usage-info.ts @@ -9,6 +9,7 @@ there is no request for a while. import { projectSubject } from "@cocalc/conat/names"; import { conat } from "@cocalc/conat/client"; import { getLogger } from "@cocalc/conat/client"; +import { type Client as ConatClient } from "@cocalc/conat/core/client"; import { type UsageInfo } from "@cocalc/util/types/project-usage-info"; import TTL from "@isaacs/ttlcache"; @@ -36,20 +37,22 @@ interface Api { } export async function get({ + client = conat(), project_id, compute_server_id = 0, path, }: { + client?: ConatClient; project_id: string; compute_server_id?: number; path: string; }) { - const c = await conat(); const subject = getSubject({ project_id, compute_server_id }); - return await c.call(subject).get(path); + return await client.call(subject).get(path); } interface Options { + client?: ConatClient; project_id: string; compute_server_id: number; createUsageInfoServer: Function; @@ -70,7 +73,7 @@ export class UsageInfoService { private createService = async () => { const subject = getSubject(this.options); logger.debug("starting usage-info service", { subject }); - const client = await conat(); + const client = this.options.client ?? conat(); this.service = await client.service(subject, { get: this.get, }); diff --git a/src/packages/conat/service/formatter.ts b/src/packages/conat/service/formatter.ts deleted file mode 100644 index 4fa5b5a4906..00000000000 --- a/src/packages/conat/service/formatter.ts +++ /dev/null @@ -1,46 +0,0 @@ -/* -Formatting services in a project. -*/ - -import { createServiceClient, createServiceHandler } from "./typed"; - -import type { - Options as FormatterOptions, - FormatResult, -} from "@cocalc/util/code-formatter"; - -// TODO: we may change it to NOT take compute server and have this listening from -// project and all compute servers... and have only the one with the file open -// actually reply. -interface FormatterApi { - formatter: (opts: { - path: string; - options: FormatterOptions; - }) => Promise; -} - -export function formatterClient({ compute_server_id = 0, project_id }) { - return createServiceClient({ - project_id, - compute_server_id, - service: "formatter", - }); -} - -export async function createFormatterService({ - compute_server_id = 0, - project_id, - impl, -}: { - project_id: string; - compute_server_id?: number; - impl: FormatterApi; -}) { - return await createServiceHandler({ - project_id, - compute_server_id, - service: "formatter", - description: "Code formatter API", - impl, - }); -} diff --git a/src/packages/conat/service/jupyter.ts b/src/packages/conat/service/jupyter.ts deleted file mode 100644 index a581be23b99..00000000000 --- a/src/packages/conat/service/jupyter.ts +++ /dev/null @@ -1,82 +0,0 @@ -/* -Services in a project/compute server for working with a Jupyter notebook. -*/ - -import { createServiceClient, createServiceHandler } from "./typed"; -import type { KernelInfo } from "@cocalc/util/jupyter/types"; - -const service = "api"; - -export interface JupyterApi { - signal: (signal: string) => Promise; - - save_ipynb_file: (opts?: { - version?: number; - timeout?: number; - }) => Promise; - - kernel_info: () => Promise; - - more_output: (id: string) => Promise; - - complete: (opts: { code: string; cursor_pos: number }) => Promise; - - introspect: (opts: { - code: string; - cursor_pos: number; - level: 0 | 1; - }) => Promise; - - store: (opts: { key: string; value?: any }) => Promise; - - comm: (opts: { - msg_id: string; - comm_id: string; - target_name: string; - data: any; - buffers64?: string[]; - buffers?: Buffer[]; - }) => Promise; - - ipywidgetsGetBuffer: (opts: { - model_id; - buffer_path; - }) => Promise<{ buffer64: string }>; -} - -export type JupyterApiEndpoint = keyof JupyterApi; - -export function jupyterApiClient({ - project_id, - path, - timeout, -}: { - project_id: string; - path: string; - timeout?: number; -}) { - return createServiceClient({ - project_id, - path, - service, - timeout, - }); -} - -export async function createConatJupyterService({ - path, - project_id, - impl, -}: { - project_id: string; - path: string; - impl: JupyterApi; -}) { - return await createServiceHandler({ - project_id, - path, - service, - impl, - description: "Jupyter notebook compute API", - }); -} diff --git a/src/packages/conat/service/terminal.ts b/src/packages/conat/service/terminal.ts index 4956e0506bd..eb5132be258 100644 --- a/src/packages/conat/service/terminal.ts +++ b/src/packages/conat/service/terminal.ts @@ -42,7 +42,10 @@ interface TerminalApi { close: (browser_id: string) => Promise; } -export function createTerminalClient({ project_id, termPath }) { +export function createTerminalClient({ + project_id, + termPath, +}) { return createServiceClient({ project_id, path: termPath, diff --git a/src/packages/conat/socket/base.ts b/src/packages/conat/socket/base.ts index 24d57ac1484..3f98df515f2 100644 --- a/src/packages/conat/socket/base.ts +++ b/src/packages/conat/socket/base.ts @@ -116,14 +116,16 @@ export abstract class ConatSocketBase extends EventEmitter { } if (this.reconnection) { setTimeout(() => { - this.connect(); + if (this.state != "closed") { + this.connect(); + } }, RECONNECT_DELAY); } }; connect = async () => { - if (this.state != "disconnected") { - // already connected + if (this.state != "disconnected" || !this.client) { + // already connected or closed return; } this.setState("connecting"); diff --git a/src/packages/conat/socket/client.ts b/src/packages/conat/socket/client.ts index bb107bdaeb3..1f15d24cd1c 100644 --- a/src/packages/conat/socket/client.ts +++ b/src/packages/conat/socket/client.ts @@ -91,8 +91,8 @@ export class ConatSocketClient extends ConatSocketBase { }); } - waitUntilDrain = async () => { - await this.tcp?.send.waitUntilDrain(); + drain = async () => { + await this.tcp?.send.drain(); }; private sendCommandToServer = async ( @@ -220,12 +220,16 @@ export class ConatSocketClient extends ConatSocketBase { }; request = async (data, options?) => { - await this.waitUntilReady(options?.timeout); + try { + await this.waitUntilReady(options?.timeout); + } catch { + throw Error("request timed out"); + } const subject = `${this.subject}.server.${this.id}`; if (this.state == "closed") { throw Error("closed"); } - // console.log("sending request from client ", { subject, data, options }); + //console.log("sending request from client ", { subject, data, options }); return await this.client.request(subject, data, options); }; diff --git a/src/packages/conat/socket/keepalive.ts b/src/packages/conat/socket/keepalive.ts index 3ba6e8ed065..10de7e65b40 100644 --- a/src/packages/conat/socket/keepalive.ts +++ b/src/packages/conat/socket/keepalive.ts @@ -1,9 +1,6 @@ import { delay } from "awaiting"; -import { getLogger } from "@cocalc/conat/client"; import { type Role } from "./util"; -const logger = getLogger("socket:keepalive"); - export function keepAlive(opts: { role: Role; ping: () => Promise; @@ -21,6 +18,7 @@ export class KeepAlive { private ping: () => Promise, private disconnect: () => void, private keepAlive: number, + // @ts-ignore private role: Role, ) { this.run(); @@ -29,10 +27,10 @@ export class KeepAlive { private run = async () => { while (this.state == "ready") { try { - logger.silly(this.role, "keepalive -- sending ping"); + //console.log(this.role, "keepalive -- sending ping"); await this.ping?.(); } catch (err) { - logger.silly(this.role, "keepalive -- ping failed -- disconnecting"); + //console.log(this.role, "keepalive -- ping failed -- disconnecting"); this.disconnect?.(); this.close(); return; diff --git a/src/packages/conat/socket/server-socket.ts b/src/packages/conat/socket/server-socket.ts index edfdc225a11..0b678b05dbc 100644 --- a/src/packages/conat/socket/server-socket.ts +++ b/src/packages/conat/socket/server-socket.ts @@ -51,6 +51,7 @@ export class ServerSocket extends EventEmitter { this.initKeepAlive(); } + private firstPing = true; private initKeepAlive = () => { this.alive?.close(); this.alive = keepAlive({ @@ -59,10 +60,15 @@ export class ServerSocket extends EventEmitter { await this.request(null, { headers: { [SOCKET_HEADER_CMD]: "ping" }, timeout: this.conatSocket.keepAliveTimeout, - // waitForInterest is very important in a cluster -- also, obviously + // waitForInterest for the *first ping* is very important + // in a cluster -- also, obviously // if somebody just opened a socket, they probably exist. - waitForInterest: true, + // However, after the first ping, we want to fail + // very quickly if the client disappears (and hence no + // more interest). + waitForInterest: this.firstPing, }); + this.firstPing = false; }, disconnect: this.close, keepAlive: this.conatSocket.keepAlive, @@ -232,7 +238,7 @@ export class ServerSocket extends EventEmitter { } }); - waitUntilDrain = async () => { - await this.tcp?.send.waitUntilDrain(); + drain = async () => { + await this.tcp?.send.drain(); }; } diff --git a/src/packages/conat/socket/tcp.ts b/src/packages/conat/socket/tcp.ts index 7f0e2b49948..da55e76a6ac 100644 --- a/src/packages/conat/socket/tcp.ts +++ b/src/packages/conat/socket/tcp.ts @@ -275,7 +275,7 @@ export class Sender extends EventEmitter { } }; - waitUntilDrain = reuseInFlight(async () => { + drain = reuseInFlight(async () => { if (this.unsent == 0) { return; } diff --git a/src/packages/conat/socket/util.ts b/src/packages/conat/socket/util.ts index 9e81f08439f..5274ba12068 100644 --- a/src/packages/conat/socket/util.ts +++ b/src/packages/conat/socket/util.ts @@ -13,15 +13,16 @@ export type Role = "client" | "server"; // socketio and use those to manage things. This ping // is entirely a "just in case" backup if some event // were missed (e.g., a kill -9'd process...) -export const PING_PONG_INTERVAL = 90000; +export const PING_PONG_INTERVAL = 90_000; // We queue up unsent writes, but only up to a point (to not have a huge memory issue). // Any write beyond this size result in an exception. // NOTE: in nodejs the default for exactly this is "infinite=use up all RAM", so // maybe we should make this even larger (?). // Also note that this is just the *number* of messages, and a message can have -// any size. -export const DEFAULT_MAX_QUEUE_SIZE = 1000; +// any size. But determining message size is very difficult without serializing the +// message, which costs. +export const DEFAULT_MAX_QUEUE_SIZE = 1_000; export let DEFAULT_COMMAND_TIMEOUT = 10_000; export let DEFAULT_KEEP_ALIVE = 25_000; diff --git a/src/packages/conat/sync-doc/sync-client.ts b/src/packages/conat/sync-doc/sync-client.ts new file mode 100644 index 00000000000..6fffef0117b --- /dev/null +++ b/src/packages/conat/sync-doc/sync-client.ts @@ -0,0 +1,148 @@ +/* + * This file is part of CoCalc: Copyright © 2020 Sagemath, Inc. + * License: MS-RSL – see LICENSE.md for details + */ + +import { EventEmitter } from "events"; +import { Client as Client0 } from "@cocalc/sync/editor/generic/types"; +import { parseQueryWithOptions } from "@cocalc/sync/table/util"; +import { PubSub } from "@cocalc/conat/sync/pubsub"; +import { type Client as ConatClient } from "@cocalc/conat/core/client"; +import { type ConatSyncTable } from "@cocalc/conat/sync/synctable"; +import { projectApiClient } from "@cocalc/conat/project/api"; +import { base64ToBuffer } from "@cocalc/util/base64"; +import callHub from "@cocalc/conat/hub/call-hub"; + +export class SyncClient extends EventEmitter implements Client0 { + private client: ConatClient; + constructor(client: ConatClient) { + super(); + if (client == null) { + throw Error("client must be specified"); + } + this.client = client; + this.client.once("closed", this.close); + } + + close = () => { + this.emit("closed"); + // @ts-ignore + delete this.client; + }; + + is_project = (): boolean => false; + is_browser = (): boolean => true; + is_compute_server = (): boolean => false; + + dbg = (_f: string) => { + return (..._) => {}; + }; + + is_connected = (): boolean => { + return this.client.isConnected(); + }; + + is_signed_in = (): boolean => { + return this.client.isSignedIn(); + }; + + touch_project = async (project_id): Promise => { + try { + await callHub({ + client: this.client, + account_id: this.client_id(), + name: "db.touch", + args: [{ project_id, account_id: this.client_id() }], + }); + } catch (err) { + if (err.code != 503) { // 503 when hub not running yet + console.log("WARNING: issue touching project", { project_id }, err); + } + } + }; + + is_deleted = (_filename: string, _project_id?: string): boolean => { + return false; + }; + + set_deleted = (_filename: string, _project_id?: string): void => {}; + + synctable_conat = async (query0, options?): Promise => { + const { query } = parseQueryWithOptions(query0, options); + return await this.client.sync.synctable({ + ...options, + query, + }); + }; + + pubsub_conat = async (opts): Promise => { + return new PubSub({ client: this.client, ...opts }); + }; + + // account_id or project_id or hub_id or fallback client.id + client_id = (): string => { + const user = this.client.info?.user; + return ( + user?.account_id ?? user?.project_id ?? user?.hub_id ?? this.client.id + ); + }; + + server_time = (): Date => { + return new Date(); + }; + + ipywidgetsGetBuffer = async ({ + project_id, + compute_server_id = 0, + path, + model_id, + buffer_path, + }: { + project_id: string; + compute_server_id?: number; + path: string; + model_id: string; + buffer_path: string; + }): Promise => { + const api = projectApiClient({ project_id, compute_server_id }); + const { buffer64 } = await api.jupyter.ipywidgetsGetBuffer({ + path, + model_id, + buffer_path, + }); + return base64ToBuffer(buffer64); + }; + + ///////////////////////////////// + // EVERYTHING BELOW: TO REMOVE? + mark_file = (_): void => {}; + + alert_message = (_): void => {}; + + sage_session = (_): void => {}; + + shell = (_): void => {}; + + path_access = (opts): void => { + opts.cb(true); + }; + path_stat = (opts): void => { + console.log("path_state", opts.path); + opts.cb(true); + }; + + async path_read(opts): Promise { + opts.cb(true); + } + async write_file(opts): Promise { + opts.cb(true); + } + watch_file(_): any {} + + log_error = (_): void => {}; + + query = (_): void => { + throw Error("not implemented"); + }; + query_cancel = (_): void => {}; +} diff --git a/src/packages/conat/sync-doc/syncdb.ts b/src/packages/conat/sync-doc/syncdb.ts new file mode 100644 index 00000000000..ac96e032464 --- /dev/null +++ b/src/packages/conat/sync-doc/syncdb.ts @@ -0,0 +1,27 @@ +import { SyncClient } from "./sync-client"; +import { SyncDB, type SyncDBOpts0 } from "@cocalc/sync/editor/db"; +import { type Client as ConatClient } from "@cocalc/conat/core/client"; + +export type MakeOptional = Omit & + Partial>; + +export interface SyncDBOptions + extends MakeOptional, "fs"> { + client: ConatClient; + // name of the file service that hosts this file: + service?: string; +} + +export type { SyncDB }; + +export function syncdb({ client, service, ...opts }: SyncDBOptions): SyncDB { + const fs = + opts.fs ?? + client.fs({ + service, + project_id: opts.project_id, + compute_server_id: opts.compute_server_id, + }); + const syncClient = new SyncClient(client); + return new SyncDB({ ...opts, fs, client: syncClient }); +} diff --git a/src/packages/conat/sync-doc/syncstring.ts b/src/packages/conat/sync-doc/syncstring.ts new file mode 100644 index 00000000000..f12d4f8fdd3 --- /dev/null +++ b/src/packages/conat/sync-doc/syncstring.ts @@ -0,0 +1,32 @@ +import { SyncClient } from "./sync-client"; +import { + SyncString, + type SyncStringOpts, +} from "@cocalc/sync/editor/string/sync"; +import { type Client as ConatClient } from "@cocalc/conat/core/client"; +import { type MakeOptional } from "./syncdb"; + +export interface SyncStringOptions + extends MakeOptional, "fs"> { + client: ConatClient; + // name of the file server that hosts this document: + service?: string; +} + +export type { SyncString }; + +export function syncstring({ + client, + service, + ...opts +}: SyncStringOptions): SyncString { + const fs = + opts.fs ?? + client.fs({ + service, + project_id: opts.project_id, + compute_server_id: opts.compute_server_id, + }); + const syncClient = new SyncClient(client); + return new SyncString({ ...opts, fs, client: syncClient }); +} diff --git a/src/packages/conat/sync/akv.ts b/src/packages/conat/sync/akv.ts index 39ed1daf358..e00dc27ec82 100644 --- a/src/packages/conat/sync/akv.ts +++ b/src/packages/conat/sync/akv.ts @@ -116,6 +116,10 @@ export class AKV { | undefined; }; + // if you set a key with a ttl, then we check that ttl is allowed + // and if not, set it, since by default it is off. It's a little + // more expensive on the server when allowed. + private ttlKnownAllowed = false; set = async ( key: string, value: T, @@ -128,11 +132,19 @@ export class AKV { }, ): Promise<{ seq: number; time: number }> => { const { headers, ...options0 } = options ?? {}; - return await this.stream.set({ + const ret = await this.stream.set({ key, messageData: messageData(value, { headers }), ...options0, }); + if (options?.ttl && !this.ttlKnownAllowed) { + this.ttlKnownAllowed = (await this.stream.config()).allow_msg_ttl; + if (!this.ttlKnownAllowed) { + await this.stream.config({ config: { allow_msg_ttl: true } }); + } + this.ttlKnownAllowed = true; + } + return ret; }; keys = async ({ timeout }: { timeout?: number } = {}): Promise => { diff --git a/src/packages/conat/sync/dko.ts b/src/packages/conat/sync/dko.ts index f4e773a90fa..c0c04c1b065 100644 --- a/src/packages/conat/sync/dko.ts +++ b/src/packages/conat/sync/dko.ts @@ -1,7 +1,7 @@ /* Distributed eventually consistent key:object store, where changes propogate sparsely. -The "values" MUST be objects and no keys or fields of objects can container the +The "values" MUST be objects and no keys or fields of objects can container the sep character, which is '|' by default. NOTE: Whenever you do a set, the lodash isEqual function is used to see which fields @@ -38,6 +38,7 @@ export class DKO extends EventEmitter { constructor(private opts: DKVOptions) { super(); + this.setMaxListeners(1000); return new Proxy(this, { deleteProperty(target, prop) { if (typeof prop == "string") { diff --git a/src/packages/conat/sync/dkv.ts b/src/packages/conat/sync/dkv.ts index d7824cfa6e0..db8862b0cb3 100644 --- a/src/packages/conat/sync/dkv.ts +++ b/src/packages/conat/sync/dkv.ts @@ -218,7 +218,7 @@ export class DKV extends EventEmitter { } this.kv.on("change", this.handleRemoteChange); await this.kv.init(); - // allow_msg_ttl is used for deleting tombstones. + // allow_msg_ttl is used for deleting tombstones so MUST be enabled for dkv. await this.kv.config({ allow_msg_ttl: true }); this.emit("connected"); }; diff --git a/src/packages/conat/sync/dstream.ts b/src/packages/conat/sync/dstream.ts index 4bc6121c261..26d0c997f9d 100644 --- a/src/packages/conat/sync/dstream.ts +++ b/src/packages/conat/sync/dstream.ts @@ -290,6 +290,7 @@ export class DStream extends EventEmitter { }; save = reuseInFlight(async () => { + //console.log("save", this.noAutosave); await until( async () => { if (this.isClosed()) { diff --git a/src/packages/conat/sync/open-files.ts b/src/packages/conat/sync/open-files.ts deleted file mode 100644 index b82afd85786..00000000000 --- a/src/packages/conat/sync/open-files.ts +++ /dev/null @@ -1,302 +0,0 @@ -/* -Keep track of open files. - -We use the "dko" distributed key:value store because of the potential of merge -conflicts, e.g,. one client changes the compute server id and another changes -whether a file is deleted. By using dko, only the field that changed is sync'd -out, so we get last-write-wins on the level of fields. - -WARNINGS: -An old version use dkv with merge conflict resolution, but with multiple clients -and the project, feedback loops or something happened and it would start getting -slow -- basically, merge conflicts could take a few seconds to resolve, which would -make opening a file start to be slow. Instead we use DKO data type, where fields -are treated separately atomically by the storage system. A *subtle issue* is -that when you set an object, this is NOT treated atomically. E.g., if you -set 2 fields in a set operation, then 2 distinct changes are emitted as the -two fields get set. - -DEVELOPMENT: - -Change to packages/backend, since packages/conat doesn't have a way to connect: - -~/cocalc/src/packages/backend$ node - -> z = await require('@cocalc/backend/conat/sync').openFiles({project_id:cc.current().project_id}) -> z.touch({path:'a.txt'}) -> z.get({path:'a.txt'}) -{ open: true, count: 1, time:2025-02-09T16:37:20.713Z } -> z.touch({path:'a.txt'}) -> z.get({path:'a.txt'}) -{ open: true, count: 2 } -> z.time({path:'a.txt'}) -2025-02-09T16:36:58.510Z -> z.touch({path:'foo/b.md',id:0}) -> z.getAll() -{ - 'a.txt': { open: true, count: 3 }, - 'foo/b.md': { open: true, count: 1 } - -Frontend Dev in browser: - -z = await cc.client.conat_client.openFiles({project_id:cc.current().project_id)) -z.getAll() -} -*/ - -import { type State } from "@cocalc/conat/types"; -import { dko, type DKO } from "@cocalc/conat/sync/dko"; -import { EventEmitter } from "events"; -import getTime, { getSkew } from "@cocalc/conat/time"; - -// info about interest in open files (and also what was explicitly deleted) older -// than this is automatically purged. -const MAX_AGE_MS = 1000 * 60 * 60 * 24; - -interface Deleted { - // what deleted state is - deleted: boolean; - // when deleted state set - time: number; -} - -interface Backend { - // who has it opened -- the compute_server_id (0 for project) - id: number; - // when they last reported having it opened - time: number; -} - -export interface KVEntry { - // a web browser has the file open at this point in time (in ms) - time?: number; - // if the file was removed from disk (and not immmediately written back), - // then deleted gets set to the time when this happened (in ms since epoch) - // and the file is closed on the backend. It won't be re-opened until - // either (1) the file is created on disk again, or (2) deleted is cleared. - // Note: the actual time here isn't really important -- what matter is the number - // is nonzero. It's just used for a display to the user. - // We store the deleted state *and* when this was set, so that in case - // of merge conflict we can do something sensible. - deleted?: Deleted; - - // if file is actively opened on a compute server/project, then it sets - // this entry. Right when it closes the file, it clears this. - // If it gets killed/broken and doesn't have a chance to clear it, then - // backend.time can be used to decide this isn't valid. - backend?: Backend; - - // optional information - doctype?; -} - -export interface Entry extends KVEntry { - // path to file relative to HOME - path: string; -} - -interface Options { - project_id: string; - noAutosave?: boolean; - noCache?: boolean; -} - -export async function createOpenFiles(opts: Options) { - const openFiles = new OpenFiles(opts); - await openFiles.init(); - return openFiles; -} - -export class OpenFiles extends EventEmitter { - private project_id: string; - private noCache?: boolean; - private noAutosave?: boolean; - private kv?: DKO; - public state: "disconnected" | "connected" | "closed" = "disconnected"; - - constructor({ project_id, noAutosave, noCache }: Options) { - super(); - if (!project_id) { - throw Error("project_id must be specified"); - } - this.project_id = project_id; - this.noAutosave = noAutosave; - this.noCache = noCache; - } - - private setState = (state: State) => { - this.state = state; - this.emit(state); - }; - - private initialized = false; - init = async () => { - if (this.initialized) { - throw Error("init can only be called once"); - } - this.initialized = true; - const d = await dko({ - name: "open-files", - project_id: this.project_id, - config: { - max_age: MAX_AGE_MS, - }, - noAutosave: this.noAutosave, - noCache: this.noCache, - noInventory: true, - }); - this.kv = d; - d.on("change", this.handleChange); - // ensure clock is synchronized - await getSkew(); - this.setState("connected"); - }; - - private handleChange = ({ key: path }) => { - const entry = this.get(path); - if (entry != null) { - // not deleted and timestamp is set: - this.emit("change", entry as Entry); - } - }; - - close = () => { - if (this.kv == null) { - return; - } - this.setState("closed"); - this.removeAllListeners(); - this.kv.removeListener("change", this.handleChange); - this.kv.close(); - delete this.kv; - // @ts-ignore - delete this.project_id; - }; - - private getKv = () => { - const { kv } = this; - if (kv == null) { - throw Error("closed"); - } - return kv; - }; - - private set = (path, entry: KVEntry) => { - this.getKv().set(path, entry); - }; - - // When a client has a file open, they should periodically - // touch it to indicate that it is open. - // updates timestamp and ensures open=true. - touch = (path: string, doctype?) => { - if (!path) { - throw Error("path must be specified"); - } - const kv = this.getKv(); - const cur = kv.get(path); - const time = getTime(); - if (doctype) { - this.set(path, { - ...cur, - time, - doctype, - }); - } else { - this.set(path, { - ...cur, - time, - }); - } - }; - - setError = (path: string, err?: any) => { - const kv = this.getKv(); - if (!err) { - const current = { ...kv.get(path) }; - delete current.error; - this.set(path, current); - } else { - const current = { ...kv.get(path) }; - current.error = { time: Date.now(), error: `${err}` }; - this.set(path, current); - } - }; - - setDeleted = (path: string) => { - const kv = this.getKv(); - this.set(path, { - ...kv.get(path), - deleted: { deleted: true, time: getTime() }, - }); - }; - - isDeleted = (path: string) => { - return !!this.getKv().get(path)?.deleted?.deleted; - }; - - setNotDeleted = (path: string) => { - const kv = this.getKv(); - this.set(path, { - ...kv.get(path), - deleted: { deleted: false, time: getTime() }, - }); - }; - - // set that id is the backend with the file open. - // This should be called by that backend periodically - // when it has the file opened. - setBackend = (path: string, id: number) => { - const kv = this.getKv(); - this.set(path, { - ...kv.get(path), - backend: { id, time: getTime() }, - }); - }; - - // get current backend that has file opened. - getBackend = (path: string): Backend | undefined => { - return this.getKv().get(path)?.backend; - }; - - // ONLY if backend for path is currently set to id, then clear - // the backend field. - setNotBackend = (path: string, id: number) => { - const kv = this.getKv(); - const cur = { ...kv.get(path) }; - if (cur?.backend?.id == id) { - delete cur.backend; - this.set(path, cur); - } - }; - - getAll = (): Entry[] => { - const x = this.getKv().getAll(); - return Object.keys(x).map((path) => { - return { ...x[path], path }; - }); - }; - - get = (path: string): Entry | undefined => { - const x = this.getKv().get(path); - if (x == null) { - return x; - } - return { ...x, path }; - }; - - delete = (path) => { - this.getKv().delete(path); - }; - - clear = () => { - this.getKv().clear(); - }; - - save = async () => { - await this.getKv().save(); - }; - - hasUnsavedChanges = () => { - return this.getKv().hasUnsavedChanges(); - }; -} diff --git a/src/packages/conat/sync/synctable-kv.ts b/src/packages/conat/sync/synctable-kv.ts index 7950305ed44..9e56393fff5 100644 --- a/src/packages/conat/sync/synctable-kv.ts +++ b/src/packages/conat/sync/synctable-kv.ts @@ -33,6 +33,7 @@ export class SyncTableKV extends EventEmitter { private config?: Partial; private desc?: JSONValue; private ephemeral?: boolean; + private noAutosave?: boolean; constructor({ query, @@ -44,6 +45,7 @@ export class SyncTableKV extends EventEmitter { config, desc, ephemeral, + noAutosave, }: { query; client: Client; @@ -54,6 +56,7 @@ export class SyncTableKV extends EventEmitter { config?: Partial; desc?: JSONValue; ephemeral?: boolean; + noAutosave?: boolean; }) { super(); this.setMaxListeners(1000); @@ -64,6 +67,7 @@ export class SyncTableKV extends EventEmitter { this.client = client; this.desc = desc; this.ephemeral = ephemeral; + this.noAutosave = noAutosave; this.table = keys(query)[0]; if (query[this.table][0].string_id && query[this.table][0].project_id) { this.project_id = query[this.table][0].project_id; @@ -126,6 +130,7 @@ export class SyncTableKV extends EventEmitter { config: this.config, desc: this.desc, ephemeral: this.ephemeral, + noAutosave: this.noAutosave, }); } else { this.dkv = await createDko({ @@ -136,6 +141,7 @@ export class SyncTableKV extends EventEmitter { config: this.config, desc: this.desc, ephemeral: this.ephemeral, + noAutosave: this.noAutosave, }); } // For some reason this one line confuses typescript and break building the compute server package (nothing else similar happens). diff --git a/src/packages/conat/sync/synctable-stream.ts b/src/packages/conat/sync/synctable-stream.ts index 36b1b3e27eb..597cfc8d953 100644 --- a/src/packages/conat/sync/synctable-stream.ts +++ b/src/packages/conat/sync/synctable-stream.ts @@ -45,6 +45,7 @@ export class SyncTableStream extends EventEmitter { private config?: Partial; private start_seq?: number; private noInventory?: boolean; + private noAutosave?: boolean; private ephemeral?: boolean; constructor({ @@ -57,6 +58,7 @@ export class SyncTableStream extends EventEmitter { start_seq, noInventory, ephemeral, + noAutosave, }: { query; client: Client; @@ -67,10 +69,12 @@ export class SyncTableStream extends EventEmitter { start_seq?: number; noInventory?: boolean; ephemeral?: boolean; + noAutosave?: boolean; }) { super(); this.client = client; this.noInventory = noInventory; + this.noAutosave = noAutosave; this.ephemeral = ephemeral; this.setMaxListeners(1000); this.getHook = immutable ? fromJS : (x) => x; @@ -107,6 +111,7 @@ export class SyncTableStream extends EventEmitter { start_seq: this.start_seq, noInventory: this.noInventory, ephemeral: this.ephemeral, + noAutosave: this.noAutosave, }); this.dstream.on("change", (mesg) => { this.handle(mesg, true); diff --git a/src/packages/conat/sync/synctable.ts b/src/packages/conat/sync/synctable.ts index 8f69000eee9..95048b86d2f 100644 --- a/src/packages/conat/sync/synctable.ts +++ b/src/packages/conat/sync/synctable.ts @@ -43,6 +43,7 @@ export interface SyncTableOptions { start_seq?: number; noInventory?: boolean; ephemeral?: boolean; + noAutosave?: boolean; } export const createSyncTable = refCache({ diff --git a/src/packages/conat/tsconfig.json b/src/packages/conat/tsconfig.json index 687201523d0..5a8dd8655a1 100644 --- a/src/packages/conat/tsconfig.json +++ b/src/packages/conat/tsconfig.json @@ -6,5 +6,5 @@ }, "exclude": ["node_modules", "dist", "test"], "references_comment": "Do not define path:../comm because that causes a circular references.", - "references": [{ "path": "../util" }] + "references": [{ "path": "../util", "path": "../sync" }] } diff --git a/src/packages/conat/util.ts b/src/packages/conat/util.ts index 670fede4d9e..668b4b228bc 100644 --- a/src/packages/conat/util.ts +++ b/src/packages/conat/util.ts @@ -3,6 +3,32 @@ import { encode as encodeBase64, decode as decodeBase64 } from "js-base64"; export { encodeBase64, decodeBase64 }; import { reuseInFlight } from "@cocalc/util/reuse-in-flight"; +export class ConatError extends Error { + code?: string | number; + subject?: string; + constructor( + mesg: string, + { code, subject }: { code?: string | number; subject?: string } = {}, + ) { + super(mesg); + this.code = code; + this.subject = subject; + } +} + +export function headerToError(headers): ConatError { + const err = Error(headers.error); + if (headers.error_attrs) { + for (const field in headers.error_attrs) { + err[field] = headers.error_attrs[field]; + } + } + if (err["code"] === undefined && headers.code) { + err["code"] = headers.code; + } + return err; +} + export function handleErrorMessage(mesg) { if (mesg?.error) { if (mesg.error.startsWith("Error: ")) { diff --git a/src/packages/database/conat/changefeed-api.ts b/src/packages/database/conat/changefeed-api.ts index f725236d55c..b5ef6b1e2c2 100644 --- a/src/packages/database/conat/changefeed-api.ts +++ b/src/packages/database/conat/changefeed-api.ts @@ -29,7 +29,6 @@ import { changefeedServer, type ConatSocketServer, } from "@cocalc/conat/hub/changefeeds"; - import { db } from "@cocalc/database"; import { conat } from "@cocalc/backend/conat"; diff --git a/src/packages/database/package.json b/src/packages/database/package.json index 15b29d94fe3..3b08e129426 100644 --- a/src/packages/database/package.json +++ b/src/packages/database/package.json @@ -35,7 +35,7 @@ "validator": "^13.6.0" }, "devDependencies": { - "@types/lodash": "^4.14.202", + "@types/lodash": "^4.17.20", "@types/node": "^18.16.14", "@types/pg": "^8.6.1", "coffeescript": "^2.5.1" @@ -45,7 +45,7 @@ "build": "../node_modules/.bin/tsc --build && coffee -c -o dist/ ./", "clean": "rm -rf dist", "tsc": "../node_modules/.bin/tsc --watch --pretty --preserveWatchOutput", - "test": "pnpm exec jest --forceExit --runInBand", + "test": "pnpm exec jest --forceExit --maxWorkers=1", "depcheck": "pnpx depcheck | grep -Ev '\\.coffee|coffee$'", "prepublishOnly": "pnpm test" }, diff --git a/src/packages/database/pool/cached.ts b/src/packages/database/pool/cached.ts index 68677e25af5..0747c66884f 100644 --- a/src/packages/database/pool/cached.ts +++ b/src/packages/database/pool/cached.ts @@ -24,8 +24,7 @@ of multiple projects. import { reuseInFlight } from "@cocalc/util/reuse-in-flight"; import LRU from "lru-cache"; -import { Pool } from "pg"; - +import { type Pool } from "pg"; import getLogger from "@cocalc/backend/logger"; import getPool from "./pool"; diff --git a/src/packages/database/pool/pg-utc-normalize.ts b/src/packages/database/pool/pg-utc-normalize.ts new file mode 100644 index 00000000000..f091d462f1d --- /dev/null +++ b/src/packages/database/pool/pg-utc-normalize.ts @@ -0,0 +1,69 @@ +// see https://chatgpt.com/share/68c2df30-1f30-800e-94de-38fbbcdc88bd +// Basically this ensures input Date objects are formated as iso strings, +// so they are interpreted as UTC time properly. The root cause is that +// our database schema uses "timestamp without timezone" everywhere, and +// it would be painful to migrate everything. ANY query using +// pool.query('...', params) +// that potentially has Date's in params should pass the params through normalizeParams. +// This is taken care of automatically in getPool and the db class. + +import type { Pool, QueryConfig } from "pg"; + +function normalizeValue(v: any): any { + if (v instanceof Date) return v.toISOString(); + if (Array.isArray(v)) return v.map(normalizeValue); + return v; +} + +export function normalizeValues(values?: any[]): any[] | undefined { + return Array.isArray(values) ? values.map(normalizeValue) : values; +} + +function normalizeQueryArgs(args: any[]): any[] { + // Forms: + // 1) query(text) + // 2) query(text, values) + // 3) query(text, values, callback) + // 4) query(config) + // 5) query(config, callback) + if (typeof args[0] === "string") { + if (Array.isArray(args[1])) { + const v = normalizeValues(args[1]); + if (args.length === 2) return [args[0], v]; + // callback in position 2 + return [args[0], v, args[2]]; + } + // only text (or text, callback) + return args; + } else { + // config object path + const cfg: QueryConfig = { ...args[0] }; + if ("values" in cfg && Array.isArray(cfg.values)) { + cfg.values = normalizeValues(cfg.values)!; + } + if (args.length === 1) return [cfg]; + return [cfg, args[1]]; // callback passthrough + } +} + +export function patchPoolForUtc(pool: Pool): Pool { + if ((pool as any).__utcNormalized) return pool; + + // Patch pool.query + const origPoolQuery = pool.query.bind(pool); + (pool as any).query = function (...args: any[]) { + return origPoolQuery(...normalizeQueryArgs(args)); + } as typeof pool.query; + + pool.on("connect", (client) => { + if ((client as any).__utcNormalized) return; + const origQuery = client.query.bind(client); + client.query = function (...args: any[]) { + return origQuery(...normalizeQueryArgs(args)); + } as typeof client.query; + (client as any).__utcNormalized = true; + }); + + (pool as any).__utcNormalized = true; + return pool; +} diff --git a/src/packages/database/pool/pool.ts b/src/packages/database/pool/pool.ts index c4a020e32fc..d233b22f67a 100644 --- a/src/packages/database/pool/pool.ts +++ b/src/packages/database/pool/pool.ts @@ -15,13 +15,18 @@ import { getLogger } from "@cocalc/backend/logger"; import { STATEMENT_TIMEOUT_MS } from "../consts"; import getCachedPool, { CacheTime } from "./cached"; import dbPassword from "./password"; +import { types } from "pg"; +export * from "./util"; +import { patchPoolForUtc } from "./pg-utc-normalize"; const L = getLogger("db:pool"); -export * from "./util"; - let pool: Pool | undefined = undefined; +// This makes it so when we read dates out, if they are in a "timestamp with no timezone" field in the +// database, then they are interpreted as having been UTC, which is always what we do. +types.setTypeParser(1114, (str: string) => new Date(str + " UTC")); + export default function getPool(cacheTime?: CacheTime): Pool { if (cacheTime != null) { return getCachedPool(cacheTime); @@ -39,8 +44,12 @@ export default function getPool(cacheTime?: CacheTime): Pool { // the test suite assumes small pool, or there will be random failures sometimes (?) max: process.env.PGDATABASE == TEST ? 2 : undefined, ssl, + options: "-c timezone=UTC", // ← make the session time zone UTC }); + // make Dates always UTC ISO going in + patchPoolForUtc(pool); + pool.on("error", (err: Error) => { L.debug("WARNING: Unexpected error on idle client in PG pool", { err: err.message, @@ -105,7 +114,10 @@ export async function initEphemeralDatabase({ database: "smc", statement_timeout: STATEMENT_TIMEOUT_MS, ssl, + options: "-c timezone=UTC", // ← make the session time zone UTC }); + patchPoolForUtc(db); + db.on("error", (err: Error) => { L.debug("WARNING: Unexpected error on idle client in PG pool", { err: err.message, diff --git a/src/packages/database/postgres-base.coffee b/src/packages/database/postgres-base.coffee index 7326fb0e9bc..f6b917bfb91 100644 --- a/src/packages/database/postgres-base.coffee +++ b/src/packages/database/postgres-base.coffee @@ -41,6 +41,8 @@ winston = require('@cocalc/backend/logger').getLogger('postgres') { quoteField } = require('./postgres/schema/util') { primaryKey, primaryKeys } = require('./postgres/schema/table') +{ normalizeValues } = require('./pool/pg-utc-normalize') + misc_node = require('@cocalc/backend/misc_node') { sslConfigToPsqlEnv, pghost, pgdatabase, pguser, pgssl } = require("@cocalc/backend/data") @@ -271,6 +273,7 @@ class exports.PostgreSQL extends EventEmitter # emits a 'connect' event whene password : @_password database : @_database ssl : @_ssl + options : "-c timezone=UTC" # make the session time zone UTC statement_timeout: DEFAULT_STATEMENT_TIMEOUT_MS # we set a statement_timeout, to avoid queries locking up PG if @_notification? client.on('notification', @_notification) @@ -817,7 +820,7 @@ class exports.PostgreSQL extends EventEmitter # emits a 'connect' event whene dbg("run query with specific postgres parameters in a transaction") do_query_with_pg_params(client: client, query: opts.query, params: opts.params, pg_params:opts.pg_params, cb: query_cb) else - client.query(opts.query, opts.params, query_cb) + client.query(opts.query, normalizeValues(opts.params), query_cb) catch e # this should never ever happen diff --git a/src/packages/database/postgres-server-queries.coffee b/src/packages/database/postgres-server-queries.coffee index 423fedbc844..e3f0eccb421 100644 --- a/src/packages/database/postgres-server-queries.coffee +++ b/src/packages/database/postgres-server-queries.coffee @@ -1517,7 +1517,7 @@ exports.extend_PostgreSQL = (ext) -> class PostgreSQL extends ext opts = defaults opts, project_id : required path : required - listing : required # files in path [{name:..., isdir:boolean, ....}, ...] + listing : required # files in path [{name:..., isDir:boolean, ....}, ...] cb : required # Get all public paths for the given project_id, then check if path is "in" one according # to the definition in misc. diff --git a/src/packages/database/postgres-user-queries.coffee b/src/packages/database/postgres-user-queries.coffee index 2cb8f7754d7..34304cbacfe 100644 --- a/src/packages/database/postgres-user-queries.coffee +++ b/src/packages/database/postgres-user-queries.coffee @@ -1762,13 +1762,6 @@ awaken_project = (db, project_id, cb) -> cb() catch err cb("error starting project = #{err}") - (cb) -> - if not db.ensure_connection_to_project? - cb() - return - dbg("also make sure there is a connection from hub to project") - # This is so the project can find out that the user wants to save a file (etc.) - db.ensure_connection_to_project(project_id, cb) ], (err) -> if err dbg("awaken project error -- #{err}") diff --git a/src/packages/database/postgres/set-pg-params.ts b/src/packages/database/postgres/set-pg-params.ts index 12a95b8853a..08f1a756eee 100644 --- a/src/packages/database/postgres/set-pg-params.ts +++ b/src/packages/database/postgres/set-pg-params.ts @@ -8,6 +8,7 @@ import { Client } from "pg"; import { getLogger } from "@cocalc/backend/logger"; +import { normalizeValues } from "../pool/pg-utc-normalize"; const L = getLogger("db:set-pg-params").debug; @@ -33,7 +34,7 @@ export async function do_query_with_pg_params(opts: Opts): Promise { L(`Setting query param: ${k}=${v}`); await client.query(q); } - const res = await client.query(query, params); + const res = await client.query(query, normalizeValues(params)); await client.query("COMMIT"); cb(undefined, res); } catch (err) { diff --git a/src/packages/database/postgres/types.ts b/src/packages/database/postgres/types.ts index 939a4b81cff..de4208b5ef0 100644 --- a/src/packages/database/postgres/types.ts +++ b/src/packages/database/postgres/types.ts @@ -355,8 +355,6 @@ export interface PostgreSQL extends EventEmitter { projectControl?: (project_id: string) => Project; - ensure_connection_to_project?: (project_id: string, cb?: CB) => Promise; - get_blob(opts: { uuid: string; save_in_db?: boolean; diff --git a/src/packages/file-server/btrfs/bees.ts b/src/packages/file-server/btrfs/bees.ts new file mode 100644 index 00000000000..30c71fc26a5 --- /dev/null +++ b/src/packages/file-server/btrfs/bees.ts @@ -0,0 +1,85 @@ +/* +Automate running BEES on the btrfs pool. +*/ + +import { spawn } from "node:child_process"; +import { delay } from "awaiting"; +import getLogger from "@cocalc/backend/logger"; +import { sudo } from "./util"; +import { exists } from "@cocalc/backend/misc/async-utils-node"; +import { join } from "node:path"; + +const logger = getLogger("file-server:btrfs:bees"); + +interface Options { + // average load target: default=1 + loadavgTarget?: number; + // 0-8: default 1 + verbose?: number; + // hash table size: default 1G + size?: string; +} + +const children: any[] = []; +export default async function bees( + mountpoint: string, + { loadavgTarget = 1, verbose = 1, size = "1G" }: Options = {}, +) { + const beeshome = join(mountpoint, ".beeshome"); + if (!(await exists(beeshome))) { + await sudo({ command: "btrfs", args: ["subvolume", "create", beeshome] }); + // disable COW + await sudo({ command: "chattr", args: ["+C", beeshome] }); + } + const dat = join(beeshome, "beeshash.dat"); + if (!(await exists(dat))) { + await sudo({ command: "truncate", args: ["-s", size, dat] }); + await sudo({ command: "chmod", args: ["700", dat] }); + } + + const args: string[] = ["bees", "-v", `${verbose}`]; + if (loadavgTarget) { + args.push("-g", `${loadavgTarget}`); + } + args.push(mountpoint); + logger.debug(`Running 'sudo ${args.join(" ")}'`); + const child = spawn("sudo", args); + children.push(child); + let error: string = ""; + child.once("error", (err) => { + error = `${err}`; + }); + let stderr = ""; + const f = (chunk: Buffer) => { + stderr += chunk.toString(); + }; + child.stderr.on("data", f); + await delay(1000); + if (error) { + error += stderr; + } else if (child.exitCode) { + error = `failed to started bees: ${stderr}`; + } + if (error) { + logger.debug("ERROR: ", error); + child.kill("SIGKILL"); + throw error; + } + child.stderr.removeListener("data", f); + return child; +} + +export function close() { + for (const child of children) { + child.kill("SIGINT"); + setTimeout(() => child.kill("SIGKILL"), 1000); + } + children.length = 0; +} + +process.once("exit", close); +["SIGINT", "SIGTERM", "SIGQUIT"].forEach((sig) => { + process.once(sig, () => { + process.exit(); + }); +}); diff --git a/src/packages/file-server/btrfs/filesystem.ts b/src/packages/file-server/btrfs/filesystem.ts index 927fb23548f..8ebc7986c40 100644 --- a/src/packages/file-server/btrfs/filesystem.ts +++ b/src/packages/file-server/btrfs/filesystem.ts @@ -7,68 +7,77 @@ Start node, then: DEBUG="cocalc:*file-server*" DEBUG_CONSOLE=yes node -a = require('@cocalc/file-server/btrfs'); fs = await a.filesystem({device:'/tmp/btrfs.img', formatIfNeeded:true, mount:'/mnt/btrfs', uid:293597964}) +a = require('@cocalc/file-server/btrfs'); fs = await a.filesystem({image:'/tmp/btrfs.img', mount:'/mnt/btrfs', size:'2G'}) */ import refCache from "@cocalc/util/refcache"; -import { mkdirp, btrfs, sudo } from "./util"; -import { join } from "path"; +import { mkdirp, btrfs, sudo, ensureMoreLoopbackDevices } from "./util"; import { Subvolumes } from "./subvolumes"; -import { mkdir } from "fs/promises"; +import { mkdir } from "node:fs/promises"; import { exists } from "@cocalc/backend/misc/async-utils-node"; -import { executeCode } from "@cocalc/backend/execute-code"; - -// default size of btrfs filesystem if creating an image file. -const DEFAULT_FILESYSTEM_SIZE = "10G"; - -// default for newly created subvolumes -export const DEFAULT_SUBVOLUME_SIZE = "1G"; - -const MOUNT_ERROR = "wrong fs type, bad option, bad superblock"; +import rustic from "@cocalc/backend/sandbox/rustic"; +import { until } from "@cocalc/util/async-utils"; +import { delay } from "awaiting"; +import { FileSync } from "./sync"; +import bees from "./bees"; +import { type ChildProcess } from "node:child_process"; export interface Options { - // the underlying block device. - // If this is a file (or filename) ending in .img, then it's a sparse file mounted as a loopback device. - // If this starts with "/dev" then it is a raw block device. - device: string; - // formatIfNeeded -- DANGEROUS! if true, format the device or image, - // if it doesn't mount with an error containing "wrong fs type, - // bad option, bad superblock". Never use this in production. Useful - // for testing and dev. - formatIfNeeded?: boolean; - // where the btrfs filesystem is mounted + // mount = root mountpoint of the btrfs filesystem. If you specify the image + // path below, then a btrfs filesystem will get automatically created (via sudo + // and a loopback device). mount: string; - // default size of newly created subvolumes - defaultSize?: string | number; - defaultFilesystemSize?: string | number; + // image = optionally use a image file at this location for the btrfs filesystem. + // This is used for **development** (not a serious deployment). It will be + // created as a sparse image file + // with given size, and mounted at opts.mount if it does not exist. If you create + // it be sure to use mkfs.btrfs to format it. + image?: string; + size?: string | number; + + // rustic = the rustic backups path. + // If this path ends in .toml, it is the configuration file for rustic, e.g., you can + // configure rustic however you want by pointing this at a toml cofig file. + // Otherwise, if this path does not exist, it will be created a new rustic repo + // initialized here. + rustic: string; } +let mountLock = false; + export class Filesystem { public readonly opts: Options; - public readonly bup: string; public readonly subvolumes: Subvolumes; + public readonly fileSync: FileSync; + private bees?: ChildProcess; constructor(opts: Options) { - opts = { - defaultSize: DEFAULT_SUBVOLUME_SIZE, - defaultFilesystemSize: DEFAULT_FILESYSTEM_SIZE, - ...opts, - }; this.opts = opts; - this.bup = join(this.opts.mount, "bup"); this.subvolumes = new Subvolumes(this); + this.fileSync = new FileSync(this); } init = async () => { await mkdirp([this.opts.mount]); await this.initDevice(); await this.mountFilesystem(); + await this.sync(); + await this.fileSync.init(); + // 'quota enable --simple' has a lot of subtle issues, and maybe isn't for us. + // It also resets to zero when you disable then enable, and there is no efficient + // way to get the numbers. await btrfs({ - args: ["quota", "enable", "--simple", this.opts.mount], + args: ["quota", "enable", this.opts.mount], }); - await this.initBup(); + await this.initRustic(); + await this.sync(); + this.bees = await bees(this.opts.mount); + }; + + sync = async () => { + await btrfs({ args: ["filesystem", "sync", this.opts.mount] }); }; unmount = async () => { @@ -79,18 +88,22 @@ export class Filesystem { }); }; - close = () => {}; + close = () => { + this.bees?.kill("SIGQUIT"); + this.fileSync.close(); + }; private initDevice = async () => { - if (!isImageFile(this.opts.device)) { - // raw block device -- nothing to do + if (!this.opts.image) { return; } - if (!(await exists(this.opts.device))) { + if (!(await exists(this.opts.image))) { + // we create and format the sparse image await sudo({ command: "truncate", - args: ["-s", `${this.opts.defaultFilesystemSize}`, this.opts.device], + args: ["-s", `${this.opts.size ?? "10G"}`, this.opts.image], }); + await sudo({ command: "mkfs.btrfs", args: [this.opts.image] }); } }; @@ -115,82 +128,93 @@ export class Filesystem { } catch {} const { stderr, exit_code } = await this._mountFilesystem(); if (exit_code) { - if (stderr.includes(MOUNT_ERROR)) { - if (this.opts.formatIfNeeded) { - await this.formatDevice(); - const { stderr, exit_code } = await this._mountFilesystem(); - if (exit_code) { - throw Error(stderr); - } else { - return; - } - } - } throw Error(stderr); } }; - private formatDevice = async () => { - await sudo({ command: "mkfs.btrfs", args: [this.opts.device] }); - }; - private _mountFilesystem = async () => { - const args: string[] = isImageFile(this.opts.device) ? ["-o", "loop"] : []; - args.push( - "-o", - "compress=zstd", - "-o", - "noatime", - "-o", - "space_cache=v2", - "-o", - "autodefrag", - this.opts.device, - "-t", - "btrfs", - this.opts.mount, - ); - { + if (!this.opts.image) { + throw Error(`there must be a btrfs image at ${this.opts.image}`); + } + await until(() => !mountLock); + try { + mountLock = true; + const args: string[] = ["-o", "loop"]; + args.push( + "-o", + "compress=zstd", + "-o", + "noatime", + "-o", + "space_cache=v2", + "-o", + "autodefrag", + this.opts.image, + "-t", + "btrfs", + this.opts.mount, + ); + { + const { exit_code: failed } = await sudo({ + command: "mount", + args, + err_on_exit: false, + }); + if (failed) { + // try again with more loopback devices + await ensureMoreLoopbackDevices(); + const { stderr, exit_code } = await sudo({ + command: "mount", + args, + err_on_exit: false, + }); + if (exit_code) { + return { stderr, exit_code }; + } + } + } + await until( + async () => { + try { + await sudo({ + command: "df", + args: ["-t", "btrfs", this.opts.mount], + }); + return true; + } catch (err) { + console.log(err); + return false; + } + }, + { min: 250 }, + ); const { stderr, exit_code } = await sudo({ - command: "mount", - args, + command: "chown", + args: [ + `${process.getuid?.() ?? 0}:${process.getgid?.() ?? 0}`, + this.opts.mount, + ], err_on_exit: false, }); - if (exit_code) { - return { stderr, exit_code }; - } + return { stderr, exit_code }; + } finally { + await delay(1000); + mountLock = false; } - const { stderr, exit_code } = await sudo({ - command: "chown", - args: [ - `${process.getuid?.() ?? 0}:${process.getgid?.() ?? 0}`, - this.opts.mount, - ], - err_on_exit: false, - }); - return { stderr, exit_code }; }; - private initBup = async () => { - if (!(await exists(this.bup))) { - await mkdir(this.bup); + private initRustic = async () => { + if (!this.opts.rustic || (await exists(this.opts.rustic))) { + return; } - await executeCode({ - command: "bup", - args: ["init"], - env: { BUP_DIR: this.bup }, - }); + if (this.opts.rustic.endsWith(".toml")) { + throw Error(`file not found: ${this.opts.rustic}`); + } + await mkdir(this.opts.rustic); + await rustic(["init"], { repo: this.opts.rustic }); }; } -function isImageFile(name: string) { - if (name.startsWith("/dev")) { - return false; - } - // TODO: could probably check os for a device with given name? - return name.endsWith(".img"); -} - const cache = refCache({ name: "btrfs-filesystems", createObject: async (options: Options) => { diff --git a/src/packages/file-server/btrfs/index.ts b/src/packages/file-server/btrfs/index.ts index edfd27c3a9b..92d7a65afb5 100644 --- a/src/packages/file-server/btrfs/index.ts +++ b/src/packages/file-server/btrfs/index.ts @@ -1 +1,3 @@ export { filesystem } from "./filesystem"; +export { type Filesystem } from "./filesystem"; +export { type FileSync } from "./sync"; diff --git a/src/packages/file-server/btrfs/snapshots.ts b/src/packages/file-server/btrfs/snapshots.ts index daf8e09212c..215811a2480 100644 --- a/src/packages/file-server/btrfs/snapshots.ts +++ b/src/packages/file-server/btrfs/snapshots.ts @@ -1,40 +1,26 @@ import { type SubvolumeSnapshots } from "./subvolume-snapshots"; +import { type SubvolumeRustic } from "./subvolume-rustic"; +import { + SNAPSHOT_INTERVALS_MS, + DEFAULT_SNAPSHOT_COUNTS, + type SnapshotCounts, +} from "@cocalc/util/consts/snapshots"; import getLogger from "@cocalc/backend/logger"; +import { isISODate } from "@cocalc/util/misc"; -const logger = getLogger("file-server:btrfs:snapshots"); - -const DATE_REGEXP = /^\d{4}-\d{2}-\d{2}T\d{2}:\d{2}:\d{2}\.\d{3}Z$/; - -// Lengths of time in minutes to keep snapshots -// (code below assumes these are listed in ORDER from shortest to longest) -export const SNAPSHOT_INTERVALS_MS = { - frequent: 15 * 1000 * 60, - daily: 60 * 24 * 1000 * 60, - weekly: 60 * 24 * 7 * 1000 * 60, - monthly: 60 * 24 * 7 * 4 * 1000 * 60, -}; +export { type SnapshotCounts }; -// How many of each type of snapshot to retain -export const DEFAULT_SNAPSHOT_COUNTS = { - frequent: 24, - daily: 14, - weekly: 7, - monthly: 4, -} as SnapshotCounts; - -export interface SnapshotCounts { - frequent: number; - daily: number; - weekly: number; - monthly: number; -} +const logger = getLogger("file-server:btrfs:snapshots"); export async function updateRollingSnapshots({ snapshots, counts, + opts, }: { - snapshots: SubvolumeSnapshots; + snapshots: SubvolumeSnapshots | SubvolumeRustic; counts?: Partial; + // options to create + opts?; }) { counts = { ...DEFAULT_SNAPSHOT_COUNTS, ...counts }; @@ -44,46 +30,69 @@ export async function updateRollingSnapshots({ counts, changed, }); - if (!changed) { - // definitely no data written since most recent snapshot, so nothing to do - return; - } // get exactly the iso timestamp snapshot names: - const snapshotNames = (await snapshots.ls()) - .map((x) => x.name) - .filter((name) => DATE_REGEXP.test(name)); + const snapshotNames = (await snapshots.readdir()).filter(isISODate); snapshotNames.sort(); - if (snapshotNames.length > 0) { - const age = Date.now() - new Date(snapshotNames.slice(-1)[0]).valueOf(); + let needNewSnapshot = false; + if (changed) { + const timeSinceLastSnapshot = + snapshotNames.length == 0 + ? 1e12 // infinitely old + : Date.now() - new Date(snapshotNames.slice(-1)[0]).valueOf(); for (const key in SNAPSHOT_INTERVALS_MS) { - if (counts[key]) { - if (age < SNAPSHOT_INTERVALS_MS[key]) { - // no need to snapshot since there is already a sufficiently recent snapshot - logger.debug("updateRollingSnapshots: no need to snapshot", { - name: snapshots.subvolume.name, - }); - return; - } - // counts[key] nonzero and snapshot is old enough so we'll be making a snapshot + if (counts[key] && timeSinceLastSnapshot > SNAPSHOT_INTERVALS_MS[key]) { + // there is NOT a sufficiently recent snapshot to satisfy the constraint + // of having at least one snapshot for the given interval. + needNewSnapshot = true; break; } } } - // make a new snapshot - const name = new Date().toISOString(); - await snapshots.create(name); + // Regarding error reporting we try to do everything below and throw the + // create error or last delete error... + + let createError: any = undefined; + if (changed && needNewSnapshot) { + // make a new snapshot -- but only bother + // definitely no data written since most recent snapshot, so nothing to do + const name = new Date().toISOString(); + logger.debug( + "updateRollingSnapshots: creating snapshot of", + snapshots.subvolume.name, + ); + try { + await snapshots.create(name, opts); + snapshotNames.push(name); + } catch (err) { + createError = err; + } + } + // delete extra snapshots - snapshotNames.push(name); const toDelete = snapshotsToDelete({ counts, snapshots: snapshotNames }); - for (const expired of toDelete) { + let deleteError: any = undefined; + for (const name of toDelete) { try { - await snapshots.delete(expired); - } catch { - // some snapshots can't be deleted, e.g., they were used for the last send. + logger.debug( + "updateRollingSnapshots: deleting snapshot of", + snapshots.subvolume.name, + name, + ); + await snapshots.delete(name); + } catch (err) { + // ONLY report this if create doesn't error, to give both delete and create a chance to run. + deleteError = err; } } + + if (createError) { + throw createError; + } + if (deleteError) { + throw deleteError; + } } function snapshotsToDelete({ counts, snapshots }): string[] { diff --git a/src/packages/file-server/btrfs/subvolume-bup.ts b/src/packages/file-server/btrfs/subvolume-bup.ts deleted file mode 100644 index 21cbbff3646..00000000000 --- a/src/packages/file-server/btrfs/subvolume-bup.ts +++ /dev/null @@ -1,192 +0,0 @@ -/* - -BUP Architecture: - -There is a single global dedup'd backup archive stored in the btrfs filesystem. -Obviously, admins should rsync this regularly to a separate location as a genuine -backup strategy. - -NOTE: we use bup instead of btrfs send/recv ! - -Not used. Instead we will rely on bup (and snapshots of the underlying disk) for backups, since: - - much easier to check they are valid - - decoupled from any btrfs issues - - not tied to any specific filesystem at all - - easier to offsite via incremental rsync - - much more space efficient with *global* dedup and compression - - bup is really just git, which is much more proven than even btrfs - -The drawback is speed, but that can be managed. -*/ - -import { type DirectoryListingEntry } from "@cocalc/util/types"; -import { type Subvolume } from "./subvolume"; -import { sudo, parseBupTime } from "./util"; -import { join, normalize } from "path"; -import getLogger from "@cocalc/backend/logger"; - -const BUP_SNAPSHOT = "temp-bup-snapshot"; - -const logger = getLogger("file-server:btrfs:subvolume-bup"); - -export class SubvolumeBup { - constructor(private subvolume: Subvolume) {} - - // create a new bup backup - save = async ({ - // timeout used for bup index and bup save commands - timeout = 30 * 60 * 1000, - }: { timeout?: number } = {}) => { - if (await this.subvolume.snapshots.exists(BUP_SNAPSHOT)) { - logger.debug(`createBupBackup: deleting existing ${BUP_SNAPSHOT}`); - await this.subvolume.snapshots.delete(BUP_SNAPSHOT); - } - try { - logger.debug( - `createBackup: creating ${BUP_SNAPSHOT} to get a consistent backup`, - ); - await this.subvolume.snapshots.create(BUP_SNAPSHOT); - const target = this.subvolume.normalize( - this.subvolume.snapshots.path(BUP_SNAPSHOT), - ); - - logger.debug(`createBupBackup: indexing ${BUP_SNAPSHOT}`); - await sudo({ - command: "bup", - args: [ - "-d", - this.subvolume.filesystem.bup, - "index", - "--exclude", - join(target, ".snapshots"), - "-x", - target, - ], - timeout, - }); - - logger.debug(`createBackup: saving ${BUP_SNAPSHOT}`); - await sudo({ - command: "bup", - args: [ - "-d", - this.subvolume.filesystem.bup, - "save", - "--strip", - "-n", - this.subvolume.name, - target, - ], - timeout, - }); - } finally { - logger.debug(`createBupBackup: deleting temporary ${BUP_SNAPSHOT}`); - await this.subvolume.snapshots.delete(BUP_SNAPSHOT); - } - }; - - restore = async (path: string) => { - // path -- branch/revision/path/to/dir - if (path.startsWith("/")) { - path = path.slice(1); - } - path = normalize(path); - // ... but to avoid potential data loss, we make a snapshot before deleting it. - await this.subvolume.snapshots.create(); - const i = path.indexOf("/"); // remove the commit name - // remove the target we're about to restore - await this.subvolume.fs.rm(path.slice(i + 1), { recursive: true }); - await sudo({ - command: "bup", - args: [ - "-d", - this.subvolume.filesystem.bup, - "restore", - "-C", - this.subvolume.path, - join(`/${this.subvolume.name}`, path), - "--quiet", - ], - }); - }; - - ls = async (path: string = ""): Promise => { - if (!path) { - const { stdout } = await sudo({ - command: "bup", - args: ["-d", this.subvolume.filesystem.bup, "ls", this.subvolume.name], - }); - const v: DirectoryListingEntry[] = []; - let newest = 0; - for (const x of stdout.trim().split("\n")) { - const name = x.split(" ").slice(-1)[0]; - if (name == "latest") { - continue; - } - const mtime = parseBupTime(name).valueOf() / 1000; - newest = Math.max(mtime, newest); - v.push({ name, isdir: true, mtime }); - } - if (v.length > 0) { - v.push({ name: "latest", isdir: true, mtime: newest }); - } - return v; - } - - path = normalize(path); - const { stdout } = await sudo({ - command: "bup", - args: [ - "-d", - this.subvolume.filesystem.bup, - "ls", - "--almost-all", - "--file-type", - "-l", - join(`/${this.subvolume.name}`, path), - ], - }); - const v: DirectoryListingEntry[] = []; - for (const x of stdout.split("\n")) { - // [-rw-------","6b851643360e435eb87ef9a6ab64a8b1/6b851643360e435eb87ef9a6ab64a8b1","5","2025-07-15","06:12","a.txt"] - const w = x.split(/\s+/); - if (w.length >= 6) { - let isdir, name; - if (w[5].endsWith("@") || w[5].endsWith("=") || w[5].endsWith("|")) { - w[5] = w[5].slice(0, -1); - } - if (w[5].endsWith("/")) { - isdir = true; - name = w[5].slice(0, -1); - } else { - name = w[5]; - isdir = false; - } - const size = parseInt(w[2]); - const mtime = new Date(w[3] + "T" + w[4]).valueOf() / 1000; - v.push({ name, size, mtime, isdir }); - } - } - return v; - }; - - prune = async ({ - dailies = "1w", - monthlies = "4m", - all = "3d", - }: { dailies?: string; monthlies?: string; all?: string } = {}) => { - await sudo({ - command: "bup", - args: [ - "-d", - this.subvolume.filesystem.bup, - "prune-older", - `--keep-dailies-for=${dailies}`, - `--keep-monthlies-for=${monthlies}`, - `--keep-all-for=${all}`, - "--unsafe", - this.subvolume.name, - ], - }); - }; -} diff --git a/src/packages/file-server/btrfs/subvolume-fs.ts b/src/packages/file-server/btrfs/subvolume-fs.ts deleted file mode 100644 index f1f2dd36772..00000000000 --- a/src/packages/file-server/btrfs/subvolume-fs.ts +++ /dev/null @@ -1,150 +0,0 @@ -import { - appendFile, - chmod, - cp, - copyFile, - link, - readFile, - realpath, - rename, - rm, - rmdir, - mkdir, - stat, - symlink, - truncate, - writeFile, - unlink, - utimes, - watch, -} from "node:fs/promises"; -import { exists } from "@cocalc/backend/misc/async-utils-node"; -import { type DirectoryListingEntry } from "@cocalc/util/types"; -import getListing from "@cocalc/backend/get-listing"; -import { type Subvolume } from "./subvolume"; -import { isdir, sudo } from "./util"; - -export class SubvolumeFilesystem { - constructor(private subvolume: Subvolume) {} - - private normalize = this.subvolume.normalize; - - ls = async ( - path: string, - { hidden, limit }: { hidden?: boolean; limit?: number } = {}, - ): Promise => { - return await getListing(this.normalize(path), hidden, { - limit, - home: "/", - }); - }; - - readFile = async (path: string, encoding?: any): Promise => { - return await readFile(this.normalize(path), encoding); - }; - - writeFile = async (path: string, data: string | Buffer) => { - return await writeFile(this.normalize(path), data); - }; - - appendFile = async (path: string, data: string | Buffer, encoding?) => { - return await appendFile(this.normalize(path), data, encoding); - }; - - unlink = async (path: string) => { - await unlink(this.normalize(path)); - }; - - stat = async (path: string) => { - return await stat(this.normalize(path)); - }; - - exists = async (path: string) => { - return await exists(this.normalize(path)); - }; - - // hard link - link = async (existingPath: string, newPath: string) => { - return await link(this.normalize(existingPath), this.normalize(newPath)); - }; - - symlink = async (target: string, path: string) => { - return await symlink(this.normalize(target), this.normalize(path)); - }; - - realpath = async (path: string) => { - const x = await realpath(this.normalize(path)); - return x.slice(this.subvolume.path.length + 1); - }; - - rename = async (oldPath: string, newPath: string) => { - await rename(this.normalize(oldPath), this.normalize(newPath)); - }; - - utimes = async ( - path: string, - atime: number | string | Date, - mtime: number | string | Date, - ) => { - await utimes(this.normalize(path), atime, mtime); - }; - - watch = (filename: string, options?) => { - return watch(this.normalize(filename), options); - }; - - truncate = async (path: string, len?: number) => { - await truncate(this.normalize(path), len); - }; - - copyFile = async (src: string, dest: string) => { - await copyFile(this.normalize(src), this.normalize(dest)); - }; - - cp = async (src: string, dest: string, options?) => { - await cp(this.normalize(src), this.normalize(dest), options); - }; - - chmod = async (path: string, mode: string | number) => { - await chmod(this.normalize(path), mode); - }; - - mkdir = async (path: string, options?) => { - await mkdir(this.normalize(path), options); - }; - - rsync = async ({ - src, - target, - args = ["-axH"], - timeout = 5 * 60 * 1000, - }: { - src: string; - target: string; - args?: string[]; - timeout?: number; - }): Promise<{ stdout: string; stderr: string; exit_code: number }> => { - let srcPath = this.normalize(src); - let targetPath = this.normalize(target); - if (!srcPath.endsWith("/") && (await isdir(srcPath))) { - srcPath += "/"; - if (!targetPath.endsWith("/")) { - targetPath += "/"; - } - } - return await sudo({ - command: "rsync", - args: [...args, srcPath, targetPath], - err_on_exit: false, - timeout: timeout / 1000, - }); - }; - - rmdir = async (path: string, options?) => { - await rmdir(this.normalize(path), options); - }; - - rm = async (path: string, options?) => { - await rm(this.normalize(path), options); - }; -} diff --git a/src/packages/file-server/btrfs/subvolume-quota.ts b/src/packages/file-server/btrfs/subvolume-quota.ts index b4288cfc57a..43a76e6f989 100644 --- a/src/packages/file-server/btrfs/subvolume-quota.ts +++ b/src/packages/file-server/btrfs/subvolume-quota.ts @@ -38,6 +38,11 @@ export class SubvolumeQuota { await btrfs({ args: ["qgroup", "limit", `${size}`, this.subvolume.path], }); + // also set the exact same quota for the total of all snapshots: + const id = await this.subvolume.getSubvolumeId(); + await btrfs({ + args: ["qgroup", "limit", `${size}`, `1/${id}`, this.subvolume.path], + }); }; du = async () => { diff --git a/src/packages/file-server/btrfs/subvolume-rustic.ts b/src/packages/file-server/btrfs/subvolume-rustic.ts new file mode 100644 index 00000000000..2c91aa0f7b1 --- /dev/null +++ b/src/packages/file-server/btrfs/subvolume-rustic.ts @@ -0,0 +1,179 @@ +/* +Rustic Architecture: + +The minimal option is a single global repo stored in the btrfs filesystem. +Obviously, admins should rsync this regularly to a separate location as a +genuine backup strategy. It's better to configure repo on separate +storage. Rustic has a very wide range of options. + +Instead of using btrfs send/recv for backups, we use Rustic because: + - much easier to check backups are valid + - globally compressed and dedup'd! btrfs send/recv is NOT globally dedupd + - decoupled from any btrfs issues + - rustic has full support for using cloud buckets as hot/cold storage + - not tied to any specific filesystem at all + - easier to offsite via incremental rsync + - much more space efficient with *global* dedup and compression + - rustic "is" restic, which is very mature and proven + - rustic is VERY fast, being parallel and in rust. +*/ + +import { type Subvolume } from "./subvolume"; +import getLogger from "@cocalc/backend/logger"; +import { parseOutput } from "@cocalc/backend/sandbox/exec"; +import { field_cmp } from "@cocalc/util/misc"; +import { type SnapshotCounts, updateRollingSnapshots } from "./snapshots"; +import { reuseInFlight } from "@cocalc/util/reuse-in-flight"; +import { ConatError } from "@cocalc/conat/core/client"; +import { DEFAULT_BACKUP_COUNTS } from "@cocalc/util/consts/snapshots"; + +export const RUSTIC = "rustic"; + +const RUSTIC_SNAPSHOT = "temp-rustic-snapshot"; + +const logger = getLogger("file-server:btrfs:subvolume-rustic"); + +interface Snapshot { + id: string; + time: Date; +} + +export class SubvolumeRustic { + constructor(public readonly subvolume: Subvolume) {} + + // create a new rustic backup + backup = async ({ + limit, + timeout = 30 * 60 * 1000, + }: { timeout?: number; limit?: number } = {}): Promise => { + if (limit != null && (await this.snapshots()).length >= limit) { + // 507 = "insufficient storage" for http + throw new ConatError(`there is a limit of ${limit} backups`, { + code: 507, + }); + } + if (await this.subvolume.snapshots.exists(RUSTIC_SNAPSHOT)) { + logger.debug(`backup: deleting existing ${RUSTIC_SNAPSHOT}`); + await this.subvolume.snapshots.delete(RUSTIC_SNAPSHOT); + } + const target = this.subvolume.snapshots.path(RUSTIC_SNAPSHOT); + try { + logger.debug( + `backup: creating ${RUSTIC_SNAPSHOT} to get a consistent backup`, + ); + await this.subvolume.snapshots.create(RUSTIC_SNAPSHOT); + logger.debug(`backup: backing up ${RUSTIC_SNAPSHOT} using rustic`); + const { stdout } = parseOutput( + await this.subvolume.fs.rustic(["backup", "-x", "--json", "."], { + timeout, + cwd: target, + }), + ); + const { time, id } = JSON.parse(stdout); + return { time: new Date(time), id }; + } finally { + this.snapshotsCache = null; + logger.debug(`backup: deleting temporary ${RUSTIC_SNAPSHOT}`); + try { + await this.subvolume.snapshots.delete(RUSTIC_SNAPSHOT); + } catch {} + } + }; + + restore = async ({ + id, + path = "", + dest, + timeout = 30 * 60 * 1000, + }: { + id: string; + path?: string; + dest?: string; + timeout?: number; + }) => { + dest ??= path; + const { stdout } = parseOutput( + await this.subvolume.fs.rustic( + ["restore", `${id}${path != null ? ":" + path : ""}`, dest], + { timeout }, + ), + ); + return stdout; + }; + + // returns list of backups, sorted from oldest to newest + private snapshotsCache: Snapshot[] | null = null; + snapshots = reuseInFlight(async (): Promise => { + if (this.snapshotsCache) { + // potentially very expensive to get list -- we clear this on delete or create + return this.snapshotsCache; + } + const { stdout } = parseOutput( + await this.subvolume.fs.rustic(["snapshots", "--json"]), + ); + const x = JSON.parse(stdout); + const v = !x[0] + ? [] + : x[0][1].map(({ time, id }) => { + return { time: new Date(time), id }; + }); + v.sort(field_cmp("time")); + this.snapshotsCache = v; + return v; + }); + + // return list of paths of files in this backup, as paths relative + // to HOME, and sorted in alphabetical order. + ls = async ({ id }: { id: string }) => { + const { stdout } = parseOutput( + await this.subvolume.fs.rustic(["ls", "--json", id]), + ); + return JSON.parse(stdout).sort(); + }; + + // Delete this backup. It's genuinely not accessible anymore, though + // this doesn't actually clean up disk space -- purge must be done separately + // later. Rustic likes the purge to happen maybe a day later, so it + // can better support concurrent writes. + forget = async ({ id }: { id: string }) => { + const { stdout } = parseOutput( + await this.subvolume.fs.rustic(["forget", id]), + ); + this.snapshotsCache = null; + return stdout; + }; + + update = async (counts?: Partial, opts?) => { + return await updateRollingSnapshots({ + snapshots: this, + counts: { ...DEFAULT_BACKUP_COUNTS, ...counts }, + opts, + }); + }; + + // Snapshot compat api, which is useful for rolling backups. + + create = async (_name?: string, { limit }: { limit?: number } = {}) => { + await this.backup({ limit }); + }; + + readdir = async (): Promise => { + return (await this.snapshots()).map(({ time }) => time.toISOString()); + }; + + // TODO -- for now just always assume we do... + hasUnsavedChanges = async () => { + return true; + }; + + delete = async (name) => { + const v = await this.snapshots(); + for (const { id, time } of v) { + if (time.toISOString() == name) { + await this.forget({ id }); + return; + } + } + throw Error(`backup ${name} not found`); + }; +} diff --git a/src/packages/file-server/btrfs/subvolume-snapshots.ts b/src/packages/file-server/btrfs/subvolume-snapshots.ts index ffe71fe6fc7..b76459c4e52 100644 --- a/src/packages/file-server/btrfs/subvolume-snapshots.ts +++ b/src/packages/file-server/btrfs/subvolume-snapshots.ts @@ -2,16 +2,18 @@ import { type Subvolume } from "./subvolume"; import { btrfs } from "./util"; import getLogger from "@cocalc/backend/logger"; import { join } from "path"; -import { type DirectoryListingEntry } from "@cocalc/util/types"; -import { SnapshotCounts, updateRollingSnapshots } from "./snapshots"; +import { type SnapshotCounts, updateRollingSnapshots } from "./snapshots"; +import { ConatError } from "@cocalc/conat/core/client"; +import { type SnapshotUsage } from "@cocalc/conat/files/file-server"; +import { SNAPSHOTS } from "@cocalc/util/consts/snapshots"; +import { getSubvolumeField, getSubvolumeId } from "./subvolume"; -export const SNAPSHOTS = ".snapshots"; const logger = getLogger("file-server:btrfs:subvolume-snapshots"); export class SubvolumeSnapshots { public readonly snapshotsDir: string; - constructor(public subvolume: Subvolume) { + constructor(public readonly subvolume: Subvolume) { this.snapshotsDir = join(this.subvolume.path, SNAPSHOTS); } @@ -27,30 +29,55 @@ export class SubvolumeSnapshots { return; } await this.subvolume.fs.mkdir(SNAPSHOTS); - await this.subvolume.fs.chmod(SNAPSHOTS, "0550"); + await this.subvolume.fs.chmod(SNAPSHOTS, "0700"); }; - create = async (name?: string) => { + create = async ( + name?: string, + { limit, readOnly }: { limit?: number; readOnly?: boolean } = {}, + ) => { if (name?.startsWith(".")) { throw Error("snapshot name must not start with '.'"); } name ??= new Date().toISOString(); logger.debug("create", { name, subvolume: this.subvolume.name }); await this.makeSnapshotsDir(); + + if (limit != null) { + if ((await this.readdir()).length >= limit) { + // 507 = "insufficient storage" for http + throw new ConatError(`there is a limit of ${limit} snapshots`, { + code: 507, + }); + } + } + + const args = ["subvolume", "snapshot"]; + if (readOnly) { + args.push("-r"); + } + const snapshotPath = join(this.snapshotsDir, name); + args.push(this.subvolume.path, snapshotPath); + + await btrfs({ args }); + + // also add snapshot to the snapshot quota group + const snapshotId = await getSubvolumeId(snapshotPath); + const subvolumeId = await this.subvolume.getSubvolumeId(); await btrfs({ args: [ - "subvolume", - "snapshot", - "-r", + "qgroup", + "assign", + `0/${snapshotId}`, + `1/${subvolumeId}`, this.subvolume.path, - join(this.snapshotsDir, name), ], }); }; - ls = async (): Promise => { + readdir = async (): Promise => { await this.makeSnapshotsDir(); - return await this.subvolume.fs.ls(SNAPSHOTS, { hidden: false }); + return await this.subvolume.fs.readdir(SNAPSHOTS); }; lock = async (name: string) => { @@ -78,34 +105,47 @@ export class SubvolumeSnapshots { }); }; - // update the rolling snapshots schedule - update = async (counts?: Partial) => { - return await updateRollingSnapshots({ snapshots: this, counts }); + // update the rolling snapshots scheduleGener + update = async (counts?: Partial, opts?) => { + return await updateRollingSnapshots({ snapshots: this, counts, opts }); }; // has newly written changes since last snapshot hasUnsavedChanges = async (): Promise => { - const s = await this.ls(); + const s = await this.readdir(); if (s.length == 0) { // more than just the SNAPSHOTS directory? - const v = await this.subvolume.fs.ls("", { hidden: true }); - if (v.length == 0 || (v.length == 1 && v[0].name == SNAPSHOTS)) { + const v = await this.subvolume.fs.readdir(""); + if (v.length == 0 || (v.length == 1 && v[0] == SNAPSHOTS)) { return false; } return true; } const pathGen = await getGeneration(this.subvolume.path); const snapGen = await getGeneration( - join(this.snapshotsDir, s[s.length - 1].name), + join(this.snapshotsDir, s[s.length - 1]), ); return snapGen < pathGen; }; + + usage = async (name: string): Promise => { + // btrfs --format=json qgroup show -reF --raw project-eac5b48a-70aa-4401-a54d-0f58c5eb09ba/.snapshots/cocalc + const snapshotPath = join(this.snapshotsDir, name); + const { stdout } = await btrfs({ + args: ["--format=json", "qgroup", "show", "-ref", "--raw", snapshotPath], + }); + const x = JSON.parse(stdout); + const { referenced, max_referenced, exclusive } = x["qgroup-show"][0]; + return { name, used: referenced, quota: max_referenced, exclusive }; + }; + + allUsage = async (): Promise => { + // get quota/usage information about all snapshots + const snaps = await this.readdir(); + return Promise.all(snaps.map(this.usage)); + }; } -async function getGeneration(path: string): Promise { - const { stdout } = await btrfs({ - args: ["subvolume", "show", path], - verbose: false, - }); - return parseInt(stdout.split("Generation:")[1].split("\n")[0].trim()); +export async function getGeneration(path: string): Promise { + return parseInt(await getSubvolumeField(path, "Generation")); } diff --git a/src/packages/file-server/btrfs/subvolume.ts b/src/packages/file-server/btrfs/subvolume.ts index 3f77abd9c40..4088cb5b68c 100644 --- a/src/packages/file-server/btrfs/subvolume.ts +++ b/src/packages/file-server/btrfs/subvolume.ts @@ -2,15 +2,16 @@ A subvolume */ -import { type Filesystem, DEFAULT_SUBVOLUME_SIZE } from "./filesystem"; +import { type Filesystem } from "./filesystem"; import refCache from "@cocalc/util/refcache"; import { sudo } from "./util"; -import { join, normalize } from "path"; -import { SubvolumeFilesystem } from "./subvolume-fs"; -import { SubvolumeBup } from "./subvolume-bup"; +import { join } from "path"; +import { SubvolumeRustic } from "./subvolume-rustic"; import { SubvolumeSnapshots } from "./subvolume-snapshots"; import { SubvolumeQuota } from "./subvolume-quota"; +import { SandboxedFilesystem } from "@cocalc/backend/sandbox"; import { exists } from "@cocalc/backend/misc/async-utils-node"; +import { btrfs } from "./util"; import getLogger from "@cocalc/backend/logger"; @@ -23,11 +24,10 @@ interface Options { export class Subvolume { public readonly name: string; - public readonly filesystem: Filesystem; public readonly path: string; - public readonly fs: SubvolumeFilesystem; - public readonly bup: SubvolumeBup; + public readonly fs: SandboxedFilesystem; + public readonly rustic: SubvolumeRustic; public readonly snapshots: SubvolumeSnapshots; public readonly quota: SubvolumeQuota; @@ -35,8 +35,11 @@ export class Subvolume { this.filesystem = filesystem; this.name = name; this.path = join(filesystem.opts.mount, name); - this.fs = new SubvolumeFilesystem(this); - this.bup = new SubvolumeBup(this); + this.fs = new SandboxedFilesystem(this.path, { + rusticRepo: filesystem.opts.rustic, + host: this.name, + }); + this.rustic = new SubvolumeRustic(this); this.snapshots = new SubvolumeSnapshots(this); this.quota = new SubvolumeQuota(this); } @@ -44,17 +47,19 @@ export class Subvolume { init = async () => { if (!(await exists(this.path))) { logger.debug(`creating ${this.name} at ${this.path}`); - await sudo({ - command: "btrfs", + await btrfs({ args: ["subvolume", "create", this.path], }); await this.chown(this.path); - await this.quota.set( - this.filesystem.opts.defaultSize ?? DEFAULT_SUBVOLUME_SIZE, - ); + const id = await this.getSubvolumeId(); + await btrfs({ args: ["qgroup", "create", `1/${id}`, this.path] }); } }; + getSubvolumeId = async (): Promise => { + return await getSubvolumeId(this.path); + }; + close = () => { // @ts-ignore delete this.filesystem; @@ -64,7 +69,7 @@ export class Subvolume { delete this.path; // @ts-ignore delete this.snapshotsDir; - for (const sub of ["fs", "bup", "snapshots", "quota"]) { + for (const sub of ["fs", "rustic", "snapshots", "quota"]) { this[sub].close?.(); delete this[sub]; } @@ -76,13 +81,6 @@ export class Subvolume { args: [`${process.getuid?.() ?? 0}:${process.getgid?.() ?? 0}`, path], }); }; - - // this should provide a path that is guaranteed to be - // inside this.path on the filesystem or throw error - // [ ] TODO: not sure if the code here is sufficient!! - normalize = (path: string) => { - return join(this.path, normalize(path)); - }; } const cache = refCache({ @@ -100,3 +98,19 @@ export async function subvolume( ): Promise { return await cache(options); } + +export async function getSubvolumeField( + path: string, + field: string, +): Promise { + const { stdout } = await btrfs({ + args: ["subvolume", "show", path], + verbose: false, + }); + // avoid any possibilitiy of a sneaky named snapshot breaking this + return stdout.split(`${field}:`)[1].split("\n")[0].trim(); +} + +export async function getSubvolumeId(path: string): Promise { + return parseInt(await getSubvolumeField(path, "Subvolume ID")); +} diff --git a/src/packages/file-server/btrfs/subvolumes.ts b/src/packages/file-server/btrfs/subvolumes.ts index 0f8da1468f5..314bd4c711a 100644 --- a/src/packages/file-server/btrfs/subvolumes.ts +++ b/src/packages/file-server/btrfs/subvolumes.ts @@ -1,29 +1,34 @@ import { type Filesystem } from "./filesystem"; import { subvolume, type Subvolume } from "./subvolume"; import getLogger from "@cocalc/backend/logger"; -import { SNAPSHOTS } from "./subvolume-snapshots"; +import { SNAPSHOTS } from "@cocalc/util/consts/snapshots"; import { exists } from "@cocalc/backend/misc/async-utils-node"; -import { join, normalize } from "path"; -import { btrfs, isdir } from "./util"; +import { join } from "path"; +import { btrfs } from "./util"; import { chmod, rename, rm } from "node:fs/promises"; -import { executeCode } from "@cocalc/backend/execute-code"; +import { SandboxedFilesystem } from "@cocalc/backend/sandbox"; +import { RUSTIC } from "./subvolume-rustic"; +import { SYNC_STATE } from "./sync"; -const RESERVED = new Set(["bup", SNAPSHOTS]); +const RESERVED = new Set([RUSTIC, SNAPSHOTS, SYNC_STATE]); const logger = getLogger("file-server:btrfs:subvolumes"); export class Subvolumes { - constructor(public filesystem: Filesystem) {} + public readonly fs: SandboxedFilesystem; - get = async (name: string): Promise => { - if (RESERVED.has(name)) { + constructor(public filesystem: Filesystem) { + this.fs = new SandboxedFilesystem(this.filesystem.opts.mount); + } + + get = async (name: string, force = false): Promise => { + if (RESERVED.has(name) && !force) { throw Error(`${name} is reserved`); } return await subvolume({ filesystem: this.filesystem, name }); }; - // create a subvolume by cloning an existing one. - clone = async (source: string, dest: string) => { + clone = async (source: string, dest: string): Promise => { logger.debug("clone ", { source, dest }); if (RESERVED.has(dest)) { throw Error(`${dest} is reserved`); @@ -79,37 +84,4 @@ export class Subvolumes { .filter((x) => x) .sort(); }; - - rsync = async ({ - src, - target, - args = ["-axH"], - timeout = 5 * 60 * 1000, - }: { - src: string; - target: string; - args?: string[]; - timeout?: number; - }): Promise<{ stdout: string; stderr: string; exit_code: number }> => { - let srcPath = normalize(join(this.filesystem.opts.mount, src)); - if (!srcPath.startsWith(this.filesystem.opts.mount)) { - throw Error("suspicious source"); - } - let targetPath = normalize(join(this.filesystem.opts.mount, target)); - if (!targetPath.startsWith(this.filesystem.opts.mount)) { - throw Error("suspicious target"); - } - if (!srcPath.endsWith("/") && (await isdir(srcPath))) { - srcPath += "/"; - if (!targetPath.endsWith("/")) { - targetPath += "/"; - } - } - return await executeCode({ - command: "rsync", - args: [...args, srcPath, targetPath], - err_on_exit: false, - timeout: timeout / 1000, - }); - }; } diff --git a/src/packages/file-server/btrfs/sync.ts b/src/packages/file-server/btrfs/sync.ts new file mode 100644 index 00000000000..962cd0ddc18 --- /dev/null +++ b/src/packages/file-server/btrfs/sync.ts @@ -0,0 +1,207 @@ +/* +Implementation of path sync *inside* volumes on the file server. + +NOTE: I'm aware that we could use bind mounts instead of mutagen +to accomplish something very similar. There are a huge list of pros +and cons to using mutagen versus bind mounts to solve this problem. +We've gone with mutagen, since it's entirely in user space (so maximally +flexible), and doesn't involve any cross filesystem mount issues. +Basically, for security it's better. + +*/ + +import getLogger from "@cocalc/backend/logger"; +import { executeCode } from "@cocalc/backend/execute-code"; +import { join, resolve } from "node:path"; +import { type Sync } from "@cocalc/conat/files/file-server"; +import { type Filesystem } from "./filesystem"; +import { sha1 } from "@cocalc/backend/sha1"; +import { type MutagenSyncSession } from "@cocalc/conat/project/mutagen/types"; + +export const SYNC_STATE = "sync-state"; + +const logger = getLogger("file-server:btrfs:sync"); + +async function mutagen( + args: string[], + { HOME, err_on_exit = true }: { HOME: string; err_on_exit?: boolean }, +) { + return await executeCode({ + command: "mutagen", + args: ["sync"].concat(args), + verbose: true, + err_on_exit, + env: { ...process.env, HOME }, + }); +} + +// Return s a valid mutagen name, which will work no matter how long +// src and dest are (e.g., paths could be 1000+ characters), +// and can be deduced from src/dest with no database needed. +function mutagenName({ src, dest }: Sync): string { + const s = parse(src); + const d = parse(dest); + return `fs-${sha1(JSON.stringify([s.name, s.path, d.name, d.path]))}`; +} + +// spec is of the form {volume-name}:{relative path into volume} +function parse(spec: string): { name: string; path: string } { + const i = spec.indexOf(":"); + if (i == -1) { + return { name: spec, path: "" }; + } + const name = spec.slice(0, i); + if (name.length > 63) { + throw Error("volume name must be at most 63 characters long"); + } + const path = spec.slice(i + 1); + if (resolve("/", path).slice(1) != path) { + throw Error(`invalid path ${path} -- must resolve to itself`); + } + return { name, path }; +} + +function parseSync(sync: Sync): { + src: { name: string; path: string }; + dest: { name: string; path: string }; +} { + return { src: parse(sync.src), dest: parse(sync.dest) }; +} + +function encode({ name, path }: { name: string; path: string }) { + return `${name}:${path}`; +} + +// if the absolute path weirdly happened to have the base path twice in it this could +// break, but I'm not worrying about that right now... +function makeRelative(path: string, base?: string): string { + if (!base) { + return path; + } + const i = path.indexOf(base); + if (i == -1) { + return path; + } + return path.slice(i + base.length + 1); +} + +// enhance MutagenSyncSession with extra data in the Sync object; +// This is a convenience function to connect mutagen's description +// of a sync session with the properties we use (src, dest, replica) +// to define one. +function addSync(session: MutagenSyncSession): Sync & MutagenSyncSession { + return { + ...session, + src: encode({ + name: session.labels?.src ?? "", + path: makeRelative(session.alpha.path, session.labels?.src), + }), + dest: encode({ + name: session.labels?.dest ?? "", + path: makeRelative(session.beta.path, session.labels?.dest), + }), + replica: session.mode == "one-way-replica", + }; +} + +// all sync with this as the source + +export class FileSync { + constructor(public readonly fs: Filesystem) {} + + init = async () => { + await this.mutagen(["daemon", "start"]); + }; + + close = async () => { + try { + await this.mutagen(["daemon", "stop"]); + } catch (err) { + console.warn("Error stopping mutagen daemon", err); + } + }; + + private HOME?: string; + private mutagen = async (args: string[], err_on_exit = true) => { + if (!this.HOME) { + this.HOME = (await this.fs.subvolumes.get(SYNC_STATE, true)).path; + } + return await mutagen(args, { HOME: this.HOME, err_on_exit }); + }; + + create = async ({ ignores = [], ...sync }: Sync & { ignores?: string[] }) => { + const cur = await this.get(sync); + if (cur != null) { + return; + } + logger.debug("create", sync); + const { src, dest } = parseSync(sync); + const srcVolume = await this.fs.subvolumes.get(src.name); + const destVolume = await this.fs.subvolumes.get(dest.name); + const alpha = join(srcVolume.path, src.path); + const beta = join(destVolume.path, dest.path); + const args = [ + "create", + "--mode", + // no possible conflicts: + sync.replica ? "one-way-replica" : "two-way-resolved", + "--label", + `src=${src.name}`, + "--label", + `dest=${dest.name}`, + `--name=${mutagenName(sync)}`, + "--watch-polling-interval-alpha=10", + "--watch-polling-interval-beta=10", + "--symlink-mode=posix-raw", + ]; + for (const ignore of Array.from(new Set(ignores))) { + args.push(`--ignore=${ignore}`); + } + args.push(alpha, beta); + await this.mutagen(args); + }; + + command = async ( + command: "flush" | "reset" | "pause" | "resume" | "terminate", + sync: Sync, + ) => { + logger.debug("command: ", command, sync); + return await this.mutagen([command, mutagenName(sync)]); + }; + + getAll = async ({ + name, + }: { + name: string; + }): Promise<(Sync & MutagenSyncSession)[]> => { + const { stdout } = await this.mutagen([ + "list", + `--label-selector`, + `src=${name}`, + "--template", + "{{json .}}", + ]); + const { stdout: stdout2 } = await this.mutagen([ + "list", + `--label-selector`, + `dest=${name}`, + "--template", + "{{json .}}", + ]); + const v = JSON.parse(stdout).concat(JSON.parse(stdout2)); + return v.map(addSync); + }; + + get = async ( + sync: Sync, + ): Promise => { + const { stdout } = await this.mutagen( + ["list", mutagenName(sync), "--template", "{{json .}}"], + false, + ); + if (!stdout.trim()) { + return undefined; // doesn't exist + } + return addSync(JSON.parse(stdout)[0]); + }; +} diff --git a/src/packages/file-server/btrfs/test/filesystem-stress.test.ts b/src/packages/file-server/btrfs/test/filesystem-stress.test.ts index a2e1de95318..2302785d14b 100644 --- a/src/packages/file-server/btrfs/test/filesystem-stress.test.ts +++ b/src/packages/file-server/btrfs/test/filesystem-stress.test.ts @@ -90,6 +90,11 @@ describe("stress operations with subvolumes", () => { `deleted ${Math.round((count2 / (Date.now() - t)) * 1000)} subvolumes per second in parallel`, ); }); + + it("everything should be gone except the clones", async () => { + const v = await fs.subvolumes.list(); + expect(v.length).toBe(count1 + count2); + }); }); afterAll(after); diff --git a/src/packages/file-server/btrfs/test/filesystem.test.ts b/src/packages/file-server/btrfs/test/filesystem.test.ts index 673980af897..c4eea6992a1 100644 --- a/src/packages/file-server/btrfs/test/filesystem.test.ts +++ b/src/packages/file-server/btrfs/test/filesystem.test.ts @@ -1,5 +1,6 @@ import { before, after, fs } from "./setup"; import { isValidUUID } from "@cocalc/util/misc"; +import { RUSTIC } from "@cocalc/file-server/btrfs/subvolume-rustic"; beforeAll(before); @@ -22,7 +23,7 @@ describe("some basic tests", () => { describe("operations with subvolumes", () => { it("can't use a reserved subvolume name", async () => { expect(async () => { - await fs.subvolumes.get("bup"); + await fs.subvolumes.get(RUSTIC); }).rejects.toThrow("is reserved"); }); @@ -30,7 +31,7 @@ describe("operations with subvolumes", () => { const vol = await fs.subvolumes.get("cocalc"); expect(vol.name).toBe("cocalc"); // it has no snapshots - expect(await vol.snapshots.ls()).toEqual([]); + expect(await vol.snapshots.readdir()).toEqual([]); }); it("our subvolume is in the list", async () => { @@ -62,24 +63,37 @@ describe("operations with subvolumes", () => { ]); }); - it("rsync from one volume to another", async () => { - await fs.subvolumes.rsync({ src: "sagemath", target: "cython" }); + it("cp from one volume to another", async () => { + await fs.subvolumes.fs.cp("sagemath", "cython", { + recursive: true, + reflink: true, + }); }); - it("rsync an actual file", async () => { + it("cp an actual file", async () => { const sagemath = await fs.subvolumes.get("sagemath"); const cython = await fs.subvolumes.get("cython"); - await sagemath.fs.writeFile("README.md", "hi"); - await fs.subvolumes.rsync({ src: "sagemath", target: "cython" }); + await sagemath.fs.writeFile("README.md", "hi5"); + await fs.subvolumes.fs.cp("sagemath/README.md", "cython/README.md", { + reflink: true, + }); const copy = await cython.fs.readFile("README.md", "utf8"); - expect(copy).toEqual("hi"); + expect(copy).toEqual("hi5"); + + // also one without reflink + await sagemath.fs.writeFile("README2.md", "hi2"); + await fs.subvolumes.fs.cp("sagemath/README2.md", "cython/README2.md", { + reflink: false, + }); + const copy2 = await cython.fs.readFile("README2.md", "utf8"); + expect(copy2).toEqual("hi2"); }); it("clone a subvolume with contents", async () => { await fs.subvolumes.clone("cython", "pyrex"); const pyrex = await fs.subvolumes.get("pyrex"); const clone = await pyrex.fs.readFile("README.md", "utf8"); - expect(clone).toEqual("hi"); + expect(clone).toEqual("hi5"); }); }); @@ -97,7 +111,7 @@ describe("clone of a subvolume with snapshots should have no snapshots", () => { it("clone has no snapshots", async () => { const clone = await fs.subvolumes.get("my-clone"); expect(await clone.fs.readFile("abc.txt", "utf8")).toEqual("hi"); - expect(await clone.snapshots.ls()).toEqual([]); + expect(await clone.snapshots.readdir()).toEqual([]); await clone.snapshots.create("my-clone-snap"); }); }); diff --git a/src/packages/file-server/btrfs/test/rustic-stress.test.ts b/src/packages/file-server/btrfs/test/rustic-stress.test.ts new file mode 100644 index 00000000000..2eb0514df71 --- /dev/null +++ b/src/packages/file-server/btrfs/test/rustic-stress.test.ts @@ -0,0 +1,45 @@ +import { before, after, fs } from "./setup"; +import { type Subvolume } from "../subvolume"; + +beforeAll(before); + +const count = 10; +describe(`make backups of ${count} different volumes at the same time`, () => { + const vols: Subvolume[] = []; + it(`creates ${count} volumes`, async () => { + for (let i = 0; i < count; i++) { + const vol = await fs.subvolumes.get(`rustic-multi-${i}`); + await vol.fs.writeFile(`a-${i}.txt`, `hello-${i}`); + vols.push(vol); + } + }); + + it(`create ${count} rustic backup in parallel`, async () => { + await Promise.all(vols.map((vol) => vol.rustic.backup())); + }); + + it("delete file from each volume, then restore them all in parallel and confirm restore worked", async () => { + const snapshots = await Promise.all( + vols.map((vol) => vol.rustic.snapshots()), + ); + const ids = snapshots.map((x) => x[0].id); + for (let i = 0; i < count; i++) { + await vols[i].fs.unlink(`a-${i}.txt`); + } + + const v: any[] = []; + for (let i = 0; i < count; i++) { + v.push(vols[i].rustic.restore({ id: ids[i] })); + } + await Promise.all(v); + + for (let i = 0; i < count; i++) { + const vol = vols[i]; + expect((await vol.fs.readFile(`a-${i}.txt`)).toString("utf8")).toEqual( + `hello-${i}`, + ); + } + }); +}); + +afterAll(after); diff --git a/src/packages/file-server/btrfs/test/rustic.test.ts b/src/packages/file-server/btrfs/test/rustic.test.ts new file mode 100644 index 00000000000..3c12e4608f3 --- /dev/null +++ b/src/packages/file-server/btrfs/test/rustic.test.ts @@ -0,0 +1,76 @@ +import { before, after, fs } from "./setup"; +import { type Subvolume } from "../subvolume"; +import { SNAPSHOTS } from "@cocalc/util/consts/snapshots"; + +beforeAll(before); + +describe("test rustic backups", () => { + let vol: Subvolume; + it("creates a volume", async () => { + vol = await fs.subvolumes.get("rustic-test"); + await vol.fs.writeFile("a.txt", "hello"); + }); + + let x; + it("create a rustic backup", async () => { + x = await vol.rustic.backup(); + }); + + it("confirm a.txt is in our backup", async () => { + const v = await vol.rustic.snapshots(); + expect(v.length == 1); + expect(v[0]).toEqual(x); + expect(Math.abs(Date.now() - v[0].time.valueOf())).toBeLessThan(10000); + const { id } = v[0]; + const w = await vol.rustic.ls({ id }); + expect(w).toEqual([SNAPSHOTS, "a.txt"]); + }); + + it("delete a.txt, then restore it from the backup", async () => { + await vol.fs.unlink("a.txt"); + const { id } = (await vol.rustic.snapshots())[0]; + await vol.rustic.restore({ id }); + expect((await vol.fs.readFile("a.txt")).toString("utf8")).toEqual("hello"); + }); + + it("create a directory, make second backup, delete directory, then restore it from backup, and also restore just one file", async () => { + await vol.fs.mkdir("my-dir"); + await vol.fs.writeFile("my-dir/file.txt", "hello"); + await vol.fs.writeFile("my-dir/file2.txt", "hello2"); + await vol.rustic.backup(); + const v = await vol.rustic.snapshots(); + expect(v.length == 2); + const { id } = v[1]; + const w = await vol.rustic.ls({ id }); + expect(w).toEqual([ + SNAPSHOTS, + "a.txt", + "my-dir", + "my-dir/file.txt", + "my-dir/file2.txt", + ]); + await vol.fs.rm("my-dir", { recursive: true }); + + // rustic all, including the path we just deleted + await vol.rustic.restore({ id }); + expect((await vol.fs.readFile("my-dir/file.txt")).toString("utf8")).toEqual( + "hello", + ); + + // restore just one specific file overwriting current version + await vol.fs.unlink("my-dir/file2.txt"); + await vol.fs.writeFile("my-dir/file.txt", "changed"); + await vol.rustic.restore({ id, path: "my-dir/file2.txt" }); + expect( + (await vol.fs.readFile("my-dir/file2.txt")).toString("utf8"), + ).toEqual("hello2"); + + // forget the second snapshot + await vol.rustic.forget({ id }); + const v2 = await vol.rustic.snapshots(); + expect(v2.length).toBe(1); + expect(v2[0].id).not.toEqual(id); + }); +}); + +afterAll(after); diff --git a/src/packages/file-server/btrfs/test/setup.ts b/src/packages/file-server/btrfs/test/setup.ts index c9736c8b79b..72dc8939ffe 100644 --- a/src/packages/file-server/btrfs/test/setup.ts +++ b/src/packages/file-server/btrfs/test/setup.ts @@ -2,11 +2,11 @@ import { filesystem, type Filesystem, } from "@cocalc/file-server/btrfs/filesystem"; -import { chmod, mkdtemp, mkdir, rm, stat } from "node:fs/promises"; +import { chmod, mkdtemp, mkdir, rm } from "node:fs/promises"; import { tmpdir } from "node:os"; import { join } from "path"; import { until } from "@cocalc/util/async-utils"; -import { sudo } from "../util"; +import { ensureMoreLoopbackDevices, sudo } from "../util"; export { sudo }; export { delay } from "awaiting"; @@ -15,25 +15,6 @@ let tempDir; const TEMP_PREFIX = "cocalc-test-btrfs-"; -async function ensureMoreLoops() { - // to run tests, this is helpful - //for i in $(seq 8 63); do sudo mknod -m660 /dev/loop$i b 7 $i; sudo chown root:disk /dev/loop$i; done - for (let i = 0; i < 64; i++) { - try { - await stat(`/dev/loop${i}`); - continue; - } catch {} - try { - // also try/catch this because ensureMoreLoops happens in parallel many times at once... - await sudo({ - command: "mknod", - args: ["-m660", `/dev/loop${i}`, "b", "7", `${i}`], - }); - } catch {} - await sudo({ command: "chown", args: ["root:disk", `/dev/loop${i}`] }); - } -} - export async function before() { try { const command = `umount ${join(tmpdir(), TEMP_PREFIX)}*/mnt`; @@ -41,7 +22,7 @@ export async function before() { // TODO: this could impact runs in parallel await sudo({ command, bash: true }); } catch {} - await ensureMoreLoops(); + await ensureMoreLoopbackDevices(); tempDir = await mkdtemp(join(tmpdir(), TEMP_PREFIX)); // Set world read/write/execute await chmod(tempDir, 0o777); @@ -49,10 +30,12 @@ export async function before() { await mkdir(mount); await chmod(mount, 0o777); fs = await filesystem({ - device: join(tempDir, "btrfs.img"), - formatIfNeeded: true, + image: join(tempDir, "btrfs.img"), + size: "1G", mount: join(tempDir, "mnt"), + rustic: join(tempDir, "rustic"), }); + return fs; } export async function after() { diff --git a/src/packages/file-server/btrfs/test/subvolume-stress.test.ts b/src/packages/file-server/btrfs/test/subvolume-stress.test.ts index 29bc048e695..69279d9b042 100644 --- a/src/packages/file-server/btrfs/test/subvolume-stress.test.ts +++ b/src/packages/file-server/btrfs/test/subvolume-stress.test.ts @@ -29,16 +29,16 @@ describe(`stress test creating ${numSnapshots} snapshots`, () => { `created ${Math.round((numSnapshots / (Date.now() - start)) * 1000)} snapshots per second in serial`, ); snaps.sort(); - expect((await vol.snapshots.ls()).map(({ name }) => name).sort()).toEqual( - snaps.sort(), - ); + expect( + (await vol.snapshots.readdir()).filter((x) => !x.startsWith(".")).sort(), + ).toEqual(snaps.sort()); }); it(`delete our ${numSnapshots} snapshots`, async () => { for (let i = 0; i < numSnapshots; i++) { await vol.snapshots.delete(`snap${i}`); } - expect(await vol.snapshots.ls()).toEqual([]); + expect(await vol.snapshots.readdir()).toEqual([]); }); }); @@ -58,9 +58,8 @@ describe(`create ${numFiles} files`, () => { log( `created ${Math.round((numFiles / (Date.now() - start)) * 1000)} files per second in serial`, ); - const v = await vol.fs.ls(""); - const w = v.map(({ name }) => name); - expect(w.sort()).toEqual(names.sort()); + const v = await vol.fs.readdir(""); + expect(v.sort()).toEqual(names.sort()); }); it(`creates ${numFiles} files in parallel`, async () => { @@ -77,9 +76,8 @@ describe(`create ${numFiles} files`, () => { `created ${Math.round((numFiles / (Date.now() - start)) * 1000)} files per second in parallel`, ); const t0 = Date.now(); - const v = await vol.fs.ls("p"); + const w = await vol.fs.readdir("p"); log("get listing of files took", Date.now() - t0, "ms"); - const w = v.map(({ name }) => name); expect(w.sort()).toEqual(names.sort()); }); }); diff --git a/src/packages/file-server/btrfs/test/subvolume.test.ts b/src/packages/file-server/btrfs/test/subvolume.test.ts index e586cc656bf..11121ca9e24 100644 --- a/src/packages/file-server/btrfs/test/subvolume.test.ts +++ b/src/packages/file-server/btrfs/test/subvolume.test.ts @@ -1,12 +1,12 @@ import { before, after, fs, sudo } from "./setup"; -import { mkdir } from "fs/promises"; -import { join } from "path"; import { wait } from "@cocalc/backend/conat/test/util"; import { randomBytes } from "crypto"; import { type Subvolume } from "../subvolume"; +import { SNAPSHOTS } from "@cocalc/util/consts/snapshots"; beforeAll(before); +jest.setTimeout(15000); describe("setting and getting quota of a subvolume", () => { let vol: Subvolume; it("set the quota of a subvolume to 5 M", async () => { @@ -19,7 +19,7 @@ describe("setting and getting quota of a subvolume", () => { }); it("get directory listing", async () => { - const v = await vol.fs.ls(""); + const v = await vol.fs.readdir(""); expect(v).toEqual([]); }); @@ -28,7 +28,7 @@ describe("setting and getting quota of a subvolume", () => { await vol.fs.writeFile("buf", buf); await wait({ until: async () => { - await sudo({ command: "sync" }); + await vol.filesystem.sync(); const { used } = await vol.quota.usage(); return used > 0; }, @@ -36,9 +36,8 @@ describe("setting and getting quota of a subvolume", () => { const { used } = await vol.quota.usage(); expect(used).toBeGreaterThan(0); - const v = await vol.fs.ls(""); - // size is potentially random, reflecting compression - expect(v).toEqual([{ name: "buf", mtime: v[0].mtime, size: v[0].size }]); + const v = await vol.fs.readdir(""); + expect(v).toEqual(["buf"]); }); it("fail to write a 50MB file (due to quota)", async () => { @@ -54,20 +53,20 @@ describe("the filesystem operations", () => { it("creates a volume and get empty listing", async () => { vol = await fs.subvolumes.get("fs"); - expect(await vol.fs.ls("")).toEqual([]); + expect(await vol.fs.readdir("")).toEqual([]); }); it("error listing non-existent path", async () => { vol = await fs.subvolumes.get("fs"); expect(async () => { - await vol.fs.ls("no-such-path"); + await vol.fs.readdir("no-such-path"); }).rejects.toThrow("ENOENT"); }); it("creates a text file to it", async () => { await vol.fs.writeFile("a.txt", "hello"); - const ls = await vol.fs.ls(""); - expect(ls).toEqual([{ name: "a.txt", mtime: ls[0].mtime, size: 5 }]); + const ls = await vol.fs.readdir(""); + expect(ls).toEqual(["a.txt"]); }); it("read the file we just created as utf8", async () => { @@ -87,17 +86,18 @@ describe("the filesystem operations", () => { let origStat; it("snapshot filesystem and see file is in snapshot", async () => { await vol.snapshots.create("snap"); - const s = await vol.fs.ls(vol.snapshots.path("snap")); - expect(s).toEqual([{ name: "a.txt", mtime: s[0].mtime, size: 5 }]); + const s = await vol.fs.readdir(vol.snapshots.path("snap")); + expect(s).toContain("a.txt"); - const stat = await vol.fs.stat("a.txt"); - origStat = stat; - expect(stat.mtimeMs / 1000).toBeCloseTo(s[0].mtime ?? 0); + const stat0 = await vol.fs.stat(vol.snapshots.path("snap")); + const stat1 = await vol.fs.stat("a.txt"); + origStat = stat1; + expect(stat1.mtimeMs).toBeCloseTo(stat0.mtimeMs, -2); }); it("unlink (delete) our file", async () => { await vol.fs.unlink("a.txt"); - expect(await vol.fs.ls("")).toEqual([]); + expect(await vol.fs.readdir("")).toEqual([SNAPSHOTS]); }); it("snapshot still exists", async () => { @@ -126,6 +126,7 @@ describe("the filesystem operations", () => { it("make a file readonly, then change it back", async () => { await vol.fs.writeFile("c.txt", "hi"); await vol.fs.chmod("c.txt", "440"); + await fs.sync(); expect(async () => { await vol.fs.appendFile("c.txt", " there"); }).rejects.toThrow("EACCES"); @@ -143,18 +144,17 @@ describe("the filesystem operations", () => { await vol.fs.writeFile("w.txt", "hi"); const ac = new AbortController(); const { signal } = ac; - const watcher = vol.fs.watch("w.txt", { signal }); + const watcher = await vol.fs.watch("w.txt", { signal }); vol.fs.appendFile("w.txt", " there"); // @ts-ignore const { value, done } = await watcher.next(); expect(done).toBe(false); expect(value).toEqual({ eventType: "change", filename: "w.txt" }); ac.abort(); - - expect(async () => { - // @ts-ignore - await watcher.next(); - }).rejects.toThrow("aborted"); + { + const { done } = await watcher.next(); + expect(done).toBe(true); + } }); it("rename a file", async () => { @@ -185,9 +185,9 @@ describe("test snapshots", () => { }); it("snapshot the volume", async () => { - expect(await vol.snapshots.ls()).toEqual([]); + expect(await vol.snapshots.readdir()).toEqual([]); await vol.snapshots.create("snap1"); - expect((await vol.snapshots.ls()).map((x) => x.name)).toEqual(["snap1"]); + expect(await vol.snapshots.readdir()).toEqual(["snap1"]); expect(await vol.snapshots.hasUnsavedChanges()).toBe(false); }); @@ -213,83 +213,18 @@ describe("test snapshots", () => { it("lock our snapshot and confirm it prevents deletion", async () => { await vol.snapshots.lock("snap1"); + await fs.sync(); expect(async () => { await vol.snapshots.delete("snap1"); }).rejects.toThrow("locked"); }); it("unlock our snapshot and delete it", async () => { + await fs.sync(); await vol.snapshots.unlock("snap1"); await vol.snapshots.delete("snap1"); expect(await vol.snapshots.exists("snap1")).toBe(false); - expect(await vol.snapshots.ls()).toEqual([]); - }); -}); - -describe.only("test bup backups", () => { - let vol: Subvolume; - it("creates a volume", async () => { - vol = await fs.subvolumes.get("bup-test"); - await vol.fs.writeFile("a.txt", "hello"); - }); - - it("create a bup backup", async () => { - await vol.bup.save(); - }); - - it("list bup backups of this vol -- there are 2, one for the date and 'latest'", async () => { - const v = await vol.bup.ls(); - expect(v.length).toBe(2); - const t = (v[0].mtime ?? 0) * 1000; - expect(Math.abs(t.valueOf() - Date.now())).toBeLessThan(10_000); - }); - - it("confirm a.txt is in our backup", async () => { - const x = await vol.bup.ls("latest"); - expect(x).toEqual([ - { name: "a.txt", size: 5, mtime: x[0].mtime, isdir: false }, - ]); - }); - - it("restore a.txt from our backup", async () => { - await vol.fs.writeFile("a.txt", "hello2"); - await vol.bup.restore("latest/a.txt"); - expect(await vol.fs.readFile("a.txt", "utf8")).toEqual("hello"); - }); - - it("prune bup backups does nothing since we have so few", async () => { - await vol.bup.prune(); - expect((await vol.bup.ls()).length).toBe(2); - }); - - it("add a directory and back up", async () => { - await mkdir(join(vol.path, "mydir")); - await vol.fs.writeFile(join("mydir", "file.txt"), "hello3"); - expect((await vol.fs.ls("mydir"))[0].name).toBe("file.txt"); - await vol.bup.save(); - const x = await vol.bup.ls("latest"); - expect(x).toEqual([ - { name: "a.txt", size: 5, mtime: x[0].mtime, isdir: false }, - { name: "mydir", size: 0, mtime: x[1].mtime, isdir: true }, - ]); - expect(Math.abs((x[0].mtime ?? 0) * 1000 - Date.now())).toBeLessThan( - 60_000, - ); - }); - - it("change file in the directory, then restore from backup whole dir", async () => { - await vol.fs.writeFile(join("mydir", "file.txt"), "changed"); - await vol.bup.restore("latest/mydir"); - expect(await vol.fs.readFile(join("mydir", "file.txt"), "utf8")).toEqual( - "hello3", - ); - }); - - it("most recent snapshot has a backup before the restore", async () => { - const s = await vol.snapshots.ls(); - const recent = s.slice(-1)[0].name; - const p = vol.snapshots.path(recent, "mydir", "file.txt"); - expect(await vol.fs.readFile(p, "utf8")).toEqual("changed"); + expect(await vol.snapshots.readdir()).toEqual([]); }); }); diff --git a/src/packages/file-server/btrfs/util.ts b/src/packages/file-server/btrfs/util.ts index b6408f84409..86902fd40dd 100644 --- a/src/packages/file-server/btrfs/util.ts +++ b/src/packages/file-server/btrfs/util.ts @@ -35,6 +35,8 @@ export async function sudo( ...opts, command, args, + // LC_ALL, etc. so that btrfs output we parse is not in a different language! + env: { ...process.env, LC_ALL: "C.UTF-8", LANG: "C.UTF-8" }, }); } @@ -44,7 +46,7 @@ export async function btrfs( return await sudo({ ...opts, command: "btrfs" }); } -export async function isdir(path: string) { +export async function isDir(path: string) { return (await stat(path)).isDirectory(); } @@ -63,3 +65,22 @@ export function parseBupTime(s: string): Date { Number(seconds), ); } + +export async function ensureMoreLoopbackDevices() { + // to run tests, this is helpful + //for i in $(seq 8 63); do sudo mknod -m660 /dev/loop$i b 7 $i; sudo chown root:disk /dev/loop$i; done + for (let i = 0; i < 64; i++) { + try { + await stat(`/dev/loop${i}`); + continue; + } catch {} + try { + // also try/catch this because ensureMoreLoops happens in parallel many times at once... + await sudo({ + command: "mknod", + args: ["-m660", `/dev/loop${i}`, "b", "7", `${i}`], + }); + } catch {} + await sudo({ command: "chown", args: ["root:disk", `/dev/loop${i}`] }); + } +} diff --git a/src/packages/file-server/package.json b/src/packages/file-server/package.json index 85eb83d2f19..62e9307be7b 100644 --- a/src/packages/file-server/package.json +++ b/src/packages/file-server/package.json @@ -4,13 +4,16 @@ "description": "CoCalc File Server", "exports": { "./btrfs": "./dist/btrfs/index.js", - "./btrfs/*": "./dist/btrfs/*.js" + "./btrfs/*": "./dist/btrfs/*.js", + "./ssh/*": "./dist/ssh/*.js", + "./conat/*": "./dist/conat/*.js", + "./fs/*": "./dist/fs/*.js" }, "scripts": { "preinstall": "npx only-allow pnpm", "build": "pnpm exec tsc --build", "tsc": "pnpm exec tsc --watch --pretty --preserveWatchOutput", - "test": "pnpm exec jest", + "test": "pnpm exec jest --maxWorkers=1 --forceExit", "depcheck": "pnpx depcheck", "clean": "rm -rf node_modules dist" }, @@ -28,9 +31,14 @@ "license": "SEE LICENSE.md", "dependencies": { "@cocalc/backend": "workspace:*", + "@cocalc/conat": "workspace:*", "@cocalc/file-server": "workspace:*", "@cocalc/util": "workspace:*", - "awaiting": "^3.0.0" + "@isaacs/ttlcache": "^1.4.1", + "awaiting": "^3.0.0", + "express": "^4.21.2", + "http-proxy-3": "^1.21.1", + "micro-key-producer": "^0.8.1" }, "devDependencies": { "@types/jest": "^30.0.0", diff --git a/src/packages/file-server/ssh/auth.ts b/src/packages/file-server/ssh/auth.ts new file mode 100644 index 00000000000..c01259ddba0 --- /dev/null +++ b/src/packages/file-server/ssh/auth.ts @@ -0,0 +1,298 @@ +import express from "express"; +import ssh from "micro-key-producer/ssh.js"; +import { randomBytes } from "micro-key-producer/utils.js"; +import { type Client as ConatClient } from "@cocalc/conat/core/client"; +import { once } from "node:events"; +import { projectApiClient } from "@cocalc/conat/project/api/project-client"; +import { conat } from "@cocalc/backend/conat"; +import * as container from "./container"; +import { secureRandomString } from "@cocalc/backend/misc"; +import getLogger from "@cocalc/backend/logger"; +import { + client as createFileClient, + type Fileserver, +} from "@cocalc/conat/files/file-server"; +import { client as projectRunnerClient } from "@cocalc/conat/project/runner/run"; +import { secretsPath } from "./ssh-server"; +import { join } from "node:path"; +import { readFile, writeFile } from "node:fs/promises"; +import { FILE_SERVER_NAME } from "@cocalc/conat/project/runner/constants"; +import { isValidUUID } from "@cocalc/util/misc"; + +const logger = getLogger("file-server:ssh:auth"); + +const SECRET_TOKEN_LENGTH = 32; + +export async function init({ + base_url, + port, + client, + scratch, +}: { + // as an extra level of security, it is recommended to + // make the base_url a secure random string. + base_url?: string; + port?: number; + client?: ConatClient; + scratch: string; +}) { + logger.debug("init"); + + container.init(); + + base_url ??= encodeURIComponent( + await secureRandomString(SECRET_TOKEN_LENGTH), + ); + client ??= conat(); + let sshKey; + const privKeyPath = join(secretsPath(), "id_ed25519"); + const pubKeyPath = join(secretsPath(), "id_ed25519.pub"); + + try { + sshKey = { + privateKey: await readFile(privKeyPath, "utf8"), + publicKey: await readFile(pubKeyPath, "utf8"), + }; + logger.debug(`init: loaded ssh key from ${secretsPath}...`); + } catch { + logger.debug("init: generating ssh key..."); + const seed = randomBytes(32); + sshKey = ssh(seed, "server"); + // persist to disk so stable between runs, so we can restart server without having to restart all the pods. + await writeFile(privKeyPath, sshKey.privateKey); + await writeFile(pubKeyPath, sshKey.publicKey); + logger.debug("init: public key", sshKey.publicKey); + } + + logger.debug("init: starting ssh server..."); + const app = express(); + app.use(express.json()); + + app.get(`/${base_url}/:user`, async (req, res) => { + try { + const { volume, authorizedKeys, path, target } = await handleRequest( + req.params.user, + client, + ); + + let ports; + if (target == "file-server") { + // the project is actually running, so we ensure ssh target container + // is available locally. + ports = await container.start({ + volume, + scratch, + publicKey: sshKey.publicKey, + authorizedKeys, + path, + }); + } else if (target == "sshd") { + // do NOT start if not already running, because this could be any random + // request for access to the project and user must explicitly start project + // first. Alternatively, if container weren't running we could consider + // starting the project somehow... but that could take a long time. + ports = await container.getPorts({ volume }); + } else { + throw Error(`unknown target '${target}'`); + } + + const port = ports[target]; + if (port == null) { + throw Error(`BUG -- port for target ${target} must be defined`); + } + + const resp = { + privateKey: sshKey.privateKey, + user: "root", + host: `localhost:${port}`, + authorizedKeys, + }; + + logger.debug(req.params.user, "--->", { ...resp, privateKey: "xxx" }); + + res.json(resp); + } catch (err) { + logger.debug("ERROR", err); + // Doing this crashes the ssh server, so instead we respond with '' values. + // res.status(403).json({ error: `${err}` }); + // Alternatively, we would have to rewrite the sshpiper_rest plugin. + res.json({ privateKey: "", user: "", host: "", authorizedKeys: "" }); + } + }); + + const server = app.listen(port); + await once(server, "listening"); + port = server.address().port; + const url = `http://127.0.0.1:${port}/${base_url}`; + const mesg = `sshpiper auth @ http://127.0.0.1:${port}/[...secret...]/:user`; + logger.debug("init: ", mesg); + return { server, app, url }; +} + +async function handleRequest( + user: string | undefined, + client: ConatClient, +): Promise<{ + authorizedKeys: string; + volume: string; + path: string; + target: "file-server" | "sshd"; +}> { + if (!user) { + throw Error("invalid user"); + } + const { target, project_id, compute_server_id } = parseUser(user); + const volume = `project-${project_id}`; + + // NOTE/TODO: we could have a special username that maps to a + // specific path in a project, which would change this path here, + // and require a different auth dance above. This could be for safely + // sharing folders instead of all files in a project. + const path = await getHome(client, project_id); + + const authorizedKeys = await getAuthorizedKeys({ + target, + client, + project_id, + compute_server_id, + path, + }); + + return { authorizedKeys, volume, path, target }; +} + +let fsclient: Fileserver | null = null; +function getFsClient(client) { + fsclient ??= createFileClient({ client }); + return fsclient; +} + +async function getHome(client: ConatClient, project_id: string) { + const c = getFsClient(client); + const { path } = await c.mount({ project_id }); + return path; +} + +async function getAuthorizedKeys({ + target, + client, + project_id, + compute_server_id, + path, +}: { + target: "file-server" | "sshd"; + client: ConatClient; + project_id: string; + compute_server_id: number; + path: string; +}): Promise { + if (target == "file-server") { + if (!compute_server_id) { + const runner = projectRunnerClient({ + client, + project_id, + timeout: 5000, + waitForInterest: false, + }); + const { publicKey } = await runner.status({ project_id }); + if (!publicKey) { + throw Error("no public key available for project"); + } + return publicKey; + } else { + const api = projectApiClient({ + project_id, + compute_server_id, + client, + timeout: 5000, + }); + return await api.system.sshPublicKey(); + } + } else if (target == "sshd") { + if (!compute_server_id) { + // we just read authorized_keys straight from the project + const authorized_keys = join(path, ".ssh", "authorized_keys"); + logger.debug("read project authorized_keys", { + authorized_keys, + project_id, + }); + return await readFile(authorized_keys, "utf8"); + } else { + throw Error("ssh directly to compute server not yet implemented"); + } + } else { + throw Error(`unknown target '${target}'`); + } +} + +/* +The patterns that we support here: + +- project-{uuid} --> target = 'sshd', project_id={uuid} +- {uuid} --> target = 'sshd', project_id={uuid} +- {uuid with dashes removed} --> target='sshd', project_id={uuid with dashes put back} +- {any of 3 above}-{compute_server_id} --> target = 'sshd', project_id={uuid}, compute_server_id=compute_server_id, where it is the GLOBAL compute server id. + +- {FILE_SERVER_NAME}-project-{project_id} --> target='file-server', project_id={project_id} + +*/ +function parseUser(user: string): { + project_id: string; + target: "sshd" | "file-server"; + compute_server_id: number; +} { + let target, prefix; + if (user?.startsWith("project-")) { + target = "sshd"; + prefix = "project-"; + } else if (user?.startsWith(`${FILE_SERVER_NAME}-project-`)) { + // right now we only support project volumes, but later we may + // support volumes like: + // file-server-mydata. + // which gives user access to a volume called "mydata" + // This of course just involves adding a way to lookup who + // has access to mydata which is just determined by a public key...? + target = "file-server"; + prefix = `${FILE_SERVER_NAME}-project-`; + } else if (isValidUUID(user)) { + target = "sshd"; + prefix = ""; + } else if ( + user.length >= 32 && + isValidUUID(putBackDashes(user.split("-")[0])) + ) { + target = "sshd"; + prefix = ""; + const v = user.split("-"); + user = putBackDashes(v[0]); + if (v[1]) { + // compute server id + user += "-" + v[1]; + } + } else { + throw Error(`unknown user ${user}`); + } + + const project_id = user.slice(prefix.length, prefix.length + 36); + const id = user.slice(prefix.length + 37); + const compute_server_id = parseInt(id ? id : "0"); + return { target, project_id, compute_server_id }; +} + +// 00000000-1000-4000-8000-000000000000 +export function putBackDashes(s: string) { + if (s.length != 32) { + throw Error("must have length 32"); + } + return ( + s.slice(0, 8) + + "-" + + s.slice(8, 12) + + "-" + + s.slice(12, 16) + + "-" + + s.slice(16, 20) + + "-" + + s.slice(20) + ); +} diff --git a/src/packages/file-server/ssh/container.ts b/src/packages/file-server/ssh/container.ts new file mode 100644 index 00000000000..b4a648364d3 --- /dev/null +++ b/src/packages/file-server/ssh/container.ts @@ -0,0 +1,433 @@ +/* +Manager container that is the target of ssh. +*/ + +import { reuseInFlight } from "@cocalc/util/reuse-in-flight"; +import getLogger from "@cocalc/backend/logger"; +import { build } from "@cocalc/backend/podman/build-container"; +import { getMutagenAgent } from "./mutagen"; +import { k8sCpuParser, split } from "@cocalc/util/misc"; +import { mkdir, rm, writeFile } from "node:fs/promises"; +import { join } from "node:path"; +import { delay } from "awaiting"; +import * as sandbox from "@cocalc/backend/sandbox/install"; +import { SSHD_CONFIG } from "@cocalc/conat/project/runner/constants"; +import { + FILE_SERVER_NAME, + Ports, + PORTS, +} from "@cocalc/conat/project/runner/constants"; +import { mountArg, podman } from "@cocalc/backend/podman"; +import { sha1 } from "@cocalc/backend/sha1"; + +const FAIR_CPU_MODE = true; + +const GRACE_PERIOD_S = "1"; + +const IDLE_CHECK_INTERVAL = 30_000; + +const logger = getLogger("file-server:ssh:container"); + +const APPS = ["btm", "rg", "fd", "dust", "rustic", "ouch"] as const; +const Dockerfile = ` +FROM docker.io/ubuntu:25.04 +RUN apt-get update && DEBIAN_FRONTEND=noninteractive apt-get install -y ssh rsync +COPY ${APPS.map((path) => sandbox.SPEC[path].binary).join(" ")} /usr/local/bin/ +`; + +const VERSION = "0.3.5"; +const IMAGE = `localhost/${FILE_SERVER_NAME}:${VERSION}`; + +const sshd_conf = ` +Port ${PORTS["file-server"]} +PasswordAuthentication no +ChallengeResponseAuthentication no +UsePAM no +PermitRootLogin yes +PubkeyAuthentication yes +AuthorizedKeysFile ${SSHD_CONFIG}/authorized_keys +AllowTcpForwarding yes +GatewayPorts no +X11Forwarding no +X11UseLocalhost no +PermitTTY yes +Subsystem sftp internal-sftp +`; + +function containerName(volume: string): string { + return `${FILE_SERVER_NAME}-${volume}`; +} + +export const start = reuseInFlight( + async ({ + volume, + path, + publicKey, + authorizedKeys, + // path in which to put directory for mutagen's state, which includes + // it's potentially large staging area. This should be on the same + // btrfs filesystem as projects for optimal performance. It MUST BE SET, because + // without this when the user's quota is hit, the sync just restarts, causing an infinite + // loop wasting resources. It should also obviously be big. + scratch, + // TODO: think about limits once I've benchmarked + pids = 100, + // reality: mutagen potentially uses a lot of RAM of you have + // a lot of files, and if it keeps crashing due to out of memory, + // the project is completely broken, so for now: + memory = "4000m", + cpu = "1000m", + lockdown = true, + }: { + volume: string; + path: string; + publicKey: string; + authorizedKeys: string; + scratch: string; + memory?: string; + cpu?: string; + pids?: number; + // can be nice to disable for dev and debugging + lockdown?: boolean; + }): Promise => { + if (!scratch) { + throw Error("scratch directory must be set"); + } + const name = containerName(volume); + const key = sha1(publicKey + authorizedKeys); + try { + const { stdout } = await podman([ + "inspect", + name, + "--format", + "{{json .Config.Labels.key}}|{{json .NetworkSettings.Ports}}", + ]); + const x = stdout.split("|"); + const storedKey = JSON.parse(x[0]); + if (storedKey == key) { + return jsonToPorts(JSON.parse(x[1])); + } else { + // I don't understand why things stop working with the key changes... + logger.debug(`restarting ${name} since key changed`); + await stop({ volume }); + } + } catch {} + // container does not exist -- create it + const args = ["run"]; + args.push("--detach"); + // the container is named in a way that is determined by the volume name, + // but we also use labels. + args.push("--name", name); + // I unfortunately hit multiple times cases where the pasta.avx process + // would be pegged forever at 100% cpu, even though everything was + // completely idle, so there is definitely a major bug with pasta in + // "podman version 5.4.1". + args.push("--network", "slirp4netns"); + args.push("--hostname", name); + args.push( + "--label", + `volume=${volume}`, + "--label", + `role=file-server`, + "--label", + `key=${key}`, + ); + + // mount the volume contents to the directory /root in the container. + // Since user can write arbitrary files here, this is noexec, so they + // can't somehow run them. + args.push( + mountArg({ + source: path, + target: "/root", + options: "noexec,nodev,nosuid", + }), + ); + + const sshdConfPathOnHost = join(path, SSHD_CONFIG); + await mkdir(sshdConfPathOnHost, { recursive: true, mode: 0o700 }); + await writeFile(join(sshdConfPathOnHost, "authorized_keys"), publicKey, { + mode: 0o600, + }); + await writeFile(join(sshdConfPathOnHost, "sshd.conf"), sshd_conf, { + mode: 0o600, + }); + + for (const key in PORTS) { + args.push("-p", `${PORTS[key]}`); + } + + if (lockdown) { + args.push( + "--cap-drop", + "ALL", + // SYS_CHROOT: needed for ssh each time we get a new connection + "--cap-add", + "SYS_CHROOT", + "--cap-add", + "SETGID", + "--cap-add", + "SETUID", + // CHOWN: needed to rsync rootfs and preserve uid's + "--cap-add", + "CHOWN", + // FSETID: needed to preserve setuid/setgid bits (e.g,. so ping for regular users works) + "--cap-add", + "FSETID", + // FOWNER: needed to set permissions when rsync'ing rootfs + "--cap-add", + "FOWNER", + // these two are so root can see inside non-root user paths when doing backups of rootfs + "--cap-add", + "DAC_READ_SEARCH", + "--cap-add", + "DAC_OVERRIDE", + ); + + // Limits + if (pids) { + args.push(`--pids-limit=${pids}`); + } + if (memory) { + args.push(`--memory=${memory}`); + } + if (FAIR_CPU_MODE) { + args.push("--cpu-shares=128"); + } else if (cpu) { + args.push(`--cpus=${k8sCpuParser(cpu)}`); + } + // make root filesystem readonly so can't install new software or waste space + args.push("--read-only"); + } + + const dotMutagen = join(scratch, volume); + dotMutagens[volume] = dotMutagen; + try { + await rm(dotMutagen, { force: true, recursive: true }); + } catch {} + await mkdir(dotMutagen, { recursive: true }); + args.push(mountArg({ source: dotMutagen, target: "/root/.mutagen-dev" })); + // Mutagen agent mounted in + const mutagen = await getMutagenAgent(); + args.push( + mountArg({ + source: mutagen.path, + target: `/root/.mutagen-dev/agents/${mutagen.version}`, + readOnly: true, + }), + ); + + // openssh server + // /usr/sbin/sshd -D -e -f /root/{SSHD_CONFIG}/sshd.conf + args.push( + "--rm", + IMAGE, + "/usr/sbin/sshd", + "-D", + "-e", + "-f", + `/root/${SSHD_CONFIG}/sshd.conf`, + ); + logger.debug(`Start file-system container: 'podman ${args.join(" ")}'`); + await podman(args); + logger.debug("Started file-system container", { volume }); + const ports = await getPorts({ volume }); + logger.debug("Got ports", { volume, ports }); + return ports; + }, +); + +function jsonToPorts(obj) { + // obj looks like this: + // obj = { + // "2222/tcp": [{ HostIp: "0.0.0.0", HostPort: "42419" }], + // "2223/tcp": [{ HostIp: "0.0.0.0", HostPort: "38437" }], + // "2224/tcp": [{ HostIp: "0.0.0.0", HostPort: "41165" }], + // "2225/tcp": [{ HostIp: "0.0.0.0", HostPort: "34057" }], + // }; + const portMap: { [p: number]: number } = {}; + for (const k in obj) { + const port = parseInt(k.split("/")[0]); + portMap[port] = obj[k][0].HostPort; + } + const ports: Partial = {}; + for (const k in PORTS) { + ports[k] = portMap[PORTS[k]]; + if (ports[k] == null) { + throw Error("BUG -- not all ports found"); + } + } + return ports as Ports; +} + +export const getPorts = reuseInFlight( + async ({ volume }: { volume: string }): Promise => { + const { stdout } = await podman([ + "inspect", + containerName(volume), + "--format", + "{{json .NetworkSettings.Ports}}", + ]); + return jsonToPorts(JSON.parse(stdout)); + }, +); + +const dotMutagens: { [volume: string]: string } = {}; + +export async function stop({ + volume, + force, +}: { + volume: string; + force?: boolean; +}) { + try { + logger.debug("stopping", { volume }); + await podman([ + "rm", + "-f", + "-t", + force ? "0" : GRACE_PERIOD_S, + containerName(volume), + ]); + } catch (err) { + logger.debug("stop error", { volume, err }); + } + const dotMutagen = dotMutagens[volume]; + if (dotMutagen) { + try { + await rm(dotMutagen, { force: true, recursive: true }); + } catch {} + delete dotMutagens[volume]; + } +} + +const buildContainerImage = reuseInFlight(async () => { + // make sure apps are installed + const v: any[] = []; + for (const app of APPS) { + v.push(sandbox.install(app)); + } + await Promise.all(v); + + // make sure our ssh image is available + await build({ + name: IMAGE, + Dockerfile, + files: APPS.map((name) => sandbox[name]), + }); +}); + +export async function getProcesses(name) { + try { + const { stdout } = await podman([ + "exec", + name, + "ps", + "-o", + "ucmd", + "--no-headers", + ]); + return split(stdout.toString()); + } catch { + return []; + } +} + +export async function terminateIfIdle(name): Promise { + if ((await getProcesses(name)).includes("sshd-session")) { + // has an open ssh session + return false; + } + await podman(["rm", "-f", "-t", GRACE_PERIOD_S, name]); + return true; +} + +export async function terminateAllIdle({ + minAge = 15_000, +}: { minAge?: number } = {}) { + const { stdout } = await podman([ + "ps", + "-a", + "--filter", + `name=${FILE_SERVER_NAME}-`, + "--filter=label=role=file-server", + "--format", + "{{.Names}} {{.StartedAt}}", + ]); + const tasks: any[] = []; + const now = Date.now(); + let killed = 0; + const f = async (name) => { + if (await terminateIfIdle(name)) { + killed += 1; + } + }; + let total = 0; + for (const x of stdout.toString().trim().split("\n")) { + const w = split(x); + if (w.length == 2) { + total++; + const [name, startedAt] = w; + if (now - parseInt(startedAt) * 1000 >= minAge) { + tasks.push(f(name)); + } + } + } + await Promise.all(tasks); + return { killed, total }; +} + +let monitoring = false; +export async function init() { + if (monitoring) return; + monitoring = true; + + await stopAll(); + await buildContainerImage(); + + while (monitoring) { + // wait first, otherwise everything is instantly killed + // on startup, since sshpiperd itself just got restarted + // (at least until we daemonize it). + await delay(IDLE_CHECK_INTERVAL); + try { + logger.debug(`scanning for idle file-system containers...`); + const { killed, total } = await terminateAllIdle(); + logger.debug(`file-system container idle check`, { total, killed }); + } catch (err) { + logger.debug( + "WARNING -- issue terminating idle file-system containers", + err, + ); + } + } +} + +export async function getAll(): Promise { + const { stdout } = await podman([ + "ps", + `--filter=name=${FILE_SERVER_NAME}-`, + `--filter=label=role=file-server`, + `--format={{ index .Labels "volume" }}`, + ]); + return stdout.split("\n").filter((x) => x); +} + +// important to clear on startup, because for whatever reason the file-server containers +// do NOT work if we restart sshpiperd, so best to stop them all. +// They don't work without the proxy anyways. We can't kill all on shutdown unless +// we make them child processes (we could do that). + +export async function stopAll() { + logger.debug(`stopping all ${FILE_SERVER_NAME} containers`); + monitoring = false; + const v: any[] = []; + const volumes = await getAll(); + logger.debug(volumes); + for (const volume of volumes) { + logger.debug("stopping", { volume }); + v.push(stop({ volume, force: true })); + } + await Promise.all(v); +} diff --git a/src/packages/file-server/ssh/mutagen.ts b/src/packages/file-server/ssh/mutagen.ts new file mode 100644 index 00000000000..477ed0d7b23 --- /dev/null +++ b/src/packages/file-server/ssh/mutagen.ts @@ -0,0 +1,70 @@ +import { mkdtemp } from "node:fs/promises"; +import { rmSync } from "node:fs"; +import { execFile as execFile0 } from "node:child_process"; +import { promisify } from "node:util"; +import { join } from "node:path"; +import { reuseInFlight } from "@cocalc/util/reuse-in-flight"; +import { mutagen } from "@cocalc/backend/sandbox/install"; +import { tmpdir } from "node:os"; + +import getLogger from "@cocalc/backend/logger"; +const execFile = promisify(execFile0); + +const logger = getLogger("file-server:ssh:mutagen"); + +let path = ""; +let version = ""; +export const getMutagenAgent = reuseInFlight( + async (): Promise<{ + path: string; + version: string; + }> => { + if (path && version) { + return { path, version }; + } + + logger.debug("getMutagenAgent: extracting..."); + + const tmp = await mkdtemp(join(tmpdir(), "cocalc-mutagen-")); + const agentTarball = mutagen + "-agents.tar.gz"; + + await execFile( + "tar", + [ + "xf", + agentTarball, + "--transform=s/linux_amd64/mutagen-agent/", + "linux_amd64", + ], + { cwd: tmp }, + ); + + // copy the correct agent over, extract it, and also + // note the version. + const { stdout } = await execFile(join(tmp, "mutagen-agent"), [ + "--version", + ]); + version = stdout.trim().split(" ").slice(-1)[0]; + path = tmp; + logger.debug("getMutagenAgent: created", { version, path }); + + return { path, version }; + }, +); + +export function close() { + if (!path) { + return; + } + try { + rmSync(path, { recursive: true, force: true }); + } catch {} + path = ""; +} + +process.once("exit", close); +["SIGINT", "SIGTERM", "SIGQUIT"].forEach((sig) => { + process.once(sig, () => { + process.exit(); + }); +}); diff --git a/src/packages/file-server/ssh/proxy.ts b/src/packages/file-server/ssh/proxy.ts new file mode 100644 index 00000000000..f10cb039b8d --- /dev/null +++ b/src/packages/file-server/ssh/proxy.ts @@ -0,0 +1,141 @@ +/* +This starts a very lightweight http server listening on the requested port, +which proxies all http traffic to project containers as follows: + + /{project_id}/{rest} --> localhost:{port}/{project_id}/{rest} + +where port is what is published as PORTS.proxy from +packages/conat/project/runner/constants.ts. + +This uses the http-proxy-3 library, which is a modern supported version +of the old http-proxy nodejs npm library, with the same API. + +- We set xfwd headers and support WebSockets. +*/ + +import * as http from "node:http"; +import httpProxy from "http-proxy-3"; +import { getLogger } from "@cocalc/backend/logger"; +import { isValidUUID } from "@cocalc/util/misc"; +import { getPorts } from "./container"; +import TTLCache from "@isaacs/ttlcache"; +import listen from "@cocalc/backend/misc/async-server-listen"; + +const logger = getLogger("file-server:http-proxy"); + +const CACHE_TTL = 1000; +const cache = new TTLCache({ + max: 100000, + ttl: CACHE_TTL, + updateAgeOnGet: true, +}); + +interface StartOptions { + port?: number; // default 8080 + host?: string; // default 127.0.0.1 +} + +export async function startProxyServer({ + port = 8080, + host = "127.0.0.1", +}: StartOptions = {}) { + logger.debug("startProxyServer", { port, host }); + + const { handleRequest, handleUpgrade } = createProxyHandlers(); + + const proxyServer = http.createServer(handleRequest); + proxyServer.on("upgrade", handleUpgrade); + + await listen({ + server: proxyServer, + port, + host, + desc: "file-server's HTTP proxy server", + }); + + return proxyServer; +} + +export function createProxyHandlers() { + const proxy = httpProxy.createProxyServer({ + xfwd: true, + ws: true, + // We set target per-request. + }); + + proxy.on("error", (err, req, res) => { + const url = (req as http.IncomingMessage).url; + logger.warn("proxy error", { err, url }); + // Best-effort error response (HTTP only): + if (!res || (res as http.ServerResponse).headersSent) return; + try { + (res as http.ServerResponse).writeHead(502, { + "Content-Type": "text/plain", + }); + (res as http.ServerResponse).end("Bad Gateway\n"); + } catch { + /* ignore */ + } + }); + + proxy.on("proxyReq", (proxyReq) => { + proxyReq.setHeader("X-Proxy-By", "cocalc-proxy"); + }); + + const handleRequest = async (req, res) => { + try { + const target = await getTarget(req); + proxy.web(req, res, { target }); + } catch { + // Not matched — 404 so it's obvious when a wrong base is used. + res.writeHead(404, { "Content-Type": "text/plain" }); + res.end("Not found\n"); + } + }; + + const handleUpgrade = async (req, socket, head) => { + try { + const target = await getTarget(req); + proxy.ws(req, socket, head, { + target, + }); + } catch { + // Not matched — close gracefully. + socket.write("HTTP/1.1 404 Not Found\r\nConnection: close\r\n\r\n"); + socket.destroy(); + return; + } + }; + + return { handleRequest, handleUpgrade }; +} + +async function getProxyPort(project_id: string) { + if (!isValidUUID(project_id)) { + throw Error("invalid url"); + } + if (cache.has(project_id)) { + const { proxy, err } = cache.get(project_id)!; + if (err) { + throw err; + } else { + return proxy; + } + } + let proxy; + try { + ({ proxy } = await getPorts({ volume: `project-${project_id}` })); + } catch (err) { + cache.set(project_id, { err }); + throw err; + } + cache.set(project_id, { proxy }); + return proxy; +} + +async function getTarget(req) { + const url = req.url ?? ""; + logger.debug("request", { url }); + const project_id = url.slice(1, 37); + return { port: await getProxyPort(project_id), host: "localhost" }; +} diff --git a/src/packages/file-server/ssh/ssh-server.ts b/src/packages/file-server/ssh/ssh-server.ts new file mode 100644 index 00000000000..ad761b7aa80 --- /dev/null +++ b/src/packages/file-server/ssh/ssh-server.ts @@ -0,0 +1,106 @@ +/* +Ssh server - manages how projects and their files are accessed via ssh. + +This is a service that runs directly on the btrfs file server. It: + +- listens for incoming ssh connections from: + - project + - compute server + - external users + +- uses conat to determine what public keys grant access to a user + of the above type + +- if user is valid, it creates container (if necessary) and connects + them to it via ssh. + + +./sshpiperd \ + -i server_host_key \ + --server-key-generate-mode notexist \ + ./sshpiperd-rest --url http://127.0.0.1:8443/auth + + +Security NOTE / TODO: It would be more secure to modify sshpiperd-rest +to support a UDP socket and use that instead, since we're running +the REST server on localhost. +*/ + +import { init as initAuth } from "./auth"; +import { startProxyServer, createProxyHandlers } from "./proxy"; +import { install, sshpiper } from "@cocalc/backend/sandbox/install"; +import { type Client as ConatClient } from "@cocalc/conat/core/client"; +import { secrets, sshServer } from "@cocalc/backend/data"; +import { dirname, join } from "node:path"; +import { mkdir } from "node:fs/promises"; +import { spawn } from "node:child_process"; +import getLogger from "@cocalc/backend/logger"; + +const logger = getLogger("file-server:ssh:ssh-server"); + +export function secretsPath() { + return join(secrets, "sshpiperd"); +} + +const children: any[] = []; +export async function init({ + port = sshServer.port, + client, + scratch, + proxyHandlers, +}: { + port?: number; + client?: ConatClient; + scratch: string; + proxyHandlers?: boolean; +}) { + logger.debug("init", { port, proxyHandlers }); + // ensure sshpiper is installed + await install("sshpiper"); + const projectProxyHandlers = proxyHandlers + ? createProxyHandlers() + : await startProxyServer(); + const { url } = await initAuth({ client, scratch }); + const hostKey = join(secretsPath(), "host_key"); + await mkdir(dirname(hostKey), { recursive: true }); + const args = [ + "-i", + hostKey, + `--port=${port}`, + "--log-level=warn", + "--server-key-generate-mode", + "notexist", + sshpiper + "-rest", + "--url", + url, + ]; + logger.debug(`${sshpiper} ${args.join(" ")}`); + const child = spawn(sshpiper, args); + children.push(child); + child.stdout.on("data", (chunk: Buffer) => { + logger.debug(chunk.toString()); + }); + child.stderr.on("data", (chunk: Buffer) => { + logger.debug(chunk.toString()); + }); + + return { child, projectProxyHandlers }; +} + +export function close() { + for (const child of children) { + if (child.exitCode == null) { + child.kill("SIGKILL"); + } + } + children.length = 0; +} + +// important because it kills all +// the processes that were spawned +process.once("exit", close); +["SIGINT", "SIGTERM", "SIGQUIT"].forEach((sig) => { + process.once(sig, () => { + process.exit(); + }); +}); diff --git a/src/packages/frontend/account/account-button.tsx b/src/packages/frontend/account/account-button.tsx index ccb46d510da..402ec9c6b3b 100644 --- a/src/packages/frontend/account/account-button.tsx +++ b/src/packages/frontend/account/account-button.tsx @@ -6,7 +6,6 @@ import { Popconfirm, Popover } from "antd"; import React from "react"; import { FormattedMessage, useIntl } from "react-intl"; - import { AccountActions } from "@cocalc/frontend/account"; import { labels } from "@cocalc/frontend/i18n"; import { CancelText } from "@cocalc/frontend/i18n/components"; diff --git a/src/packages/frontend/account/account-page.tsx b/src/packages/frontend/account/account-page.tsx index ff543842276..9e7ac3cc87e 100644 --- a/src/packages/frontend/account/account-page.tsx +++ b/src/packages/frontend/account/account-page.tsx @@ -12,7 +12,7 @@ and configuration. // cSpell:ignore payg -import { Flex, Menu, Space } from "antd"; +import { Button, Flex, Menu, Space } from "antd"; import { useEffect } from "react"; import { useIntl } from "react-intl"; import { SignOut } from "@cocalc/frontend/account/sign-out"; @@ -46,6 +46,8 @@ import { I18NSelector } from "./i18n-selector"; import { LicensesPage } from "./licenses/licenses-page"; import { PublicPaths } from "./public-paths/public-paths"; import { UpgradesPage } from "./upgrades/upgrades-page"; +import { lite, project_id } from "@cocalc/frontend/lite"; +import { AdminPage } from "@cocalc/frontend/admin"; // give up on trying to load account info and redirect to landing page. // Do NOT make too short, since loading account info might takes ~10 seconds, e,g., due @@ -96,6 +98,17 @@ export const AccountPage: React.FC = () => { ), }, ]; + if (lite) { + items.push({ + key: "admin", + label: ( + + Admin + + ), + children: active_page === "admin" && , + }); + } // adds a few conditional tabs if (is_anonymous) { // None of the rest make any sense for a temporary anonymous account. @@ -185,16 +198,18 @@ export const AccountPage: React.FC = () => { items.push({ type: "divider" }); } - items.push({ - key: "public-files", - label: ( - - {" "} - {intl.formatMessage(labels.published_files)} - - ), - children: active_page === "public-files" && , - }); + if (!lite) { + items.push({ + key: "public-files", + label: ( + + {" "} + {intl.formatMessage(labels.published_files)} + + ), + children: active_page === "public-files" && , + }); + } if (cloudFilesystemsEnabled()) { items.push({ key: "cloud-filesystems", @@ -236,7 +251,9 @@ export const AccountPage: React.FC = () => { {is_commercial ? : undefined} - + {!lite && ( + + )} ); } @@ -288,15 +305,35 @@ export const AccountPage: React.FC = () => { borderRight: "1px solid rgba(5, 5, 5, 0.06)", }} > -
- Account Configuration -
+ {!lite && ( +
+ Account Configuration +
+ )} + {lite && ( +
+ +
+ )} { handle_select(e.key); diff --git a/src/packages/frontend/account/actions.ts b/src/packages/frontend/account/actions.ts index 9f8a30a78d5..8cf33f3d00f 100644 --- a/src/packages/frontend/account/actions.ts +++ b/src/packages/frontend/account/actions.ts @@ -19,6 +19,7 @@ import { Actions } from "@cocalc/util/redux/Actions"; import { show_announce_end, show_announce_start } from "./dates"; import { AccountStore } from "./store"; import { AccountState } from "./types"; +import { lite } from "@cocalc/frontend/lite"; // Define account actions export class AccountActions extends Actions { @@ -140,8 +141,10 @@ export class AccountActions extends Actions { this.push_state("/" + tab); } - // Add an ssh key for this user, with the given fingerprint, title, and value - public add_ssh_key(unsafe_opts: unknown): void { + // Add an ssh key for this user, with the given fingerprint, + // title, and value. Also updates authorized_keys for all running + // projects. + add_ssh_key = async (unsafe_opts: unknown): Promise => { const opts = define<{ fingerprint: string; title: string; @@ -151,7 +154,7 @@ export class AccountActions extends Actions { title: required, value: required, }); - this.redux.getTable("account").set({ + await this.redux.getTable("account").set({ ssh_keys: { [opts.fingerprint]: { title: opts.title, @@ -160,16 +163,37 @@ export class AccountActions extends Actions { }, }, }); - } + await this.updateAuthorizedKeysForRunningProjects(true); + }; // Delete the ssh key with given fingerprint for this user. - public delete_ssh_key(fingerprint): void { - this.redux.getTable("account").set({ + // Also updates authorized_keys for all running projects. + delete_ssh_key = async (fingerprint): Promise => { + await this.redux.getTable("account").set({ ssh_keys: { [fingerprint]: null, }, }); // null is how to tell the backend/synctable to delete this... - } + await this.updateAuthorizedKeysForRunningProjects(true); + }; + + // call after adding/removing global ssh keys + updateAuthorizedKeysForRunningProjects = async (ignoreErrors = true) => { + const store = this.redux.getStore("projects"); + const f = async (project_id) => { + const api = webapp_client.conat_client.projectApi({ project_id }); + try { + await api.system.updateSshKeys(); + } catch (err) { + if (!ignoreErrors) { + throw err; + } + // it is expected for these to sometimes fail, e.g., because + // the state is listed as "running" but it is stale. + } + }; + await Promise.all(store.getRunningProjects().map(f)); + }; public set_account_table(obj: object): void { this.redux.getTable("account").set(obj); @@ -246,6 +270,9 @@ export class AccountActions extends Actions { }; addTag = async (tag: string) => { + if (lite) { + return; + } const store = this.redux.getStore("account"); if (!store) return; const tags = store.get("tags"); diff --git a/src/packages/frontend/account/editor-settings/checkboxes.tsx b/src/packages/frontend/account/editor-settings/checkboxes.tsx index b40c611d89b..421b8df6748 100644 --- a/src/packages/frontend/account/editor-settings/checkboxes.tsx +++ b/src/packages/frontend/account/editor-settings/checkboxes.tsx @@ -87,7 +87,7 @@ const EDITOR_SETTINGS_CHECKBOXES = { disable_jupyter_virtualization: defineMessage({ id: "account.editor-setting.checkbox.disable_jupyter_virtualization", defaultMessage: - "render entire Jupyter Notebook instead of just visible part (slower and not recommended)", + "render entire Jupyter Notebook instead of just visible part (slower but more reliable)", }), } as const; diff --git a/src/packages/frontend/account/other-settings.tsx b/src/packages/frontend/account/other-settings.tsx index 299bd681b37..9e93bce6b06 100644 --- a/src/packages/frontend/account/other-settings.tsx +++ b/src/packages/frontend/account/other-settings.tsx @@ -55,6 +55,7 @@ import Messages from "./messages"; import Tours from "./tours"; import { useLanguageModelSetting } from "./useLanguageModelSetting"; import { UserDefinedLLMComponent } from "./user-defined-llm"; +import { lite } from "@cocalc/frontend/lite"; const DARK_MODE_LABELS = defineMessages({ brightness: { @@ -95,6 +96,7 @@ export function OtherSettings(props: Readonly): React.JSX.Element { const { locale } = useLocalizationCtx(); const isCoCalcCom = useTypedRedux("customize", "is_cocalc_com"); const user_defined_llm = useTypedRedux("customize", "user_defined_llm"); + const is_commercial = useTypedRedux("customize", "is_commercial"); const [model, setModel] = useLanguageModelSetting(); @@ -200,7 +202,7 @@ export function OtherSettings(props: Readonly): React.JSX.Element { } function render_standby_timeout(): Rendered { - if (IS_TOUCH) { + if (IS_TOUCH || lite) { return; } return ( @@ -296,32 +298,6 @@ export function OtherSettings(props: Readonly): React.JSX.Element { ); } - function render_default_file_sort(): Rendered { - return ( - - on_change("default_file_sort", value)} - /> - - ); - } - function render_new_filenames(): Rendered { const selected = props.other_settings.get(NEW_FILENAMES) ?? DEFAULT_NEW_FILENAMES; @@ -353,24 +329,6 @@ export function OtherSettings(props: Readonly): React.JSX.Element { ); } - function render_page_size(): Rendered { - return ( - - on_change("page_size", n)} - min={1} - max={10000} - number={props.other_settings.get("page_size")} - /> - - ); - } - function render_no_free_warnings(): Rendered { let extra; if (!props.is_stripe_customer) { @@ -734,13 +692,11 @@ export function OtherSettings(props: Readonly): React.JSX.Element { {render_i18n_selector()} {render_vertical_fixed_bar_options()} {render_new_filenames()} - {render_default_file_sort()} - {render_page_size()} {render_standby_timeout()}
- + {is_commercial && } ); diff --git a/src/packages/frontend/account/settings-button.tsx b/src/packages/frontend/account/settings-button.tsx new file mode 100644 index 00000000000..6fbfc163685 --- /dev/null +++ b/src/packages/frontend/account/settings-button.tsx @@ -0,0 +1,22 @@ +/* +Button to get to the settings page. Currently used by lite mode only. +*/ + +import { Icon } from "@cocalc/frontend/components"; +import { Button } from "antd"; +import { redux } from "@cocalc/frontend/app-framework"; + +export default function SettingsButton() { + const icon = ; + return ( + + ); +} diff --git a/src/packages/frontend/account/settings/account-settings.tsx b/src/packages/frontend/account/settings/account-settings.tsx index aa0846df73b..abb16592ee6 100644 --- a/src/packages/frontend/account/settings/account-settings.tsx +++ b/src/packages/frontend/account/settings/account-settings.tsx @@ -7,7 +7,6 @@ import { Alert as AntdAlert, Space } from "antd"; import { List, Map } from "immutable"; import { join } from "path"; import { FormattedMessage, useIntl } from "react-intl"; - import { Alert, Button, @@ -53,6 +52,7 @@ import { EmailAddressSetting } from "./email-address-setting"; import { EmailVerification } from "./email-verification"; import { PasswordSetting } from "./password-setting"; import { TextSetting } from "./text-setting"; +import { lite } from "@cocalc/frontend/lite"; type ImmutablePassportStrategy = TypedMap; @@ -250,7 +250,7 @@ export function AccountSettings(props: Readonly) { } function render_sign_out_error(): Rendered { - if (!props.sign_out_error) { + if (!props.sign_out_error || lite) { return; } return ( @@ -263,6 +263,9 @@ export function AccountSettings(props: Readonly) { } function render_sign_out_buttons(): Rendered { + if (lite) { + return; + } return ( ) { } function render_linked_external_accounts(): Rendered { - if (props.strategies == null || props.strategies.size <= 1) { + if (props.strategies == null || props.strategies.size <= 1 || lite) { // not configured by server return; } @@ -324,7 +327,7 @@ export function AccountSettings(props: Readonly) { } function render_available_to_link(): Rendered { - if (props.strategies == null || props.strategies.size <= 1) { + if (props.strategies == null || props.strategies.size <= 1 || lite) { // not configured by server yet, or nothing but email return; } @@ -390,7 +393,7 @@ export function AccountSettings(props: Readonly) { } function render_anonymous_warning(): Rendered { - if (!props.is_anonymous) { + if (!props.is_anonymous || lite) { return; } // makes no sense to delete an account that is anonymous; it'll @@ -414,7 +417,7 @@ export function AccountSettings(props: Readonly) { } function render_delete_account(): Rendered { - if (props.is_anonymous) { + if (props.is_anonymous || lite) { return; } return ( @@ -434,7 +437,7 @@ export function AccountSettings(props: Readonly) { } function render_password(): Rendered { - if (!props.email_address) { + if (!props.email_address || lite) { // makes no sense to change password if don't have an email address return; } @@ -442,7 +445,7 @@ export function AccountSettings(props: Readonly) { } function render_terms_of_service(): Rendered { - if (!props.is_anonymous) { + if (!props.is_anonymous || lite) { return; } const style: React.CSSProperties = { padding: "10px 20px" }; @@ -572,7 +575,7 @@ will no longer work (automatic redirects are not implemented), so change with ca } function render_email_address(): Rendered { - if (!props.account_id) { + if (!props.account_id || lite) { return; // makes no sense to change email if there is no account } return ( @@ -586,7 +589,7 @@ will no longer work (automatic redirects are not implemented), so change with ca } function render_unlisted(): Rendered { - if (!props.account_id) { + if (!props.account_id || lite) { return; // makes no sense to change unlisted status if there is no account } return ( diff --git a/src/packages/frontend/account/ssh-keys/global-ssh-keys.tsx b/src/packages/frontend/account/ssh-keys/global-ssh-keys.tsx index 5eea57ee437..0705cd3d9ea 100644 --- a/src/packages/frontend/account/ssh-keys/global-ssh-keys.tsx +++ b/src/packages/frontend/account/ssh-keys/global-ssh-keys.tsx @@ -33,12 +33,13 @@ export default function GlobalSSHKeys() { all projects and compute servers - on which you are an owner or collaborator. + on which you are a collaborator. Alternatively, set SSH keys that grant access only to a project in the settings for that project. See the docs - or the SSH part of the settings page in a project for further instructions.`} + or the SSH part of the settings page in a project for further instructions. + Adding keys here simply automates them being added to the file ~/.ssh/authorized_keys`} values={{ strong: (c) => {c}, i: (c) => {c}, diff --git a/src/packages/frontend/account/ssh-keys/ssh-key-adder.tsx b/src/packages/frontend/account/ssh-keys/ssh-key-adder.tsx index 8400700f700..a1bfc6708e2 100644 --- a/src/packages/frontend/account/ssh-keys/ssh-key-adder.tsx +++ b/src/packages/frontend/account/ssh-keys/ssh-key-adder.tsx @@ -150,7 +150,7 @@ export default function SSHKeyAdder({ {intl.formatMessage( { id: "account.ssh-key-adder.title", - defaultMessage: "Add an SSH key", + defaultMessage: "Add SSH key", }, { A: (c) => ( @@ -177,17 +177,6 @@ export default function SSHKeyAdder({ > {extra && extra}
- Title - setKeyTitle(e.target.value)} - placeholder={intl.formatMessage({ - id: "account.ssh-key-adder.placeholder", - defaultMessage: - "Choose a name for this ssh key to help you keep track of it...", - })} - />
Key ; @@ -74,7 +75,7 @@ export default function SSHKeyList({ function render_header() { return ( - {project_id ? "Project Specific " : "Global "} + {project_id ? "Project " : ""} {intl.formatMessage(labels.ssh_keys)} {help && {help}}
@@ -164,18 +165,18 @@ interface OneSSHKeyProps { function OneSSHKey({ ssh_key, project_id, mode = "project" }: OneSSHKeyProps) { const isFlyout = mode === "flyout"; - function render_last_use(): React.JSX.Element { - const d = ssh_key.get("last_use_date"); - if (d) { - return ( - - Last used - - ); - } else { - return Never used; - } - } + // function render_last_use(): React.JSX.Element { + // const d = ssh_key.get("last_use_date"); + // if (d) { + // return ( + // + // Last used + // + // ); + // } else { + // return Never used; + // } + // } function delete_key(): void { const fingerprint = ssh_key.get("fingerprint"); @@ -232,8 +233,14 @@ function OneSSHKey({ ssh_key, project_id, mode = "project" }: OneSSHKeyProps) { {ssh_key.get("fingerprint")}
+ Added on {new Date(ssh_key.get("creation_date")).toLocaleDateString()} -
{render_last_use()} (NOTE: not all usage is tracked.)
+ {/*
{render_last_use()} (NOTE: not all usage is tracked.)
*/}
); diff --git a/src/packages/frontend/admin/page.tsx b/src/packages/frontend/admin/page.tsx index 99bba3301bc..7caa39193e0 100644 --- a/src/packages/frontend/admin/page.tsx +++ b/src/packages/frontend/admin/page.tsx @@ -10,7 +10,6 @@ import { Icon, Title } from "@cocalc/frontend/components"; import { SiteLicenses } from "../site-licenses/admin/component"; import { RegistrationToken } from "./registration-token"; import SiteSettings from "./site-settings"; -import { UsageStatistics } from "./stats/page"; import { SystemNotifications } from "./system-notifications"; import { UserSearch } from "./users/user-search"; import AIAvatar from "@cocalc/frontend/components/ai-avatar"; @@ -31,15 +30,6 @@ export function AdminPage() { ), children: , }, - { - key: "site-licenses", - label: ( -
- Licenses -
- ), - children: , - }, { key: "site-settings", label: ( @@ -55,6 +45,15 @@ export function AdminPage() { /> ), }, + { + key: "site-licenses", + label: ( +
+ Licenses +
+ ), + children: , + }, { key: "registration-tokens", label: ( @@ -75,16 +74,16 @@ export function AdminPage() { ), children: , }, - { - key: "usage-stats", - label: ( -
- Usage - Statistics -
- ), - children: , - }, + // { + // key: "usage-stats", + // label: ( + //
+ // Usage + // Statistics + //
+ // ), + // children: , + // }, { key: "llm-testing", label: ( diff --git a/src/packages/frontend/admin/site-settings/row-entry.tsx b/src/packages/frontend/admin/site-settings/row-entry.tsx index 8c964e00ca3..22e5ed48588 100644 --- a/src/packages/frontend/admin/site-settings/row-entry.tsx +++ b/src/packages/frontend/admin/site-settings/row-entry.tsx @@ -168,9 +168,8 @@ function VersionHint({ value }: { value: string }) { // The production site works differently. // TODO: make this a more sophisticated data editor. function JsonEntry({ name, data, readonly, onJsonEntryChange }) { - const jval = JSON.parse(data ?? "{}") ?? {}; - const dflt = FIELD_DEFAULTS[name]; - const quotas = { ...dflt, ...jval }; + const jsonValue = JSON.parse(data ?? "{}") ?? {}; + const quotas = { ...FIELD_DEFAULTS[name], ...jsonValue }; const value = JSON.stringify(quotas); return ( { - if (path[0] == "") { + if (!path[0]) { // Special case -- we allow passing "" for the name of the store and get out undefined. // This is useful when using the useRedux hook but when the name of the store isn't known initially. return undefined; diff --git a/src/packages/frontend/app/actions.ts b/src/packages/frontend/app/actions.ts index 1624d8053ae..22103bd2db6 100644 --- a/src/packages/frontend/app/actions.ts +++ b/src/packages/frontend/app/actions.ts @@ -17,6 +17,9 @@ import { disconnect_from_project } from "@cocalc/frontend/project/websocket/conn import { session_manager } from "@cocalc/frontend/session"; import { once } from "@cocalc/util/async-utils"; import { PageState } from "./store"; +import { lite, project_id } from "@cocalc/frontend/lite"; + +const LITE_TABS = new Set(["account", "admin"]); export class PageActions extends Actions { private session_manager?: any; @@ -140,7 +143,13 @@ export class PageActions extends Actions { disconnect_from_project(project_id); } - async set_active_tab(key, change_history = true): Promise { + set_active_tab = async (key, change_history = true): Promise => { + if (lite) { + if (!LITE_TABS.has(key)) { + key = project_id; + } + } + const prev_key = this.redux.getStore("page").get("active_top_tab"); this.setState({ active_top_tab: key }); @@ -228,7 +237,7 @@ export class PageActions extends Actions { set_window_title(""); } } - } + }; show_connection(show_connection) { this.setState({ show_connection }); diff --git a/src/packages/frontend/app/page.tsx b/src/packages/frontend/app/page.tsx index fbeda8d8719..3e5acdabbb8 100644 --- a/src/packages/frontend/app/page.tsx +++ b/src/packages/frontend/app/page.tsx @@ -12,7 +12,6 @@ declare var DEBUG: boolean; import { Spin } from "antd"; import { useIntl } from "react-intl"; - import { Avatar } from "@cocalc/frontend/account/avatar/avatar"; import { alert_message } from "@cocalc/frontend/alerts"; import { Button } from "@cocalc/frontend/antd-bootstrap"; @@ -52,6 +51,7 @@ import { HIDE_LABEL_THRESHOLD, NAV_CLASS } from "./top-nav-consts"; import { VerifyEmail } from "./verify-email-banner"; import VersionWarning from "./version-warning"; import { CookieWarning, LocalStorageWarning } from "./warnings"; +import { lite } from "@cocalc/frontend/lite"; // ipad and ios have a weird trick where they make the screen // actually smaller than 100vh and have it be scrollable, even @@ -111,7 +111,6 @@ export const Page: React.FC = () => { const fullscreen = useTypedRedux("page", "fullscreen"); const local_storage_warning = useTypedRedux("page", "local_storage_warning"); const cookie_warning = useTypedRedux("page", "cookie_warning"); - const accountIsReady = useTypedRedux("account", "is_ready"); const account_id = useTypedRedux("account", "account_id"); const is_logged_in = useTypedRedux("account", "is_logged_in"); @@ -119,7 +118,6 @@ export const Page: React.FC = () => { const when_account_created = useTypedRedux("account", "created"); const groups = useTypedRedux("account", "groups"); const show_i18n = useShowI18NBanner(); - const is_commercial = useTypedRedux("customize", "is_commercial"); const insecure_test_mode = useTypedRedux("customize", "insecure_test_mode"); @@ -213,7 +211,7 @@ export const Page: React.FC = () => { } function render_sign_in_tab(): React.JSX.Element | null { - if (is_logged_in || !showSignInTab) return null; + if (lite || is_logged_in || !showSignInTab) return null; return ( { {local_storage_warning && } {show_i18n && } - {!fullscreen && ( + {!lite && !fullscreen && ( )} {fullscreen && render_fullscreen()} - {isNarrow && ( + {!lite && isNarrow && ( )} diff --git a/src/packages/frontend/chat/chat-indicator.tsx b/src/packages/frontend/chat/chat-indicator.tsx index d1e60e67070..05481a4c4e4 100644 --- a/src/packages/frontend/chat/chat-indicator.tsx +++ b/src/packages/frontend/chat/chat-indicator.tsx @@ -20,6 +20,7 @@ import { HiddenXS } from "@cocalc/frontend/components"; import { Icon } from "@cocalc/frontend/components/icon"; import track from "@cocalc/frontend/user-tracking"; import { labels } from "../i18n"; +import { lite } from "@cocalc/frontend/lite"; export type ChatState = | "" // not opened (also undefined counts as not open) @@ -51,14 +52,15 @@ export function ChatIndicator({ project_id, path, chatState }: Props) { ...CHAT_INDICATOR_STYLE, ...{ display: "flex" }, }; - return (
- + {!lite && ( + + )}
); diff --git a/src/packages/frontend/chat/chat-log.tsx b/src/packages/frontend/chat/chat-log.tsx index e1f25a778bd..37f0d29cf10 100644 --- a/src/packages/frontend/chat/chat-log.tsx +++ b/src/packages/frontend/chat/chat-log.tsx @@ -13,11 +13,10 @@ import { Alert, Button } from "antd"; import { Set as immutableSet } from "immutable"; import { MutableRefObject, useEffect, useMemo, useRef } from "react"; import { Virtuoso, VirtuosoHandle } from "react-virtuoso"; - +import useVirtuosoScrollHook from "@cocalc/frontend/components/virtuoso-scroll-hook"; import { chatBotName, isChatBot } from "@cocalc/frontend/account/chatbot"; import { useRedux, useTypedRedux } from "@cocalc/frontend/app-framework"; import { Icon } from "@cocalc/frontend/components"; -import useVirtuosoScrollHook from "@cocalc/frontend/components/virtuoso-scroll-hook"; import { HashtagBar } from "@cocalc/frontend/editors/task-editor/hashtag-bar"; import { DivTempHeight } from "@cocalc/frontend/jupyter/cell-list"; import { diff --git a/src/packages/frontend/chat/register.ts b/src/packages/frontend/chat/register.ts index 7ff96d84ca7..c28991c1679 100644 --- a/src/packages/frontend/chat/register.ts +++ b/src/packages/frontend/chat/register.ts @@ -26,7 +26,7 @@ export function initChat(project_id: string, path: string): ChatActions { redux.getProjectActions(project_id)?.setNotDeleted(path); } - const syncdb = webapp_client.sync_client.sync_db({ + const syncdb = webapp_client.conat_client.conat().sync.db({ project_id, path, primary_keys: ["date", "sender_id", "event"], diff --git a/src/packages/frontend/client/client.ts b/src/packages/frontend/client/client.ts index 793bbbabd19..ab4be2f1389 100644 --- a/src/packages/frontend/client/client.ts +++ b/src/packages/frontend/client/client.ts @@ -35,6 +35,7 @@ import type { CreateConatServiceFunction, } from "@cocalc/conat/service"; import { randomId } from "@cocalc/conat/names"; +import api from "./api"; // This DEBUG variable comes from webpack: declare const DEBUG; @@ -91,12 +92,6 @@ export interface WebappClient extends EventEmitter { prettier: Function; exec: Function; touch_project: (project_id: string, compute_server_id?: number) => void; - ipywidgetsGetBuffer: ( - project_id: string, - path: string, - model_id: string, - buffer_path: string, - ) => Promise; log_error: (any) => void; user_tracking: Function; send: Function; @@ -114,6 +109,7 @@ export interface WebappClient extends EventEmitter { set_connected?: Function; version: Function; alert_message: Function; + nextjsApi?: typeof api; } export const WebappClient = null; // webpack + TS es2020 modules need this @@ -171,12 +167,6 @@ class Client extends EventEmitter implements WebappClient { prettier: Function; exec: Function; touch_project: (project_id: string, compute_server_id?: number) => void; - ipywidgetsGetBuffer: ( - project_id: string, - path: string, - model_id: string, - buffer_path: string, - ) => Promise; log_error: (any) => void; user_tracking: Function; @@ -194,6 +184,7 @@ class Client extends EventEmitter implements WebappClient { synctable_database: Function; async_query: Function; alert_message: Function; + nextjsApi = api; constructor() { super(); @@ -235,9 +226,6 @@ class Client extends EventEmitter implements WebappClient { this.touch_project = this.project_client.touch_project.bind( this.project_client, ); - this.ipywidgetsGetBuffer = this.project_client.ipywidgetsGetBuffer.bind( - this.project_client, - ); this.synctable_database = this.sync_client.synctable_database.bind( this.sync_client, @@ -317,26 +305,6 @@ class Client extends EventEmitter implements WebappClient { public set_deleted(): void { throw Error("not implemented for frontend"); } - - touchOpenFile = async ({ - project_id, - path, - setNotDeleted, - doctype, - }: { - project_id: string; - path: string; - id?: number; - doctype?; - // if file is deleted, this explicitly undeletes it. - setNotDeleted?: boolean; - }) => { - const x = await this.conat_client.openFiles(project_id); - if (setNotDeleted) { - x.setNotDeleted(path); - } - x.touch(path, doctype); - }; } export const webapp_client = new Client(); diff --git a/src/packages/frontend/client/idle.ts b/src/packages/frontend/client/idle.ts index 0992c9cec43..46c9fdb5959 100644 --- a/src/packages/frontend/client/idle.ts +++ b/src/packages/frontend/client/idle.ts @@ -10,6 +10,7 @@ import { redux } from "../app-framework"; import { IS_TOUCH } from "../feature"; import { WebappClient } from "./client"; import { disconnect_from_all_projects } from "../project/websocket/connect"; +import { lite } from "@cocalc/frontend/lite"; // set to true when there are no load issues. const NEVER_TIMEOUT_VISIBLE = false; @@ -40,7 +41,8 @@ export class IdleClient { // Do not bother on touch devices, since they already automatically tend to // disconnect themselves very aggressively to save battery life, and it's // sketchy trying to ensure that banner will dismiss properly. - if (IS_TOUCH) { + if (IS_TOUCH || lite) { + // never use idle timeout on touch devices (phones) or in lite mode return; } @@ -98,7 +100,7 @@ export class IdleClient { }; private idle_check = (): void => { - if (!this.idle_time) return; + if (!this.idle_time || lite) return; const remaining = this.idle_time - Date.now(); if (remaining > 0) { // console.log(`Standby in ${Math.round(remaining / 1000)}s if not active`); @@ -171,7 +173,7 @@ export class IdleClient { }; show_notification = (): void => { - if (this.notification_is_visible) return; + if (this.notification_is_visible || lite) return; const idle = $("#cocalc-idle-notification"); if (idle.length === 0) { const content = this.notification_html(); diff --git a/src/packages/frontend/client/project.ts b/src/packages/frontend/client/project.ts index d9aab297ca9..7550a796eb1 100644 --- a/src/packages/frontend/client/project.ts +++ b/src/packages/frontend/client/project.ts @@ -7,9 +7,7 @@ Functionality that mainly involves working with a specific project. */ -import { throttle } from "lodash"; import { join } from "path"; - import { readFile, type ReadFileOptions } from "@cocalc/conat/files/read"; import { writeFile, type WriteFileOptions } from "@cocalc/conat/files/write"; import { projectSubject, EXEC_STREAM_SERVICE } from "@cocalc/conat/names"; @@ -17,7 +15,6 @@ import { redux } from "@cocalc/frontend/app-framework"; import { appBasePath } from "@cocalc/frontend/customize/app-base-path"; import { dialogs } from "@cocalc/frontend/i18n"; import { getIntl } from "@cocalc/frontend/i18n/get-intl"; -import { allow_project_to_run } from "@cocalc/frontend/project/client-side-throttle"; import { ensure_project_running } from "@cocalc/frontend/project/project-start-warning"; import { API } from "@cocalc/frontend/project/websocket/api"; import { connection_to_project } from "@cocalc/frontend/project/websocket/connect"; @@ -44,18 +41,25 @@ import { import { reuseInFlight } from "@cocalc/util/reuse-in-flight"; import { DirectoryListingEntry } from "@cocalc/util/types"; import { WebappClient } from "./client"; +import { throttle } from "lodash"; +import { type ProjectApi } from "@cocalc/conat/project/api"; +import { type CopyOptions } from "@cocalc/conat/files/fs"; + +const TOUCH_THROTTLE = 30_000; import { ExecStream } from "./types"; export class ProjectClient { private client: WebappClient; - private touch_throttle: { [project_id: string]: number } = {}; constructor(client: WebappClient) { this.client = client; } - private conatApi = (project_id: string) => { - return this.client.conat_client.projectApi({ project_id }); + conatApi = (project_id: string, compute_server_id = 0): ProjectApi => { + return this.client.conat_client.projectApi({ + project_id, + compute_server_id, + }); }; // This can write small text files in one message. @@ -122,16 +126,10 @@ export class ProjectClient { return url; }; - copy_path_between_projects = async (opts: { - src_project_id: string; // id of source project - src_path: string; // relative path of director or file in the source project - target_project_id: string; // if of target project - target_path?: string; // defaults to src_path - overwrite_newer?: boolean; // overwrite newer versions of file at destination (destructive) - delete_missing?: boolean; // delete files in dest that are missing from source (destructive) - backup?: boolean; // make ~ backup files instead of overwriting changed files - timeout?: number; // **timeout in milliseconds** -- how long to wait for the copy to complete before reporting "error" (though it could still succeed) - exclude?: string[]; // list of patterns to exclude; this uses exactly the (confusing) rsync patterns + copyPathBetweenProjects = async (opts: { + src: { project_id: string; path: string | string[] }; + dest: { project_id: string; path: string }; + options?: CopyOptions; }): Promise => { await this.client.conat_client.hub.projects.copyPathBetweenProjects(opts); }; @@ -546,63 +544,43 @@ export class ProjectClient { }, ); - touch_project = async ( - // project_id where activity occurred - project_id: string, - // optional global id of a compute server (in the given project), in which case we also mark - // that compute server as active, which keeps it running in case it has idle timeout configured. - compute_server_id?: number, - ): Promise => { - if (!is_valid_uuid_string(project_id)) { - console.warn("WARNING -- touch_project takes a project_id, but got ", { - project_id, - }); - } - if (compute_server_id) { - // this is throttled, etc. and is independent of everything below. - touchComputeServer({ - project_id, - compute_server_id, - client: this.client, - }); - // that said, we do still touch the project, since if a user is actively - // using a compute server, the project should also be considered active. - } - - const state = redux.getStore("projects")?.get_state(project_id); - if (!(state == null && redux.getStore("account")?.get("is_admin"))) { - // not trying to view project as admin so do some checks - if (!(await allow_project_to_run(project_id))) return; - if (!this.client.is_signed_in()) { - // silently ignore if not signed in + touch_project = throttle( + async ( + // project_id where activity occured + project_id: string, + // optional global id of a compute server (in the given project), in which case we also mark + // that compute server as active, which keeps it running in case it has idle timeout configured. + compute_server_id?: number | null, + ): Promise => { + if (!is_valid_uuid_string(project_id)) { + console.warn("WARNING -- touch_project takes a project_id, but got ", { + project_id, + }); return; } - if (state != "running") { - // not running so don't touch (user must explicitly start first) - return; + if (compute_server_id) { + // this is independent of everything below. + touchComputeServer({ + project_id, + compute_server_id, + client: this.client, + }); + // that said, we do still touch the project, since if a user is actively + // using a compute server, the project should also be considered active. } - } - // Throttle -- so if this function is called with the same project_id - // twice in 3s, it's ignored (to avoid unnecessary network traffic). - // Do not make the timeout long, since that can mess up - // getting the hub-websocket to connect to the project. - const last = this.touch_throttle[project_id]; - if (last != null && Date.now() - last <= 3000) { - return; - } - this.touch_throttle[project_id] = Date.now(); - try { - await this.client.conat_client.hub.db.touch({ project_id }); - } catch (err) { - // silently ignore; this happens, e.g., if you touch too frequently, - // and shouldn't be fatal and break other things. - // NOTE: this is a bit ugly for now -- basically the - // hub returns an error regarding actually touching - // the project (updating the db), but it still *does* - // ensure there is a TCP connection to the project. - } - }; + try { + await this.client.conat_client.hub.db.touch({ project_id }); + } catch (err) { + // 503 would just mean the hub isn't listening yet, so expected + // sometimes. + if (err.code == 503) { + console.log("WARNING: issue touching project", err); + } + } + }, + TOUCH_THROTTLE, + ); // Print sagews to pdf // The printed version of the file will be created in the same directory @@ -628,11 +606,12 @@ export class ProjectClient { title: string; description: string; image?: string; + rootfs_image?: string; start?: boolean; // "license_id1,license_id2,..." -- if given, create project with these licenses applied license?: string; - // never use pool - noPool?: boolean; + // make exact clone of the files from this project: + src_project_id?: string; }): Promise => { const project_id = await this.client.conat_client.hub.projects.createProject(opts); @@ -650,13 +629,14 @@ export class ProjectClient { return (await this.api(opts.project_id)).realpath(opts.path); }; - isdir = async ({ + isDir = async ({ project_id, path, }: { project_id: string; path: string; }): Promise => { + // [ ] TODO: rewrite to use new fs.stat! const { stdout, exit_code } = await this.exec({ project_id, command: "file", @@ -666,21 +646,6 @@ export class ProjectClient { return !exit_code && stdout.trim() == "directory"; }; - ipywidgetsGetBuffer = reuseInFlight( - async ( - project_id: string, - path: string, - model_id: string, - buffer_path: string, - ): Promise => { - const actions = redux.getEditorActions(project_id, path); - return await actions.jupyter_actions.ipywidgetsGetBuffer( - model_id, - buffer_path, - ); - }, - ); - // getting, setting, editing, deleting, etc., the api keys for a project api_keys = async (opts: { project_id: string; diff --git a/src/packages/frontend/client/welcome-file.ts b/src/packages/frontend/client/welcome-file.ts index 4cf04b7685d..7f87037bc45 100644 --- a/src/packages/frontend/client/welcome-file.ts +++ b/src/packages/frontend/client/welcome-file.ts @@ -99,7 +99,7 @@ export class WelcomeFile { async open() { if (this.path == null) return; - await this.create_file(); + await this.createFile(); await this.extra_setup(); } @@ -147,20 +147,20 @@ export class WelcomeFile { if (cell.type == "markdown") { jactions.set_cell_type(cell_id, "markdown"); } else { - jactions.run_code_cell(cell_id); + jactions.runCells([cell_id]); } cell_id = jactions.insert_cell_adjacent(cell_id, +1); }); } - // Calling the "create file" action will properly initialize certain files, + // Calling the "createFile" action will properly initialize certain files, // in particular .tex - private async create_file(): Promise { + private async createFile(): Promise { if (this.path == null) - throw new Error("WelcomeFile::create_file – path is not defined"); + throw new Error("WelcomeFile::createFile – path is not defined"); const project_actions = redux.getProjectActions(this.project_id); const { name, ext } = separate_file_extension(this.path); - await project_actions.create_file({ + await project_actions.createFile({ name, ext, current_path: "", diff --git a/src/packages/frontend/codemirror/extensions/set-value-nojump.ts b/src/packages/frontend/codemirror/extensions/set-value-nojump.ts index 910993a0099..be78e201b55 100644 --- a/src/packages/frontend/codemirror/extensions/set-value-nojump.ts +++ b/src/packages/frontend/codemirror/extensions/set-value-nojump.ts @@ -4,7 +4,7 @@ */ import * as CodeMirror from "codemirror"; -import { diff_main } from "@cocalc/sync/editor/generic/util"; +import { diff_main } from "@cocalc/util/patch"; /* Try to set the value of the buffer to something new by replacing just the ranges diff --git a/src/packages/frontend/components/error-display.tsx b/src/packages/frontend/components/error-display.tsx index 377d02825f4..eae239d9d96 100644 --- a/src/packages/frontend/components/error-display.tsx +++ b/src/packages/frontend/components/error-display.tsx @@ -3,6 +3,8 @@ * License: MS-RSL – see LICENSE.md for details */ +// DEPRECATED -- the ShowError component in ./error.tsx is much better. + import { Alert } from "antd"; // use "style" to customize diff --git a/src/packages/frontend/components/error.tsx b/src/packages/frontend/components/error.tsx index f0841386a4c..bb53bff9dc8 100644 --- a/src/packages/frontend/components/error.tsx +++ b/src/packages/frontend/components/error.tsx @@ -8,6 +8,7 @@ interface Props { style?: CSSProperties; message?; banner?; + noMarkdown?: boolean; } export default function ShowError({ message = "Error", @@ -15,10 +16,10 @@ export default function ShowError({ setError, style, banner, + noMarkdown, }: Props) { if (!error) return null; - - const err = `${error}`.replace(/^Error:/, "").trim(); + const err = `${error}`.replace(/Error:/g, "").trim(); return ( - + {noMarkdown ? err : }
} onClose={() => setError?.("")} diff --git a/src/packages/frontend/components/fake-progress.tsx b/src/packages/frontend/components/fake-progress.tsx index d634e14ad5a..1b0f10ef854 100644 --- a/src/packages/frontend/components/fake-progress.tsx +++ b/src/packages/frontend/components/fake-progress.tsx @@ -23,7 +23,6 @@ export default function FakeProgress({ time }) { return ( null} percent={percent} strokeColor={{ "0%": "#108ee9", "100%": "#87d068" }} diff --git a/src/packages/frontend/components/loading.tsx b/src/packages/frontend/components/loading.tsx index 8e0dac07694..0abfe44ff6f 100644 --- a/src/packages/frontend/components/loading.tsx +++ b/src/packages/frontend/components/loading.tsx @@ -6,7 +6,8 @@ import { CSSProperties } from "react"; import { useIntl } from "react-intl"; import FakeProgress from "@cocalc/frontend/components/fake-progress"; -import { TypedMap, useDelayedRender } from "@cocalc/frontend/app-framework"; +import { TypedMap } from "@cocalc/frontend/app-framework"; +import useDelayedRender from "@cocalc/frontend/app-framework/delayed-render-hook"; import { labels } from "@cocalc/frontend/i18n"; import { Icon } from "./icon"; @@ -19,9 +20,9 @@ export const Estimate = null; // webpack + TS es2020 modules need this interface Props { style?: CSSProperties; text?: string; - estimate?: Estimate; + estimate?: Estimate | number; theme?: "medium" | undefined; - delay?: number; // if given, don't show anything until after delay milliseconds. The component could easily unmount by then, and hence never annoyingly flicker on screen. + delay?: number; // (default:1000) don't show anything until after delay milliseconds. The component could easily unmount by then, and hence never annoyingly flicker on screen. transparent?: boolean; } @@ -40,14 +41,13 @@ export function Loading({ text, estimate, theme, - delay, + delay = 1000, transparent = false, }: Props) { const intl = useIntl(); - const render = useDelayedRender(delay ?? 0); - if (!render) { - return <>; + if (!useDelayedRender(delay ?? 0)) { + return null; } return ( @@ -64,7 +64,13 @@ export function Loading({ {estimate != undefined && (
- +
)}
diff --git a/src/packages/frontend/components/progress-estimate.tsx b/src/packages/frontend/components/progress-estimate.tsx index 92a6ee9aea7..49c01e3231e 100644 --- a/src/packages/frontend/components/progress-estimate.tsx +++ b/src/packages/frontend/components/progress-estimate.tsx @@ -5,13 +5,15 @@ take. import { CSSProperties, useEffect, useState } from "react"; import { Progress } from "antd"; import useIsMountedRef from "@cocalc/frontend/app-framework/is-mounted-hook"; +import useDelayedRender from "@cocalc/frontend/app-framework/delayed-render-hook"; interface Props { seconds: number; style?: CSSProperties; + delay?: number; } -export default function ProgressEstimate({ seconds, style }: Props) { +export default function ProgressEstimate({ seconds, style, delay }: Props) { const isMountedRef = useIsMountedRef(); const [progress, setProgress] = useState(0); @@ -26,6 +28,10 @@ export default function ProgressEstimate({ seconds, style }: Props) { return () => clearInterval(interval); }, [progress, seconds]); + if (!useDelayedRender(delay ?? 0)) { + return null; + } + return ( void; + timeout?: number; } // definitely never show run buttons for text formats that can't possibly be run. @@ -81,8 +74,8 @@ export default function RunButton({ runRef, tag, size, - auto, setInfo, + timeout = 30_000, }: Props) { const mode = infoToMode(info); @@ -92,7 +85,6 @@ export default function RunButton({ project_id, path: filename, is_visible, - /*hasOpenAI, */ } = useFileContext(); const noRun = NO_RUN.has(mode) || disableMarkdownCodebar; const path = project_id && filename ? path_split(filename).head : undefined; @@ -134,89 +126,45 @@ export default function RunButton({ const [kernelName, setKernelName] = useState(undefined); const [showPopover, setShowPopover] = useState(false); + // determine the kernel useEffect(() => { if ( noRun || - !jupyterApiEnabled || + (!project_id && !jupyterApiEnabled) || setOutput == null || running || !info.trim() ) { return; } - const { output: messages, kernel: usedKernel } = getFromCache({ - input, - history, - info, - project_id, - path, - }); - if (!info) { - setKernelName(undefined); - } + setOutput({ old: true }); + (async () => { + let kernel; + try { + kernel = await getKernel({ input, history, info, project_id }); + } catch (err) { + // could fail, e.g., if user not signed in. shouldn't be fatal. + console.warn(`WARNING: ${err}`); + return; + } + setKernelName(kernel); + })(); + }, [input, history, info]); + + // set output based on local cache (so unmount/mount doesn't clear output) + useEffect(() => { + const messages = getFromCache({ input, history, info }); if (messages != null) { setOutput({ messages }); - setKernelName(usedKernel); - } else { - setOutput({ old: true }); - // but we try to asynchronously get the output from the - // backend, if available - (async () => { - let kernel; - try { - kernel = await getKernel({ input, history, info, project_id }); - } catch (err) { - // could fail, e.g., if user not signed in. shouldn't be fatal. - console.warn(`WARNING: ${err}`); - return; - } - setKernelName(kernel); - if (!auto && outputMessagesRef.current == null) { - // we don't initially automatically check database since auto is false. - return; - } - - const hash = computeHash({ - input, - history, - kernel, - project_id, - path, - }); - let x; - try { - x = await getFromDatabaseCache(hash); - } catch (err) { - console.warn(`WARNING: ${err}`); - return; - } - const { output: messages, created } = x; - if (messages != null) { - saveToCache({ - input, - history, - info, - output: messages, - project_id, - path, - kernel, - }); - setOutput({ messages }); - setCreated(created); - } - })(); } - }, [input, history, info]); + }, []); if (noRun || (!jupyterApiEnabled && !project_id)) { - // run button is not enabled when no project_id given, or not info at all. + // run button is not enabled when no project_id given, or no info at all. return null; } - const run = async ({ - noCache, - forceKernel, - }: { noCache?: boolean; forceKernel?: string } = {}) => { + const run = async ({ forceKernel }: { forceKernel?: string } = {}) => { try { setRunning(true); setOutput({ running: true }); @@ -236,42 +184,33 @@ export default function RunButton({ } setKernelName(kernel); } - let resp; + let messages; try { if (!kernel) { setOutput({ error: "Select a Kernel" }); return; } - resp = await api("execute", { + const opts = { input, history, kernel, - noCache, project_id, path, tag, - }); - } catch (err) { - if (resp?.error != null) { - setOutput({ error: resp.error }); + }; + if (project_id) { + const api = projectApiClient({ project_id, timeout }); + messages = await api.jupyter.apiExecute(opts); } else { - setOutput({ error: `Timeout or communication problem` }); + ({ output: messages } = await nextApi("execute", opts)); } + saveInCache({ input, history, info, messages }); + } catch (err) { + setOutput({ error: `${err}` }); return; } - if (resp.output != null) { - setOutput({ messages: resp.output }); - setCreated(resp.created); - saveToCache({ - input, - history, - info, - output: resp.output, - project_id, - path, - kernel, - }); - } + setOutput({ messages }); + setCreated(new Date()); } catch (error) { setOutput({ error }); } finally { @@ -299,7 +238,7 @@ export default function RunButton({ disabled={disabled || !kernelName} onClick={() => { setShowPopover(false); - run({ noCache: false }); + run(); }} > { setShowPopover(false); - run({ noCache: true }); + run(); }} > - {/*hasOpenAI && ( - - )*/} ); } -type GetFromCache = (hash: string) => Promise<{ - output?: object[]; - created?: Date; -}>; +const outputCache = new LRU({ max: 200 }); -const getFromDatabaseCache: GetFromCache = reuseInFlight( - async (hash) => await api("execute", { hash }), -); +function cacheKey({ input, history, info }) { + return JSON.stringify([input, history, info]); +} + +function saveInCache({ input, history, info, messages }) { + outputCache.set(cacheKey({ input, history, info }), messages); +} + +function getFromCache({ input, history, info }) { + return outputCache.get(cacheKey({ input, history, info })); +} diff --git a/src/packages/frontend/components/run-button/kernel-info.ts b/src/packages/frontend/components/run-button/kernel-info.ts index b3b9b3cff00..046993ab90a 100644 --- a/src/packages/frontend/components/run-button/kernel-info.ts +++ b/src/packages/frontend/components/run-button/kernel-info.ts @@ -4,10 +4,10 @@ */ import LRU from "lru-cache"; - import type { KernelSpec } from "@cocalc/jupyter/types"; import { capitalize } from "@cocalc/util/misc"; -import api from "./api"; +import { projectApiClient } from "@cocalc/conat/project/api"; +import nextApi from "./api"; const kernelInfoCache = new LRU({ ttl: 30000, @@ -44,10 +44,14 @@ export async function getKernelInfo( throw new Error("No information, because project is not running"); } - const { kernels } = await api( - "kernels", - project_id ? { project_id } : undefined, - ); + let kernels; + if (project_id) { + // TODO: compute server support -- would select here + const api = projectApiClient({ project_id }); + kernels = await api.jupyter.kernels(); + } else { + ({ kernels } = await nextApi("kernels")); + } if (kernels == null) { throw Error("bug"); } diff --git a/src/packages/frontend/components/run-button/output.tsx b/src/packages/frontend/components/run-button/output.tsx index 14760018525..67ae96f7e3f 100644 --- a/src/packages/frontend/components/run-button/output.tsx +++ b/src/packages/frontend/components/run-button/output.tsx @@ -31,7 +31,9 @@ export default function Output({ } return (
- {running && } + {running && ( + + )} {output != null && (
diff --git a/src/packages/frontend/components/run-button/select-kernel.tsx b/src/packages/frontend/components/run-button/select-kernel.tsx index c107bbef9dd..47bb6658733 100644 --- a/src/packages/frontend/components/run-button/select-kernel.tsx +++ b/src/packages/frontend/components/run-button/select-kernel.tsx @@ -8,7 +8,6 @@ import { OptionProps } from "antd/es/select"; import { fromJS } from "immutable"; import { sortBy } from "lodash"; import { useEffect, useState } from "react"; - import Logo from "@cocalc/frontend/jupyter/logo"; import type { KernelSpec } from "@cocalc/jupyter/types"; import { diff --git a/src/packages/frontend/components/search-input.tsx b/src/packages/frontend/components/search-input.tsx index e3839ca547b..88d017315a0 100644 --- a/src/packages/frontend/components/search-input.tsx +++ b/src/packages/frontend/components/search-input.tsx @@ -10,13 +10,7 @@ */ import { Input, InputRef } from "antd"; - -import { - React, - useEffect, - useRef, - useState, -} from "@cocalc/frontend/app-framework"; +import { CSSProperties, useEffect, useRef, useState } from "react"; interface Props { size?; @@ -31,21 +25,39 @@ interface Props { on_down?: () => void; on_up?: () => void; on_escape?: (value: string) => void; - style?: React.CSSProperties; - input_class?: string; + style?: CSSProperties; autoFocus?: boolean; autoSelect?: boolean; placeholder?: string; - focus?: number; // if this changes, focus the search box. + focus?; // if this changes, focus the search box. status?: "warning" | "error"; } -export const SearchInput: React.FC = React.memo((props) => { - const [value, setValue] = useState( - props.value ?? props.default_value ?? "", - ); +export function SearchInput({ + size, + default_value, + value: value0, + on_change, + on_clear, + on_submit, + buttonAfter, + disabled, + clear_on_submit, + on_down, + on_up, + on_escape, + style, + autoFocus, + autoSelect, + placeholder, + focus, + status, +}: Props) { + const [value, setValue] = useState(value0 ?? default_value ?? ""); // if value changes, we update as well! - useEffect(() => setValue(props.value ?? ""), [props.value]); + useEffect(() => { + setValue(value0 ?? ""); + }, [value0]); const [ctrl_down, set_ctrl_down] = useState(false); const [shift_down, set_shift_down] = useState(false); @@ -53,7 +65,7 @@ export const SearchInput: React.FC = React.memo((props) => { const input_ref = useRef(null); useEffect(() => { - if (props.autoSelect && input_ref.current) { + if (autoSelect && input_ref.current) { try { input_ref.current?.select(); } catch (_) {} @@ -71,20 +83,20 @@ export const SearchInput: React.FC = React.memo((props) => { function clear_value(): void { setValue(""); - props.on_change?.("", get_opts()); - props.on_clear?.(); + on_change?.("", get_opts()); + on_clear?.(); } function submit(e?): void { if (e != null) { e.preventDefault(); } - if (typeof props.on_submit === "function") { - props.on_submit(value, get_opts()); + if (typeof on_submit === "function") { + on_submit(value, get_opts()); } - if (props.clear_on_submit) { + if (clear_on_submit) { clear_value(); - props.on_change?.(value, get_opts()); + on_change?.(value, get_opts()); } } @@ -94,10 +106,10 @@ export const SearchInput: React.FC = React.memo((props) => { escape(); break; case 40: - props.on_down?.(); + on_down?.(); break; case 38: - props.on_up?.(); + on_up?.(); break; case 17: set_ctrl_down(true); @@ -120,35 +132,35 @@ export const SearchInput: React.FC = React.memo((props) => { } function escape(): void { - if (typeof props.on_escape === "function") { - props.on_escape(value); + if (typeof on_escape === "function") { + on_escape(value); } clear_value(); } return ( { e.preventDefault(); const value = e.target?.value ?? ""; setValue(value); - props.on_change?.(value, get_opts()); + on_change?.(value, get_opts()); if (!value) clear_value(); }} onKeyDown={key_down} onKeyUp={key_up} - disabled={props.disabled} - enterButton={props.buttonAfter} - status={props.status} + disabled={disabled} + enterButton={buttonAfter} + status={status} /> ); -}); +} diff --git a/src/packages/frontend/components/setting-box.tsx b/src/packages/frontend/components/setting-box.tsx index 63563d7725e..8c7c7b33c7f 100644 --- a/src/packages/frontend/components/setting-box.tsx +++ b/src/packages/frontend/components/setting-box.tsx @@ -41,8 +41,9 @@ export function SettingBox({ title={ show_header ? (
- - {icon && <Icon name={icon} />} {title} + <Title level={4} style={{ display: "flex" }}> + {icon && <Icon name={icon} style={{ marginRight: "5px" }} />} +  {title} {subtitle} {/* subtitle must be outside of the Typography.Title -- this is assumed, e.g., in frontend/project/new/project-new-form.tsx */} diff --git a/src/packages/frontend/components/terminal.tsx b/src/packages/frontend/components/terminal.tsx new file mode 100644 index 00000000000..674d7229b06 --- /dev/null +++ b/src/packages/frontend/components/terminal.tsx @@ -0,0 +1,90 @@ +/* +Easy React Terminal display output. + +TODO: NOT FINISHED/USED YET! +*/ + +import { useEffect, useRef } from "react"; +import { Terminal as Terminal0 } from "@xterm/xterm"; +import { useTypedRedux } from "@cocalc/frontend/app-framework"; +import { setTheme } from "@cocalc/frontend/frame-editors/terminal-editor/themes"; +import { WebLinksAddon } from "@xterm/addon-web-links"; +import { handleLink } from "@cocalc/frontend/frame-editors/terminal-editor/connected-terminal"; + +const WIDTH = 160; +const HEIGHT = 40; + +interface Options { + value?: string; + scrollback?: number; + width?: number; + height?: number; + style?; + scrollToBottom?: boolean; +} + +export default function Terminal({ + value = "", + scrollback = 1_000_000, + width = WIDTH, + height = HEIGHT, + style, + scrollToBottom = true, +}: Options) { + const { font_size, font, color_scheme } = useTypedRedux( + "account", + "terminal", + )?.toJS() ?? { font: "monospace", font_size: 13, color_scheme: "default" }; + const eltRef = useRef(null); + const termRef = useRef(null); + + useEffect(() => { + const elt = eltRef.current; + if (elt == null) { + return; + } + const term = new Terminal0({ + fontSize: font_size, + fontFamily: font, + scrollback, + }); + term.loadAddon(new WebLinksAddon(handleLink)); + //term.loadAddon(new FitAddon()); + termRef.current = term; + setTheme(term, color_scheme); + term.resize(width, height); + term.open(elt); + term.write(value); + if (scrollToBottom) { + term.scrollToBottom(); + } + + return () => { + term.dispose(); + }; + }, [scrollback, width, height, font_size, font, color_scheme]); + + useEffect(() => { + termRef.current?.write(value); + if (scrollToBottom) { + termRef.current?.scrollToBottom(); + } + }, [value, scrollToBottom]); + + return ( +
+ 

+    

+  );
+}
diff --git a/src/packages/frontend/components/time-ago.tsx b/src/packages/frontend/components/time-ago.tsx
index 7259a5a75a9..00352b38d2a 100644
--- a/src/packages/frontend/components/time-ago.tsx
+++ b/src/packages/frontend/components/time-ago.tsx
@@ -166,6 +166,9 @@ export const TimeAgoElement: React.FC = ({
   }
 
   const d = is_date(date) ? (date as Date) : new Date(date);
+  if (!d.valueOf()) {
+    return null;
+  }
   try {
     d.toISOString();
   } catch (error) {
@@ -203,7 +206,7 @@ export const TimeAgo: React.FC = React.memo(
   }: TimeAgoElementProps) => {
     const { timeAgoAbsolute } = useAppContext();
 
-    if (date == null) {
+    if (!date?.valueOf()) {
       return <>;
     }
 
diff --git a/src/packages/frontend/components/virtuoso-scroll-hook.ts b/src/packages/frontend/components/virtuoso-scroll-hook.ts
index fab8ed41c92..43a20506235 100644
--- a/src/packages/frontend/components/virtuoso-scroll-hook.ts
+++ b/src/packages/frontend/components/virtuoso-scroll-hook.ts
@@ -15,6 +15,8 @@ the upstream Virtuoso project:  https://github.com/petyosi/react-virtuoso/blob/m
 import LRU from "lru-cache";
 import { useCallback, useMemo, useRef } from "react";
 
+const DEFAULT_VIEWPORT = 1000;
+
 export interface ScrollState {
   index: number;
   offset: number;
@@ -46,7 +48,7 @@ export default function useVirtuosoScrollHook({
   }, []);
   if (disabled) return {};
   const lastScrollRef = useRef(
-    initialState ?? { index: 0, offset: 0 }
+    initialState ?? { index: 0, offset: 0 },
   );
   const recordScrollState = useMemo(() => {
     return (state: ScrollState) => {
@@ -64,8 +66,9 @@ export default function useVirtuosoScrollHook({
   }, [onScroll, cacheId]);
 
   return {
+    increaseViewportBy: DEFAULT_VIEWPORT,
     initialTopMostItemIndex:
-      (cacheId ? cache.get(cacheId) ?? initialState : initialState) ?? 0,
+      (cacheId ? (cache.get(cacheId) ?? initialState) : initialState) ?? 0,
     scrollerRef: handleScrollerRef,
     onScroll: () => {
       const scrollTop = scrollerRef.current?.scrollTop;
diff --git a/src/packages/frontend/compute/action.tsx b/src/packages/frontend/compute/action.tsx
index a7e552f75c7..2d648734b69 100644
--- a/src/packages/frontend/compute/action.tsx
+++ b/src/packages/frontend/compute/action.tsx
@@ -51,7 +51,7 @@ export default function getActions({
     const a = ACTION_INFO[action];
     if (!a) continue;
     if (action == "suspend") {
-      if (configuration.cloud != "google-cloud") {
+      if (configuration?.cloud != "google-cloud") {
         continue;
       }
       if (getArchitecture(configuration) == "arm64") {
@@ -61,12 +61,12 @@ export default function getActions({
         continue;
       }
       // must have no gpu and <= 208GB of RAM -- https://cloud.google.com/compute/docs/instances/suspend-resume-instance
-      if (configuration.acceleratorType) {
+      if (configuration?.acceleratorType) {
         continue;
       }
       // [ ] TODO: we don't have an easy way to check the RAM requirement right now.
     }
-    if (!editModal && configuration.ephemeral && action == "stop") {
+    if (!editModal && configuration?.ephemeral && action == "stop") {
       continue;
     }
     const {
@@ -79,10 +79,10 @@ export default function getActions({
       confirmMessage,
       clouds,
     } = a;
-    if (danger && !configuration.ephemeral && !editModal) {
+    if (danger && !configuration?.ephemeral && !editModal) {
       continue;
     }
-    if (clouds && !clouds.includes(configuration.cloud)) {
+    if (clouds && !clouds.includes(configuration?.cloud)) {
       continue;
     }
     v.push(
@@ -179,7 +179,7 @@ function ActionButton({
       }
     }
 
-    if (configuration.cloud == "onprem") {
+    if (configuration?.cloud == "onprem") {
       if (action == "start") {
         setShowOnPremStart(true);
       } else if (action == "stop") {
@@ -231,10 +231,10 @@ function ActionButton({
     >
        {label}{" "}
       {doing && (
-        <>
+        
           

           
-        
+        
       )}
     
   );
@@ -244,7 +244,7 @@ function ActionButton({
         onOpenChange={setPopConfirm}
         placement="right"
         okButtonProps={{
-          disabled: !configuration.ephemeral && danger && !understand,
+          disabled: !configuration?.ephemeral && danger && !understand,
         }}
         title={
           

@@ -269,7 +269,7 @@ function ActionButton({
                 }`}
               />
             )}
-            {!configuration.ephemeral && danger && (
+            {!configuration?.ephemeral && danger && (
               

                 {/* ATTN: Not using a checkbox here to WORKAROUND A BUG IN CHROME that I see after a day or so! */}
                 

             }
           />
-          {configuration.gpu && (
+          {configuration?.gpu && (
             
               Since you clicked GPU, you must also have an NVIDIA GPU and the
               Cuda drivers installed and working.{" "}
diff --git a/src/packages/frontend/compute/api.ts b/src/packages/frontend/compute/api.ts
index f22340431ac..f12a03c8845 100644
--- a/src/packages/frontend/compute/api.ts
+++ b/src/packages/frontend/compute/api.ts
@@ -1,7 +1,4 @@
-import api from "@cocalc/frontend/client/api";
 import type {
-  Action,
-  Cloud,
   ComputeServerTemplate,
   ComputeServerUserInfo,
   Configuration,
@@ -16,86 +13,68 @@ import type {
 } from "@cocalc/util/compute/templates";
 import { reuseInFlight } from "@cocalc/util/reuse-in-flight";
 import TTL from "@isaacs/ttlcache";
+import { webapp_client } from "@cocalc/frontend/webapp-client";
+import { type Compute } from "@cocalc/conat/hub/api/compute";
 
-export async function createServer(opts: {
-  project_id: string;
-  title?: string;
-  color?: string;
-  autorestart?: boolean;
-  cloud?: Cloud;
-  configuration?: Configuration;
-  notes?: string;
-  course_project_id?: string;
-  course_server_id?: number;
-}): Promise {
-  return await api("compute/create-server", opts);
+function compute(): Compute {
+  return webapp_client.conat_client.hub.compute;
 }
 
-export async function computeServerAction(opts: {
-  id: number;
-  action: Action;
-}) {
-  await api("compute/compute-server-action", opts);
+export async function createServer(opts): Promise {
+  return await compute().createServer(opts);
 }
 
-// Get servers across potentially different projects by their global unique id.
-// Use the fields parameter to restrict to a much smaller subset of information
-// about each server (e.g., just the state field).  Caller must be a collaborator
-// on each project containing the servers.
-// If you give an id of a server that doesn't exist, it'll just be excluded in the result.
-// Similarly, if you give a field that doesn't exist, it is excluded.
-// The order of the returned servers and count probably will NOT match that in
-// ids, so you should include 'id' in fields.
-export async function getServersById(opts: {
-  ids: number[];
-  fields?: string[];
-}): Promise[]> {
-  return await api("compute/get-servers-by-id", opts);
+export async function computeServerAction(opts): Promise {
+  await compute().computeServerAction(opts);
 }
 
-export async function getServers(opts: {
-  id?: number;
-  project_id: string;
-}): Promise {
-  return await api("compute/get-servers", opts);
+export async function getServersById(
+  opts,
+): Promise[]> {
+  return await compute().getServersById(opts);
 }
 
-export async function getServerState(id: number) {
-  return await api("compute/get-server-state", { id });
+export async function getServers(opts): Promise {
+  return await compute().getServers(opts);
 }
 
-export async function getSerialPortOutput(id: number) {
-  return await api("compute/get-serial-port-output", { id });
+export async function getServerState(
+  id: number,
+): Promise {
+  return await compute().getServerState({ id });
+}
+
+export async function getSerialPortOutput(id: number): Promise {
+  return await compute().getSerialPortOutput({ id });
 }
 
 export async function deleteServer(id: number) {
-  return await api("compute/delete-server", { id });
+  return await compute().deleteServer({ id });
 }
 
 export async function isDnsAvailable(dns: string) {
-  const { isAvailable } = await api("compute/is-dns-available", { dns });
-  return isAvailable;
+  return await compute().isDnsAvailable({ dns });
 }
 
 export async function undeleteServer(id: number) {
-  await api("compute/undelete-server", { id });
+  return await compute().undeleteServer({ id });
 }
 
 // only owner can change properties of a compute server.
 
 export async function setServerColor(opts: { id: number; color: string }) {
-  return await api("compute/set-server-color", opts);
+  await compute().setServerColor(opts);
 }
 
 export async function setServerTitle(opts: { id: number; title: string }) {
-  return await api("compute/set-server-title", opts);
+  await compute().setServerTitle(opts);
 }
 
 export async function setServerConfiguration(opts: {
   id: number;
   configuration: Partial;
 }) {
-  return await api("compute/set-server-configuration", opts);
+  await compute().setServerConfiguration(opts);
 }
 
 // only for admins!
@@ -103,7 +82,7 @@ export async function setTemplate(opts: {
   id: number;
   template: ComputeServerTemplate;
 }) {
-  return await api("compute/set-template", opts);
+  await compute().setTemplate(opts);
 }
 
 // 5-minute client side ttl cache of all and specific template, since
@@ -115,7 +94,7 @@ export async function getTemplate(id: number): Promise {
   if (templatesCache.has(id)) {
     return templatesCache.get(id)!;
   }
-  const x = await api("compute/get-template", { id });
+  const x = await compute().getTemplate({ id });
   templatesCache.set(id, x);
   return x;
 }
@@ -124,20 +103,20 @@ export async function getTemplates(): Promise {
   if (templatesCache.has("templates")) {
     return templatesCache.get("templates")!;
   }
-  const x = await api("compute/get-templates");
+  const x = await compute().getTemplates();
   templatesCache.set("templates", x);
   return x;
 }
 
 export async function setServerCloud(opts: { id: number; cloud: string }) {
-  return await api("compute/set-server-cloud", opts);
+  await compute().setServerCloud(opts);
 }
 
 export async function setServerOwner(opts: {
   id: number;
   new_account_id: string;
 }) {
-  return await api("compute/set-server-owner", opts);
+  await compute().setServerOwner(opts);
 }
 
 // Cache for 12 hours
@@ -149,7 +128,7 @@ export const getGoogleCloudPriceData = reuseInFlight(
       googleCloudPriceData == null ||
       Date.now() >= googleCloudPriceDataExpire
     ) {
-      googleCloudPriceData = await api("compute/google-cloud/get-pricing-data");
+      googleCloudPriceData = await compute().getGoogleCloudPriceData();
       googleCloudPriceDataExpire = Date.now() + 1000 * 60 * 60 * 12; // 12 hour cache
     }
     if (googleCloudPriceData == null) {
@@ -186,7 +165,7 @@ export const getHyperstackPriceData = reuseInFlight(
       hyperstackPriceData == null ||
       Date.now() >= hyperstackPriceDataExpire
     ) {
-      hyperstackPriceData = await api("compute/get-hyperstack-pricing-data");
+      hyperstackPriceData = await compute().getHyperstackPriceData();
       hyperstackPriceDataExpire = Date.now() + 1000 * 60 * 5; // 5 minute cache
     }
     if (hyperstackPriceData == null) {
@@ -203,22 +182,22 @@ export async function getNetworkUsage(opts: {
   start: Date;
   end: Date;
 }): Promise<{ amount: number; cost: number }> {
-  return await api("compute/get-network-usage", opts);
+  return await compute().getNetworkUsage(opts);
 }
 
 // Get the current api key for a specific (on prem) server.
 // We only need this for on prem, so we are restricting to that right now.
 // If no key is allocated, one will be created.
 export async function getApiKey(opts: { id }): Promise {
-  return await api("compute/get-api-key", opts);
+  return await compute().getApiKey(opts);
 }
-export async function deleteApiKey(opts: { id }): Promise {
-  return await api("compute/delete-api-key", opts);
+export async function deleteApiKey(opts: { id }): Promise {
+  await compute().deleteApiKey(opts);
 }
 
 // Get the project log entries directly for just one compute server
 export async function getLog(opts: { id; type: "activity" | "files" }) {
-  return await api("compute/get-log", opts);
+  return await compute().getLog(opts);
 }
 
 export const getTitle = reuseInFlight(
@@ -229,7 +208,7 @@ export const getTitle = reuseInFlight(
     color: string;
     project_specific_id: number;
   }> => {
-    return await api("compute/get-server-title", opts);
+    return await compute().getTitle(opts);
   },
 );
 
@@ -243,7 +222,7 @@ export async function setDetailedState(opts: {
   timeout?: number;
   progress?: number;
 }) {
-  return await api("compute/set-detailed-state", opts);
+  await compute().setDetailedState(opts);
 }
 
 // We cache images for 5 minutes.
@@ -271,7 +250,6 @@ function cacheGet(cloud) {
 function cacheSet(cloud, images) {
   imagesCache[cloud] = { images, timestamp: Date.now() };
 }
-
 async function getImagesFor({
   cloud,
   endpoint,
@@ -286,11 +264,14 @@ async function getImagesFor({
   }
 
   try {
-    const images = await api(
-      endpoint,
-      // admin reload forces fetch data from github and/or google cloud - normal users just have their cache ignored above
-      reload ? { noCache: true } : undefined,
-    );
+    let images;
+    if (endpoint == "compute/get-images") {
+      images = await compute().getImages({ noCache: !!reload });
+    } else if (endpoint == "compute/get-images-google") {
+      images = await compute().getGoogleCloudImages({ noCache: !!reload });
+    } else {
+      throw Error(`unknown endpoint ${endpoint}`);
+    }
     cacheSet(cloud, images);
     return images;
   } catch (err) {
@@ -328,5 +309,5 @@ export async function setImageTested(opts: {
   id: number; // server id
   tested: boolean;
 }) {
-  return await api("compute/set-image-tested", opts);
+  await compute().setImageTested(opts);
 }
diff --git a/src/packages/frontend/compute/clone.tsx b/src/packages/frontend/compute/clone.tsx
index 6c7c2815260..c2c5dcc6961 100644
--- a/src/packages/frontend/compute/clone.tsx
+++ b/src/packages/frontend/compute/clone.tsx
@@ -78,7 +78,7 @@ export async function cloneConfiguration({
   const server = servers[0] as ComputeServerUserInfo;
   if (!noChange) {
     let n = 1;
-    let title = `Clone of ${server.title}`;
+    let title = `Fork of ${server.title}`;
     const titles = new Set(servers.map((x) => x.title));
     if (titles.has(title)) {
       while (titles.has(title + ` (${n})`)) {
diff --git a/src/packages/frontend/compute/configuration.tsx b/src/packages/frontend/compute/configuration.tsx
index bf9b9ac1d98..631231939e8 100644
--- a/src/packages/frontend/compute/configuration.tsx
+++ b/src/packages/frontend/compute/configuration.tsx
@@ -131,11 +131,9 @@ function Config({
         template={template}
       />
     );
+  } else if (configuration == null) {
+    return Not Configured;
   } else {
-    return (
-      
-        Configuration not implemented: {JSON.stringify(configuration)}
-      
-    );
+    return Unknown Cloud: '{JSON.stringify(configuration?.cloud)}';
   }
 }
diff --git a/src/packages/frontend/compute/doc-status.tsx b/src/packages/frontend/compute/doc-status.tsx
index e0da1d5b55b..e2c911fbea0 100644
--- a/src/packages/frontend/compute/doc-status.tsx
+++ b/src/packages/frontend/compute/doc-status.tsx
@@ -7,13 +7,6 @@ server.  It does the following:
 
 - If id is as requested and is not the project, draw line in color of that compute server.
 
-- If not where we want to be, defines how close via a percentage
-
-- If compute server not running:
-    - if exists and you own it, prompts user to start it and also shows the
-      compute server's component so they can do so.
-    - if not exists (or deleted), say so
-    - if owned by somebody else, say so
 */
 
 import Inline from "./inline";
@@ -29,6 +22,7 @@ import { avatar_fontcolor } from "@cocalc/frontend/account/avatar/font-color";
 import { DisplayImage } from "./select-image";
 import Menu from "./menu";
 import { SpendLimitStatus } from "./spend-limit";
+import useComputeServerApiState from "./use-compute-server-api-state";
 
 interface Props {
   project_id: string;
@@ -48,6 +42,11 @@ export function ComputeServerDocStatus({
   if (requestedId == null) {
     requestedId = id;
   }
+  const apiState = useComputeServerApiState({
+    project_id,
+    compute_server_id: requestedId,
+  });
+
   const [showDetails, setShowDetails] = useState(null);
   const computeServers = useTypedRedux({ project_id }, "compute_servers");
   const account_id = useTypedRedux("account", "account_id");
@@ -210,11 +209,15 @@ export function ComputeServerDocStatus({
   const { progress, message, status } = getProgress(
     server,
     account_id,
-    id,
     requestedId,
+    apiState,
   );
   if (!showDetails) {
-    if (showDetails == null && progress < 100) {
+    if (
+      showDetails == null &&
+      progress < 100 &&
+      (apiState != null || status == "exception")
+    ) {
       setShowDetails(true);
     }
     return topBar(progress);
@@ -283,12 +286,14 @@ export function ComputeServerDocStatus({
   );
 }
 
-// gets progress of starting the compute server with given id and having it actively available to host this file.
+// gets progress of starting the compute server with given id and
+// having it actively available to host this file.
+
 function getProgress(
   server: ComputeServerUserInfo | undefined,
   account_id,
-  id,
   requestedId,
+  apiState,
 ): {
   progress: number;
   message: string;
@@ -301,6 +306,15 @@ function getProgress(
       status: "active",
     };
   }
+
+  if (apiState == "running") {
+    return {
+      progress: 100,
+      message: "Compute server is fully connected!",
+      status: "success",
+    };
+  }
+
   if (server == null) {
     return {
       progress: 0,
@@ -320,17 +334,26 @@ function getProgress(
   if (
     server.account_id != account_id &&
     server.state != "running" &&
-    server.state != "starting"
+    server.state != "starting" &&
+    !server.configuration?.allowCollaboratorControl
   ) {
     return {
       progress: 0,
       message:
-        "This is not your compute server, and it is not running. Only the owner of a compute server can start it.",
+        "This is not your compute server, and it is not running. Only the owner of this compute server can start it.",
       status: "exception",
     };
   }
 
-  // below here it isn't our server, it is running.
+  if (apiState != null) {
+    if (apiState == "starting") {
+      return {
+        progress: 75,
+        message: "Compute server is starting.",
+        status: "active",
+      };
+    }
+  }
 
   if (server.state == "deprovisioned") {
     return {
@@ -371,63 +394,20 @@ function getProgress(
     };
   }
 
-  // below it is running
-
-  const computeIsLive = server.detailed_state?.compute?.state == "ready";
-  if (computeIsLive) {
-    if (id == requestedId) {
-      return {
-        progress: 100,
-        message: "Compute server is fully connected!",
-        status: "success",
-      };
-    } else {
-      return {
-        progress: 90,
-        message:
-          "Compute server is connected and should attach to this file soon...",
-        status: "success",
-      };
-    }
-  }
-  const filesystemIsLive =
-    server.detailed_state?.["filesystem-sync"]?.state == "ready";
-  const computeIsRecent = isRecent(server.detailed_state?.compute?.time);
-  const filesystemIsRecent = isRecent(
-    server.detailed_state?.["filesystem-sync"]?.time,
-  );
-  if (filesystemIsRecent) {
-    return {
-      progress: 70,
-      message: "Waiting for filesystem to connect.",
-      status: "normal",
-    };
-  }
-  if (filesystemIsLive) {
-    if (computeIsRecent) {
-      return {
-        progress: 80,
-        message: "Waiting for compute to connect.",
-        status: "normal",
-      };
-    }
-  }
-
   return {
     progress: 50,
-    message:
-      "Compute server is running, but filesystem and compute components aren't connected. Waiting...",
+    message: "Compute server is starting...",
     status: "active",
   };
 }
 
 // This is useful elsewhere to give a sense of how the compute server
 // is doing as it progresses from running to really being fully available.
-function getRunningStatus(server) {
+function getRunningStatus(server, apiState) {
   if (server == null) {
     return { progress: 0, message: "Loading...", status: "exception" };
   }
-  return getProgress(server, webapp_client.account_id, server.id, server.id);
+  return getProgress(server, webapp_client.account_id, server.id, apiState);
 }
 
 export function RunningProgress({
@@ -437,8 +417,12 @@ export function RunningProgress({
   server: ComputeServerUserInfo | undefined;
   style?;
 }) {
+  const apiState = useComputeServerApiState({
+    project_id: server?.project_id,
+    compute_server_id: server?.id,
+  });
   const { progress, message } = useMemo(() => {
-    return getRunningStatus(server);
+    return getRunningStatus(server, apiState);
   }, [server]);
 
   return (
@@ -452,7 +436,3 @@ export function RunningProgress({
     
   );
 }
-
-function isRecent(expire = 0) {
-  return Date.now() - expire < 60 * 1000;
-}
diff --git a/src/packages/frontend/compute/public-templates.tsx b/src/packages/frontend/compute/public-templates.tsx
index 9f1adb79ea8..259bda4f0b1 100644
--- a/src/packages/frontend/compute/public-templates.tsx
+++ b/src/packages/frontend/compute/public-templates.tsx
@@ -1,6 +1,7 @@
 import { Select, Spin, Tag, Tooltip } from "antd";
 import { useEffect, useState } from "react";
-import { getTemplates } from "@cocalc/frontend/compute/api";
+import api from "@cocalc/frontend/client/api";
+import TTL from "@isaacs/ttlcache";
 import type { ConfigurationTemplate } from "@cocalc/util/compute/templates";
 import type { HyperstackConfiguration } from "@cocalc/util/db-schema/compute-servers";
 import { CLOUDS_BY_NAME } from "@cocalc/util/compute/cloud/clouds";
@@ -13,6 +14,17 @@ import { filterOption } from "@cocalc/frontend/compute/util";
 import DisplayCloud from "./display-cloud";
 import { Icon } from "@cocalc/frontend/components/icon";
 
+const templatesCache = new TTL({ ttl: 60 * 1000 * 15 });
+export async function getTemplates() {
+  if (templatesCache.has("templates")) {
+    return templatesCache.get("templates")!;
+  }
+  // use nextjs api instead of conat, since this component is used on the landing page.
+  const x = await api("compute/get-templates");
+  templatesCache.set("templates", x);
+  return x;
+}
+
 const { CheckableTag } = Tag;
 
 const TAGS = {
diff --git a/src/packages/frontend/compute/select-server.tsx b/src/packages/frontend/compute/select-server.tsx
index ffb7d09981f..bf271898446 100644
--- a/src/packages/frontend/compute/select-server.tsx
+++ b/src/packages/frontend/compute/select-server.tsx
@@ -300,8 +300,8 @@ export default function SelectServer({
             : `${value}`
         }
         onDropdownVisibleChange={setOpen}
+        popupMatchSelectWidth={350}
         style={{
-          width: open ? "300px" : undefined,
           background,
           color: avatar_fontcolor(background),
           ...style,
diff --git a/src/packages/frontend/compute/serial-port-output.tsx b/src/packages/frontend/compute/serial-port-output.tsx
index 00b48bd298e..d47737d6739 100644
--- a/src/packages/frontend/compute/serial-port-output.tsx
+++ b/src/packages/frontend/compute/serial-port-output.tsx
@@ -9,6 +9,9 @@ Autorefresh exponential backoff algorithm:
    - if there is a change, refresh again in MIN_INTERVAL_MS
    - if there is no change, refresh in cur*EXPONENTIAL_BACKOFF seconds,
      up to MAX_INTERVAL_MS.
+
+It might be nice to replace the terminal rendering with:
+frontend/components/terminal.tsx
 */
 
 const MIN_INTERVAL_MS = 2000;
diff --git a/src/packages/frontend/compute/use-compute-server-api-state.ts b/src/packages/frontend/compute/use-compute-server-api-state.ts
new file mode 100644
index 00000000000..c0906abf832
--- /dev/null
+++ b/src/packages/frontend/compute/use-compute-server-api-state.ts
@@ -0,0 +1,93 @@
+/*
+
+- If compute server api isReady() returns true, then 'running'.
+
+- If compute server api not ready, check state of compute server itself, as stored
+  in the database (this is managed by a backend service).
+  If compute server state is 'starting' or 'running', then api state is 'starting',
+  otherwise api state is 'off'.
+
+- If the server is in the running state but isReady() returned false, we
+  wait a few seconds for the api to be ready, then try again.
+*/
+
+import { webapp_client } from "@cocalc/frontend/webapp-client";
+import { getServerState } from "./api";
+import { useEffect, useRef, useState } from "react";
+import { until } from "@cocalc/util/async-utils";
+import { useIsMountedRef } from "@cocalc/frontend/app-framework";
+
+export type State = "off" | "starting" | "running";
+const TIMEOUT = 5000;
+
+export default function useComputeServerApiState({
+  project_id,
+  compute_server_id,
+}: {
+  project_id?: string;
+  compute_server_id?: number;
+}): State | null {
+  const isMountedRef = useIsMountedRef();
+  const [state, setState] = useState(null);
+  const currentRef = useRef<{
+    project_id?: string;
+    compute_server_id?: number;
+  }>({
+    project_id,
+    compute_server_id,
+  });
+  currentRef.current.project_id = project_id;
+  currentRef.current.compute_server_id = compute_server_id;
+
+  useEffect(() => {
+    if (project_id == null || compute_server_id == null) {
+      setState(null);
+      return;
+    }
+    const isCurrent = () =>
+      isMountedRef.current &&
+      currentRef.current.project_id == project_id &&
+      currentRef.current.compute_server_id == compute_server_id;
+    const projectApi = webapp_client.conat_client.projectApi({
+      project_id,
+      compute_server_id,
+    });
+    (async () => {
+      await until(
+        async () => {
+          if (!isCurrent()) return true;
+          if (await projectApi.isReady()) {
+            if (!isCurrent()) return true;
+            setState("running");
+            return false;
+          }
+          const s = await getServerState(compute_server_id);
+          if (!isCurrent()) return true;
+          if (s == "running" || s == "starting") {
+            setState("starting");
+          } else {
+            setState("off");
+          }
+
+          // watch for change to running
+          try {
+            await projectApi.waitUntilReady({ timeout: TIMEOUT });
+            if (!isCurrent()) return true;
+            // didn't throw and is current, so must be running
+            setState("running");
+            return false;
+          } catch {}
+          if (!isCurrent()) return true;
+          return false;
+        },
+        { min: 3000, max: 6000 },
+      );
+    })();
+
+    return () => {
+      setState(null);
+    };
+  }, [project_id, compute_server_id]);
+
+  return state;
+}
diff --git a/src/packages/frontend/conat/client.ts b/src/packages/frontend/conat/client.ts
index 5ee541f0f15..b68c1d85ec5 100644
--- a/src/packages/frontend/conat/client.ts
+++ b/src/packages/frontend/conat/client.ts
@@ -9,9 +9,8 @@ import { randomId, inboxPrefix } from "@cocalc/conat/names";
 import { projectSubject } from "@cocalc/conat/names";
 import { parseQueryWithOptions } from "@cocalc/sync/table/util";
 import { type HubApi, initHubApi } from "@cocalc/conat/hub/api";
-import { type ProjectApi, initProjectApi } from "@cocalc/conat/project/api";
+import { type ProjectApi, projectApiClient } from "@cocalc/conat/project/api";
 import { isValidUUID } from "@cocalc/util/misc";
-import { createOpenFiles, OpenFiles } from "@cocalc/conat/sync/open-files";
 import { PubSub } from "@cocalc/conat/sync/pubsub";
 import type { ChatOptions } from "@cocalc/util/types/llm";
 import { dkv } from "@cocalc/conat/sync/dkv";
@@ -46,6 +45,10 @@ import {
   deleteRememberMe,
   setRememberMe,
 } from "@cocalc/frontend/misc/remember-me";
+import { client as projectRunnerClient } from "@cocalc/conat/project/runner/run";
+import { get as getBootlog } from "@cocalc/conat/project/runner/bootlog";
+import { terminalClient } from "@cocalc/conat/project/terminal";
+import { lite } from "@cocalc/frontend/lite";
 
 export interface ConatConnectionStatus {
   state: "connected" | "disconnected";
@@ -62,14 +65,20 @@ export class ConatClient extends EventEmitter {
   client: WebappClient;
   public hub: HubApi;
   public sessionId = randomId();
-  private openFilesCache: { [project_id: string]: OpenFiles } = {};
   private clientWithState: ClientWithState;
   private _conatClient: null | ReturnType;
   public numConnectionAttempts = 0;
   private automaticallyReconnect;
+  public address: string;
+  private remote: boolean;
 
-  constructor(client: WebappClient) {
+  constructor(
+    client: WebappClient,
+    { address, remote }: { address?: string; remote?: boolean } = {},
+  ) {
     super();
+    this.address = address ?? location.origin + appBasePath;
+    this.remote = !!remote;
     this.setMaxListeners(100);
     this.client = client;
     this.hub = initHubApi(this.callHub);
@@ -92,9 +101,8 @@ export class ConatClient extends EventEmitter {
   conat = () => {
     if (this._conatClient == null) {
       this.startStatsReporter();
-      const address = location.origin + appBasePath;
       this._conatClient = connectToConat({
-        address,
+        address: this.address,
         inboxPrefix: inboxPrefix({ account_id: this.client.account_id }),
         // it is necessary to manually managed reconnects due to a bugs
         // in socketio that has stumped their devs
@@ -155,21 +163,25 @@ export class ConatClient extends EventEmitter {
   };
 
   private initConatClient = async () => {
-    setConatClient({
-      account_id: this.client.account_id,
-      conat: this.conat,
-      reconnect: async () => this.reconnect(),
-      getLogger: DEBUG
-        ? (name) => {
-            return {
-              info: (...args) => console.info(name, ...args),
-              debug: (...args) => console.log(name, ...args),
-              warn: (...args) => console.warn(name, ...args),
-              silly: (...args) => console.log(name, ...args),
-            };
-          }
-        : undefined,
-    });
+    if (!this.remote) {
+      // only initialize if not making a remote connection, since this is
+      // the default connection to our local server
+      setConatClient({
+        account_id: this.client.account_id,
+        conat: this.conat,
+        reconnect: async () => this.reconnect(),
+        getLogger: DEBUG
+          ? (name) => {
+              return {
+                info: (...args) => console.info(name, ...args),
+                debug: (...args) => console.log(name, ...args),
+                warn: (...args) => console.warn(name, ...args),
+                silly: (...args) => console.log(name, ...args),
+              };
+            }
+          : undefined,
+      });
+    }
     this.clientWithState = getClientWithState();
     this.clientWithState.on("state", (state) => {
       if (state != "closed") {
@@ -178,6 +190,10 @@ export class ConatClient extends EventEmitter {
     });
     initTime();
     const client = this.conat();
+    client.inboxPrefixHook = (info) => {
+      return info?.user ? inboxPrefix(info?.user) : undefined;
+    };
+
     client.on("info", (info) => {
       if (client.info?.user?.account_id) {
         console.log("Connected as ", JSON.stringify(client.info?.user));
@@ -186,15 +202,27 @@ export class ConatClient extends EventEmitter {
           hub: info.id ?? "",
         });
         const cookie = Cookies.get(ACCOUNT_ID_COOKIE);
-        if (cookie && cookie != client.info.user.account_id) {
+        if (!lite && cookie && cookie != client.info.user.account_id) {
           // make sure account_id cookie is set to the actual account we're
           // signed in as, then refresh since some things are going to be
           // broken otherwise. To test this use dev tools and just change the account_id
           // cookies value to something random.
           Cookies.set(ACCOUNT_ID_COOKIE, client.info.user.account_id);
           // and we're out of here:
-          location.reload();
+          const wait = 5000;
+          console.log(`COOKIE ISSUE -- RELOAD IN ${wait / 1000} SECONDS...`, {
+            cookie,
+          });
+          setTimeout(() => {
+            if (lite) {
+              return;
+            }
+            location.reload();
+          }, 5000);
         }
+      } else if (lite && client.info?.user?.project_id) {
+        // we *also* sign in as the PROJECT in lite mode.
+        console.log("lite: created project client");
       } else {
         console.log("Sign in failed -- ", client.info);
         this.signInFailed(client.info?.user?.error ?? "Failed to sign in.");
@@ -236,6 +264,14 @@ export class ConatClient extends EventEmitter {
   // if there is a connection, resume it
   resume = () => {
     this.connect();
+    // sometimes due to a race (?) the above connect fails or
+    // is disconnected immedaitely. So we call connect more times,
+    // which are no-ops once connected.
+    for (const delay of [3_500, 10_000, 20_000]) {
+      setTimeout(() => {
+        this.connect();
+      }, delay);
+    }
   };
 
   // keep trying until connected.
@@ -259,10 +295,8 @@ export class ConatClient extends EventEmitter {
         await delay(750);
         await waitForOnline();
         attempts += 1;
-        console.log(
-          `Connecting to ${this._conatClient?.options.address}: attempts ${attempts}`,
-        );
-        this._conatClient?.conn.io.connect();
+        console.log(`Connecting to ${this.address}: attempts ${attempts}`);
+        this._conatClient?.connect();
         return false;
       },
       { min: 3000, max: 15000 },
@@ -314,7 +348,13 @@ export class ConatClient extends EventEmitter {
       const resp = await cn.request(subject, data, { timeout });
       return resp.data;
     } catch (err) {
-      err.message = `${err.message} - callHub: subject='${subject}', name='${name}', code='${err.code}' `;
+      try {
+        err.message = `${err.message} - callHub: subject='${subject}', name='${name}', code='${err.code}'`;
+      } catch {
+        err = new Error(
+          `${err.message} - callHub: subject='${subject}', name='${name}', code='${err.code}'`,
+        );
+      }
       throw err;
     }
   };
@@ -342,60 +382,20 @@ export class ConatClient extends EventEmitter {
       const actions = redux.getProjectActions(project_id);
       if (path != null) {
         compute_server_id =
-          actions.getComputeServerIdForFile({ path }) ??
+          actions.getComputeServerIdForFile(path) ??
           actions.getComputeServerId();
       } else {
         compute_server_id = actions.getComputeServerId();
       }
     }
-    const callProjectApi = async ({ name, args }) => {
-      return await this.callProject({
-        project_id,
-        compute_server_id,
-        timeout,
-        service: "api",
-        name,
-        args,
-      });
-    };
-    return initProjectApi(callProjectApi);
-  };
-
-  private callProject = async ({
-    service = "api",
-    project_id,
-    compute_server_id,
-    name,
-    args = [],
-    timeout = DEFAULT_TIMEOUT,
-  }: {
-    service?: string;
-    project_id: string;
-    compute_server_id?: number;
-    name: string;
-    args: any[];
-    timeout?: number;
-  }) => {
-    const cn = this.conat();
-    const subject = projectSubject({ project_id, compute_server_id, service });
-    const resp = await cn.request(
-      subject,
-      { name, args },
-      // we use waitForInterest because often the project hasn't
-      // quite fully started.
-      { timeout, waitForInterest: true },
-    );
-    return resp.data;
+    return projectApiClient({ project_id, compute_server_id, timeout });
   };
 
   synctable: ConatSyncTableFunction = async (
     query0,
     options?,
   ): Promise => {
-    const { query, table } = parseQueryWithOptions(query0, options);
-    if (options?.project_id != null && query[table][0]["project_id"] === null) {
-      query[table][0]["project_id"] = options.project_id;
-    }
+    const { query } = parseQueryWithOptions(query0, options);
     return await this.conat().sync.synctable({
       ...options,
       query,
@@ -425,42 +425,6 @@ export class ConatClient extends EventEmitter {
     });
   };
 
-  openFiles = reuseInFlight(async (project_id: string) => {
-    if (this.openFilesCache[project_id] == null) {
-      const openFiles = await createOpenFiles({
-        project_id,
-      });
-      this.openFilesCache[project_id] = openFiles;
-      openFiles.on("closed", () => {
-        delete this.openFilesCache[project_id];
-      });
-      openFiles.on("change", (entry) => {
-        if (entry.deleted?.deleted) {
-          setDeleted({
-            project_id,
-            path: entry.path,
-            deleted: entry.deleted.time,
-          });
-        } else {
-          setNotDeleted({ project_id, path: entry.path });
-        }
-      });
-      const recentlyDeletedPaths: any = {};
-      for (const { path, deleted } of openFiles.getAll()) {
-        if (deleted?.deleted) {
-          recentlyDeletedPaths[path] = deleted.time;
-        }
-      }
-      const store = redux.getProjectStore(project_id);
-      store.setState({ recentlyDeletedPaths });
-    }
-    return this.openFilesCache[project_id]!;
-  });
-
-  closeOpenFiles = (project_id) => {
-    this.openFilesCache[project_id]?.close();
-  };
-
   pubsub = async ({
     project_id,
     path,
@@ -515,22 +479,35 @@ export class ConatClient extends EventEmitter {
   };
 
   refCacheInfo = () => refCacheInfo();
-}
 
-function setDeleted({ project_id, path, deleted }) {
-  if (!redux.hasProjectStore(project_id)) {
-    return;
-  }
-  const actions = redux.getProjectActions(project_id);
-  actions.setRecentlyDeleted(path, deleted);
-}
+  projectRunner = (
+    project_id: string,
+    { timeout = 30 * 1000 * 60 }: { timeout?: number } = {},
+  ) => {
+    return projectRunnerClient({
+      project_id,
+      client: this.conat(),
+      timeout,
+    });
+  };
 
-function setNotDeleted({ project_id, path }) {
-  if (!redux.hasProjectStore(project_id)) {
-    return;
-  }
-  const actions = redux.getProjectActions(project_id);
-  actions?.setRecentlyDeleted(path, 0);
+  projectBootlog = (opts: {
+    project_id: string;
+    compute_server_id?: number;
+  }) => {
+    return getBootlog({ ...opts, client: this.conat() });
+  };
+
+  terminalClient = (opts: {
+    project_id: string;
+    compute_server_id?: number;
+    getSize?: () => undefined | { rows: number; cols: number };
+  }) => {
+    return terminalClient({
+      client: this.conat(),
+      ...opts,
+    });
+  };
 }
 
 async function waitForOnline(): Promise {
diff --git a/src/packages/frontend/course/actions.ts b/src/packages/frontend/course/actions.ts
index 3334d929aac..ae59ff1c754 100644
--- a/src/packages/frontend/course/actions.ts
+++ b/src/packages/frontend/course/actions.ts
@@ -127,7 +127,7 @@ export class CourseActions extends Actions {
     }
     this.syncdb.set(obj);
     if (commit) {
-      this.syncdb.commit(emitChangeImmediately);
+      this.syncdb.commit({ emitChangeImmediately });
     }
   };
 
diff --git a/src/packages/frontend/course/assignments/actions.ts b/src/packages/frontend/course/assignments/actions.ts
index a50934da009..cd0197b558f 100644
--- a/src/packages/frontend/course/assignments/actions.ts
+++ b/src/packages/frontend/course/assignments/actions.ts
@@ -8,7 +8,7 @@ Actions involving working with assignments:
   - assigning, collecting, setting feedback, etc.
 */
 
-import { delay, map } from "awaiting";
+import { map } from "awaiting";
 import { Map } from "immutable";
 import { debounce } from "lodash";
 import { join } from "path";
@@ -39,7 +39,6 @@ import {
   uuid,
 } from "@cocalc/util/misc";
 import { CourseActions } from "../actions";
-import { COPY_TIMEOUT_MS } from "../consts";
 import { export_assignment } from "../export/export-assignment";
 import { export_student_file_use_times } from "../export/file-use-times";
 import { grading_state } from "../nbgrader/util";
@@ -456,15 +455,13 @@ export class AssignmentsActions {
       desc: `Copying assignment from ${student_name}`,
     });
     try {
-      await webapp_client.project_client.copy_path_between_projects({
-        src_project_id: student_project_id,
-        src_path: assignment.get("target_path"),
-        target_project_id: store.get("course_project_id"),
-        target_path,
-        overwrite_newer: true,
-        backup: true,
-        delete_missing: false,
-        timeout: COPY_TIMEOUT_MS,
+      await webapp_client.project_client.copyPathBetweenProjects({
+        src: {
+          project_id: student_project_id,
+          path: assignment.get("target_path"),
+        },
+        dest: { project_id: store.get("course_project_id"), path: target_path },
+        options: { recursive: true },
       });
       // write their name to a file
       const name = store.get_student_name_extra(student_id);
@@ -615,17 +612,25 @@ ${details}
         path: src_path + "/GRADE.md",
         content,
       });
-      await webapp_client.project_client.copy_path_between_projects({
-        src_project_id: store.get("course_project_id"),
-        src_path,
-        target_project_id: student_project_id,
-        target_path: assignment.get("graded_path"),
-        overwrite_newer: true,
-        backup: true,
-        delete_missing: false,
-        exclude: peer_graded ? ["*GRADER*.txt"] : undefined,
-        timeout: COPY_TIMEOUT_MS,
+      await webapp_client.project_client.copyPathBetweenProjects({
+        src: { project_id: store.get("course_project_id"), path: src_path },
+        dest: {
+          project_id: student_project_id,
+          path: assignment.get("graded_path"),
+        },
+        options: {
+          recursive: true,
+        },
       });
+      if (peer_graded) {
+        const actions = redux.getProjectActions(student_project_id);
+        await actions.deleteMatchingFiles({
+          path: assignment.get("graded_path"),
+          recursive: true,
+          compute_server_id: 0,
+          filter: (p) => p.includes("GRADER"),
+        });
+      }
       finish("");
     } catch (err) {
       finish(err);
@@ -853,21 +858,19 @@ ${details}
         desc: `Copying files to ${student_name}'s project`,
       });
       const opts = {
-        src_project_id: store.get("course_project_id"),
-        src_path,
-        target_project_id: student_project_id,
-        target_path: assignment.get("target_path"),
-        overwrite_newer: !!overwrite, // default is "false"
-        delete_missing: !!overwrite, // default is "false"
-        backup: !!!overwrite, // default is "true"
-        timeout: COPY_TIMEOUT_MS,
+        src: { project_id: store.get("course_project_id"), path: src_path },
+        dest: {
+          project_id: student_project_id,
+          path: assignment.get("target_path"),
+        },
+        options: { recursive: true, force: !!overwrite },
       };
-      await webapp_client.project_client.copy_path_between_projects(opts);
+      await webapp_client.project_client.copyPathBetweenProjects(opts);
       await this.course_actions.compute.setComputeServerAssociations({
         student_id,
-        src_path: opts.src_path,
-        target_project_id: opts.target_project_id,
-        target_path: opts.target_path,
+        src_path,
+        target_project_id: student_project_id,
+        target_path: assignment.get("target_path"),
         unit_id: assignment_id,
       });
 
@@ -1007,28 +1010,6 @@ ${details}
     });
   };
 
-  private start_all_for_peer_grading = async (): Promise => {
-    // On cocalc.com, if the student projects get started specifically
-    // for the purposes of copying files to/from them, then they stop
-    // around a minute later.  This is very bad for peer grading, since
-    // so much copying occurs, and we end up with conflicts between
-    // projects starting to peer grade, then stop, then needing to be
-    // started again all at once.  We thus request that they all start,
-    // wait a few seconds for that "reason" for them to be running to
-    // take effect, and then do the copy.  This way the projects aren't
-    // automatically stopped after the copies happen.
-    const id = this.course_actions.set_activity({
-      desc: "Warming up all student projects for peer grading...",
-    });
-    this.course_actions.student_projects.action_all_student_projects("start");
-    // We request to start all projects simultaneously, and the system
-    // will start doing that.  I think it's not so much important that
-    // the projects are actually running, but that they were started
-    // before the copy operations started.
-    await delay(5 * 1000);
-    this.course_actions.clear_activity(id);
-  };
-
   async peer_copy_to_all_students(
     assignment_id: string,
     new_only: boolean,
@@ -1049,7 +1030,6 @@ ${details}
       this.course_actions.set_error(`${short_desc} -- ${err}`);
       return;
     }
-    await this.start_all_for_peer_grading();
     // OK, now do the assignment... in parallel.
     await this.assignment_action_all_students({
       assignment_id,
@@ -1070,7 +1050,6 @@ ${details}
       desc += " from whom we have not already copied it";
     }
     const short_desc = "copy peer grading from students";
-    await this.start_all_for_peer_grading();
     await this.assignment_action_all_students({
       assignment_id,
       new_only,
@@ -1370,15 +1349,19 @@ ${details}
       // peer grading is anonymous; also, remove original
       // due date to avoid confusion.
       // copy the files to be peer graded into place for this student
-      await webapp_client.project_client.copy_path_between_projects({
-        src_project_id: store.get("course_project_id"),
-        src_path,
-        target_project_id: student_project_id,
-        target_path,
-        overwrite_newer: false,
-        delete_missing: false,
-        exclude: ["*STUDENT*.txt", "*" + DUE_DATE_FILENAME + "*"],
-        timeout: COPY_TIMEOUT_MS,
+      await webapp_client.project_client.copyPathBetweenProjects({
+        src: { project_id: store.get("course_project_id"), path: src_path },
+        dest: { project_id: student_project_id, path: target_path },
+        options: { recursive: true, force: false },
+      });
+
+      const actions = redux.getProjectActions(student_project_id);
+      await actions.deleteMatchingFiles({
+        path: target_path,
+        recursive: true,
+        compute_server_id: 0,
+        filter: (path) =>
+          path.includes("STUDENT") || path.includes(DUE_DATE_FILENAME),
       });
     };
 
@@ -1454,14 +1437,10 @@ ${details}
       }
 
       // copy the files over from the student who did the peer grading
-      await webapp_client.project_client.copy_path_between_projects({
-        src_project_id,
-        src_path,
-        target_project_id: store.get("course_project_id"),
-        target_path,
-        overwrite_newer: false,
-        delete_missing: false,
-        timeout: COPY_TIMEOUT_MS,
+      await webapp_client.project_client.copyPathBetweenProjects({
+        src: { project_id: src_project_id, path: src_path },
+        dest: { project_id: store.get("course_project_id"), path: target_path },
+        options: { force: false, recursive: true },
       });
 
       // write local file identifying the grader
@@ -1598,7 +1577,7 @@ ${details}
 
     let has_student_subdir: boolean = false;
     for (const entry of listing) {
-      if (entry.isdir && entry.name == STUDENT_SUBDIR) {
+      if (entry.isDir && entry.name == STUDENT_SUBDIR) {
         has_student_subdir = true;
         break;
       }
@@ -1641,10 +1620,8 @@ ${details}
     const project_id = store.get("course_project_id");
     let files;
     try {
-      files = await redux
-        .getProjectStore(project_id)
-        .get_listings()
-        .getListingDirectly(path);
+      const { fs } = this.course_actions.syncdb;
+      files = await fs.readdir(path, { withFileTypes: true });
     } catch (err) {
       // This happens, e.g., if the instructor moves the directory
       // that contains their version of the ipynb file.
@@ -1658,7 +1635,7 @@ ${details}
     if (this.course_actions.is_closed()) return result;
 
     const to_read = files
-      .filter((entry) => !entry.isdir && endswith(entry.name, ".ipynb"))
+      .filter((entry) => entry.isFile() && endswith(entry.name, ".ipynb"))
       .map((entry) => entry.name);
 
     const f: (file: string) => Promise = async (file) => {
@@ -2008,15 +1985,10 @@ ${details}
             // This is necessary because grading the assignment may depend on
             // data files that are sent as part of the assignment.  Also,
             // student's might have some code in text files next to the ipynb.
-            await webapp_client.project_client.copy_path_between_projects({
-              src_project_id: course_project_id,
-              src_path: student_path,
-              target_project_id: grade_project_id,
-              target_path: student_path,
-              overwrite_newer: true,
-              delete_missing: true,
-              backup: false,
-              timeout: COPY_TIMEOUT_MS,
+            await webapp_client.project_client.copyPathBetweenProjects({
+              src: { project_id: course_project_id, path: student_path },
+              dest: { project_id: grade_project_id, path: student_path },
+              options: { recursive: true },
             });
           } else {
             ephemeralGradePath = false;
diff --git a/src/packages/frontend/course/assignments/assignment.tsx b/src/packages/frontend/course/assignments/assignment.tsx
index 276aaa1364b..6ae1f5a8c8c 100644
--- a/src/packages/frontend/course/assignments/assignment.tsx
+++ b/src/packages/frontend/course/assignments/assignment.tsx
@@ -407,7 +407,7 @@ export function Assignment({
     );
   }
 
-  function open_assignment_path(): void {
+  async function open_assignment_path() {
     if (assignment.get("listing")?.size == 0) {
       // there are no files yet, so we *close* the assignment
       // details panel.  This is just **a hack** so that the user
@@ -421,7 +421,7 @@ export function Assignment({
         assignment.get("assignment_id"),
       );
     }
-    return redux
+    await redux
       .getProjectActions(project_id)
       .open_directory(assignment.get("path"));
   }
@@ -704,11 +704,8 @@ export function Assignment({
         return (
           
             This will recopy all of the files to them. CAUTION: if you update a
-            file that a student has also worked on, their work will get copied
-            to a backup file ending in a tilde, or possibly only be available in
-            snapshots. Select "Replace student files!" in case you do not{" "}
-            want to create any backups and also delete all other files in
-            the assignment folder of their projects.{" "}
+            file that a student has also worked on, their work will get
+            overwritten. They can use TimeTravel to get it back.
             
         );
       case "collect":
-        return "This will recollect all of the homework from them.  CAUTION: if you have graded/edited a file that a student has updated, your work will get copied to a backup file ending in a tilde, or possibly only be available in snapshots.";
+        return "This will recollect all of the homework from them.  CAUTION: if you have graded/edited a file that a student has updated, your work will get overwritten. Use TimeTravel to get it back.";
       case "return_graded":
         return "This will rereturn all of the graded files to them.";
       case "peer_assignment":
-        return "This will recopy all of the files to them.  CAUTION: if there is a file a student has also worked on grading, their work will get copied to a backup file ending in a tilde, or possibly be only available in snapshots.";
+        return "This will recopy all of the files to them.  CAUTION: if there is a file a student has also worked on grading, their work will get overwritten. Use TimeTravel to get it back.";
       case "peer_collect":
-        return "This will recollect all of the peer-graded homework from the students.  CAUTION: if you have graded/edited a previously collected file that a student has updated, your work will get copied to a backup file ending in a tilde, or possibly only be available in snapshots.";
+        return "This will recollect all of the peer-graded homework from the students.  CAUTION: if you have graded/edited a previously collected file that a student has updated, your work will get overwritten.  Use TimeTravel to get it back.";
     }
   }
 
@@ -743,18 +740,8 @@ export function Assignment({
             disabled={copy_assignment_confirm_overwrite}
             onClick={() => copy_assignment(step, false)}
           >
-            Yes, do it (with backup)
+            Yes. Replaces student files!
           
-          {step === "assignment" ? (
-            
-          ) : undefined}
         
         {render_copy_assignment_confirm_overwrite(step)}
       

diff --git a/src/packages/frontend/course/common/student-assignment-info.tsx b/src/packages/frontend/course/common/student-assignment-info.tsx
index 507c6db86c8..17a06ad9461 100644
--- a/src/packages/frontend/course/common/student-assignment-info.tsx
+++ b/src/packages/frontend/course/common/student-assignment-info.tsx
@@ -10,7 +10,6 @@ import { FormattedMessage, useIntl } from "react-intl";
 import { useActions } from "@cocalc/frontend/app-framework";
 import { Gap, Icon, Markdown, Tip } from "@cocalc/frontend/components";
 import ShowError from "@cocalc/frontend/components/error";
-import { COPY_TIMEOUT_MS } from "@cocalc/frontend/course/consts";
 import { MarkdownInput } from "@cocalc/frontend/editors/markdown-input";
 import { labels } from "@cocalc/frontend/i18n";
 import { NotebookScores } from "@cocalc/frontend/jupyter/nbgrader/autograde";
@@ -95,7 +94,7 @@ export function StudentAssignmentInfo({
   nbgrader_run_info,
 }: StudentAssignmentInfoProps) {
   const intl = useIntl();
-  const clicked_nbgrader = useRef(undefined);
+  const clicked_nbgrader = useRef(undefined);
   const actions = useActions({ name });
   const size = useButtonSize();
   const [recopy, set_recopy] = useRecopy();
@@ -512,7 +511,7 @@ export function StudentAssignmentInfo({
     const do_stop = () => stop(type, info.assignment_id, info.student_id);
     const v: React.JSX.Element[] = [];
     if (enable_copy) {
-      if (webapp_client.server_time() - (data.start ?? 0) < COPY_TIMEOUT_MS) {
+      if (webapp_client.server_time() - (data.start ?? 0) < 15_000) {
         v.push(render_open_copying(step, do_open, do_stop));
       } else if (data.time) {
         v.push(
diff --git a/src/packages/frontend/course/configuration/configuration-copying.tsx b/src/packages/frontend/course/configuration/configuration-copying.tsx
index 4510ad0f238..41df5121e60 100644
--- a/src/packages/frontend/course/configuration/configuration-copying.tsx
+++ b/src/packages/frontend/course/configuration/configuration-copying.tsx
@@ -35,19 +35,15 @@ import {
 } from "antd";
 import { useMemo, useState } from "react";
 import { FormattedMessage, useIntl } from "react-intl";
-
 import { labels } from "@cocalc/frontend/i18n";
 import {
   redux,
   useFrameContext,
-  useTypedRedux,
 } from "@cocalc/frontend/app-framework";
 import { Icon } from "@cocalc/frontend/components";
 import ShowError from "@cocalc/frontend/components/error";
 import { COMMANDS } from "@cocalc/frontend/course/commands";
-import { exec } from "@cocalc/frontend/frame-editors/generic/client";
 import { IntlMessage } from "@cocalc/frontend/i18n";
-import { pathExists } from "@cocalc/frontend/project/directory-selector";
 import { ProjectTitle } from "@cocalc/frontend/projects/project-title";
 import { isIntlMessage } from "@cocalc/util/i18n";
 import { plural } from "@cocalc/util/misc";
@@ -446,10 +442,6 @@ function AddTarget({ settings, actions, project_id }) {
   const [path, setPath] = useState("");
   const [error, setError] = useState("");
   const [create, setCreate] = useState("");
-  const directoryListings = useTypedRedux(
-    { project_id },
-    "directory_listings",
-  )?.get(0);
 
   const add = async () => {
     try {
@@ -458,19 +450,14 @@ function AddTarget({ settings, actions, project_id }) {
         throw Error(`'${path} is the current course'`);
       }
       setLoading(true);
-      const exists = await pathExists(project_id, path, directoryListings);
-      if (!exists) {
+      const projectActions = redux.getProjectActions(project_id);
+      const fs = projectActions.fs();
+      if (!(await fs.exists(path))) {
         if (create) {
-          await exec({
-            command: "touch",
-            args: [path],
-            project_id,
-            filesystem: true,
-          });
-        } else {
-          setCreate(path);
-          return;
+          await fs.writeFile(path, "");
         }
+      } else {
+        setCreate(path);
       }
       const copy_config_targets = getTargets(settings);
       copy_config_targets[`${project_id}/${path}`] = true;
diff --git a/src/packages/frontend/course/consts.ts b/src/packages/frontend/course/consts.ts
deleted file mode 100644
index 96e0001fb19..00000000000
--- a/src/packages/frontend/course/consts.ts
+++ /dev/null
@@ -1,6 +0,0 @@
-// All copy operations (e.g., assigning, collecting, etc.) is set to timeout after this long.
-// Also in the UI displaying that a copy is ongoing also times out after this long, e.g, if
-// the user refreshes their browser and nothing is going to update things again.
-// TODO: make this a configurable parameter, e.g., maybe users have very large assignments
-// or things are very slow.
-export const COPY_TIMEOUT_MS = 2 * 60 * 1000; // 2 minutes, for now -- starting project can take time.
diff --git a/src/packages/frontend/course/export/export-assignment.ts b/src/packages/frontend/course/export/export-assignment.ts
index 0ab46b502ea..35e14ea09bb 100644
--- a/src/packages/frontend/course/export/export-assignment.ts
+++ b/src/packages/frontend/course/export/export-assignment.ts
@@ -82,7 +82,7 @@ async function export_one_directory(
   let x: any;
   const timeout = 60; // 60 seconds
   for (x of listing) {
-    if (x.isdir) continue; // we ignore subdirectories...
+    if (x.isDir) continue; // we ignore subdirectories...
     const { name } = x;
     if (startswith(name, "STUDENT")) continue;
     if (startswith(name, ".")) continue;
diff --git a/src/packages/frontend/course/handouts/actions.ts b/src/packages/frontend/course/handouts/actions.ts
index 302e074bab0..8f604f14c1f 100644
--- a/src/packages/frontend/course/handouts/actions.ts
+++ b/src/packages/frontend/course/handouts/actions.ts
@@ -16,7 +16,6 @@ import { map } from "awaiting";
 import type { SyncDBRecordHandout } from "../types";
 import { exec } from "../../frame-editors/generic/client";
 import { export_student_file_use_times } from "../export/file-use-times";
-import { COPY_TIMEOUT_MS } from "../consts";
 
 export class HandoutsActions {
   private course_actions: CourseActions;
@@ -253,22 +252,20 @@ export class HandoutsActions {
       });
 
       const opts = {
-        src_project_id: course_project_id,
-        src_path,
-        target_project_id: student_project_id,
-        target_path: handout.get("target_path"),
-        overwrite_newer: !!overwrite, // default is "false"
-        delete_missing: !!overwrite, // default is "false"
-        backup: !!!overwrite, // default is "true"
-        timeout: COPY_TIMEOUT_MS,
+        src: { project_id: course_project_id, path: src_path },
+        dest: {
+          project_id: student_project_id,
+          path: handout.get("target_path"),
+        },
+        options: { force: !!overwrite },
       };
-      await webapp_client.project_client.copy_path_between_projects(opts);
+      await webapp_client.project_client.copyPathBetweenProjects(opts);
 
       await this.course_actions.compute.setComputeServerAssociations({
         student_id,
-        src_path: opts.src_path,
-        target_project_id: opts.target_project_id,
-        target_path: opts.target_path,
+        src_path,
+        target_project_id: student_project_id,
+        target_path: handout.get("target_path"),
         unit_id: handout_id,
       });
 
diff --git a/src/packages/frontend/course/handouts/handout.tsx b/src/packages/frontend/course/handouts/handout.tsx
index aaf3ab001e2..be744611ee3 100644
--- a/src/packages/frontend/course/handouts/handout.tsx
+++ b/src/packages/frontend/course/handouts/handout.tsx
@@ -324,8 +324,7 @@ export function Handout({
       case "handout":
         return `\
   This will recopy all of the files to them.
-  CAUTION: if you update a file that a student has also worked on, their work will get copied to a backup file ending in a tilde, or possibly only be available in snapshots.
-  Select "Replace student files!" in case you do not want to create any backups and also delete all other files in the handout directory of their projects.\
+  CAUTION: if you update a file that a student has also worked on, their work will get overwritten. They can recover it using TimeTravel.\
   `;
     }
   }
diff --git a/src/packages/frontend/course/handouts/handouts-info-panel.tsx b/src/packages/frontend/course/handouts/handouts-info-panel.tsx
index cfdf4d09020..f4999c8f772 100644
--- a/src/packages/frontend/course/handouts/handouts-info-panel.tsx
+++ b/src/packages/frontend/course/handouts/handouts-info-panel.tsx
@@ -11,7 +11,6 @@ import { useIntl } from "react-intl";
 
 import { Icon, Tip } from "@cocalc/frontend/components";
 import ShowError from "@cocalc/frontend/components/error";
-import { COPY_TIMEOUT_MS } from "@cocalc/frontend/course/consts";
 import { labels } from "@cocalc/frontend/i18n";
 import { webapp_client } from "@cocalc/frontend/webapp-client";
 import { CourseActions } from "../actions";
@@ -155,7 +154,7 @@ export function StudentHandoutInfo({
     }
     const v: any[] = [];
     if (enable_copy) {
-      if (webapp_client.server_time() - (obj.start ?? 0) < COPY_TIMEOUT_MS) {
+      if (webapp_client.server_time() - (obj.start ?? 0) < 15_000) {
         v.push(render_open_copying(do_open, do_stop));
       } else if (obj.time) {
         v.push(render_open_recopy(name, do_open, do_copy, copy_tip, open_tip));
diff --git a/src/packages/frontend/course/shared-project/actions.ts b/src/packages/frontend/course/shared-project/actions.ts
index 3b6254fcd21..918ef34f6fe 100644
--- a/src/packages/frontend/course/shared-project/actions.ts
+++ b/src/packages/frontend/course/shared-project/actions.ts
@@ -176,10 +176,10 @@ export class SharedProjectActions {
     if (!shared_project_id) {
       return; // no shared project
     }
-    const dflt_img = await redux.getStore("customize").getDefaultComputeImage();
-    const img_id = store.get("settings").get("custom_image") ?? dflt_img;
+    const defaultImage = await redux.getStore("customize").getDefaultComputeImage();
+    const imageId = store.get("settings").get("custom_image") ?? defaultImage;
     const actions = redux.getProjectActions(shared_project_id);
-    await actions.set_compute_image(img_id);
+    await actions.set_compute_image(imageId);
   };
 
   set_datastore_and_envvars = async (): Promise => {
diff --git a/src/packages/frontend/course/student-projects/actions.ts b/src/packages/frontend/course/student-projects/actions.ts
index c02afa34bd8..dcba5f97136 100644
--- a/src/packages/frontend/course/student-projects/actions.ts
+++ b/src/packages/frontend/course/student-projects/actions.ts
@@ -66,14 +66,13 @@ export class StudentProjectsActions {
     const id = this.course_actions.set_activity({
       desc: `Create project for ${store.get_student_name(student_id)}.`,
     });
-    const dflt_img = await redux.getStore("customize").getDefaultComputeImage();
+    const defaultImage = await redux.getStore("customize").getDefaultComputeImage();
     let project_id: string;
     try {
       project_id = await redux.getActions("projects").create_project({
         title: store.get("settings").get("title"),
         description: store.get("settings").get("description"),
-        image: store.get("settings").get("custom_image") ?? dflt_img,
-        noPool: true, // student is unlikely to use the project right *now*
+        image: store.get("settings").get("custom_image") ?? defaultImage,
       });
     } catch (err) {
       this.course_actions.set_error(
@@ -607,8 +606,8 @@ export class StudentProjectsActions {
   ): Promise => {
     const store = this.get_store();
     if (store == null) return;
-    const dflt_img = await redux.getStore("customize").getDefaultComputeImage();
-    const img_id = store.get("settings").get("custom_image") ?? dflt_img;
+    const defaultImage = await redux.getStore("customize").getDefaultComputeImage();
+    const img_id = store.get("settings").get("custom_image") ?? defaultImage;
     const actions = redux.getProjectActions(student_project_id);
     await actions.set_compute_image(img_id);
   };
diff --git a/src/packages/frontend/course/sync.ts b/src/packages/frontend/course/sync.ts
index 69a470e1f81..6fbfe3df11c 100644
--- a/src/packages/frontend/course/sync.ts
+++ b/src/packages/frontend/course/sync.ts
@@ -31,7 +31,7 @@ export function create_sync_db(
   const path = store.get("course_filename");
   actions.setState({ loading: true });
 
-  const syncdb = webapp_client.sync_client.sync_db({
+  const syncdb = webapp_client.conat_client.conat().sync.db({
     project_id,
     path,
     primary_keys: ["table", "handout_id", "student_id", "assignment_id"],
diff --git a/src/packages/frontend/cspell.json b/src/packages/frontend/cspell.json
index f393183b970..065cdbdb7b6 100644
--- a/src/packages/frontend/cspell.json
+++ b/src/packages/frontend/cspell.json
@@ -91,6 +91,7 @@
     "isabs",
     "isactive",
     "isdir",
+    "isDir",
     "isopen",
     "issymlink",
     "kernelspec",
diff --git a/src/packages/frontend/custom-software/reset-bar.tsx b/src/packages/frontend/custom-software/reset-bar.tsx
index 2442a38bbe9..5195c76649f 100644
--- a/src/packages/frontend/custom-software/reset-bar.tsx
+++ b/src/packages/frontend/custom-software/reset-bar.tsx
@@ -6,15 +6,15 @@
 import { Button as AntdButton, Card } from "antd";
 import React from "react";
 import { FormattedMessage, useIntl } from "react-intl";
-
 import { A, Icon } from "@cocalc/frontend/components";
 import { labels } from "@cocalc/frontend/i18n";
 import { ProjectMap } from "@cocalc/frontend/todo-types";
 import { COLORS, SITE_NAME } from "@cocalc/util/theme";
-
 import { Available as AvailableFeatures } from "../project_configuration";
 import { ComputeImages } from "./init";
 import { props2img, RESET_ICON } from "./util";
+import { useTypedRedux } from "@cocalc/frontend/app-framework";
+import { type ProjectActions } from "@cocalc/frontend/project_store";
 
 const doc_snap = "https://doc.cocalc.com/project-files.html#snapshots";
 const doc_tt = "https://doc.cocalc.com/time-travel.html";
@@ -36,13 +36,13 @@ interface Props {
   project_id: string;
   images: ComputeImages;
   project_map?: ProjectMap;
-  actions: any;
+  actions: ProjectActions;
   available_features?: AvailableFeatures;
-  site_name?: string;
 }
 
 export const CustomSoftwareReset: React.FC = (props: Props) => {
-  const { actions, site_name } = props;
+  const { actions } = props;
+  const site_name = useTypedRedux("customize", "site_name");
 
   const intl = useIntl();
 
@@ -83,7 +83,7 @@ export const CustomSoftwareReset: React.FC = (props: Props) => {
               Note, that this will overwrite any changes you did to these
               accompanying files, but does not modify or delete any other files.
               However, nothing is lost: you can still access the previous version
-              via Snapshot Backups or TimeTravel.
+              via Snapshots or TimeTravel.
             

             
This action will also restart your project!
`}
           values={{
diff --git a/src/packages/frontend/custom-software/selector.tsx b/src/packages/frontend/custom-software/selector.tsx
index 9ff6988cd79..8d31921bbcf 100644
--- a/src/packages/frontend/custom-software/selector.tsx
+++ b/src/packages/frontend/custom-software/selector.tsx
@@ -3,7 +3,7 @@
  *  License: MS-RSL – see LICENSE.md for details
  */
 
-// cSpell:ignore descr disp dflt
+// cSpell:ignore descr disp
 
 import { Col, Form } from "antd";
 import { FormattedMessage, useIntl } from "react-intl";
@@ -43,11 +43,11 @@ export async function derive_project_img_name(
   custom_software: SoftwareEnvironmentState,
 ): Promise {
   const { image_type, image_selected } = custom_software;
-  const dflt_software_img = await redux
+  const defaultSoftwareImage = await redux
     .getStore("customize")
     .getDefaultComputeImage();
   if (image_selected == null || image_type == null) {
-    return dflt_software_img;
+    return defaultSoftwareImage;
   }
   switch (image_type) {
     case "custom":
@@ -56,7 +56,7 @@ export async function derive_project_img_name(
       return image_selected;
     default:
       unreachable(image_type);
-      return dflt_software_img; // make TS happy
+      return defaultSoftwareImage; // make TS happy
   }
 }
 
@@ -77,7 +77,7 @@ export function SoftwareEnvironment(props: Props) {
   const onCoCalcCom = customize_kucalc === KUCALC_COCALC_COM;
   const customize_software = useTypedRedux("customize", "software");
   const organization_name = useTypedRedux("customize", "organization_name");
-  const dflt_software_img = customize_software.get("default");
+  const defaultSoftwareImage = customize_software.get("default");
   const software_images = customize_software.get("environments");
 
   const haveSoftwareImages: boolean = useMemo(
@@ -111,7 +111,7 @@ export function SoftwareEnvironment(props: Props) {
 
   // initialize selection, if there is a default image set
   React.useEffect(() => {
-    if (default_image == null || default_image === dflt_software_img) {
+    if (default_image == null || default_image === defaultSoftwareImage) {
       // do nothing, that's the initial state already!
     } else if (is_custom_image(default_image)) {
       if (images == null) return;
@@ -125,7 +125,7 @@ export function SoftwareEnvironment(props: Props) {
     } else {
       // must be standard image
       const img = software_images.get(default_image);
-      const display = img != null ? img.get("title") ?? "" : "";
+      const display = img != null ? (img.get("title") ?? "") : "";
       setState(default_image, display, "standard");
     }
   }, []);
@@ -172,7 +172,7 @@ export function SoftwareEnvironment(props: Props) {
   }
 
   function render_onprem() {
-    const selected = image_selected ?? dflt_software_img;
+    const selected = image_selected ?? defaultSoftwareImage;
     return (
       <>
         
@@ -221,7 +221,7 @@ export function SoftwareEnvironment(props: Props) {
   }
 
   function render_standard_image_selector() {
-    const isCustom = is_custom_image(image_selected ?? dflt_software_img);
+    const isCustom = is_custom_image(image_selected ?? defaultSoftwareImage);
     return (
       <>
         
@@ -232,7 +232,7 @@ export function SoftwareEnvironment(props: Props) {
             >
                {
diff --git a/src/packages/frontend/customize.tsx b/src/packages/frontend/customize.tsx
index ca560e3d91d..835793059ab 100644
--- a/src/packages/frontend/customize.tsx
+++ b/src/packages/frontend/customize.tsx
@@ -9,7 +9,6 @@
 import { fromJS, List, Map } from "immutable";
 import { join } from "path";
 import { useIntl } from "react-intl";
-
 import {
   Actions,
   rclass,
@@ -61,6 +60,7 @@ import { CustomLLMPublic } from "@cocalc/util/types/llm";
 import { DefaultQuotaSetting, Upgrades } from "@cocalc/util/upgrades/quota";
 export { TermsOfService } from "@cocalc/frontend/customize/terms-of-service";
 import { delay } from "awaiting";
+import { init as initLite } from "./lite";
 
 // update every 2 minutes.
 const UPDATE_INTERVAL = 2 * 60000;
@@ -103,6 +103,7 @@ export type SoftwareEnvironments = TypedMap<{
 export interface CustomizeState {
   time: number; // this will always get set once customize has loaded.
   is_commercial: boolean;
+
   openai_enabled: boolean;
   google_vertexai_enabled: boolean;
   mistral_enabled: boolean;
@@ -188,6 +189,12 @@ export interface CustomizeState {
   i18n?: List;
 
   user_tracking?: string;
+
+  lite?: boolean;
+  account_id?: string;
+  project_id?: string;
+  compute_server_id?: number;
+  remote_sync?: boolean;
 }
 
 export class CustomizeStore extends Store {
@@ -261,7 +268,7 @@ export class CustomizeActions extends Actions {
       unlicensed_project_timetravel_limit: undefined,
     });
   };
-  
+
   reload = async () => {
     await loadCustomizeState();
   };
@@ -305,6 +312,7 @@ async function loadCustomizeState() {
     ollama = null, // the derived public information
     custom_openai = null,
   } = customize;
+  processLite(configuration);
   process_kucalc(configuration);
   process_software(software, configuration.is_cocalc_com);
   process_customize(configuration); // this sets _is_configured to true
@@ -704,3 +712,12 @@ async function init_analytics() {
 }
 
 init_analytics();
+
+let liteInitialized = false;
+function processLite(configuration) {
+  if (!configuration.lite || liteInitialized) {
+    return;
+  }
+  liteInitialized = true;
+  initLite(redux, configuration);
+}
diff --git a/src/packages/frontend/editors/archive/component.tsx b/src/packages/frontend/editors/archive/component.tsx
index 7828ab71a28..c7512d60624 100644
--- a/src/packages/frontend/editors/archive/component.tsx
+++ b/src/packages/frontend/editors/archive/component.tsx
@@ -4,7 +4,6 @@
  */
 
 import { Button, Card } from "antd";
-
 import { useActions, useRedux } from "@cocalc/frontend/app-framework";
 import { A, ErrorDisplay, Icon, Loading } from "@cocalc/frontend/components";
 import { ArchiveActions } from "./actions";
diff --git a/src/packages/frontend/editors/file-info-dropdown.tsx b/src/packages/frontend/editors/file-info-dropdown.tsx
index bcbf3530a2a..e25de0188d0 100644
--- a/src/packages/frontend/editors/file-info-dropdown.tsx
+++ b/src/packages/frontend/editors/file-info-dropdown.tsx
@@ -11,7 +11,7 @@ import { CSS, React, useActions } from "@cocalc/frontend/app-framework";
 import { DropdownMenu, Icon, IconName } from "@cocalc/frontend/components";
 import { MenuItems } from "@cocalc/frontend/components/dropdown-menu";
 import { useStudentProjectFunctionality } from "@cocalc/frontend/course";
-import { file_actions } from "@cocalc/frontend/project_store";
+import { file_actions, type FileAction } from "@cocalc/frontend/project_store";
 import { capitalize, filename_extension } from "@cocalc/util/misc";
 
 interface Props {
@@ -57,9 +57,9 @@ const EditorFileInfoDropdown: React.FC = React.memo(
       }
       for (const key in file_actions) {
         if (key === name) {
-          actions.show_file_action_panel({
+          actions.showFileActionPanel({
             path: filename,
-            action: key,
+            action: key as FileAction,
           });
           break;
         }
diff --git a/src/packages/frontend/editors/slate/elements/code-block/editable.tsx b/src/packages/frontend/editors/slate/elements/code-block/editable.tsx
index ba3c876e108..4a1ed439a83 100644
--- a/src/packages/frontend/editors/slate/elements/code-block/editable.tsx
+++ b/src/packages/frontend/editors/slate/elements/code-block/editable.tsx
@@ -129,7 +129,6 @@ function Element({ attributes, children, element }: RenderElementProps) {
                   )}
                   {!disableMarkdownCodebar && (
                      = ({
                 
               )}
                = (props: EditableProps) => {
                   }
                 }
               }
-              1;
             }
           },
           [readOnly, attributes.onCut],
diff --git a/src/packages/frontend/editors/slate/upload.tsx b/src/packages/frontend/editors/slate/upload.tsx
index 8f4dd9f4316..dff8f593076 100644
--- a/src/packages/frontend/editors/slate/upload.tsx
+++ b/src/packages/frontend/editors/slate/upload.tsx
@@ -76,8 +76,9 @@ export default function useUpload(
           return;
         }
         let node;
-        const { dataURL, height, upload } = file;
-        if (!height && !dataURL?.startsWith("data:image")) {
+        const { height, upload } = file;
+        const type = upload.chunks[0]?.file.type;
+        if (!height && !type?.startsWith("image")) {
           node = {
             type: "link",
             isInline: true,
diff --git a/src/packages/frontend/editors/unknown/editor.tsx b/src/packages/frontend/editors/unknown/editor.tsx
index 65944c17db4..5d02dbf37b8 100644
--- a/src/packages/frontend/editors/unknown/editor.tsx
+++ b/src/packages/frontend/editors/unknown/editor.tsx
@@ -226,13 +226,16 @@ export const UnknownEditor: React.FC = (props: Props) => {
   function render_register() {
     return (
       <>
-        

-          {NAME} detected that the file's content has the MIME code{" "}
-          
-            {mime}
-          
-          . {explanation}
-        

+        {mime && (
+          

+            {NAME} detected that the file's content has the MIME code{" "}
+            
+              {mime}
+            
+            . {explanation}
+          

+        )}
+        {!mime && 
{NAME} was not able to detect the file's type.
}
         
The following editors are available: